agentaudit 3.12.1 → 3.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli.mjs +550 -293
- package/index.mjs +1 -1
- package/package.json +1 -1
package/cli.mjs
CHANGED
|
@@ -150,6 +150,36 @@ const USER_CRED_DIR = path.join(xdgConfig, 'agentaudit');
|
|
|
150
150
|
const USER_CRED_FILE = path.join(USER_CRED_DIR, 'credentials.json');
|
|
151
151
|
const SKILL_CRED_FILE = path.join(SKILL_DIR, 'config', 'credentials.json');
|
|
152
152
|
const PROFILE_CACHE_FILE = path.join(USER_CRED_DIR, 'profile-cache.json');
|
|
153
|
+
const HISTORY_DIR = path.join(USER_CRED_DIR, 'history');
|
|
154
|
+
|
|
155
|
+
function saveHistory(report) {
|
|
156
|
+
try {
|
|
157
|
+
fs.mkdirSync(HISTORY_DIR, { recursive: true });
|
|
158
|
+
const slug = report.skill_slug || 'unknown';
|
|
159
|
+
const model = (report.audit_model || 'unknown').replace(/[^a-z0-9-]/gi, '-').slice(0, 30);
|
|
160
|
+
const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
|
161
|
+
const filename = `${ts}_${slug}_${model}.json`;
|
|
162
|
+
fs.writeFileSync(path.join(HISTORY_DIR, filename), JSON.stringify(report, null, 2));
|
|
163
|
+
} catch {}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function loadHistory(limit = 20) {
|
|
167
|
+
try {
|
|
168
|
+
if (!fs.existsSync(HISTORY_DIR)) return [];
|
|
169
|
+
const files = fs.readdirSync(HISTORY_DIR)
|
|
170
|
+
.filter(f => f.endsWith('.json'))
|
|
171
|
+
.sort()
|
|
172
|
+
.reverse()
|
|
173
|
+
.slice(0, limit);
|
|
174
|
+
return files.map(f => {
|
|
175
|
+
try {
|
|
176
|
+
const data = JSON.parse(fs.readFileSync(path.join(HISTORY_DIR, f), 'utf8'));
|
|
177
|
+
data._file = f;
|
|
178
|
+
return data;
|
|
179
|
+
} catch { return null; }
|
|
180
|
+
}).filter(Boolean);
|
|
181
|
+
} catch { return []; }
|
|
182
|
+
}
|
|
153
183
|
|
|
154
184
|
function loadCredentials() {
|
|
155
185
|
for (const f of [SKILL_CRED_FILE, USER_CRED_FILE]) {
|
|
@@ -222,6 +252,37 @@ function resolveProvider() {
|
|
|
222
252
|
return LLM_PROVIDERS.find(p => process.env[p.key]) || null;
|
|
223
253
|
}
|
|
224
254
|
|
|
255
|
+
function resolveModel(modelName) {
|
|
256
|
+
// model with '/' → OpenRouter
|
|
257
|
+
if (modelName.includes('/')) {
|
|
258
|
+
const p = LLM_PROVIDERS.find(p => p.provider === 'openrouter' && process.env[p.key]);
|
|
259
|
+
if (p) return { ...p, model: modelName };
|
|
260
|
+
return null;
|
|
261
|
+
}
|
|
262
|
+
// Known prefix → native provider
|
|
263
|
+
const prefixes = [
|
|
264
|
+
['claude', 'anthropic'], ['gemini', 'google'], ['gpt', 'openai'],
|
|
265
|
+
['deepseek', 'deepseek'], ['mistral', 'mistral'], ['grok', 'xai'], ['glm', 'zhipu'],
|
|
266
|
+
];
|
|
267
|
+
for (const [prefix, prov] of prefixes) {
|
|
268
|
+
if (modelName.toLowerCase().startsWith(prefix)) {
|
|
269
|
+
const p = LLM_PROVIDERS.find(p => p.provider === prov && process.env[p.key]);
|
|
270
|
+
if (p) return { ...p, model: modelName };
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
// Check PROVIDER_MODELS for exact match
|
|
274
|
+
for (const [prov, models] of Object.entries(PROVIDER_MODELS)) {
|
|
275
|
+
if (models.some(m => m.value === modelName)) {
|
|
276
|
+
const p = LLM_PROVIDERS.find(p => p.provider === prov && process.env[p.key]);
|
|
277
|
+
if (p) return { ...p, model: modelName };
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
// Last resort: OpenRouter
|
|
281
|
+
const or = LLM_PROVIDERS.find(p => p.provider === 'openrouter' && process.env[p.key]);
|
|
282
|
+
if (or) return { ...or, model: modelName };
|
|
283
|
+
return null;
|
|
284
|
+
}
|
|
285
|
+
|
|
225
286
|
function saveCredentials(data) {
|
|
226
287
|
const json = JSON.stringify(data, null, 2);
|
|
227
288
|
fs.mkdirSync(USER_CRED_DIR, { recursive: true });
|
|
@@ -2509,6 +2570,91 @@ function loadAuditPrompt() {
|
|
|
2509
2570
|
return null;
|
|
2510
2571
|
}
|
|
2511
2572
|
|
|
2573
|
+
async function callLlm(llmConfig, systemPrompt, userMessage) {
|
|
2574
|
+
const apiKey = process.env[llmConfig.key];
|
|
2575
|
+
if (!apiKey) return { error: `Missing API key: ${llmConfig.key}` };
|
|
2576
|
+
const start = Date.now();
|
|
2577
|
+
let _text = '';
|
|
2578
|
+
try {
|
|
2579
|
+
let data;
|
|
2580
|
+
if (llmConfig.type === 'anthropic') {
|
|
2581
|
+
const res = await fetch(llmConfig.url, {
|
|
2582
|
+
method: 'POST',
|
|
2583
|
+
headers: { 'x-api-key': apiKey, 'anthropic-version': '2023-06-01', 'content-type': 'application/json' },
|
|
2584
|
+
body: JSON.stringify({ model: llmConfig.model, max_tokens: 8192, system: systemPrompt, messages: [{ role: 'user', content: userMessage }] }),
|
|
2585
|
+
signal: AbortSignal.timeout(120_000),
|
|
2586
|
+
});
|
|
2587
|
+
data = await res.json();
|
|
2588
|
+
if (data.error) {
|
|
2589
|
+
const friendly = formatApiError(data.error, llmConfig.provider, res.status);
|
|
2590
|
+
return { error: friendly?.text || data.error.message || JSON.stringify(data.error), hint: friendly?.hint, duration: Date.now() - start };
|
|
2591
|
+
}
|
|
2592
|
+
_text = data.content?.[0]?.text || '';
|
|
2593
|
+
const report = extractJSON(_text);
|
|
2594
|
+
if (report) {
|
|
2595
|
+
report.audit_model = data.model || llmConfig.model;
|
|
2596
|
+
report.audit_provider = llmConfig.provider;
|
|
2597
|
+
if (data.id) report.provider_msg_id = data.id;
|
|
2598
|
+
if (data.usage) { report.input_tokens = data.usage.input_tokens; report.output_tokens = data.usage.output_tokens; }
|
|
2599
|
+
}
|
|
2600
|
+
return { report, text: _text, duration: Date.now() - start };
|
|
2601
|
+
} else if (llmConfig.type === 'gemini') {
|
|
2602
|
+
const res = await fetch(`${llmConfig.url}/${llmConfig.model}:generateContent?key=${apiKey}`, {
|
|
2603
|
+
method: 'POST',
|
|
2604
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2605
|
+
body: JSON.stringify({
|
|
2606
|
+
systemInstruction: { parts: [{ text: systemPrompt }] },
|
|
2607
|
+
contents: [{ role: 'user', parts: [{ text: userMessage }] }],
|
|
2608
|
+
generationConfig: { maxOutputTokens: 8192 },
|
|
2609
|
+
}),
|
|
2610
|
+
signal: AbortSignal.timeout(120_000),
|
|
2611
|
+
});
|
|
2612
|
+
data = await res.json();
|
|
2613
|
+
if (data.error) {
|
|
2614
|
+
const friendly = formatApiError(data.error, llmConfig.provider, res.status);
|
|
2615
|
+
return { error: friendly?.text || data.error.message || JSON.stringify(data.error), hint: friendly?.hint, duration: Date.now() - start };
|
|
2616
|
+
}
|
|
2617
|
+
_text = data.candidates?.[0]?.content?.parts?.[0]?.text || '';
|
|
2618
|
+
const report = extractJSON(_text);
|
|
2619
|
+
if (report) {
|
|
2620
|
+
report.audit_model = data.modelVersion || llmConfig.model;
|
|
2621
|
+
report.audit_provider = llmConfig.provider;
|
|
2622
|
+
if (data.usageMetadata) { report.input_tokens = data.usageMetadata.promptTokenCount; report.output_tokens = data.usageMetadata.candidatesTokenCount; }
|
|
2623
|
+
}
|
|
2624
|
+
return { report, text: _text, duration: Date.now() - start };
|
|
2625
|
+
} else {
|
|
2626
|
+
const headers = { 'Authorization': `Bearer ${apiKey}`, 'Content-Type': 'application/json' };
|
|
2627
|
+
if (llmConfig.provider === 'openrouter') { headers['HTTP-Referer'] = 'https://agentaudit.dev'; headers['X-Title'] = 'AgentAudit CLI'; }
|
|
2628
|
+
const res = await fetch(llmConfig.url, {
|
|
2629
|
+
method: 'POST',
|
|
2630
|
+
headers,
|
|
2631
|
+
body: JSON.stringify({ model: llmConfig.model, max_tokens: 8192, messages: [{ role: 'system', content: systemPrompt }, { role: 'user', content: userMessage }] }),
|
|
2632
|
+
signal: AbortSignal.timeout(120_000),
|
|
2633
|
+
});
|
|
2634
|
+
data = await res.json();
|
|
2635
|
+
if (data.error) {
|
|
2636
|
+
const friendly = formatApiError(data.error, llmConfig.provider, res.status);
|
|
2637
|
+
return { error: friendly?.text || data.error.message || JSON.stringify(data.error), hint: friendly?.hint, duration: Date.now() - start };
|
|
2638
|
+
}
|
|
2639
|
+
_text = data.choices?.[0]?.message?.content || '';
|
|
2640
|
+
const report = extractJSON(_text);
|
|
2641
|
+
if (report) {
|
|
2642
|
+
report.audit_model = data.model || llmConfig.model;
|
|
2643
|
+
report.audit_provider = llmConfig.provider;
|
|
2644
|
+
if (data.id) report.provider_msg_id = data.id;
|
|
2645
|
+
if (data.system_fingerprint) report.provider_fingerprint = data.system_fingerprint;
|
|
2646
|
+
if (data.usage) { report.input_tokens = data.usage.prompt_tokens; report.output_tokens = data.usage.completion_tokens; }
|
|
2647
|
+
}
|
|
2648
|
+
return { report, text: _text, duration: Date.now() - start };
|
|
2649
|
+
}
|
|
2650
|
+
} catch (err) {
|
|
2651
|
+
const dur = Date.now() - start;
|
|
2652
|
+
if (err.name === 'TimeoutError' || err.message?.includes('timeout')) return { error: 'Request timed out (120s)', hint: 'Try again or use a faster model', duration: dur };
|
|
2653
|
+
if (err.code === 'ENOTFOUND' || err.code === 'ECONNREFUSED' || err.message?.includes('fetch failed')) return { error: `Network error: could not reach ${llmConfig.provider}`, hint: 'Check your internet connection', duration: dur };
|
|
2654
|
+
return { error: err.message, duration: dur };
|
|
2655
|
+
}
|
|
2656
|
+
}
|
|
2657
|
+
|
|
2512
2658
|
async function auditRepo(url) {
|
|
2513
2659
|
const start = Date.now();
|
|
2514
2660
|
const slug = slugFromUrl(url);
|
|
@@ -2547,72 +2693,24 @@ async function auditRepo(url) {
|
|
|
2547
2693
|
}
|
|
2548
2694
|
console.log(` ${c.green}done${c.reset}`);
|
|
2549
2695
|
|
|
2550
|
-
// Step 4:
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
const
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
const
|
|
2558
|
-
const
|
|
2559
|
-
const
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
if (
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
if (!activeLlm) {
|
|
2567
|
-
// No LLM API key — compact explanation
|
|
2568
|
-
console.log();
|
|
2569
|
-
console.log(` ${c.yellow}No LLM API key found.${c.reset} The ${c.bold}audit${c.reset} command needs an LLM to analyze code.`);
|
|
2570
|
-
console.log();
|
|
2571
|
-
console.log(` ${c.bold}Set an API key${c.reset} (e.g. ${c.cyan}export OPENROUTER_API_KEY=sk-or-...${c.reset})`);
|
|
2572
|
-
console.log(` ${c.dim}Run "agentaudit model" to configure provider + model interactively${c.reset}`);
|
|
2573
|
-
console.log();
|
|
2574
|
-
console.log(` ${c.bold}Or export for manual review:${c.reset} ${c.cyan}agentaudit audit ${url} --export${c.reset}`);
|
|
2575
|
-
console.log(` ${c.bold}Or use as MCP server${c.reset} in Cursor/Claude ${c.dim}(no extra API key needed)${c.reset}`);
|
|
2576
|
-
console.log(` ${c.dim}{ "agentaudit": { "command": "npx", "args": ["-y", "agentaudit"] } }${c.reset}`);
|
|
2577
|
-
console.log();
|
|
2578
|
-
|
|
2579
|
-
// Check if --export flag
|
|
2580
|
-
if (process.argv.includes('--export')) {
|
|
2581
|
-
const exportPath = path.join(process.cwd(), `audit-${slug}.md`);
|
|
2582
|
-
const exportContent = [
|
|
2583
|
-
`# Security Audit: ${slug}`,
|
|
2584
|
-
`**Source:** ${url}`,
|
|
2585
|
-
`**Files:** ${files.length}`,
|
|
2586
|
-
``,
|
|
2587
|
-
`## Audit Instructions`,
|
|
2588
|
-
``,
|
|
2589
|
-
auditPrompt || '(audit prompt not found)',
|
|
2590
|
-
``,
|
|
2591
|
-
`## Report Format`,
|
|
2592
|
-
``,
|
|
2593
|
-
`After analysis, produce a JSON report:`,
|
|
2594
|
-
'```json',
|
|
2595
|
-
`{ "skill_slug": "${slug}", "source_url": "${url}", "risk_score": 0, "result": "safe", "findings": [] }`,
|
|
2596
|
-
'```',
|
|
2597
|
-
``,
|
|
2598
|
-
`## Source Code`,
|
|
2599
|
-
``,
|
|
2600
|
-
codeBlock,
|
|
2601
|
-
].join('\n');
|
|
2602
|
-
fs.writeFileSync(exportPath, exportContent);
|
|
2603
|
-
console.log(` ${icons.safe} Exported to ${c.bold}${exportPath}${c.reset}`);
|
|
2604
|
-
console.log(` ${c.dim}Paste this into any LLM (Claude, ChatGPT, etc.) for analysis${c.reset}`);
|
|
2605
|
-
}
|
|
2606
|
-
|
|
2607
|
-
// Cleanup
|
|
2608
|
-
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
|
|
2609
|
-
return null;
|
|
2610
|
-
}
|
|
2611
|
-
|
|
2612
|
-
// We have an API key — run LLM audit
|
|
2613
|
-
const modelLabel = modelOverride ? `${activeProvider} → ${activeLlm.model}` : activeProvider;
|
|
2614
|
-
process.stdout.write(` ${stepProgress(4, 4)} Running LLM analysis ${c.dim}(${modelLabel})${c.reset}...`);
|
|
2696
|
+
// Step 4: Provenance + type detection (needs repoPath on disk)
|
|
2697
|
+
let commitSha = '';
|
|
2698
|
+
try { commitSha = execSync('git rev-parse HEAD', { cwd: repoPath, encoding: 'utf8' }).trim(); } catch {}
|
|
2699
|
+
const sourceHash = crypto.createHash('sha256').update(
|
|
2700
|
+
files.slice().sort((a, b) => a.path.localeCompare(b.path))
|
|
2701
|
+
.map(f => f.path + '\n' + f.content).join('\n')
|
|
2702
|
+
).digest('hex');
|
|
2703
|
+
const pkgInfo = detectPackageInfo(repoPath, files);
|
|
2704
|
+
const KNOWN_MCP_LIBS = new Set(['fastmcp', 'jlowin-fastmcp', 'mcp-go', 'fastapi-mcp', 'fastapi_mcp', 'mcp-use', 'mcp-agent']);
|
|
2705
|
+
const KNOWN_CLI = new Set(['mcp-cli', 'mcp-scan', 'inspector']);
|
|
2706
|
+
let detectedType = pkgInfo.type === 'unknown' ? 'other' : pkgInfo.type;
|
|
2707
|
+
if (KNOWN_MCP_LIBS.has(slug)) detectedType = 'library';
|
|
2708
|
+
if (KNOWN_CLI.has(slug)) detectedType = 'cli-tool';
|
|
2709
|
+
|
|
2710
|
+
// Cleanup repo (files in memory, provenance captured)
|
|
2711
|
+
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
|
|
2615
2712
|
|
|
2713
|
+
// Build prompts
|
|
2616
2714
|
const systemPrompt = auditPrompt || 'You are a security auditor. Analyze the code and report findings as JSON.';
|
|
2617
2715
|
const userMessage = [
|
|
2618
2716
|
`Audit this package: **${slug}** (${url})`,
|
|
@@ -2628,205 +2726,278 @@ async function auditRepo(url) {
|
|
|
2628
2726
|
codeBlock,
|
|
2629
2727
|
].join('\n');
|
|
2630
2728
|
|
|
2631
|
-
|
|
2632
|
-
|
|
2729
|
+
// Helper: add provenance to a report
|
|
2730
|
+
const enrichReport = (report, duration) => {
|
|
2731
|
+
report.skill_slug = slug;
|
|
2732
|
+
report.package_type = detectedType;
|
|
2733
|
+
report.audit_duration_ms = duration || (Date.now() - start);
|
|
2734
|
+
report.files_scanned = files.length;
|
|
2735
|
+
if (commitSha) report.commit_sha = commitSha;
|
|
2736
|
+
report.source_hash = sourceHash;
|
|
2737
|
+
};
|
|
2633
2738
|
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
if (
|
|
2637
|
-
|
|
2638
|
-
|
|
2739
|
+
// Helper: upload one report
|
|
2740
|
+
const uploadReport = async (report, creds) => {
|
|
2741
|
+
if (!creds) return;
|
|
2742
|
+
process.stdout.write(` Uploading report${report.audit_model ? ` (${report.audit_model})` : ''}...`);
|
|
2743
|
+
try {
|
|
2744
|
+
const res = await fetch(`${REGISTRY_URL}/api/reports`, {
|
|
2639
2745
|
method: 'POST',
|
|
2640
|
-
headers: {
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
'content-type': 'application/json',
|
|
2644
|
-
},
|
|
2645
|
-
body: JSON.stringify({
|
|
2646
|
-
model: activeLlm.model,
|
|
2647
|
-
max_tokens: 8192,
|
|
2648
|
-
system: systemPrompt,
|
|
2649
|
-
messages: [{ role: 'user', content: userMessage }],
|
|
2650
|
-
}),
|
|
2651
|
-
signal: AbortSignal.timeout(120_000),
|
|
2746
|
+
headers: { 'Authorization': `Bearer ${creds.api_key}`, 'Content-Type': 'application/json' },
|
|
2747
|
+
body: JSON.stringify(report),
|
|
2748
|
+
signal: AbortSignal.timeout(15_000),
|
|
2652
2749
|
});
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
console.log(` ${c.dim}${friendly.hint}${c.reset}`);
|
|
2660
|
-
} else {
|
|
2661
|
-
console.log(` ${c.red}API error: ${data.error.message || JSON.stringify(data.error)}${c.reset}`);
|
|
2662
|
-
}
|
|
2663
|
-
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
|
|
2664
|
-
return null;
|
|
2750
|
+
if (res.ok) {
|
|
2751
|
+
console.log(` ${c.green}done${c.reset}`);
|
|
2752
|
+
} else {
|
|
2753
|
+
let errBody = ''; try { errBody = await res.text(); } catch {}
|
|
2754
|
+
console.log(` ${c.yellow}failed (HTTP ${res.status})${c.reset}`);
|
|
2755
|
+
if (errBody && process.argv.includes('--debug')) console.log(` ${c.dim}Server: ${errBody.slice(0, 300)}${c.reset}`);
|
|
2665
2756
|
}
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2757
|
+
} catch { console.log(` ${c.yellow}failed${c.reset}`); }
|
|
2758
|
+
};
|
|
2759
|
+
|
|
2760
|
+
// Step 5: Resolve models
|
|
2761
|
+
const modelsArgIdx = process.argv.indexOf('--models');
|
|
2762
|
+
const modelsFlag = modelsArgIdx !== -1 ? process.argv[modelsArgIdx + 1] : null;
|
|
2763
|
+
const modelNames = modelsFlag ? modelsFlag.split(',').map(m => m.trim()).filter(Boolean) : [];
|
|
2764
|
+
const isMultiModel = modelNames.length > 1;
|
|
2765
|
+
|
|
2766
|
+
// ── Multi-Model Path ─────────────────────────────────────
|
|
2767
|
+
if (isMultiModel) {
|
|
2768
|
+
const resolvedModels = [];
|
|
2769
|
+
const failedModels = [];
|
|
2770
|
+
for (const name of modelNames) {
|
|
2771
|
+
const config = resolveModel(name);
|
|
2772
|
+
if (!config) { failedModels.push(name); continue; }
|
|
2773
|
+
resolvedModels.push({ name, config });
|
|
2774
|
+
}
|
|
2775
|
+
|
|
2776
|
+
if (resolvedModels.length === 0) {
|
|
2777
|
+
console.log();
|
|
2778
|
+
console.log(` ${c.red}No API keys available for requested models${c.reset}`);
|
|
2779
|
+
for (const name of failedModels) console.log(` ${c.dim}${name}: no matching API key${c.reset}`);
|
|
2780
|
+
console.log(` ${c.dim}Run "agentaudit model" to configure providers${c.reset}`);
|
|
2781
|
+
return null;
|
|
2782
|
+
}
|
|
2783
|
+
|
|
2784
|
+
// Progress
|
|
2785
|
+
const totalSteps = resolvedModels.length;
|
|
2786
|
+
console.log(` ${stepProgress(4, 4)} Running LLM analysis ${c.dim}(${totalSteps} models in parallel)${c.reset}`);
|
|
2787
|
+
if (failedModels.length > 0) {
|
|
2788
|
+
for (const name of failedModels) console.log(` ${c.yellow}⚠${c.reset} ${name.padEnd(30)} ${c.dim}skipped (no API key)${c.reset}`);
|
|
2789
|
+
}
|
|
2790
|
+
|
|
2791
|
+
// Parallel LLM calls
|
|
2792
|
+
const results = await Promise.allSettled(
|
|
2793
|
+
resolvedModels.map(async ({ name, config }) => {
|
|
2794
|
+
const result = await callLlm(config, systemPrompt, userMessage);
|
|
2795
|
+
return { name, ...result };
|
|
2796
|
+
})
|
|
2797
|
+
);
|
|
2798
|
+
|
|
2799
|
+
// Process results
|
|
2800
|
+
const reports = [];
|
|
2801
|
+
for (let i = 0; i < results.length; i++) {
|
|
2802
|
+
const name = resolvedModels[i].name;
|
|
2803
|
+
const r = results[i];
|
|
2804
|
+
if (r.status === 'rejected') {
|
|
2805
|
+
console.log(` ${c.red}✗${c.reset} ${name.padEnd(30)} ${c.red}error${c.reset}`);
|
|
2806
|
+
continue;
|
|
2676
2807
|
}
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
signal: AbortSignal.timeout(120_000),
|
|
2688
|
-
});
|
|
2689
|
-
data = await res.json();
|
|
2690
|
-
if (data.error) {
|
|
2691
|
-
console.log(` ${c.red}failed${c.reset}`);
|
|
2692
|
-
const friendly = formatApiError(data.error, activeLlm.provider, res.status);
|
|
2693
|
-
if (friendly) {
|
|
2694
|
-
console.log(` ${c.red}${friendly.text}${c.reset}`);
|
|
2695
|
-
console.log(` ${c.dim}${friendly.hint}${c.reset}`);
|
|
2696
|
-
} else {
|
|
2697
|
-
console.log(` ${c.red}API error: ${data.error.message || JSON.stringify(data.error)}${c.reset}`);
|
|
2808
|
+
const { report, text, error, hint, duration } = r.value;
|
|
2809
|
+
if (error) {
|
|
2810
|
+
console.log(` ${c.red}✗${c.reset} ${name.padEnd(30)} ${c.red}${error}${c.reset}`);
|
|
2811
|
+
if (hint) console.log(` ${c.dim}${hint}${c.reset}`);
|
|
2812
|
+
continue;
|
|
2813
|
+
}
|
|
2814
|
+
if (!report) {
|
|
2815
|
+
console.log(` ${c.yellow}✗${c.reset} ${name.padEnd(30)} ${c.yellow}JSON parse failed${c.reset}`);
|
|
2816
|
+
if (process.argv.includes('--debug') && text) {
|
|
2817
|
+
console.log(` ${c.dim}${text.slice(0, 200)}...${c.reset}`);
|
|
2698
2818
|
}
|
|
2699
|
-
|
|
2700
|
-
return null;
|
|
2819
|
+
continue;
|
|
2701
2820
|
}
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2821
|
+
const durSec = Math.round((duration || 0) / 1000);
|
|
2822
|
+
console.log(` ${c.green}✓${c.reset} ${name.padEnd(30)} ${c.green}done${c.reset} ${c.dim}(${durSec}s)${c.reset}`);
|
|
2823
|
+
enrichReport(report, duration);
|
|
2824
|
+
saveHistory(report);
|
|
2825
|
+
reports.push({ name, report });
|
|
2826
|
+
}
|
|
2827
|
+
|
|
2828
|
+
if (reports.length === 0) {
|
|
2829
|
+
console.log();
|
|
2830
|
+
console.log(` ${c.red}No models returned valid results${c.reset}`);
|
|
2831
|
+
return null;
|
|
2832
|
+
}
|
|
2833
|
+
|
|
2834
|
+
// Display per-model results
|
|
2835
|
+
console.log();
|
|
2836
|
+
for (const { name, report } of reports) {
|
|
2837
|
+
console.log(sectionHeader(name));
|
|
2838
|
+
console.log(` ${riskBadge(report.risk_score || 0)}`);
|
|
2839
|
+
const fc = report.findings?.length || 0;
|
|
2840
|
+
if (fc > 0) {
|
|
2841
|
+
const counts = {};
|
|
2842
|
+
for (const f of report.findings) { const s = (f.severity || 'info').toLowerCase(); counts[s] = (counts[s] || 0) + 1; }
|
|
2843
|
+
const parts = [];
|
|
2844
|
+
for (const sev of ['critical', 'high', 'medium', 'low', 'info']) { if (counts[sev]) parts.push(`${counts[sev]} ${sev}`); }
|
|
2845
|
+
console.log(` ${c.dim}${fc} findings: ${parts.join(', ')}${c.reset}`);
|
|
2846
|
+
} else {
|
|
2847
|
+
console.log(` ${c.green}No findings${c.reset}`);
|
|
2848
|
+
}
|
|
2849
|
+
console.log();
|
|
2850
|
+
}
|
|
2851
|
+
|
|
2852
|
+
// Consensus comparison
|
|
2853
|
+
if (reports.length > 1) {
|
|
2854
|
+
console.log(sectionHeader('Consensus'));
|
|
2855
|
+
|
|
2856
|
+
// Risk range
|
|
2857
|
+
const risks = reports.map(r => r.report.risk_score || 0);
|
|
2858
|
+
const minRisk = Math.min(...risks);
|
|
2859
|
+
const maxRisk = Math.max(...risks);
|
|
2860
|
+
const avgRisk = Math.round(risks.reduce((a, b) => a + b, 0) / risks.length);
|
|
2861
|
+
console.log(` Risk: ${riskBadge(avgRisk)} ${c.dim}(range ${minRisk}–${maxRisk})${c.reset}`);
|
|
2862
|
+
console.log();
|
|
2863
|
+
|
|
2864
|
+
// Severity agreement
|
|
2865
|
+
const severities = reports.map(r => (r.report.max_severity || 'none').toLowerCase());
|
|
2866
|
+
const allSameSev = severities.every(s => s === severities[0]);
|
|
2867
|
+
if (allSameSev) {
|
|
2868
|
+
console.log(` ${c.green}${reports.length}/${reports.length} models agree:${c.reset} ${severities[0].toUpperCase()}`);
|
|
2869
|
+
} else {
|
|
2870
|
+
console.log(` ${c.yellow}Models disagree on severity:${c.reset}`);
|
|
2871
|
+
for (const { name, report } of reports) {
|
|
2872
|
+
const sev = (report.max_severity || 'none').toUpperCase();
|
|
2873
|
+
const sc = severityColor(report.max_severity);
|
|
2874
|
+
console.log(` ${sc}${sev.padEnd(10)}${c.reset} ${c.dim}${name}${c.reset}`);
|
|
2710
2875
|
}
|
|
2711
2876
|
}
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2877
|
+
console.log();
|
|
2878
|
+
|
|
2879
|
+
// Finding intersection (match by normalized title)
|
|
2880
|
+
const findingsByTitle = new Map();
|
|
2881
|
+
for (const { name, report } of reports) {
|
|
2882
|
+
for (const f of (report.findings || [])) {
|
|
2883
|
+
const key = (f.title || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
|
|
2884
|
+
if (!key) continue;
|
|
2885
|
+
if (!findingsByTitle.has(key)) findingsByTitle.set(key, { title: f.title, severity: f.severity, models: [] });
|
|
2886
|
+
findingsByTitle.get(key).models.push(name);
|
|
2887
|
+
}
|
|
2722
2888
|
}
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
],
|
|
2733
|
-
}),
|
|
2734
|
-
signal: AbortSignal.timeout(120_000),
|
|
2735
|
-
});
|
|
2736
|
-
data = await res.json();
|
|
2737
|
-
if (data.error) {
|
|
2738
|
-
console.log(` ${c.red}failed${c.reset}`);
|
|
2739
|
-
const friendly = formatApiError(data.error, activeLlm.provider, res.status);
|
|
2740
|
-
if (friendly) {
|
|
2741
|
-
console.log(` ${c.red}${friendly.text}${c.reset}`);
|
|
2742
|
-
console.log(` ${c.dim}${friendly.hint}${c.reset}`);
|
|
2743
|
-
} else {
|
|
2744
|
-
console.log(` ${c.red}API error: ${data.error.message || JSON.stringify(data.error)}${c.reset}`);
|
|
2889
|
+
|
|
2890
|
+
const shared = [...findingsByTitle.values()].filter(f => f.models.length > 1);
|
|
2891
|
+
const unique = [...findingsByTitle.values()].filter(f => f.models.length === 1);
|
|
2892
|
+
|
|
2893
|
+
if (shared.length > 0) {
|
|
2894
|
+
console.log(` ${c.bold}Shared findings (${shared.length}):${c.reset}`);
|
|
2895
|
+
for (const f of shared) {
|
|
2896
|
+
const sc = severityColor(f.severity);
|
|
2897
|
+
console.log(` ${sc}┃${c.reset} ${sc}${(f.severity || '').toUpperCase().padEnd(8)}${c.reset} ${f.title} ${c.dim}(${f.models.length}/${reports.length})${c.reset}`);
|
|
2745
2898
|
}
|
|
2746
|
-
|
|
2747
|
-
return null;
|
|
2899
|
+
console.log();
|
|
2748
2900
|
}
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
if (data.system_fingerprint) report.provider_fingerprint = data.system_fingerprint;
|
|
2756
|
-
if (data.usage) {
|
|
2757
|
-
report.input_tokens = data.usage.prompt_tokens;
|
|
2758
|
-
report.output_tokens = data.usage.completion_tokens;
|
|
2901
|
+
|
|
2902
|
+
if (unique.length > 0) {
|
|
2903
|
+
console.log(` ${c.bold}Unique findings (${unique.length}):${c.reset}`);
|
|
2904
|
+
for (const f of unique) {
|
|
2905
|
+
const sc = severityColor(f.severity);
|
|
2906
|
+
console.log(` ${sc}┃${c.reset} ${sc}${(f.severity || '').toUpperCase().padEnd(8)}${c.reset} ${f.title} ${c.dim}(${f.models[0]} only)${c.reset}`);
|
|
2759
2907
|
}
|
|
2908
|
+
console.log();
|
|
2760
2909
|
}
|
|
2761
2910
|
}
|
|
2762
|
-
|
|
2763
|
-
|
|
2764
|
-
|
|
2765
|
-
|
|
2766
|
-
if (
|
|
2767
|
-
|
|
2768
|
-
console.log(` ${c.dim}
|
|
2769
|
-
} else if (
|
|
2770
|
-
console.log(` ${c.
|
|
2771
|
-
|
|
2772
|
-
|
|
2773
|
-
|
|
2911
|
+
|
|
2912
|
+
// Upload each report
|
|
2913
|
+
const noUpload = process.argv.includes('--no-upload');
|
|
2914
|
+
const creds = loadCredentials();
|
|
2915
|
+
if (!noUpload && creds) {
|
|
2916
|
+
for (const { report } of reports) await uploadReport(report, creds);
|
|
2917
|
+
console.log(` ${c.dim}Reports: ${REGISTRY_URL}/packages/${slug}${c.reset}`);
|
|
2918
|
+
} else if (!noUpload && !creds) {
|
|
2919
|
+
console.log(` ${c.dim}Run ${c.cyan}agentaudit setup${c.dim} to upload reports to agentaudit.dev${c.reset}`);
|
|
2920
|
+
}
|
|
2921
|
+
|
|
2922
|
+
console.log();
|
|
2923
|
+
return reports.map(r => r.report);
|
|
2924
|
+
}
|
|
2925
|
+
|
|
2926
|
+
// ── Single-Model Path ────────────────────────────────────
|
|
2927
|
+
// If --models has exactly 1 model, use it; otherwise resolve via --model / config / env
|
|
2928
|
+
let activeLlm;
|
|
2929
|
+
if (modelNames.length === 1) {
|
|
2930
|
+
activeLlm = resolveModel(modelNames[0]);
|
|
2931
|
+
} else {
|
|
2932
|
+
activeLlm = resolveProvider();
|
|
2933
|
+
// Model override: --model flag > AGENTAUDIT_MODEL env > credentials.json > provider default
|
|
2934
|
+
const modelArgIdx2 = process.argv.indexOf('--model');
|
|
2935
|
+
const modelFlag2 = modelArgIdx2 !== -1 ? process.argv[modelArgIdx2 + 1] : null;
|
|
2936
|
+
const modelOverride = modelFlag2 || process.env.AGENTAUDIT_MODEL || loadLlmConfig()?.llm_model || null;
|
|
2937
|
+
if (activeLlm && modelOverride) activeLlm.model = modelOverride;
|
|
2938
|
+
}
|
|
2939
|
+
|
|
2940
|
+
if (!activeLlm) {
|
|
2941
|
+
console.log();
|
|
2942
|
+
console.log(` ${c.yellow}No LLM API key found.${c.reset} The ${c.bold}audit${c.reset} command needs an LLM to analyze code.`);
|
|
2943
|
+
console.log();
|
|
2944
|
+
console.log(` ${c.bold}Set an API key${c.reset} (e.g. ${c.cyan}export OPENROUTER_API_KEY=sk-or-...${c.reset})`);
|
|
2945
|
+
console.log(` ${c.dim}Run "agentaudit model" to configure provider + model interactively${c.reset}`);
|
|
2946
|
+
console.log();
|
|
2947
|
+
console.log(` ${c.bold}Or export for manual review:${c.reset} ${c.cyan}agentaudit audit ${url} --export${c.reset}`);
|
|
2948
|
+
console.log(` ${c.bold}Or use as MCP server${c.reset} in Cursor/Claude ${c.dim}(no extra API key needed)${c.reset}`);
|
|
2949
|
+
console.log(` ${c.dim}{ "agentaudit": { "command": "npx", "args": ["-y", "agentaudit"] } }${c.reset}`);
|
|
2950
|
+
console.log();
|
|
2951
|
+
if (process.argv.includes('--export')) {
|
|
2952
|
+
const exportPath = path.join(process.cwd(), `audit-${slug}.md`);
|
|
2953
|
+
const exportContent = [
|
|
2954
|
+
`# Security Audit: ${slug}`, `**Source:** ${url}`, `**Files:** ${files.length}`, ``,
|
|
2955
|
+
`## Audit Instructions`, ``, auditPrompt || '(audit prompt not found)', ``,
|
|
2956
|
+
`## Report Format`, ``, `After analysis, produce a JSON report:`,
|
|
2957
|
+
'```json', `{ "skill_slug": "${slug}", "source_url": "${url}", "risk_score": 0, "result": "safe", "findings": [] }`, '```',
|
|
2958
|
+
``, `## Source Code`, ``, codeBlock,
|
|
2959
|
+
].join('\n');
|
|
2960
|
+
fs.writeFileSync(exportPath, exportContent);
|
|
2961
|
+
console.log(` ${icons.safe} Exported to ${c.bold}${exportPath}${c.reset}`);
|
|
2962
|
+
console.log(` ${c.dim}Paste this into any LLM (Claude, ChatGPT, etc.) for analysis${c.reset}`);
|
|
2774
2963
|
}
|
|
2775
|
-
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
|
|
2776
2964
|
return null;
|
|
2777
2965
|
}
|
|
2778
|
-
|
|
2779
|
-
// Provenance: compute BEFORE cleanup (needs repoPath on disk)
|
|
2780
|
-
let commitSha = '';
|
|
2781
|
-
try {
|
|
2782
|
-
commitSha = execSync('git rev-parse HEAD', { cwd: repoPath, encoding: 'utf8' }).trim();
|
|
2783
|
-
} catch { /* shallow clone without HEAD — unlikely but safe */ }
|
|
2784
|
-
const sourceHash = crypto.createHash('sha256').update(
|
|
2785
|
-
files.slice().sort((a, b) => a.path.localeCompare(b.path))
|
|
2786
|
-
.map(f => f.path + '\n' + f.content).join('\n')
|
|
2787
|
-
).digest('hex');
|
|
2788
|
-
// Code-based type detection (uses files array in memory + repoPath for context)
|
|
2789
|
-
const pkgInfo = detectPackageInfo(repoPath, files);
|
|
2790
|
-
// Known MCP frameworks are libraries, not servers (they contain MCP patterns but ARE the SDK)
|
|
2791
|
-
const KNOWN_MCP_LIBS = new Set(['fastmcp', 'jlowin-fastmcp', 'mcp-go', 'fastapi-mcp', 'fastapi_mcp', 'mcp-use', 'mcp-agent']);
|
|
2792
|
-
const KNOWN_CLI = new Set(['mcp-cli', 'mcp-scan', 'inspector']);
|
|
2793
|
-
let detectedType = pkgInfo.type === 'unknown' ? 'other' : pkgInfo.type;
|
|
2794
|
-
if (KNOWN_MCP_LIBS.has(slug)) detectedType = 'library';
|
|
2795
|
-
if (KNOWN_CLI.has(slug)) detectedType = 'cli-tool';
|
|
2796
2966
|
|
|
2797
|
-
//
|
|
2798
|
-
|
|
2967
|
+
// Single LLM call via callLlm()
|
|
2968
|
+
const modelLabel = `${activeLlm.name} → ${activeLlm.model}`;
|
|
2969
|
+
process.stdout.write(` ${stepProgress(4, 4)} Running LLM analysis ${c.dim}(${modelLabel})${c.reset}...`);
|
|
2970
|
+
|
|
2971
|
+
const llmResult = await callLlm(activeLlm, systemPrompt, userMessage);
|
|
2972
|
+
|
|
2973
|
+
if (llmResult.error) {
|
|
2974
|
+
console.log(` ${c.red}failed${c.reset}`);
|
|
2975
|
+
console.log(` ${c.red}${llmResult.error}${c.reset}`);
|
|
2976
|
+
if (llmResult.hint) console.log(` ${c.dim}${llmResult.hint}${c.reset}`);
|
|
2977
|
+
return null;
|
|
2978
|
+
}
|
|
2799
2979
|
|
|
2980
|
+
console.log(` ${c.green}done${c.reset} ${c.dim}(${elapsed(start)})${c.reset}`);
|
|
2981
|
+
|
|
2982
|
+
const report = llmResult.report;
|
|
2800
2983
|
if (!report) {
|
|
2801
2984
|
console.log(` ${c.red}Could not parse LLM response as JSON${c.reset}`);
|
|
2802
2985
|
console.log(` ${c.dim}Hint: run with --debug to see the raw LLM response${c.reset}`);
|
|
2803
2986
|
if (process.argv.includes('--debug')) {
|
|
2804
2987
|
console.log(` ${c.dim}--- Raw LLM response (first 2000 chars) ---${c.reset}`);
|
|
2805
|
-
console.log((
|
|
2988
|
+
console.log((llmResult.text || '(empty)').slice(0, 2000));
|
|
2806
2989
|
console.log(` ${c.dim}--- end ---${c.reset}`);
|
|
2807
2990
|
}
|
|
2808
2991
|
return null;
|
|
2809
2992
|
}
|
|
2810
2993
|
|
|
2811
|
-
|
|
2812
|
-
report
|
|
2813
|
-
|
|
2814
|
-
// Force package_type from code detection — never trust LLM-provided type
|
|
2815
|
-
report.package_type = detectedType;
|
|
2816
|
-
|
|
2817
|
-
// Add scan metadata for benchmarking
|
|
2818
|
-
report.audit_duration_ms = Date.now() - start;
|
|
2819
|
-
report.files_scanned = files.length;
|
|
2820
|
-
|
|
2821
|
-
// Set provenance data
|
|
2822
|
-
if (commitSha) report.commit_sha = commitSha;
|
|
2823
|
-
report.source_hash = sourceHash;
|
|
2994
|
+
enrichReport(report);
|
|
2995
|
+
saveHistory(report);
|
|
2824
2996
|
|
|
2825
2997
|
// Display results
|
|
2826
2998
|
console.log();
|
|
2827
|
-
const riskScore = report.risk_score || 0;
|
|
2828
2999
|
console.log(sectionHeader('Result'));
|
|
2829
|
-
console.log(` ${riskBadge(
|
|
3000
|
+
console.log(` ${riskBadge(report.risk_score || 0)}`);
|
|
2830
3001
|
console.log();
|
|
2831
3002
|
|
|
2832
3003
|
if (report.findings && report.findings.length > 0) {
|
|
@@ -2839,8 +3010,6 @@ async function auditRepo(url) {
|
|
|
2839
3010
|
if (f.description) console.log(` ${sc}┃${c.reset} ${c.dim}${f.description.slice(0, 120)}${c.reset}`);
|
|
2840
3011
|
console.log();
|
|
2841
3012
|
}
|
|
2842
|
-
|
|
2843
|
-
// Severity histogram
|
|
2844
3013
|
const histLines = severityHistogram(report.findings);
|
|
2845
3014
|
if (histLines.length > 1) {
|
|
2846
3015
|
console.log(sectionHeader('Severity'));
|
|
@@ -2851,41 +3020,16 @@ async function auditRepo(url) {
|
|
|
2851
3020
|
console.log(` ${c.green}No findings — package looks clean.${c.reset}`);
|
|
2852
3021
|
console.log();
|
|
2853
3022
|
}
|
|
2854
|
-
|
|
2855
|
-
// Upload to registry
|
|
3023
|
+
|
|
3024
|
+
// Upload to registry
|
|
2856
3025
|
const noUpload = process.argv.includes('--no-upload');
|
|
2857
3026
|
let creds = loadCredentials();
|
|
2858
3027
|
if (noUpload) {
|
|
2859
3028
|
// Skip silently
|
|
2860
3029
|
} else if (creds) {
|
|
2861
|
-
|
|
2862
|
-
|
|
2863
|
-
const res = await fetch(`${REGISTRY_URL}/api/reports`, {
|
|
2864
|
-
method: 'POST',
|
|
2865
|
-
headers: {
|
|
2866
|
-
'Authorization': `Bearer ${creds.api_key}`,
|
|
2867
|
-
'Content-Type': 'application/json',
|
|
2868
|
-
},
|
|
2869
|
-
body: JSON.stringify(report),
|
|
2870
|
-
signal: AbortSignal.timeout(15_000),
|
|
2871
|
-
});
|
|
2872
|
-
if (res.ok) {
|
|
2873
|
-
const data = await res.json();
|
|
2874
|
-
console.log(` ${c.green}done${c.reset}`);
|
|
2875
|
-
console.log(` ${c.dim}Report: ${REGISTRY_URL}/packages/${slug}${c.reset}`);
|
|
2876
|
-
} else {
|
|
2877
|
-
let errBody = '';
|
|
2878
|
-
try { errBody = await res.text(); } catch {}
|
|
2879
|
-
console.log(` ${c.yellow}failed (HTTP ${res.status})${c.reset}`);
|
|
2880
|
-
if (errBody && process.argv.includes('--debug')) {
|
|
2881
|
-
console.log(` ${c.dim}Server: ${errBody.slice(0, 300)}${c.reset}`);
|
|
2882
|
-
}
|
|
2883
|
-
}
|
|
2884
|
-
} catch (err) {
|
|
2885
|
-
console.log(` ${c.yellow}failed${c.reset}`);
|
|
2886
|
-
}
|
|
3030
|
+
await uploadReport(report, creds);
|
|
3031
|
+
console.log(` ${c.dim}Report: ${REGISTRY_URL}/packages/${slug}${c.reset}`);
|
|
2887
3032
|
} else if (process.stdin.isTTY) {
|
|
2888
|
-
// No credentials — prompt to paste key or set up
|
|
2889
3033
|
console.log();
|
|
2890
3034
|
console.log(` ${c.bold}Want to upload this report to agentaudit.dev?${c.reset}`);
|
|
2891
3035
|
console.log(` ${c.dim}Create an API key at ${c.cyan}${REGISTRY_URL}/profile${c.dim} (sign in with GitHub)${c.reset}`);
|
|
@@ -2899,27 +3043,8 @@ async function auditRepo(url) {
|
|
|
2899
3043
|
saveCredentials({ api_key: pastedKey.trim(), agent_name: agentName });
|
|
2900
3044
|
creds = { api_key: pastedKey.trim(), agent_name: agentName };
|
|
2901
3045
|
console.log(` ${c.green}valid!${c.reset}`);
|
|
2902
|
-
|
|
2903
|
-
|
|
2904
|
-
const res = await fetch(`${REGISTRY_URL}/api/reports`, {
|
|
2905
|
-
method: 'POST',
|
|
2906
|
-
headers: {
|
|
2907
|
-
'Authorization': `Bearer ${creds.api_key}`,
|
|
2908
|
-
'Content-Type': 'application/json',
|
|
2909
|
-
},
|
|
2910
|
-
body: JSON.stringify(report),
|
|
2911
|
-
signal: AbortSignal.timeout(15_000),
|
|
2912
|
-
});
|
|
2913
|
-
if (res.ok) {
|
|
2914
|
-
console.log(` ${c.green}done${c.reset}`);
|
|
2915
|
-
console.log(` ${c.dim}Report: ${REGISTRY_URL}/packages/${slug}${c.reset}`);
|
|
2916
|
-
} else {
|
|
2917
|
-
console.log(` ${c.yellow}failed (HTTP ${res.status})${c.reset}`);
|
|
2918
|
-
}
|
|
2919
|
-
} catch (err) {
|
|
2920
|
-
console.log(` ${c.red}failed${c.reset}`);
|
|
2921
|
-
console.log(` ${c.dim}${err.message}${c.reset}`);
|
|
2922
|
-
}
|
|
3046
|
+
await uploadReport(report, creds);
|
|
3047
|
+
console.log(` ${c.dim}Report: ${REGISTRY_URL}/packages/${slug}${c.reset}`);
|
|
2923
3048
|
} else {
|
|
2924
3049
|
console.log(` ${c.red}invalid key${c.reset}`);
|
|
2925
3050
|
console.log(` ${c.dim}Run ${c.cyan}agentaudit setup${c.dim} to configure.${c.reset}`);
|
|
@@ -2928,7 +3053,7 @@ async function auditRepo(url) {
|
|
|
2928
3053
|
} else {
|
|
2929
3054
|
console.log(` ${c.dim}Run ${c.cyan}agentaudit setup${c.dim} to configure your API key and upload reports${c.reset}`);
|
|
2930
3055
|
}
|
|
2931
|
-
|
|
3056
|
+
|
|
2932
3057
|
console.log();
|
|
2933
3058
|
return report;
|
|
2934
3059
|
}
|
|
@@ -3812,9 +3937,11 @@ async function main() {
|
|
|
3812
3937
|
// Strip global flags from args (including --model <value>)
|
|
3813
3938
|
const globalFlags = new Set(['--json', '--quiet', '-q', '--no-color', '--no-upload']);
|
|
3814
3939
|
let args = rawArgs.filter(a => !globalFlags.has(a));
|
|
3815
|
-
// Remove --model <value>
|
|
3940
|
+
// Remove --model <value> and --models <value> pairs
|
|
3816
3941
|
const modelIdx = args.indexOf('--model');
|
|
3817
3942
|
if (modelIdx !== -1) args.splice(modelIdx, 2);
|
|
3943
|
+
const modelsIdx = args.indexOf('--models');
|
|
3944
|
+
if (modelsIdx !== -1) args.splice(modelsIdx, 2);
|
|
3818
3945
|
|
|
3819
3946
|
// Detect per-command --help BEFORE stripping (e.g. `agentaudit model --help`)
|
|
3820
3947
|
const wantsHelp = args.includes('--help') || args.includes('-h');
|
|
@@ -3864,15 +3991,16 @@ async function main() {
|
|
|
3864
3991
|
`Deep LLM-powered 3-pass security audit (~30s). Requires an LLM API key.`,
|
|
3865
3992
|
``,
|
|
3866
3993
|
`${c.bold}Options:${c.reset}`,
|
|
3867
|
-
` --model <name>
|
|
3868
|
-
` --
|
|
3869
|
-
` --
|
|
3870
|
-
` --
|
|
3994
|
+
` --model <name> Override LLM model for this run`,
|
|
3995
|
+
` --models <a,b,c> Multi-model audit (parallel calls, consensus comparison)`,
|
|
3996
|
+
` --no-upload Skip uploading report to registry`,
|
|
3997
|
+
` --export Export audit payload as markdown (for manual LLM review)`,
|
|
3998
|
+
` --debug Show raw LLM response on parse errors`,
|
|
3871
3999
|
``,
|
|
3872
4000
|
`${c.bold}Examples:${c.reset}`,
|
|
3873
4001
|
` agentaudit audit https://github.com/owner/repo`,
|
|
3874
|
-
` agentaudit audit https://github.com/owner/repo --no-upload`,
|
|
3875
4002
|
` agentaudit audit https://github.com/owner/repo --model gpt-4o`,
|
|
4003
|
+
` agentaudit audit https://github.com/owner/repo --models gemini-2.5-flash,claude-sonnet-4-20250514`,
|
|
3876
4004
|
` agentaudit audit https://github.com/owner/repo --export`,
|
|
3877
4005
|
],
|
|
3878
4006
|
lookup: [
|
|
@@ -3986,10 +4114,32 @@ async function main() {
|
|
|
3986
4114
|
` agentaudit benchmark --json`,
|
|
3987
4115
|
],
|
|
3988
4116
|
bench: null, // alias → benchmark
|
|
4117
|
+
consensus: [
|
|
4118
|
+
`${c.bold}agentaudit consensus${c.reset} <package-name>`,
|
|
4119
|
+
``,
|
|
4120
|
+
`View multi-model consensus status from the AgentAudit registry.`,
|
|
4121
|
+
`Shows agreement across different LLM models and peer reviewers.`,
|
|
4122
|
+
``,
|
|
4123
|
+
`${c.bold}Options:${c.reset}`,
|
|
4124
|
+
` --json Machine-readable JSON output`,
|
|
4125
|
+
``,
|
|
4126
|
+
`${c.bold}Examples:${c.reset}`,
|
|
4127
|
+
` agentaudit consensus nanobanana-mcp-server`,
|
|
4128
|
+
` agentaudit consensus fastmcp --json`,
|
|
4129
|
+
],
|
|
4130
|
+
history: [
|
|
4131
|
+
`${c.bold}agentaudit history${c.reset} [options]`,
|
|
4132
|
+
``,
|
|
4133
|
+
`Show your local audit history. Results are stored in ~/.config/agentaudit/history/`,
|
|
4134
|
+
`after every audit run. No internet connection required.`,
|
|
4135
|
+
``,
|
|
4136
|
+
`${c.bold}Options:${c.reset}`,
|
|
4137
|
+
` --json Machine-readable JSON output`,
|
|
4138
|
+
],
|
|
3989
4139
|
activity: [
|
|
3990
4140
|
`${c.bold}agentaudit activity${c.reset} [options]`,
|
|
3991
4141
|
``,
|
|
3992
|
-
`Show your recent audits and findings from the AgentAudit registry.`,
|
|
4142
|
+
`Show your recent audits and findings from the AgentAudit registry (online).`,
|
|
3993
4143
|
`Requires being logged in (run ${c.cyan}agentaudit setup${c.reset} first).`,
|
|
3994
4144
|
``,
|
|
3995
4145
|
`${c.bold}Options:${c.reset}`,
|
|
@@ -4087,12 +4237,14 @@ async function main() {
|
|
|
4087
4237
|
console.log(` ${c.cyan}audit${c.reset} <url> [url...] Deep LLM-powered security audit (~30s)`);
|
|
4088
4238
|
console.log(` ${c.cyan}validate${c.reset} [path] Validate SKILL.md format & security`);
|
|
4089
4239
|
console.log(` ${c.cyan}lookup${c.reset} <name> Look up package in registry`);
|
|
4240
|
+
console.log(` ${c.cyan}consensus${c.reset} <name> View multi-model consensus for a package`);
|
|
4090
4241
|
console.log();
|
|
4091
4242
|
console.log(` ${c.bold}COMMUNITY${c.reset}`);
|
|
4092
4243
|
console.log(` ${c.cyan}dashboard${c.reset} Interactive dashboard (full-screen)`);
|
|
4093
4244
|
console.log(` ${c.cyan}leaderboard${c.reset} Top contributors ranking`);
|
|
4094
4245
|
console.log(` ${c.cyan}benchmark${c.reset} LLM model performance comparison`);
|
|
4095
|
-
console.log(` ${c.cyan}
|
|
4246
|
+
console.log(` ${c.cyan}history${c.reset} Your local audit history`);
|
|
4247
|
+
console.log(` ${c.cyan}activity${c.reset} Your recent audits & findings (online)`);
|
|
4096
4248
|
console.log(` ${c.cyan}search${c.reset} <query> Search packages in registry`);
|
|
4097
4249
|
console.log();
|
|
4098
4250
|
console.log(` ${c.bold}CONFIGURATION${c.reset}`);
|
|
@@ -4106,6 +4258,7 @@ async function main() {
|
|
|
4106
4258
|
console.log(` ${c.dim}--quiet Suppress banner${c.reset}`);
|
|
4107
4259
|
console.log(` ${c.dim}--no-color Disable ANSI colors (also: NO_COLOR env)${c.reset}`);
|
|
4108
4260
|
console.log(` ${c.dim}--model <name> Override LLM model for this run${c.reset}`);
|
|
4261
|
+
console.log(` ${c.dim}--models <a,b,c> Multi-model audit (parallel, with consensus)${c.reset}`);
|
|
4109
4262
|
console.log(` ${c.dim}--no-upload Skip uploading report to registry${c.reset}`);
|
|
4110
4263
|
console.log(` ${c.dim}--export Export audit payload as markdown${c.reset}`);
|
|
4111
4264
|
console.log(` ${c.dim}--debug Show raw LLM response on parse errors${c.reset}`);
|
|
@@ -4114,6 +4267,7 @@ async function main() {
|
|
|
4114
4267
|
console.log(` agentaudit discover --quick`);
|
|
4115
4268
|
console.log(` agentaudit scan https://github.com/owner/repo`);
|
|
4116
4269
|
console.log(` agentaudit audit https://github.com/owner/repo`);
|
|
4270
|
+
console.log(` agentaudit audit <url> --models gemini-2.5-flash,claude-sonnet-4-20250514`);
|
|
4117
4271
|
console.log(` agentaudit lookup fastmcp --json`);
|
|
4118
4272
|
console.log();
|
|
4119
4273
|
console.log(` ${c.bold}LEARN MORE${c.reset}`);
|
|
@@ -4140,6 +4294,37 @@ async function main() {
|
|
|
4140
4294
|
await benchmarkCommand(targets);
|
|
4141
4295
|
return;
|
|
4142
4296
|
}
|
|
4297
|
+
if (command === 'history') {
|
|
4298
|
+
banner();
|
|
4299
|
+
const entries = loadHistory(30);
|
|
4300
|
+
if (entries.length === 0) {
|
|
4301
|
+
console.log(` ${c.dim}No local audit history yet. Run ${c.cyan}agentaudit audit <url>${c.dim} to start.${c.reset}`);
|
|
4302
|
+
console.log();
|
|
4303
|
+
return;
|
|
4304
|
+
}
|
|
4305
|
+
|
|
4306
|
+
if (jsonMode) {
|
|
4307
|
+
console.log(JSON.stringify(entries, null, 2));
|
|
4308
|
+
return;
|
|
4309
|
+
}
|
|
4310
|
+
|
|
4311
|
+
console.log(sectionHeader(`Local History (${entries.length})`));
|
|
4312
|
+
console.log();
|
|
4313
|
+
|
|
4314
|
+
for (const entry of entries) {
|
|
4315
|
+
const slug = entry.skill_slug || 'unknown';
|
|
4316
|
+
const risk = entry.risk_score ?? '?';
|
|
4317
|
+
const sev = entry.max_severity || 'none';
|
|
4318
|
+
const sc = severityColor(sev);
|
|
4319
|
+
const model = entry.audit_model || '?';
|
|
4320
|
+
const fc = entry.findings?.length || 0;
|
|
4321
|
+
const ts = entry._file?.slice(0, 10) || '';
|
|
4322
|
+
console.log(` ${sc}┃${c.reset} ${c.bold}${slug.padEnd(30)}${c.reset} ${riskBadge(risk)} ${c.dim}${model}${c.reset}`);
|
|
4323
|
+
console.log(` ${sc}┃${c.reset} ${c.dim}${ts} ${fc} findings ${sev.toUpperCase()}${c.reset}`);
|
|
4324
|
+
console.log();
|
|
4325
|
+
}
|
|
4326
|
+
return;
|
|
4327
|
+
}
|
|
4143
4328
|
if (command === 'activity' || command === 'my') {
|
|
4144
4329
|
await activityCommand(targets);
|
|
4145
4330
|
return;
|
|
@@ -4148,6 +4333,73 @@ async function main() {
|
|
|
4148
4333
|
await searchCommand(targets);
|
|
4149
4334
|
return;
|
|
4150
4335
|
}
|
|
4336
|
+
if (command === 'consensus') {
|
|
4337
|
+
banner();
|
|
4338
|
+
const pkg = targets[0];
|
|
4339
|
+
if (!pkg) {
|
|
4340
|
+
console.log(` ${c.red}Error: package name required${c.reset}`);
|
|
4341
|
+
console.log(` ${c.dim}Usage: ${c.cyan}agentaudit consensus <package-name>${c.reset}`);
|
|
4342
|
+
process.exitCode = 2;
|
|
4343
|
+
return;
|
|
4344
|
+
}
|
|
4345
|
+
const slug = pkg.toLowerCase().replace(/[^a-z0-9-]/g, '-');
|
|
4346
|
+
if (!jsonMode) console.log(` Fetching consensus for ${c.bold}${slug}${c.reset}...`);
|
|
4347
|
+
try {
|
|
4348
|
+
const res = await fetch(`${REGISTRY_URL}/api/packages/${slug}/consensus`, { signal: AbortSignal.timeout(10_000) });
|
|
4349
|
+
if (!res.ok) {
|
|
4350
|
+
if (res.status === 404) {
|
|
4351
|
+
console.log(` ${c.yellow}Not found${c.reset} — "${slug}" hasn't been audited yet.`);
|
|
4352
|
+
console.log(` ${c.dim}Run: ${c.cyan}agentaudit audit <repo-url>${c.dim} to create the first audit${c.reset}`);
|
|
4353
|
+
} else {
|
|
4354
|
+
console.log(` ${c.red}API error (HTTP ${res.status})${c.reset}`);
|
|
4355
|
+
}
|
|
4356
|
+
return;
|
|
4357
|
+
}
|
|
4358
|
+
const data = await res.json();
|
|
4359
|
+
if (jsonMode) { console.log(JSON.stringify(data, null, 2)); return; }
|
|
4360
|
+
|
|
4361
|
+
console.log();
|
|
4362
|
+
console.log(sectionHeader(`Consensus: ${slug}`));
|
|
4363
|
+
console.log();
|
|
4364
|
+
|
|
4365
|
+
// Status
|
|
4366
|
+
const status = data.consensus_status || data.status || 'pending';
|
|
4367
|
+
const statusColor = status === 'reached' ? c.green : status === 'disputed' ? c.yellow : c.dim;
|
|
4368
|
+
console.log(` Status: ${statusColor}${status.toUpperCase()}${c.reset}`);
|
|
4369
|
+
|
|
4370
|
+
// Risk + Severity
|
|
4371
|
+
if (data.consensus_risk_score != null) console.log(` Risk: ${riskBadge(data.consensus_risk_score)}`);
|
|
4372
|
+
if (data.consensus_severity) {
|
|
4373
|
+
const sc = severityColor(data.consensus_severity);
|
|
4374
|
+
console.log(` Severity: ${sc}${data.consensus_severity.toUpperCase()}${c.reset}`);
|
|
4375
|
+
}
|
|
4376
|
+
|
|
4377
|
+
// Models
|
|
4378
|
+
if (data.models && data.models.length > 0) {
|
|
4379
|
+
console.log();
|
|
4380
|
+
console.log(` ${c.bold}Models (${data.models.length}):${c.reset}`);
|
|
4381
|
+
for (const m of data.models) {
|
|
4382
|
+
const sc = severityColor(m.severity || m.max_severity);
|
|
4383
|
+
const risk = m.risk_score ?? '?';
|
|
4384
|
+
console.log(` ${sc}┃${c.reset} ${(m.model || m.audit_model || '?').padEnd(30)} ${c.dim}risk ${risk}${c.reset} ${sc}${(m.severity || m.max_severity || '').toUpperCase()}${c.reset}`);
|
|
4385
|
+
}
|
|
4386
|
+
}
|
|
4387
|
+
|
|
4388
|
+
// Reviewers
|
|
4389
|
+
if (data.reviews != null || data.reviewer_count != null) {
|
|
4390
|
+
const count = data.reviewer_count || data.reviews?.length || 0;
|
|
4391
|
+
console.log();
|
|
4392
|
+
console.log(` ${c.dim}Reviews: ${count} | Threshold: 5 reviewers, >60% agreement${c.reset}`);
|
|
4393
|
+
}
|
|
4394
|
+
|
|
4395
|
+
console.log();
|
|
4396
|
+
console.log(` ${c.dim}Full details: ${REGISTRY_URL}/packages/${slug}${c.reset}`);
|
|
4397
|
+
console.log();
|
|
4398
|
+
} catch (err) {
|
|
4399
|
+
console.log(` ${c.red}Failed: ${err.message}${c.reset}`);
|
|
4400
|
+
}
|
|
4401
|
+
return;
|
|
4402
|
+
}
|
|
4151
4403
|
|
|
4152
4404
|
banner();
|
|
4153
4405
|
|
|
@@ -4729,8 +4981,13 @@ async function main() {
|
|
|
4729
4981
|
|
|
4730
4982
|
let hasFindings = false;
|
|
4731
4983
|
for (const url of urls) {
|
|
4732
|
-
const
|
|
4733
|
-
|
|
4984
|
+
const result = await auditRepo(url);
|
|
4985
|
+
// Multi-model returns array, single-model returns object
|
|
4986
|
+
if (Array.isArray(result)) {
|
|
4987
|
+
if (result.some(r => r?.findings?.length > 0)) hasFindings = true;
|
|
4988
|
+
} else if (result?.findings?.length > 0) {
|
|
4989
|
+
hasFindings = true;
|
|
4990
|
+
}
|
|
4734
4991
|
}
|
|
4735
4992
|
process.exitCode = hasFindings ? 1 : 0;
|
|
4736
4993
|
return;
|
package/index.mjs
CHANGED
|
@@ -343,7 +343,7 @@ async function checkRegistry(slug) {
|
|
|
343
343
|
// ── MCP Server ───────────────────────────────────────────
|
|
344
344
|
|
|
345
345
|
const server = new Server(
|
|
346
|
-
{ name: 'agentaudit', version: '3.12.
|
|
346
|
+
{ name: 'agentaudit', version: '3.12.2' },
|
|
347
347
|
{ capabilities: { tools: {} } }
|
|
348
348
|
);
|
|
349
349
|
|