@kevinrabun/judges 3.38.0 → 3.41.0

package/dist/commands/interactive-fix.js

@@ -0,0 +1,140 @@
+/**
+ * `judges fix --interactive` — Interactive fix mode.
+ *
+ * Presents each auto-fixable finding one by one, letting the developer
+ * accept, skip, or view the diff before applying each patch.
+ * Reduces trust barrier by giving full control over what gets changed.
+ *
+ * Usage:
+ *   judges fix src/app.ts --interactive       # Interactive per-finding review
+ *   judges fix src/app.ts -I                  # Short form
+ */
+import { readFileSync, writeFileSync } from "fs";
+import { createInterface } from "readline";
+import { applyPatches } from "./fix.js";
+// ─── Diff Display ───────────────────────────────────────────────────────────
+function formatPatchDiff(patch) {
+    const lines = [];
+    lines.push(`  Line ${patch.startLine}–${patch.endLine}:`);
+    for (const line of patch.oldText.split("\n")) {
+        lines.push(`  \x1b[31m- ${line}\x1b[0m`);
+    }
+    for (const line of patch.newText.split("\n")) {
+        lines.push(`  \x1b[32m+ ${line}\x1b[0m`);
+    }
+    return lines.join("\n");
+}
+function formatFindingHeader(finding, index, total) {
+    const sevColors = {
+        critical: "\x1b[31;1m",
+        high: "\x1b[31m",
+        medium: "\x1b[33m",
+        low: "\x1b[36m",
+        info: "\x1b[37m",
+    };
+    const color = sevColors[finding.severity] || "\x1b[37m";
+    const reset = "\x1b[0m";
+    return [
+        "",
+        `═══════════════════════════════════════════════════════════════`,
+        `  [${index + 1}/${total}] ${color}${finding.severity.toUpperCase()}${reset} ${finding.ruleId} — ${finding.title}`,
+        `  ${finding.description}`,
+        `───────────────────────────────────────────────────────────────`,
+    ].join("\n");
+}
+// ─── Interactive Prompt ─────────────────────────────────────────────────────
+async function promptUser(question) {
+    const rl = createInterface({ input: process.stdin, output: process.stdout });
+    return new Promise((resolve) => {
+        rl.question(question, (answer) => {
+            rl.close();
+            resolve(answer.trim().toLowerCase());
+        });
+    });
+}
+// ─── Interactive Fix Runner ─────────────────────────────────────────────────
+/**
+ * Run interactive fix mode — present each fixable finding and let the user
+ * accept or skip it before applying.
+ */
+export async function runInteractiveFix(filePath, findings) {
+    const fixable = findings.filter((f) => f.patch);
+    if (fixable.length === 0) {
+        console.log("\n  No auto-fixable findings.\n");
+        return { accepted: 0, skipped: 0, total: 0 };
+    }
+    console.log(`\n  Found ${fixable.length} auto-fixable finding(s). Review each one:\n`);
+    const accepted = [];
+    let skipped = 0;
+    for (let i = 0; i < fixable.length; i++) {
+        const finding = fixable[i];
+        console.log(formatFindingHeader(finding, i, fixable.length));
+        console.log(formatPatchDiff(finding.patch));
+        console.log("");
+        const answer = await promptUser("  Apply this fix? [y]es / [n]o / [a]ll / [q]uit: ");
+        switch (answer) {
+            case "y":
+            case "yes":
+                accepted.push({
+                    ruleId: finding.ruleId,
+                    title: finding.title,
+                    severity: finding.severity,
+                    patch: finding.patch,
+                    lineNumbers: finding.lineNumbers,
+                });
+                console.log("  ✓ Accepted\n");
+                break;
+            case "a":
+            case "all":
+                // Accept this and all remaining
+                accepted.push({
+                    ruleId: finding.ruleId,
+                    title: finding.title,
+                    severity: finding.severity,
+                    patch: finding.patch,
+                    lineNumbers: finding.lineNumbers,
+                });
+                for (let j = i + 1; j < fixable.length; j++) {
+                    accepted.push({
+                        ruleId: fixable[j].ruleId,
+                        title: fixable[j].title,
+                        severity: fixable[j].severity,
+                        patch: fixable[j].patch,
+                        lineNumbers: fixable[j].lineNumbers,
+                    });
+                }
+                console.log(`  ✓ Accepted all remaining ${fixable.length - i} fix(es)\n`);
+                i = fixable.length; // break out of loop
+                break;
+            case "q":
+            case "quit":
+                skipped += fixable.length - i;
+                console.log("  Quit — remaining fixes skipped.\n");
+                i = fixable.length; // break out of loop
+                break;
+            case "n":
+            case "no":
+            default:
+                skipped++;
+                console.log("  ⏭ Skipped\n");
+                break;
+        }
+    }
+    // Apply accepted patches
+    if (accepted.length > 0) {
+        const code = readFileSync(filePath, "utf-8");
+        const result = applyPatches(code, accepted);
+        writeFileSync(filePath, result.result, "utf-8");
+        console.log(`  ✅ Applied ${result.applied} fix(es) to ${filePath}`);
+        if (result.skipped > 0) {
+            console.log(`  ⏭ ${result.skipped} fix(es) could not be applied (source changed)`);
+        }
+    }
+    console.log(`\n  Summary: ${accepted.length} accepted, ${skipped} skipped out of ${fixable.length}\n`);
+    return {
+        accepted: accepted.length,
+        skipped,
+        total: fixable.length,
+    };
+}
+//# sourceMappingURL=interactive-fix.js.map

package/dist/commands/interactive-fix.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"interactive-fix.js","sourceRoot":"","sources":["../../src/commands/interactive-fix.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,YAAY,EAAuB,MAAM,UAAU,CAAC;AAU7D,+EAA+E;AAE/E,SAAS,eAAe,CAAC,KAAY;IACnC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC;IAC1D,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,eAAe,IAAI,SAAS,CAAC,CAAC;IAC3C,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,eAAe,IAAI,SAAS,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAgB,EAAE,KAAa,EAAE,KAAa;IACzE,MAAM,SAAS,GAA2B;QACxC,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,UAAU;QAChB,MAAM,EAAE,UAAU;QAClB,GAAG,EAAE,UAAU;QACf,IAAI,EAAE,UAAU;KACjB,CAAC;IACF,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,UAAU,CAAC;IACxD,MAAM,KAAK,GAAG,SAAS,CAAC;IAExB,OAAO;QACL,EAAE;QACF,iEAAiE;QACjE,MAAM,KAAK,GAAG,CAAC,IAAI,KAAK,KAAK,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,WAAW,EAAE,GAAG,KAAK,IAAI,OAAO,CAAC,MAAM,MAAM,OAAO,CAAC,KAAK,EAAE;QAClH,KAAK,OAAO,CAAC,WAAW,EAAE;QAC1B,iEAAiE;KAClE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,+EAA+E;AAE/E,KAAK,UAAU,UAAU,CAAC,QAAgB;IACxC,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7E,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE;YAC/B,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB,EAAE,QAAmB;IAC3E,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAEhD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;QAC/C,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAC/C,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,aAAa,OAAO,CAAC,MAAM,8CAA8C,CAAC,CAAC;IAEvF,MAAM,QAAQ,GAAqB,EAAE,CAAC;IACtC,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE3B,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,KAAM,CAAC,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAEhB,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,mDAAmD,CAAC,CAAC;QAErF,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,GAAG,CAAC;YACT,KAAK,KAAK;gBACR,QAAQ,CAAC,IAAI,CAAC;oBACZ,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,KAAK,EAAE,OAAO,CAAC,KAAM;oBACrB,WAAW,EAAE,OAAO,CAAC,WAAW;iBACjC,CAAC,CAAC;gBACH,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;gBAC9B,MAAM;YAER,KAAK,GAAG,CAAC;YACT,KAAK,KAAK;gBACR,gCAAgC;gBAChC,QAAQ,CAAC,IAAI,CAAC;oBACZ,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,KAAK,EAAE,OAAO,CAAC,KAAM;oBACrB,WAAW,EAAE,OAAO,CAAC,WAAW;iBACjC,CAAC,CAAC;gBACH,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC5C,QAAQ,CAAC,IAAI,CAAC;wBACZ,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM;wBACzB,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK;wBACvB,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ;wBAC7B,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,KAAM;wBACxB,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW;qBACpC,CAAC,CAAC;gBACL,CAAC;gBACD,OAAO,CAAC,GAAG,CAAC,8BAA8B,OAAO,CAAC,MAAM,GAAG,CAAC,YAAY,CAAC,CAAC;gBAC1E,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,oBAAoB;gBACxC,MAAM;YAER,KAAK,GAAG,CAAC;YACT,KAAK,MAAM;gBACT,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;gBAC9B,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;gBACnD,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,oBAAoB;gBACxC,MAAM;YAER,KAAK,GAAG,CAAC;YACT,KAAK,IAAI,CAAC;YACV;gBACE,OAAO,EAAE,CAAC;gBACV,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;gBAC7B,MAAM;QACV,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC5C,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,eAAe,QAAQ,EAAE,CAAC,CAAC;QACpE,IAAI,MAAM,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,OAAO,MAAM,CAAC,OAAO,gDAAgD,CAAC,CAAC;QACrF,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,gBAAgB,QAAQ,CAAC,MAAM,cAAc,OAAO,mBAAmB,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;IAEvG,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,MAAM;QACzB,OAAO;QACP,KAAK,EAAE,OAAO,CAAC,MAAM;KACtB,CAAC;AACJ,CAAC"}

package/dist/commands/llm-benchmark.d.ts

@@ -0,0 +1,119 @@
+/**
+ * LLM Benchmark — types, scoring, and formatting for the probabilistic
+ * (LLM prompt) layer of the Judges Panel benchmark.
+ *
+ * This module does NOT make LLM API calls. It provides:
+ * - Types for LLM benchmark snapshots
+ * - Rule-ID parsing from LLM response text
+ * - Prompt construction (mirrors MCP-served prompts exactly)
+ * - Scoring logic (same methodology as L1 deterministic benchmark)
+ *
+ * The actual LLM API calls live in scripts/run-llm-benchmark.ts,
+ * keeping the npm package free of LLM API dependencies.
+ */
+import type { JudgeDefinition } from "../types.js";
+import type { BenchmarkCase, CategoryResult, JudgeBenchmarkResult, DifficultyResult } from "./benchmark.js";
+export interface LlmBenchmarkSnapshot {
+    /** Timestamp of this LLM benchmark run */
+    timestamp: string;
+    /** Version of judges used */
+    version: string;
+    /** LLM model used (e.g. "gpt-4o", "claude-sonnet-4-20250514") */
+    model: string;
+    /** API provider (e.g. "openai", "anthropic") */
+    provider: string;
+    /** Prompt mode used: full-tribunal or per-judge */
+    promptMode: "tribunal" | "per-judge";
+    /** Total test cases run */
+    totalCases: number;
+    /** Cases where at least one expected rule was detected */
+    detected: number;
+    /** Cases where no expected rule was detected */
+    missed: number;
+    /** Total expected findings across all cases */
+    totalExpected: number;
+    /** True positives (prefix-based matching) */
+    truePositives: number;
+    /** False negatives (prefix-based matching) */
+    falseNegatives: number;
+    /** False positives */
+    falsePositives: number;
+    /** Precision: TP / (TP + FP) */
+    precision: number;
+    /** Recall: TP / (TP + FN) */
+    recall: number;
+    /** F1 Score */
+    f1Score: number;
+    /** Detection rate: cases detected / total cases */
+    detectionRate: number;
+    /** Per-category breakdown */
+    perCategory: Record<string, CategoryResult>;
+    /** Per-judge breakdown */
+    perJudge: Record<string, JudgeBenchmarkResult>;
+    /** Per-difficulty breakdown */
+    perDifficulty: Record<string, DifficultyResult>;
+    /** Individual case results */
+    cases: LlmCaseResult[];
+    /** Total tokens used (if available from API) */
+    totalTokensUsed?: number;
+    /** Total run duration in seconds */
+    durationSeconds: number;
+}
+export interface LlmCaseResult {
+    caseId: string;
+    category: string;
+    difficulty: string;
+    passed: boolean;
+    expectedRuleIds: string[];
+    detectedRuleIds: string[];
+    missedRuleIds: string[];
+    falsePositiveRuleIds: string[];
+    /** Raw LLM response text */
+    rawResponse: string;
+    /** Tokens used for this case */
+    tokensUsed?: number;
+}
+/**
+ * Extract unique rule IDs from LLM response text.
+ * Matches patterns like CYBER-001, SEC-003, AUTH-001, etc.
+ */
+export declare function parseLlmRuleIds(response: string): string[];
+/**
+ * Construct a per-judge prompt — identical to the MCP-served `judge-{id}` prompt.
+ */
+export declare function constructPerJudgePrompt(judge: JudgeDefinition, code: string, language: string): string;
+/**
+ * Construct the full-tribunal prompt — identical to the MCP-served `full-tribunal` prompt.
+ */
+export declare function constructTribunalPrompt(code: string, language: string): string;
+/**
+ * Select a stratified sample of benchmark cases, ensuring representation
+ * across categories, difficulties, and both clean/dirty cases.
+ */
+export declare function selectStratifiedSample(cases: BenchmarkCase[], targetSize: number): BenchmarkCase[];
+/**
+ * Score a single LLM benchmark case using prefix-based matching.
+ * Returns a fully populated LlmCaseResult.
+ */
+export declare function scoreLlmCase(tc: BenchmarkCase, detectedRuleIds: string[], rawResponse: string, tokensUsed?: number): LlmCaseResult;
+/**
+ * Compute aggregate metrics for an LLM benchmark snapshot from raw case results.
+ * Uses the same prefix-based matching methodology as the L1 benchmark.
+ */
+export declare function computeLlmMetrics(rawCases: LlmCaseResult[], version: string, model: string, provider: string, promptMode: "tribunal" | "per-judge", durationSeconds: number, totalTokensUsed?: number): LlmBenchmarkSnapshot;
+/**
+ * Format an LLM benchmark snapshot as a markdown section for the benchmark report.
+ */
+export declare function formatLlmSnapshotMarkdown(snapshot: LlmBenchmarkSnapshot): string;
+/**
+ * Format a layer comparison table showing L1 vs L2 side by side.
+ */
+export declare function formatLayerComparisonMarkdown(l1: {
+    detectionRate: number;
+    precision: number;
+    recall: number;
+    f1Score: number;
+    falsePositives: number;
+    totalCases: number;
+}, l2: LlmBenchmarkSnapshot): string;
+//# sourceMappingURL=llm-benchmark.d.ts.map

package/dist/commands/llm-benchmark.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-benchmark.d.ts","sourceRoot":"","sources":["../../src/commands/llm-benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAM5G,MAAM,WAAW,oBAAoB;IACnC,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,6BAA6B;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,iEAAiE;IACjE,KAAK,EAAE,MAAM,CAAC;IACd,gDAAgD;IAChD,QAAQ,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,UAAU,EAAE,UAAU,GAAG,WAAW,CAAC;IACrC,2BAA2B;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,gDAAgD;IAChD,MAAM,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,aAAa,EAAE,MAAM,CAAC;IACtB,6CAA6C;IAC7C,aAAa,EAAE,MAAM,CAAC;IACtB,8CAA8C;IAC9C,cAAc,EAAE,MAAM,CAAC;IACvB,sBAAsB;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,gCAAgC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,6BAA6B;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,eAAe;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,aAAa,EAAE,MAAM,CAAC;IACtB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC5C,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC;IAC/C,+BAA+B;IAC/B,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAChD,8BAA8B;IAC9B,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,gDAAgD;IAChD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oCAAoC;IACpC,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;IAChB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,oBAAoB,EAAE,MAAM,EAAE,CAAC;IAC/B,4BAA4B;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAID;;;GAGG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAW1D;AAOD;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAMtG;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAqB9E;AAID;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,aAAa,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE,CAqDlG;AAID;;;GAGG;AACH,wBAAgB,YAAY,CAC1B,EAAE,EAAE,aAAa,EACjB,eAAe,EAAE,MAAM,EAAE,EACzB,WAAW,EAAE,MAAM,EACnB,UAAU,CAAC,EAAE,MAAM,GAClB,aAAa,CAmCf;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,aAAa,EAAE,EACzB,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,UAAU,GAAG,WAAW,EACpC,eAAe,EAAE,MAAM,EACvB,eAAe,CAAC,EAAE,MAAM,GACvB,oBAAoB,CA+HtB;AAID;;GAEG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,oBAAoB,GAAG,MAAM,CAkGhF;AAED;;GAEG;AACH,wBAAgB,6BAA6B,CAC3C,EAAE,EAAE;IACF,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;CACpB,EACD,EAAE,EAAE,oBAAoB,GACvB,MAAM,CAsBR"}

package/dist/commands/llm-benchmark.js

@@ -0,0 +1,396 @@
+/**
+ * LLM Benchmark — types, scoring, and formatting for the probabilistic
+ * (LLM prompt) layer of the Judges Panel benchmark.
+ *
+ * This module does NOT make LLM API calls. It provides:
+ * - Types for LLM benchmark snapshots
+ * - Rule-ID parsing from LLM response text
+ * - Prompt construction (mirrors MCP-served prompts exactly)
+ * - Scoring logic (same methodology as L1 deterministic benchmark)
+ *
+ * The actual LLM API calls live in scripts/run-llm-benchmark.ts,
+ * keeping the npm package free of LLM API dependencies.
+ */
+import { JUDGES } from "../judges/index.js";
+import { getCondensedCriteria, SHARED_ADVERSARIAL_MANDATE, PRECISION_MANDATE } from "../tools/prompts.js";
+// ─── Rule ID Parsing ────────────────────────────────────────────────────────
+/**
+ * Extract unique rule IDs from LLM response text.
+ * Matches patterns like CYBER-001, SEC-003, AUTH-001, etc.
+ */
+export function parseLlmRuleIds(response) {
+    const validPrefixes = new Set(JUDGES.map((j) => j.rulePrefix));
+    const pattern = /\b([A-Z]{2,})-(\d{3})\b/g;
+    const found = new Set();
+    let match;
+    while ((match = pattern.exec(response)) !== null) {
+        if (validPrefixes.has(match[1])) {
+            found.add(match[0]);
+        }
+    }
+    return [...found];
+}
+// ─── Prompt Construction ────────────────────────────────────────────────────
+// These construct the exact same prompts served via MCP, ensuring the
+// benchmark tests the same prompts real users experience.
+// ─────────────────────────────────────────────────────────────────────────────
+/**
+ * Construct a per-judge prompt — identical to the MCP-served `judge-{id}` prompt.
+ */
+export function constructPerJudgePrompt(judge, code, language) {
+    return (`${judge.systemPrompt}\n\n${PRECISION_MANDATE}\n\n` +
+        `Please evaluate the following ${language} code:\n\n\`\`\`${language}\n${code}\n\`\`\`` +
+        `\n\nProvide your evaluation as structured findings with rule IDs (prefix: ${judge.rulePrefix}-), severity levels (critical/high/medium/low/info), descriptions, and actionable recommendations. End with an overall score (0-100) and verdict (pass/warning/fail).`);
+}
+/**
+ * Construct the full-tribunal prompt — identical to the MCP-served `full-tribunal` prompt.
+ */
+export function constructTribunalPrompt(code, language) {
+    const judgeInstructions = JUDGES.map((j) => `### ${j.name} — ${j.domain}\n**Rule prefix:** \`${j.rulePrefix}-\`\n\n${getCondensedCriteria(j.systemPrompt)}`).join("\n\n---\n\n");
+    return (`You are the Judges Panel — a panel of ${JUDGES.length} expert judges who independently evaluate code for quality, security, and operational readiness.\n\n` +
+        `## Universal Evaluation Directives\n\n` +
+        `${SHARED_ADVERSARIAL_MANDATE}\n\n` +
+        `${PRECISION_MANDATE}\n\n` +
+        `## Evaluation Instructions\n\n` +
+        `Evaluate the following ${language} code from the perspective of ALL ${JUDGES.length} judges below. For each judge, provide:\n` +
+        `1. Judge name and domain\n` +
+        `2. Verdict (PASS / WARNING / FAIL)\n` +
+        `3. Score (0-100)\n` +
+        `4. Specific findings with rule IDs (using each judge's rule prefix), severity, and recommendations\n\n` +
+        `Then provide an OVERALL TRIBUNAL VERDICT that synthesizes all judges' input.\n\n` +
+        `## The Judges\n\n${judgeInstructions}\n\n` +
+        `## Code to Evaluate\n\n\`\`\`${language}\n${code}\n\`\`\``);
+}
+// ─── Stratified Sampling ────────────────────────────────────────────────────
+/**
+ * Select a stratified sample of benchmark cases, ensuring representation
+ * across categories, difficulties, and both clean/dirty cases.
+ */
+export function selectStratifiedSample(cases, targetSize) {
+    if (targetSize >= cases.length)
+        return [...cases];
+    // Group by category
+    const byCategory = {};
+    for (const c of cases) {
+        if (!byCategory[c.category])
+            byCategory[c.category] = [];
+        byCategory[c.category].push(c);
+    }
+    const categories = Object.keys(byCategory);
+    const perCategory = Math.max(2, Math.floor(targetSize / categories.length));
+    const selected = [];
+    const usedIds = new Set();
+    for (const cat of categories) {
+        const pool = byCategory[cat];
+        // Ensure difficulty coverage within each category
+        const byDiff = {};
+        for (const c of pool) {
+            if (!byDiff[c.difficulty])
+                byDiff[c.difficulty] = [];
+            byDiff[c.difficulty].push(c);
+        }
+        let taken = 0;
+        for (const diff of ["easy", "medium", "hard"]) {
+            if (taken >= perCategory)
+                break;
+            const diffPool = byDiff[diff] || [];
+            const remaining = perCategory - taken;
+            const diffCount = diff === "easy" ? 3 : diff === "medium" ? 2 : 1;
+            const toTake = Math.max(1, Math.ceil(remaining / diffCount));
+            for (let i = 0; i < Math.min(toTake, diffPool.length) && taken < perCategory; i++) {
+                if (!usedIds.has(diffPool[i].id)) {
+                    selected.push(diffPool[i]);
+                    usedIds.add(diffPool[i].id);
+                    taken++;
+                }
+            }
+        }
+    }
+    // Fill remaining slots if under target
+    if (selected.length < targetSize) {
+        for (const c of cases) {
+            if (selected.length >= targetSize)
+                break;
+            if (!usedIds.has(c.id)) {
+                selected.push(c);
+                usedIds.add(c.id);
+            }
+        }
+    }
+    return selected;
+}
+// ─── Scoring ────────────────────────────────────────────────────────────────
+/**
+ * Score a single LLM benchmark case using prefix-based matching.
+ * Returns a fully populated LlmCaseResult.
+ */
+export function scoreLlmCase(tc, detectedRuleIds, rawResponse, tokensUsed) {
+    const _expectedPrefixes = new Set(tc.expectedRuleIds.map((r) => r.split("-")[0]));
+    const detectedPrefixes = new Set(detectedRuleIds.map((r) => r.split("-")[0]));
+    const matchedExpected = tc.expectedRuleIds.filter((expected) => {
+        const prefix = expected.split("-")[0];
+        return detectedPrefixes.has(prefix);
+    });
+    const missedExpected = tc.expectedRuleIds.filter((expected) => {
+        const prefix = expected.split("-")[0];
+        return !detectedPrefixes.has(prefix);
+    });
+    const falsePositiveIds = tc.unexpectedRuleIds
+        ? detectedRuleIds.filter((found) => {
+            const prefix = found.split("-")[0];
+            return tc.unexpectedRuleIds.some((u) => u.split("-")[0] === prefix);
+        })
+        : [];
+    const casePassed = tc.expectedRuleIds.length === 0 ? falsePositiveIds.length === 0 : matchedExpected.length > 0;
+    return {
+        caseId: tc.id,
+        category: tc.category,
+        difficulty: tc.difficulty,
+        passed: casePassed,
+        expectedRuleIds: tc.expectedRuleIds,
+        detectedRuleIds,
+        missedRuleIds: missedExpected,
+        falsePositiveRuleIds: falsePositiveIds,
+        rawResponse,
+        tokensUsed,
+    };
+}
+/**
+ * Compute aggregate metrics for an LLM benchmark snapshot from raw case results.
+ * Uses the same prefix-based matching methodology as the L1 benchmark.
+ */
+export function computeLlmMetrics(rawCases, version, model, provider, promptMode, durationSeconds, totalTokensUsed) {
+    const perCategory = {};
+    const perJudge = {};
+    const perDifficulty = {};
+    let totalTP = 0;
+    let totalFN = 0;
+    let totalFP = 0;
+    let totalDetected = 0;
+    for (const c of rawCases) {
+        const caseTP = c.expectedRuleIds.length - c.missedRuleIds.length;
+        const caseFN = c.missedRuleIds.length;
+        const caseFP = c.falsePositiveRuleIds.length;
+        if (c.passed)
+            totalDetected++;
+        totalTP += caseTP;
+        totalFN += caseFN;
+        totalFP += caseFP;
+        // Per-difficulty
+        if (!perDifficulty[c.difficulty]) {
+            perDifficulty[c.difficulty] = { difficulty: c.difficulty, total: 0, detected: 0, detectionRate: 0 };
+        }
+        perDifficulty[c.difficulty].total++;
+        if (c.passed)
+            perDifficulty[c.difficulty].detected++;
+        // Per-category
+        if (!perCategory[c.category]) {
+            perCategory[c.category] = {
+                category: c.category,
+                total: 0,
+                detected: 0,
+                truePositives: 0,
+                falseNegatives: 0,
+                falsePositives: 0,
+                precision: 0,
+                recall: 0,
+                f1Score: 0,
+            };
+        }
+        const cat = perCategory[c.category];
+        cat.total++;
+        if (c.passed)
+            cat.detected++;
+        cat.truePositives += caseTP;
+        cat.falseNegatives += caseFN;
+        cat.falsePositives += caseFP;
+        // Per-judge
+        const expectedPrefixes = new Set(c.expectedRuleIds.map((r) => r.split("-")[0]));
+        const isCleanCase = c.expectedRuleIds.length === 0;
+        for (const ruleId of c.detectedRuleIds) {
+            const prefix = ruleId.split("-")[0];
+            if (!perJudge[prefix]) {
+                perJudge[prefix] = {
+                    judgeId: prefix,
+                    total: 0,
+                    truePositives: 0,
+                    falseNegatives: 0,
+                    falsePositives: 0,
+                    precision: 0,
+                    recall: 0,
+                    f1Score: 0,
+                };
+            }
+            const jb = perJudge[prefix];
+            jb.total++;
+            if (expectedPrefixes.has(prefix)) {
+                jb.truePositives++;
+            }
+            else if (isCleanCase) {
+                jb.falsePositives++;
+            }
+        }
+    }
+    // Compute aggregate metrics
+    const precision = totalTP + totalFP > 0 ? totalTP / (totalTP + totalFP) : 1;
+    const recall = totalTP + totalFN > 0 ? totalTP / (totalTP + totalFN) : 1;
+    const f1Score = precision + recall > 0 ? (2 * precision * recall) / (precision + recall) : 0;
+    // Compute per-difficulty rates
+    for (const d of Object.values(perDifficulty)) {
+        d.detectionRate = d.total > 0 ? d.detected / d.total : 0;
+    }
+    // Compute per-category metrics
+    for (const cat of Object.values(perCategory)) {
+        cat.precision =
+            cat.truePositives + cat.falsePositives > 0 ? cat.truePositives / (cat.truePositives + cat.falsePositives) : 1;
+        cat.recall =
+            cat.truePositives + cat.falseNegatives > 0 ? cat.truePositives / (cat.truePositives + cat.falseNegatives) : 1;
+        cat.f1Score = cat.precision + cat.recall > 0 ? (2 * cat.precision * cat.recall) / (cat.precision + cat.recall) : 0;
+    }
+    // Compute per-judge metrics
+    for (const jb of Object.values(perJudge)) {
+        jb.precision =
+            jb.truePositives + jb.falsePositives > 0 ? jb.truePositives / (jb.truePositives + jb.falsePositives) : 1;
+        jb.recall =
+            jb.truePositives + jb.falseNegatives > 0 ? jb.truePositives / (jb.truePositives + jb.falseNegatives) : 1;
+        jb.f1Score = jb.precision + jb.recall > 0 ? (2 * jb.precision * jb.recall) / (jb.precision + jb.recall) : 0;
+    }
+    return {
+        timestamp: new Date().toISOString(),
+        version,
+        model,
+        provider,
+        promptMode,
+        totalCases: rawCases.length,
+        detected: totalDetected,
+        missed: rawCases.length - totalDetected,
+        totalExpected: rawCases.reduce((s, c) => s + c.expectedRuleIds.length, 0),
+        truePositives: totalTP,
+        falseNegatives: totalFN,
+        falsePositives: totalFP,
+        precision,
+        recall,
+        f1Score,
+        detectionRate: rawCases.length > 0 ? totalDetected / rawCases.length : 0,
+        perCategory,
+        perJudge,
+        perDifficulty,
+        cases: rawCases,
+        durationSeconds,
+        ...(totalTokensUsed ? { totalTokensUsed } : {}),
+    };
+}
+// ─── Markdown Formatting ────────────────────────────────────────────────────
+/**
+ * Format an LLM benchmark snapshot as a markdown section for the benchmark report.
+ */
+export function formatLlmSnapshotMarkdown(snapshot) {
+    const lines = [];
+    const pct = (n) => `${(n * 100).toFixed(1)}%`;
+    const l2Grade = snapshot.f1Score >= 0.9
+        ? "A"
+        : snapshot.f1Score >= 0.8
+            ? "B"
+            : snapshot.f1Score >= 0.7
+                ? "C"
+                : snapshot.f1Score >= 0.6
+                    ? "D"
+                    : "F";
+    const l2GradeEmoji = l2Grade === "A" ? "🟢" : l2Grade === "B" ? "🟡" : l2Grade === "C" ? "🟠" : "🔴";
+    lines.push("## Layer 2 — LLM Deep Review");
+    lines.push("");
+    lines.push(`> Model: **${snapshot.model}** · Provider: ${snapshot.provider} · ${snapshot.totalCases} test cases · ${new Date(snapshot.timestamp).toLocaleDateString()}`);
+    lines.push("");
+    lines.push(`| Metric | Value |`);
+    lines.push(`|--------|-------|`);
+    lines.push(`| Overall Grade | ${l2GradeEmoji} **${l2Grade}** |`);
+    lines.push(`| Test Cases | ${snapshot.totalCases} |`);
+    lines.push(`| Detection Rate | ${pct(snapshot.detectionRate)} (${snapshot.detected}/${snapshot.totalCases}) |`);
+    lines.push(`| Precision | ${pct(snapshot.precision)} |`);
+    lines.push(`| Recall | ${pct(snapshot.recall)} |`);
+    lines.push(`| F1 Score | ${pct(snapshot.f1Score)} |`);
+    lines.push(`| True Positives | ${snapshot.truePositives} |`);
+    lines.push(`| False Negatives | ${snapshot.falseNegatives} |`);
+    lines.push(`| False Positives | ${snapshot.falsePositives} |`);
+    if (snapshot.totalTokensUsed) {
+        lines.push(`| Total Tokens | ${snapshot.totalTokensUsed.toLocaleString()} |`);
+    }
+    lines.push(`| Run Duration | ${snapshot.durationSeconds}s |`);
+    lines.push("");
+    // Per-difficulty
+    if (Object.keys(snapshot.perDifficulty).length > 0) {
+        lines.push("### L2 Detection by Difficulty");
+        lines.push("");
+        lines.push("| Difficulty | Detected | Total | Rate |");
+        lines.push("|------------|----------|-------|------|");
+        for (const diff of ["easy", "medium", "hard"]) {
+            const d = snapshot.perDifficulty[diff];
+            if (d) {
+                lines.push(`| ${diff} | ${d.detected} | ${d.total} | ${pct(d.detectionRate)} |`);
+            }
+        }
+        lines.push("");
+    }
+    // Per-category
+    if (Object.keys(snapshot.perCategory).length > 0) {
+        lines.push("### L2 Results by Category");
+        lines.push("");
+        lines.push("| Category | Detected | Total | Precision | Recall | F1 |");
+        lines.push("|----------|----------|-------|-----------|--------|-----|");
+        for (const [cat, stats] of Object.entries(snapshot.perCategory).sort(([a], [b]) => a.localeCompare(b))) {
+            lines.push(`| ${cat} | ${stats.detected} | ${stats.total} | ${pct(stats.precision)} | ${pct(stats.recall)} | ${pct(stats.f1Score)} |`);
+        }
+        lines.push("");
+    }
+    // Per-judge
+    if (Object.keys(snapshot.perJudge).length > 0) {
+        lines.push("### L2 Results by Judge");
+        lines.push("");
+        lines.push("| Judge | Findings | TP | FP | Precision |");
+        lines.push("|-------|----------|-----|-----|-----------|");
+        for (const [judgeId, stats] of Object.entries(snapshot.perJudge).sort(([a], [b]) => a.localeCompare(b))) {
+            lines.push(`| ${judgeId} | ${stats.total} | ${stats.truePositives} | ${stats.falsePositives} | ${pct(stats.precision)} |`);
+        }
+        lines.push("");
+    }
+    // Failed cases
+    const failed = snapshot.cases.filter((c) => !c.passed);
+    if (failed.length > 0 && failed.length <= 50) {
+        lines.push("### L2 Failed Cases");
+        lines.push("");
+        lines.push("| Case | Difficulty | Category | Missed Rules | False Positives |");
+        lines.push("|------|------------|----------|--------------|-----------------|");
+        for (const c of failed) {
+            const missed = c.missedRuleIds.length > 0 ? c.missedRuleIds.join(", ") : "—";
+            const fps = c.falsePositiveRuleIds.length > 0 ? c.falsePositiveRuleIds.join(", ") : "—";
+            lines.push(`| ${c.caseId} | ${c.difficulty} | ${c.category} | ${missed} | ${fps} |`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+/**
+ * Format a layer comparison table showing L1 vs L2 side by side.
+ */
+export function formatLayerComparisonMarkdown(l1, l2) {
+    const lines = [];
+    const pct = (n) => `${(n * 100).toFixed(1)}%`;
+    lines.push("## Layer Comparison");
+    lines.push("");
+    lines.push(`| Metric | L1 (Deterministic) | L2 (${l2.model}) |`);
+    lines.push(`|--------|-------------------|${"-".repeat(Math.max(l2.model.length + 4, 5))}|`);
+    lines.push(`| Test Cases | ${l1.totalCases} | ${l2.totalCases} |`);
+    lines.push(`| Detection Rate | ${pct(l1.detectionRate)} | ${pct(l2.detectionRate)} |`);
+    lines.push(`| Precision | ${pct(l1.precision)} | ${pct(l2.precision)} |`);
+    lines.push(`| Recall | ${pct(l1.recall)} | ${pct(l2.recall)} |`);
+    lines.push(`| F1 Score | ${pct(l1.f1Score)} | ${pct(l2.f1Score)} |`);
+    lines.push(`| False Positives | ${l1.falsePositives} | ${l2.falsePositives} |`);
+    lines.push("");
+    lines.push("The two layers are **complementary**:");
+    lines.push("- **L1** provides fast, reliable baseline detection with high precision and zero cost");
+    lines.push("- **L2** catches sophisticated issues that patterns miss, at the cost of API calls and latency");
+    lines.push("- Together, they deliver defense-in-depth code analysis");
+    lines.push("");
+    return lines.join("\n");
+}
+//# sourceMappingURL=llm-benchmark.js.map