@kevinrabun/judges-cli 3.124.0

package/dist/commands/llm-benchmark.js

@@ -0,0 +1,670 @@
+/**
+ * LLM Benchmark — types, scoring, and formatting for the probabilistic
+ * (LLM prompt) layer of the Judges Panel benchmark.
+ *
+ * This module does NOT make LLM API calls. It provides:
+ * - Types for LLM benchmark snapshots
+ * - Rule-ID parsing from LLM response text
+ * - Prompt construction (mirrors MCP-served prompts exactly)
+ * - Scoring logic (same methodology as L1 deterministic benchmark)
+ *
+ * LLM API calling is intentionally kept out of the npm package. Wire this
+ * to your preferred provider in a thin runner script (or use the CLI
+ * command `judges llm-benchmark`). The former helper script
+ * `scripts/run-llm-benchmark.ts` has been removed.
+ */
+import { JUDGES } from "../judges/index.js";
+import { getCondensedCriteria, SHARED_ADVERSARIAL_MANDATE, PRECISION_MANDATE, CLEAN_CODE_GATE, } from "../tools/prompts.js";
+import { extractAndValidateLlmFindings, mergeFindings } from "../probabilistic/llm-response-validator.js";
+import { formatAmendmentSection } from "./llm-benchmark-optimizer.js";
+// ─── Tribunal Judge Filtering ───────────────────────────────────────────────
+// Meta-judges that assess analysis quality rather than code quality produce
+// near-100% false positives in single-pass tribunal mode and are excluded.
+const TRIBUNAL_EXCLUDED_PREFIXES = new Set(["INTENT", "COH", "MFPR", "FPR", "OVER"]);
+export const TRIBUNAL_JUDGES = JUDGES.filter((j) => !TRIBUNAL_EXCLUDED_PREFIXES.has(j.rulePrefix));
+// ─── Category → Acceptable Prefixes Mapping ────────────────────────────────
+// For each benchmark case category, these judge prefixes are domain-relevant
+// and findings from them should NOT count as false positives even when not
+// in expectedRuleIds. This prevents legitimate cross-domain observations
+// from inflating the FP metric.
+// ─────────────────────────────────────────────────────────────────────────────
+const CATEGORY_ACCEPTABLE_PREFIXES = {
+    injection: ["SEC", "CYBER", "DB", "DATA", "ERR", "FW", "LOGIC"],
+    xss: ["SEC", "CYBER", "FW", "DATA", "ERR", "LOGIC"],
+    auth: ["AUTH", "SEC", "CYBER", "DATA", "CFG", "ERR", "LOGIC"],
+    "rate-limiting": ["RATE", "PERF", "SCALE", "REL", "SEC", "ERR"],
+    "error-handling": ["ERR", "REL", "OBS", "LOGIC", "MAINT", "STRUCT"],
+    "data-security": ["DATA", "SEC", "CYBER", "LOGPRIV", "SOV", "CFG", "ERR"],
+    security: ["SEC", "CYBER", "AUTH", "DATA", "FW", "CFG", "ERR", "LOGIC"],
+    concurrency: ["CONC", "PERF", "REL", "LOGIC", "ERR", "MAINT"],
+    performance: ["PERF", "SCALE", "CACHE", "DB", "CONC", "LOGIC", "MAINT"],
+    database: ["DB", "SEC", "DATA", "PERF", "ERR", "LOGIC"],
+    "api-design": ["API", "ERR", "AUTH", "SEC", "STRUCT", "LOGIC", "MAINT"],
+    observability: ["OBS", "LOGPRIV", "REL", "ERR", "CFG", "MAINT"],
+    reliability: ["REL", "ERR", "CONC", "PERF", "OBS", "LOGIC"],
+    scalability: ["SCALE", "PERF", "CACHE", "CLOUD", "CONC", "STRUCT"],
+    "cloud-readiness": ["CLOUD", "CFG", "CICD", "SCALE", "PORTA", "SEC"],
+    configuration: ["CFG", "SEC", "DATA", "CLOUD", "ERR"],
+    maintainability: ["MAINT", "STRUCT", "SWDEV", "DOC", "LOGIC", "ERR"],
+    "code-structure": ["STRUCT", "MAINT", "SWDEV", "LOGIC", "ERR"],
+    documentation: ["DOC", "MAINT", "SWDEV", "STRUCT"],
+    testing: ["TEST", "SWDEV", "LOGIC", "ERR", "MAINT"],
+    "cost-effectiveness": ["COST", "CLOUD", "SCALE", "PERF", "IAC"],
+    compliance: ["COMP", "DATA", "SOV", "LOGPRIV", "SEC", "CYBER"],
+    accessibility: ["A11Y", "UX", "I18N", "STRUCT", "LOGIC"],
+    internationalization: ["I18N", "A11Y", "UX", "STRUCT"],
+    "dependency-health": ["DEPS", "SEC", "COMPAT", "MAINT"],
+    "logging-privacy": ["LOGPRIV", "DATA", "OBS", "SEC", "ERR"],
+    "backwards-compatibility": ["COMPAT", "API", "STRUCT", "LOGIC"],
+    caching: ["CACHE", "PERF", "SCALE", "REL", "LOGIC"],
+    "ethics-bias": ["ETHICS", "DATA", "COMP", "SEC"],
+    portability: ["PORTA", "CLOUD", "STRUCT", "CFG"],
+    "ci-cd": ["CICD", "SEC", "CFG", "CLOUD", "TEST"],
+    "iac-security": ["IAC", "SEC", "CYBER", "CFG", "CLOUD"],
+    cloud: ["CLOUD", "IAC", "SEC", "CYBER", "CFG", "SCALE"],
+    ethics: ["ETHICS", "A11Y", "UX", "DATA", "COMP"],
+    "framework-safety": ["FW", "SEC", "CYBER", "ERR", "LOGIC"],
+    "framework-security": ["FW", "SEC", "CYBER", "AUTH", "ERR", "API", "COMP", "OBS", "COMPAT", "CONC", "DOC"],
+    "agent-instructions": ["AGENT", "SEC", "CYBER", "AICS", "ERR", "LOGIC"],
+    cicd: ["CICD", "SEC", "CFG", "CLOUD", "TEST", "PORTA"],
+    ux: ["UX", "ERR", "SEC", "A11Y", "I18N", "LOGIC"],
+    "software-practices": ["SWDEV", "MAINT", "STRUCT", "DOC", "LOGIC", "ERR"],
+    "software-development": ["SWDEV", "MAINT", "STRUCT", "DOC", "LOGIC", "ERR"],
+    "code-quality": ["MAINT", "API", "STRUCT", "SWDEV", "LOGIC", "ERR"],
+    "supply-chain": ["DEPS", "SEC", "COMPAT", "MAINT"],
+    "ai-security": ["AICS", "SEC", "CYBER", "DATA", "ERR", "LOGIC"],
+    structure: [
+        "STRUCT",
+        "MAINT",
+        "SWDEV",
+        "ERR",
+        "LOGIC",
+        "SEC",
+        "PERF",
+        "API",
+        "DB",
+        "OBS",
+        "AUTH",
+        "CACHE",
+        "SCALE",
+        "CYBER",
+        "AICS",
+        "RATE",
+        "CONC",
+        "COST",
+        "UX",
+        "COMPAT",
+    ],
+    clean: [], // Clean code — no acceptable prefixes, all findings are FPs
+};
+/**
+ * Get acceptable prefixes for a benchmark case. Uses the case's explicit
+ * acceptablePrefixes if defined, otherwise falls back to the category map.
+ * Expected prefixes are always included (they're TPs, not FPs).
+ */
+export function getAcceptablePrefixes(tc) {
+    const explicit = tc.acceptablePrefixes;
+    const fromCategory = CATEGORY_ACCEPTABLE_PREFIXES[tc.category] ?? [];
+    const combined = new Set([...tc.expectedRuleIds.map((r) => r.split("-")[0]), ...(explicit ?? fromCategory)]);
+    return combined;
+}
+// ─── Core Judges (always included in routed tribunals) ──────────────────────
+// These judges provide universal code quality signals and should always be
+// part of the tribunal regardless of category.
+const CORE_JUDGE_PREFIXES = new Set(["SEC", "ERR", "LOGIC", "STRUCT", "MAINT"]);
+/**
+ * Select a focused subset of tribunal judges relevant to a benchmark case's
+ * category. Returns core judges + category-specific judges, typically 8-15
+ * instead of the full 35. Returns undefined if no routing is possible
+ * (unknown category), signalling the caller to use all tribunal judges.
+ */
+export function selectJudgesForCategory(category) {
+    const acceptable = CATEGORY_ACCEPTABLE_PREFIXES[category];
+    if (!acceptable || acceptable.length === 0)
+        return undefined;
+    const targetPrefixes = new Set([...CORE_JUDGE_PREFIXES, ...acceptable]);
+    const selected = TRIBUNAL_JUDGES.filter((j) => targetPrefixes.has(j.rulePrefix));
+    // Only route if we meaningfully reduced the set (at least 40% fewer)
+    return selected.length < TRIBUNAL_JUDGES.length * 0.6 ? selected : undefined;
+}
+// ─── Rule ID Parsing ────────────────────────────────────────────────────────
+/**
+ * Extract unique rule IDs from LLM response text.
+ * Matches patterns like CYBER-001, SEC-003, AUTH-001, etc.
+ */
+export function getValidRulePrefixes() {
+    return new Set(JUDGES.map((j) => j.rulePrefix));
+}
+/**
+ * Valid prefixes for tribunal mode — excludes meta-judges that are not
+ * included in the tribunal prompt (INTENT, COH, MFPR, FPR, OVER).
+ * Findings with these prefixes are hallucinated by the LLM and should
+ * not be scored.
+ */
+export function getTribunalValidPrefixes() {
+    return new Set(TRIBUNAL_JUDGES.map((j) => j.rulePrefix));
+}
+export function parseLlmRuleIds(response) {
+    const validPrefixes = getValidRulePrefixes();
+    const pattern = /\b([A-Z][A-Z0-9]+)-(\d{1,3})\b/g;
+    const found = new Set();
+    let match;
+    while ((match = pattern.exec(response)) !== null) {
+        if (validPrefixes.has(match[1])) {
+            found.add(match[0]);
+        }
+    }
+    // Secondary pass: extract known prefixes from compound IDs like DEPS-TYPO-001
+    const compoundPattern = /\b([A-Z][A-Z0-9]+)-[A-Z][A-Z0-9]+-(\d{1,3})\b/g;
+    while ((match = compoundPattern.exec(response)) !== null) {
+        if (validPrefixes.has(match[1])) {
+            found.add(`${match[1]}-${match[2]}`);
+        }
+    }
+    return [...found];
+}
+/**
+ * Preferred entrypoint: extract findings from raw LLM text with validation. Falls back to regex rule-id scan.
+ */
+export function extractValidatedLlmFindings(response, prefixes) {
+    const validPrefixes = prefixes ?? getValidRulePrefixes();
+    const primary = extractAndValidateLlmFindings(response, validPrefixes);
+    // Fallback regex scan (for unstructured responses)
+    const fallbackRuleIds = parseLlmRuleIds(response);
+    return mergeFindings(primary, fallbackRuleIds);
+}
+// ─── Prompt Construction ────────────────────────────────────────────────────
+// These construct the exact same prompts served via MCP, ensuring the
+// benchmark tests the same prompts real users experience.
+// ─────────────────────────────────────────────────────────────────────────────
+/**
+ * Construct a per-judge prompt — identical to the MCP-served `judge-{id}` prompt.
+ * Uses condensed criteria (adversarial mandate stripped) plus shared mandates,
+ * mirroring the tribunal architecture for consistency and better precision.
+ */
+export function constructPerJudgePrompt(judge, code, language, contextSnippets = [], amendments) {
+    const persona = judge.systemPrompt.substring(0, judge.systemPrompt.indexOf("\n\n"));
+    const criteria = getCondensedCriteria(judge.systemPrompt);
+    const contextSection = contextSnippets.length
+        ? `## Repository Context\n\n${contextSnippets.map((s) => `- ${s.replace(/\n/g, " ")}`).join("\n")}\n\n`
+        : "";
+    // Filter amendments to only those relevant to this judge
+    const relevantAmendments = (amendments ?? []).filter((a) => a.judgePrefix === judge.rulePrefix);
+    const amendmentSection = formatAmendmentSection(relevantAmendments);
+    return (`${persona}\n\n` +
+        `${SHARED_ADVERSARIAL_MANDATE}\n\n` +
+        `${PRECISION_MANDATE}\n\n` +
+        (amendmentSection ? `${amendmentSection}\n` : "") +
+        contextSection +
+        `${criteria}\n\n` +
+        `${CLEAN_CODE_GATE}\n\n` +
+        `Please evaluate the following ${language} code:\n\n\`\`\`${language}\n${code}\n\`\`\`` +
+        `\n\nProvide your evaluation as structured findings with rule IDs (prefix: ${judge.rulePrefix}-), severity levels (critical/high/medium/low/info), descriptions, and actionable recommendations. If no issues meet the confidence threshold, report zero findings explicitly. End with an overall score (0-100) and verdict (pass/warning/fail).`);
+}
+/**
+ * Construct the full-tribunal prompt — identical to the MCP-served `full-tribunal` prompt.
+ * When `judges` is provided, uses that filtered list instead of all tribunal judges.
+ */
+export function constructTribunalPrompt(code, language, contextSnippets = [], amendments, judges) {
+    const activeJudges = judges ?? TRIBUNAL_JUDGES;
+    const judgeInstructions = activeJudges
+        .map((j) => `### ${j.name} — ${j.domain}\n**Rule prefix:** \`${j.rulePrefix}-\`\n\n${getCondensedCriteria(j.systemPrompt)}`)
+        .join("\n\n---\n\n");
+    const contextSection = contextSnippets.length
+        ? `## Repository Context\n\n${contextSnippets.map((s) => `- ${s.replace(/\n/g, " ")}`).join("\n")}\n\n`
+        : "";
+    const amendmentSection = formatAmendmentSection(amendments ?? []);
+    return (`You are the Judges Panel — a panel of ${activeJudges.length} expert judges who independently evaluate code for quality, security, and operational readiness.\n\n` +
+        `## Universal Evaluation Directives\n\n` +
+        `${SHARED_ADVERSARIAL_MANDATE}\n\n` +
+        `${PRECISION_MANDATE}\n\n` +
+        `${CLEAN_CODE_GATE}\n\n` +
+        `DOMAIN SCOPE DIRECTIVE (applies to ALL judges):\n` +
+        `- Each judge MUST only report findings within their stated domain expertise.\n` +
+        `- A CI/CD judge should NOT report authentication findings. An ethics judge should NOT report performance findings.\n` +
+        `- If code falls entirely outside your domain (e.g., a YAML CI workflow being evaluated by the Database judge), report ZERO findings for that judge.\n` +
+        `- Cross-domain observations should ONLY be reported by the judge whose domain they fall under.\n` +
+        `- HARD LIMIT: Each judge may report AT MOST 2 findings. If a judge has more than 2 potential findings, keep only the 2 highest-severity, highest-confidence ones and discard the rest.\n\n` +
+        (amendmentSection ? `${amendmentSection}\n` : "") +
+        contextSection +
+        `## Evaluation Instructions\n\n` +
+        `Evaluate the following ${language} code from the perspective of ALL ${activeJudges.length} judges below.\n\n` +
+        `### Output Format — Tiered Findings\n` +
+        `Organize ALL findings into three tiers:\n\n` +
+        `**🔴 MUST FIX** (critical/high severity — blocks merge):\n` +
+        `These are real bugs, security vulnerabilities, data loss risks, or correctness issues. ` +
+        `Report at most 5 findings here. Each must have concrete code evidence.\n\n` +
+        `**🟡 WORTH REVIEWING** (medium severity — warrants discussion):\n` +
+        `Design flaws, maintainability concerns, or reliability risks that a senior reviewer would flag. ` +
+        `Only include findings with specific code evidence.\n\n` +
+        `**🟢 INFORMATIONAL** (low/info severity — optional improvements):\n` +
+        `Minor style or optimization suggestions. Limit to the most impactful 3. Omit if none are genuinely useful.\n\n` +
+        `For each finding, provide:\n` +
+        `1. Rule ID (using the judge's prefix)\n` +
+        `2. Severity (critical/high/medium/low/info)\n` +
+        `3. Confidence (0-100%): How certain are you this is a real issue? Only include findings ≥80%.\n` +
+        `4. Judge name and domain\n` +
+        `5. Specific code evidence (line numbers, patterns)\n` +
+        `6. Description and recommendation\n\n` +
+        `For judges where no issues meet the confidence threshold, report a PASS verdict with zero findings.\n\n` +
+        `Then provide an OVERALL TRIBUNAL VERDICT that synthesizes all judges' input.\n\n` +
+        `## The Judges\n\n${judgeInstructions}\n\n` +
+        `## Code to Evaluate\n\n\`\`\`${language}\n${code}\n\`\`\``);
+}
+// ─── Stratified Sampling ────────────────────────────────────────────────────
+/**
+ * Select a stratified sample of benchmark cases, ensuring representation
+ * across categories, difficulties, and both clean/dirty cases.
+ */
+export function selectStratifiedSample(cases, targetSize) {
+    if (targetSize >= cases.length)
+        return [...cases];
+    // Group by category
+    const byCategory = {};
+    for (const c of cases) {
+        if (!byCategory[c.category])
+            byCategory[c.category] = [];
+        byCategory[c.category].push(c);
+    }
+    const categories = Object.keys(byCategory);
+    const perCategory = Math.max(2, Math.floor(targetSize / categories.length));
+    const selected = [];
+    const usedIds = new Set();
+    for (const cat of categories) {
+        const pool = byCategory[cat];
+        // Ensure difficulty coverage within each category
+        const byDiff = {};
+        for (const c of pool) {
+            if (!byDiff[c.difficulty])
+                byDiff[c.difficulty] = [];
+            byDiff[c.difficulty].push(c);
+        }
+        let taken = 0;
+        for (const diff of ["easy", "medium", "hard"]) {
+            if (taken >= perCategory)
+                break;
+            const diffPool = byDiff[diff] || [];
+            const remaining = perCategory - taken;
+            const diffCount = diff === "easy" ? 3 : diff === "medium" ? 2 : 1;
+            const toTake = Math.max(1, Math.ceil(remaining / diffCount));
+            for (let i = 0; i < Math.min(toTake, diffPool.length) && taken < perCategory; i++) {
+                if (!usedIds.has(diffPool[i].id)) {
+                    selected.push(diffPool[i]);
+                    usedIds.add(diffPool[i].id);
+                    taken++;
+                }
+            }
+        }
+    }
+    // Fill remaining slots if under target
+    if (selected.length < targetSize) {
+        for (const c of cases) {
+            if (selected.length >= targetSize)
+                break;
+            if (!usedIds.has(c.id)) {
+                selected.push(c);
+                usedIds.add(c.id);
+            }
+        }
+    }
+    return selected;
+}
+// ─── Scoring ────────────────────────────────────────────────────────────────
+/**
+ * Score a single LLM benchmark case using prefix-based matching.
+ * Returns a fully populated LlmCaseResult.
+ *
+ * @param topKPrefixes - If set, only keep the first `topKPrefixes` unique
+ *   detected prefixes (in the order they appear in the LLM response).
+ *   This prevents verbose tribunal output from inflating FP counts.
+ */
+export function scoreLlmCase(tc, detectedRuleIds, rawResponse, tokensUsed, topKPrefixes) {
+    // ── Optional top-K prefix cap ─────────────────────────────────────────
+    let filteredDetected = detectedRuleIds;
+    if (topKPrefixes !== undefined && topKPrefixes > 0) {
+        const seenPrefixes = new Set();
+        filteredDetected = detectedRuleIds.filter((id) => {
+            const prefix = id.split("-")[0];
+            if (seenPrefixes.size >= topKPrefixes && !seenPrefixes.has(prefix))
+                return false;
+            seenPrefixes.add(prefix);
+            return true;
+        });
+    }
+    // ── Prefix-level FP deduplication ─────────────────────────────────────
+    // TPs are counted per-expected-rule using prefix matching: a single
+    // detected CYBER-xxx satisfies all expected CYBER-yyy rules.
+    // FPs should use symmetric counting: each erroneously-firing judge prefix
+    // counts as ONE false positive regardless of how many rule IDs the LLM
+    // generates for that prefix.  This prevents verbose LLM output from
+    // inflating the FP metric (e.g. CYBER-001…005 on clean code = 1 FP,
+    // not 5).
+    const detectedPrefixes = new Set(filteredDetected.map((r) => r.split("-")[0]));
+    const matchedExpected = tc.expectedRuleIds.filter((expected) => {
+        const prefix = expected.split("-")[0];
+        return detectedPrefixes.has(prefix);
+    });
+    const missedExpected = tc.expectedRuleIds.filter((expected) => {
+        const prefix = expected.split("-")[0];
+        return !detectedPrefixes.has(prefix);
+    });
+    // For clean cases (no expected findings), ALL detections are false positives.
+    // For dirty cases with unexpectedRuleIds, FPs are detections matching those prefixes.
+    // For dirty cases WITHOUT unexpectedRuleIds, FPs are detections whose prefix
+    // doesn't match any expected or acceptable prefix.
+    const isCleanCase = tc.expectedRuleIds.length === 0;
+    const acceptablePrefixes = getAcceptablePrefixes(tc);
+    const falsePositiveIdsRaw = isCleanCase
+        ? filteredDetected
+        : tc.unexpectedRuleIds
+            ? filteredDetected.filter((found) => {
+                const prefix = found.split("-")[0];
+                return tc.unexpectedRuleIds.some((u) => u.split("-")[0] === prefix);
+            })
+            : filteredDetected.filter((found) => {
+                const prefix = found.split("-")[0];
+                return !acceptablePrefixes.has(prefix);
+            });
+    // Deduplicate FPs by prefix — keep one representative rule ID per prefix
+    const fpPrefixSeen = new Set();
+    const falsePositiveIds = falsePositiveIdsRaw.filter((id) => {
+        const prefix = id.split("-")[0];
+        if (fpPrefixSeen.has(prefix))
+            return false;
+        fpPrefixSeen.add(prefix);
+        return true;
+    });
+    const casePassed = isCleanCase ? falsePositiveIds.length === 0 : matchedExpected.length > 0;
+    return {
+        caseId: tc.id,
+        category: tc.category,
+        difficulty: tc.difficulty,
+        passed: casePassed,
+        expectedRuleIds: tc.expectedRuleIds,
+        detectedRuleIds: filteredDetected,
+        missedRuleIds: missedExpected,
+        falsePositiveRuleIds: falsePositiveIds,
+        rawResponse,
+        tokensUsed,
+    };
+}
+/**
+ * Compute aggregate metrics for an LLM benchmark snapshot from raw case results.
+ * Uses the same prefix-based matching methodology as the L1 benchmark.
+ */
+export function computeLlmMetrics(rawCases, version, model, provider, promptMode, durationSeconds, totalTokensUsed) {
+    const perCategory = {};
+    const perJudge = {};
+    const perDifficulty = {};
+    let totalTP = 0;
+    let totalFN = 0;
+    let totalFP = 0;
+    let totalDetected = 0;
+    for (const c of rawCases) {
+        const caseTP = c.expectedRuleIds.length - c.missedRuleIds.length;
+        const caseFN = c.missedRuleIds.length;
+        const caseFP = c.falsePositiveRuleIds.length;
+        if (c.passed)
+            totalDetected++;
+        totalTP += caseTP;
+        totalFN += caseFN;
+        totalFP += caseFP;
+        // Per-difficulty
+        if (!perDifficulty[c.difficulty]) {
+            perDifficulty[c.difficulty] = { difficulty: c.difficulty, total: 0, detected: 0, detectionRate: 0 };
+        }
+        perDifficulty[c.difficulty].total++;
+        if (c.passed)
+            perDifficulty[c.difficulty].detected++;
+        // Per-category
+        if (!perCategory[c.category]) {
+            perCategory[c.category] = {
+                category: c.category,
+                total: 0,
+                detected: 0,
+                truePositives: 0,
+                falseNegatives: 0,
+                falsePositives: 0,
+                precision: 0,
+                recall: 0,
+                f1Score: 0,
+            };
+        }
+        const cat = perCategory[c.category];
+        cat.total++;
+        if (c.passed)
+            cat.detected++;
+        cat.truePositives += caseTP;
+        cat.falseNegatives += caseFN;
+        cat.falsePositives += caseFP;
+        // Per-judge (deduplicate by prefix per case to match case-level FP counting)
+        // Use pre-computed falsePositiveRuleIds to stay consistent with scoreLlmCase
+        const fpPrefixes = new Set(c.falsePositiveRuleIds.map((r) => r.split("-")[0]));
+        const expectedPrefixes = new Set(c.expectedRuleIds.map((r) => r.split("-")[0]));
+        const seenPrefixes = new Set();
+        for (const ruleId of c.detectedRuleIds) {
+            const prefix = ruleId.split("-")[0];
+            if (seenPrefixes.has(prefix))
+                continue;
+            seenPrefixes.add(prefix);
+            if (!perJudge[prefix]) {
+                perJudge[prefix] = {
+                    judgeId: prefix,
+                    total: 0,
+                    truePositives: 0,
+                    falseNegatives: 0,
+                    falsePositives: 0,
+                    precision: 0,
+                    recall: 0,
+                    f1Score: 0,
+                };
+            }
+            const jb = perJudge[prefix];
+            jb.total++;
+            if (expectedPrefixes.has(prefix)) {
+                jb.truePositives++;
+            }
+            else if (fpPrefixes.has(prefix)) {
+                jb.falsePositives++;
+            }
+            // Acceptable (non-expected, non-FP) detections are silently ignored
+        }
+    }
+    // Compute aggregate metrics
+    const precision = totalTP + totalFP > 0 ? totalTP / (totalTP + totalFP) : 1;
+    const recall = totalTP + totalFN > 0 ? totalTP / (totalTP + totalFN) : 1;
+    const f1Score = precision + recall > 0 ? (2 * precision * recall) / (precision + recall) : 0;
+    // Severity-weighted F1 — re-extract findings from raw responses to get
+    // severity info, then weight FPs: critical/high=3x, medium=1x, low/info=0.3x
+    const SEVERITY_WEIGHTS = {
+        critical: 3,
+        high: 3,
+        medium: 1,
+        low: 0.3,
+        info: 0.3,
+    };
+    let weightedFP = 0;
+    const tribunalPrefixes = getTribunalValidPrefixes();
+    for (const c of rawCases) {
+        if (c.falsePositiveRuleIds.length === 0)
+            continue;
+        const fpSet = new Set(c.falsePositiveRuleIds.map((r) => r.split("-")[0]));
+        const validation = extractValidatedLlmFindings(c.rawResponse, tribunalPrefixes);
+        // Map finding ruleId prefix → max severity weight
+        const prefixMaxWeight = new Map();
+        for (const f of validation.findings) {
+            const prefix = f.ruleId.split("-")[0];
+            if (!fpSet.has(prefix))
+                continue;
+            const weight = SEVERITY_WEIGHTS[f.severity] ?? 1;
+            prefixMaxWeight.set(prefix, Math.max(prefixMaxWeight.get(prefix) ?? 0, weight));
+        }
+        // Sum weights for FP prefixes (use weight=1 default if severity unknown)
+        for (const prefix of fpSet) {
+            weightedFP += prefixMaxWeight.get(prefix) ?? 1;
+        }
+    }
+    const weightedPrecision = totalTP + weightedFP > 0 ? totalTP / (totalTP + weightedFP) : 1;
+    const weightedF1Score = weightedPrecision + recall > 0 ? (2 * weightedPrecision * recall) / (weightedPrecision + recall) : 0;
+    // Compute per-difficulty rates
+    for (const d of Object.values(perDifficulty)) {
+        d.detectionRate = d.total > 0 ? d.detected / d.total : 0;
+    }
+    // Compute per-category metrics
+    for (const cat of Object.values(perCategory)) {
+        cat.precision =
+            cat.truePositives + cat.falsePositives > 0 ? cat.truePositives / (cat.truePositives + cat.falsePositives) : 1;
+        cat.recall =
+            cat.truePositives + cat.falseNegatives > 0 ? cat.truePositives / (cat.truePositives + cat.falseNegatives) : 1;
+        cat.f1Score = cat.precision + cat.recall > 0 ? (2 * cat.precision * cat.recall) / (cat.precision + cat.recall) : 0;
+    }
+    // Compute per-judge metrics
+    for (const jb of Object.values(perJudge)) {
+        jb.precision =
+            jb.truePositives + jb.falsePositives > 0 ? jb.truePositives / (jb.truePositives + jb.falsePositives) : 1;
+        jb.recall =
+            jb.truePositives + jb.falseNegatives > 0 ? jb.truePositives / (jb.truePositives + jb.falseNegatives) : 1;
+        jb.f1Score = jb.precision + jb.recall > 0 ? (2 * jb.precision * jb.recall) / (jb.precision + jb.recall) : 0;
+    }
+    return {
+        timestamp: new Date().toISOString(),
+        version,
+        model,
+        provider,
+        promptMode,
+        totalCases: rawCases.length,
+        detected: totalDetected,
+        missed: rawCases.length - totalDetected,
+        totalExpected: rawCases.reduce((s, c) => s + c.expectedRuleIds.length, 0),
+        truePositives: totalTP,
+        falseNegatives: totalFN,
+        falsePositives: totalFP,
+        precision,
+        recall,
+        f1Score,
+        weightedF1Score,
+        detectionRate: rawCases.length > 0 ? totalDetected / rawCases.length : 0,
+        perCategory,
+        perJudge,
+        perDifficulty,
+        cases: rawCases,
+        durationSeconds,
+        ...(totalTokensUsed ? { totalTokensUsed } : {}),
+    };
+}
+// ─── Markdown Formatting ────────────────────────────────────────────────────
+/**
+ * Format an LLM benchmark snapshot as a markdown section for the benchmark report.
+ */
+export function formatLlmSnapshotMarkdown(snapshot) {
+    const lines = [];
+    const pct = (n) => `${(n * 100).toFixed(1)}%`;
+    const l2Grade = snapshot.f1Score >= 0.9
+        ? "A"
+        : snapshot.f1Score >= 0.8
+            ? "B"
+            : snapshot.f1Score >= 0.7
+                ? "C"
+                : snapshot.f1Score >= 0.6
+                    ? "D"
+                    : "F";
+    const l2GradeEmoji = l2Grade === "A" ? "🟢" : l2Grade === "B" ? "🟡" : l2Grade === "C" ? "🟠" : "🔴";
+    lines.push("## Layer 2 — LLM Deep Review");
+    lines.push("");
+    lines.push(`> Model: **${snapshot.model}** · Provider: ${snapshot.provider} · ${snapshot.totalCases} test cases · ${new Date(snapshot.timestamp).toLocaleDateString()}`);
+    lines.push("");
+    lines.push(`| Metric | Value |`);
+    lines.push(`|--------|-------|`);
+    lines.push(`| Overall Grade | ${l2GradeEmoji} **${l2Grade}** |`);
+    lines.push(`| Test Cases | ${snapshot.totalCases} |`);
+    lines.push(`| Detection Rate | ${pct(snapshot.detectionRate)} (${snapshot.detected}/${snapshot.totalCases}) |`);
+    lines.push(`| Precision | ${pct(snapshot.precision)} |`);
+    lines.push(`| Recall | ${pct(snapshot.recall)} |`);
+    lines.push(`| F1 Score | ${pct(snapshot.f1Score)} |`);
+    if (snapshot.weightedF1Score !== null && snapshot.weightedF1Score !== undefined) {
+        lines.push(`| Weighted F1 | ${pct(snapshot.weightedF1Score)} |`);
+    }
+    lines.push(`| True Positives | ${snapshot.truePositives} |`);
+    lines.push(`| False Negatives | ${snapshot.falseNegatives} |`);
+    lines.push(`| False Positives | ${snapshot.falsePositives} |`);
+    if (snapshot.totalTokensUsed) {
+        lines.push(`| Total Tokens | ${snapshot.totalTokensUsed.toLocaleString()} |`);
+    }
+    lines.push(`| Run Duration | ${snapshot.durationSeconds}s |`);
+    lines.push("");
+    // Per-difficulty
+    if (Object.keys(snapshot.perDifficulty).length > 0) {
+        lines.push("### L2 Detection by Difficulty");
+        lines.push("");
+        lines.push("| Difficulty | Detected | Total | Rate |");
+        lines.push("|------------|----------|-------|------|");
+        for (const diff of ["easy", "medium", "hard"]) {
+            const d = snapshot.perDifficulty[diff];
+            if (d) {
+                lines.push(`| ${diff} | ${d.detected} | ${d.total} | ${pct(d.detectionRate)} |`);
+            }
+        }
+        lines.push("");
+    }
+    // Per-category
+    if (Object.keys(snapshot.perCategory).length > 0) {
+        lines.push("### L2 Results by Category");
+        lines.push("");
+        lines.push("| Category | Detected | Total | Precision | Recall | F1 |");
+        lines.push("|----------|----------|-------|-----------|--------|-----|");
+        for (const [cat, stats] of Object.entries(snapshot.perCategory).sort(([a], [b]) => a.localeCompare(b))) {
+            lines.push(`| ${cat} | ${stats.detected} | ${stats.total} | ${pct(stats.precision)} | ${pct(stats.recall)} | ${pct(stats.f1Score)} |`);
+        }
+        lines.push("");
+    }
+    // Per-judge
+    if (Object.keys(snapshot.perJudge).length > 0) {
+        lines.push("### L2 Results by Judge");
+        lines.push("");
+        lines.push("| Judge | Findings | TP | FP | Precision |");
+        lines.push("|-------|----------|-----|-----|-----------|");
+        for (const [judgeId, stats] of Object.entries(snapshot.perJudge).sort(([a], [b]) => a.localeCompare(b))) {
+            lines.push(`| ${judgeId} | ${stats.total} | ${stats.truePositives} | ${stats.falsePositives} | ${pct(stats.precision)} |`);
+        }
+        lines.push("");
+    }
+    // Failed cases
+    const failed = snapshot.cases.filter((c) => !c.passed);
+    if (failed.length > 0 && failed.length <= 50) {
+        lines.push("### L2 Failed Cases");
+        lines.push("");
+        lines.push("| Case | Difficulty | Category | Missed Rules | False Positives |");
+        lines.push("|------|------------|----------|--------------|-----------------|");
+        for (const c of failed) {
+            const missed = c.missedRuleIds.length > 0 ? c.missedRuleIds.join(", ") : "—";
+            const fps = c.falsePositiveRuleIds.length > 0 ? c.falsePositiveRuleIds.join(", ") : "—";
+            lines.push(`| ${c.caseId} | ${c.difficulty} | ${c.category} | ${missed} | ${fps} |`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+/**
+ * Format a layer comparison table showing L1 vs L2 side by side.
+ */
+export function formatLayerComparisonMarkdown(l1, l2) {
+    const lines = [];
+    const pct = (n) => `${(n * 100).toFixed(1)}%`;
+    lines.push("## Layer Comparison");
+    lines.push("");
+    lines.push(`| Metric | L1 (Deterministic) | L2 (${l2.model}) |`);
+    lines.push(`|--------|-------------------|${"-".repeat(Math.max(l2.model.length + 4, 5))}|`);
+    lines.push(`| Test Cases | ${l1.totalCases} | ${l2.totalCases} |`);
+    lines.push(`| Detection Rate | ${pct(l1.detectionRate)} | ${pct(l2.detectionRate)} |`);
+    lines.push(`| Precision | ${pct(l1.precision)} | ${pct(l2.precision)} |`);
+    lines.push(`| Recall | ${pct(l1.recall)} | ${pct(l2.recall)} |`);
+    lines.push(`| F1 Score | ${pct(l1.f1Score)} | ${pct(l2.f1Score)} |`);
+    lines.push(`| False Positives | ${l1.falsePositives} | ${l2.falsePositives} |`);
+    lines.push("");
+    lines.push("The two layers are **complementary**:");
+    lines.push("- **L1** provides fast, reliable baseline detection with high precision and zero cost");
+    lines.push("- **L2** catches sophisticated issues that patterns miss, at the cost of API calls and latency");
+    lines.push("- Together, they deliver defense-in-depth code analysis");
+    lines.push("");
+    return lines.join("\n");
+}