@kevinrabun/judges-cli 3.128.3 → 3.129.0

package/dist/commands/openssf-cve-benchmark.js

@@ -0,0 +1,659 @@
+/**
+ * OpenSSF CVE Benchmark Integration
+ *
+ * Runs the Judges evaluation engine against the OpenSSF CVE Benchmark dataset
+ * (https://github.com/ossf-cve-benchmark/ossf-cve-benchmark) — 200+ real-world
+ * JavaScript/TypeScript CVEs with pre-patch (vulnerable) and post-patch (fixed)
+ * git commits.
+ *
+ * Two modes:
+ *   1. Deterministic (L1): Runs Judges' pattern-based evaluators against each CVE.
+ *   2. LLM integration: Converts passing CVE cases into BenchmarkCase format
+ *      for inclusion in the LLM benchmark pipeline.
+ *
+ * Usage:
+ *   judges openssf-cve run [--repo <path>] [--cve <id>] [--format json|text|markdown]
+ *   judges openssf-cve convert [--repo <path>]  # Convert to BenchmarkCase[]
+ */
+import { existsSync, readFileSync, readdirSync, writeFileSync, mkdirSync } from "fs";
+import { resolve, join, extname } from "path";
+import { execSync } from "child_process";
+import { evaluateWithTribunal } from "../evaluators/index.js";
+import { EXT_TO_LANG } from "../ext-to-lang.js";
+import { registerBenchmarkAdapter } from "./external-benchmarks.js";
+// ─── CWE → Judge Prefix Mapping ────────────────────────────────────────────
+/**
+ * Maps CWE IDs to the judge rule prefixes that are expected to detect them.
+ * This is the reverse of the PREFIX_MAP in security-ids.ts, extended with
+ * additional CWE coverage from individual judges.
+ */
+const CWE_TO_PREFIXES = {
+    // Injection
+    "CWE-078": ["CYBER", "SEC"], // OS Command Injection
+    "CWE-079": ["CYBER", "SEC", "XSS", "FW"], // XSS
+    "CWE-089": ["CYBER", "SEC", "DB"], // SQL Injection
+    "CWE-094": ["CYBER", "SEC"], // Code Injection
+    "CWE-095": ["CYBER", "SEC"], // Eval Injection
+    "CWE-917": ["CYBER", "SEC"], // Expression Language Injection
+    "CWE-134": ["CYBER", "SEC"], // Format String
+    "CWE-943": ["DB", "SEC"], // NoSQL Injection
+    // Path Traversal / File Access
+    "CWE-022": ["CYBER", "SEC"], // Path Traversal
+    "CWE-073": ["CYBER", "SEC"], // External Control of File Name
+    "CWE-434": ["CYBER", "SEC"], // Unrestricted Upload
+    // Auth / Crypto
+    "CWE-287": ["AUTH", "CYBER"], // Improper Authentication
+    "CWE-798": ["AUTH", "CFG", "DATA", "CYBER"], // Hard-coded Credentials
+    "CWE-327": ["CRYPTO", "CYBER"], // Use of Broken Crypto Algorithm
+    "CWE-328": ["CRYPTO", "CYBER"], // Weak Hash
+    "CWE-330": ["CRYPTO", "CYBER", "AICS"], // Insufficient Randomness
+    "CWE-916": ["CRYPTO", "AUTH", "CYBER"], // Weak Password Hash
+    // Access Control
+    "CWE-284": ["CYBER", "AUTH"], // Improper Access Control
+    "CWE-269": ["CYBER", "AUTH"], // Improper Privilege Management
+    "CWE-862": ["AUTH", "CYBER"], // Missing Authorization
+    "CWE-863": ["AUTH", "CYBER"], // Incorrect Authorization
+    // Data Exposure
+    "CWE-200": ["DATA", "SEC", "LOGPRIV"], // Information Exposure
+    "CWE-209": ["ERR", "SEC"], // Error Message Info Exposure
+    "CWE-312": ["DATA", "CFG"], // Cleartext Storage of Sensitive Info
+    "CWE-319": ["CYBER", "SEC", "DATA"], // Cleartext Transmission
+    "CWE-532": ["LOGPRIV"], // Insertion of Sensitive Info into Log
+    // Deserialization / Prototype Pollution
+    "CWE-502": ["CYBER", "SEC"], // Deserialization of Untrusted Data
+    "CWE-915": ["CYBER", "SEC"], // Improperly Controlled Modification (prototype pollution)
+    "CWE-471": ["CYBER", "SEC"], // Modification of Assumed-Immutable Data
+    // Input / Validation
+    "CWE-020": ["SEC", "CYBER"], // Improper Input Validation
+    "CWE-400": ["RATE", "CYBER"], // Uncontrolled Resource Consumption (ReDoS, etc.)
+    "CWE-770": ["RATE"], // Allocation of Resources Without Limits
+    // Race Conditions
+    "CWE-362": ["CONC"], // Race Condition
+    "CWE-667": ["CONC"], // Improper Locking
+    // Configuration
+    "CWE-016": ["CFG"], // Configuration
+    "CWE-1188": ["CFG"], // Insecure Default Initialization
+    // SSRF
+    "CWE-918": ["CYBER", "SEC"], // Server-Side Request Forgery
+    // Denial of Service
+    "CWE-185": ["SEC", "CYBER"], // Incorrect Regular Expression
+    "CWE-1333": ["SEC", "CYBER"], // Inefficient Regular Expression (ReDoS)
+};
+// ─── Helpers ────────────────────────────────────────────────────────────────
+function normalizeCwe(cwe) {
+    return cwe.replace(/^CWE-0*/, "CWE-");
+}
+function getExpectedPrefixes(cwes) {
+    const prefixes = new Set();
+    for (const cwe of cwes) {
+        const normalized = normalizeCwe(cwe);
+        const mapped = CWE_TO_PREFIXES[normalized];
+        if (mapped) {
+            for (const p of mapped)
+                prefixes.add(p);
+        }
+    }
+    // Always consider CYBER and SEC as relevant for any security CVE
+    if (prefixes.size === 0) {
+        prefixes.add("CYBER");
+        prefixes.add("SEC");
+    }
+    return [...prefixes];
+}
+function findingMatchesCwes(finding, cwes) {
+    const normalizedCwes = new Set(cwes.map(normalizeCwe));
+    // Check direct CWE match on the finding
+    if (finding.cweIds) {
+        for (const fCwe of finding.cweIds) {
+            if (normalizedCwes.has(normalizeCwe(fCwe)))
+                return true;
+        }
+    }
+    // Check prefix match — if the finding's rule prefix maps to one of the CVE's CWEs
+    const prefix = finding.ruleId.split("-")[0];
+    for (const cwe of normalizedCwes) {
+        const expectedPrefixes = CWE_TO_PREFIXES[cwe] ?? [];
+        if (expectedPrefixes.includes(prefix))
+            return true;
+    }
+    return false;
+}
+function detectLanguage(filePath) {
+    const ext = extname(filePath).toLowerCase();
+    return EXT_TO_LANG[ext] ?? "javascript";
+}
+// ─── CVE Loading ────────────────────────────────────────────────────────────
+export function loadCveFiles(repoPath) {
+    const cvesDir = join(repoPath, "CVEs");
+    if (!existsSync(cvesDir)) {
+        throw new Error(`OpenSSF CVE Benchmark not found at ${cvesDir}. Clone it with:\n  git clone https://github.com/ossf-cve-benchmark/ossf-cve-benchmark.git`);
+    }
+    const files = readdirSync(cvesDir).filter((f) => f.endsWith(".json"));
+    const cves = [];
+    for (const file of files) {
+        const raw = readFileSync(join(cvesDir, file), "utf-8");
+        const cve = JSON.parse(raw);
+        // Only include complete, published CVEs
+        if (cve.state === "PUBLISHED" && cve.prePatch?.weaknesses?.length > 0 && cve.postPatch?.commit) {
+            cves.push(cve);
+        }
+    }
+    return cves;
+}
+// ─── Git Checkout Helpers ───────────────────────────────────────────────────
+function ensureSourcesDir(repoPath) {
+    const sourcesDir = join(repoPath, "work", "sources");
+    if (!existsSync(sourcesDir)) {
+        mkdirSync(sourcesDir, { recursive: true });
+    }
+    return sourcesDir;
+}
+function checkoutCommit(cve, commit, sourcesDir) {
+    const cveDir = join(sourcesDir, cve.CVE);
+    // Check if the CVE-specific repo exists at the ossf-cve-benchmark org
+    const repoUrl = `https://github.com/ossf-cve-benchmark/${cve.CVE}.git`;
+    if (!existsSync(cveDir)) {
+        try {
+            execSync(`git clone --quiet "${repoUrl}" "${cveDir}"`, {
+                stdio: "pipe",
+                timeout: 60_000,
+            });
+        }
+        catch {
+            // Fallback to the original repository URL from the CVE metadata
+            try {
+                execSync(`git clone --quiet "${cve.repository}" "${cveDir}"`, {
+                    stdio: "pipe",
+                    timeout: 120_000,
+                });
+            }
+            catch {
+                return undefined;
+            }
+        }
+    }
+    try {
+        execSync(`git checkout --quiet "${commit}"`, {
+            cwd: cveDir,
+            stdio: "pipe",
+            timeout: 30_000,
+        });
+        return cveDir;
+    }
+    catch {
+        return undefined;
+    }
+}
+function readWeaknessFiles(checkoutDir, weaknesses) {
+    const files = [];
+    const seen = new Set();
+    for (const w of weaknesses) {
+        if (seen.has(w.location.file))
+            continue;
+        seen.add(w.location.file);
+        const fullPath = join(checkoutDir, w.location.file);
+        if (!existsSync(fullPath))
+            continue;
+        const content = readFileSync(fullPath, "utf-8");
+        const language = detectLanguage(w.location.file);
+        files.push({ path: w.location.file, content, language });
+    }
+    return files;
+}
+// ─── Evaluation ─────────────────────────────────────────────────────────────
+function evaluateCveCode(files, cwes) {
+    const allFindings = [];
+    for (const file of files) {
+        const verdict = evaluateWithTribunal(file.content, file.language);
+        allFindings.push(...verdict.findings);
+    }
+    const normalizedCwes = cwes.map(normalizeCwe);
+    const matchedCwes = [];
+    const missedCwes = [];
+    for (const cwe of normalizedCwes) {
+        const expectedPrefixes = CWE_TO_PREFIXES[cwe] ?? ["CYBER", "SEC"];
+        const matched = allFindings.some((f) => findingMatchesCwes(f, [cwe]) || expectedPrefixes.includes(f.ruleId.split("-")[0]));
+        if (matched) {
+            matchedCwes.push(cwe);
+        }
+        else {
+            missedCwes.push(cwe);
+        }
+    }
+    return { findings: allFindings, matchedCwes, missedCwes };
+}
+export function evaluateSingleCve(cve, sourcesDir) {
+    const language = detectLanguage(cve.prePatch.weaknesses[0]?.location.file ?? "index.js");
+    // Checkout pre-patch (vulnerable) code
+    const prePatchDir = checkoutCommit(cve, cve.prePatch.commit, sourcesDir);
+    if (!prePatchDir) {
+        return {
+            cve: cve.CVE,
+            cwes: cve.CWEs,
+            language,
+            detected: false,
+            cleanOnPatch: true,
+            prePatchFindings: [],
+            postPatchFindings: [],
+            matchedCwes: [],
+            missedCwes: cve.CWEs.map(normalizeCwe),
+            error: "Failed to checkout pre-patch commit",
+        };
+    }
+    const prePatchFiles = readWeaknessFiles(prePatchDir, cve.prePatch.weaknesses);
+    if (prePatchFiles.length === 0) {
+        return {
+            cve: cve.CVE,
+            cwes: cve.CWEs,
+            language,
+            detected: false,
+            cleanOnPatch: true,
+            prePatchFindings: [],
+            postPatchFindings: [],
+            matchedCwes: [],
+            missedCwes: cve.CWEs.map(normalizeCwe),
+            error: "Weakness files not found in checkout",
+        };
+    }
+    // Evaluate pre-patch
+    const prePatchEval = evaluateCveCode(prePatchFiles, cve.CWEs);
+    const detected = prePatchEval.matchedCwes.length > 0;
+    // Checkout post-patch (fixed) code
+    const postPatchDir = checkoutCommit(cve, cve.postPatch.commit, sourcesDir);
+    let postPatchFindings = [];
+    let cleanOnPatch = true;
+    if (postPatchDir) {
+        const postPatchFiles = readWeaknessFiles(postPatchDir, cve.prePatch.weaknesses);
+        if (postPatchFiles.length > 0) {
+            const expectedPrefixes = getExpectedPrefixes(cve.CWEs);
+            const postEval = evaluateCveCode(postPatchFiles, cve.CWEs);
+            // Only count findings with relevant prefixes as FPs on patched code
+            postPatchFindings = postEval.findings.filter((f) => expectedPrefixes.includes(f.ruleId.split("-")[0]));
+            cleanOnPatch = postPatchFindings.length === 0;
+        }
+    }
+    return {
+        cve: cve.CVE,
+        cwes: cve.CWEs.map(normalizeCwe),
+        language,
+        detected,
+        cleanOnPatch,
+        prePatchFindings: prePatchEval.findings,
+        postPatchFindings,
+        matchedCwes: prePatchEval.matchedCwes,
+        missedCwes: prePatchEval.missedCwes,
+    };
+}
+// ─── Aggregate Results ──────────────────────────────────────────────────────
+export function computeOpenSSFMetrics(results) {
+    const evaluated = results.filter((r) => !r.error);
+    const detected = evaluated.filter((r) => r.detected);
+    const cleanOnPatch = evaluated.filter((r) => r.cleanOnPatch);
+    // Per-CWE breakdown
+    const perCwe = {};
+    for (const r of evaluated) {
+        for (const cwe of r.cwes) {
+            if (!perCwe[cwe])
+                perCwe[cwe] = { total: 0, detected: 0, rate: 0 };
+            perCwe[cwe].total++;
+            if (r.matchedCwes.includes(cwe))
+                perCwe[cwe].detected++;
+        }
+    }
+    for (const entry of Object.values(perCwe)) {
+        entry.rate = entry.total > 0 ? entry.detected / entry.total : 0;
+    }
+    const detectionRate = evaluated.length > 0 ? detected.length / evaluated.length : 0;
+    // Precision: among detected CVEs, how many had no FP on patch
+    const truePositives = detected.filter((r) => r.cleanOnPatch).length;
+    const falsePositives = detected.filter((r) => !r.cleanOnPatch).length;
+    const falseNegatives = evaluated.length - detected.length;
+    const precision = truePositives + falsePositives > 0 ? truePositives / (truePositives + falsePositives) : 1;
+    const recall = truePositives + falseNegatives > 0 ? truePositives / (truePositives + falseNegatives) : 1;
+    const f1Score = precision + recall > 0 ? (2 * precision * recall) / (precision + recall) : 0;
+    return {
+        timestamp: new Date().toISOString(),
+        totalCves: results.length,
+        evaluated: evaluated.length,
+        skipped: results.length - evaluated.length,
+        detected: detected.length,
+        missed: evaluated.length - detected.length,
+        cleanOnPatch: cleanOnPatch.length,
+        falsePositiveOnPatch: evaluated.length - cleanOnPatch.length,
+        detectionRate,
+        precision,
+        recall,
+        f1Score,
+        perCwe,
+        results,
+    };
+}
+// ─── BenchmarkCase Conversion ───────────────────────────────────────────────
+/**
+ * Convert OpenSSF CVE results into BenchmarkCase[] format for use
+ * in the Judges LLM benchmark pipeline. Only includes CVEs where:
+ * - The vulnerable code was successfully checked out
+ * - At least one weakness file was found
+ * - CWEs map to known judge prefixes
+ */
+export function convertToBenchmarkCases(cves, sourcesDir) {
+    const cases = [];
+    for (const cve of cves) {
+        const prePatchDir = checkoutCommit(cve, cve.prePatch.commit, sourcesDir);
+        if (!prePatchDir)
+            continue;
+        const files = readWeaknessFiles(prePatchDir, cve.prePatch.weaknesses);
+        if (files.length === 0)
+            continue;
+        const expectedPrefixes = getExpectedPrefixes(cve.CWEs);
+        if (expectedPrefixes.length === 0)
+            continue;
+        // Generate expected rule IDs from CWE→prefix mapping
+        const expectedRuleIds = expectedPrefixes.map((p) => `${p}-001`);
+        // Use the primary weakness file as the main code
+        const primaryFile = files[0];
+        const code = primaryFile.content.length > 4000
+            ? primaryFile.content.slice(0, 4000) + "\n// ... truncated for benchmark"
+            : primaryFile.content;
+        // Determine category from CWE
+        const category = inferCategory(cve.CWEs);
+        const benchmarkCase = {
+            id: `openssf-${cve.CVE.toLowerCase()}`,
+            description: `Real-world CVE: ${cve.CVE} (${cve.CWEs.join(", ")}) — ${cve.prePatch.weaknesses[0]?.explanation ?? "security vulnerability"}`,
+            language: primaryFile.language,
+            code,
+            expectedRuleIds,
+            acceptablePrefixes: [...new Set([...expectedPrefixes, "CYBER", "SEC"])],
+            category,
+            difficulty: "hard",
+            aiSource: "openssf-cve-benchmark",
+        };
+        // Include additional files for multi-file cases
+        if (files.length > 1) {
+            benchmarkCase.files = files.slice(1).map((f) => ({
+                path: f.path,
+                content: f.content.length > 4000 ? f.content.slice(0, 4000) + "\n// ... truncated" : f.content,
+                language: f.language,
+            }));
+        }
+        cases.push(benchmarkCase);
+    }
+    return cases;
+}
+function inferCategory(cwes) {
+    const normalized = cwes.map(normalizeCwe);
+    for (const cwe of normalized) {
+        if (["CWE-89", "CWE-78", "CWE-94", "CWE-134"].includes(cwe))
+            return "injection";
+        if (["CWE-79"].includes(cwe))
+            return "xss";
+        if (["CWE-22", "CWE-73"].includes(cwe))
+            return "path-traversal";
+        if (["CWE-287", "CWE-798", "CWE-862", "CWE-863"].includes(cwe))
+            return "auth";
+        if (["CWE-327", "CWE-328", "CWE-916", "CWE-330"].includes(cwe))
+            return "crypto";
+        if (["CWE-502", "CWE-915", "CWE-471"].includes(cwe))
+            return "prototype-pollution";
+        if (["CWE-200", "CWE-209", "CWE-312", "CWE-532"].includes(cwe))
+            return "data-exposure";
+        if (["CWE-400", "CWE-770", "CWE-185", "CWE-1333"].includes(cwe))
+            return "denial-of-service";
+        if (["CWE-918"].includes(cwe))
+            return "ssrf";
+        if (["CWE-362", "CWE-667"].includes(cwe))
+            return "concurrency";
+    }
+    return "security";
+}
+// ─── Report Formatting ──────────────────────────────────────────────────────
+export function formatOpenSSFReport(result) {
+    const lines = [];
+    lines.push("# OpenSSF CVE Benchmark Report");
+    lines.push("");
+    lines.push(`**Date:** ${result.timestamp}`);
+    lines.push(`**CVEs Evaluated:** ${result.evaluated} of ${result.totalCves} (${result.skipped} skipped)`);
+    lines.push("");
+    lines.push("## Summary");
+    lines.push("");
+    lines.push(`| Metric | Value |`);
+    lines.push(`|--------|-------|`);
+    lines.push(`| Detection Rate | ${(result.detectionRate * 100).toFixed(1)}% (${result.detected}/${result.evaluated}) |`);
+    lines.push(`| Precision | ${(result.precision * 100).toFixed(1)}% |`);
+    lines.push(`| Recall | ${(result.recall * 100).toFixed(1)}% |`);
+    lines.push(`| F1 Score | ${(result.f1Score * 100).toFixed(1)}% |`);
+    lines.push(`| Clean on Patch | ${result.cleanOnPatch}/${result.evaluated} |`);
+    lines.push(`| False Positives on Patch | ${result.falsePositiveOnPatch} |`);
+    lines.push("");
+    // Per-CWE breakdown
+    const cweEntries = Object.entries(result.perCwe).sort((a, b) => b[1].total - a[1].total);
+    if (cweEntries.length > 0) {
+        lines.push("## Per-CWE Detection Rates");
+        lines.push("");
+        lines.push("| CWE | Total | Detected | Rate |");
+        lines.push("|-----|-------|----------|------|");
+        for (const [cwe, data] of cweEntries) {
+            lines.push(`| ${cwe} | ${data.total} | ${data.detected} | ${(data.rate * 100).toFixed(0)}% |`);
+        }
+        lines.push("");
+    }
+    // Missed CVEs
+    const missed = result.results.filter((r) => !r.error && !r.detected);
+    if (missed.length > 0) {
+        lines.push("## Missed CVEs");
+        lines.push("");
+        for (const r of missed.slice(0, 20)) {
+            lines.push(`- **${r.cve}** (${r.cwes.join(", ")}): ${r.language}`);
+        }
+        if (missed.length > 20) {
+            lines.push(`- ... and ${missed.length - 20} more`);
+        }
+        lines.push("");
+    }
+    // False positives on patched code
+    const fpOnPatch = result.results.filter((r) => !r.error && !r.cleanOnPatch);
+    if (fpOnPatch.length > 0) {
+        lines.push("## False Positives on Patched Code");
+        lines.push("");
+        for (const r of fpOnPatch.slice(0, 10)) {
+            const fpRules = [...new Set(r.postPatchFindings.map((f) => f.ruleId))];
+            lines.push(`- **${r.cve}**: ${fpRules.join(", ")}`);
+        }
+        if (fpOnPatch.length > 10) {
+            lines.push(`- ... and ${fpOnPatch.length - 10} more`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+// ─── CLI Entry Point ────────────────────────────────────────────────────────
+export function runOpenSSFCveBenchmark(argv) {
+    const subcommand = argv[3] || "run";
+    if (subcommand === "--help" || subcommand === "-h") {
+        console.log(`
+Judges Panel — OpenSSF CVE Benchmark
+
+Evaluates Judges against 200+ real-world CVEs from the OpenSSF CVE Benchmark.
+Requires the benchmark repo to be cloned locally.
+
+USAGE:
+  judges openssf-cve run [options]       Run benchmark against all CVEs
+  judges openssf-cve convert [options]   Convert CVEs to BenchmarkCase format
+  judges openssf-cve report [options]    Generate markdown report from results
+
+OPTIONS:
+  --repo, -r <path>    Path to the ossf-cve-benchmark repo (default: ../ossf-cve-benchmark)
+  --cve <id>           Evaluate a single CVE (e.g. CVE-2018-16492)
+  --output, -o <path>  Save results to file
+  --format <fmt>       Output: text, json, markdown (default: text)
+
+SETUP:
+  git clone https://github.com/ossf-cve-benchmark/ossf-cve-benchmark.git
+  cd ossf-cve-benchmark && npm i && npm run build
+`);
+        process.exit(0);
+    }
+    let repoPath = resolve("..", "ossf-cve-benchmark");
+    let singleCve;
+    let outputPath;
+    let format = "text";
+    for (let i = 4; i < argv.length; i++) {
+        const arg = argv[i];
+        if (arg === "--repo" || arg === "-r")
+            repoPath = resolve(argv[++i]);
+        else if (arg === "--cve")
+            singleCve = argv[++i];
+        else if (arg === "--output" || arg === "-o")
+            outputPath = argv[++i];
+        else if (arg === "--format")
+            format = argv[++i];
+    }
+    if (!existsSync(repoPath)) {
+        console.error(`OpenSSF CVE Benchmark repo not found at: ${repoPath}`);
+        console.error("Clone it with:");
+        console.error("  git clone https://github.com/ossf-cve-benchmark/ossf-cve-benchmark.git");
+        process.exit(1);
+    }
+    const cves = loadCveFiles(repoPath);
+    console.log(`Loaded ${cves.length} published CVEs from ${repoPath}`);
+    const sourcesDir = ensureSourcesDir(repoPath);
+    if (subcommand === "convert") {
+        console.log("Converting CVEs to BenchmarkCase format...");
+        const cases = convertToBenchmarkCases(cves, sourcesDir);
+        const output = JSON.stringify(cases, null, 2);
+        if (outputPath) {
+            writeFileSync(outputPath, output, "utf-8");
+            console.log(`Wrote ${cases.length} benchmark cases to ${outputPath}`);
+        }
+        else {
+            console.log(output);
+        }
+        return;
+    }
+    // Run evaluation
+    const targetCves = singleCve ? cves.filter((c) => c.CVE === singleCve) : cves;
+    if (targetCves.length === 0) {
+        console.error(singleCve ? `CVE ${singleCve} not found in dataset` : "No CVEs to evaluate");
+        process.exit(1);
+    }
+    console.log(`Evaluating ${targetCves.length} CVEs...`);
+    const results = [];
+    for (let i = 0; i < targetCves.length; i++) {
+        const cve = targetCves[i];
+        const pct = Math.round(((i + 1) / targetCves.length) * 100);
+        process.stdout.write(`\r[${i + 1}/${targetCves.length}] ${pct}% ${cve.CVE}`);
+        const result = evaluateSingleCve(cve, sourcesDir);
+        results.push(result);
+        const icon = result.error ? "⚠️" : result.detected ? "✅" : "❌";
+        process.stdout.write(`\r[${i + 1}/${targetCves.length}] ${pct}% ${icon} ${cve.CVE}                    \n`);
+    }
+    const metrics = computeOpenSSFMetrics(results);
+    if (format === "json") {
+        const output = JSON.stringify(metrics, null, 2);
+        if (outputPath) {
+            writeFileSync(outputPath, output, "utf-8");
+            console.log(`Results saved to ${outputPath}`);
+        }
+        else {
+            console.log(output);
+        }
+    }
+    else if (format === "markdown") {
+        const report = formatOpenSSFReport(metrics);
+        if (outputPath) {
+            writeFileSync(outputPath, report, "utf-8");
+            console.log(`Report saved to ${outputPath}`);
+        }
+        else {
+            console.log(report);
+        }
+    }
+    else {
+        // Text summary
+        console.log("\n─── OpenSSF CVE Benchmark Results ───\n");
+        console.log(`  Evaluated:       ${metrics.evaluated}/${metrics.totalCves} CVEs`);
+        console.log(`  Detected:        ${metrics.detected} (${(metrics.detectionRate * 100).toFixed(1)}%)`);
+        console.log(`  Missed:          ${metrics.missed}`);
+        console.log(`  Clean on Patch:  ${metrics.cleanOnPatch}/${metrics.evaluated}`);
+        console.log(`  FP on Patch:     ${metrics.falsePositiveOnPatch}`);
+        console.log(`  Precision:       ${(metrics.precision * 100).toFixed(1)}%`);
+        console.log(`  Recall:          ${(metrics.recall * 100).toFixed(1)}%`);
+        console.log(`  F1 Score:        ${(metrics.f1Score * 100).toFixed(1)}%`);
+        if (outputPath) {
+            writeFileSync(outputPath, JSON.stringify(metrics, null, 2), "utf-8");
+            console.log(`\nFull results saved to ${outputPath}`);
+        }
+    }
+}
+// ─── Adapter Registration ───────────────────────────────────────────────────
+function readJudgesVersion() {
+    try {
+        const pkg = JSON.parse(readFileSync(resolve("package.json"), "utf-8"));
+        return pkg.version ?? "unknown";
+    }
+    catch {
+        return "unknown";
+    }
+}
+/**
+ * Convert OpenSSFBenchmarkResult → ExternalBenchmarkResult for the registry.
+ */
+export function toExternalResult(metrics) {
+    return {
+        suiteId: "openssf-cve",
+        suiteName: "OpenSSF CVE Benchmark",
+        suiteUrl: "https://github.com/ossf-cve-benchmark/ossf-cve-benchmark",
+        timestamp: metrics.timestamp,
+        judgesVersion: readJudgesVersion(),
+        totalItems: metrics.totalCves,
+        evaluatedItems: metrics.evaluated,
+        skippedItems: metrics.skipped,
+        precision: metrics.precision,
+        recall: metrics.recall,
+        f1Score: metrics.f1Score,
+        detectionRate: metrics.detectionRate,
+        truePositives: metrics.detected,
+        falsePositives: metrics.falsePositiveOnPatch,
+        falseNegatives: metrics.missed,
+        perCategory: metrics.perCwe,
+        rawData: metrics,
+    };
+}
+const openSSFAdapter = {
+    suiteId: "openssf-cve",
+    suiteName: "OpenSSF CVE Benchmark",
+    suiteUrl: "https://github.com/ossf-cve-benchmark/ossf-cve-benchmark",
+    defaultRepoPath: "../ossf-cve-benchmark",
+    description: "200+ real-world JS/TS CVEs with pre-patch and post-patch commits",
+    validate(repoPath) {
+        if (!existsSync(repoPath)) {
+            return `Repo not found at ${repoPath}. Clone with: git clone https://github.com/ossf-cve-benchmark/ossf-cve-benchmark.git`;
+        }
+        if (!existsSync(join(repoPath, "CVEs"))) {
+            return `CVEs directory not found at ${repoPath}/CVEs. Is this the correct repo?`;
+        }
+        return undefined;
+    },
+    run(config) {
+        const cves = loadCveFiles(config.repoPath);
+        console.log(`  Loaded ${cves.length} published CVEs`);
+        const sourcesDir = ensureSourcesDir(config.repoPath);
+        const targetCves = config.singleItem ? cves.filter((c) => c.CVE === config.singleItem) : cves;
+        if (targetCves.length === 0) {
+            return toExternalResult(computeOpenSSFMetrics([]));
+        }
+        console.log(`  Evaluating ${targetCves.length} CVEs...`);
+        const results = [];
+        for (let i = 0; i < targetCves.length; i++) {
+            const cve = targetCves[i];
+            const pct = Math.round(((i + 1) / targetCves.length) * 100);
+            process.stdout.write(`\r  [${i + 1}/${targetCves.length}] ${pct}% ${cve.CVE}`);
+            const result = evaluateSingleCve(cve, sourcesDir);
+            results.push(result);
+            const icon = result.error ? "⚠️" : result.detected ? "✅" : "❌";
+            process.stdout.write(`\r  [${i + 1}/${targetCves.length}] ${pct}% ${icon} ${cve.CVE}                    \n`);
+        }
+        return toExternalResult(computeOpenSSFMetrics(results));
+    },
+};
+registerBenchmarkAdapter(openSSFAdapter);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kevinrabun/judges-cli",
-  "version": "3.128.3",
+  "version": "3.129.0",
   "description": "CLI wrapper for the Judges code review toolkit.",
   "type": "module",
   "main": "dist/cli.js",