@kevinrabun/judges-cli 3.128.3 → 3.129.1

package/dist/commands/openssf-cve-benchmark.d.ts

@@ -0,0 +1,96 @@
+/**
+ * OpenSSF CVE Benchmark Integration
+ *
+ * Runs the Judges evaluation engine against the OpenSSF CVE Benchmark dataset
+ * (https://github.com/ossf-cve-benchmark/ossf-cve-benchmark) — 200+ real-world
+ * JavaScript/TypeScript CVEs with pre-patch (vulnerable) and post-patch (fixed)
+ * git commits.
+ *
+ * Two modes:
+ *   1. Deterministic (L1): Runs Judges' pattern-based evaluators against each CVE.
+ *   2. LLM integration: Converts passing CVE cases into BenchmarkCase format
+ *      for inclusion in the LLM benchmark pipeline.
+ *
+ * Usage:
+ *   judges openssf-cve run [--repo <path>] [--cve <id>] [--format json|text|markdown]
+ *   judges openssf-cve convert [--repo <path>]  # Convert to BenchmarkCase[]
+ */
+import type { Finding } from "../types.js";
+import type { BenchmarkCase } from "./benchmark.js";
+import type { ExternalBenchmarkResult } from "./external-benchmarks.js";
+/** Raw JSON from a CVE file in the OpenSSF benchmark repo */
+export interface OpenSSFCve {
+    CVE: string;
+    state: "PUBLISHED" | "DRAFT" | "RESERVED";
+    repository: string;
+    prePatch: {
+        commit: string;
+        weaknesses: Array<{
+            location: {
+                file: string;
+                line: number;
+            };
+            explanation: string;
+        }>;
+    };
+    postPatch: {
+        commit: string;
+    };
+    CWEs: string[];
+}
+export interface CveEvalResult {
+    cve: string;
+    cwes: string[];
+    language: string;
+    /** Did Judges detect at least one finding matching a relevant CWE? */
+    detected: boolean;
+    /** Did Judges produce no false positives on the patched version? */
+    cleanOnPatch: boolean;
+    /** Relevant findings on the pre-patch (vulnerable) code */
+    prePatchFindings: Finding[];
+    /** Findings on the post-patch (fixed) code — ideally empty */
+    postPatchFindings: Finding[];
+    /** Which CWEs from the CVE were matched by findings */
+    matchedCwes: string[];
+    /** Which CWEs from the CVE were NOT matched */
+    missedCwes: string[];
+    /** Error message if evaluation failed */
+    error?: string;
+}
+export interface OpenSSFBenchmarkResult {
+    timestamp: string;
+    totalCves: number;
+    evaluated: number;
+    skipped: number;
+    detected: number;
+    missed: number;
+    cleanOnPatch: number;
+    falsePositiveOnPatch: number;
+    detectionRate: number;
+    precision: number;
+    recall: number;
+    f1Score: number;
+    perCwe: Record<string, {
+        total: number;
+        detected: number;
+        rate: number;
+    }>;
+    results: CveEvalResult[];
+}
+export declare function loadCveFiles(repoPath: string): OpenSSFCve[];
+export declare function evaluateSingleCve(cve: OpenSSFCve, sourcesDir: string): CveEvalResult;
+export declare function computeOpenSSFMetrics(results: CveEvalResult[]): OpenSSFBenchmarkResult;
+/**
+ * Convert OpenSSF CVE results into BenchmarkCase[] format for use
+ * in the Judges LLM benchmark pipeline. Only includes CVEs where:
+ * - The vulnerable code was successfully checked out
+ * - At least one weakness file was found
+ * - CWEs map to known judge prefixes
+ */
+export declare function convertToBenchmarkCases(cves: OpenSSFCve[], sourcesDir: string): BenchmarkCase[];
+export declare function formatOpenSSFReport(result: OpenSSFBenchmarkResult): string;
+export declare function runOpenSSFCveBenchmark(argv: string[]): void;
+/**
+ * Convert OpenSSFBenchmarkResult → ExternalBenchmarkResult for the registry.
+ */
+export declare function toExternalResult(metrics: OpenSSFBenchmarkResult): ExternalBenchmarkResult;