@kevinrabun/judges-cli 3.124.0

package/dist/finding-lifecycle.js

@@ -0,0 +1,547 @@
+/**
+ * Finding Lifecycle Tracking
+ *
+ * Tracks individual findings across multiple evaluation runs, enabling:
+ * - New vs. recurring finding classification
+ * - "Fixed" detection when a finding disappears
+ * - Finding trend statistics (are things getting better or worse?)
+ * - Age tracking (how long has a finding been open?)
+ *
+ * Data stored in .judges-findings.json
+ */
+import { readFileSync, writeFileSync, existsSync } from "fs";
+import { resolve } from "path";
+import { getDataAdapter } from "./data-adapter.js";
+// ─── Fingerprinting ─────────────────────────────────────────────────────────
+/**
+ * Generate a stable fingerprint for a finding that survives minor code edits.
+ * Based on ruleId + file path + approximate code location.
+ */
+export function generateFindingFingerprint(finding, filePath) {
+    // Use ruleId + file + line range bucket (groups of 5 lines to handle minor shifts)
+    const lineBucket = finding.lineNumbers?.[0] ? Math.floor(finding.lineNumbers[0] / 5) * 5 : 0;
+    // Simple hash-like string for fast comparison
+    const key = `${finding.ruleId}::${filePath}::${lineBucket}::${finding.title.slice(0, 50)}`;
+    return key;
+}
+// ─── Store I/O ───────────────────────────────────────────────────────────────
+const FINDINGS_FILE = ".judges-findings.json";
+export function loadFindingStore(dir = ".") {
+    const filePath = resolve(dir, FINDINGS_FILE);
+    if (!existsSync(filePath)) {
+        return {
+            version: "1.0.0",
+            lastRunAt: new Date().toISOString(),
+            runNumber: 0,
+            findings: [],
+        };
+    }
+    try {
+        return JSON.parse(readFileSync(filePath, "utf-8"));
+    }
+    catch {
+        return {
+            version: "1.0.0",
+            lastRunAt: new Date().toISOString(),
+            runNumber: 0,
+            findings: [],
+        };
+    }
+}
+export function saveFindingStore(store, dir = ".") {
+    const filePath = resolve(dir, FINDINGS_FILE);
+    writeFileSync(filePath, JSON.stringify(store, null, 2) + "\n", "utf-8");
+}
+/**
+ * Load finding store via the configured DataAdapter.
+ */
+export async function loadFindingsViaAdapter(projectDir, adapter) {
+    const da = adapter ?? getDataAdapter();
+    return da.loadFindings(projectDir);
+}
+/**
+ * Save finding store via the configured DataAdapter.
+ */
+export async function saveFindingsViaAdapter(store, projectDir, adapter) {
+    const da = adapter ?? getDataAdapter();
+    return da.saveFindings(store, projectDir);
+}
+// ─── Lifecycle Operations ────────────────────────────────────────────────────
+/**
+ * Update the finding store with results from a new evaluation run.
+ * Returns a delta describing what changed since the last run.
+ *
+ * When `dirOrStore` is a string, reads/writes `.judges-findings.json` in that
+ * directory.  When a `FindingStore` object is passed directly, operates
+ * in-memory (useful for testing) and does NOT persist to disk.
+ */
+export function updateFindings(currentFindings, dirOrStore = ".") {
+    const inMemory = typeof dirOrStore !== "string";
+    const store = inMemory ? dirOrStore : loadFindingStore(dirOrStore);
+    const now = new Date().toISOString();
+    store.runNumber++;
+    store.lastRunAt = now;
+    // Build fingerprint → finding map for current run
+    const currentMap = new Map();
+    for (const entry of currentFindings) {
+        const fp = generateFindingFingerprint(entry.finding, entry.filePath);
+        currentMap.set(fp, entry);
+    }
+    // Build fingerprint → tracked finding map for existing store
+    const existingMap = new Map();
+    for (const tracked of store.findings) {
+        existingMap.set(tracked.fingerprint, tracked);
+    }
+    const delta = {
+        introduced: [],
+        recurring: [],
+        fixed: [],
+        stats: {
+            totalOpen: 0,
+            totalFixed: 0,
+            introduced: 0,
+            recurring: 0,
+            fixed: 0,
+            trend: "stable",
+        },
+    };
+    // Process current findings
+    for (const [fp, entry] of currentMap) {
+        const existing = existingMap.get(fp);
+        if (existing) {
+            // Recurring — update last seen and run count
+            existing.lastSeen = now;
+            existing.runCount++;
+            existing.status = "open";
+            existing.fixedAt = undefined;
+            delta.recurring.push(existing);
+        }
+        else {
+            // New finding
+            const tracked = {
+                fingerprint: fp,
+                ruleId: entry.finding.ruleId,
+                severity: entry.finding.severity,
+                filePath: entry.filePath,
+                title: entry.finding.title,
+                firstSeen: now,
+                lastSeen: now,
+                runCount: 1,
+                status: "open",
+            };
+            store.findings.push(tracked);
+            delta.introduced.push(tracked);
+        }
+    }
+    // Detect fixed findings (were open, no longer present)
+    // Triaged findings (accepted-risk, deferred, wont-fix, false-positive) are
+    // NOT auto-marked as fixed — their triage decision is preserved.
+    const triageStatuses = new Set(["accepted-risk", "deferred", "wont-fix", "false-positive"]);
+    for (const tracked of store.findings) {
+        if (triageStatuses.has(tracked.status))
+            continue;
+        if (tracked.status === "open" && !currentMap.has(tracked.fingerprint)) {
+            tracked.status = "fixed";
+            tracked.fixedAt = now;
+            delta.fixed.push(tracked);
+        }
+    }
+    // Compute stats
+    const openFindings = store.findings.filter((f) => f.status === "open");
+    const fixedFindings = store.findings.filter((f) => f.status === "fixed");
+    delta.stats.totalOpen = openFindings.length;
+    delta.stats.totalFixed = fixedFindings.length;
+    delta.stats.introduced = delta.introduced.length;
+    delta.stats.recurring = delta.recurring.length;
+    delta.stats.fixed = delta.fixed.length;
+    // Determine trend
+    if (delta.fixed.length > delta.introduced.length) {
+        delta.stats.trend = "improving";
+    }
+    else if (delta.introduced.length > delta.fixed.length) {
+        delta.stats.trend = "degrading";
+    }
+    else {
+        delta.stats.trend = "stable";
+    }
+    // Prune very old fixed findings (> 30 days)
+    const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000).toISOString();
+    const triageStatusSet = new Set(["accepted-risk", "deferred", "wont-fix", "false-positive"]);
+    store.findings = store.findings.filter((f) => f.status === "open" || triageStatusSet.has(f.status) || (f.fixedAt && f.fixedAt > thirtyDaysAgo));
+    if (!inMemory) {
+        saveFindingStore(store, dirOrStore);
+    }
+    return delta;
+}
+/**
+ * Get summary statistics from the finding store.
+ */
+export function getFindingStats(dirOrStore = ".") {
+    const store = typeof dirOrStore === "string" ? loadFindingStore(dirOrStore) : dirOrStore;
+    const openFindings = store.findings.filter((f) => f.status === "open");
+    const fixedFindings = store.findings.filter((f) => f.status === "fixed");
+    const triageStatuses = new Set(["accepted-risk", "deferred", "wont-fix", "false-positive"]);
+    const triagedFindings = store.findings.filter((f) => triageStatuses.has(f.status));
+    const bySeverity = {};
+    const byRule = {};
+    const byTriageStatus = {};
+    let totalAgeDays = 0;
+    for (const f of openFindings) {
+        bySeverity[f.severity] = (bySeverity[f.severity] || 0) + 1;
+        byRule[f.ruleId] = (byRule[f.ruleId] || 0) + 1;
+        const age = (Date.now() - new Date(f.firstSeen).getTime()) / (1000 * 60 * 60 * 24);
+        totalAgeDays += age;
+    }
+    for (const f of triagedFindings) {
+        byTriageStatus[f.status] = (byTriageStatus[f.status] || 0) + 1;
+    }
+    const sortedOpen = [...openFindings].sort((a, b) => new Date(a.firstSeen).getTime() - new Date(b.firstSeen).getTime());
+    return {
+        totalOpen: openFindings.length,
+        totalFixed: fixedFindings.length,
+        totalTriaged: triagedFindings.length,
+        byTriageStatus,
+        oldestOpen: sortedOpen[0]?.firstSeen,
+        bySeverity,
+        byRule,
+        avgAge: openFindings.length > 0 ? totalAgeDays / openFindings.length : 0,
+        runCount: store.runNumber,
+    };
+}
+/**
+ * Format finding delta as a human-readable summary for CLI output.
+ */
+export function formatDelta(delta) {
+    const trendEmoji = delta.stats.trend === "improving" ? "📈" : delta.stats.trend === "degrading" ? "📉" : "➡️";
+    const lines = [
+        `  Finding Lifecycle: ${trendEmoji} ${delta.stats.trend}`,
+        `  Open: ${delta.stats.totalOpen} | Fixed: ${delta.stats.totalFixed}`,
+    ];
+    if (delta.stats.introduced > 0) {
+        lines.push(`  🆕 New: ${delta.stats.introduced}`);
+        for (const f of delta.introduced.slice(0, 5)) {
+            lines.push(`     + [${f.severity.toUpperCase()}] ${f.ruleId}: ${f.title}`);
+        }
+        if (delta.introduced.length > 5) {
+            lines.push(`     ... and ${delta.introduced.length - 5} more`);
+        }
+    }
+    if (delta.stats.fixed > 0) {
+        lines.push(`  ✅ Fixed: ${delta.stats.fixed}`);
+        for (const f of delta.fixed.slice(0, 5)) {
+            lines.push(`     - [${f.severity.toUpperCase()}] ${f.ruleId}: ${f.title}`);
+        }
+        if (delta.fixed.length > 5) {
+            lines.push(`     ... and ${delta.fixed.length - 5} more`);
+        }
+    }
+    if (delta.stats.recurring > 0) {
+        lines.push(`  🔄 Recurring: ${delta.stats.recurring}`);
+    }
+    return lines.join("\n");
+}
+/**
+ * Triage a finding by its fingerprint or ruleId+filePath.
+ * Returns the triaged finding, or null if not found.
+ */
+export function triageFinding(dirOrStore, match, status, reason, triagedBy) {
+    const inMemory = typeof dirOrStore !== "string";
+    const store = inMemory ? dirOrStore : loadFindingStore(dirOrStore);
+    const now = new Date().toISOString();
+    let target;
+    if (match.fingerprint) {
+        target = store.findings.find((f) => f.fingerprint === match.fingerprint && f.status === "open");
+    }
+    else if (match.ruleId) {
+        target = store.findings.find((f) => f.ruleId === match.ruleId && f.status === "open" && (!match.filePath || f.filePath === match.filePath));
+    }
+    if (!target)
+        return null;
+    target.status = status;
+    target.triagedAt = now;
+    target.triagedBy = triagedBy;
+    target.triageReason = reason;
+    if (!inMemory) {
+        saveFindingStore(store, dirOrStore);
+    }
+    return target;
+}
+/**
+ * List all findings with a specific triage status.
+ */
+export function getTriagedFindings(dirOrStore, status) {
+    const store = typeof dirOrStore === "string" ? loadFindingStore(dirOrStore) : dirOrStore;
+    const triageStatuses = new Set(["accepted-risk", "deferred", "wont-fix", "false-positive"]);
+    return store.findings.filter((f) => (status ? f.status === status : triageStatuses.has(f.status)));
+}
+/**
+ * Format triaged findings as a human-readable summary.
+ */
+export function formatTriageSummary(dirOrStore) {
+    const store = typeof dirOrStore === "string" ? loadFindingStore(dirOrStore) : dirOrStore;
+    const triageStatuses = new Set(["accepted-risk", "deferred", "wont-fix", "false-positive"]);
+    const triaged = store.findings.filter((f) => triageStatuses.has(f.status));
+    if (triaged.length === 0) {
+        return "  No triaged findings.";
+    }
+    const lines = [`  Triaged Findings: ${triaged.length}`, ""];
+    const grouped = new Map();
+    for (const f of triaged) {
+        const list = grouped.get(f.status) || [];
+        list.push(f);
+        grouped.set(f.status, list);
+    }
+    const statusLabels = {
+        "accepted-risk": "⚠️  Accepted Risk",
+        deferred: "⏳ Deferred",
+        "wont-fix": "🚫 Won't Fix",
+        "false-positive": "❌ False Positive",
+    };
+    for (const [status, findings] of grouped) {
+        lines.push(`  ${statusLabels[status] || status} (${findings.length}):`);
+        for (const f of findings.slice(0, 10)) {
+            lines.push(`    [${f.severity.toUpperCase()}] ${f.ruleId}: ${f.title}`);
+            if (f.triageReason) {
+                lines.push(`      Reason: ${f.triageReason}`);
+            }
+        }
+        if (findings.length > 10) {
+            lines.push(`    ... and ${findings.length - 10} more`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+// ─── Triage → Auto-Tune Bridge ──────────────────────────────────────────────
+/**
+ * Convert triage decisions from the finding store into feedback entries
+ * suitable for auto-tune. This bridges lifecycle triage (users marking
+ * findings as false-positive, wont-fix, etc.) into the calibration loop.
+ *
+ * - false-positive → verdict "fp"
+ * - wont-fix → verdict "fp" (not useful enough to keep)
+ * - accepted-risk → verdict "tp" (real finding, intentionally accepted)
+ * - deferred → verdict "tp" (real finding, just not fixing now)
+ */
+export function triageToFeedbackEntries(dirOrStore) {
+    const store = typeof dirOrStore === "string" ? loadFindingStore(dirOrStore) : dirOrStore;
+    const entries = [];
+    for (const f of store.findings) {
+        if (f.status === "false-positive" || f.status === "wont-fix") {
+            entries.push({
+                ruleId: f.ruleId,
+                verdict: "fp",
+                severity: f.severity,
+                timestamp: f.triagedAt || f.lastSeen,
+            });
+        }
+        else if (f.status === "accepted-risk" || f.status === "deferred") {
+            entries.push({
+                ruleId: f.ruleId,
+                verdict: "tp",
+                severity: f.severity,
+                timestamp: f.triagedAt || f.lastSeen,
+            });
+        }
+    }
+    return entries;
+}
+/**
+ * Get the set of rule IDs that should be auto-suppressed based on
+ * accumulated triage history. Uses a simple threshold: if ≥ `threshold`
+ * fraction of triaged findings for a rule are FP/wont-fix, suppress it.
+ */
+export function getTriageBasedSuppressions(dirOrStore, options = {}) {
+    const threshold = options.threshold ?? 0.8;
+    const minSamples = options.minSamples ?? 3;
+    const entries = triageToFeedbackEntries(dirOrStore);
+    const byRule = new Map();
+    for (const e of entries) {
+        const stats = byRule.get(e.ruleId) ?? { fp: 0, total: 0 };
+        stats.total++;
+        if (e.verdict === "fp")
+            stats.fp++;
+        byRule.set(e.ruleId, stats);
+    }
+    const suppressed = new Set();
+    for (const [ruleId, stats] of byRule) {
+        if (stats.total >= minSamples && stats.fp / stats.total >= threshold) {
+            suppressed.add(ruleId);
+        }
+    }
+    return suppressed;
+}
+/**
+ * Compute detailed suppression analytics from the finding store.
+ * Identifies patterns in what gets suppressed and recommends tuning actions.
+ */
+export function getSuppressionAnalytics(dirOrStore, options = {}) {
+    const fpThreshold = options.fpThreshold ?? 0.7;
+    const minSamples = options.minSamples ?? 3;
+    const store = typeof dirOrStore === "string" ? loadFindingStore(dirOrStore) : dirOrStore;
+    const totalTracked = store.findings.length;
+    const suppressed = store.findings.filter((f) => f.status === "false-positive" || f.status === "wont-fix");
+    // ── Per-rule analytics ──
+    const ruleStats = new Map();
+    for (const f of store.findings) {
+        const stats = ruleStats.get(f.ruleId) ?? { total: 0, fp: 0, wontFix: 0, deferred: 0, acceptedRisk: 0 };
+        stats.total++;
+        if (f.status === "false-positive")
+            stats.fp++;
+        if (f.status === "wont-fix")
+            stats.wontFix++;
+        if (f.status === "deferred")
+            stats.deferred++;
+        if (f.status === "accepted-risk")
+            stats.acceptedRisk++;
+        ruleStats.set(f.ruleId, stats);
+    }
+    const topSuppressedRules = [...ruleStats.entries()]
+        .map(([ruleId, s]) => ({
+        ruleId,
+        judgePrefix: ruleId.replace(/-\d+$/, ""),
+        suppressedCount: s.fp + s.wontFix,
+        totalCount: s.total,
+        fpRate: s.total > 0 ? (s.fp + s.wontFix) / s.total : 0,
+    }))
+        .filter((r) => r.suppressedCount > 0)
+        .sort((a, b) => b.suppressedCount - a.suppressedCount);
+    // ── Per-judge analytics ──
+    const judgeStats = new Map();
+    for (const [ruleId, s] of ruleStats) {
+        const prefix = ruleId.replace(/-\d+$/, "");
+        const js = judgeStats.get(prefix) ?? { total: 0, fp: 0, wontFix: 0, deferred: 0, acceptedRisk: 0 };
+        js.total += s.total;
+        js.fp += s.fp;
+        js.wontFix += s.wontFix;
+        js.deferred += s.deferred;
+        js.acceptedRisk += s.acceptedRisk;
+        judgeStats.set(prefix, js);
+    }
+    const byJudge = [...judgeStats.entries()]
+        .map(([judgePrefix, s]) => ({
+        judgePrefix,
+        totalFindings: s.total,
+        fpCount: s.fp,
+        wontFixCount: s.wontFix,
+        deferredCount: s.deferred,
+        acceptedRiskCount: s.acceptedRisk,
+        fpRate: s.total > 0 ? (s.fp + s.wontFix) / s.total : 0,
+    }))
+        .sort((a, b) => b.fpRate - a.fpRate);
+    // ── Auto-suppress candidates ──
+    const autoSuppressCandidates = topSuppressedRules
+        .filter((r) => r.totalCount >= minSamples && r.fpRate >= fpThreshold)
+        .map((r) => r.ruleId);
+    // ── Frequently deferred ──
+    const frequentlyDeferred = [...ruleStats.entries()]
+        .filter(([, s]) => s.deferred >= minSamples)
+        .map(([ruleId]) => ruleId);
+    // ── Recommendations ──
+    const recommendations = [];
+    if (autoSuppressCandidates.length > 0) {
+        recommendations.push(`${autoSuppressCandidates.length} rule(s) have FP rate ≥ ${(fpThreshold * 100).toFixed(0)}% — consider auto-suppressing: ${autoSuppressCandidates.join(", ")}`);
+    }
+    for (const jp of byJudge) {
+        if (jp.totalFindings >= minSamples * 2 && jp.fpRate >= 0.5) {
+            recommendations.push(`Judge ${jp.judgePrefix} has ${(jp.fpRate * 100).toFixed(0)}% suppression rate across ${jp.totalFindings} findings — review its detection patterns`);
+        }
+    }
+    if (frequentlyDeferred.length > 0) {
+        recommendations.push(`${frequentlyDeferred.length} rule(s) are frequently deferred — consider lowering severity: ${frequentlyDeferred.join(", ")}`);
+    }
+    if (suppressed.length === 0 && totalTracked > 10) {
+        recommendations.push("No findings have been triaged yet — use triage tools to improve calibration over time");
+    }
+    return {
+        totalTracked,
+        totalSuppressed: suppressed.length,
+        suppressionRate: totalTracked > 0 ? suppressed.length / totalTracked : 0,
+        topSuppressedRules,
+        byJudge,
+        autoSuppressCandidates,
+        frequentlyDeferred,
+        recommendations,
+    };
+}
+/**
+ * Format suppression analytics as a human-readable summary.
+ */
+export function formatSuppressionAnalytics(analytics) {
+    const lines = [
+        `Suppression Analytics`,
+        `━━━━━━━━━━━━━━━━━━━━`,
+        `Total tracked: ${analytics.totalTracked} | Suppressed: ${analytics.totalSuppressed} (${(analytics.suppressionRate * 100).toFixed(1)}%)`,
+        "",
+    ];
+    if (analytics.topSuppressedRules.length > 0) {
+        lines.push("Top Suppressed Rules:");
+        for (const r of analytics.topSuppressedRules.slice(0, 10)) {
+            lines.push(`  ${r.ruleId}: ${r.suppressedCount}/${r.totalCount} suppressed (${(r.fpRate * 100).toFixed(0)}% FP rate)`);
+        }
+        lines.push("");
+    }
+    if (analytics.byJudge.length > 0) {
+        lines.push("By Judge:");
+        for (const j of analytics.byJudge.slice(0, 10)) {
+            lines.push(`  ${j.judgePrefix}: ${j.totalFindings} total, ${j.fpCount} FP, ${j.wontFixCount} won't-fix (${(j.fpRate * 100).toFixed(0)}% suppression rate)`);
+        }
+        lines.push("");
+    }
+    if (analytics.recommendations.length > 0) {
+        lines.push("Recommendations:");
+        for (const r of analytics.recommendations) {
+            lines.push(`  💡 ${r}`);
+        }
+    }
+    return lines.join("\n");
+}
+// ─── Triage Feedback Learning Loop ──────────────────────────────────────────
+/**
+ * Compute per-rule false positive rates from historical triage data.
+ * Returns a map from ruleId to a confidence adjustment factor
+ * (negative number indicating how much to reduce confidence).
+ */
+export function computeTriageFeedback(store) {
+    const ruleCounts = new Map();
+    for (const tracked of store.findings) {
+        const ruleId = tracked.ruleId;
+        const entry = ruleCounts.get(ruleId) ?? { total: 0, fp: 0 };
+        entry.total++;
+        if (tracked.status === "false-positive" || tracked.status === "wont-fix") {
+            entry.fp++;
+        }
+        ruleCounts.set(ruleId, entry);
+    }
+    const adjustments = new Map();
+    for (const [ruleId, { total, fp }] of ruleCounts) {
+        if (total < 3)
+            continue; // Need at least 3 observations to adjust
+        const fpRate = fp / total;
+        // High FP rate → reduce confidence proportionally (max -0.3)
+        if (fpRate > 0.3) {
+            adjustments.set(ruleId, -Math.min(fpRate * 0.5, 0.3));
+        }
+    }
+    return adjustments;
+}
+/**
+ * Apply triage feedback to a set of findings, adjusting confidence scores
+ * downward for rules that have historically high false positive rates.
+ */
+export function applyTriageFeedback(findings, store) {
+    const adjustments = computeTriageFeedback(store);
+    if (adjustments.size === 0)
+        return findings;
+    return findings.map((f) => {
+        const adj = adjustments.get(f.ruleId);
+        if (adj === undefined)
+            return f;
+        const adjusted = Math.max(0.1, (f.confidence ?? 0.8) + adj);
+        return {
+            ...f,
+            confidence: adjusted,
+            needsHumanReview: true, // Flag for human review if confidence was reduced
+        };
+    });
+}

package/dist/fingerprint.d.ts

@@ -0,0 +1,39 @@
+/**
+ * AI Code Fingerprinting — Detect AI-generated code patterns
+ *
+ * Identifies patterns commonly produced by AI code generators (GitHub Copilot,
+ * ChatGPT, Claude, etc.) to flag code that may need extra review.
+ *
+ * Signals include:
+ * - Generic variable naming patterns
+ * - Overly verbose/uniform comment styles
+ * - Common AI boilerplate patterns
+ * - Missing error handling in generated code
+ * - Suspiciously complete implementations without tests
+ */
+import type { Finding } from "./types.js";
+export interface AiFingerprint {
+    /** Confidence that code is AI-generated (0-1) */
+    aiProbability: number;
+    /** Individual signals detected */
+    signals: AiSignal[];
+    /** Risk level based on probability */
+    riskLevel: "high" | "medium" | "low" | "none";
+    /** Summary text */
+    summary: string;
+}
+export interface AiSignal {
+    name: string;
+    description: string;
+    weight: number;
+    matches: number;
+    evidence?: string;
+}
+/**
+ * Analyze code for AI-generated patterns and return a fingerprint.
+ */
+export declare function fingerprintCode(code: string, language?: string): AiFingerprint;
+/**
+ * Convert AI fingerprint to findings for inclusion in evaluation results.
+ */
+export declare function fingerprintToFindings(fingerprint: AiFingerprint): Finding[];