@kevinrabun/judges-cli 3.124.0

package/dist/commands/judge-learn.js

@@ -0,0 +1,217 @@
+/**
+ * Judge-learn — interactive wizard that ingests false positive/negative
+ * feedback to automatically generate domain-specific judges with
+ * detection rules and confidence calibration.
+ *
+ * All data stored locally in `.judges-learned/`.
+ */
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from "fs";
+import { join } from "path";
+// ─── Storage ────────────────────────────────────────────────────────────────
+const DATA_DIR = ".judges-learned";
+function ensureDir() {
+    if (!existsSync(DATA_DIR))
+        mkdirSync(DATA_DIR, { recursive: true });
+}
+function loadFeedback() {
+    const file = join(DATA_DIR, "feedback.json");
+    if (!existsSync(file))
+        return [];
+    try {
+        return JSON.parse(readFileSync(file, "utf-8"));
+    }
+    catch {
+        return [];
+    }
+}
+function saveFeedback(entries) {
+    ensureDir();
+    writeFileSync(join(DATA_DIR, "feedback.json"), JSON.stringify(entries, null, 2));
+}
+function loadJudges() {
+    const file = join(DATA_DIR, "judges.json");
+    if (!existsSync(file))
+        return [];
+    try {
+        return JSON.parse(readFileSync(file, "utf-8"));
+    }
+    catch {
+        return [];
+    }
+}
+function saveJudges(judges) {
+    ensureDir();
+    writeFileSync(join(DATA_DIR, "judges.json"), JSON.stringify(judges, null, 2));
+}
+// ─── Pattern extraction ─────────────────────────────────────────────────────
+function extractPatterns(feedback) {
+    const patternMap = new Map();
+    for (const entry of feedback) {
+        const key = `${entry.pattern}:${entry.language}`;
+        const existing = patternMap.get(key) || { entries: [], fpCount: 0, fnCount: 0 };
+        existing.entries.push(entry);
+        if (entry.type === "false-positive")
+            existing.fpCount++;
+        if (entry.type === "false-negative")
+            existing.fnCount++;
+        patternMap.set(key, existing);
+    }
+    const rules = [];
+    for (const [key, data] of patternMap) {
+        const [pattern, language] = key.split(":");
+        const total = data.entries.length;
+        const correctRate = total > 0 ? (total - data.fpCount) / total : 0;
+        const confidence = Math.round(correctRate * 100);
+        // Only create rules for patterns with enough feedback
+        if (total >= 2) {
+            rules.push({
+                id: `learned-${rules.length + 1}`,
+                name: `${pattern}-detector`,
+                pattern,
+                language: language || "any",
+                confidence,
+                feedbackCount: total,
+                description: `Detects "${pattern}" pattern (${total} feedback entries, ${confidence}% confidence)`,
+            });
+        }
+    }
+    return rules.sort((a, b) => b.confidence - a.confidence);
+}
+// ─── CLI ────────────────────────────────────────────────────────────────────
+export function runJudgeLearn(argv) {
+    if (argv.includes("--help") || argv.includes("-h")) {
+        console.log(`
+judges judge-learn — Generate custom judges from feedback
+
+Usage:
+  judges judge-learn --feedback --type fp --pattern "unused-import" --lang ts
+  judges judge-learn --feedback --type fn --pattern "sql-injection" --lang py --context "raw query"
+  judges judge-learn --generate --name "my-team-judge"
+  judges judge-learn --show
+
+Options:
+  --feedback            Record feedback entry
+  --type <fp|fn|ok>     Feedback type: fp (false positive), fn (false negative), ok (correct)
+  --pattern <name>      Pattern/rule that triggered
+  --lang <language>     Language context
+  --context <text>      Additional context
+  --generate            Generate judge from accumulated feedback
+  --name <name>         Judge name for generation
+  --show                Show learned judges and feedback stats
+  --format json         JSON output
+  --help, -h            Show this help
+
+Workflow:
+  1. Record feedback as you review findings (--feedback)
+  2. Generate a custom judge once patterns emerge (--generate)
+  3. Use generated judge in .judgesrc config
+`);
+        return;
+    }
+    const format = argv.find((_a, i) => argv[i - 1] === "--format") || "text";
+    const isFeedback = argv.includes("--feedback");
+    const isGenerate = argv.includes("--generate");
+    const _isShow = argv.includes("--show");
+    if (isFeedback) {
+        const typeMap = {
+            fp: "false-positive",
+            fn: "false-negative",
+            ok: "correct",
+        };
+        const typeArg = argv.find((_a, i) => argv[i - 1] === "--type") || "fp";
+        const pattern = argv.find((_a, i) => argv[i - 1] === "--pattern") || "";
+        const lang = argv.find((_a, i) => argv[i - 1] === "--lang") || "any";
+        const context = argv.find((_a, i) => argv[i - 1] === "--context") || "";
+        if (!pattern) {
+            console.error("  --pattern is required");
+            return;
+        }
+        const entry = {
+            id: `fb-${Date.now()}`,
+            type: typeMap[typeArg] || "false-positive",
+            pattern,
+            language: lang,
+            context,
+            timestamp: new Date().toISOString(),
+        };
+        const feedback = loadFeedback();
+        feedback.push(entry);
+        saveFeedback(feedback);
+        console.log(`  ✅ Recorded ${entry.type} feedback for "${pattern}" (${lang})`);
+        return;
+    }
+    if (isGenerate) {
+        const name = argv.find((_a, i) => argv[i - 1] === "--name") || "custom-judge";
+        const feedback = loadFeedback();
+        if (feedback.length < 3) {
+            console.log(`  Need at least 3 feedback entries to generate a judge (have ${feedback.length}).`);
+            return;
+        }
+        const rules = extractPatterns(feedback);
+        if (rules.length === 0) {
+            console.log("  No patterns with enough feedback to generate rules.");
+            return;
+        }
+        const judge = {
+            id: `learned-${name}`,
+            name,
+            description: `Auto-generated judge from ${feedback.length} feedback entries`,
+            rules,
+            createdAt: new Date().toISOString(),
+            updatedAt: new Date().toISOString(),
+        };
+        const judges = loadJudges();
+        const existing = judges.findIndex((j) => j.id === judge.id);
+        if (existing >= 0) {
+            judges[existing] = judge;
+        }
+        else {
+            judges.push(judge);
+        }
+        saveJudges(judges);
+        if (format === "json") {
+            console.log(JSON.stringify(judge, null, 2));
+        }
+        else {
+            console.log(`\n  ✅ Generated judge: ${name}`);
+            console.log(`     ${rules.length} rule(s) from ${feedback.length} feedback entries\n`);
+            for (const rule of rules) {
+                const icon = rule.confidence >= 80 ? "🟢" : rule.confidence >= 50 ? "🟡" : "🔴";
+                console.log(`     ${icon} ${rule.name} — ${rule.confidence}% confidence (${rule.feedbackCount} entries)`);
+            }
+            console.log(`\n     Add to .judgesrc: { "customJudges": ["${judge.id}"] }\n`);
+        }
+        return;
+    }
+    // Show
+    const feedback = loadFeedback();
+    const judges = loadJudges();
+    if (format === "json") {
+        console.log(JSON.stringify({ feedback: feedback.length, judges, timestamp: new Date().toISOString() }, null, 2));
+        return;
+    }
+    console.log(`\n  Judge Learn — ${feedback.length} feedback entries, ${judges.length} generated judges\n  ──────────────────────────`);
+    if (feedback.length > 0) {
+        const fpCount = feedback.filter((f) => f.type === "false-positive").length;
+        const fnCount = feedback.filter((f) => f.type === "false-negative").length;
+        const okCount = feedback.filter((f) => f.type === "correct").length;
+        console.log(`    Feedback: ${fpCount} false positives, ${fnCount} false negatives, ${okCount} correct`);
+        const patterns = extractPatterns(feedback);
+        if (patterns.length > 0) {
+            console.log(`    Learnable patterns: ${patterns.length}`);
+            for (const p of patterns.slice(0, 5)) {
+                console.log(`      • ${p.name} — ${p.confidence}% confidence`);
+            }
+        }
+    }
+    if (judges.length > 0) {
+        console.log(`\n    Generated Judges:`);
+        for (const j of judges) {
+            console.log(`      📋 ${j.name} — ${j.rules.length} rules (${j.updatedAt})`);
+        }
+    }
+    if (feedback.length === 0 && judges.length === 0) {
+        console.log("    No data yet. Use --feedback to record findings feedback.");
+    }
+    console.log("");
+}

package/dist/commands/judge-reputation.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Per-judge reputation tracking — FP rates, confidence calibration,
+ * historical accuracy metrics for each judge.
+ *
+ * Stored locally in .judges-reputation.json.
+ */
+interface FeedbackEntry {
+    findingId: string;
+    verdict: "true-positive" | "false-positive";
+    timestamp: string;
+}
+export interface JudgeReputation {
+    judgeId: string;
+    totalFindings: number;
+    truePositives: number;
+    falsePositives: number;
+    fpRate: number;
+    accuracy: number;
+    avgConfidence: number;
+    confidenceCalibration: number;
+    trend: "improving" | "stable" | "declining";
+    lastUpdated: string;
+    feedback: FeedbackEntry[];
+}
+export declare function recordFeedback(judgeId: string, findingId: string, verdict: "true-positive" | "false-positive", confidence?: number): JudgeReputation;
+export declare function getReputations(): JudgeReputation[];
+export declare function runJudgeReputation(argv: string[]): void;
+export {};

package/dist/commands/judge-reputation.js

@@ -0,0 +1,198 @@
+/**
+ * Per-judge reputation tracking — FP rates, confidence calibration,
+ * historical accuracy metrics for each judge.
+ *
+ * Stored locally in .judges-reputation.json.
+ */
+import { readFileSync, writeFileSync, existsSync } from "fs";
+const REPUTATION_FILE = ".judges-reputation.json";
+// ─── Core ───────────────────────────────────────────────────────────────────
+function loadDb() {
+    if (!existsSync(REPUTATION_FILE))
+        return { judges: [] };
+    return JSON.parse(readFileSync(REPUTATION_FILE, "utf-8"));
+}
+function saveDb(db) {
+    writeFileSync(REPUTATION_FILE, JSON.stringify(db, null, 2));
+}
+function computeTrend(feedback) {
+    if (feedback.length < 10)
+        return "stable";
+    const recent = feedback.slice(-10);
+    const older = feedback.slice(-20, -10);
+    if (older.length < 5)
+        return "stable";
+    const recentFpRate = recent.filter((f) => f.verdict === "false-positive").length / recent.length;
+    const olderFpRate = older.filter((f) => f.verdict === "false-positive").length / older.length;
+    if (recentFpRate < olderFpRate - 0.1)
+        return "improving";
+    if (recentFpRate > olderFpRate + 0.1)
+        return "declining";
+    return "stable";
+}
+export function recordFeedback(judgeId, findingId, verdict, confidence) {
+    const db = loadDb();
+    let judge = db.judges.find((j) => j.judgeId === judgeId);
+    if (!judge) {
+        judge = {
+            judgeId,
+            totalFindings: 0,
+            truePositives: 0,
+            falsePositives: 0,
+            fpRate: 0,
+            accuracy: 1,
+            avgConfidence: 0,
+            confidenceCalibration: 0,
+            trend: "stable",
+            lastUpdated: new Date().toISOString(),
+            feedback: [],
+        };
+        db.judges.push(judge);
+    }
+    judge.feedback.push({ findingId, verdict, timestamp: new Date().toISOString() });
+    judge.totalFindings = judge.feedback.length;
+    judge.truePositives = judge.feedback.filter((f) => f.verdict === "true-positive").length;
+    judge.falsePositives = judge.feedback.filter((f) => f.verdict === "false-positive").length;
+    judge.fpRate = Math.round((judge.falsePositives / judge.totalFindings) * 100) / 100;
+    judge.accuracy = Math.round((judge.truePositives / judge.totalFindings) * 100) / 100;
+    judge.trend = computeTrend(judge.feedback);
+    judge.lastUpdated = new Date().toISOString();
+    if (confidence !== undefined) {
+        // Running average of confidence
+        const prevTotal = judge.totalFindings - 1;
+        judge.avgConfidence =
+            Math.round(((judge.avgConfidence * prevTotal + confidence) / judge.totalFindings) * 100) / 100;
+        // Calibration: |accuracy - avgConfidence| — lower is better
+        judge.confidenceCalibration = Math.round(Math.abs(judge.accuracy - judge.avgConfidence) * 100) / 100;
+    }
+    saveDb(db);
+    return judge;
+}
+export function getReputations() {
+    return loadDb().judges;
+}
+// ─── CLI ────────────────────────────────────────────────────────────────────
+export function runJudgeReputation(argv) {
+    if (argv.includes("--help") || argv.includes("-h")) {
+        console.log(`
+judges judge-reputation — Per-judge accuracy and FP tracking
+
+Usage:
+  judges judge-reputation --record <judgeId> --finding <id> --verdict tp
+  judges judge-reputation --record <judgeId> --finding <id> --verdict fp --confidence 0.8
+  judges judge-reputation --list
+  judges judge-reputation --rank
+  judges judge-reputation --judge <id>
+  judges judge-reputation --flagged
+
+Options:
+  --record <judgeId>   Record feedback for a judge
+  --finding <id>       Finding ID
+  --verdict <v>        tp (true-positive) | fp (false-positive)
+  --confidence <n>     Finding confidence (0.0–1.0)
+  --list               List all judge reputations
+  --rank               Rank judges by accuracy
+  --judge <id>         Show details for a specific judge
+  --flagged            Show judges with FP rate > 30%
+  --format json        JSON output
+  --help, -h           Show this help
+`);
+        return;
+    }
+    const format = argv.find((_a, i) => argv[i - 1] === "--format") || "text";
+    // Record feedback
+    const recordJudge = argv.find((_a, i) => argv[i - 1] === "--record");
+    const findingId = argv.find((_a, i) => argv[i - 1] === "--finding");
+    const verdictStr = argv.find((_a, i) => argv[i - 1] === "--verdict");
+    const confStr = argv.find((_a, i) => argv[i - 1] === "--confidence");
+    if (recordJudge && findingId && verdictStr) {
+        const verdict = verdictStr === "fp" ? "false-positive" : "true-positive";
+        const confidence = confStr ? parseFloat(confStr) : undefined;
+        const rep = recordFeedback(recordJudge, findingId, verdict, confidence);
+        if (format === "json") {
+            console.log(JSON.stringify(rep, null, 2));
+        }
+        else {
+            console.log(`  ✅ Recorded ${verdict} for ${recordJudge}`);
+            console.log(`     Accuracy: ${(rep.accuracy * 100).toFixed(0)}%, FP rate: ${(rep.fpRate * 100).toFixed(0)}%, Trend: ${rep.trend}`);
+        }
+        return;
+    }
+    const db = loadDb();
+    // Show specific judge
+    const judgeId = argv.find((_a, i) => argv[i - 1] === "--judge");
+    if (judgeId) {
+        const judge = db.judges.find((j) => j.judgeId === judgeId);
+        if (!judge) {
+            console.error(`  ❌ No reputation data for judge "${judgeId}"`);
+            return;
+        }
+        if (format === "json") {
+            console.log(JSON.stringify(judge, null, 2));
+        }
+        else {
+            console.log(`\n  Judge: ${judge.judgeId}`);
+            console.log(`  ──────────────────────`);
+            console.log(`  Total findings:  ${judge.totalFindings}`);
+            console.log(`  True positives:  ${judge.truePositives}`);
+            console.log(`  False positives: ${judge.falsePositives}`);
+            console.log(`  Accuracy:        ${(judge.accuracy * 100).toFixed(1)}%`);
+            console.log(`  FP rate:         ${(judge.fpRate * 100).toFixed(1)}%`);
+            console.log(`  Avg confidence:  ${(judge.avgConfidence * 100).toFixed(1)}%`);
+            console.log(`  Calibration:     ${(judge.confidenceCalibration * 100).toFixed(1)}%`);
+            console.log(`  Trend:           ${judge.trend}`);
+            console.log(`  Last updated:    ${judge.lastUpdated}\n`);
+        }
+        return;
+    }
+    // Flagged judges (FP > 30%)
+    if (argv.includes("--flagged")) {
+        const flagged = db.judges.filter((j) => j.fpRate > 0.3 && j.totalFindings >= 5);
+        if (format === "json") {
+            console.log(JSON.stringify(flagged, null, 2));
+        }
+        else if (flagged.length === 0) {
+            console.log("\n  No flagged judges (all under 30% FP rate).\n");
+        }
+        else {
+            console.log(`\n  Flagged Judges — FP Rate > 30% (${flagged.length})\n  ──────────────────────────────────`);
+            for (const j of flagged) {
+                console.log(`    ⚠️  ${j.judgeId.padEnd(20)} FP: ${(j.fpRate * 100).toFixed(0)}% (${j.falsePositives}/${j.totalFindings}) ${j.trend}`);
+            }
+            console.log("");
+        }
+        return;
+    }
+    // Rank by accuracy
+    if (argv.includes("--rank")) {
+        const ranked = [...db.judges].sort((a, b) => b.accuracy - a.accuracy);
+        if (format === "json") {
+            console.log(JSON.stringify(ranked, null, 2));
+        }
+        else {
+            console.log(`\n  Judge Rankings (${ranked.length})\n  ────────────────`);
+            ranked.forEach((j, i) => {
+                const icon = j.trend === "improving" ? "📈" : j.trend === "declining" ? "📉" : "➡️";
+                console.log(`    ${String(i + 1).padStart(2)}. ${j.judgeId.padEnd(20)} accuracy: ${(j.accuracy * 100).toFixed(0)}% FP: ${(j.fpRate * 100).toFixed(0)}% ${icon}`);
+            });
+            console.log("");
+        }
+        return;
+    }
+    // List all
+    if (db.judges.length === 0) {
+        console.log("\n  No reputation data. Use --record to log feedback.\n");
+        return;
+    }
+    if (format === "json") {
+        console.log(JSON.stringify(db.judges, null, 2));
+    }
+    else {
+        console.log(`\n  Judge Reputations (${db.judges.length})\n  ───────────────────`);
+        for (const j of db.judges) {
+            const icon = j.trend === "improving" ? "📈" : j.trend === "declining" ? "📉" : "➡️";
+            console.log(`    ${j.judgeId.padEnd(20)} acc: ${(j.accuracy * 100).toFixed(0)}% FP: ${(j.fpRate * 100).toFixed(0)}% (${j.totalFindings} total) ${icon}`);
+        }
+        console.log("");
+    }
+}

package/dist/commands/kb.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Team knowledge base — store team decisions about rules, exceptions,
+ * known patterns, and contextual knowledge that informs evaluations.
+ *
+ * Stored locally in .judges-kb.json.
+ */
+export interface KbEntry {
+    id: string;
+    /** Rule ID or prefix this applies to */
+    rulePattern: string;
+    /** Decision type */
+    decision: "not-applicable" | "accepted-risk" | "deferred" | "exception" | "custom-guidance";
+    /** Why the team made this decision */
+    reason: string;
+    /** Who approved this decision */
+    approvedBy: string;
+    /** When this decision was made */
+    createdIso: string;
+    /** Optional expiry date for temporary exceptions */
+    expiresIso?: string;
+    /** Scope (e.g., file glob or project area) */
+    scope?: string;
+    /** Tags for categorization */
+    tags?: string[];
+    /** Active or archived */
+    active: boolean;
+}
+export declare function addKbEntry(opts: {
+    rulePattern: string;
+    decision: KbEntry["decision"];
+    reason: string;
+    approvedBy: string;
+    scope?: string;
+    tags?: string[];
+    expiresIn?: number;
+}): KbEntry;
+export declare function archiveKbEntry(id: string): boolean;
+export declare function searchKb(query: string): KbEntry[];
+export declare function getApplicableEntries(ruleId: string): KbEntry[];
+export declare function runKnowledgeBase(argv: string[]): void;