@kevinrabun/judges 3.126.2 → 3.127.1

package/dist/api.d.ts

@@ -75,6 +75,7 @@ export { parseLlmRuleIds, constructPerJudgePrompt, constructTribunalPrompt, sele
 export type { LlmBenchmarkSnapshot, LlmCaseResult } from "./commands/llm-benchmark.js";
 export type { LlmFinding, ValidationResult } from "./probabilistic/llm-response-validator.js";
 export type { PromptAmendment, OptimizerInsight, OptimizationResult, AmendmentStore, } from "./commands/llm-benchmark-optimizer.js";
+export { optimizeBenchmark, mergeAmendments, createEmptyStore, formatAmendmentSection, } from "./commands/llm-benchmark-optimizer.js";
 export { runReviewAutopilot, dedupeComments, filterAlreadyPostedComments } from "./commands/review.js";
 export { buildContextSnippets } from "./context/context-snippets.js";
 export { EmbeddingCache, FallbackEmbeddingProvider, getOrCreateEmbedding } from "./context/embedding-cache.js";

package/dist/api.js

@@ -83,6 +83,7 @@ export { compareCapabilities, formatComparisonReport, formatFullComparisonMatrix
 export { runBenchmarkSuite, benchmarkGate, formatBenchmarkReport, formatBenchmarkMarkdown, analyzeL2Coverage, formatL2CoverageReport, ingestFindingsAsBenchmarkCases, deduplicateIngestCases, BENCHMARK_CASES, } from "./commands/benchmark.js";
 // ─── LLM Benchmark ──────────────────────────────────────────────────────────
 export { parseLlmRuleIds, constructPerJudgePrompt, constructTribunalPrompt, selectStratifiedSample, selectJudgesForCategory, scoreLlmCase, computeLlmMetrics, formatLlmSnapshotMarkdown, formatLayerComparisonMarkdown, extractValidatedLlmFindings, getValidRulePrefixes, getTribunalValidPrefixes, } from "./commands/llm-benchmark.js";
+export { optimizeBenchmark, mergeAmendments, createEmptyStore, formatAmendmentSection, } from "./commands/llm-benchmark-optimizer.js";
 // Review autopilot (GitHub App / scripts)
 export { runReviewAutopilot, dedupeComments, filterAlreadyPostedComments } from "./commands/review.js";
 export { buildContextSnippets } from "./context/context-snippets.js";

package/dist/cli-dispatch.js

@@ -7,6 +7,7 @@
 export const COMMAND_TABLE = {
     "adoption-report": ["./commands/adoption-report.js", "runAdoptionReport"],
     "adoption-track": ["./commands/adoption-track.js", "runAdoptionTrack"],
+    "codify-amendments": ["./commands/codify-amendments.js", "runCodifyAmendments"],
     "ai-gate": ["./commands/ai-gate.js", "runAiGate"],
     "ai-model-trust": ["./commands/ai-model-trust.js", "runAiModelTrust"],
     "ai-output-compare": ["./commands/ai-output-compare.js", "runAiOutputCompare"],

package/dist/commands/codify-amendments.d.ts

@@ -0,0 +1,20 @@
+/**
+ * `judges codify-amendments` — Bake benchmark-learned amendments into judge source files.
+ *
+ * Reads amendments from either:
+ *   - The VS Code global storage (llm-benchmark-amendments.json)
+ *   - A specified JSON file (--file path)
+ *
+ * For each amendment, appends a BENCHMARK-LEARNED section to the judge's
+ * FALSE POSITIVE AVOIDANCE block in the agent .judge.md file. Then runs
+ * generate:agents to sync .ts files.
+ *
+ * This makes self-teaching improvements part of the distributed package
+ * rather than local-only amendment files.
+ *
+ * Usage:
+ *   judges codify-amendments                    # from VS Code global storage
+ *   judges codify-amendments --file amend.json  # from file
+ *   judges codify-amendments --dry-run          # preview without writing
+ */
+export declare function runCodifyAmendments(argv: string[]): void;

package/dist/commands/codify-amendments.js

@@ -0,0 +1,162 @@
+/**
+ * `judges codify-amendments` — Bake benchmark-learned amendments into judge source files.
+ *
+ * Reads amendments from either:
+ *   - The VS Code global storage (llm-benchmark-amendments.json)
+ *   - A specified JSON file (--file path)
+ *
+ * For each amendment, appends a BENCHMARK-LEARNED section to the judge's
+ * FALSE POSITIVE AVOIDANCE block in the agent .judge.md file. Then runs
+ * generate:agents to sync .ts files.
+ *
+ * This makes self-teaching improvements part of the distributed package
+ * rather than local-only amendment files.
+ *
+ * Usage:
+ *   judges codify-amendments                    # from VS Code global storage
+ *   judges codify-amendments --file amend.json  # from file
+ *   judges codify-amendments --dry-run          # preview without writing
+ */
+import { readFileSync, writeFileSync, existsSync } from "fs";
+import { resolve, join } from "path";
+import { fileURLToPath } from "url";
+import { dirname } from "path";
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+/**
+ * Find the agents directory relative to this source file.
+ */
+function findAgentsDir() {
+    // In dist/commands/ → go up to repo root
+    const candidates = [
+        resolve(__dirname, "..", "..", "agents"),
+        resolve(__dirname, "..", "agents"),
+        resolve(process.cwd(), "agents"),
+    ];
+    for (const dir of candidates) {
+        if (existsSync(dir))
+            return dir;
+    }
+    throw new Error("Cannot find agents/ directory. Run from the repo root.");
+}
+/**
+ * Load amendments from a file or the default VS Code global storage location.
+ */
+function loadAmendments(filePath) {
+    if (filePath) {
+        const store = JSON.parse(readFileSync(resolve(filePath), "utf8"));
+        return store.amendments;
+    }
+    // Try VS Code global storage
+    const appdata = process.env.APPDATA || process.env.HOME;
+    if (!appdata)
+        throw new Error("Cannot determine global storage path. Use --file to specify.");
+    const globalPath = join(appdata, "Code", "User", "globalStorage", "kevinrabun.judges-panel", "llm-benchmark-amendments.json");
+    if (!existsSync(globalPath)) {
+        throw new Error(`No amendments found at ${globalPath}. Run an LLM benchmark first, or use --file.`);
+    }
+    const store = JSON.parse(readFileSync(globalPath, "utf8"));
+    return store.amendments;
+}
+/**
+ * Codify a single amendment into a judge's .judge.md file by appending
+ * to the FALSE POSITIVE AVOIDANCE section (or creating one if missing).
+ */
+function codifyAmendment(agentsDir, amendment, dryRun) {
+    // Map prefix to judge file
+    const files = existsSync(agentsDir) ? readdirSync(agentsDir) : [];
+    let targetFile;
+    for (const file of files) {
+        if (!file.endsWith(".judge.md"))
+            continue;
+        const content = readFileSync(join(agentsDir, file), "utf8");
+        // Match judge by rule prefix in frontmatter
+        if (content.includes(`rulePrefix: ${amendment.judgePrefix}`) ||
+            content.includes(`rulePrefix: "${amendment.judgePrefix}"`)) {
+            targetFile = file;
+            break;
+        }
+    }
+    if (!targetFile) {
+        console.error(`  ⚠ No agent file found for prefix ${amendment.judgePrefix} — skipping`);
+        return false;
+    }
+    const filePath = join(agentsDir, targetFile);
+    let content = readFileSync(filePath, "utf8");
+    // Build the codified section
+    const codifiedBlock = [
+        "",
+        `BENCHMARK-LEARNED PRECISION GUIDANCE (${amendment.judgePrefix}):`,
+        `- ${amendment.amendment}`,
+        `- Source: ${amendment.generatedFrom} | FP rate: ${(amendment.fpRate * 100).toFixed(0)}% | Generated: ${amendment.timestamp.slice(0, 10)}`,
+    ].join("\n");
+    // Check if already codified for this prefix
+    if (content.includes(`BENCHMARK-LEARNED PRECISION GUIDANCE (${amendment.judgePrefix})`)) {
+        console.log(`  ♻ ${targetFile} — already has codified amendment for ${amendment.judgePrefix}, replacing`);
+        // Replace existing block
+        content = content.replace(new RegExp(`\\nBENCHMARK-LEARNED PRECISION GUIDANCE \\(${amendment.judgePrefix}\\):[\\s\\S]*?(?=\\n[A-Z]|$)`), codifiedBlock);
+    }
+    else {
+        // Insert before ADVERSARIAL MANDATE if exists, otherwise append
+        if (content.includes("ADVERSARIAL MANDATE:")) {
+            content = content.replace("ADVERSARIAL MANDATE:", codifiedBlock + "\n\nADVERSARIAL MANDATE:");
+        }
+        else {
+            content = content.trimEnd() + "\n" + codifiedBlock + "\n";
+        }
+    }
+    if (dryRun) {
+        console.log(`  📝 [DRY RUN] Would update ${targetFile}`);
+        console.log(`     ${codifiedBlock.split("\n").slice(1, 3).join("\n     ")}`);
+    }
+    else {
+        writeFileSync(filePath, content);
+        console.log(`  ✅ Updated ${targetFile}`);
+    }
+    return true;
+}
+import { readdirSync } from "fs";
+export function runCodifyAmendments(argv) {
+    const dryRun = argv.includes("--dry-run");
+    const fileIdx = argv.indexOf("--file");
+    const filePath = fileIdx >= 0 ? argv[fileIdx + 1] : undefined;
+    console.log("");
+    console.log("╔══════════════════════════════════════════════════════════════╗");
+    console.log("║        Judges — Codify Benchmark Amendments                 ║");
+    console.log("╚══════════════════════════════════════════════════════════════╝");
+    console.log("");
+    let amendments;
+    try {
+        amendments = loadAmendments(filePath);
+    }
+    catch (e) {
+        console.error(`Error: ${e instanceof Error ? e.message : String(e)}`);
+        process.exit(1);
+    }
+    if (amendments.length === 0) {
+        console.log("  No amendments to codify. Run an LLM benchmark to generate them.");
+        process.exit(0);
+    }
+    console.log(`  Found ${amendments.length} amendment(s) to codify${dryRun ? " (dry run)" : ""}:`);
+    for (const a of amendments) {
+        console.log(`    ${a.judgePrefix}: ${a.reason}`);
+    }
+    console.log("");
+    const agentsDir = findAgentsDir();
+    let codified = 0;
+    for (const amendment of amendments) {
+        if (codifyAmendment(agentsDir, amendment, dryRun)) {
+            codified++;
+        }
+    }
+    console.log("");
+    console.log(`  ${dryRun ? "Would codify" : "Codified"} ${codified}/${amendments.length} amendment(s) into agent files.`);
+    if (!dryRun && codified > 0) {
+        console.log("  Next steps:");
+        console.log("    1. npm run generate:agents:force  — sync .ts files from .judge.md");
+        console.log("    2. npm run build                  — rebuild");
+        console.log("    3. npm test                       — verify");
+        console.log("    4. Commit and release");
+    }
+    console.log("");
+}

package/dist/commands/llm-benchmark-optimizer.js

@@ -11,9 +11,9 @@
 import { JUDGES } from "../judges/index.js";
 // ─── Thresholds ─────────────────────────────────────────────────────────────
 /** Judges below this precision get amendments */
-const AMENDMENT_PRECISION_THRESHOLD = 0.4;
+const AMENDMENT_PRECISION_THRESHOLD = 0.7;
 /** Minimum findings before generating amendment (avoid noise) */
-const MIN_FINDINGS_FOR_AMENDMENT = 5;
+const MIN_FINDINGS_FOR_AMENDMENT = 3;
 /** Categories below this F1 get flagged */
 const CATEGORY_F1_THRESHOLD = 0.5;
 /** Difficulty detection rate below this gets flagged */
@@ -109,6 +109,7 @@ function generateAmendment(prefix, precision, fpCount, total, snapshot) {
     const domain = judge?.domain ?? "its domain";
     // Analyze what the FPs look like — which categories get falsely flagged
     const fpCategories = new Map();
+    const tpCategories = new Map();
     // Collect specific FP case IDs for pattern extraction
     const fpCaseExamples = [];
     for (const c of snapshot.cases) {
@@ -120,35 +121,36 @@ function generateAmendment(prefix, precision, fpCount, total, snapshot) {
                 }
             }
         }
+        // Also track where this judge produces TRUE positives
+        for (const det of c.detectedRuleIds) {
+            if (det.startsWith(prefix + "-") && !c.falsePositiveRuleIds.includes(det)) {
+                tpCategories.set(c.category, (tpCategories.get(c.category) ?? 0) + 1);
+            }
+        }
     }
-    const topFpCategories = [...fpCategories.entries()]
+    // Identify categories that are FP-only (no TPs) — safe to suppress
+    const fpOnlyCategories = [...fpCategories.entries()]
+        .filter(([cat]) => !tpCategories.has(cat))
         .sort((a, b) => b[1] - a[1])
         .slice(0, 5)
         .map(([cat]) => cat);
-    // Build specific anti-FP instructions based on observed patterns
-    const categoryBlocklist = topFpCategories.length > 0
-        ? `\nDo NOT report ${prefix}- findings on code in these categories: ${topFpCategories.join(", ")}. ` +
-            `These categories fall outside ${domain} and historically produce false positives.`
-        : "";
-    // Extract specific FP patterns for concrete guidance
-    const fpRuleIds = new Set(fpCaseExamples.map((e) => e.ruleId));
-    const specificRules = [...fpRuleIds].slice(0, 5).join(", ");
-    const ruleWarning = specificRules
-        ? `\nSpecific rule IDs with high FP rates: ${specificRules}. Require >=80% confidence with exact line citations before reporting these.`
-        : "";
-    // Identify if clean cases are a problem for this judge
+    // Build targeted anti-FP instructions — only suppress on clean/FP-only categories
     const cleanFPs = fpCaseExamples.filter((e) => e.category === "clean" || e.category.startsWith("ai-negative")).length;
+    const nonCleanFPOnlyWarning = fpOnlyCategories.length > 0
+        ? `\nHistorically produces false positives on: ${fpOnlyCategories.join(", ")}. Apply extra scrutiny on these categories — require concrete evidence before reporting.`
+        : "";
     const cleanWarning = cleanFPs > 0
-        ? `\nThis judge produced ${cleanFPs} false positives on CLEAN code. Well-written code using standard patterns exists. If the code follows established best practices, report ZERO ${prefix}- findings.`
+        ? `\nThis judge produced ${cleanFPs} false positive(s) on CLEAN code. If code uses standard patterns correctly (proper error handling, established libraries, framework conventions), report ZERO ${prefix}- findings. Clean, well-written code exists — do not manufacture findings.`
         : "";
-    const amendment = `PRECISION OVERRIDE for ${judgeName} (${prefix}-): ` +
-        `Empirical precision: ${pct(precision)} (${fpCount} FP in ${total} findings). ` +
-        `SCOPE: Only report ${prefix}- findings for code that specifically involves ${domain}. ` +
-        `EVIDENCE: Every ${prefix}- finding MUST cite exact line numbers and specific code patterns.` +
-        categoryBlocklist +
-        ruleWarning +
+    // IMPORTANT: Do NOT restrict the judge from detecting real issues in vulnerable code.
+    // Only add caution for clean-code patterns, not a blanket confidence floor.
+    const amendment = `PRECISION CALIBRATION for ${judgeName} (${prefix}-): ` +
+        `Empirical precision: ${pct(precision)} in recent benchmarks. ` +
+        `IMPORTANT: Continue detecting genuine ${domain} issues in vulnerable code — do NOT reduce sensitivity to real problems. ` +
+        `CALIBRATION: The false positives come from flagging well-written code that correctly uses established patterns. ` +
+        `Before reporting ${prefix}- findings, verify the code actually has a deficiency — not just a theoretical improvement opportunity.` +
         cleanWarning +
-        ` When confidence is below 80%, OMIT the ${prefix}- finding.`;
+        nonCleanFPOnlyWarning;
     return {
         judgePrefix: prefix,
         amendment,
@@ -167,11 +169,12 @@ export function formatAmendmentSection(amendments) {
     if (amendments.length === 0)
         return "";
     const lines = [
-        "## Precision Overrides — Based on Empirical Benchmark Data",
+        "## Precision Calibration — Based on Empirical Benchmark Data",
         "",
-        "The following judges have been identified as having high false positive rates. " +
-            "Apply EXTRA scrutiny before reporting findings with these prefixes. " +
-            "False positives erode developer trust more than missed findings.",
+        "The following judges have historically produced false positives on clean code. " +
+            "Apply the calibration guidance below to avoid repeating these errors. " +
+            "IMPORTANT: These calibrations target CLEAN CODE false positives only — " +
+            "continue detecting genuine issues in vulnerable code with full sensitivity.",
         "",
     ];
     for (const a of amendments) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kevinrabun/judges",
-  "version": "3.126.2",
+  "version": "3.127.1",
   "description": "45 specialized judges that evaluate AI-generated code for security, cost, and quality.",
   "mcpName": "io.github.KevinRabun/judges",
   "type": "module",

package/server.json

@@ -16,12 +16,12 @@
       "mimeType": "image/png"
     }
   ],
-  "version": "3.126.2",
+  "version": "3.127.1",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "@kevinrabun/judges",
-      "version": "3.126.2",
+      "version": "3.127.1",
       "transport": {
         "type": "stdio"
       }