@kevinrabun/judges 3.126.1 → 3.127.0

package/dist/api.d.ts

@@ -75,6 +75,7 @@ export { parseLlmRuleIds, constructPerJudgePrompt, constructTribunalPrompt, sele
 export type { LlmBenchmarkSnapshot, LlmCaseResult } from "./commands/llm-benchmark.js";
 export type { LlmFinding, ValidationResult } from "./probabilistic/llm-response-validator.js";
 export type { PromptAmendment, OptimizerInsight, OptimizationResult, AmendmentStore, } from "./commands/llm-benchmark-optimizer.js";
+export { optimizeBenchmark, mergeAmendments, createEmptyStore, formatAmendmentSection, } from "./commands/llm-benchmark-optimizer.js";
 export { runReviewAutopilot, dedupeComments, filterAlreadyPostedComments } from "./commands/review.js";
 export { buildContextSnippets } from "./context/context-snippets.js";
 export { EmbeddingCache, FallbackEmbeddingProvider, getOrCreateEmbedding } from "./context/embedding-cache.js";

package/dist/api.js

@@ -83,6 +83,7 @@ export { compareCapabilities, formatComparisonReport, formatFullComparisonMatrix
 export { runBenchmarkSuite, benchmarkGate, formatBenchmarkReport, formatBenchmarkMarkdown, analyzeL2Coverage, formatL2CoverageReport, ingestFindingsAsBenchmarkCases, deduplicateIngestCases, BENCHMARK_CASES, } from "./commands/benchmark.js";
 // ─── LLM Benchmark ──────────────────────────────────────────────────────────
 export { parseLlmRuleIds, constructPerJudgePrompt, constructTribunalPrompt, selectStratifiedSample, selectJudgesForCategory, scoreLlmCase, computeLlmMetrics, formatLlmSnapshotMarkdown, formatLayerComparisonMarkdown, extractValidatedLlmFindings, getValidRulePrefixes, getTribunalValidPrefixes, } from "./commands/llm-benchmark.js";
+export { optimizeBenchmark, mergeAmendments, createEmptyStore, formatAmendmentSection, } from "./commands/llm-benchmark-optimizer.js";
 // Review autopilot (GitHub App / scripts)
 export { runReviewAutopilot, dedupeComments, filterAlreadyPostedComments } from "./commands/review.js";
 export { buildContextSnippets } from "./context/context-snippets.js";

package/dist/cli-dispatch.js

@@ -7,6 +7,7 @@
 export const COMMAND_TABLE = {
     "adoption-report": ["./commands/adoption-report.js", "runAdoptionReport"],
     "adoption-track": ["./commands/adoption-track.js", "runAdoptionTrack"],
+    "codify-amendments": ["./commands/codify-amendments.js", "runCodifyAmendments"],
     "ai-gate": ["./commands/ai-gate.js", "runAiGate"],
     "ai-model-trust": ["./commands/ai-model-trust.js", "runAiModelTrust"],
     "ai-output-compare": ["./commands/ai-output-compare.js", "runAiOutputCompare"],

package/dist/commands/codify-amendments.d.ts

@@ -0,0 +1,20 @@
+/**
+ * `judges codify-amendments` — Bake benchmark-learned amendments into judge source files.
+ *
+ * Reads amendments from either:
+ *   - The VS Code global storage (llm-benchmark-amendments.json)
+ *   - A specified JSON file (--file path)
+ *
+ * For each amendment, appends a BENCHMARK-LEARNED section to the judge's
+ * FALSE POSITIVE AVOIDANCE block in the agent .judge.md file. Then runs
+ * generate:agents to sync .ts files.
+ *
+ * This makes self-teaching improvements part of the distributed package
+ * rather than local-only amendment files.
+ *
+ * Usage:
+ *   judges codify-amendments                    # from VS Code global storage
+ *   judges codify-amendments --file amend.json  # from file
+ *   judges codify-amendments --dry-run          # preview without writing
+ */
+export declare function runCodifyAmendments(argv: string[]): void;

package/dist/commands/codify-amendments.js

@@ -0,0 +1,162 @@
+/**
+ * `judges codify-amendments` — Bake benchmark-learned amendments into judge source files.
+ *
+ * Reads amendments from either:
+ *   - The VS Code global storage (llm-benchmark-amendments.json)
+ *   - A specified JSON file (--file path)
+ *
+ * For each amendment, appends a BENCHMARK-LEARNED section to the judge's
+ * FALSE POSITIVE AVOIDANCE block in the agent .judge.md file. Then runs
+ * generate:agents to sync .ts files.
+ *
+ * This makes self-teaching improvements part of the distributed package
+ * rather than local-only amendment files.
+ *
+ * Usage:
+ *   judges codify-amendments                    # from VS Code global storage
+ *   judges codify-amendments --file amend.json  # from file
+ *   judges codify-amendments --dry-run          # preview without writing
+ */
+import { readFileSync, writeFileSync, existsSync } from "fs";
+import { resolve, join } from "path";
+import { fileURLToPath } from "url";
+import { dirname } from "path";
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+/**
+ * Find the agents directory relative to this source file.
+ */
+function findAgentsDir() {
+    // In dist/commands/ → go up to repo root
+    const candidates = [
+        resolve(__dirname, "..", "..", "agents"),
+        resolve(__dirname, "..", "agents"),
+        resolve(process.cwd(), "agents"),
+    ];
+    for (const dir of candidates) {
+        if (existsSync(dir))
+            return dir;
+    }
+    throw new Error("Cannot find agents/ directory. Run from the repo root.");
+}
+/**
+ * Load amendments from a file or the default VS Code global storage location.
+ */
+function loadAmendments(filePath) {
+    if (filePath) {
+        const store = JSON.parse(readFileSync(resolve(filePath), "utf8"));
+        return store.amendments;
+    }
+    // Try VS Code global storage
+    const appdata = process.env.APPDATA || process.env.HOME;
+    if (!appdata)
+        throw new Error("Cannot determine global storage path. Use --file to specify.");
+    const globalPath = join(appdata, "Code", "User", "globalStorage", "kevinrabun.judges-panel", "llm-benchmark-amendments.json");
+    if (!existsSync(globalPath)) {
+        throw new Error(`No amendments found at ${globalPath}. Run an LLM benchmark first, or use --file.`);
+    }
+    const store = JSON.parse(readFileSync(globalPath, "utf8"));
+    return store.amendments;
+}
+/**
+ * Codify a single amendment into a judge's .judge.md file by appending
+ * to the FALSE POSITIVE AVOIDANCE section (or creating one if missing).
+ */
+function codifyAmendment(agentsDir, amendment, dryRun) {
+    // Map prefix to judge file
+    const files = existsSync(agentsDir) ? readdirSync(agentsDir) : [];
+    let targetFile;
+    for (const file of files) {
+        if (!file.endsWith(".judge.md"))
+            continue;
+        const content = readFileSync(join(agentsDir, file), "utf8");
+        // Match judge by rule prefix in frontmatter
+        if (content.includes(`rulePrefix: ${amendment.judgePrefix}`) ||
+            content.includes(`rulePrefix: "${amendment.judgePrefix}"`)) {
+            targetFile = file;
+            break;
+        }
+    }
+    if (!targetFile) {
+        console.error(`  ⚠ No agent file found for prefix ${amendment.judgePrefix} — skipping`);
+        return false;
+    }
+    const filePath = join(agentsDir, targetFile);
+    let content = readFileSync(filePath, "utf8");
+    // Build the codified section
+    const codifiedBlock = [
+        "",
+        `BENCHMARK-LEARNED PRECISION GUIDANCE (${amendment.judgePrefix}):`,
+        `- ${amendment.amendment}`,
+        `- Source: ${amendment.generatedFrom} | FP rate: ${(amendment.fpRate * 100).toFixed(0)}% | Generated: ${amendment.timestamp.slice(0, 10)}`,
+    ].join("\n");
+    // Check if already codified for this prefix
+    if (content.includes(`BENCHMARK-LEARNED PRECISION GUIDANCE (${amendment.judgePrefix})`)) {
+        console.log(`  ♻ ${targetFile} — already has codified amendment for ${amendment.judgePrefix}, replacing`);
+        // Replace existing block
+        content = content.replace(new RegExp(`\\nBENCHMARK-LEARNED PRECISION GUIDANCE \\(${amendment.judgePrefix}\\):[\\s\\S]*?(?=\\n[A-Z]|$)`), codifiedBlock);
+    }
+    else {
+        // Insert before ADVERSARIAL MANDATE if exists, otherwise append
+        if (content.includes("ADVERSARIAL MANDATE:")) {
+            content = content.replace("ADVERSARIAL MANDATE:", codifiedBlock + "\n\nADVERSARIAL MANDATE:");
+        }
+        else {
+            content = content.trimEnd() + "\n" + codifiedBlock + "\n";
+        }
+    }
+    if (dryRun) {
+        console.log(`  📝 [DRY RUN] Would update ${targetFile}`);
+        console.log(`     ${codifiedBlock.split("\n").slice(1, 3).join("\n     ")}`);
+    }
+    else {
+        writeFileSync(filePath, content);
+        console.log(`  ✅ Updated ${targetFile}`);
+    }
+    return true;
+}
+import { readdirSync } from "fs";
+export function runCodifyAmendments(argv) {
+    const dryRun = argv.includes("--dry-run");
+    const fileIdx = argv.indexOf("--file");
+    const filePath = fileIdx >= 0 ? argv[fileIdx + 1] : undefined;
+    console.log("");
+    console.log("╔══════════════════════════════════════════════════════════════╗");
+    console.log("║        Judges — Codify Benchmark Amendments                 ║");
+    console.log("╚══════════════════════════════════════════════════════════════╝");
+    console.log("");
+    let amendments;
+    try {
+        amendments = loadAmendments(filePath);
+    }
+    catch (e) {
+        console.error(`Error: ${e instanceof Error ? e.message : String(e)}`);
+        process.exit(1);
+    }
+    if (amendments.length === 0) {
+        console.log("  No amendments to codify. Run an LLM benchmark to generate them.");
+        process.exit(0);
+    }
+    console.log(`  Found ${amendments.length} amendment(s) to codify${dryRun ? " (dry run)" : ""}:`);
+    for (const a of amendments) {
+        console.log(`    ${a.judgePrefix}: ${a.reason}`);
+    }
+    console.log("");
+    const agentsDir = findAgentsDir();
+    let codified = 0;
+    for (const amendment of amendments) {
+        if (codifyAmendment(agentsDir, amendment, dryRun)) {
+            codified++;
+        }
+    }
+    console.log("");
+    console.log(`  ${dryRun ? "Would codify" : "Codified"} ${codified}/${amendments.length} amendment(s) into agent files.`);
+    if (!dryRun && codified > 0) {
+        console.log("  Next steps:");
+        console.log("    1. npm run generate:agents:force  — sync .ts files from .judge.md");
+        console.log("    2. npm run build                  — rebuild");
+        console.log("    3. npm test                       — verify");
+        console.log("    4. Commit and release");
+    }
+    console.log("");
+}

package/dist/commands/llm-benchmark-optimizer.js

@@ -11,9 +11,9 @@
 import { JUDGES } from "../judges/index.js";
 // ─── Thresholds ─────────────────────────────────────────────────────────────
 /** Judges below this precision get amendments */
-const AMENDMENT_PRECISION_THRESHOLD = 0.4;
+const AMENDMENT_PRECISION_THRESHOLD = 0.7;
 /** Minimum findings before generating amendment (avoid noise) */
-const MIN_FINDINGS_FOR_AMENDMENT = 5;
+const MIN_FINDINGS_FOR_AMENDMENT = 3;
 /** Categories below this F1 get flagged */
 const CATEGORY_F1_THRESHOLD = 0.5;
 /** Difficulty detection rate below this gets flagged */

package/dist/commands/llm-benchmark.js

@@ -148,13 +148,15 @@ export function parseLlmRuleIds(response) {
     const validPrefixes = getValidRulePrefixes();
     const pattern = /\b([A-Z][A-Z0-9]+)-(\d{1,3})\b/g;
     const found = new Set();
-    // Split response into paragraphs/sections and skip sections that explicitly
-    // declare zero findings — rule IDs mentioned in "zero findings" rationale
-    // are explanatory references, not actual detections.
-    const sections = response.split(/\n{2,}/);
-    const zeroFindingsPattern = /\*?\*?(?:ZERO|zero|0|no)\s+findings?\*?\*?|(?:findings?|issues?)[\s:]*\*?\*?(?:none|0|zero)\*?\*?|no\s+(?:issues?|findings?|problems?|concerns?)\s+(?:found|detected|identified|reported)/i;
+    // Split response into judge sections (separated by --- dividers or ## headers)
+    // and skip entire sections that declare zero/no findings. This prevents rule
+    // IDs mentioned in rationale text or findings tables of "clean" judge sections
+    // from being counted as detections.
+    const sections = response.split(/(?:^|\n)---\s*\n|(?=^## )/m);
+    const zeroFindingsPattern = /\*?\*?(?:ZERO|zero|0|no)\s+findings?\*?\*?|(?:findings?|issues?)[\s:]*\*?\*?(?:none|0|zero)\*?\*?|no\s+(?:issues?|findings?|problems?|concerns?)\s+(?:found|detected|identified|reported)|report(?:ing)?\s+zero|Score\s*[|:]\s*\*?\*?100\s*\/?\s*100\*?\*?/i;
     for (const section of sections) {
-        // If this section explicitly declares zero/no findings, skip rule ID extraction
+        // If this section explicitly declares zero/no findings or a perfect score,
+        // skip rule ID extraction — any rule IDs are explanatory references
         if (zeroFindingsPattern.test(section))
             continue;
         let match;
@@ -165,13 +167,17 @@ export function parseLlmRuleIds(response) {
             }
         }
     }
-    // Secondary pass on full text: extract known prefixes from compound IDs like DEPS-TYPO-001
-    // These are almost always in findings tables, not rationale
-    const compoundPattern = /\b([A-Z][A-Z0-9]+)-[A-Z][A-Z0-9]+-(\d{1,3})\b/g;
-    let match;
-    while ((match = compoundPattern.exec(response)) !== null) {
-        if (validPrefixes.has(match[1])) {
-            found.add(`${match[1]}-${match[2]}`);
+    // Secondary pass: extract compound IDs like DEPS-TYPO-001 from sections
+    // that did NOT declare zero findings (reuse the filtered sections)
+    for (const section of sections) {
+        if (zeroFindingsPattern.test(section))
+            continue;
+        const compoundPattern = /\b([A-Z][A-Z0-9]+)-[A-Z][A-Z0-9]+-(\d{1,3})\b/g;
+        let match;
+        while ((match = compoundPattern.exec(section)) !== null) {
+            if (validPrefixes.has(match[1])) {
+                found.add(`${match[1]}-${match[2]}`);
+            }
         }
     }
     return [...found];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kevinrabun/judges",
-  "version": "3.126.1",
+  "version": "3.127.0",
   "description": "45 specialized judges that evaluate AI-generated code for security, cost, and quality.",
   "mcpName": "io.github.KevinRabun/judges",
   "type": "module",

package/server.json

@@ -16,12 +16,12 @@
       "mimeType": "image/png"
     }
   ],
-  "version": "3.126.1",
+  "version": "3.127.0",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "@kevinrabun/judges",
-      "version": "3.126.1",
+      "version": "3.127.0",
       "transport": {
         "type": "stdio"
       }