@kevinrabun/judges 3.115.3 → 3.116.0

package/dist/index.js

@@ -6,6 +6,7 @@ import("@modelcontextprotocol/sdk/server/mcp.js")
     const { StdioServerTransport } = await import("@modelcontextprotocol/sdk/server/stdio.js");
     const { registerTools } = await import("./tools/register.js");
     const { registerPrompts } = await import("./tools/prompts.js");
+    const { registerResources } = await import("./tools/register-resources.js");
     const { readFileSync } = await import("fs");
     const { resolve, dirname } = await import("path");
     const { fileURLToPath } = await import("url");
@@ -25,6 +26,7 @@ import("@modelcontextprotocol/sdk/server/mcp.js")
     });
     registerTools(server);
     registerPrompts(server);
+    registerResources(server);
     const transport = new StdioServerTransport();
     await server.connect(transport);
     console.error("Judges Panel MCP server running on stdio");

package/dist/judges/index.d.ts

@@ -1,17 +1,62 @@
 /**
- * Judge registry bootstrap (agent-native).
+ * Judge registry bootstrap.
  *
- * Judges are now sourced from `.judge.md` files in the `agents/` folder (legacy
- * `.agent.md` still supported). Each agent frontmatter references an evaluator
- * script (in `src/evaluators/`), and the agent loader registers them with the
- * unified `JudgeRegistry`.
+ * Judges are dual-registered:
+ *  1. Static side-effect imports below — each module calls
+ *     `defaultRegistry.register()` at load time.  These are inlined by
+ *     esbuild and work in both ESM and CJS bundles.
+ *  2. Agent-native `.judge.md` files loaded at runtime from the `agents/`
+ *     directory (when available).  This enriches / overrides metadata.
  *
- * Legacy side-effect imports have been removed. If you need to add a judge, add
- * an agent file and (optionally) an evaluator script, then run:
- *   - `npm run generate:agents` (to sync)
- *   - `npm run validate:agents`
+ * The static imports guarantee that judges are always available, even in
+ * bundled environments (VS Code extension) where `agents/` is absent.
  */
 import type { JudgeDefinition } from "../types.js";
+import "./accessibility.js";
+import "./agent-instructions.js";
+import "./ai-code-safety.js";
+import "./api-contract.js";
+import "./api-design.js";
+import "./authentication.js";
+import "./backwards-compatibility.js";
+import "./caching.js";
+import "./ci-cd.js";
+import "./cloud-readiness.js";
+import "./code-structure.js";
+import "./compliance.js";
+import "./concurrency.js";
+import "./configuration-management.js";
+import "./cost-effectiveness.js";
+import "./cybersecurity.js";
+import "./data-security.js";
+import "./data-sovereignty.js";
+import "./database.js";
+import "./dependency-health.js";
+import "./documentation.js";
+import "./error-handling.js";
+import "./ethics-bias.js";
+import "./false-positive-review.js";
+import "./framework-safety.js";
+import "./hallucination-detection.js";
+import "./iac-security.js";
+import "./intent-alignment.js";
+import "./internationalization.js";
+import "./logging-privacy.js";
+import "./logic-review.js";
+import "./maintainability.js";
+import "./model-fingerprint.js";
+import "./multi-turn-coherence.js";
+import "./observability.js";
+import "./over-engineering.js";
+import "./performance.js";
+import "./portability.js";
+import "./rate-limiting.js";
+import "./reliability.js";
+import "./scalability.js";
+import "./security.js";
+import "./software-practices.js";
+import "./testing.js";
+import "./ux.js";
 /**
  * Load judges (agent-native). Loads agents from the default `agents/` folder
  * and returns the current registry snapshot.

package/dist/judges/index.js

@@ -1,30 +1,85 @@
 /**
- * Judge registry bootstrap (agent-native).
+ * Judge registry bootstrap.
  *
- * Judges are now sourced from `.judge.md` files in the `agents/` folder (legacy
- * `.agent.md` still supported). Each agent frontmatter references an evaluator
- * script (in `src/evaluators/`), and the agent loader registers them with the
- * unified `JudgeRegistry`.
+ * Judges are dual-registered:
+ *  1. Static side-effect imports below — each module calls
+ *     `defaultRegistry.register()` at load time.  These are inlined by
+ *     esbuild and work in both ESM and CJS bundles.
+ *  2. Agent-native `.judge.md` files loaded at runtime from the `agents/`
+ *     directory (when available).  This enriches / overrides metadata.
  *
- * Legacy side-effect imports have been removed. If you need to add a judge, add
- * an agent file and (optionally) an evaluator script, then run:
- *   - `npm run generate:agents` (to sync)
- *   - `npm run validate:agents`
+ * The static imports guarantee that judges are always available, even in
+ * bundled environments (VS Code extension) where `agents/` is absent.
  */
 import { defaultRegistry } from "../judge-registry.js";
 import { loadAndRegisterAgents } from "../agent-loader.js";
 import { resolve, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
+// ─── Static side-effect imports (self-registering) ──────────────────────────
+import "./accessibility.js";
+import "./agent-instructions.js";
+import "./ai-code-safety.js";
+import "./api-contract.js";
+import "./api-design.js";
+import "./authentication.js";
+import "./backwards-compatibility.js";
+import "./caching.js";
+import "./ci-cd.js";
+import "./cloud-readiness.js";
+import "./code-structure.js";
+import "./compliance.js";
+import "./concurrency.js";
+import "./configuration-management.js";
+import "./cost-effectiveness.js";
+import "./cybersecurity.js";
+import "./data-security.js";
+import "./data-sovereignty.js";
+import "./database.js";
+import "./dependency-health.js";
+import "./documentation.js";
+import "./error-handling.js";
+import "./ethics-bias.js";
+import "./false-positive-review.js";
+import "./framework-safety.js";
+import "./hallucination-detection.js";
+import "./iac-security.js";
+import "./intent-alignment.js";
+import "./internationalization.js";
+import "./logging-privacy.js";
+import "./logic-review.js";
+import "./maintainability.js";
+import "./model-fingerprint.js";
+import "./multi-turn-coherence.js";
+import "./observability.js";
+import "./over-engineering.js";
+import "./performance.js";
+import "./portability.js";
+import "./rate-limiting.js";
+import "./reliability.js";
+import "./scalability.js";
+import "./security.js";
+import "./software-practices.js";
+import "./testing.js";
+import "./ux.js";
 // Support both ESM (import.meta.url) and CJS (esbuild bundle) environments.
 const _importMetaUrl = typeof import.meta?.url === "string" ? import.meta.url : undefined;
 const __filename = _importMetaUrl ? fileURLToPath(_importMetaUrl) : "";
-const __dirname = __filename ? dirname(__filename) : process.cwd();
+const __dirname = __filename ? dirname(__filename) : "";
 let agentsLoaded = false;
 function loadDefaultAgents() {
     if (agentsLoaded)
         return;
-    const agentsDir = resolve(__dirname, "..", "..", "agents");
-    loadAndRegisterAgents(agentsDir, defaultRegistry);
+    // Static side-effect imports above already registered all built-in judges.
+    // In ESM mode, also load from agents/ directory for metadata enrichment.
+    if (__dirname) {
+        try {
+            const agentsDir = resolve(__dirname, "..", "..", "agents");
+            loadAndRegisterAgents(agentsDir, defaultRegistry);
+        }
+        catch {
+            // agents/ directory may not exist — built-in judges are already loaded
+        }
+    }
     agentsLoaded = true;
 }
 // ─── Optional Agent Loader Integration ──────────────────────────────────────
@@ -42,9 +97,12 @@ export async function loadJudges() {
  * agents can augment or replace built-in judges. If a judge is already
  * registered, it is skipped.
  */
-export function loadAgentJudges(dir = resolve(__dirname, "..", "..", "agents")) {
+export function loadAgentJudges(dir) {
+    const agentsDir = dir ?? (__dirname ? resolve(__dirname, "..", "..", "agents") : "");
+    if (!agentsDir)
+        return 0; // CJS bundle — no agents directory available
     agentsLoaded = false; // allow re-run to pick up new agents if dir changes
-    const count = loadAndRegisterAgents(dir, defaultRegistry);
+    const count = loadAndRegisterAgents(agentsDir, defaultRegistry);
     agentsLoaded = true;
     return count;
 }

package/dist/tools/register-evaluation.js

@@ -5,10 +5,12 @@ import { z } from "zod";
 import { readFileSync, existsSync } from "fs";
 import { extname } from "path";
 import { JUDGES, getJudge, getJudgeSummaries } from "../judges/index.js";
-import { evaluateWithJudge, evaluateWithTribunal, formatVerdictAsMarkdown, formatEvaluationAsMarkdown, } from "../evaluators/index.js";
+import { evaluateWithJudge, evaluateWithTribunal, evaluateWithTribunalStreaming, formatVerdictAsMarkdown, formatEvaluationAsMarkdown, } from "../evaluators/index.js";
 import { evaluateCodeV2, evaluateProjectV2, getSupportedPolicyProfiles } from "../evaluators/v2.js";
 import { detectProjectContext } from "../evaluators/shared.js";
+import { getGlobalSession } from "../evaluation-session.js";
 import { configSchema, toJudgesConfig } from "./schemas.js";
+import { validateCodeSize } from "./validation.js";
 import { buildSingleJudgeDeepReviewSection, buildTribunalDeepReviewSection } from "./deep-review.js";
 /**
  * Register evaluation-focused tools: get_judges, evaluate_code,
@@ -20,6 +22,7 @@ export function registerEvaluationTools(server) {
     registerEvaluateSingleJudge(server);
     registerEvaluateV2(server);
     registerEvaluateFile(server);
+    registerEvaluateCodeStreaming(server);
 }
 // ─── get_judges ──────────────────────────────────────────────────────────────
 function registerGetJudges(server) {
@@ -34,6 +37,15 @@ function registerGetJudges(server) {
                     type: "text",
                     text: `# Judges Panel\n\n${text}`,
                 },
+                {
+                    type: "text",
+                    text: "```json\n" +
+                        JSON.stringify({
+                            judgeCount: judges.length,
+                            judges: judges.map((j) => ({ id: j.id, name: j.name, domain: j.domain })),
+                        }, null, 2) +
+                        "\n```",
+                },
             ],
         };
     });
@@ -70,20 +82,52 @@ function registerEvaluateCode(server) {
         config: configSchema,
     }, async ({ code, language, context, includeAstFindings, minConfidence, relatedFiles, config }) => {
         try {
+            const sizeError = validateCodeSize(code);
+            if (sizeError) {
+                return { content: [{ type: "text", text: `Error: ${sizeError}` }], isError: true };
+            }
+            const session = getGlobalSession();
             const verdict = evaluateWithTribunal(code, language, context, {
                 includeAstFindings,
                 minConfidence,
                 config: toJudgesConfig(config),
+                adaptiveSelection: true,
+                filePath: context,
             });
+            // Track evaluation in session
+            session.recordEvaluation(context ?? `<inline:${language}>`, code, verdict);
             const projectContext = detectProjectContext(code, language);
             const patternResults = formatVerdictAsMarkdown(verdict);
             const deepReview = buildTribunalDeepReviewSection(JUDGES, language, context, relatedFiles, projectContext);
+            // Structured JSON content block for programmatic consumption
+            const structuredData = {
+                score: verdict.overallScore,
+                verdict: verdict.overallVerdict,
+                findingCount: verdict.findings.length,
+                criticalCount: verdict.findings.filter((f) => f.severity === "critical").length,
+                highCount: verdict.findings.filter((f) => f.severity === "high").length,
+                judgesRun: verdict.evaluations.length,
+                findings: verdict.findings.map((f) => ({
+                    ruleId: f.ruleId,
+                    severity: f.severity,
+                    title: f.title,
+                    lineNumbers: f.lineNumbers,
+                    confidence: f.confidence,
+                })),
+                sessionStats: {
+                    evaluationCount: session.evaluationCount,
+                },
+            };
             return {
                 content: [
                     {
                         type: "text",
                         text: patternResults + deepReview,
                     },
+                    {
+                        type: "text",
+                        text: "```json\n" + JSON.stringify(structuredData, null, 2) + "\n```",
+                    },
                 ],
             };
         }
@@ -130,6 +174,10 @@ function registerEvaluateSingleJudge(server) {
         config: configSchema,
     }, async ({ code, language, judgeId, context, minConfidence, relatedFiles, config }) => {
         try {
+            const sizeError = validateCodeSize(code);
+            if (sizeError) {
+                return { content: [{ type: "text", text: `Error: ${sizeError}` }], isError: true };
+            }
             const judge = getJudge(judgeId);
             if (!judge) {
                 return {
@@ -149,12 +197,25 @@ function registerEvaluateSingleJudge(server) {
             const projectContext = detectProjectContext(code, language);
             const patternResults = formatEvaluationAsMarkdown(evaluation);
             const deepReview = buildSingleJudgeDeepReviewSection(judge, language, context, relatedFiles, projectContext);
+            const structured = {
+                judgeId,
+                judgeName: judge.name,
+                domain: judge.domain,
+                score: evaluation.score,
+                verdict: evaluation.verdict,
+                findingCount: evaluation.findings.length,
+                findings: evaluation.findings.map((f) => ({
+                    ruleId: f.ruleId,
+                    severity: f.severity,
+                    title: f.title,
+                    lineNumbers: f.lineNumbers,
+                    confidence: f.confidence,
+                })),
+            };
             return {
                 content: [
-                    {
-                        type: "text",
-                        text: patternResults + deepReview,
-                    },
+                    { type: "text", text: patternResults + deepReview },
+                    { type: "text", text: "```json\n" + JSON.stringify(structured, null, 2) + "\n```" },
                 ],
             };
         }
@@ -173,7 +234,7 @@ function registerEvaluateSingleJudge(server) {
 }
 // ─── evaluate_v2 ─────────────────────────────────────────────────────────────
 function registerEvaluateV2(server) {
-    server.tool("evaluate_v2", "Run V2 context-aware tribunal evaluation with policy profiles, evidence calibration, specialty feedback, confidence scoring, and uncertainty reporting.", {
+    server.tool("evaluate_policy_aware", "Run policy-aware tribunal evaluation with named policy profiles (startup, regulated, healthcare, fintech, public-sector), evidence calibration from runtime metrics, specialty-per-judge feedback, confidence scoring, and uncertainty reporting. Use this when code must meet specific compliance or vertical requirements.", {
         code: z.string().optional().describe("Source code for single-file mode"),
         language: z.string().optional().describe("Language for single-file mode"),
         files: z
@@ -263,7 +324,7 @@ function registerEvaluateV2(server) {
                     evaluationContext,
                     evidence,
                 });
-            let md = `# V2 Tribunal Evaluation\n\n`;
+            let md = `# Policy-Aware Tribunal Evaluation\n\n`;
             md += `**Policy Profile:** ${result.policyProfile}\n`;
             md += `**Calibrated Verdict:** ${result.calibratedVerdict.toUpperCase()} (${result.calibratedScore}/100)\n`;
             md += `**Base Verdict:** ${result.baseVerdict.overallVerdict.toUpperCase()} (${result.baseVerdict.overallScore}/100)\n`;
@@ -310,7 +371,28 @@ function registerEvaluateV2(server) {
             md += `\n## Supported Policy Profiles\n\n`;
             md += supportedProfiles.map((profile) => `- ${profile}`).join("\n");
             md += "\n";
-            return { content: [{ type: "text", text: md }] };
+            const structured = {
+                policyProfile: result.policyProfile,
+                calibratedScore: result.calibratedScore,
+                calibratedVerdict: result.calibratedVerdict,
+                baseScore: result.baseVerdict.overallScore,
+                baseVerdict: result.baseVerdict.overallVerdict,
+                confidence: result.confidence,
+                findingCount: result.findings.length,
+                findings: result.findings.map((f) => ({
+                    ruleId: f.ruleId,
+                    severity: f.severity,
+                    title: f.title,
+                    confidence: f.confidence,
+                })),
+                uncertainty: result.uncertainty,
+            };
+            return {
+                content: [
+                    { type: "text", text: md },
+                    { type: "text", text: "```json\n" + JSON.stringify(structured, null, 2) + "\n```" },
+                ],
+            };
         }
         catch (error) {
             return {
@@ -382,20 +464,60 @@ function registerEvaluateFile(server) {
             }
             const code = readFileSync(filePath, "utf-8");
             const detectedLang = language || detectLanguageFromPath(filePath);
+            const session = getGlobalSession();
+            // Skip re-evaluation if verdict is stable for this file
+            if (session.isVerdictStable(filePath)) {
+                const history = session.getVerdictHistory(filePath);
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `# Evaluation: ${filePath}\n\n` +
+                                `> ⚡ **Verdict stable** — score has converged at **${history[0]?.score ?? 0}/100** ` +
+                                `across last evaluations. Skipping redundant re-evaluation.\n\n` +
+                                `Use \`evaluate_code\` with the code directly to force a fresh evaluation.`,
+                        },
+                    ],
+                };
+            }
             const verdict = evaluateWithTribunal(code, detectedLang, context, {
                 includeAstFindings,
                 minConfidence,
                 config: toJudgesConfig(config),
+                adaptiveSelection: true,
+                filePath,
             });
+            session.recordEvaluation(filePath, code, verdict);
             const projectContext = detectProjectContext(code, detectedLang, filePath);
             const patternResults = formatVerdictAsMarkdown(verdict);
             const deepReview = buildTribunalDeepReviewSection(JUDGES, detectedLang, context, undefined, projectContext);
+            const structuredData = {
+                filePath,
+                language: detectedLang,
+                score: verdict.overallScore,
+                verdict: verdict.overallVerdict,
+                findingCount: verdict.findings.length,
+                criticalCount: verdict.findings.filter((f) => f.severity === "critical").length,
+                highCount: verdict.findings.filter((f) => f.severity === "high").length,
+                judgesRun: verdict.evaluations.length,
+                findings: verdict.findings.map((f) => ({
+                    ruleId: f.ruleId,
+                    severity: f.severity,
+                    title: f.title,
+                    lineNumbers: f.lineNumbers,
+                    confidence: f.confidence,
+                })),
+            };
             return {
                 content: [
                     {
                         type: "text",
                         text: `# Evaluation: ${filePath}\n\n` + patternResults + deepReview,
                     },
+                    {
+                        type: "text",
+                        text: "```json\n" + JSON.stringify(structuredData, null, 2) + "\n```",
+                    },
                 ],
             };
         }
@@ -412,3 +534,81 @@ function registerEvaluateFile(server) {
         }
     });
 }
+// ─── evaluate_code_streaming ─────────────────────────────────────────────────
+function registerEvaluateCodeStreaming(server) {
+    server.tool("evaluate_code_streaming", `Submit code for streaming evaluation — returns per-judge results as each judge completes, with running aggregate scores. Ideal for long evaluations where you want progressive feedback. All ${JUDGES.length} judges run sequentially with per-judge results accumulated into a single structured response.`, {
+        code: z.string().describe("The source code to evaluate."),
+        language: z.string().describe("The programming language (e.g., 'typescript', 'python', 'javascript')."),
+        context: z.string().optional().describe("Optional context about the code."),
+        includeAstFindings: z.boolean().optional().describe("Include AST/code-structure findings (default: true)"),
+        minConfidence: z
+            .number()
+            .min(0)
+            .max(1)
+            .optional()
+            .describe("Minimum finding confidence to include (0-1, default: 0)"),
+        config: configSchema,
+    }, async ({ code, language, context, includeAstFindings, minConfidence, config }) => {
+        try {
+            const session = getGlobalSession();
+            const batches = [];
+            let finalBatch;
+            for await (const batch of evaluateWithTribunalStreaming(code, language, context, {
+                includeAstFindings,
+                minConfidence,
+                config: toJudgesConfig(config),
+                adaptiveSelection: true,
+            })) {
+                batches.push({
+                    judgeId: batch.judgeId,
+                    judgeName: batch.judgeName,
+                    findingCount: batch.evaluation.findings.length,
+                    durationMs: batch.evaluation.durationMs ?? 0,
+                    runningScore: batch.aggregate.currentScore,
+                    runningVerdict: batch.aggregate.currentVerdict,
+                });
+                finalBatch = batch;
+            }
+            // Build progressive markdown
+            let md = `# Streaming Evaluation Results\n\n`;
+            md += `**Final Score:** ${finalBatch?.aggregate.currentScore ?? 0}/100\n`;
+            md += `**Verdict:** ${(finalBatch?.aggregate.currentVerdict ?? "pass").toUpperCase()}\n`;
+            md += `**Judges Run:** ${finalBatch?.aggregate.completedJudges ?? 0}/${finalBatch?.aggregate.totalJudges ?? 0}\n`;
+            md += `**Total Findings:** ${finalBatch?.aggregate.findingsSoFar ?? 0}\n\n`;
+            md += `## Per-Judge Breakdown\n\n`;
+            md += `| Judge | Findings | Time (ms) | Running Score |\n`;
+            md += `|-------|----------|-----------|---------------|\n`;
+            for (const b of batches) {
+                md += `| ${b.judgeName} | ${b.findingCount} | ${b.durationMs} | ${b.runningScore}/100 |\n`;
+            }
+            const structuredData = {
+                score: finalBatch?.aggregate.currentScore ?? 0,
+                verdict: finalBatch?.aggregate.currentVerdict ?? "pass",
+                totalFindings: finalBatch?.aggregate.findingsSoFar ?? 0,
+                criticalFindings: finalBatch?.aggregate.criticalSoFar ?? 0,
+                highFindings: finalBatch?.aggregate.highSoFar ?? 0,
+                judgesRun: finalBatch?.aggregate.completedJudges ?? 0,
+                totalJudges: finalBatch?.aggregate.totalJudges ?? 0,
+                perJudge: batches,
+                sessionEvaluationCount: session.evaluationCount,
+            };
+            return {
+                content: [
+                    { type: "text", text: md },
+                    { type: "text", text: "```json\n" + JSON.stringify(structuredData, null, 2) + "\n```" },
+                ],
+            };
+        }
+        catch (error) {
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: error instanceof Error ? `Error: ${error.message}` : "Error: Streaming evaluation failed",
+                    },
+                ],
+                isError: true,
+            };
+        }
+    });
+}

package/dist/tools/register-fix.js

@@ -7,6 +7,7 @@ import { evaluateWithTribunal, evaluateWithJudge } from "../evaluators/index.js"
 import { getJudge, JUDGES } from "../judges/index.js";
 import { applyPatches } from "../commands/fix.js";
 import { configSchema, toJudgesConfig } from "./schemas.js";
+import { validateCodeSize } from "./validation.js";
 /**
  * Register the fix_code tool for one-shot code evaluation + auto-fix.
  */
@@ -38,6 +39,10 @@ function registerFixCode(server) {
         config: configSchema,
     }, async ({ code, language, judgeId, context, minConfidence, config }) => {
         try {
+            const sizeError = validateCodeSize(code);
+            if (sizeError) {
+                return { content: [{ type: "text", text: `Error: ${sizeError}` }], isError: true };
+            }
             const effectiveMinConfidence = minConfidence ?? 0.5;
             // ── Evaluate ────────────────────────────────────────────────
             let allFindings;
@@ -133,8 +138,26 @@ function registerFixCode(server) {
                 text += `\n`;
             }
             text += `### Fixed Code\n\n\`\`\`${language}\n${fixedCode}\n\`\`\`\n`;
+            const structured = {
+                totalFindings: allFindings.length,
+                autoFixable: fixable.length,
+                applied,
+                skipped,
+                remaining: remaining.length,
+                patches: fixable.map((p) => ({
+                    ruleId: p.ruleId,
+                    severity: p.severity,
+                    title: p.title,
+                    line: p.patch.startLine,
+                    oldText: p.patch.oldText,
+                    newText: p.patch.newText,
+                })),
+            };
             return {
-                content: [{ type: "text", text }],
+                content: [
+                    { type: "text", text },
+                    { type: "text", text: "```json\n" + JSON.stringify(structured, null, 2) + "\n```" },
+                ],
             };
         }
         catch (error) {

package/dist/tools/register-resources.d.ts

@@ -0,0 +1,6 @@
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+/**
+ * Register MCP resources: judges catalog, presets, session state,
+ * and parameterized templates for single-judge / single-preset lookups.
+ */
+export declare function registerResources(server: McpServer): void;