@kevinrabun/judges 3.115.4 → 3.117.0

package/dist/tools/register-evaluation.js

@@ -5,10 +5,12 @@ import { z } from "zod";
 import { readFileSync, existsSync } from "fs";
 import { extname } from "path";
 import { JUDGES, getJudge, getJudgeSummaries } from "../judges/index.js";
-import { evaluateWithJudge, evaluateWithTribunal, formatVerdictAsMarkdown, formatEvaluationAsMarkdown, } from "../evaluators/index.js";
+import { evaluateWithJudge, evaluateWithTribunal, evaluateWithTribunalStreaming, formatVerdictAsMarkdown, formatEvaluationAsMarkdown, } from "../evaluators/index.js";
 import { evaluateCodeV2, evaluateProjectV2, getSupportedPolicyProfiles } from "../evaluators/v2.js";
 import { detectProjectContext } from "../evaluators/shared.js";
+import { getGlobalSession } from "../evaluation-session.js";
 import { configSchema, toJudgesConfig } from "./schemas.js";
+import { validateCodeSize } from "./validation.js";
 import { buildSingleJudgeDeepReviewSection, buildTribunalDeepReviewSection } from "./deep-review.js";
 /**
  * Register evaluation-focused tools: get_judges, evaluate_code,
@@ -20,6 +22,7 @@ export function registerEvaluationTools(server) {
     registerEvaluateSingleJudge(server);
     registerEvaluateV2(server);
     registerEvaluateFile(server);
+    registerEvaluateCodeStreaming(server);
 }
 // ─── get_judges ──────────────────────────────────────────────────────────────
 function registerGetJudges(server) {
@@ -34,6 +37,15 @@ function registerGetJudges(server) {
                     type: "text",
                     text: `# Judges Panel\n\n${text}`,
                 },
+                {
+                    type: "text",
+                    text: "```json\n" +
+                        JSON.stringify({
+                            judgeCount: judges.length,
+                            judges: judges.map((j) => ({ id: j.id, name: j.name, domain: j.domain })),
+                        }, null, 2) +
+                        "\n```",
+                },
             ],
         };
     });
@@ -70,20 +82,52 @@ function registerEvaluateCode(server) {
         config: configSchema,
     }, async ({ code, language, context, includeAstFindings, minConfidence, relatedFiles, config }) => {
         try {
+            const sizeError = validateCodeSize(code);
+            if (sizeError) {
+                return { content: [{ type: "text", text: `Error: ${sizeError}` }], isError: true };
+            }
+            const session = getGlobalSession();
             const verdict = evaluateWithTribunal(code, language, context, {
                 includeAstFindings,
                 minConfidence,
                 config: toJudgesConfig(config),
+                adaptiveSelection: true,
+                filePath: context,
             });
+            // Track evaluation in session
+            session.recordEvaluation(context ?? `<inline:${language}>`, code, verdict);
             const projectContext = detectProjectContext(code, language);
             const patternResults = formatVerdictAsMarkdown(verdict);
             const deepReview = buildTribunalDeepReviewSection(JUDGES, language, context, relatedFiles, projectContext);
+            // Structured JSON content block for programmatic consumption
+            const structuredData = {
+                score: verdict.overallScore,
+                verdict: verdict.overallVerdict,
+                findingCount: verdict.findings.length,
+                criticalCount: verdict.findings.filter((f) => f.severity === "critical").length,
+                highCount: verdict.findings.filter((f) => f.severity === "high").length,
+                judgesRun: verdict.evaluations.length,
+                findings: verdict.findings.map((f) => ({
+                    ruleId: f.ruleId,
+                    severity: f.severity,
+                    title: f.title,
+                    lineNumbers: f.lineNumbers,
+                    confidence: f.confidence,
+                })),
+                sessionStats: {
+                    evaluationCount: session.evaluationCount,
+                },
+            };
             return {
                 content: [
                     {
                         type: "text",
                         text: patternResults + deepReview,
                     },
+                    {
+                        type: "text",
+                        text: "```json\n" + JSON.stringify(structuredData, null, 2) + "\n```",
+                    },
                 ],
             };
         }
@@ -130,6 +174,10 @@ function registerEvaluateSingleJudge(server) {
         config: configSchema,
     }, async ({ code, language, judgeId, context, minConfidence, relatedFiles, config }) => {
         try {
+            const sizeError = validateCodeSize(code);
+            if (sizeError) {
+                return { content: [{ type: "text", text: `Error: ${sizeError}` }], isError: true };
+            }
             const judge = getJudge(judgeId);
             if (!judge) {
                 return {
@@ -149,12 +197,25 @@ function registerEvaluateSingleJudge(server) {
             const projectContext = detectProjectContext(code, language);
             const patternResults = formatEvaluationAsMarkdown(evaluation);
             const deepReview = buildSingleJudgeDeepReviewSection(judge, language, context, relatedFiles, projectContext);
+            const structured = {
+                judgeId,
+                judgeName: judge.name,
+                domain: judge.domain,
+                score: evaluation.score,
+                verdict: evaluation.verdict,
+                findingCount: evaluation.findings.length,
+                findings: evaluation.findings.map((f) => ({
+                    ruleId: f.ruleId,
+                    severity: f.severity,
+                    title: f.title,
+                    lineNumbers: f.lineNumbers,
+                    confidence: f.confidence,
+                })),
+            };
             return {
                 content: [
-                    {
-                        type: "text",
-                        text: patternResults + deepReview,
-                    },
+                    { type: "text", text: patternResults + deepReview },
+                    { type: "text", text: "```json\n" + JSON.stringify(structured, null, 2) + "\n```" },
                 ],
             };
         }
@@ -173,7 +234,7 @@ function registerEvaluateSingleJudge(server) {
 }
 // ─── evaluate_v2 ─────────────────────────────────────────────────────────────
 function registerEvaluateV2(server) {
-    server.tool("evaluate_v2", "Run V2 context-aware tribunal evaluation with policy profiles, evidence calibration, specialty feedback, confidence scoring, and uncertainty reporting.", {
+    server.tool("evaluate_policy_aware", "Run policy-aware tribunal evaluation with named policy profiles (startup, regulated, healthcare, fintech, public-sector), evidence calibration from runtime metrics, specialty-per-judge feedback, confidence scoring, and uncertainty reporting. Use this when code must meet specific compliance or vertical requirements.", {
         code: z.string().optional().describe("Source code for single-file mode"),
         language: z.string().optional().describe("Language for single-file mode"),
         files: z
@@ -263,7 +324,7 @@ function registerEvaluateV2(server) {
                     evaluationContext,
                     evidence,
                 });
-            let md = `# V2 Tribunal Evaluation\n\n`;
+            let md = `# Policy-Aware Tribunal Evaluation\n\n`;
             md += `**Policy Profile:** ${result.policyProfile}\n`;
             md += `**Calibrated Verdict:** ${result.calibratedVerdict.toUpperCase()} (${result.calibratedScore}/100)\n`;
             md += `**Base Verdict:** ${result.baseVerdict.overallVerdict.toUpperCase()} (${result.baseVerdict.overallScore}/100)\n`;
@@ -310,7 +371,28 @@ function registerEvaluateV2(server) {
             md += `\n## Supported Policy Profiles\n\n`;
             md += supportedProfiles.map((profile) => `- ${profile}`).join("\n");
             md += "\n";
-            return { content: [{ type: "text", text: md }] };
+            const structured = {
+                policyProfile: result.policyProfile,
+                calibratedScore: result.calibratedScore,
+                calibratedVerdict: result.calibratedVerdict,
+                baseScore: result.baseVerdict.overallScore,
+                baseVerdict: result.baseVerdict.overallVerdict,
+                confidence: result.confidence,
+                findingCount: result.findings.length,
+                findings: result.findings.map((f) => ({
+                    ruleId: f.ruleId,
+                    severity: f.severity,
+                    title: f.title,
+                    confidence: f.confidence,
+                })),
+                uncertainty: result.uncertainty,
+            };
+            return {
+                content: [
+                    { type: "text", text: md },
+                    { type: "text", text: "```json\n" + JSON.stringify(structured, null, 2) + "\n```" },
+                ],
+            };
         }
         catch (error) {
             return {
@@ -382,20 +464,60 @@ function registerEvaluateFile(server) {
             }
             const code = readFileSync(filePath, "utf-8");
             const detectedLang = language || detectLanguageFromPath(filePath);
+            const session = getGlobalSession();
+            // Skip re-evaluation if verdict is stable for this file
+            if (session.isVerdictStable(filePath)) {
+                const history = session.getVerdictHistory(filePath);
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `# Evaluation: ${filePath}\n\n` +
+                                `> ⚡ **Verdict stable** — score has converged at **${history[0]?.score ?? 0}/100** ` +
+                                `across last evaluations. Skipping redundant re-evaluation.\n\n` +
+                                `Use \`evaluate_code\` with the code directly to force a fresh evaluation.`,
+                        },
+                    ],
+                };
+            }
             const verdict = evaluateWithTribunal(code, detectedLang, context, {
                 includeAstFindings,
                 minConfidence,
                 config: toJudgesConfig(config),
+                adaptiveSelection: true,
+                filePath,
             });
+            session.recordEvaluation(filePath, code, verdict);
             const projectContext = detectProjectContext(code, detectedLang, filePath);
             const patternResults = formatVerdictAsMarkdown(verdict);
             const deepReview = buildTribunalDeepReviewSection(JUDGES, detectedLang, context, undefined, projectContext);
+            const structuredData = {
+                filePath,
+                language: detectedLang,
+                score: verdict.overallScore,
+                verdict: verdict.overallVerdict,
+                findingCount: verdict.findings.length,
+                criticalCount: verdict.findings.filter((f) => f.severity === "critical").length,
+                highCount: verdict.findings.filter((f) => f.severity === "high").length,
+                judgesRun: verdict.evaluations.length,
+                findings: verdict.findings.map((f) => ({
+                    ruleId: f.ruleId,
+                    severity: f.severity,
+                    title: f.title,
+                    lineNumbers: f.lineNumbers,
+                    confidence: f.confidence,
+                })),
+            };
             return {
                 content: [
                     {
                         type: "text",
                         text: `# Evaluation: ${filePath}\n\n` + patternResults + deepReview,
                     },
+                    {
+                        type: "text",
+                        text: "```json\n" + JSON.stringify(structuredData, null, 2) + "\n```",
+                    },
                 ],
             };
         }
@@ -412,3 +534,81 @@ function registerEvaluateFile(server) {
         }
     });
 }
+// ─── evaluate_code_streaming ─────────────────────────────────────────────────
+function registerEvaluateCodeStreaming(server) {
+    server.tool("evaluate_code_streaming", `Submit code for streaming evaluation — returns per-judge results as each judge completes, with running aggregate scores. Ideal for long evaluations where you want progressive feedback. All ${JUDGES.length} judges run sequentially with per-judge results accumulated into a single structured response.`, {
+        code: z.string().describe("The source code to evaluate."),
+        language: z.string().describe("The programming language (e.g., 'typescript', 'python', 'javascript')."),
+        context: z.string().optional().describe("Optional context about the code."),
+        includeAstFindings: z.boolean().optional().describe("Include AST/code-structure findings (default: true)"),
+        minConfidence: z
+            .number()
+            .min(0)
+            .max(1)
+            .optional()
+            .describe("Minimum finding confidence to include (0-1, default: 0)"),
+        config: configSchema,
+    }, async ({ code, language, context, includeAstFindings, minConfidence, config }) => {
+        try {
+            const session = getGlobalSession();
+            const batches = [];
+            let finalBatch;
+            for await (const batch of evaluateWithTribunalStreaming(code, language, context, {
+                includeAstFindings,
+                minConfidence,
+                config: toJudgesConfig(config),
+                adaptiveSelection: true,
+            })) {
+                batches.push({
+                    judgeId: batch.judgeId,
+                    judgeName: batch.judgeName,
+                    findingCount: batch.evaluation.findings.length,
+                    durationMs: batch.evaluation.durationMs ?? 0,
+                    runningScore: batch.aggregate.currentScore,
+                    runningVerdict: batch.aggregate.currentVerdict,
+                });
+                finalBatch = batch;
+            }
+            // Build progressive markdown
+            let md = `# Streaming Evaluation Results\n\n`;
+            md += `**Final Score:** ${finalBatch?.aggregate.currentScore ?? 0}/100\n`;
+            md += `**Verdict:** ${(finalBatch?.aggregate.currentVerdict ?? "pass").toUpperCase()}\n`;
+            md += `**Judges Run:** ${finalBatch?.aggregate.completedJudges ?? 0}/${finalBatch?.aggregate.totalJudges ?? 0}\n`;
+            md += `**Total Findings:** ${finalBatch?.aggregate.findingsSoFar ?? 0}\n\n`;
+            md += `## Per-Judge Breakdown\n\n`;
+            md += `| Judge | Findings | Time (ms) | Running Score |\n`;
+            md += `|-------|----------|-----------|---------------|\n`;
+            for (const b of batches) {
+                md += `| ${b.judgeName} | ${b.findingCount} | ${b.durationMs} | ${b.runningScore}/100 |\n`;
+            }
+            const structuredData = {
+                score: finalBatch?.aggregate.currentScore ?? 0,
+                verdict: finalBatch?.aggregate.currentVerdict ?? "pass",
+                totalFindings: finalBatch?.aggregate.findingsSoFar ?? 0,
+                criticalFindings: finalBatch?.aggregate.criticalSoFar ?? 0,
+                highFindings: finalBatch?.aggregate.highSoFar ?? 0,
+                judgesRun: finalBatch?.aggregate.completedJudges ?? 0,
+                totalJudges: finalBatch?.aggregate.totalJudges ?? 0,
+                perJudge: batches,
+                sessionEvaluationCount: session.evaluationCount,
+            };
+            return {
+                content: [
+                    { type: "text", text: md },
+                    { type: "text", text: "```json\n" + JSON.stringify(structuredData, null, 2) + "\n```" },
+                ],
+            };
+        }
+        catch (error) {
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: error instanceof Error ? `Error: ${error.message}` : "Error: Streaming evaluation failed",
+                    },
+                ],
+                isError: true,
+            };
+        }
+    });
+}

package/dist/tools/register-fix.js

@@ -7,6 +7,7 @@ import { evaluateWithTribunal, evaluateWithJudge } from "../evaluators/index.js"
 import { getJudge, JUDGES } from "../judges/index.js";
 import { applyPatches } from "../commands/fix.js";
 import { configSchema, toJudgesConfig } from "./schemas.js";
+import { validateCodeSize } from "./validation.js";
 /**
  * Register the fix_code tool for one-shot code evaluation + auto-fix.
  */
@@ -38,6 +39,10 @@ function registerFixCode(server) {
         config: configSchema,
     }, async ({ code, language, judgeId, context, minConfidence, config }) => {
         try {
+            const sizeError = validateCodeSize(code);
+            if (sizeError) {
+                return { content: [{ type: "text", text: `Error: ${sizeError}` }], isError: true };
+            }
             const effectiveMinConfidence = minConfidence ?? 0.5;
             // ── Evaluate ────────────────────────────────────────────────
             let allFindings;
@@ -133,8 +138,26 @@ function registerFixCode(server) {
                 text += `\n`;
             }
             text += `### Fixed Code\n\n\`\`\`${language}\n${fixedCode}\n\`\`\`\n`;
+            const structured = {
+                totalFindings: allFindings.length,
+                autoFixable: fixable.length,
+                applied,
+                skipped,
+                remaining: remaining.length,
+                patches: fixable.map((p) => ({
+                    ruleId: p.ruleId,
+                    severity: p.severity,
+                    title: p.title,
+                    line: p.patch.startLine,
+                    oldText: p.patch.oldText,
+                    newText: p.patch.newText,
+                })),
+            };
             return {
-                content: [{ type: "text", text }],
+                content: [
+                    { type: "text", text },
+                    { type: "text", text: "```json\n" + JSON.stringify(structured, null, 2) + "\n```" },
+                ],
             };
         }
         catch (error) {

package/dist/tools/register-resources.d.ts

@@ -0,0 +1,6 @@
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+/**
+ * Register MCP resources: judges catalog, presets, session state,
+ * and parameterized templates for single-judge / single-preset lookups.
+ */
+export declare function registerResources(server: McpServer): void;

package/dist/tools/register-resources.js

@@ -0,0 +1,177 @@
+// ─── MCP Resource Registration ───────────────────────────────────────────────
+// Expose judges metadata, presets, and session state as MCP resources.
+// Includes both static resources and parameterized resource templates for
+// efficient single-item lookups (judges://judge/{id}, judges://preset/{key}).
+// ──────────────────────────────────────────────────────────────────────────────
+import { ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { getJudge, getJudgeSummaries, JUDGES } from "../judges/index.js";
+import { getPreset, PRESETS } from "../presets.js";
+import { getGlobalSession } from "../evaluation-session.js";
+/**
+ * Register MCP resources: judges catalog, presets, session state,
+ * and parameterized templates for single-judge / single-preset lookups.
+ */
+export function registerResources(server) {
+    registerJudgesCatalog(server);
+    registerPresetsResource(server);
+    registerSessionResource(server);
+    registerJudgeTemplate(server);
+    registerPresetTemplate(server);
+}
+// ─── judges://catalog ────────────────────────────────────────────────────────
+function registerJudgesCatalog(server) {
+    server.resource("judges-catalog", "judges://catalog", { description: "Full catalog of all judges on the panel — IDs, names, domains, and descriptions." }, async (uri) => {
+        const judges = getJudgeSummaries();
+        const data = judges.map((j) => ({
+            id: j.id,
+            name: j.name,
+            domain: j.domain,
+            description: j.description,
+        }));
+        return {
+            contents: [
+                {
+                    uri: uri.href,
+                    mimeType: "application/json",
+                    text: JSON.stringify(data, null, 2),
+                },
+            ],
+        };
+    });
+}
+// ─── judges://presets ────────────────────────────────────────────────────────
+function registerPresetsResource(server) {
+    server.resource("presets", "judges://presets", { description: "Available evaluation presets with names, descriptions, and configuration overrides." }, async (uri) => {
+        const data = Object.entries(PRESETS).map(([key, preset]) => ({
+            key,
+            name: preset.name,
+            description: preset.description,
+            config: preset.config,
+        }));
+        return {
+            contents: [
+                {
+                    uri: uri.href,
+                    mimeType: "application/json",
+                    text: JSON.stringify(data, null, 2),
+                },
+            ],
+        };
+    });
+}
+// ─── judges://session ────────────────────────────────────────────────────────
+function registerSessionResource(server) {
+    server.resource("session", "judges://session", {
+        description: "Current evaluation session state — evaluation count, detected frameworks, verdict history, and stability indicators.",
+    }, async (uri) => {
+        const session = getGlobalSession();
+        const ctx = session.getContext();
+        const filesEvaluated = [...ctx.verdictHistory.entries()].map(([file, history]) => ({
+            file,
+            evaluations: history.length,
+            latestScore: history[history.length - 1]?.score ?? 0,
+            stable: session.isVerdictStable(file),
+        }));
+        const data = {
+            evaluationCount: ctx.evaluationCount,
+            startedAt: ctx.startedAt,
+            frameworks: ctx.frameworks,
+            capabilities: [...ctx.capabilities],
+            filesEvaluated,
+        };
+        return {
+            contents: [
+                {
+                    uri: uri.href,
+                    mimeType: "application/json",
+                    text: JSON.stringify(data, null, 2),
+                },
+            ],
+        };
+    });
+}
+// ─── judges://judge/{id} (template) ─────────────────────────────────────────
+function registerJudgeTemplate(server) {
+    const judgeIds = JUDGES.map((j) => j.id);
+    server.resource("judge-detail", new ResourceTemplate("judges://judge/{id}", {
+        list: async () => ({
+            resources: judgeIds.map((id) => ({
+                uri: `judges://judge/${id}`,
+                name: id,
+            })),
+        }),
+        complete: {
+            id: (value) => judgeIds.filter((id) => id.startsWith(value)),
+        },
+    }), { description: "Detailed info for a single judge — rules, domain, system prompt summary." }, async (uri, { id }) => {
+        const judgeId = Array.isArray(id) ? id[0] : id;
+        const judge = getJudge(judgeId);
+        if (!judge) {
+            return {
+                contents: [
+                    {
+                        uri: uri.href,
+                        mimeType: "application/json",
+                        text: JSON.stringify({ error: `Judge '${judgeId}' not found` }),
+                    },
+                ],
+            };
+        }
+        const data = {
+            id: judge.id,
+            name: judge.name,
+            domain: judge.domain,
+            description: judge.description,
+            rulePrefix: judge.rulePrefix,
+            tableDescription: judge.tableDescription,
+            promptDescription: judge.promptDescription,
+        };
+        return {
+            contents: [
+                {
+                    uri: uri.href,
+                    mimeType: "application/json",
+                    text: JSON.stringify(data, null, 2),
+                },
+            ],
+        };
+    });
+}
+// ─── judges://preset/{key} (template) ───────────────────────────────────────
+function registerPresetTemplate(server) {
+    const presetKeys = Object.keys(PRESETS);
+    server.resource("preset-detail", new ResourceTemplate("judges://preset/{key}", {
+        list: async () => ({
+            resources: presetKeys.map((key) => ({
+                uri: `judges://preset/${key}`,
+                name: key,
+            })),
+        }),
+        complete: {
+            key: (value) => presetKeys.filter((k) => k.startsWith(value)),
+        },
+    }), { description: "Detailed configuration for a single evaluation preset." }, async (uri, { key }) => {
+        const presetKey = Array.isArray(key) ? key[0] : key;
+        const preset = getPreset(presetKey);
+        if (!preset) {
+            return {
+                contents: [
+                    {
+                        uri: uri.href,
+                        mimeType: "application/json",
+                        text: JSON.stringify({ error: `Preset '${presetKey}' not found` }),
+                    },
+                ],
+            };
+        }
+        return {
+            contents: [
+                {
+                    uri: uri.href,
+                    mimeType: "application/json",
+                    text: JSON.stringify({ key: presetKey, ...preset }, null, 2),
+                },
+            ],
+        };
+    });
+}

package/dist/tools/register-review.js

@@ -115,8 +115,21 @@ function registerExplainFinding(server) {
             sections.push(`\n## Remediation\n${remediation}`);
         }
         sections.push(`\n## Next steps\n- Use \`triage_finding\` to accept, defer, or dismiss this finding\n- Use \`fix_code\` to auto-fix if a patch is available\n- Use \`evaluate_code\` to re-evaluate after fixing`);
+        const structured = {
+            ruleId,
+            prefix,
+            title: title ?? null,
+            severity: severity ?? null,
+            owasp: ctx?.owasp ?? null,
+            cwe: ctx?.cwe ?? null,
+            learn: ctx?.learn ?? null,
+            remediation: getRemediationGuidance(prefix) ?? null,
+        };
         return {
-            content: [{ type: "text", text: sections.join("\n") }],
+            content: [
+                { type: "text", text: sections.join("\n") },
+                { type: "text", text: "```json\n" + JSON.stringify(structured, null, 2) + "\n```" },
+            ],
         };
     });
 }
@@ -173,6 +186,18 @@ function registerTriageFinding(server) {
                         type: "text",
                         text: `✓ Triaged finding \`${result.ruleId}\` in ${result.filePath} as **${status}**${reason ? `\n\nReason: ${reason}` : ""}${triagedBy ? `\nTriaged by: ${triagedBy}` : ""}`,
                     },
+                    {
+                        type: "text",
+                        text: "```json\n" +
+                            JSON.stringify({
+                                ruleId: result.ruleId,
+                                filePath: result.filePath,
+                                status,
+                                reason: reason ?? null,
+                                triagedBy: triagedBy ?? null,
+                            }, null, 2) +
+                            "\n```",
+                    },
                 ],
             };
         }