@kevinrabun/judges 3.115.4 → 3.117.0

package/dist/escalation.js

@@ -0,0 +1,292 @@
+/**
+ * Human Escalation Protocol
+ *
+ * Routes low-confidence findings to human reviewers instead of auto-actioning.
+ * Provides a structured escalation workflow with reasons, routing suggestions,
+ * and a persistent escalation queue.
+ *
+ * Data stored in .judges-escalations.json
+ */
+import { readFileSync, writeFileSync, existsSync } from "fs";
+import { resolve } from "path";
+import { getDataAdapter } from "./data-adapter.js";
+// ─── Constants ───────────────────────────────────────────────────────────────
+const ESCALATION_FILE = ".judges-escalations.json";
+const DEFAULT_CONFIDENCE_THRESHOLD = 0.5;
+/** Rule prefixes that route to security team */
+const SECURITY_PREFIXES = new Set(["SEC-", "CYBER-", "AUTH-", "DATA-", "AICS-", "LOGPRIV-"]);
+/** Rule prefixes that route to compliance officer */
+const COMPLIANCE_PREFIXES = new Set(["COMP-", "DSOV-", "ETH-"]);
+// ─── Escalation Store I/O ────────────────────────────────────────────────────
+export function loadEscalationStore(dir = ".") {
+    const filePath = resolve(dir, ESCALATION_FILE);
+    if (!existsSync(filePath)) {
+        return { version: "1.0.0", escalations: [], lastUpdated: new Date().toISOString() };
+    }
+    try {
+        return JSON.parse(readFileSync(filePath, "utf-8"));
+    }
+    catch {
+        return { version: "1.0.0", escalations: [], lastUpdated: new Date().toISOString() };
+    }
+}
+export function saveEscalationStore(store, dir = ".") {
+    store.lastUpdated = new Date().toISOString();
+    const filePath = resolve(dir, ESCALATION_FILE);
+    writeFileSync(filePath, JSON.stringify(store, null, 2) + "\n", "utf-8");
+}
+export async function loadEscalationsViaAdapter(projectDir, adapter) {
+    const da = adapter ?? getDataAdapter();
+    return ((await da.loadJson("escalations", projectDir)) ?? {
+        version: "1.0.0",
+        escalations: [],
+        lastUpdated: new Date().toISOString(),
+    });
+}
+export async function saveEscalationsViaAdapter(store, projectDir, adapter) {
+    const da = adapter ?? getDataAdapter();
+    return da.saveJson("escalations", store, projectDir);
+}
+// ─── Escalation Logic ────────────────────────────────────────────────────────
+/**
+ * Determine why a finding should be escalated.
+ */
+function classifyEscalationReasons(finding, verdict) {
+    const reasons = [];
+    const conf = finding.confidence ?? 0.5;
+    if (conf < DEFAULT_CONFIDENCE_THRESHOLD) {
+        reasons.push("low-confidence");
+    }
+    // Check for conflicting judge signals
+    if (verdict) {
+        const judgeVerdicts = verdict.evaluations.map((e) => e.verdict);
+        const hasPass = judgeVerdicts.includes("pass");
+        const hasFail = judgeVerdicts.includes("fail");
+        if (hasPass && hasFail) {
+            reasons.push("conflicting-judges");
+        }
+    }
+    // AI-generated code detection
+    if (finding.ruleId.startsWith("MFPR")) {
+        reasons.push("ai-generated-code");
+    }
+    // Novel pattern — absence-based with no prior feedback data
+    if (finding.isAbsenceBased && finding.provenance === "absence-of-pattern") {
+        reasons.push("cross-file-uncertainty");
+    }
+    // Compliance-sensitive rules
+    if (finding.ruleId.startsWith("COMP-") || finding.ruleId.startsWith("DSOV-") || finding.ruleId.startsWith("ETH-")) {
+        reasons.push("compliance-sensitive");
+    }
+    // High-severity with low evidence
+    if ((finding.severity === "critical" || finding.severity === "high") && conf < 0.7) {
+        reasons.push("security-critical-low-evidence");
+    }
+    return reasons.length > 0 ? reasons : ["low-confidence"];
+}
+/**
+ * Determine which team/role should review this escalation.
+ */
+function determineRouting(finding) {
+    const ruleId = finding.ruleId;
+    for (const prefix of SECURITY_PREFIXES) {
+        if (ruleId.startsWith(prefix))
+            return "security-team";
+    }
+    for (const prefix of COMPLIANCE_PREFIXES) {
+        if (ruleId.startsWith(prefix))
+            return "compliance-officer";
+    }
+    if (finding.severity === "critical")
+        return "senior-developer";
+    if (finding.severity === "high")
+        return "tech-lead";
+    return "any-human";
+}
+/**
+ * Generate a human-readable explanation for why this finding was escalated.
+ */
+function buildEscalationExplanation(finding, reasons) {
+    const parts = [];
+    if (reasons.includes("low-confidence")) {
+        const conf = finding.confidence ?? 0;
+        parts.push(`Confidence is ${Math.round(conf * 100)}%, below the escalation threshold`);
+    }
+    if (reasons.includes("conflicting-judges")) {
+        parts.push("Judges disagree on the verdict for this file");
+    }
+    if (reasons.includes("ai-generated-code")) {
+        parts.push("AI-generated code detected — requires human verification of correctness");
+    }
+    if (reasons.includes("cross-file-uncertainty")) {
+        parts.push("Finding depends on cross-file context that could not be verified");
+    }
+    if (reasons.includes("compliance-sensitive")) {
+        parts.push("Compliance-sensitive finding requires human sign-off");
+    }
+    if (reasons.includes("security-critical-low-evidence")) {
+        parts.push(`High-severity security finding (${finding.severity}) with insufficient evidence — needs expert analysis`);
+    }
+    if (reasons.includes("novel-pattern")) {
+        parts.push("Pattern not seen before — no historical data to calibrate confidence");
+    }
+    return `[${finding.ruleId}] ${finding.title}: ${parts.join("; ")}.`;
+}
+let escalationCounter = 0;
+/**
+ * Generate a unique escalation ID.
+ */
+function generateEscalationId() {
+    escalationCounter++;
+    const ts = Date.now().toString(36);
+    const seq = escalationCounter.toString(36).padStart(4, "0");
+    return `ESC-${ts}-${seq}`;
+}
+/**
+ * Evaluate which findings in a tribunal verdict need human escalation.
+ * Mutates findings to set `needsHumanReview` and returns the escalation records.
+ */
+export function evaluateEscalations(verdict, filePath, policy) {
+    const threshold = policy?.confidenceThreshold ?? DEFAULT_CONFIDENCE_THRESHOLD;
+    const alwaysSeverities = new Set(policy?.alwaysEscalateSeverities ?? []);
+    const alwaysPrefixes = policy?.alwaysEscalatePrefixes ?? [];
+    const escalations = [];
+    const now = new Date().toISOString();
+    for (const finding of verdict.findings) {
+        const conf = finding.confidence ?? 0.5;
+        let shouldEscalate = false;
+        // Confidence below threshold
+        if (conf < threshold)
+            shouldEscalate = true;
+        // Always-escalate severities
+        if (alwaysSeverities.has(finding.severity))
+            shouldEscalate = true;
+        // Always-escalate rule prefixes
+        if (alwaysPrefixes.some((p) => finding.ruleId.startsWith(p)))
+            shouldEscalate = true;
+        if (shouldEscalate) {
+            finding.needsHumanReview = true;
+            const reasons = classifyEscalationReasons(finding, verdict);
+            const routing = determineRouting(finding);
+            const explanation = buildEscalationExplanation(finding, reasons);
+            escalations.push({
+                escalationId: generateEscalationId(),
+                finding,
+                filePath,
+                reasons,
+                routing,
+                explanation,
+                status: "pending",
+                createdAt: now,
+            });
+        }
+    }
+    return escalations;
+}
+/**
+ * Resolve an escalation — mark it as resolved or dismissed.
+ */
+export function resolveEscalation(store, escalationId, resolution) {
+    const esc = store.escalations.find((e) => e.escalationId === escalationId);
+    if (!esc || esc.status === "resolved" || esc.status === "dismissed")
+        return false;
+    esc.status = resolution.status;
+    esc.resolvedAt = new Date().toISOString();
+    esc.resolvedBy = resolution.resolvedBy;
+    esc.resolutionNotes = resolution.notes;
+    return true;
+}
+/**
+ * Compute summary statistics for the escalation queue.
+ */
+export function computeEscalationSummary(store) {
+    const byRouting = {};
+    const byReason = {};
+    let pending = 0;
+    let acknowledged = 0;
+    let resolved = 0;
+    let dismissed = 0;
+    let oldestPendingMs = 0;
+    const now = Date.now();
+    for (const esc of store.escalations) {
+        switch (esc.status) {
+            case "pending":
+                pending++;
+                break;
+            case "acknowledged":
+                acknowledged++;
+                break;
+            case "resolved":
+                resolved++;
+                break;
+            case "dismissed":
+                dismissed++;
+                break;
+        }
+        if (esc.status === "pending") {
+            const age = now - new Date(esc.createdAt).getTime();
+            if (age > oldestPendingMs)
+                oldestPendingMs = age;
+        }
+        byRouting[esc.routing] = (byRouting[esc.routing] ?? 0) + 1;
+        for (const reason of esc.reasons) {
+            byReason[reason] = (byReason[reason] ?? 0) + 1;
+        }
+    }
+    return {
+        total: store.escalations.length,
+        pending,
+        acknowledged,
+        resolved,
+        dismissed,
+        byRouting,
+        byReason,
+        oldestPendingHours: Math.round((oldestPendingMs / (1000 * 60 * 60)) * 10) / 10,
+    };
+}
+/**
+ * Check whether the escalation queue should block a merge.
+ * Blocks when pending escalations exceed the policy limit.
+ */
+export function shouldBlockOnEscalations(store, policy) {
+    const maxPending = policy?.maxPendingBeforeBlock ?? 0;
+    if (maxPending <= 0)
+        return false;
+    const pending = store.escalations.filter((e) => e.status === "pending").length;
+    return pending >= maxPending;
+}
+/**
+ * Enhance a ReviewDecision with escalation information.
+ * When escalations exist, the review action may be upgraded to "request-changes"
+ * to ensure a human signs off.
+ */
+export function enhanceReviewWithEscalations(decision, escalations) {
+    if (escalations.length === 0)
+        return decision;
+    const pendingCount = escalations.filter((e) => e.status === "pending").length;
+    // If there are pending escalations, upgrade to at least "comment"
+    let action = decision.action;
+    if (pendingCount > 0 && action === "approve") {
+        action = "comment";
+    }
+    // Critical escalations force request-changes
+    const hasCriticalEscalation = escalations.some((e) => e.status === "pending" &&
+        (e.reasons.includes("security-critical-low-evidence") || e.reasons.includes("compliance-sensitive")));
+    if (hasCriticalEscalation) {
+        action = "request-changes";
+    }
+    const escalationSummary = `\n\n**Escalation Notice**: ${pendingCount} finding(s) flagged for human review. ` +
+        `Routing: ${[...new Set(escalations.map((e) => e.routing))].join(", ")}.`;
+    return {
+        ...decision,
+        action,
+        summary: decision.summary + escalationSummary,
+        blockingIssues: [
+            ...decision.blockingIssues,
+            ...escalations
+                .filter((e) => e.status === "pending")
+                .slice(0, 3)
+                .map((e) => `[ESCALATED] ${e.explanation}`),
+        ],
+    };
+}

package/dist/evaluation-session.d.ts

@@ -0,0 +1,74 @@
+/**
+ * Evaluation session — persistent context that survives across multiple
+ * evaluation calls within the same session (MCP connection, VS Code
+ * extension lifetime, or CLI watch mode).
+ *
+ * Avoids redundant framework detection, capability scanning, and feedback
+ * loading. Tracks verdict evolution per file for stability detection.
+ */
+import type { SessionContext, TribunalVerdict } from "./types.js";
+/**
+ * An evaluation session that accumulates project knowledge across calls.
+ */
+export declare class EvaluationSession {
+    private ctx;
+    constructor();
+    /** Get the current session context (read-only snapshot). */
+    getContext(): Readonly<SessionContext>;
+    /** Number of evaluations performed. */
+    get evaluationCount(): number;
+    /** Record detected frameworks (deduplicated). */
+    addFrameworks(frameworks: string[]): void;
+    /** Record detected project capabilities (e.g. "rate-limiting", "auth"). */
+    addCapabilities(caps: Iterable<string>): void;
+    /** Get accumulated capabilities for absence-based finding suppression. */
+    getCapabilities(): Set<string>;
+    /**
+     * Record an evaluation result for a file. Tracks verdict history
+     * so repeated evaluations can detect stability (converging scores).
+     */
+    recordEvaluation(filePath: string, code: string, verdict: TribunalVerdict): void;
+    /**
+     * Check if a file's verdict is stable — same score and finding count
+     * across the last N evaluations. Returns true if stable (skip re-eval).
+     */
+    isVerdictStable(filePath: string, minRuns?: number): boolean;
+    /**
+     * Check if a file has already been evaluated with the same content.
+     */
+    hasEvaluated(filePath: string, code: string): boolean;
+    /**
+     * Get verdict history for a file — most recent first.
+     */
+    getVerdictHistory(filePath: string): Array<{
+        score: number;
+        findingCount: number;
+        timestamp: string;
+    }>;
+    /** Reset the session (clear all accumulated context). */
+    reset(): void;
+    /**
+     * Record user feedback for a finding rule.
+     * tp = true positive, fp = false positive, wontfix = acknowledged but skipped.
+     */
+    recordFeedback(ruleId: string, verdict: "tp" | "fp" | "wontfix"): void;
+    /**
+     * Get a confidence penalty for a rule based on accumulated FP feedback.
+     * Returns a multiplier in (0, 1] — 1.0 means no penalty, lower means
+     * the rule has been flagged as FP frequently and confidence should be reduced.
+     *
+     * Formula: 1 / (1 + fpCount) — degrades smoothly as FP reports accumulate.
+     * A single FP report halves confidence; two reports reduce it to 1/3, etc.
+     */
+    getConfidencePenalty(ruleId: string): number;
+    /** Get the raw feedback tally for all rules. */
+    getFeedbackTally(): ReadonlyMap<string, {
+        tp: number;
+        fp: number;
+        wontfix: number;
+    }>;
+}
+/** Get or create the global evaluation session (shared across MCP calls). */
+export declare function getGlobalSession(): EvaluationSession;
+/** Reset the global session (for testing or explicit reset). */
+export declare function resetGlobalSession(): void;

package/dist/evaluation-session.js

@@ -0,0 +1,152 @@
+/**
+ * Evaluation session — persistent context that survives across multiple
+ * evaluation calls within the same session (MCP connection, VS Code
+ * extension lifetime, or CLI watch mode).
+ *
+ * Avoids redundant framework detection, capability scanning, and feedback
+ * loading. Tracks verdict evolution per file for stability detection.
+ */
+import { contentHash } from "./cache.js";
+/**
+ * An evaluation session that accumulates project knowledge across calls.
+ */
+export class EvaluationSession {
+    ctx;
+    constructor() {
+        this.ctx = {
+            frameworks: [],
+            capabilities: new Set(),
+            verdictHistory: new Map(),
+            evaluatedFiles: new Map(),
+            startedAt: new Date().toISOString(),
+            evaluationCount: 0,
+            feedbackTally: new Map(),
+        };
+    }
+    /** Get the current session context (read-only snapshot). */
+    getContext() {
+        return this.ctx;
+    }
+    /** Number of evaluations performed. */
+    get evaluationCount() {
+        return this.ctx.evaluationCount;
+    }
+    /** Record detected frameworks (deduplicated). */
+    addFrameworks(frameworks) {
+        const existing = new Set(this.ctx.frameworks);
+        for (const fw of frameworks) {
+            if (!existing.has(fw)) {
+                this.ctx.frameworks.push(fw);
+                existing.add(fw);
+            }
+        }
+    }
+    /** Record detected project capabilities (e.g. "rate-limiting", "auth"). */
+    addCapabilities(caps) {
+        for (const cap of caps) {
+            this.ctx.capabilities.add(cap);
+        }
+    }
+    /** Get accumulated capabilities for absence-based finding suppression. */
+    getCapabilities() {
+        return this.ctx.capabilities;
+    }
+    /**
+     * Record an evaluation result for a file. Tracks verdict history
+     * so repeated evaluations can detect stability (converging scores).
+     */
+    recordEvaluation(filePath, code, verdict) {
+        this.ctx.evaluationCount++;
+        const hash = contentHash(code, filePath);
+        this.ctx.evaluatedFiles.set(hash, filePath);
+        const history = this.ctx.verdictHistory.get(filePath) ?? [];
+        history.push({
+            score: verdict.overallScore,
+            findingCount: verdict.findings.length,
+            timestamp: verdict.timestamp,
+        });
+        // Keep last 10 evaluations per file
+        if (history.length > 10)
+            history.shift();
+        this.ctx.verdictHistory.set(filePath, history);
+    }
+    /**
+     * Check if a file's verdict is stable — same score and finding count
+     * across the last N evaluations. Returns true if stable (skip re-eval).
+     */
+    isVerdictStable(filePath, minRuns = 3) {
+        const history = this.ctx.verdictHistory.get(filePath);
+        if (!history || history.length < minRuns)
+            return false;
+        const recent = history.slice(-minRuns);
+        const firstScore = recent[0].score;
+        const firstCount = recent[0].findingCount;
+        return recent.every((h) => h.score === firstScore && h.findingCount === firstCount);
+    }
+    /**
+     * Check if a file has already been evaluated with the same content.
+     */
+    hasEvaluated(filePath, code) {
+        const hash = contentHash(code, filePath);
+        return this.ctx.evaluatedFiles.has(hash);
+    }
+    /**
+     * Get verdict history for a file — most recent first.
+     */
+    getVerdictHistory(filePath) {
+        return [...(this.ctx.verdictHistory.get(filePath) ?? [])].reverse();
+    }
+    /** Reset the session (clear all accumulated context). */
+    reset() {
+        this.ctx = {
+            frameworks: [],
+            capabilities: new Set(),
+            verdictHistory: new Map(),
+            evaluatedFiles: new Map(),
+            startedAt: new Date().toISOString(),
+            evaluationCount: 0,
+            feedbackTally: new Map(),
+        };
+    }
+    /**
+     * Record user feedback for a finding rule.
+     * tp = true positive, fp = false positive, wontfix = acknowledged but skipped.
+     */
+    recordFeedback(ruleId, verdict) {
+        const existing = this.ctx.feedbackTally.get(ruleId) ?? { tp: 0, fp: 0, wontfix: 0 };
+        existing[verdict]++;
+        this.ctx.feedbackTally.set(ruleId, existing);
+    }
+    /**
+     * Get a confidence penalty for a rule based on accumulated FP feedback.
+     * Returns a multiplier in (0, 1] — 1.0 means no penalty, lower means
+     * the rule has been flagged as FP frequently and confidence should be reduced.
+     *
+     * Formula: 1 / (1 + fpCount) — degrades smoothly as FP reports accumulate.
+     * A single FP report halves confidence; two reports reduce it to 1/3, etc.
+     */
+    getConfidencePenalty(ruleId) {
+        const tally = this.ctx.feedbackTally.get(ruleId);
+        if (!tally || tally.fp === 0)
+            return 1.0;
+        return 1 / (1 + tally.fp);
+    }
+    /** Get the raw feedback tally for all rules. */
+    getFeedbackTally() {
+        return this.ctx.feedbackTally;
+    }
+}
+// ─── Singleton for MCP Server / Extension lifetime ──────────────────────────
+let _globalSession;
+/** Get or create the global evaluation session (shared across MCP calls). */
+export function getGlobalSession() {
+    if (!_globalSession) {
+        _globalSession = new EvaluationSession();
+    }
+    return _globalSession;
+}
+/** Reset the global session (for testing or explicit reset). */
+export function resetGlobalSession() {
+    _globalSession?.reset();
+    _globalSession = undefined;
+}

package/dist/evaluators/index.d.ts

@@ -1,4 +1,4 @@
-import type { JudgeDefinition, JudgeEvaluation, TribunalVerdict, ProjectVerdict, DiffVerdict, Finding, MustFixGateOptions, JudgesConfig, SuppressionResult } from "../types.js";
+import type { JudgeDefinition, JudgeEvaluation, TribunalVerdict, ProjectVerdict, DiffVerdict, Finding, MustFixGateOptions, JudgesConfig, SuppressionResult, StreamingBatch } from "../types.js";
 import type { CodeStructure } from "../ast/types.js";
 import type { TaintFlow } from "../ast/taint-tracker.js";
 import { formatVerdictAsMarkdown, formatEvaluationAsMarkdown } from "./shared.js";
@@ -53,6 +53,12 @@ export interface EvaluationOptions {
      * Generated by `scanProjectCapabilities()` from the project evaluator.
      */
     projectCapabilities?: Set<string>;
+    /**
+     * Enable adaptive judge selection — automatically skip judges that are
+     * irrelevant to the file's language, framework, or role. Reduces noise
+     * and improves performance. Defaults to false (run all judges).
+     */
+    adaptiveSelection?: boolean;
     /** @internal — pre-computed AST structure for the file (set by evaluateWithTribunal) */
     _astCache?: CodeStructure;
     /** @internal — pre-computed taint flows for the file (set by evaluateWithTribunal) */
@@ -88,6 +94,22 @@ export declare function evaluateWithJudge(judge: JudgeDefinition, code: string,
  * Run the full tribunal — all judges evaluate the code.
  */
 export declare function evaluateWithTribunal(code: string, language: string, context?: string, options?: EvaluationOptions): TribunalVerdict;
+/**
+ * Streaming tribunal evaluation — yields per-judge results as each judge
+ * completes, enabling progressive UI updates and early termination.
+ *
+ * Each yielded `StreamingBatch` contains the judge evaluation, execution
+ * trace, and running aggregate statistics.
+ *
+ * Usage:
+ * ```ts
+ * for await (const batch of evaluateWithTribunalStreaming(code, lang)) {
+ *   console.log(`${batch.judgeName}: ${batch.evaluation.findings.length} findings`);
+ *   if (batch.aggregate.criticalSoFar > 10) break; // early termination
+ * }
+ * ```
+ */
+export declare function evaluateWithTribunalStreaming(code: string, language: string, context?: string, options?: EvaluationOptions): AsyncGenerator<StreamingBatch>;
 export { scanProjectWideSecurityPatterns } from "./project.js";
 export declare function evaluateProject(files: Array<{
     path: string;