@kevinrabun/judges 3.115.0 → 3.115.2

package/dist/commands/sla-track.js

@@ -0,0 +1,268 @@
+/**
+ * SLA tracking — define response-time SLAs per severity and track
+ * violation rates across runs.
+ *
+ * Data is stored locally in .judges-sla.json — no remote storage.
+ */
+import { readFileSync, writeFileSync, existsSync } from "fs";
+// ─── Default SLA Policies ───────────────────────────────────────────────────
+const DEFAULT_POLICIES = [
+    { severity: "critical", responseHours: 4, resolutionHours: 24 },
+    { severity: "high", responseHours: 24, resolutionHours: 72 },
+    { severity: "medium", responseHours: 72, resolutionHours: 168 },
+    { severity: "low", responseHours: 168, resolutionHours: 720 },
+    { severity: "info", responseHours: 720, resolutionHours: 2160 },
+];
+const SLA_FILE = ".judges-sla.json";
+// ─── Core Functions ─────────────────────────────────────────────────────────
+function loadDb(file) {
+    if (!existsSync(file))
+        return { policies: DEFAULT_POLICIES, entries: [] };
+    return JSON.parse(readFileSync(file, "utf-8"));
+}
+function saveDb(file, db) {
+    writeFileSync(file, JSON.stringify(db, null, 2));
+}
+function findingId(f) {
+    return `${f.ruleId}:${f.title}`;
+}
+export function trackFindings(findings, dbPath = SLA_FILE) {
+    const db = loadDb(dbPath);
+    const now = new Date().toISOString();
+    const seen = new Set(db.entries.map((e) => e.findingId));
+    for (const f of findings) {
+        const id = findingId(f);
+        if (!seen.has(id)) {
+            db.entries.push({
+                findingId: id,
+                ruleId: f.ruleId,
+                severity: f.severity,
+                title: f.title,
+                firstSeenIso: now,
+                status: "open",
+            });
+            seen.add(id);
+        }
+    }
+    saveDb(dbPath, db);
+    return db;
+}
+export function triageEntry(id, dbPath = SLA_FILE) {
+    const db = loadDb(dbPath);
+    const entry = db.entries.find((e) => e.findingId === id);
+    if (!entry)
+        throw new Error(`Entry not found: ${id}`);
+    entry.triagedIso = new Date().toISOString();
+    entry.status = "triaged";
+    saveDb(dbPath, db);
+}
+export function resolveEntry(id, dbPath = SLA_FILE) {
+    const db = loadDb(dbPath);
+    const entry = db.entries.find((e) => e.findingId === id);
+    if (!entry)
+        throw new Error(`Entry not found: ${id}`);
+    entry.resolvedIso = new Date().toISOString();
+    entry.status = "resolved";
+    saveDb(dbPath, db);
+}
+export function checkViolations(dbPath = SLA_FILE) {
+    const db = loadDb(dbPath);
+    const now = Date.now();
+    const violations = [];
+    for (const entry of db.entries) {
+        const policy = db.policies.find((p) => p.severity === entry.severity);
+        if (!policy)
+            continue;
+        const firstSeen = new Date(entry.firstSeenIso).getTime();
+        const elapsedMs = now - firstSeen;
+        const elapsedHours = Math.round((elapsedMs / 3_600_000) * 10) / 10;
+        if (entry.status === "open" && elapsedHours > policy.responseHours) {
+            violations.push({
+                findingId: entry.findingId,
+                ruleId: entry.ruleId,
+                severity: entry.severity,
+                type: "response",
+                elapsedHours,
+                allowedHours: policy.responseHours,
+            });
+        }
+        if (entry.status !== "resolved" && elapsedHours > policy.resolutionHours) {
+            violations.push({
+                findingId: entry.findingId,
+                ruleId: entry.ruleId,
+                severity: entry.severity,
+                type: "resolution",
+                elapsedHours,
+                allowedHours: policy.resolutionHours,
+            });
+        }
+    }
+    return violations;
+}
+export function getSlaStats(dbPath = SLA_FILE) {
+    const db = loadDb(dbPath);
+    const violations = checkViolations(dbPath);
+    const stats = {
+        total: db.entries.length,
+        open: db.entries.filter((e) => e.status === "open").length,
+        triaged: db.entries.filter((e) => e.status === "triaged").length,
+        resolved: db.entries.filter((e) => e.status === "resolved").length,
+        violations: violations.length,
+        bySeverity: {},
+        avgResponseHours: 0,
+        avgResolutionHours: 0,
+    };
+    for (const entry of db.entries) {
+        if (!stats.bySeverity[entry.severity]) {
+            stats.bySeverity[entry.severity] = { total: 0, open: 0, violations: 0 };
+        }
+        stats.bySeverity[entry.severity].total++;
+        if (entry.status === "open")
+            stats.bySeverity[entry.severity].open++;
+    }
+    for (const v of violations) {
+        if (stats.bySeverity[v.severity]) {
+            stats.bySeverity[v.severity].violations++;
+        }
+    }
+    const triaged = db.entries.filter((e) => e.triagedIso);
+    if (triaged.length > 0) {
+        const totalResponseMs = triaged.reduce((sum, e) => {
+            return sum + (new Date(e.triagedIso).getTime() - new Date(e.firstSeenIso).getTime());
+        }, 0);
+        stats.avgResponseHours = Math.round((totalResponseMs / triaged.length / 3_600_000) * 10) / 10;
+    }
+    const resolved = db.entries.filter((e) => e.resolvedIso);
+    if (resolved.length > 0) {
+        const totalResMs = resolved.reduce((sum, e) => {
+            return sum + (new Date(e.resolvedIso).getTime() - new Date(e.firstSeenIso).getTime());
+        }, 0);
+        stats.avgResolutionHours = Math.round((totalResMs / resolved.length / 3_600_000) * 10) / 10;
+    }
+    return stats;
+}
+// ─── CLI ────────────────────────────────────────────────────────────────────
+export async function runSlaTrack(argv) {
+    if (argv.includes("--help") || argv.includes("-h")) {
+        console.log(`
+judges sla-track — SLA tracking for security findings
+
+Usage:
+  judges sla-track --input results.json             Track findings from a results file
+  judges sla-track --check                           Check for SLA violations
+  judges sla-track --triage <finding-id>             Mark finding as triaged
+  judges sla-track --resolve <finding-id>            Mark finding as resolved
+  judges sla-track --stats                           Show SLA statistics
+  judges sla-track --set-policy <severity> <resp-h> <res-h>
+
+Options:
+  --input <path>        Results JSON to track
+  --check               Check for SLA violations
+  --triage <id>         Mark a finding as triaged
+  --resolve <id>        Resolve a finding
+  --stats               Show statistics
+  --set-policy          Set SLA for a severity level
+  --format json         JSON output
+  --help, -h            Show this help
+`);
+        return;
+    }
+    const format = argv.find((_a, i) => argv[i - 1] === "--format") || "text";
+    // Track findings from input file
+    const inputPath = argv.find((_a, i) => argv[i - 1] === "--input");
+    if (inputPath) {
+        if (!existsSync(inputPath)) {
+            console.error(`Error: file not found: ${inputPath}`);
+            process.exit(1);
+        }
+        const data = JSON.parse(readFileSync(inputPath, "utf-8"));
+        const findings = data.evaluations
+            ? data.evaluations.flatMap((e) => e.findings || [])
+            : data.findings || data;
+        const db = trackFindings(findings);
+        if (format === "json") {
+            console.log(JSON.stringify(db, null, 2));
+        }
+        else {
+            console.log(`\n  Tracked ${findings.length} findings (${db.entries.length} total in DB)\n`);
+        }
+        return;
+    }
+    // Triage
+    const triageTarget = argv.find((_a, i) => argv[i - 1] === "--triage");
+    if (triageTarget) {
+        triageEntry(triageTarget);
+        console.log(`  Triaged: ${triageTarget}`);
+        return;
+    }
+    // Resolve
+    const resolveTarget = argv.find((_a, i) => argv[i - 1] === "--resolve");
+    if (resolveTarget) {
+        resolveEntry(resolveTarget);
+        console.log(`  Resolved: ${resolveTarget}`);
+        return;
+    }
+    // Set policy
+    if (argv.includes("--set-policy")) {
+        const idx = argv.indexOf("--set-policy");
+        const severity = argv[idx + 1];
+        const respH = parseFloat(argv[idx + 2]);
+        const resH = parseFloat(argv[idx + 3]);
+        if (!severity || isNaN(respH) || isNaN(resH)) {
+            console.error("Error: --set-policy <severity> <response-hours> <resolution-hours>");
+            process.exit(1);
+        }
+        const db = loadDb(SLA_FILE);
+        const existing = db.policies.find((p) => p.severity === severity);
+        if (existing) {
+            existing.responseHours = respH;
+            existing.resolutionHours = resH;
+        }
+        else {
+            db.policies.push({ severity, responseHours: respH, resolutionHours: resH });
+        }
+        saveDb(SLA_FILE, db);
+        console.log(`  SLA policy set: ${severity} → response ${respH}h, resolution ${resH}h`);
+        return;
+    }
+    // Check violations
+    if (argv.includes("--check")) {
+        const violations = checkViolations();
+        if (format === "json") {
+            console.log(JSON.stringify(violations, null, 2));
+        }
+        else if (violations.length === 0) {
+            console.log("\n  ✅ No SLA violations\n");
+        }
+        else {
+            console.log(`\n  ⚠️  ${violations.length} SLA violation(s)\n`);
+            for (const v of violations) {
+                console.log(`    ${v.severity.padEnd(8)} ${v.type.padEnd(10)} ${v.ruleId} — ${v.elapsedHours}h / ${v.allowedHours}h allowed`);
+            }
+            console.log("");
+        }
+        return;
+    }
+    // Stats (default if --stats or no other flag)
+    const s = getSlaStats();
+    if (format === "json") {
+        console.log(JSON.stringify(s, null, 2));
+    }
+    else {
+        console.log(`
+  SLA Statistics
+  ──────────────────
+  Total tracked:       ${s.total}
+  Open:                ${s.open}
+  Triaged:             ${s.triaged}
+  Resolved:            ${s.resolved}
+  Violations:          ${s.violations}
+  Avg response time:   ${s.avgResponseHours}h
+  Avg resolution time: ${s.avgResolutionHours}h
+`);
+        for (const [sev, data] of Object.entries(s.bySeverity)) {
+            console.log(`    ${sev.padEnd(10)} ${data.total} total, ${data.open} open, ${data.violations} violations`);
+        }
+        console.log("");
+    }
+}

package/dist/commands/smart-select.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Smart judge selection — auto-select relevant judges based on file content.
+ *
+ * Avoids running irrelevant judges (e.g., SQL judge on .tsx files,
+ * IaC judge on .py files) to improve evaluation speed and reduce noise.
+ *
+ * Used internally by the evaluation pipeline when `smartSelect: true`.
+ */
+export interface JudgeRelevance {
+    judgeId: string;
+    relevant: boolean;
+    reason: string;
+}
+/**
+ * Select relevant judges for a given file based on its language and content.
+ * Returns the list of judge IDs that should be run.
+ */
+export declare function selectJudgesForFile(language: string, code: string, availableJudges?: string[]): JudgeRelevance[];
+/**
+ * Get just the relevant judge IDs for a file.
+ */
+export declare function getRelevantJudges(language: string, code: string, availableJudges?: string[]): string[];
+/**
+ * CLI: Show judge selection for a file.
+ */
+export declare function runSmartSelect(argv: string[]): void;

package/dist/commands/smart-select.js

@@ -0,0 +1,345 @@
+/**
+ * Smart judge selection — auto-select relevant judges based on file content.
+ *
+ * Avoids running irrelevant judges (e.g., SQL judge on .tsx files,
+ * IaC judge on .py files) to improve evaluation speed and reduce noise.
+ *
+ * Used internally by the evaluation pipeline when `smartSelect: true`.
+ */
+import { defaultRegistry } from "../judge-registry.js";
+// ─── Language → Judge Relevance ─────────────────────────────────────────────
+/** Judges that are always relevant regardless of language */
+const UNIVERSAL_JUDGES = new Set([
+    "code-structure",
+    "error-handling",
+    "documentation",
+    "maintainability",
+    "testing",
+    "logic-review",
+    "intent-alignment",
+    "ai-code-safety",
+    "software-practices",
+    "over-engineering",
+]);
+/** Judges relevant only for specific language families */
+const LANGUAGE_SPECIFIC = {
+    // Security judges — relevant for all code languages
+    cybersecurity: new Set([
+        "typescript",
+        "javascript",
+        "python",
+        "go",
+        "rust",
+        "java",
+        "csharp",
+        "ruby",
+        "php",
+        "swift",
+        "kotlin",
+        "c",
+        "cpp",
+        "dart",
+        "scala",
+    ]),
+    "data-security": new Set([
+        "typescript",
+        "javascript",
+        "python",
+        "go",
+        "rust",
+        "java",
+        "csharp",
+        "ruby",
+        "php",
+        "swift",
+        "kotlin",
+        "c",
+        "cpp",
+        "dart",
+        "scala",
+    ]),
+    authentication: new Set([
+        "typescript",
+        "javascript",
+        "python",
+        "go",
+        "java",
+        "csharp",
+        "ruby",
+        "php",
+        "kotlin",
+        "scala",
+    ]),
+    security: new Set([
+        "typescript",
+        "javascript",
+        "python",
+        "go",
+        "rust",
+        "java",
+        "csharp",
+        "ruby",
+        "php",
+        "swift",
+        "kotlin",
+        "c",
+        "cpp",
+        "dart",
+        "scala",
+    ]),
+    // Infrastructure judges
+    "iac-security": new Set(["terraform", "bicep", "arm", "yaml", "dockerfile"]),
+    "ci-cd": new Set(["yaml", "dockerfile", "bash", "powershell"]),
+    "cloud-readiness": new Set([
+        "typescript",
+        "javascript",
+        "python",
+        "go",
+        "java",
+        "csharp",
+        "yaml",
+        "terraform",
+        "dockerfile",
+    ]),
+    // Database judge
+    database: new Set(["typescript", "javascript", "python", "go", "java", "csharp", "ruby", "php", "sql", "kotlin"]),
+    // Performance judges
+    performance: new Set([
+        "typescript",
+        "javascript",
+        "python",
+        "go",
+        "rust",
+        "java",
+        "csharp",
+        "ruby",
+        "c",
+        "cpp",
+        "kotlin",
+        "scala",
+    ]),
+    scalability: new Set(["typescript", "javascript", "python", "go", "java", "csharp", "ruby", "kotlin", "scala"]),
+    caching: new Set(["typescript", "javascript", "python", "go", "java", "csharp", "ruby", "kotlin"]),
+    "cost-effectiveness": new Set(["typescript", "javascript", "python", "go", "java", "csharp", "yaml", "terraform"]),
+    // Reliability
+    reliability: new Set([
+        "typescript",
+        "javascript",
+        "python",
+        "go",
+        "rust",
+        "java",
+        "csharp",
+        "ruby",
+        "kotlin",
+        "scala",
+    ]),
+    observability: new Set(["typescript", "javascript", "python", "go", "java", "csharp", "kotlin"]),
+    "rate-limiting": new Set(["typescript", "javascript", "python", "go", "java", "csharp", "ruby", "kotlin"]),
+    // Compliance judges
+    compliance: new Set(["typescript", "javascript", "python", "go", "java", "csharp", "ruby", "php", "kotlin"]),
+    "data-sovereignty": new Set(["typescript", "javascript", "python", "go", "java", "csharp", "ruby", "kotlin"]),
+    // UX / Accessibility — only frontend
+    accessibility: new Set(["typescript", "javascript", "html"]),
+    ux: new Set(["typescript", "javascript", "html"]),
+    internationalization: new Set(["typescript", "javascript", "python", "java", "csharp", "ruby", "kotlin"]),
+    // Backwards compatibility
+    "backwards-compatibility": new Set(["typescript", "javascript", "python", "go", "java", "csharp", "ruby", "kotlin"]),
+    // API judges
+    "api-contract": new Set(["typescript", "javascript", "python", "go", "java", "csharp", "ruby", "kotlin", "scala"]),
+    "api-design": new Set(["typescript", "javascript", "python", "go", "java", "csharp", "ruby", "kotlin", "scala"]),
+    // Concurrency
+    concurrency: new Set([
+        "typescript",
+        "javascript",
+        "python",
+        "go",
+        "rust",
+        "java",
+        "csharp",
+        "c",
+        "cpp",
+        "kotlin",
+        "scala",
+    ]),
+    // Framework safety
+    "framework-safety": new Set(["typescript", "javascript", "python", "java", "csharp", "ruby", "kotlin"]),
+    // AI-specific
+    "hallucination-detection": new Set([
+        "typescript",
+        "javascript",
+        "python",
+        "go",
+        "rust",
+        "java",
+        "csharp",
+        "ruby",
+        "kotlin",
+    ]),
+    "model-fingerprint": new Set([
+        "typescript",
+        "javascript",
+        "python",
+        "go",
+        "rust",
+        "java",
+        "csharp",
+        "ruby",
+        "kotlin",
+    ]),
+    "agent-instructions": new Set(["typescript", "javascript", "python", "yaml"]),
+    "multi-turn-coherence": new Set(["typescript", "javascript", "python"]),
+    // Other
+    "ethics-bias": new Set(["typescript", "javascript", "python", "go", "java", "csharp", "ruby", "kotlin"]),
+    portability: new Set(["typescript", "javascript", "python", "go", "rust", "java", "csharp", "c", "cpp"]),
+    "logging-privacy": new Set(["typescript", "javascript", "python", "go", "java", "csharp", "ruby", "kotlin"]),
+    "configuration-management": new Set(["typescript", "javascript", "python", "go", "java", "csharp", "yaml", "json"]),
+    "dependency-health": new Set(["typescript", "javascript", "python", "go", "rust", "java", "csharp", "ruby"]),
+};
+/** Content-based signals that boost judge relevance */
+const CONTENT_SIGNALS = {
+    database: /\b(SELECT|INSERT|UPDATE|DELETE|CREATE TABLE|mongoose|prisma|typeorm|sequelize|knex|drizzle|sqlalchemy|django\.db|ActiveRecord)\b/i,
+    authentication: /\b(jwt|oauth|passport|bcrypt|argon2|session|cookie|token|login|signup|auth|credential|password)\b/i,
+    "rate-limiting": /\b(rate[_-]?limit|throttl|express[_-]?rate|ratelimit|bucket|leaky|sliding[_-]?window)\b/i,
+    caching: /\b(redis|memcache|cache|lru|ttl|invalidat|memoiz)\b/i,
+    "iac-security": /\b(resource|provider|module|azurerm|aws_|google_|terraform|apiVersion|kind:\s*Deployment)\b/i,
+    accessibility: /\b(aria-|role=|alt=|tabindex|a11y|wcag|screen[_-]?reader)\b/i,
+    internationalization: /\b(i18n|l10n|intl\.|gettext|ngettext|t\(|useTranslation|formatMessage)\b/i,
+    concurrency: /\b(mutex|semaphor|lock|atomic|channel|goroutine|thread|async|await|Promise\.all|worker|race condition)\b/i,
+};
+// ─── Selection Logic ────────────────────────────────────────────────────────
+/**
+ * Select relevant judges for a given file based on its language and content.
+ * Returns the list of judge IDs that should be run.
+ */
+export function selectJudgesForFile(language, code, availableJudges) {
+    const judges = availableJudges || defaultRegistry.getJudges().map((j) => j.id);
+    const results = [];
+    for (const judgeId of judges) {
+        // Universal judges are always relevant
+        if (UNIVERSAL_JUDGES.has(judgeId)) {
+            results.push({ judgeId, relevant: true, reason: "Universal judge" });
+            continue;
+        }
+        // Check language relevance
+        const langSet = LANGUAGE_SPECIFIC[judgeId];
+        if (langSet) {
+            if (langSet.has(language)) {
+                results.push({ judgeId, relevant: true, reason: `Relevant for ${language}` });
+            }
+            else {
+                // Check content-based override — maybe the code contains relevant patterns
+                const signal = CONTENT_SIGNALS[judgeId];
+                if (signal && signal.test(code)) {
+                    results.push({ judgeId, relevant: true, reason: `Content signal detected` });
+                }
+                else {
+                    results.push({ judgeId, relevant: false, reason: `Not relevant for ${language}` });
+                }
+            }
+            continue;
+        }
+        // Unknown judge — include it to be safe
+        results.push({ judgeId, relevant: true, reason: "Default include" });
+    }
+    return results;
+}
+/**
+ * Get just the relevant judge IDs for a file.
+ */
+export function getRelevantJudges(language, code, availableJudges) {
+    return selectJudgesForFile(language, code, availableJudges)
+        .filter((r) => r.relevant)
+        .map((r) => r.judgeId);
+}
+/**
+ * CLI: Show judge selection for a file.
+ */
+export function runSmartSelect(argv) {
+    if (argv.includes("--help") || argv.includes("-h")) {
+        console.log(`
+judges smart-select — Show which judges would run for a file
+
+Usage:
+  judges smart-select <file>              Show relevant judges
+  judges smart-select --language <lang>   Specify language
+  judges smart-select --all               Show all judges including skipped
+
+Options:
+  --language <lang>    Override detected language
+  --all                Show all judges (relevant + skipped)
+  --format json        JSON output
+  --help, -h           Show this help
+`);
+        return;
+    }
+    const { readFileSync } = require("fs");
+    const { extname } = require("path");
+    const file = argv.find((a, i) => i > 1 && !a.startsWith("-") && argv[i - 1] !== "--language" && argv[i - 1] !== "--format");
+    const langOverride = argv.find((_a, i) => argv[i - 1] === "--language");
+    const format = argv.find((_a, i) => argv[i - 1] === "--format") || "text";
+    const showAll = argv.includes("--all");
+    const EXT_MAP = {
+        ".ts": "typescript",
+        ".tsx": "typescript",
+        ".js": "javascript",
+        ".jsx": "javascript",
+        ".py": "python",
+        ".go": "go",
+        ".rs": "rust",
+        ".java": "java",
+        ".cs": "csharp",
+        ".rb": "ruby",
+        ".php": "php",
+        ".swift": "swift",
+        ".kt": "kotlin",
+        ".scala": "scala",
+        ".c": "c",
+        ".cpp": "cpp",
+        ".h": "c",
+        ".hpp": "cpp",
+        ".yaml": "yaml",
+        ".yml": "yaml",
+        ".tf": "terraform",
+        ".bicep": "bicep",
+        ".sql": "sql",
+        ".sh": "bash",
+        ".ps1": "powershell",
+        ".dockerfile": "dockerfile",
+    };
+    let language = langOverride || "typescript";
+    let code = "";
+    if (file) {
+        try {
+            code = readFileSync(file, "utf-8");
+        }
+        catch {
+            /* use empty */
+        }
+        if (!langOverride) {
+            const ext = extname(file).toLowerCase();
+            language = EXT_MAP[ext] || "typescript";
+            if (file.toLowerCase().includes("dockerfile"))
+                language = "dockerfile";
+        }
+    }
+    const results = selectJudgesForFile(language, code);
+    if (format === "json") {
+        console.log(JSON.stringify({ language, results: showAll ? results : results.filter((r) => r.relevant) }, null, 2));
+        return;
+    }
+    const relevant = results.filter((r) => r.relevant);
+    const skipped = results.filter((r) => !r.relevant);
+    console.log(`\n  Smart Judge Selection — ${language}\n`);
+    console.log(`  Relevant judges (${relevant.length}):`);
+    for (const r of relevant) {
+        console.log(`    ✅ ${r.judgeId.padEnd(30)} ${r.reason}`);
+    }
+    if (showAll && skipped.length > 0) {
+        console.log(`\n  Skipped judges (${skipped.length}):`);
+        for (const r of skipped) {
+            console.log(`    ⏭️  ${r.judgeId.padEnd(30)} ${r.reason}`);
+        }
+    }
+    console.log(`\n  ${relevant.length}/${results.length} judges selected (${skipped.length} skipped)\n`);
+}

package/dist/commands/snippet-eval.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Snippet eval — evaluate a code snippet from stdin or a string
+ * argument without needing a file, with instant formatted output.
+ *
+ * Zero-friction entry point for evaluating AI-generated code snippets.
+ */
+export declare function runSnippetEval(argv: string[]): void;