@kevinrabun/judges-cli 3.124.0

package/dist/commands/migration-safety.js

@@ -0,0 +1,256 @@
+/**
+ * Migration safety — validate framework/language migration PRs for
+ * compatibility gaps, data-loss risks, and breaking changes.
+ */
+import { readFileSync, readdirSync, statSync } from "fs";
+import { join, extname } from "path";
+// ─── File Collection ────────────────────────────────────────────────────────
+const CODE_EXTS = new Set([
+    ".ts",
+    ".tsx",
+    ".js",
+    ".jsx",
+    ".mjs",
+    ".cjs",
+    ".py",
+    ".java",
+    ".go",
+    ".rs",
+    ".json",
+    ".yaml",
+    ".yml",
+]);
+function collectFiles(dir, max = 300) {
+    const files = [];
+    function walk(d) {
+        if (files.length >= max)
+            return;
+        let entries;
+        try {
+            entries = readdirSync(d);
+        }
+        catch {
+            return;
+        }
+        for (const e of entries) {
+            if (files.length >= max)
+                return;
+            if (e.startsWith(".") || e === "node_modules" || e === "dist" || e === "build")
+                continue;
+            const full = join(d, e);
+            try {
+                if (statSync(full).isDirectory())
+                    walk(full);
+                else if (CODE_EXTS.has(extname(full)))
+                    files.push(full);
+            }
+            catch {
+                /* skip */
+            }
+        }
+    }
+    walk(dir);
+    return files;
+}
+// ─── Analysis ───────────────────────────────────────────────────────────────
+const MIGRATION_PATTERNS = [
+    // Framework migrations
+    {
+        pattern: /(?:from|import)\s+['"]react-router(?:-dom)?['"]/i,
+        risk: "React Router migration",
+        severity: "high",
+        detail: "React Router v5→v6: <Switch> replaced by <Routes>, useHistory→useNavigate",
+    },
+    {
+        pattern: /(?:from|import)\s+['"]@angular\/(?:core|common)['"]/i,
+        risk: "Angular version migration",
+        severity: "high",
+        detail: "Angular major versions change DI, module system, and template syntax",
+    },
+    {
+        pattern: /(?:from|import)\s+['"]vue['"]/i,
+        risk: "Vue migration",
+        severity: "high",
+        detail: "Vue 2→3: Options API defaults change, Composition API, Proxy-based reactivity",
+    },
+    {
+        pattern: /(?:from|import)\s+['"]express['"].*(?:from|import)\s+['"]fastify['"]/is,
+        risk: "Express→Fastify migration",
+        severity: "high",
+        detail: "Middleware model differs — req/res API changes, plugin system replaces middleware chain",
+    },
+    // TypeScript/JavaScript evolution
+    {
+        pattern: /require\s*\(.*\).*(?:import|from).*['"]/i,
+        risk: "CJS/ESM mixed imports",
+        severity: "medium",
+        detail: "Mixed require() and import — choose one module system",
+    },
+    {
+        pattern: /"type"\s*:\s*"module"/i,
+        risk: "ESM migration",
+        severity: "medium",
+        detail: "ESM migration: ensure all imports have extensions, __dirname unavailable",
+    },
+    // Database migrations
+    {
+        pattern: /(?:ALTER|DROP)\s+(?:TABLE|COLUMN|INDEX)/i,
+        risk: "Schema migration",
+        severity: "critical",
+        detail: "Schema changes may lose data — run on copy first, verify rollback",
+    },
+    {
+        pattern: /(?:mongoose|sequelize|typeorm|prisma).*(?:migrate|migration)/i,
+        risk: "ORM migration",
+        severity: "high",
+        detail: "ORM migrations may have different default behaviors across versions",
+    },
+    // Config format changes
+    {
+        pattern: /(?:webpack|rollup|vite|esbuild|parcel).*(?:config|\.config)/i,
+        risk: "Bundler migration",
+        severity: "medium",
+        detail: "Bundler configs are not portable — verify plugin equivalents",
+    },
+    // API surface changes
+    {
+        pattern: /deprecated|@deprecated|TODO.*migrate|FIXME.*upgrade/i,
+        risk: "Deprecated API usage",
+        severity: "medium",
+        detail: "Deprecated APIs may be removed in next version — update now",
+    },
+    // Runtime changes
+    {
+        pattern: /engines.*node.*['"]\d+/i,
+        risk: "Node.js version requirement",
+        severity: "high",
+        detail: "Node version changes affect API availability, performance, and security",
+    },
+    {
+        pattern: /python_requires.*['"]\s*>=?\s*3\./i,
+        risk: "Python version requirement",
+        severity: "high",
+        detail: "Python version changes affect syntax, stdlib, and type hints",
+    },
+    // Data serialization changes
+    {
+        pattern: /(?:protobuf|proto3|avro|thrift).*(?:schema|definition)/i,
+        risk: "Serialization schema change",
+        severity: "critical",
+        detail: "Schema changes can break wire compatibility — ensure backward compat",
+    },
+    // Authentication changes
+    {
+        pattern: /(?:passport|auth0|firebase[.-]auth|cognito).*(?:migrate|upgrade|v\d)/i,
+        risk: "Auth provider migration",
+        severity: "critical",
+        detail: "Auth migrations risk locking out users — run parallel during transition",
+    },
+];
+function analyzeFile(filepath) {
+    const risks = [];
+    let content;
+    try {
+        content = readFileSync(filepath, "utf-8");
+    }
+    catch {
+        return risks;
+    }
+    const lines = content.split("\n");
+    for (let i = 0; i < lines.length; i++) {
+        for (const mp of MIGRATION_PATTERNS) {
+            if (mp.pattern.test(lines[i])) {
+                risks.push({ file: filepath, line: i + 1, risk: mp.risk, severity: mp.severity, detail: mp.detail });
+                break; // One finding per line
+            }
+        }
+    }
+    return risks;
+}
+function detectMigrationContext(dir) {
+    const signals = [];
+    const files = collectFiles(dir, 50);
+    const allContent = files
+        .map((f) => {
+        try {
+            return readFileSync(f, "utf-8");
+        }
+        catch {
+            return "";
+        }
+    })
+        .join("\n");
+    if (/codemod|jscodeshift|ast-grep/i.test(allContent))
+        signals.push("Codemod tooling detected — automated migration in progress");
+    if (/compatibility|compat|shim|polyfill/i.test(allContent))
+        signals.push("Compatibility layer/shim detected");
+    if (/canary|feature.?flag|gradual.?rollout/i.test(allContent))
+        signals.push("Gradual rollout strategy detected");
+    if (/(?:upgrade|migration).?guide|MIGRATION/i.test(allContent))
+        signals.push("Migration guide referenced");
+    return signals;
+}
+// ─── CLI ────────────────────────────────────────────────────────────────────
+export function runMigrationSafety(argv) {
+    if (argv.includes("--help") || argv.includes("-h")) {
+        console.log(`
+judges migration-safety — Validate migration PRs for compatibility and data-loss risks
+
+Usage:
+  judges migration-safety [dir]
+  judges migration-safety src/ --format json
+
+Options:
+  [dir]                 Directory to scan (default: .)
+  --format json         JSON output
+  --help, -h            Show this help
+
+Detects: framework version breaks, CJS/ESM mixed imports, schema migrations,
+bundler config changes, deprecated APIs, runtime version changes, serialization
+schema breaks, auth provider migrations.
+`);
+        return;
+    }
+    const format = argv.find((_a, i) => argv[i - 1] === "--format") || "text";
+    const dir = argv.find((a) => !a.startsWith("-") && argv.indexOf(a) > 0) || ".";
+    const files = collectFiles(dir);
+    const allRisks = [];
+    for (const f of files)
+        allRisks.push(...analyzeFile(f));
+    const signals = detectMigrationContext(dir);
+    const critCount = allRisks.filter((r) => r.severity === "critical").length;
+    const highCount = allRisks.filter((r) => r.severity === "high").length;
+    const score = allRisks.length === 0 ? 100 : Math.max(0, 100 - critCount * 20 - highCount * 8);
+    if (format === "json") {
+        console.log(JSON.stringify({
+            risks: allRisks,
+            signals,
+            score,
+            summary: { critical: critCount, high: highCount, total: allRisks.length },
+            timestamp: new Date().toISOString(),
+        }, null, 2));
+    }
+    else {
+        const badge = critCount > 0 ? "🚫 HIGH RISK" : highCount > 2 ? "⚠️  CAUTION" : "✅ SAFE";
+        console.log(`\n  Migration Safety: ${badge} (${score}/100)\n  ─────────────────────────────`);
+        if (signals.length > 0) {
+            console.log("\n    Context:");
+            for (const s of signals)
+                console.log(`      📌 ${s}`);
+        }
+        if (allRisks.length === 0) {
+            console.log("\n    No migration risks detected.\n");
+            return;
+        }
+        console.log("\n    Risks:");
+        for (const r of allRisks.slice(0, 25)) {
+            const icon = r.severity === "critical" ? "🚫" : r.severity === "high" ? "🔴" : "🟡";
+            console.log(`      ${icon} [${r.severity.toUpperCase()}] ${r.risk}`);
+            console.log(`          ${r.file}:${r.line}`);
+            console.log(`          ${r.detail}`);
+        }
+        if (allRisks.length > 25)
+            console.log(`      ... and ${allRisks.length - 25} more`);
+        console.log(`\n    Total: ${allRisks.length} | Critical: ${critCount} | High: ${highCount} | Score: ${score}/100\n`);
+    }
+}

package/dist/commands/model-report.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Model report — track which AI model/version produced each finding
+ * across evaluation runs. Generates model scorecards with pass rates,
+ * failure categories, and vendor-specific blind spots.
+ *
+ * All data stored locally in `.judges-model-reports/`.
+ */
+export declare function runModelReport(argv: string[]): void;

package/dist/commands/model-report.js

@@ -0,0 +1,194 @@
+/**
+ * Model report — track which AI model/version produced each finding
+ * across evaluation runs. Generates model scorecards with pass rates,
+ * failure categories, and vendor-specific blind spots.
+ *
+ * All data stored locally in `.judges-model-reports/`.
+ */
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from "fs";
+import { join } from "path";
+// ─── Storage ────────────────────────────────────────────────────────────────
+const DATA_DIR = ".judges-model-reports";
+function ensureDir() {
+    if (!existsSync(DATA_DIR))
+        mkdirSync(DATA_DIR, { recursive: true });
+}
+function loadEntries() {
+    const file = join(DATA_DIR, "history.json");
+    if (!existsSync(file))
+        return [];
+    try {
+        return JSON.parse(readFileSync(file, "utf-8"));
+    }
+    catch {
+        return [];
+    }
+}
+function saveEntries(entries) {
+    ensureDir();
+    writeFileSync(join(DATA_DIR, "history.json"), JSON.stringify(entries, null, 2));
+}
+// ─── Analysis ───────────────────────────────────────────────────────────────
+function categorizeFindings(content) {
+    const cats = {};
+    const patterns = [
+        ["security", /injection|xss|csrf|auth|password|secret|token|key/gi],
+        ["error-handling", /catch|error|exception|throw|reject|finally/gi],
+        ["performance", /loop|recursion|allocation|memory|cache|lazy|eager/gi],
+        ["naming", /naming|variable|identifier|abbreviation|convention/gi],
+        ["complexity", /complex|nested|depth|cyclomatic|conditional/gi],
+        ["duplication", /duplicate|copy|clone|repeat|redundant/gi],
+        ["documentation", /comment|doc|readme|jsdoc|annotation/gi],
+        ["testing", /test|assert|mock|stub|coverage|spec/gi],
+    ];
+    for (const [cat, re] of patterns) {
+        const matches = (content.match(re) || []).length;
+        if (matches > 0)
+            cats[cat] = matches;
+    }
+    return cats;
+}
+function generateReports(entries) {
+    const models = new Map();
+    for (const entry of entries) {
+        const list = models.get(entry.model) || [];
+        list.push(entry);
+        models.set(entry.model, list);
+    }
+    const reports = [];
+    for (const [model, evalList] of models) {
+        const avgScore = Math.round(evalList.reduce((s, e) => s + e.score, 0) / evalList.length);
+        const avgFindings = Math.round(evalList.reduce((s, e) => s + e.totalFindings, 0) / evalList.length);
+        // Aggregate categories
+        const catTotals = {};
+        for (const e of evalList) {
+            for (const [cat, count] of Object.entries(e.categories)) {
+                catTotals[cat] = (catTotals[cat] || 0) + count;
+            }
+        }
+        const topCategories = Object.entries(catTotals)
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, 5)
+            .map(([category, count]) => ({ category, count }));
+        // Trend
+        let trend = "stable";
+        if (evalList.length >= 3) {
+            const recent = evalList.slice(-3);
+            const scoreDiff = recent[recent.length - 1].score - recent[0].score;
+            if (scoreDiff > 5)
+                trend = "improving";
+            else if (scoreDiff < -5)
+                trend = "declining";
+        }
+        reports.push({ model, evaluations: evalList.length, avgScore, avgFindings, topCategories, trend });
+    }
+    return reports.sort((a, b) => b.avgScore - a.avgScore);
+}
+// ─── CLI ────────────────────────────────────────────────────────────────────
+export function runModelReport(argv) {
+    if (argv.includes("--help") || argv.includes("-h")) {
+        console.log(`
+judges model-report — AI model scorecard and comparison
+
+Usage:
+  judges model-report --record --model "gpt-4" --score 85 --findings 3
+  judges model-report --record --model "claude-4" --file eval-output.json
+  judges model-report --show
+  judges model-report --compare
+
+Options:
+  --record              Record a new evaluation entry
+  --model <name>        Model name/version
+  --score <n>           Evaluation score (0-100)
+  --findings <n>        Number of findings
+  --file <path>         Import findings from evaluation output
+  --show                Display model scorecards
+  --compare             Side-by-side model comparison
+  --format json         JSON output
+  --help, -h            Show this help
+`);
+        return;
+    }
+    const format = argv.find((_a, i) => argv[i - 1] === "--format") || "text";
+    const record = argv.includes("--record");
+    const show = argv.includes("--show");
+    const compare = argv.includes("--compare");
+    const modelName = argv.find((_a, i) => argv[i - 1] === "--model") || "";
+    const scoreArg = argv.find((_a, i) => argv[i - 1] === "--score");
+    const findingsArg = argv.find((_a, i) => argv[i - 1] === "--findings");
+    const fileArg = argv.find((_a, i) => argv[i - 1] === "--file");
+    if (record) {
+        if (!modelName) {
+            console.error("  --model is required for --record");
+            return;
+        }
+        let totalFindings = parseInt(findingsArg || "0");
+        let score = parseInt(scoreArg || "0");
+        let categories = {};
+        if (fileArg && existsSync(fileArg)) {
+            try {
+                const data = readFileSync(fileArg, "utf-8");
+                categories = categorizeFindings(data);
+                const parsed = JSON.parse(data);
+                if (parsed.findings)
+                    totalFindings = Array.isArray(parsed.findings) ? parsed.findings.length : totalFindings;
+                if (parsed.overallScore)
+                    score = parsed.overallScore;
+            }
+            catch {
+                /* use manual values */
+            }
+        }
+        const entry = {
+            model: modelName,
+            timestamp: new Date().toISOString(),
+            totalFindings,
+            criticalCount: 0,
+            highCount: 0,
+            mediumCount: Math.round(totalFindings * 0.6),
+            lowCount: Math.round(totalFindings * 0.4),
+            score,
+            categories,
+        };
+        const entries = loadEntries();
+        entries.push(entry);
+        saveEntries(entries);
+        console.log(`  ✅ Recorded evaluation for ${modelName} (score: ${score}, findings: ${totalFindings})`);
+        return;
+    }
+    const entries = loadEntries();
+    if (entries.length === 0) {
+        console.log("  No model data recorded yet. Use --record to add evaluations.");
+        return;
+    }
+    const reports = generateReports(entries);
+    if (format === "json") {
+        console.log(JSON.stringify({ reports, totalEntries: entries.length, timestamp: new Date().toISOString() }, null, 2));
+        return;
+    }
+    if (compare) {
+        console.log(`\n  Model Comparison — ${reports.length} models\n  ──────────────────────────`);
+        console.log(`    ${"Model".padEnd(25)} ${"Evals".padEnd(8)} ${"Avg Score".padEnd(12)} ${"Avg Findings".padEnd(14)} Trend`);
+        console.log(`    ${"─".repeat(25)} ${"─".repeat(8)} ${"─".repeat(12)} ${"─".repeat(14)} ${"─".repeat(10)}`);
+        for (const r of reports) {
+            const trendIcon = r.trend === "improving" ? "📈" : r.trend === "declining" ? "📉" : "➡️";
+            console.log(`    ${r.model.padEnd(25)} ${String(r.evaluations).padEnd(8)} ${String(r.avgScore).padEnd(12)} ${String(r.avgFindings).padEnd(14)} ${trendIcon} ${r.trend}`);
+        }
+        console.log("");
+        return;
+    }
+    // Show (default)
+    if (show || !compare) {
+        console.log(`\n  Model Report Cards — ${entries.length} evaluations across ${reports.length} models\n  ──────────────────────────`);
+        for (const r of reports) {
+            const icon = r.avgScore >= 80 ? "🟢" : r.avgScore >= 60 ? "🟡" : "🔴";
+            const trendIcon = r.trend === "improving" ? "📈" : r.trend === "declining" ? "📉" : "➡️";
+            console.log(`\n    ${icon} ${r.model}`);
+            console.log(`        Score: ${r.avgScore}/100  |  Findings: ${r.avgFindings} avg  |  Evals: ${r.evaluations}  |  ${trendIcon} ${r.trend}`);
+            if (r.topCategories.length > 0) {
+                console.log(`        Top issues: ${r.topCategories.map((c) => `${c.category}(${c.count})`).join(", ")}`);
+            }
+        }
+        console.log("");
+    }
+}

package/dist/commands/model-risk.d.ts

@@ -0,0 +1,27 @@
+/**
+ * AI model risk profile — analyze which vulnerability patterns different
+ * AI code generation models tend to introduce.
+ *
+ * Uses local benchmark and scan history data.
+ */
+interface ModelProfile {
+    model: string;
+    language?: string;
+    topVulnerabilities: {
+        rulePrefix: string;
+        category: string;
+        frequency: number;
+        avgSeverity: string;
+    }[];
+    riskScore: number;
+    recommendedJudges: string[];
+    suggestedOverrides: {
+        ruleId: string;
+        action: string;
+        reason: string;
+    }[];
+    sampleSize: number;
+}
+export declare function getModelRisk(model: string, language?: string): ModelProfile;
+export declare function runModelRisk(argv: string[]): void;
+export {};