@kevinrabun/judges-cli 3.124.0

package/dist/comparison.js

@@ -0,0 +1,253 @@
+/**
+ * Comparison Benchmarks — Compare judges against other code review tools
+ *
+ * Provides structured comparison data showing how judges performs relative to
+ * ESLint, SonarQube, Semgrep, CodeQL, and Bandit across multiple dimensions.
+ *
+ * This is data-driven: real comparisons require running the other tools,
+ * but we provide the framework and known capability matrices.
+ */
+// ─── Tool Profiles ───────────────────────────────────────────────────────────
+export const TOOL_PROFILES = [
+    {
+        name: "ESLint",
+        type: "linter",
+        languages: ["javascript", "typescript"],
+        categories: ["code-style", "best-practices", "potential-errors"],
+        strengths: ["Massive plugin ecosystem", "Auto-fixable rules", "Deep JS/TS knowledge", "IDE integration"],
+        weaknesses: ["JS/TS only", "No security focus by default", "No architectural analysis", "No AI-code detection"],
+        pricing: "Free (OSS)",
+    },
+    {
+        name: "SonarQube",
+        type: "sast",
+        languages: ["javascript", "typescript", "python", "java", "csharp", "go", "rust", "cpp"],
+        categories: ["bugs", "vulnerabilities", "code-smells", "security-hotspots"],
+        strengths: ["Multi-language", "Quality gates", "Historical tracking", "Enterprise features"],
+        weaknesses: ["Heavy infrastructure", "Slow analysis", "No AI-code detection", "Complex setup"],
+        pricing: "Free (Community) / $150+/mo (Enterprise)",
+    },
+    {
+        name: "Semgrep",
+        type: "sast",
+        languages: ["javascript", "typescript", "python", "java", "go", "rust", "ruby"],
+        categories: ["security", "best-practices", "correctness"],
+        strengths: ["Pattern-based rules", "Fast analysis", "Custom rules easy to write", "Good security coverage"],
+        weaknesses: [
+            "Limited architectural analysis",
+            "No AI-code detection",
+            "Cloud rules require account",
+            "No code review personas",
+        ],
+        pricing: "Free (OSS) / Team plans",
+    },
+    {
+        name: "CodeQL",
+        type: "sast",
+        languages: ["javascript", "typescript", "python", "java", "csharp", "go", "cpp", "ruby"],
+        categories: ["security", "correctness", "data-flow"],
+        strengths: [
+            "Deep data-flow analysis",
+            "GitHub-native integration",
+            "Powerful query language",
+            "Excellent security coverage",
+        ],
+        weaknesses: [
+            "Requires compilation for some languages",
+            "Complex query authoring",
+            "Slow analysis",
+            "No architectural/design review",
+        ],
+        pricing: "Free for OSS / GitHub Advanced Security",
+    },
+    {
+        name: "Bandit",
+        type: "sast",
+        languages: ["python"],
+        categories: ["security"],
+        strengths: ["Python-focused security", "Fast", "Simple setup", "CI-friendly"],
+        weaknesses: ["Python only", "Limited to security", "No architecture analysis", "High false-positive rate"],
+        pricing: "Free (OSS)",
+    },
+];
+// ─── Capability Matrix ───────────────────────────────────────────────────────
+/**
+ * The 10 trust dimensions that judges covers, mapped against other tools.
+ */
+export const CAPABILITY_MATRIX = [
+    // Security
+    { tool: "judges", category: "Security (OWASP Top 10)", coverage: "full" },
+    { tool: "ESLint", category: "Security (OWASP Top 10)", coverage: "partial", notes: "With eslint-plugin-security" },
+    { tool: "SonarQube", category: "Security (OWASP Top 10)", coverage: "full" },
+    { tool: "Semgrep", category: "Security (OWASP Top 10)", coverage: "full" },
+    { tool: "CodeQL", category: "Security (OWASP Top 10)", coverage: "full" },
+    { tool: "Bandit", category: "Security (OWASP Top 10)", coverage: "partial", notes: "Python only" },
+    // Reliability
+    { tool: "judges", category: "Reliability (error handling, edge cases)", coverage: "full" },
+    { tool: "ESLint", category: "Reliability (error handling, edge cases)", coverage: "partial" },
+    { tool: "SonarQube", category: "Reliability (error handling, edge cases)", coverage: "full" },
+    { tool: "Semgrep", category: "Reliability (error handling, edge cases)", coverage: "partial" },
+    { tool: "CodeQL", category: "Reliability (error handling, edge cases)", coverage: "partial" },
+    { tool: "Bandit", category: "Reliability (error handling, edge cases)", coverage: "none" },
+    // Performance
+    { tool: "judges", category: "Performance (N+1, caching, async)", coverage: "full" },
+    { tool: "ESLint", category: "Performance (N+1, caching, async)", coverage: "partial", notes: "Async patterns only" },
+    { tool: "SonarQube", category: "Performance (N+1, caching, async)", coverage: "partial" },
+    { tool: "Semgrep", category: "Performance (N+1, caching, async)", coverage: "none" },
+    { tool: "CodeQL", category: "Performance (N+1, caching, async)", coverage: "none" },
+    { tool: "Bandit", category: "Performance (N+1, caching, async)", coverage: "none" },
+    // Accessibility
+    { tool: "judges", category: "Accessibility (WCAG)", coverage: "full" },
+    { tool: "ESLint", category: "Accessibility (WCAG)", coverage: "partial", notes: "jsx-a11y plugin" },
+    { tool: "SonarQube", category: "Accessibility (WCAG)", coverage: "none" },
+    { tool: "Semgrep", category: "Accessibility (WCAG)", coverage: "none" },
+    { tool: "CodeQL", category: "Accessibility (WCAG)", coverage: "none" },
+    { tool: "Bandit", category: "Accessibility (WCAG)", coverage: "none" },
+    // Architecture / Code Structure
+    { tool: "judges", category: "Architecture / Code Structure", coverage: "full" },
+    { tool: "ESLint", category: "Architecture / Code Structure", coverage: "partial", notes: "Complexity rules only" },
+    { tool: "SonarQube", category: "Architecture / Code Structure", coverage: "partial" },
+    { tool: "Semgrep", category: "Architecture / Code Structure", coverage: "none" },
+    { tool: "CodeQL", category: "Architecture / Code Structure", coverage: "none" },
+    { tool: "Bandit", category: "Architecture / Code Structure", coverage: "none" },
+    // AI Code Detection
+    { tool: "judges", category: "AI Code Detection", coverage: "full" },
+    { tool: "ESLint", category: "AI Code Detection", coverage: "none" },
+    { tool: "SonarQube", category: "AI Code Detection", coverage: "none" },
+    { tool: "Semgrep", category: "AI Code Detection", coverage: "none" },
+    { tool: "CodeQL", category: "AI Code Detection", coverage: "none" },
+    { tool: "Bandit", category: "AI Code Detection", coverage: "none" },
+    // Compliance
+    { tool: "judges", category: "Compliance (FedRAMP, GDPR)", coverage: "full" },
+    { tool: "ESLint", category: "Compliance (FedRAMP, GDPR)", coverage: "none" },
+    { tool: "SonarQube", category: "Compliance (FedRAMP, GDPR)", coverage: "partial", notes: "Some OWASP mapping" },
+    { tool: "Semgrep", category: "Compliance (FedRAMP, GDPR)", coverage: "partial", notes: "Via custom rules" },
+    { tool: "CodeQL", category: "Compliance (FedRAMP, GDPR)", coverage: "none" },
+    { tool: "Bandit", category: "Compliance (FedRAMP, GDPR)", coverage: "none" },
+    // API Design
+    { tool: "judges", category: "API Design Review", coverage: "full" },
+    { tool: "ESLint", category: "API Design Review", coverage: "none" },
+    { tool: "SonarQube", category: "API Design Review", coverage: "none" },
+    { tool: "Semgrep", category: "API Design Review", coverage: "none" },
+    { tool: "CodeQL", category: "API Design Review", coverage: "none" },
+    { tool: "Bandit", category: "API Design Review", coverage: "none" },
+    // Cost & Cloud Readiness
+    { tool: "judges", category: "Cost & Cloud Readiness", coverage: "full" },
+    { tool: "ESLint", category: "Cost & Cloud Readiness", coverage: "none" },
+    { tool: "SonarQube", category: "Cost & Cloud Readiness", coverage: "none" },
+    { tool: "Semgrep", category: "Cost & Cloud Readiness", coverage: "none" },
+    { tool: "CodeQL", category: "Cost & Cloud Readiness", coverage: "none" },
+    { tool: "Bandit", category: "Cost & Cloud Readiness", coverage: "none" },
+    // CI/CD Pipeline Review
+    { tool: "judges", category: "CI/CD Pipeline Review", coverage: "full" },
+    { tool: "ESLint", category: "CI/CD Pipeline Review", coverage: "none" },
+    { tool: "SonarQube", category: "CI/CD Pipeline Review", coverage: "none" },
+    { tool: "Semgrep", category: "CI/CD Pipeline Review", coverage: "partial", notes: "Dockerfile rules" },
+    { tool: "CodeQL", category: "CI/CD Pipeline Review", coverage: "none" },
+    { tool: "Bandit", category: "CI/CD Pipeline Review", coverage: "none" },
+];
+// ─── Comparison Logic ────────────────────────────────────────────────────────
+/**
+ * Generate a capability summary comparing judges to a specific tool.
+ */
+export function compareCapabilities(toolName) {
+    const categories = [...new Set(CAPABILITY_MATRIX.map((c) => c.category))];
+    const judgesOnly = [];
+    const otherOnly = [];
+    const both = [];
+    const judgesPartial = [];
+    const otherPartial = [];
+    for (const cat of categories) {
+        const judgesCap = CAPABILITY_MATRIX.find((c) => c.tool === "judges" && c.category === cat);
+        const otherCap = CAPABILITY_MATRIX.find((c) => c.tool.toLowerCase() === toolName.toLowerCase() && c.category === cat);
+        if (!otherCap || otherCap.coverage === "none") {
+            if (judgesCap && judgesCap.coverage !== "none") {
+                judgesOnly.push(cat);
+            }
+        }
+        else if (judgesCap?.coverage === "full" && otherCap.coverage === "full") {
+            both.push(cat);
+        }
+        else if (judgesCap?.coverage === "full" && otherCap.coverage === "partial") {
+            judgesPartial.push(cat);
+        }
+        else if (judgesCap?.coverage === "partial" && otherCap.coverage === "full") {
+            otherPartial.push(cat);
+        }
+    }
+    return { judgesOnly, otherOnly, both, judgesPartial, otherPartial };
+}
+/**
+ * Format a comparison report as text.
+ */
+export function formatComparisonReport(toolName) {
+    const profile = TOOL_PROFILES.find((t) => t.name.toLowerCase() === toolName.toLowerCase());
+    const comparison = compareCapabilities(toolName);
+    const lines = [];
+    lines.push(`╔══════════════════════════════════════════════════════════════╗`);
+    lines.push(`║  judges vs ${(profile?.name || toolName).padEnd(48)} ║`);
+    lines.push(`╚══════════════════════════════════════════════════════════════╝`);
+    lines.push("");
+    if (profile) {
+        lines.push(`Type: ${profile.type} | Languages: ${profile.languages.length} | Pricing: ${profile.pricing}`);
+        lines.push("");
+    }
+    if (comparison.judgesOnly.length > 0) {
+        lines.push(`✅ Unique to judges (${comparison.judgesOnly.length} categories):`);
+        for (const cat of comparison.judgesOnly) {
+            lines.push(`   • ${cat}`);
+        }
+        lines.push("");
+    }
+    if (comparison.both.length > 0) {
+        lines.push(`🤝 Both tools cover fully (${comparison.both.length} categories):`);
+        for (const cat of comparison.both) {
+            lines.push(`   • ${cat}`);
+        }
+        lines.push("");
+    }
+    if (comparison.judgesPartial.length > 0) {
+        lines.push(`📊 judges covers fully, ${toolName} partially:`);
+        for (const cat of comparison.judgesPartial) {
+            lines.push(`   • ${cat}`);
+        }
+        lines.push("");
+    }
+    if (comparison.otherPartial.length > 0) {
+        lines.push(`📊 ${toolName} covers fully, judges partially:`);
+        for (const cat of comparison.otherPartial) {
+            lines.push(`   • ${cat}`);
+        }
+        lines.push("");
+    }
+    lines.push("─".repeat(62));
+    lines.push(`judges covers ${comparison.judgesOnly.length + comparison.both.length + comparison.judgesPartial.length}/10 categories ` +
+        `vs ${toolName}'s ${comparison.both.length + comparison.otherPartial.length}/10`);
+    return lines.join("\n");
+}
+/**
+ * Generate a full comparison matrix across all tools.
+ */
+export function formatFullComparisonMatrix() {
+    const tools = ["judges", ...TOOL_PROFILES.map((t) => t.name)];
+    const categories = [...new Set(CAPABILITY_MATRIX.map((c) => c.category))];
+    const lines = [];
+    lines.push("Capability Matrix: judges vs Other Tools");
+    lines.push("═".repeat(90));
+    // Header
+    const header = "Category".padEnd(40) + tools.map((t) => t.substring(0, 8).padEnd(10)).join("");
+    lines.push(header);
+    lines.push("─".repeat(90));
+    for (const cat of categories) {
+        let row = cat.substring(0, 38).padEnd(40);
+        for (const tool of tools) {
+            const cap = CAPABILITY_MATRIX.find((c) => c.tool.toLowerCase() === tool.toLowerCase() && c.category === cat);
+            const icon = cap?.coverage === "full" ? "●" : cap?.coverage === "partial" ? "◐" : "○";
+            row += icon.padEnd(10);
+        }
+        lines.push(row);
+    }
+    lines.push("─".repeat(90));
+    lines.push("● = Full coverage  ◐ = Partial  ○ = None");
+    return lines.join("\n");
+}

package/dist/config.d.ts

@@ -0,0 +1,108 @@
+import type { JudgesConfig, JudgeDefinition } from "./types.js";
+export declare function expandEnvPlaceholders(content: string): string;
+/**
+ * Validate a JudgeDefinition object. Returns an array of error messages. Empty
+ * array means validation passed.
+ */
+export declare function validateJudgeDefinition(def: Partial<JudgeDefinition>): string[];
+/**
+ * Parse a JSON string into a JudgesConfig, with validation.
+ */
+export declare function parseConfig(jsonStr: string): JudgesConfig;
+/**
+ * Create a default (empty) configuration.
+ */
+export declare function defaultConfig(): JudgesConfig;
+/**
+ * Discover .judgesrc files by walking up from `startDir` to `rootDir`.
+ * Returns configs from root → leaf order (leaf overrides root).
+ */
+export declare function discoverCascadingConfigs(startDir: string, rootDir?: string): JudgesConfig[];
+/**
+ * Merge multiple configs (root → leaf order). Later entries override earlier.
+ * Arrays (disabledRules, disabledJudges, languages, exclude, include, plugins) are
+ * concatenated (union). Scalars (minSeverity, maxFiles, preset, failOnFindings,
+ * baseline, format) use the leaf value.
+ * ruleOverrides are deep-merged.
+ *
+ * Org-level policy enforcement:
+ * - `lockedRules`: locked rules cannot be added to `disabledRules`
+ * - `lockedJudges`: locked judges cannot be added to `disabledJudges`
+ * - `lockedMinSeverity`: child configs cannot set a more lenient minSeverity
+ */
+export declare function mergeConfigs(...configs: JudgesConfig[]): JudgesConfig;
+/** Lightweight loader for tests and CLI helpers.
+ * Reads a JSON config file, expands env placeholders (${VARS}), and parses it.
+ * Returns {} on any failure to keep CLI flows resilient.
+ */
+export declare function loadConfigFile(path: string): Promise<JudgesConfig>;
+/**
+ * Resolve `extends` references in a config. Each value in `extends` is treated
+ * as a file path (relative to `baseDir`). The referenced configs are loaded
+ * and merged (left-to-right), then the current config is applied on top.
+ *
+ * Circular extends are detected and rejected with a ConfigError.
+ *
+ * @param config - The config to resolve
+ * @param baseDir - Directory for resolving relative extends paths
+ * @param seen - Set of already-resolved paths (for cycle detection)
+ * @returns The fully resolved and merged config
+ */
+export declare function resolveExtendsConfig(config: JudgesConfig, baseDir: string, seen?: Set<string>): JudgesConfig;
+/**
+ * Load a cascading config for a specific file path.
+ * Walks from the file's directory up to rootDir, merging all .judgesrc files.
+ */
+export declare function loadCascadingConfig(filePath: string, rootDir?: string): JudgesConfig;
+/**
+ * Validate that a value looks like a JudgeDefinition.
+ * Checks required string fields (id, name, domain, description, systemPrompt,
+ * rulePrefix) and an optional analyze function.
+ */
+export declare function isValidJudgeDefinition(val: unknown): val is JudgeDefinition;
+/**
+ * Load and register plugins from module specifiers declared in config.
+ *
+ * Each plugin is a module specifier (npm package name or relative/absolute
+ * file path) that must export one of:
+ *   - `default: JudgesPlugin`       — a full plugin object (name, version, judges, rules, hooks)
+ *   - `plugin: JudgesPlugin`        — named export
+ *   - `judges: JudgeDefinition[]`   — an array of judge definitions (auto-wrapped)
+ *   - `default: JudgeDefinition[]`  — default export as an array (auto-wrapped)
+ *
+ * Plugins that export a full JudgesPlugin object are registered via
+ * registerPlugin() to enable hooks, custom rules, and full lifecycle.
+ * Simple judge arrays are wrapped into a minimal plugin automatically.
+ *
+ * @param pluginSpecifiers - Array of module specifiers from config.plugins
+ * @param baseDir - Base directory for resolving relative paths
+ * @param logError - Optional error logger (defaults to console.error)
+ * @returns Array of plugin judge definitions that were loaded
+ */
+export declare function loadPluginJudges(pluginSpecifiers: string[], baseDir?: string, logError?: (msg: string) => void): Promise<JudgeDefinition[]>;
+/**
+ * Synchronously validate plugin specifiers without loading them.
+ * Returns an array of validation error messages (empty if all valid).
+ */
+export declare function validatePluginSpecifiers(specifiers: string[]): string[];
+/**
+ * Apply path-scoped overrides to a base config for a specific file.
+ *
+ * Each override entry carries a glob `files` pattern (e.g. `"**\/*.test.ts"`).
+ * If the file path matches, the override's partial config is merged on top
+ * of the base config. Multiple matching overrides are applied in order.
+ *
+ * @param config  - The resolved base config
+ * @param filePath - The relative file path to match against override globs
+ * @returns A new config with matching overrides applied
+ */
+export declare function applyOverridesForFile(config: JudgesConfig, filePath: string): JudgesConfig;
+/**
+ * Apply language-specific config overrides from languageProfiles.
+ * The language string is normalised to a LangFamily before lookup.
+ *
+ * @param config   - Base config (may include languageProfiles)
+ * @param language - The detected language (e.g. "typescript", "python")
+ * @returns A new config with the matching language profile merged on top
+ */
+export declare function applyLanguageProfile(config: JudgesConfig, language: string): JudgesConfig;