@kevinrabun/judges-cli 3.124.0

package/dist/commands/hallucination-score.js

@@ -0,0 +1,316 @@
+/**
+ * Hallucination score — assign a hallucination risk score (0–100)
+ * to AI-generated code based on detected patterns: generic naming,
+ * suspicious imports, implausible logic, unverified API usage.
+ *
+ * All analysis local.
+ */
+import { existsSync, readFileSync, readdirSync } from "fs";
+import { join, extname } from "path";
+// ─── Detectors ──────────────────────────────────────────────────────────────
+function detectHallucinationSignals(content, lines) {
+    const signals = [];
+    // 1. Non-existent or suspicious imports
+    const importLines = lines
+        .map((l, i) => ({ line: i + 1, text: l }))
+        .filter((l) => /\bimport\s|require\s*\(/.test(l.text));
+    for (const imp of importLines) {
+        // Suspicious package names (very generic or implausible)
+        if (/["'](?:utils|helpers|common|shared|lib|core|base)["']/.test(imp.text) && !/from\s*["']\./.test(imp.text)) {
+            signals.push({
+                id: "suspicious-import",
+                description: "Generic import name — may reference non-existent package",
+                weight: 15,
+                line: imp.line,
+                evidence: imp.text.trim(),
+            });
+        }
+        // Version-specific imports (hallucinated API paths)
+        if (/["'][^"']+\/v\d+\/[^"']+["']/.test(imp.text)) {
+            signals.push({
+                id: "versioned-import-path",
+                description: "Version-specific import path — verify API exists",
+                weight: 10,
+                line: imp.line,
+                evidence: imp.text.trim(),
+            });
+        }
+    }
+    // 2. Generic/placeholder variable names
+    let genericNames = 0;
+    for (let i = 0; i < lines.length; i++) {
+        if (/\b(?:const|let|var)\s+(?:data|result|value|item|thing|stuff|temp|tmp|foo|bar|baz|x|y|z)\s*[=:]/.test(lines[i])) {
+            genericNames++;
+        }
+    }
+    if (genericNames > 3) {
+        signals.push({
+            id: "generic-naming",
+            description: `${genericNames} generic variable names — AI may have generated placeholder code`,
+            weight: 10,
+        });
+    }
+    // 3. TODO/FIXME comments left by AI
+    let todoCount = 0;
+    for (let i = 0; i < lines.length; i++) {
+        if (/\/\/\s*(?:TODO|FIXME|HACK|XXX|PLACEHOLDER)/i.test(lines[i])) {
+            todoCount++;
+        }
+    }
+    if (todoCount > 2) {
+        signals.push({
+            id: "excessive-todos",
+            description: `${todoCount} TODO/FIXME comments — AI left implementation gaps`,
+            weight: 20,
+        });
+    }
+    // 4. Unreachable or dead code patterns
+    for (let i = 0; i < lines.length; i++) {
+        if (/\breturn\b/.test(lines[i]) &&
+            i + 1 < lines.length &&
+            /^\s*\S/.test(lines[i + 1]) &&
+            !/^\s*[}\])]/.test(lines[i + 1]) &&
+            !/^\s*(?:case|default|\/\/|\/?\*)/.test(lines[i + 1])) {
+            signals.push({
+                id: "dead-code",
+                description: "Code after return statement — likely hallucinated",
+                weight: 15,
+                line: i + 2,
+                evidence: lines[i + 1].trim().substring(0, 60),
+            });
+        }
+    }
+    // 5. Contradictory logic
+    for (let i = 0; i < lines.length; i++) {
+        if (/if\s*\(\s*true\s*\)|if\s*\(\s*false\s*\)/.test(lines[i])) {
+            signals.push({
+                id: "tautology",
+                description: "Tautological condition (always true/false)",
+                weight: 20,
+                line: i + 1,
+            });
+        }
+        if (/=\s*null.*\.\w+|=\s*undefined.*\.\w+/.test(lines[i])) {
+            signals.push({
+                id: "null-access",
+                description: "Property access on value just assigned null/undefined",
+                weight: 25,
+                line: i + 1,
+            });
+        }
+    }
+    // 6. Copy-paste artifacts
+    const lineSet = new Map();
+    for (let i = 0; i < lines.length; i++) {
+        const trimmed = lines[i].trim();
+        if (trimmed.length > 20) {
+            if (!lineSet.has(trimmed))
+                lineSet.set(trimmed, []);
+            lineSet.get(trimmed).push(i + 1);
+        }
+    }
+    let duplicateBlocks = 0;
+    for (const [, lineNums] of lineSet) {
+        if (lineNums.length >= 3)
+            duplicateBlocks++;
+    }
+    if (duplicateBlocks > 2) {
+        signals.push({
+            id: "copy-paste-artifact",
+            description: `${duplicateBlocks} repeated code blocks — AI may have duplicated patterns`,
+            weight: 15,
+        });
+    }
+    // 7. Magic numbers/strings
+    let magicCount = 0;
+    for (let i = 0; i < lines.length; i++) {
+        if (/\b(?:0x[0-9a-f]{4,}|\d{5,})\b/i.test(lines[i]) && !/\bconst\b/.test(lines[i]) && !/\/\//.test(lines[i])) {
+            magicCount++;
+        }
+    }
+    if (magicCount > 3) {
+        signals.push({
+            id: "magic-numbers",
+            description: `${magicCount} unexplained magic numbers — AI may have hallucinated constants`,
+            weight: 10,
+        });
+    }
+    // 8. Inconsistent error handling
+    const tryCatchCount = (content.match(/\btry\s*{/g) || []).length;
+    const catchCount = (content.match(/\bcatch\s*\(/g) || []).length;
+    const emptyHandlers = (content.match(/catch\s*\([^)]*\)\s*{\s*}/g) || []).length;
+    if (emptyHandlers > 0) {
+        signals.push({
+            id: "empty-catch",
+            description: `${emptyHandlers} empty catch block(s) — errors silently swallowed`,
+            weight: 15,
+        });
+    }
+    if (tryCatchCount > 0 && catchCount < tryCatchCount) {
+        signals.push({
+            id: "unmatched-try",
+            description: "More try blocks than catch blocks — incomplete error handling",
+            weight: 10,
+        });
+    }
+    // 9. Commented-out code (AI artifact)
+    let commentedCode = 0;
+    for (let i = 0; i < lines.length; i++) {
+        if (/^\s*\/\/\s*(?:const|let|var|function|class|import|return|if|for|while)\b/.test(lines[i])) {
+            commentedCode++;
+        }
+    }
+    if (commentedCode > 3) {
+        signals.push({
+            id: "commented-code",
+            description: `${commentedCode} lines of commented-out code — AI left alternative implementations`,
+            weight: 10,
+        });
+    }
+    // 10. Functions with no implementation
+    for (let i = 0; i < lines.length; i++) {
+        if (/\bfunction\s+\w+\s*\([^)]*\)\s*{\s*}/.test(lines[i]) || /=>\s*{\s*}/.test(lines[i])) {
+            signals.push({
+                id: "empty-function",
+                description: "Empty function body — stub not implemented",
+                weight: 20,
+                line: i + 1,
+            });
+        }
+    }
+    return signals;
+}
+// ─── Scanner ────────────────────────────────────────────────────────────────
+const SKIP = new Set(["node_modules", ".git", "dist", "build", "coverage"]);
+const EXTS = new Set([".ts", ".js", ".py", ".java", ".cs", ".go", ".rb", ".php", ".rs", ".swift", ".kt"]);
+function collectFiles(dir) {
+    const result = [];
+    function walk(d) {
+        let entries;
+        try {
+            entries = readdirSync(d);
+        }
+        catch {
+            return;
+        }
+        for (const name of entries) {
+            if (SKIP.has(name) || name.startsWith("."))
+                continue;
+            const full = join(d, name);
+            try {
+                const sub = readdirSync(full);
+                void sub;
+                walk(full);
+            }
+            catch {
+                if (EXTS.has(extname(name).toLowerCase()))
+                    result.push(full);
+            }
+        }
+    }
+    walk(dir);
+    return result;
+}
+function computeScore(signals) {
+    const raw = signals.reduce((sum, s) => sum + s.weight, 0);
+    return Math.min(100, raw);
+}
+function riskLevel(score) {
+    if (score >= 70)
+        return "critical";
+    if (score >= 45)
+        return "high";
+    if (score >= 20)
+        return "medium";
+    return "low";
+}
+// ─── CLI ────────────────────────────────────────────────────────────────────
+export function runHallucinationScore(argv) {
+    if (argv.includes("--help") || argv.includes("-h")) {
+        console.log(`
+judges hallucination-score — Assess hallucination risk in AI-generated code
+
+Usage:
+  judges hallucination-score <file-or-dir>
+  judges hallucination-score src/ --min-risk medium
+
+Options:
+  --min-risk <level>  Only show files at or above this risk (low, medium, high, critical)
+  --format json       JSON output
+  --help, -h          Show this help
+
+Signals detected:
+  • Suspicious/non-existent imports
+  • Generic placeholder naming
+  • Excessive TODO/FIXME comments
+  • Dead code after return statements
+  • Tautological conditions
+  • Copy-paste artifacts
+  • Magic numbers
+  • Empty catch blocks
+  • Commented-out code
+  • Empty function stubs
+`);
+        return;
+    }
+    const format = argv.find((_a, i) => argv[i - 1] === "--format") || "text";
+    const minRisk = argv.find((_a, i) => argv[i - 1] === "--min-risk");
+    const target = argv.find((a) => !a.startsWith("--") && !argv[argv.indexOf(a) - 1]?.startsWith("--")) || ".";
+    if (!existsSync(target)) {
+        console.error(`  Path not found: ${target}`);
+        return;
+    }
+    let files;
+    try {
+        readdirSync(target);
+        files = collectFiles(target);
+    }
+    catch {
+        files = [target];
+    }
+    const scored = [];
+    for (const f of files) {
+        let content;
+        try {
+            content = readFileSync(f, "utf-8");
+        }
+        catch {
+            continue;
+        }
+        const lines = content.split("\n");
+        const signals = detectHallucinationSignals(content, lines);
+        const score = computeScore(signals);
+        scored.push({ file: f, score, riskLevel: riskLevel(score), signals });
+    }
+    let filtered = scored;
+    if (minRisk) {
+        const order = ["low", "medium", "high", "critical"];
+        const minIdx = order.indexOf(minRisk);
+        if (minIdx >= 0)
+            filtered = scored.filter((s) => order.indexOf(s.riskLevel) >= minIdx);
+    }
+    filtered.sort((a, b) => b.score - a.score);
+    if (format === "json") {
+        console.log(JSON.stringify({ files: filtered, scannedFiles: files.length, timestamp: new Date().toISOString() }, null, 2));
+    }
+    else {
+        console.log(`\n  Hallucination Risk Assessment — ${files.length} files`);
+        console.log(`  ──────────────────────────`);
+        if (filtered.length === 0) {
+            console.log(`    ✅ No hallucination risks above threshold\n`);
+            return;
+        }
+        for (const f of filtered.slice(0, 20)) {
+            const icon = f.riskLevel === "critical" ? "🔴" : f.riskLevel === "high" ? "🟠" : f.riskLevel === "medium" ? "🟡" : "🟢";
+            console.log(`\n    ${icon} ${f.file} — Score: ${f.score}/100 (${f.riskLevel})`);
+            for (const s of f.signals) {
+                console.log(`        • ${s.description}${s.line ? ` (line ${s.line})` : ""}`);
+            }
+        }
+        if (filtered.length > 20)
+            console.log(`    ... and ${filtered.length - 20} more files`);
+        const avgScore = Math.round(filtered.reduce((sum, f) => sum + f.score, 0) / filtered.length);
+        console.log(`\n    Average risk score: ${avgScore}/100`);
+        console.log(`    Critical: ${filtered.filter((f) => f.riskLevel === "critical").length} | High: ${filtered.filter((f) => f.riskLevel === "high").length} | Medium: ${filtered.filter((f) => f.riskLevel === "medium").length} | Low: ${filtered.filter((f) => f.riskLevel === "low").length}\n`);
+    }
+}

package/dist/commands/help.d.ts

@@ -0,0 +1,7 @@
+/**
+ * `judges help [topic]` — contextual help and getting-started guides.
+ *
+ * Provides quick, actionable guidance for each major capability area.
+ * Run without arguments to see all available topics.
+ */
+export declare function runHelp(argv: string[]): void;

package/dist/commands/help.js

@@ -0,0 +1,302 @@
+/**
+ * `judges help [topic]` — contextual help and getting-started guides.
+ *
+ * Provides quick, actionable guidance for each major capability area.
+ * Run without arguments to see all available topics.
+ */
+const TOPICS = {
+    "getting-started": {
+        title: "Getting Started",
+        summary: "First steps with Judges Panel",
+        body: `
+  Quick Start
+  ───────────
+  1. Evaluate a single file:
+       judges eval server.ts
+
+  2. Scan an entire directory:
+       judges eval src/
+
+  3. Generate a config file:
+       judges init
+
+  4. Use a preset for CI:
+       judges eval src/ --preset security
+
+  5. Auto-fix findings:
+       judges eval src/ --fix
+
+  Common Flags
+  ────────────
+    --format json|sarif|html|markdown   Output format
+    --fix                               Apply auto-fix patches
+    --baseline .judges-baseline.json    Suppress known findings
+    --incremental                       Cache results for unchanged files
+    --changed-only                      Only scan git-changed files
+    --quiet                             Suppress progress output
+    --explain                           Show CWE/OWASP context for findings
+`,
+    },
+    feedback: {
+        title: "Feedback & False Positive Reporting",
+        summary: "How to improve accuracy with feedback",
+        body: `
+  Providing Feedback
+  ──────────────────
+  Feedback trains Judges to reduce false positives over time.
+
+  CLI:
+    judges feedback add --rule SEC-001 --verdict fp --comment "Not exploitable"
+    judges feedback add --rule AUTH-003 --verdict tp
+    judges feedback list
+    judges feedback stats
+
+  VS Code:
+    Click the thumbs-up/down icons on any finding's code action menu.
+    Feedback is stored in .judges-feedback.json in your workspace.
+
+  Verdicts:
+    tp         True positive (real issue)
+    fp         False positive (not a real issue)
+    wontfix    Acknowledged but won't fix
+
+  Your feedback is stored locally and never sent externally.
+  Configure a DataAdapter in .judgesrc to share feedback with your team.
+`,
+    },
+    calibration: {
+        title: "Calibration & Confidence Tuning",
+        summary: "How to tune judge accuracy for your codebase",
+        body: `
+  Auto-Calibration
+  ────────────────
+    judges tune             Run auto-calibration on your feedback data
+    judges tune --dry-run   Preview what would change
+
+  Calibration Dashboard
+  ─────────────────────
+    judges calibration-dashboard          Open interactive HTML dashboard
+    judges calibration-dashboard --output report.html
+
+  How It Works
+  ────────────
+  Judges uses your feedback (tp/fp verdicts) to adjust confidence thresholds
+  per rule. Rules with many false positives get their confidence reduced;
+  rules with consistent true positives are boosted.
+
+  Team Sharing
+  ────────────
+  Calibration profiles are stored in .judges-calibration.json.
+  Commit this to your repo so team members benefit from shared tuning.
+`,
+    },
+    ci: {
+        title: "CI/CD Integration",
+        summary: "Setting up Judges in CI pipelines",
+        body: `
+  Generate CI Config
+  ──────────────────
+    judges ci-templates              Show all available templates
+    judges ci-templates --github     GitHub Actions workflow
+    judges ci-templates --gitlab     GitLab CI config
+    judges ci-templates --azure      Azure Pipelines config
+    judges ci-templates --bitbucket  Bitbucket Pipelines config
+
+  Recommended CI Setup
+  ────────────────────
+  1. Create a baseline to suppress existing findings:
+       judges baseline create --dir src/
+
+  2. Run incremental scans in PRs:
+       judges eval src/ --changed-only --fail-on-findings --format sarif
+
+  3. Upload SARIF to GitHub Code Scanning:
+       - uses: github/codeql-action/upload-sarif@v3
+         with:
+           sarif_file: judges-report.sarif.json
+
+  Environment Variables
+  ─────────────────────
+    JUDGES_CACHE_DIR    Custom cache directory (default: .judges-cache)
+    JUDGES_NO_COLOR     Disable colored output
+    JUDGES_CONFIG       Path to .judgesrc config file
+`,
+    },
+    presets: {
+        title: "Presets & Configuration",
+        summary: "Using and composing evaluation presets",
+        body: `
+  Available Presets
+  ────────────────
+    judges eval --preset security       Security-focused evaluation
+    judges eval --preset quality        Code quality focus
+    judges eval --preset compliance     Compliance & governance
+    judges eval --preset full           All judges enabled
+
+  List All Presets
+  ────────────────
+    judges config presets
+
+  Composing Presets
+  ────────────────
+  In .judgesrc, you can compose multiple presets:
+    {
+      "presets": ["security", "quality"],
+      "disabledRules": ["DOC-001"],
+      "minSeverity": "medium"
+    }
+
+  Custom Config (.judgesrc)
+  ────────────────────────
+    judges init                    Generate a starter .judgesrc
+    judges config validate         Validate your config
+    judges doctor                  Diagnose configuration issues
+`,
+    },
+    plugins: {
+        title: "Plugins & Custom Rules",
+        summary: "Extending Judges with custom rules and plugins",
+        body: `
+  Creating a Plugin
+  ─────────────────
+    judges scaffold-plugin my-rules    Generate a plugin template
+
+  Plugin Structure
+  ────────────────
+    my-rules/
+      package.json        # name, version, "judges-plugin" keyword
+      index.js            # exports { rules: [...] }
+
+  Installing Plugins
+  ──────────────────
+  Add to .judgesrc:
+    {
+      "plugins": ["@myorg/judges-plugin-custom"]
+    }
+
+  Community Patterns
+  ──────────────────
+    judges community-patterns          Browse community rule patterns
+
+  See docs/plugin-guide.md for the full authoring guide.
+`,
+    },
+    "data-adapter": {
+        title: "Data Adapters (Team Storage)",
+        summary: "Configure external storage for team-wide sharing",
+        body: `
+  What Are Data Adapters?
+  ──────────────────────
+  By default, Judges stores feedback, findings, and metrics as local JSON
+  files. Data Adapters let your team share this data via your own backend.
+
+  Judges never hosts or processes your data — you bring your own storage.
+
+  Configuration (.judgesrc)
+  ────────────────────────
+  Local filesystem (default):
+    {
+      "dataAdapter": { "type": "filesystem" }
+    }
+
+  HTTP backend (your own API):
+    {
+      "dataAdapter": {
+        "type": "http",
+        "url": "https://your-team-api.example.com/judges",
+        "headers": {
+          "Authorization": "Bearer \${JUDGES_API_TOKEN}"
+        }
+      }
+    }
+
+  Environment variables in header values are resolved automatically.
+  The HTTP adapter uses standard REST endpoints:
+    GET  /feedback       Load feedback store
+    PUT  /feedback       Save feedback store
+    GET  /findings       Load finding store
+    PUT  /findings       Save finding store
+`,
+    },
+    "security-ids": {
+        title: "CWE & OWASP Identifiers",
+        summary: "How findings map to CWE and OWASP standards",
+        body: `
+  Structured Security IDs
+  ──────────────────────
+  Every finding is automatically enriched with CWE and OWASP identifiers
+  based on its rule prefix and specific rule ID.
+
+  Examples:
+    SEC-001  →  CWE-89 (SQL Injection), OWASP A03:2021
+    AUTH-001 →  CWE-798 (Hard-coded Credentials), OWASP A07:2021
+    XSS-*    →  CWE-79 (Cross-site Scripting), OWASP A03:2021
+    SSRF-*   →  CWE-918 (Server-Side Request Forgery), OWASP A10:2021
+
+  These IDs appear in:
+    - SARIF output (as rule tags)
+    - JSON output (cweIds / owaspIds fields)
+    - The --explain flag output
+    - VS Code extension hover details
+
+  Use --format sarif for tool integration with GitHub Code Scanning,
+  SonarQube, Snyk, and other platforms that consume CWE/OWASP data.
+`,
+    },
+    metrics: {
+        title: "Metrics & ROI Tracking",
+        summary: "Measuring the value Judges provides",
+        body: `
+  Quick Metrics
+  ─────────────
+    judges metrics                    Summary to stdout
+    judges metrics --format json      Machine-readable output
+    judges metrics --since 30d        Last 30 days only
+
+  Visual Dashboard
+  ────────────────
+    judges metrics-dashboard                  Generate HTML dashboard
+    judges metrics-dashboard --output dash.html
+
+  What's Tracked
+  ─────────────
+    - Findings detected / fixed / open / false-positive
+    - Auto-fix adoption rate
+    - Estimated time saved (conservative industry averages)
+    - Trend direction (improving / stable / degrading)
+    - Fix rates by severity
+
+  All metrics are computed from local data. Configure a DataAdapter
+  to aggregate metrics across your team (see: judges help data-adapter).
+`,
+    },
+};
+// ─── Runner ─────────────────────────────────────────────────────────────────
+export function runHelp(argv) {
+    const topic = argv.find((a, i) => i > 2 && !a.startsWith("-"))?.toLowerCase();
+    if (!topic) {
+        printTopicList();
+        return;
+    }
+    const entry = TOPICS[topic];
+    if (!entry) {
+        console.error(`  Unknown help topic: "${topic}"\n`);
+        printTopicList();
+        process.exit(1);
+    }
+    console.log("");
+    console.log(`  ${entry.title}`);
+    console.log(`  ${"─".repeat(entry.title.length)}`);
+    console.log(entry.body);
+}
+function printTopicList() {
+    console.log("");
+    console.log("  Judges Panel — Help Topics");
+    console.log("  ──────────────────────────");
+    console.log("");
+    const maxKey = Math.max(...Object.keys(TOPICS).map((k) => k.length));
+    for (const [key, { summary }] of Object.entries(TOPICS)) {
+        console.log(`    judges help ${key.padEnd(maxKey)}   ${summary}`);
+    }
+    console.log("");
+}

package/dist/commands/hook-install.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Hook install — install and configure git hooks for pre-commit
+ * and pre-push evaluation with Judges.
+ *
+ * Supports husky, simple-git-hooks, and direct .git/hooks installation.
+ */
+export interface HookConfig {
+    /** Hook type: pre-commit or pre-push */
+    hook: "pre-commit" | "pre-push";
+    /** Only evaluate staged files */
+    stagedOnly: boolean;
+    /** Fail on findings of this severity or higher */
+    failSeverity: string;
+    /** Timeout in seconds */
+    timeout: number;
+    /** Run in parallel */
+    parallel: boolean;
+    /** Use smart judge selection */
+    smartSelect: boolean;
+}
+export declare function runHookInstall(argv: string[]): void;