@kevinrabun/judges 3.115.0 → 3.115.2

package/dist/agent-loader.d.ts

@@ -0,0 +1,107 @@
+/**
+ * Agent Markdown Loader — reads `.judge.md` files (legacy `.agent.md` also
+ * accepted) and converts them into JudgeDefinition objects that register with
+ * the unified JudgeRegistry.
+ *
+ * This is the bridge between the file-based agent paradigm and the existing
+ * TypeScript judge system. Agent files use YAML frontmatter for metadata
+ * and markdown body for the system prompt (persona + evaluation criteria).
+ *
+ * ## File Format
+ *
+ * ```markdown
+ * ---
+ * id: cybersecurity
+ * name: Judge Cybersecurity
+ * domain: Cybersecurity & Threat Defense
+ * rulePrefix: CYBER
+ * description: Evaluates code for vulnerability...
+ * tableDescription: "Injection attacks, XSS, CSRF, auth flaws"
+ * promptDescription: Deep cybersecurity review
+ * script: ../src/evaluators/cybersecurity.ts   # optional
+ * priority: 10                                  # optional, default 10
+ * ---
+ *
+ * You are Judge Cybersecurity — a principal application security engineer...
+ *
+ * ## Evaluation Criteria
+ * ...
+ * ```
+ *
+ * - `script` is a relative path to the evaluator module (must export a
+ *   function matching `(code: string, language: string, context?) => Finding[]`).
+ *   If omitted, the judge is LLM-only (no deterministic layer).
+ * - `priority` controls ordering. Higher = later. 999 is reserved for
+ *   false-positive-review (always last). Default is 10.
+ */
+import type { JudgeDefinition, Finding, AnalyzeContext } from "./types.js";
+/** Parsed YAML frontmatter from a `.judge.md` file (legacy `.agent.md`). */
+export interface AgentFrontmatter {
+    id: string;
+    name: string;
+    domain: string;
+    rulePrefix: string;
+    description: string;
+    tableDescription: string;
+    promptDescription: string;
+    script?: string;
+    priority?: number;
+}
+/** A parsed agent file — metadata + the markdown body (system prompt). */
+export interface ParsedAgent {
+    frontmatter: AgentFrontmatter;
+    /** The markdown body below the frontmatter — becomes the systemPrompt. */
+    body: string;
+    /** Absolute path of the source `.judge.md` file (legacy `.agent.md`). */
+    sourcePath: string;
+}
+/**
+ * Parse YAML frontmatter from a string. Handles the subset of YAML used
+ * by agent files: simple key-value pairs, quoted strings, and multi-line
+ * `>` folded scalars. No arrays, nested objects, or anchors.
+ */
+export declare function parseFrontmatter(raw: string): {
+    meta: Record<string, string>;
+    body: string;
+};
+/**
+ * Validate and coerce parsed frontmatter into a typed AgentFrontmatter.
+ * Throws on missing required fields.
+ */
+export declare function validateFrontmatter(meta: Record<string, string>, sourcePath: string): AgentFrontmatter;
+/**
+ * Parse a single `.judge.md` file into its frontmatter and body (legacy `.agent.md`).
+ */
+export declare function parseAgentFile(filePath: string): ParsedAgent;
+/**
+ * Resolve the `script` path to an analyze function.
+ *
+ * Requirements:
+ * - Synchronous (to support existing synchronous evaluation paths)
+ * - Works both from source (`tsx`/ts-node) and compiled `dist`
+ */
+type AnalyzeFn = (code: string, language: string, context?: AnalyzeContext) => Finding[];
+export declare function resolveEvaluator(agent: ParsedAgent): AnalyzeFn | undefined;
+/**
+ * Convert a parsed agent file to a JudgeDefinition, reconstructing the
+ * systemPrompt from the markdown body with the standard adversarial
+ * mandate appended.
+ */
+export declare function agentToJudgeDefinition(agent: ParsedAgent, analyze?: (code: string, language: string, context?: AnalyzeContext) => Finding[]): JudgeDefinition;
+/**
+ * Load all `.judge.md` files from a directory (legacy `.agent.md` supported)
+ * and return parsed agents sorted by priority (ascending — lower number =
+ * earlier in pipeline).
+ */
+export declare function loadAgentDirectory(dirPath: string): ParsedAgent[];
+/**
+ * Load all agent files from a directory and register them with the
+ * JudgeRegistry. This is the main entry point for the hybrid phase.
+ *
+ * Returns the number of agents loaded.
+ */
+export declare function loadAndRegisterAgents(dirPath: string, registry: {
+    register: (judge: JudgeDefinition) => void;
+    getJudge: (id: string) => JudgeDefinition | undefined;
+}): number;
+export {};

package/dist/agent-loader.js

@@ -0,0 +1,254 @@
+/**
+ * Agent Markdown Loader — reads `.judge.md` files (legacy `.agent.md` also
+ * accepted) and converts them into JudgeDefinition objects that register with
+ * the unified JudgeRegistry.
+ *
+ * This is the bridge between the file-based agent paradigm and the existing
+ * TypeScript judge system. Agent files use YAML frontmatter for metadata
+ * and markdown body for the system prompt (persona + evaluation criteria).
+ *
+ * ## File Format
+ *
+ * ```markdown
+ * ---
+ * id: cybersecurity
+ * name: Judge Cybersecurity
+ * domain: Cybersecurity & Threat Defense
+ * rulePrefix: CYBER
+ * description: Evaluates code for vulnerability...
+ * tableDescription: "Injection attacks, XSS, CSRF, auth flaws"
+ * promptDescription: Deep cybersecurity review
+ * script: ../src/evaluators/cybersecurity.ts   # optional
+ * priority: 10                                  # optional, default 10
+ * ---
+ *
+ * You are Judge Cybersecurity — a principal application security engineer...
+ *
+ * ## Evaluation Criteria
+ * ...
+ * ```
+ *
+ * - `script` is a relative path to the evaluator module (must export a
+ *   function matching `(code: string, language: string, context?) => Finding[]`).
+ *   If omitted, the judge is LLM-only (no deterministic layer).
+ * - `priority` controls ordering. Higher = later. 999 is reserved for
+ *   false-positive-review (always last). Default is 10.
+ */
+import { readFileSync, readdirSync, existsSync } from "node:fs";
+import { join, resolve, dirname } from "node:path";
+import { createRequire } from "node:module";
+// ─── Frontmatter Parser ─────────────────────────────────────────────────────
+/**
+ * Parse YAML frontmatter from a string. Handles the subset of YAML used
+ * by agent files: simple key-value pairs, quoted strings, and multi-line
+ * `>` folded scalars. No arrays, nested objects, or anchors.
+ */
+export function parseFrontmatter(raw) {
+    const match = raw.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/);
+    if (!match) {
+        return { meta: {}, body: raw };
+    }
+    const yamlBlock = match[1];
+    const body = match[2].trim();
+    const meta = {};
+    const lines = yamlBlock.split(/\r?\n/);
+    let i = 0;
+    while (i < lines.length) {
+        const line = lines[i];
+        // Skip empty lines and comments
+        if (!line.trim() || line.trim().startsWith("#")) {
+            i++;
+            continue;
+        }
+        const kvMatch = line.match(/^([a-zA-Z_][a-zA-Z0-9_]*)\s*:\s*(.*)/);
+        if (!kvMatch) {
+            i++;
+            continue;
+        }
+        const key = kvMatch[1];
+        let value = kvMatch[2].trim();
+        // Handle folded scalar (>)
+        if (value === ">") {
+            const parts = [];
+            i++;
+            while (i < lines.length && (lines[i].startsWith("  ") || lines[i].trim() === "")) {
+                if (lines[i].trim() === "") {
+                    parts.push("");
+                }
+                else {
+                    parts.push(lines[i].trimStart());
+                }
+                i++;
+            }
+            // Folded scalar: join non-empty lines with spaces, blank lines become newlines
+            value = parts
+                .reduce((acc, part) => {
+                if (part === "") {
+                    acc.push("\n");
+                }
+                else if (acc.length > 0 && acc[acc.length - 1] !== "\n") {
+                    acc[acc.length - 1] += " " + part;
+                }
+                else {
+                    acc.push(part);
+                }
+                return acc;
+            }, [])
+                .join("")
+                .trim();
+        }
+        else {
+            // Strip surrounding quotes
+            if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
+                value = value.slice(1, -1);
+            }
+            i++;
+        }
+        meta[key] = value;
+    }
+    return { meta, body };
+}
+// ─── Validation ──────────────────────────────────────────────────────────────
+const REQUIRED_FIELDS = [
+    "id",
+    "name",
+    "domain",
+    "rulePrefix",
+    "description",
+    "tableDescription",
+    "promptDescription",
+];
+/**
+ * Validate and coerce parsed frontmatter into a typed AgentFrontmatter.
+ * Throws on missing required fields.
+ */
+export function validateFrontmatter(meta, sourcePath) {
+    for (const field of REQUIRED_FIELDS) {
+        if (!meta[field]) {
+            throw new Error(`Agent file ${sourcePath} is missing required field: "${field}"`);
+        }
+    }
+    return {
+        id: meta.id,
+        name: meta.name,
+        domain: meta.domain,
+        rulePrefix: meta.rulePrefix,
+        description: meta.description,
+        tableDescription: meta.tableDescription,
+        promptDescription: meta.promptDescription,
+        script: meta.script || undefined,
+        priority: meta.priority ? parseInt(meta.priority, 10) : 10,
+    };
+}
+// ─── Agent File Parsing ──────────────────────────────────────────────────────
+/**
+ * Parse a single `.judge.md` file into its frontmatter and body (legacy `.agent.md`).
+ */
+export function parseAgentFile(filePath) {
+    const absPath = resolve(filePath);
+    const raw = readFileSync(absPath, "utf-8");
+    const { meta, body } = parseFrontmatter(raw);
+    const frontmatter = validateFrontmatter(meta, absPath);
+    return {
+        frontmatter,
+        body,
+        sourcePath: absPath,
+    };
+}
+export function resolveEvaluator(agent) {
+    if (!agent.frontmatter.script)
+        return undefined;
+    const scriptPath = resolve(dirname(agent.sourcePath), agent.frontmatter.script);
+    const candidatePaths = [
+        scriptPath,
+        scriptPath.replace(/\.ts$/, ".js"),
+        scriptPath
+            .replace(/\\src\\/g, "\\dist\\")
+            .replace(/\/src\//g, "/dist/")
+            .replace(/\.ts$/, ".js"),
+        resolve(process.cwd(), "dist", "evaluators", `${agent.frontmatter.id}.js`),
+    ];
+    const req = createRequire(import.meta.url);
+    for (const candidate of candidatePaths) {
+        try {
+            const mod = req(candidate);
+            const pascalId = agent.frontmatter.id
+                .split("-")
+                .map((s) => s.charAt(0).toUpperCase() + s.slice(1))
+                .join("");
+            const fnName = `analyze${pascalId}`;
+            const maybeFn = mod?.[fnName];
+            if (typeof maybeFn === "function")
+                return maybeFn;
+            for (const key of Object.keys(mod || {})) {
+                const candidateFn = mod[key];
+                if (typeof candidateFn === "function" && key.startsWith("analyze"))
+                    return candidateFn;
+            }
+        }
+        catch {
+            // swallow and try next
+        }
+    }
+    return undefined;
+}
+// ─── Conversion to JudgeDefinition ───────────────────────────────────────────
+/**
+ * Convert a parsed agent file to a JudgeDefinition, reconstructing the
+ * systemPrompt from the markdown body with the standard adversarial
+ * mandate appended.
+ */
+export function agentToJudgeDefinition(agent, analyze) {
+    const fm = agent.frontmatter;
+    // The markdown body IS the system prompt content. We prepend the persona
+    // line (which is typically the first line of the body) and leave the
+    // rest as structured evaluation criteria.
+    const systemPrompt = agent.body;
+    return {
+        id: fm.id,
+        name: fm.name,
+        domain: fm.domain,
+        description: fm.description,
+        rulePrefix: fm.rulePrefix,
+        tableDescription: fm.tableDescription,
+        promptDescription: fm.promptDescription,
+        systemPrompt,
+        ...(analyze ? { analyze } : {}),
+    };
+}
+// ─── Directory Loading ───────────────────────────────────────────────────────
+/**
+ * Load all `.judge.md` files from a directory (legacy `.agent.md` supported)
+ * and return parsed agents sorted by priority (ascending — lower number =
+ * earlier in pipeline).
+ */
+export function loadAgentDirectory(dirPath) {
+    const absDir = resolve(dirPath);
+    if (!existsSync(absDir))
+        return [];
+    const files = readdirSync(absDir).filter((f) => /\.(agent|judge)\.md$/i.test(f));
+    return files
+        .map((f) => parseAgentFile(join(absDir, f)))
+        .sort((a, b) => (a.frontmatter.priority ?? 10) - (b.frontmatter.priority ?? 10));
+}
+/**
+ * Load all agent files from a directory and register them with the
+ * JudgeRegistry. This is the main entry point for the hybrid phase.
+ *
+ * Returns the number of agents loaded.
+ */
+export function loadAndRegisterAgents(dirPath, registry) {
+    const agents = loadAgentDirectory(dirPath);
+    let count = 0;
+    for (const agent of agents) {
+        // Skip if a judge with this ID already exists (built-ins or previously loaded agents)
+        if (registry.getJudge(agent.frontmatter.id)) {
+            continue;
+        }
+        const analyze = resolveEvaluator(agent);
+        const judge = agentToJudgeDefinition(agent, analyze);
+        registry.register(judge);
+        count++;
+    }
+    return count;
+}

package/dist/cli-helpers.d.ts

@@ -0,0 +1,29 @@
+export interface ParsedCliArgs {
+    command?: string;
+    file?: string;
+    format?: string;
+    judge?: string;
+    baseline?: string;
+    config?: string;
+    preset?: string;
+    minScore?: number;
+    language?: string;
+    plugins?: string[];
+    include?: string[];
+    exclude?: string[];
+    maxFiles?: number;
+    failOnFindings?: boolean;
+    help?: boolean;
+}
+export declare function parseCliArgs(argv: string[]): ParsedCliArgs;
+export declare function globToRegex(pattern: string): RegExp;
+export declare function matchesGlob(filePath: string, patterns: string[]): boolean;
+export interface CollectOptions {
+    include?: string[];
+    exclude?: string[];
+}
+export declare function collectFiles(target: string, options?: CollectOptions): string[];
+export interface DispatchTable {
+    [command: string]: (argv: string[]) => Promise<void> | void;
+}
+export declare function dispatchCommand(command: string | undefined, argv: string[], dispatchTable: DispatchTable): Promise<void>;

package/dist/cli-helpers.js

@@ -0,0 +1,129 @@
+import { existsSync, readdirSync, statSync } from "fs";
+import { resolve, relative } from "path";
+export function parseCliArgs(argv) {
+    const args = { include: [], exclude: [], plugins: [] };
+    let i = 0;
+    if (argv.length > 0 && !argv[0].startsWith("-")) {
+        args.command = argv[i++];
+    }
+    for (; i < argv.length; i++) {
+        const arg = argv[i];
+        switch (arg) {
+            case "--file":
+            case "-f":
+                args.file = argv[++i];
+                break;
+            case "--format":
+                args.format = argv[++i];
+                break;
+            case "--judge":
+            case "-j":
+                args.judge = argv[++i];
+                break;
+            case "--baseline":
+                args.baseline = argv[++i];
+                break;
+            case "--config":
+                args.config = argv[++i];
+                break;
+            case "--preset":
+                args.preset = argv[++i];
+                break;
+            case "--min-score":
+                args.minScore = parseInt(argv[++i], 10);
+                break;
+            case "--language":
+            case "-l":
+                args.language = argv[++i];
+                break;
+            case "--plugins":
+                args.plugins.push(argv[++i]);
+                break;
+            case "--include":
+                args.include.push(argv[++i]);
+                break;
+            case "--exclude":
+                args.exclude.push(argv[++i]);
+                break;
+            case "--max-files":
+                args.maxFiles = parseInt(argv[++i], 10);
+                break;
+            case "--fail-on-findings":
+                args.failOnFindings = true;
+                break;
+            case "--help":
+            case "-h":
+                args.help = true;
+                break;
+            default:
+                // ignore unknown flags; let dispatch handle it
+                break;
+        }
+    }
+    return args;
+}
+export function globToRegex(pattern) {
+    // Normalize Windows backslashes in the pattern
+    const normalized = pattern.replace(/\\/g, "/");
+    // Escape all regex meta characters EXCEPT our glob wildcards (*, ?)
+    const escapeRegexChars = /[.+^${}()|[\]\\]/g; // safe char class
+    let escaped = normalized.replace(escapeRegexChars, "\\$&");
+    // Temporarily protect globstar so the single-star replacement won't clobber it
+    escaped = escaped.replace(/\*\*/g, "§§GLOBSTAR§§");
+    // Replace single * with a single path-segment wildcard
+    escaped = escaped.replace(/\*/g, "[^/]*");
+    // Replace globstar with an any-depth wildcard (including empty)
+    escaped = escaped.replace(/§§GLOBSTAR§§/g, ".*");
+    // Optional: support ? for single-character wildcard
+    escaped = escaped.replace(/\?/g, ".");
+    // Special-case patterns that start with globstar+slash (e.g., **/*.ts) to also
+    // match files in the root (i.e., allow the slash segment to be optional).
+    escaped = escaped.replace(/^\.\*\//, "(?:.*\/)?");
+    return new RegExp(`^${escaped}$`);
+}
+export function matchesGlob(filePath, patterns) {
+    const normalizedPath = filePath.replace(/\\/g, "/");
+    return patterns.some((p) => globToRegex(p).test(normalizedPath));
+}
+export function collectFiles(target, options = {}) {
+    const root = resolve(target);
+    const include = options.include ?? ["**/*"];
+    const exclude = options.exclude ?? [];
+    const files = [];
+    const rootBasename = root.split(/[/\\]/).filter(Boolean).pop() ?? "";
+    function pathVariants(rel) {
+        const normalized = rel.replace(/\\/g, "/");
+        const withRoot = rootBasename ? `${rootBasename}/${normalized}` : normalized;
+        return [normalized, withRoot];
+    }
+    function matchesAny(rel, globs) {
+        return pathVariants(rel).some((candidate) => matchesGlob(candidate, globs));
+    }
+    function walk(dir) {
+        for (const entry of readdirSync(dir)) {
+            const full = resolve(dir, entry);
+            const rel = relative(root, full);
+            const st = statSync(full);
+            if (st.isDirectory()) {
+                walk(full);
+            }
+            else {
+                if (!matchesAny(rel, include))
+                    continue;
+                if (exclude.length && matchesAny(rel, exclude))
+                    continue;
+                files.push(full);
+            }
+        }
+    }
+    if (existsSync(root)) {
+        walk(root);
+    }
+    return files;
+}
+export async function dispatchCommand(command, argv, dispatchTable) {
+    if (!command || !dispatchTable[command]) {
+        throw new Error("UNKNOWN_COMMAND");
+    }
+    await dispatchTable[command](argv);
+}

package/dist/cli.d.ts

@@ -0,0 +1,30 @@
+#!/usr/bin/env node
+/**
+ * Judges Panel — CLI Evaluator
+ *
+ * Evaluate code directly from the command line without MCP setup.
+ *
+ * Usage:
+ *   judges eval --file src/app.ts                         # evaluate a file
+ *   judges eval --file src/app.ts --language python       # explicit language
+ *   judges eval --file src/app.ts --format sarif          # SARIF output
+ *   judges eval --file src/app.ts --format html           # HTML report
+ *   judges eval --file src/app.ts --judge cybersecurity   # single judge
+ *   judges eval --fail-on-findings src/app.ts             # exit 1 on fail
+ *   cat src/app.ts | judges eval --language typescript    # stdin pipe
+ *   judges init                                           # interactive setup
+ *   judges fix src/app.ts --apply                         # auto-fix findings
+ *   judges watch src/                                     # watch mode
+ *   judges report .                                       # project report
+ *   judges hook install                                   # install pre-commit
+ *   judges eval --help                                    # show help
+ */
+export { globToRegex, matchesGlob } from "./cli-helpers.js";
+interface CollectOptions {
+    exclude?: string[];
+    include?: string[];
+    maxFiles?: number;
+    sample?: boolean;
+}
+export declare function collectFiles(target: string, options?: CollectOptions): string[];
+export declare function runCli(argv: string[]): Promise<void>;