@nexrall/code-core 1.4.74 → 1.4.76

package/dist/plugins/eval.js

@@ -0,0 +1,470 @@
+"use strict";
+// S21 — `nex plugin eval`: run a plugin against a suite of test cases and score
+// the results (Claude Code's `claude plugin eval`, shipped in their W37).
+//
+// This file is the PURE half: case discovery/parsing, the grader vocabulary,
+// the trace-based graders, and the scoring/aggregation. The caller owns ALL
+// I/O — spawning each run, reading the child's event log, and judging (the
+// `llm`/`baseline` graders call a model through an injected `judge` function so
+// this module stays unit-testable without a backend).
+//
+// Suite layout (CC's, minus what we do not support yet — see below):
+//
+//   my-plugin/
+//     evals/
+//       first-case/
+//         prompt.md          # frontmatter = case fields; body = the prompt
+//         graders/
+//           criteria.md      # frontmatter = grader; body = llm criteria
+//           skill-fired.md
+//       results/             # written by each run; never discovered as a case
+//
+// Deliberately NOT supported yet (refused loudly, never silently skipped):
+//   · case.yaml — needs a real YAML parser, which this repo does not carry
+//     (the frontmatter reader is "not a full YAML parser" by design). A case
+//     that ships one is an ERROR, because silently ignoring its fields would
+//     run a DIFFERENT suite than the author wrote.
+//   · mock MCP servers (`evals/mocks/`) — a follow-up; a case using them is
+//     likewise refused rather than run without the mocks.
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EVAL_MAX_TIMEOUT_S = exports.EVAL_MAX_TURNS_CEILING = exports.EVAL_DEFAULT_TIMEOUT_S = exports.EVAL_DEFAULT_MAX_TURNS = exports.EVAL_DEFAULT_THRESHOLD = exports.EVAL_MAX_RUNS = exports.EVAL_DEFAULT_RUNS = exports.EVAL_SCHEMA_VERSION = void 0;
+exports.parseList = parseList;
+exports.parseGraderFile = parseGraderFile;
+exports.parseEvalCase = parseEvalCase;
+exports.discoverEvalCases = discoverEvalCases;
+exports.targetText = targetText;
+exports.evaluateGrader = evaluateGrader;
+exports.gradersForArm = gradersForArm;
+exports.scoreRun = scoreRun;
+exports.aggregateReport = aggregateReport;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const frontmatter_1 = require("../util/frontmatter");
+const nestedInstructions_1 = require("../agent/nestedInstructions");
+exports.EVAL_SCHEMA_VERSION = 1;
+/** CC's default: each case runs three times. */
+exports.EVAL_DEFAULT_RUNS = 3;
+exports.EVAL_MAX_RUNS = 50;
+exports.EVAL_DEFAULT_THRESHOLD = 1.0;
+exports.EVAL_DEFAULT_MAX_TURNS = 10;
+exports.EVAL_DEFAULT_TIMEOUT_S = 300;
+/** CC caps both; larger values are refused, not clamped silently. */
+exports.EVAL_MAX_TURNS_CEILING = 200;
+exports.EVAL_MAX_TIMEOUT_S = 3600;
+/** `match: count:N` in a regex grader. */
+const COUNT_RE = /^count:(\d+)$/;
+// ─── Parsing helpers ─────────────────────────────────────────────────────────
+/** `[a, b]` / `a, b` / `a` → ['a','b'] — the two shapes real files use. */
+function parseList(value) {
+    if (value === undefined)
+        return [];
+    const inner = value.trim().replace(/^\[|\]$/g, '');
+    return inner
+        .split(',')
+        .map((s) => s.trim().replace(/^["']|["']$/g, ''))
+        .filter(Boolean);
+}
+function parsePositiveInt(value, dflt, ceiling, what) {
+    if (value === undefined || value.trim() === '')
+        return dflt;
+    const n = Number(value.trim());
+    if (!Number.isInteger(n) || n < 1)
+        throw new Error(`${what} must be a positive integer (got "${value}")`);
+    if (n > ceiling)
+        throw new Error(`${what} ${n} exceeds the ceiling of ${ceiling}`);
+    return n;
+}
+/**
+ * One grader file → EvalGrader. Throws on anything that would make the grader
+ * silently unscorable (unknown type, missing pattern/tool/path, bad count).
+ */
+function parseGraderFile(file, raw) {
+    const { meta, body } = (0, frontmatter_1.parseFrontmatter)(raw);
+    const type = (meta.type ?? '').trim();
+    if (!['regex', 'tool_used', 'tool_order', 'file_exists', 'llm', 'baseline'].includes(type)) {
+        throw new Error(`grader "${file}": type must be regex|tool_used|tool_order|file_exists|llm|baseline (got "${meta.type ?? ''}")`);
+    }
+    const weightRaw = (meta.weight ?? '1').trim();
+    const weight = Number(weightRaw);
+    if (!Number.isFinite(weight) || weight <= 0)
+        throw new Error(`grader "${file}": weight must be a positive number`);
+    const arm = (meta.arm ?? 'both').trim() === 'with-only' ? 'with-only' : 'both';
+    if (meta.arm !== undefined && !['both', 'with-only'].includes(meta.arm.trim())) {
+        throw new Error(`grader "${file}": arm must be both or with-only (got "${meta.arm}")`);
+    }
+    const g = { file, type, weight, arm };
+    if (type === 'regex') {
+        if (!meta.pattern)
+            throw new Error(`grader "${file}": regex grader needs a \`pattern\``);
+        g.pattern = meta.pattern;
+        g.flags = meta.flags?.trim() || '';
+        const match = (meta.match ?? 'contains').trim();
+        if (!(match === 'contains' || match === 'not_contains' || COUNT_RE.test(match))) {
+            throw new Error(`grader "${file}": match must be contains | not_contains | count:<n> (got "${match}")`);
+        }
+        g.match = match;
+        const target = (meta.target ?? 'last_message').trim();
+        if (!['last_message', 'trace', 'file'].includes(target)) {
+            throw new Error(`grader "${file}": target must be last_message | trace | file (got "${target}")`);
+        }
+        g.target = target;
+        if (target === 'file') {
+            if (!meta.path)
+                throw new Error(`grader "${file}": target "file" needs a \`path\``);
+            g.pathPattern = meta.path;
+        }
+    }
+    else if (type === 'tool_used') {
+        if (!meta.tool)
+            throw new Error(`grader "${file}": tool_used grader needs a \`tool\``);
+        g.tool = meta.tool.trim();
+        g.inputMatch = meta.input_match?.trim() || undefined;
+        g.min = parsePositiveInt(meta.min, 1, 10000, 'min');
+        g.max = meta.max !== undefined && meta.max.trim() !== '' ? parsePositiveInt(meta.max, 1, 10000, 'max') : undefined;
+        if (g.max !== undefined && g.max < (g.min ?? 1))
+            throw new Error(`grader "${file}": max < min`);
+    }
+    else if (type === 'tool_order') {
+        if (!meta.before || !meta.after)
+            throw new Error(`grader "${file}": tool_order needs \`before\` and \`after\``);
+        g.before = meta.before.trim();
+        g.after = meta.after.trim();
+    }
+    else if (type === 'file_exists') {
+        if (!meta.path)
+            throw new Error(`grader "${file}": file_exists needs a \`path\``);
+        g.path = meta.path.trim();
+        const ex = (meta.exists ?? 'true').trim().toLowerCase();
+        g.exists = !(ex === 'false' || ex === '0' || ex === 'no');
+    }
+    else {
+        // llm / baseline: criteria from the body (the file IS the criteria text).
+        const criteria = body.trim();
+        if (!criteria)
+            throw new Error(`grader "${file}": ${type} grader needs criteria in the file body`);
+        g.criteria = criteria;
+        if (type === 'llm') {
+            const focus = (meta.focus ?? 'last_message').trim();
+            if (!['last_message', 'trace'].includes(focus)) {
+                throw new Error(`grader "${file}": focus must be last_message | trace (got "${focus}")`);
+            }
+            g.focus = focus;
+        }
+        else {
+            if (!meta.baseline_file)
+                throw new Error(`grader "${file}": baseline grader needs \`baseline_file\``);
+            g.baselineFile = meta.baseline_file.trim();
+        }
+    }
+    return g;
+}
+/** One case directory → EvalCase. See the header for the layout. */
+function parseEvalCase(dir) {
+    const promptFile = path.join(dir, 'prompt.md');
+    const caseYaml = path.join(dir, 'case.yaml');
+    if (fs.existsSync(caseYaml)) {
+        throw new Error('case.yaml is not supported yet (this repo carries no YAML parser) — move its fields into prompt.md frontmatter');
+    }
+    if (fs.existsSync(path.join(dir, 'mocks'))) {
+        throw new Error('mock MCP servers (mocks/) are not supported yet');
+    }
+    let raw;
+    try {
+        raw = fs.readFileSync(promptFile, 'utf-8');
+    }
+    catch {
+        throw new Error('no prompt.md (a case needs one; case.yaml alone is not supported yet)');
+    }
+    const { meta, body } = (0, frontmatter_1.parseFrontmatter)(raw);
+    const prompt = body.trim();
+    if (!prompt)
+        throw new Error('prompt.md has an empty prompt body');
+    const gradersDir = path.join(dir, 'graders');
+    let graderFiles;
+    try {
+        graderFiles = fs
+            .readdirSync(gradersDir)
+            .filter((f) => f.endsWith('.md'))
+            .sort();
+    }
+    catch {
+        throw new Error('no graders/ directory (at least one grader is required)');
+    }
+    if (!graderFiles.length)
+        throw new Error('graders/ holds no .md files (at least one grader is required)');
+    const graders = graderFiles.map((f) => parseGraderFile(f, fs.readFileSync(path.join(gradersDir, f), 'utf-8')));
+    return {
+        name: meta.name?.trim() || path.basename(dir),
+        dir,
+        prompt,
+        runs: parsePositiveInt(meta.runs, exports.EVAL_DEFAULT_RUNS, exports.EVAL_MAX_RUNS, 'runs'),
+        maxTurns: parsePositiveInt(meta.max_turns, exports.EVAL_DEFAULT_MAX_TURNS, exports.EVAL_MAX_TURNS_CEILING, 'max_turns'),
+        timeoutSeconds: parsePositiveInt(meta.timeout_seconds, exports.EVAL_DEFAULT_TIMEOUT_S, exports.EVAL_MAX_TIMEOUT_S, 'timeout_seconds'),
+        allowedTools: parseList(meta.allowed_tools),
+        tags: parseList(meta.tags),
+        expectedOutcome: meta.expected_outcome?.trim() || undefined,
+        graders,
+    };
+}
+/** Every case under `<pluginDir>/evals` (the results dir is never a case). */
+function discoverEvalCases(pluginDir) {
+    const evalsDir = path.join(pluginDir, 'evals');
+    const errors = [];
+    let names;
+    try {
+        names = fs
+            .readdirSync(evalsDir)
+            .filter((n) => n !== 'results' && !n.startsWith('.'))
+            .sort();
+    }
+    catch {
+        return { cases: [], errors: [] };
+    }
+    const cases = [];
+    for (const n of names) {
+        const dir = path.join(evalsDir, n);
+        let isDir;
+        try {
+            isDir = fs.statSync(dir).isDirectory();
+        }
+        catch {
+            isDir = false;
+        }
+        if (!isDir)
+            continue;
+        try {
+            cases.push(parseEvalCase(dir));
+        }
+        catch (e) {
+            errors.push({ dir, error: e.message });
+        }
+    }
+    return { cases, errors };
+}
+// ─── The graders ─────────────────────────────────────────────────────────────
+/** The text a grader's target names. Exposed for tests and the report's `why`. */
+function targetText(g, trace) {
+    if (g.type === 'regex' && g.target === 'trace') {
+        return trace.toolCalls.map((c) => `${c.tool} ${JSON.stringify(c.input)}`).join('\n');
+    }
+    if (g.type === 'regex' && g.target === 'file') {
+        const content = trace.readFile(g.pathPattern ?? '');
+        return content; // null = no matching file; the grader fails with that reason
+    }
+    if (g.type === 'llm' && g.focus === 'trace') {
+        return trace.toolCalls.map((c) => `${c.tool} ${JSON.stringify(c.input)}`).join('\n');
+    }
+    return trace.lastMessage;
+}
+/** null = the pattern does not compile; the grader FAILS with that reason. */
+function compileRegex(g) {
+    try {
+        return new RegExp(g.pattern ?? '', g.flags ?? '');
+    }
+    catch {
+        return null;
+    }
+}
+function countMatches(re, text) {
+    // A global clone: a non-global regex .match() returns the first hit only, and
+    // count:N must count every occurrence.
+    const flags = re.flags.includes('g') ? re.flags : re.flags + 'g';
+    return (text.match(new RegExp(re.source, flags)) ?? []).length;
+}
+/**
+ * Score ONE grader over ONE run. Async because llm/baseline call the judge.
+ * A grader that cannot be evaluated FAILS (with the reason in `why`) — it must
+ * never pass by accident, or a broken suite would look green.
+ */
+async function evaluateGrader(g, trace, judge) {
+    try {
+        switch (g.type) {
+            case 'regex': {
+                const text = targetText(g, trace);
+                if (text === null)
+                    return { passed: false, why: `no file matches ${g.pathPattern} (regex ${g.pattern})` };
+                const re = compileRegex(g);
+                if (!re)
+                    return { passed: false, why: `invalid pattern: ${g.pattern}` };
+                const match = g.match ?? 'contains';
+                if (match === 'contains') {
+                    return { passed: re.test(text), why: re.test(text) ? 'found' : `not found in ${g.target ?? 'last_message'}` };
+                }
+                if (match === 'not_contains') {
+                    return {
+                        passed: !re.test(text),
+                        why: re.test(text) ? `unexpectedly found in ${g.target ?? 'last_message'}` : 'absent, as required',
+                    };
+                }
+                const cm = COUNT_RE.exec(match);
+                const want = Number(cm[1]);
+                const got = countMatches(re, text);
+                return { passed: got === want, why: `count ${got}, wanted ${want}` };
+            }
+            case 'tool_used': {
+                const n = trace.toolCalls.filter((c) => {
+                    if (c.tool !== g.tool)
+                        return false;
+                    if (!g.inputMatch)
+                        return true;
+                    try {
+                        return new RegExp(g.inputMatch).test(JSON.stringify(c.input));
+                    }
+                    catch {
+                        return false;
+                    }
+                }).length;
+                const min = g.min ?? 1;
+                const ok = n >= min && (g.max === undefined || n <= g.max);
+                return {
+                    passed: ok,
+                    why: `${g.tool} used ${n} time(s)` +
+                        (ok ? '' : ` (wanted ${g.max === undefined ? `>=${min}` : `${min}..${g.max}`})`),
+                };
+            }
+            case 'tool_order': {
+                const iBefore = trace.toolCalls.findIndex((c) => c.tool === g.before);
+                const iAfter = trace.toolCalls.findIndex((c) => c.tool === g.after);
+                if (iBefore === -1)
+                    return { passed: false, why: `${g.before} never called` };
+                if (iAfter === -1)
+                    return { passed: false, why: `${g.after} never called` };
+                return iBefore < iAfter
+                    ? { passed: true, why: `${g.before} before ${g.after}` }
+                    : { passed: false, why: `${g.after} came before ${g.before}` };
+            }
+            case 'file_exists': {
+                const re = (0, nestedInstructions_1.pathGlobToRegExp)(g.path ?? '');
+                const hit = re ? trace.files.some((f) => re.test(f)) : false;
+                const want = g.exists !== false;
+                return {
+                    passed: hit === want,
+                    why: want ? (hit ? 'present' : 'missing') : hit ? 'present, wanted absent' : 'absent, as required',
+                };
+            }
+            case 'llm': {
+                if (!judge)
+                    return { passed: false, why: 'no judge available (pass --judge-model or provide NEXRALL_EVAL_JUDGE_MODEL)' };
+                const focusText = targetText(g, trace) ?? '';
+                const verdict = await judge({ criteria: g.criteria ?? '', focusText });
+                if (verdict === null)
+                    return { passed: false, why: 'judge could not be reached' };
+                return { passed: verdict, why: verdict ? 'judge: pass' : 'judge: fail' };
+            }
+            case 'baseline': {
+                if (!judge)
+                    return { passed: false, why: 'no judge available (pass --judge-model)' };
+                const baselineText = trace.readFile(g.baselineFile ?? '');
+                if (baselineText === null)
+                    return { passed: false, why: `baseline file ${g.baselineFile} not found` };
+                const verdict = await judge({ criteria: g.criteria ?? '', focusText: trace.lastMessage, baselineText });
+                if (verdict === null)
+                    return { passed: false, why: 'judge could not be reached' };
+                return { passed: verdict, why: verdict ? 'at least as good as the baseline' : 'worse than the baseline' };
+            }
+        }
+    }
+    catch (e) {
+        return { passed: false, why: e.message };
+    }
+}
+/** Which graders count in an arm: with-only graders never score the baseline. */
+function gradersForArm(graders, arm) {
+    return arm === 'with' ? graders : graders.filter((g) => g.arm !== 'with-only');
+}
+/** Score one run: weighted fraction of its graders that passed. */
+async function scoreRun(graders, trace, judge, arm) {
+    const active = gradersForArm(graders, arm);
+    if (!active.length)
+        return { score: 0, results: [] };
+    const results = [];
+    let passedWeight = 0;
+    let totalWeight = 0;
+    for (const g of active) {
+        const v = await evaluateGrader(g, trace, judge);
+        totalWeight += g.weight;
+        if (v.passed)
+            passedWeight += g.weight;
+        results.push({ file: g.file, type: g.type, weight: g.weight, passed: v.passed, why: v.why });
+    }
+    return { score: totalWeight > 0 ? passedWeight / totalWeight : 0, results };
+}
+const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);
+/**
+ * Fold run outcomes into the report. Case score = mean of the plugin-arm run
+ * scores; `delta` = with − without (null when the baseline did not run).
+ * A case PASSES at ≥ threshold (CC's default: every grader, every run).
+ */
+function aggregateReport(i) {
+    const cases = i.cases.map((c) => {
+        const withScore = mean(c.with.map((r) => r.score));
+        const withoutScore = mean(c.without.map((r) => r.score));
+        return {
+            name: c.name,
+            with: c.with,
+            without: c.without,
+            withScore,
+            withoutScore,
+            delta: withScore !== null && withoutScore !== null ? withScore - withoutScore : null,
+            passed: withScore !== null && withScore >= i.threshold,
+        };
+    });
+    const scores = cases.map((c) => c.withScore).filter((s) => s !== null);
+    const deltas = cases.map((c) => c.delta).filter((d) => d !== null);
+    const allRuns = cases.flatMap((c) => [...c.with, ...c.without]);
+    const reported = allRuns.filter((r) => r.costReported && r.costUsd !== null);
+    return {
+        schemaVersion: exports.EVAL_SCHEMA_VERSION,
+        plugin: i.plugin,
+        startedAt: i.startedAt,
+        durationMs: i.durationMs,
+        threshold: i.threshold,
+        runs: i.runs,
+        ...(i.judgeModel ? { judgeModel: i.judgeModel } : {}),
+        ...(i.partial ? { partial: true, partialReason: i.partial.reason } : {}),
+        cases,
+        errors: i.errors,
+        aggregates: {
+            casesPassed: cases.filter((c) => c.passed).length,
+            casesTotal: cases.length,
+            overallScore: scores.length ? scores.reduce((a, b) => a + b, 0) / scores.length : 0,
+            meanDelta: deltas.length ? deltas.reduce((a, b) => a + b, 0) / deltas.length : null,
+        },
+        costUsd: reported.reduce((a, r) => a + (r.costUsd ?? 0), 0),
+        costMissing: allRuns.length - reported.length,
+    };
+}
+//# sourceMappingURL=eval.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nexrall/code-core",
-  "version": "1.4.74",
+  "version": "1.4.76",
   "description": "Core agent loop, tools, and extension primitives for Nexrall Code — embed an AI coding agent in any Node.js application.",
   "license": "MIT",
   "author": "Nexrall <support@nexrall.com> (https://nexrall.com)",