@nexrall/code-core 1.4.73 → 1.4.75

package/dist/agent/taskEvents.js

@@ -0,0 +1,297 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports._taskEventsInternal = void 0;
+exports.taskScopeRoot = taskScopeRoot;
+exports.taskScopeDir = taskScopeDir;
+exports.taskEventsFile = taskEventsFile;
+exports.emitTaskEvent = emitTaskEvent;
+exports.takeNewTaskEvents = takeNewTaskEvents;
+exports.formatTaskEventNotes = formatTaskEventNotes;
+/**
+ * Task board events — the notification side of the shared task list
+ * (@see ./sharedTasks.ts for the board itself).
+ *
+ * WHY THIS EXISTS: the board is a directory of files, so a change made by one
+ * session becomes visible to every other session the moment it lands — and
+ * visibility with no signal is a poll tax. The concrete failure it removes:
+ * session B creates "write the release notes" blocked by "cut the release",
+ * session A runs `nex tasks done cut-the-release` (or its own tool call), and
+ * B — sitting at its prompt or mid-turn — has no way to learn its task just
+ * became claimable except by re-listing the board on a hunch. Claude Code's
+ * agent teams has no cross-session signal at all: teammates learn about each
+ * other's progress only by polling the task files.
+ *
+ * HOW: every mutation appends one JSON line to `events.jsonl` in the board
+ * scope directory (same directory as the task files; `listSharedTasks` skips
+ * it — it only reads `*.json`). Readers keep a per-(scope, session) BYTE
+ * cursor and pull what is new; there is no watcher, no socket and no queue,
+ * which is what lets the same mechanism work no matter who made the change —
+ * a tool call inside another interactive session, a headless `nex --bg` run,
+ * a VS Code window, or a person typing `nex tasks done` in a shell.
+ *
+ * DELIVERY SEMANTICS, on purpose:
+ *   • A reader adopts the CURRENT end of the file the first time it looks, so
+ *     a fresh session is never handed a replay of the board's history — the
+ *     board STATE is what tools read; events are only about what changed
+ *     while you were watching. (chat.ts primes the cursor at session start —
+ *     see the takeNewTaskEvents call next to its CheckpointManager — rather
+ *     than letting the first in-loop drain anchor, so an event that lands
+ *     while the session's opening request is in flight is still delivered.)
+ *   • A session never hears about its own actions: events carry the writing
+ *     process's pid and readers drop their own pid. (Frame: events are
+ *     notifications from OTHER processes. Within one process, e.g. two VS
+ *     Code chat panels on the same workspace, the panel that made a change
+ *     will not self-notify — its own tool result already said what happened.)
+ *   • A torn line (a reader racing a writer mid-append) is never half-parsed:
+ *     the cursor only advances past the last complete line.
+ *
+ * The module must never make a board mutation fail — every write here is
+ * best-effort and swallows its own errors. The board is the product; the
+ * events are a courtesy on top of it.
+ */
+const crypto = __importStar(require("crypto"));
+const fs = __importStar(require("fs"));
+const os = __importStar(require("os"));
+const path = __importStar(require("path"));
+// ─── scope ────────────────────────────────────────────────────────────────────
+// Moved here (from sharedTasks.ts) so this module can address the board
+// directory without importing the mutators — one canonical hash, no cycle:
+// sharedTasks imports THIS module, never the other way round.
+function tasksRoot() {
+    return process.env.NEXRALL_SHARED_TASKS_DIR || path.join(os.homedir(), '.nexrall', 'peers', 'tasks');
+}
+/** The root directory all board scopes live under (env-overridable — tests redirect it). */
+function taskScopeRoot() {
+    return tasksRoot();
+}
+/** The board scope directory for a project — one per resolved workDir, shared by every process. */
+function taskScopeDir(workDir) {
+    const hash = crypto.createHash('sha256').update(path.resolve(workDir)).digest('hex').slice(0, 24);
+    return path.join(tasksRoot(), hash);
+}
+function taskEventsFile(workDir) {
+    return path.join(taskScopeDir(workDir), 'events.jsonl');
+}
+// ─── write ────────────────────────────────────────────────────────────────────
+/**
+ * Append one event to the board's log. Called by every mutator in sharedTasks
+ * right after its atomic write lands. Best-effort by contract: a failure here
+ * must NEVER surface to the caller — the mutation itself already succeeded.
+ */
+function emitTaskEvent(dir, event) {
+    try {
+        fs.mkdirSync(dir, { recursive: true });
+        const line = JSON.stringify({ ...event, ts: event.ts ?? Date.now(), pid: event.pid ?? process.pid }) + '\n';
+        // One appendFileSync per event: O_APPEND, so concurrent writers cannot
+        // overwrite each other, and a single small write lands whole (readers
+        // still guard against the torn-line case rather than trusting that).
+        fs.appendFileSync(eventsFileIn(dir), line);
+    }
+    catch {
+        /* courtesy, not contract */
+    }
+}
+function eventsFileIn(dir) {
+    return path.join(dir, 'events.jsonl');
+}
+// ─── read ─────────────────────────────────────────────────────────────────────
+/** Per-(scope, session) byte cursors, in memory: a process restarts → a session restarts. */
+const _cursors = new Map();
+/**
+ * New events for this reader — everything appended since the last call, minus
+ * this process's own writes. The FIRST call for a (scope, session) adopts the
+ * current end of the log and returns nothing: events describe change relative
+ * to when you started looking, never the board's backstory.
+ */
+function takeNewTaskEvents(workDir, opts = {}) {
+    const dir = taskScopeDir(workDir);
+    const key = `${dir}|${opts.sessionKey ?? ''}`;
+    const file = eventsFileIn(dir);
+    let size;
+    try {
+        size = fs.statSync(file).size;
+    }
+    catch {
+        // No log yet. Still ANCHOR (at zero): the file appearing later is exactly
+        // the "new events" case — without this, a brand-new board's first activity
+        // would be adopted-away by the next call's first-sight rule.
+        if (!_cursors.has(key))
+            _cursors.set(key, 0);
+        return [];
+    }
+    const from = _cursors.get(key);
+    if (from === undefined) {
+        _cursors.set(key, size);
+        return [];
+    }
+    if (size <= from) {
+        // Equal: nothing new. Shrunk: the log was replaced under us (someone
+        // cleared the scope directory) — re-anchor rather than replay, the only
+        // two options that cannot double-deliver.
+        if (size < from)
+            _cursors.set(key, size);
+        return [];
+    }
+    const length = size - from;
+    let buf;
+    try {
+        const fd = fs.openSync(file, 'r');
+        try {
+            buf = Buffer.alloc(length);
+            const read = fs.readSync(fd, buf, 0, length, from);
+            if (read < length)
+                buf = buf.subarray(0, read);
+        }
+        finally {
+            fs.closeSync(fd);
+        }
+    }
+    catch {
+        return [];
+    }
+    // Only advance past the last COMPLETE line — a writer may be mid-append.
+    const lastNewline = buf.lastIndexOf(0x0a);
+    if (lastNewline === -1)
+        return [];
+    const complete = buf.subarray(0, lastNewline + 1);
+    _cursors.set(key, from + complete.length);
+    const selfPid = opts.selfPid ?? process.pid;
+    const events = [];
+    for (const line of complete.toString('utf-8').split('\n')) {
+        if (!line.trim())
+            continue;
+        try {
+            const ev = JSON.parse(line);
+            if (ev && typeof ev.taskId === 'string' && typeof ev.type === 'string' && typeof ev.subject === 'string') {
+                events.push(ev);
+            }
+        }
+        catch {
+            /* a corrupt line is skipped, never fatal */
+        }
+    }
+    return events.filter((ev) => ev.pid !== selfPid);
+}
+/** Deps of `task` still NOT completed, given the live board. */
+function unmetDeps(task, byId) {
+    return (task.blockedBy ?? []).filter((id) => byId.get(id)?.status !== 'completed');
+}
+function quoteRef(id, content) {
+    const subject = content.replace(/\s+/g, ' ').trim();
+    return `[${id}] "${subject.length > 80 ? subject.slice(0, 79) + '…' : subject}"`;
+}
+function quote(t) {
+    return quoteRef(t.id, t.content);
+}
+/** An event carries the subject directly — the task file may already be gone (removed). */
+function quoteEvent(e) {
+    return quoteRef(e.taskId, e.subject);
+}
+/**
+ * Turn raw events into model-bound notes — but only for the events that
+ * actually changed what somebody can DO, because the rest are board gossip:
+ *
+ *   completed → every open task waiting on it is now unblocked (or one dep
+ *               closer), for whoever gets there first;
+ *   reopened  → the reverse: waiting tasks are stuck again;
+ *   removed   → waiting tasks are stuck FOREVER (a dep that can never
+ *               complete) unless it comes back — the one case the board
+ *               itself cannot express, since the task file is gone.
+ *
+ * created / claimed / released produce no note: they change no one's options
+ * (released → the task is pending again, which its `blockedBy` already gates).
+ * Hooks still see every event — the matcher decides — but the model only gets
+ * told about consequences.
+ */
+function formatTaskEventNotes(events, tasks) {
+    const byId = new Map(tasks.map((t) => [t.id, t]));
+    const open = tasks.filter((t) => t.status !== 'completed');
+    const notes = [];
+    for (const event of events) {
+        if (event.type !== 'completed' && event.type !== 'reopened' && event.type !== 'removed')
+            continue;
+        const waiters = open.filter((t) => t.id !== event.taskId && (t.blockedBy ?? []).includes(event.taskId));
+        if (!waiters.length)
+            continue;
+        const actor = event.actor ? ` by ${event.actor}` : '';
+        if (event.type === 'completed') {
+            const nowFree = [];
+            const stillStuck = [];
+            for (const t of waiters) {
+                (unmetDeps(t, byId).length === 0 ? nowFree : stillStuck).push(quote(t));
+            }
+            const lines = [`Task ${quoteEvent(event)} was completed${actor}.`];
+            if (nowFree.length) {
+                lines.push(`Now unblocked: ${nowFree.join(', ')} — claim${nowFree.length === 1 ? ' it' : ' them'} when ready.`);
+            }
+            if (stillStuck.length) {
+                lines.push(`Still waiting on other tasks: ${stillStuck.join(', ')}.`);
+            }
+            notes.push({
+                event,
+                text: envelope(lines.join('\n')),
+                unblocked: waiters.filter((t) => unmetDeps(t, byId).length === 0).map((t) => t.id),
+            });
+        }
+        else if (event.type === 'reopened') {
+            notes.push({
+                event,
+                text: envelope(`Task ${quoteEvent(event)} was reopened${actor} — it must be completed again before: ${waiters.map(quote).join(', ')}.`),
+                unblocked: [],
+            });
+        }
+        else {
+            notes.push({
+                event,
+                text: envelope(`Task ${quoteEvent(event)} was removed${actor} while ${waiters.map(quote).join(', ')} still ${waiters.length === 1 ? 'waits' : 'wait'} on it. A removed task can never complete, so those stay blocked until it is recreated (or they are re-created without it).`),
+                unblocked: [],
+            });
+        }
+    }
+    return notes;
+}
+function envelope(body) {
+    return (`<task-event source="shared-tasks">\n` +
+        `${body}\n` +
+        `This is an automated notification, not a message from the user.\n` +
+        `</task-event>`);
+}
+/** Test-only handles — the cursor map is module state and must be observable/resettable. */
+exports._taskEventsInternal = {
+    _cursors,
+    eventsFileIn,
+};
+//# sourceMappingURL=taskEvents.js.map

package/dist/index.d.ts

@@ -36,11 +36,13 @@ export * from './plugins/index';
 export * from './plugins/installer';
 export * from './plugins/sources';
 export * from './plugins/data';
+export * from './plugins/eval';
 export * from './agent/trust';
 export * from './agent/crossProcessLock';
 export * from './agent/worktree';
 export * from './agent/worktreeEnforcement';
 export * from './agent/sharedTasks';
+export * from './agent/taskEvents';
 export * from './agent/peerRegistry';
 export * from './agent/peerTransport';
 export * from './agent/askOnce';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,4BAA4B,CAAC;AAC3C,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,eAAe,CAAC;AAC9B,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC;AACpC,cAAc,mBAAmB,CAAC;AAClC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,eAAe,CAAC;AAC9B,cAAc,0BAA0B,CAAC;AACzC,cAAc,kBAAkB,CAAC;AACjC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,qBAAqB,CAAC;AACpC,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,iBAAiB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,4BAA4B,CAAC;AAC3C,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,eAAe,CAAC;AAC9B,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC;AACpC,cAAc,mBAAmB,CAAC;AAClC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,eAAe,CAAC;AAC9B,cAAc,0BAA0B,CAAC;AACzC,cAAc,kBAAkB,CAAC;AACjC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,iBAAiB,CAAC"}

package/dist/index.js

@@ -52,11 +52,13 @@ __exportStar(require("./plugins/index"), exports);
 __exportStar(require("./plugins/installer"), exports);
 __exportStar(require("./plugins/sources"), exports);
 __exportStar(require("./plugins/data"), exports);
+__exportStar(require("./plugins/eval"), exports);
 __exportStar(require("./agent/trust"), exports);
 __exportStar(require("./agent/crossProcessLock"), exports);
 __exportStar(require("./agent/worktree"), exports);
 __exportStar(require("./agent/worktreeEnforcement"), exports);
 __exportStar(require("./agent/sharedTasks"), exports);
+__exportStar(require("./agent/taskEvents"), exports);
 __exportStar(require("./agent/peerRegistry"), exports);
 __exportStar(require("./agent/peerTransport"), exports);
 __exportStar(require("./agent/askOnce"), exports);

package/dist/plugins/eval.d.ts

@@ -0,0 +1,199 @@
+export declare const EVAL_SCHEMA_VERSION = 1;
+/** CC's default: each case runs three times. */
+export declare const EVAL_DEFAULT_RUNS = 3;
+export declare const EVAL_MAX_RUNS = 50;
+export declare const EVAL_DEFAULT_THRESHOLD = 1;
+export declare const EVAL_DEFAULT_MAX_TURNS = 10;
+export declare const EVAL_DEFAULT_TIMEOUT_S = 300;
+/** CC caps both; larger values are refused, not clamped silently. */
+export declare const EVAL_MAX_TURNS_CEILING = 200;
+export declare const EVAL_MAX_TIMEOUT_S = 3600;
+export type GraderType = 'regex' | 'tool_used' | 'tool_order' | 'file_exists' | 'llm' | 'baseline';
+/**
+ * Which arm a grader scores in.
+ *  · 'both' (default) — counted for the plugin arm AND the no-plugin baseline.
+ *  · 'with-only' — counted only when the plugin was loaded (CC's rule for the
+ *    two-arm runs: a grader that can only pass WITH the plugin must not drag
+ *    the baseline down).
+ */
+export type GraderArm = 'both' | 'with-only';
+export interface EvalGrader {
+    /** File name inside graders/, for reporting. */
+    file: string;
+    type: GraderType;
+    weight: number;
+    arm: GraderArm;
+    pattern?: string;
+    flags?: string;
+    /** 'contains' (default) | 'not_contains' | 'count:<n>' */
+    match?: string;
+    /** 'last_message' (default) | 'trace' | 'file' */
+    target?: string;
+    /** target 'file': project-relative path or glob to read. */
+    pathPattern?: string;
+    tool?: string;
+    inputMatch?: string;
+    min?: number;
+    max?: number;
+    before?: string;
+    after?: string;
+    path?: string;
+    exists?: boolean;
+    criteria?: string;
+    /** llm: 'last_message' (default) | 'trace'. */
+    focus?: string;
+    /** baseline: the reference transcript file (case-relative). */
+    baselineFile?: string;
+}
+export interface EvalCase {
+    name: string;
+    dir: string;
+    prompt: string;
+    runs: number;
+    maxTurns: number;
+    timeoutSeconds: number;
+    allowedTools: string[];
+    tags: string[];
+    expectedOutcome?: string;
+    graders: EvalGrader[];
+}
+export interface DiscoveredSuite {
+    cases: EvalCase[];
+    /** Case dirs that failed to parse — reported, never silently dropped. */
+    errors: Array<{
+        dir: string;
+        error: string;
+    }>;
+}
+/** One tool call as the child's stream reported it. */
+export interface EvalToolCall {
+    tool: string;
+    input: unknown;
+}
+/**
+ * What ONE run produced. `readFile` resolves a project-relative path or glob
+ * against the run's workspace — the caller owns the filesystem, so the graders
+ * stay pure.
+ */
+export interface EvalRunTrace {
+    lastMessage: string;
+    toolCalls: EvalToolCall[];
+    files: string[];
+    readFile: (relOrGlob: string) => string | null;
+    /** USD when the server reported it; null = unreported (never invented). */
+    costUsd: number | null;
+    tokens: number;
+    exitCode: number;
+    error?: string;
+}
+export interface GraderVerdict {
+    passed: boolean;
+    why: string;
+}
+/** The injected model call for llm/baseline graders. null = could not judge. */
+export type EvalJudge = (args: {
+    criteria: string;
+    focusText: string;
+    baselineText?: string;
+}) => Promise<boolean | null>;
+export interface CaseRunOutcome {
+    arm: 'with' | 'without';
+    run: number;
+    score: number;
+    graderResults: Array<{
+        file: string;
+        type: GraderType;
+        weight: number;
+        passed: boolean;
+        why: string;
+    }>;
+    costUsd: number | null;
+    costReported: boolean;
+    durationMs: number;
+    error?: string;
+}
+export interface CaseOutcome {
+    name: string;
+    with: CaseRunOutcome[];
+    without: CaseRunOutcome[];
+    withScore: number | null;
+    withoutScore: number | null;
+    delta: number | null;
+    passed: boolean;
+}
+export interface EvalReport {
+    schemaVersion: number;
+    plugin: string;
+    startedAt: number;
+    durationMs: number;
+    threshold: number;
+    runs: number;
+    judgeModel?: string;
+    partial?: boolean;
+    partialReason?: string;
+    cases: CaseOutcome[];
+    errors: Array<{
+        dir: string;
+        error: string;
+    }>;
+    aggregates: {
+        casesPassed: number;
+        casesTotal: number;
+        overallScore: number;
+        meanDelta: number | null;
+    };
+    costUsd: number;
+    costMissing: number;
+}
+/** `[a, b]` / `a, b` / `a` → ['a','b'] — the two shapes real files use. */
+export declare function parseList(value: string | undefined): string[];
+/**
+ * One grader file → EvalGrader. Throws on anything that would make the grader
+ * silently unscorable (unknown type, missing pattern/tool/path, bad count).
+ */
+export declare function parseGraderFile(file: string, raw: string): EvalGrader;
+/** One case directory → EvalCase. See the header for the layout. */
+export declare function parseEvalCase(dir: string): EvalCase;
+/** Every case under `<pluginDir>/evals` (the results dir is never a case). */
+export declare function discoverEvalCases(pluginDir: string): DiscoveredSuite;
+/** The text a grader's target names. Exposed for tests and the report's `why`. */
+export declare function targetText(g: EvalGrader, trace: EvalRunTrace): string | null;
+/**
+ * Score ONE grader over ONE run. Async because llm/baseline call the judge.
+ * A grader that cannot be evaluated FAILS (with the reason in `why`) — it must
+ * never pass by accident, or a broken suite would look green.
+ */
+export declare function evaluateGrader(g: EvalGrader, trace: EvalRunTrace, judge: EvalJudge | undefined): Promise<GraderVerdict>;
+/** Which graders count in an arm: with-only graders never score the baseline. */
+export declare function gradersForArm(graders: EvalGrader[], arm: 'with' | 'without'): EvalGrader[];
+/** Score one run: weighted fraction of its graders that passed. */
+export declare function scoreRun(graders: EvalGrader[], trace: EvalRunTrace, judge: EvalJudge | undefined, arm: 'with' | 'without'): Promise<{
+    score: number;
+    results: CaseRunOutcome['graderResults'];
+}>;
+/**
+ * Fold run outcomes into the report. Case score = mean of the plugin-arm run
+ * scores; `delta` = with − without (null when the baseline did not run).
+ * A case PASSES at ≥ threshold (CC's default: every grader, every run).
+ */
+export declare function aggregateReport(i: {
+    plugin: string;
+    startedAt: number;
+    durationMs: number;
+    threshold: number;
+    runs: number;
+    judgeModel?: string;
+    cases: Array<{
+        name: string;
+        with: CaseRunOutcome[];
+        without: CaseRunOutcome[];
+    }>;
+    errors: Array<{
+        dir: string;
+        error: string;
+    }>;
+    partial?: {
+        reason: string;
+    };
+}): EvalReport;
+//# sourceMappingURL=eval.d.ts.map

package/dist/plugins/eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../src/plugins/eval.ts"],"names":[],"mappings":"AAiCA,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC,gDAAgD;AAChD,eAAO,MAAM,iBAAiB,IAAI,CAAC;AACnC,eAAO,MAAM,aAAa,KAAK,CAAC;AAChC,eAAO,MAAM,sBAAsB,IAAM,CAAC;AAC1C,eAAO,MAAM,sBAAsB,KAAK,CAAC;AACzC,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAC1C,qEAAqE;AACrE,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAC1C,eAAO,MAAM,kBAAkB,OAAO,CAAC;AAMvC,MAAM,MAAM,UAAU,GAAG,OAAO,GAAG,WAAW,GAAG,YAAY,GAAG,aAAa,GAAG,KAAK,GAAG,UAAU,CAAC;AAEnG;;;;;;GAMG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,WAAW,CAAC;AAE7C,MAAM,WAAW,UAAU;IACzB,gDAAgD;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,UAAU,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,SAAS,CAAC;IAEf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0DAA0D;IAC1D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,kDAAkD;IAClD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4DAA4D;IAC5D,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,OAAO,CAAC;IAEjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+CAA+C;IAC/C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+DAA+D;IAC/D,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,yEAAyE;IACzE,MAAM,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC/C;AAED,uDAAuD;AACvD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IAC/C,2EAA2E;IAC3E,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,OAAO,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;CACb;AAED,gFAAgF;AAChF,MAAM,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,KAAK,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;AAE9B,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,UAAU,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACvG,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,YAAY,EAAE,OAAO,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,cAAc,EAAE,CAAC;IACvB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,WAAW,EAAE,CAAC;IACrB,MAAM,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9C,UAAU,EAAE;QACV,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;KAC1B,CAAC;IACF,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;CACrB;AAID,2EAA2E;AAC3E,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,EAAE,CAO7D;AAUD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,UAAU,CAoErE;AAED,oEAAoE;AACpE,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,QAAQ,CAmDnD;AAED,8EAA8E;AAC9E,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,eAAe,CA6BpE;AAID,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,YAAY,GAAG,MAAM,GAAG,IAAI,CAY5E;AAkBD;;;;GAIG;AACH,wBAAsB,cAAc,CAClC,CAAC,EAAE,UAAU,EACb,KAAK,EAAE,YAAY,EACnB,KAAK,EAAE,SAAS,GAAG,SAAS,GAC3B,OAAO,CAAC,aAAa,CAAC,CAgFxB;AAED,iFAAiF;AACjF,wBAAgB,aAAa,CAAC,OAAO,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,SAAS,GAAG,UAAU,EAAE,CAE1F;AAED,mEAAmE;AACnE,wBAAsB,QAAQ,CAC5B,OAAO,EAAE,UAAU,EAAE,EACrB,KAAK,EAAE,YAAY,EACnB,KAAK,EAAE,SAAS,GAAG,SAAS,EAC5B,GAAG,EAAE,MAAM,GAAG,SAAS,GACtB,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,cAAc,CAAC,eAAe,CAAC,CAAA;CAAE,CAAC,CAatE;AAID;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,KAAK,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,cAAc,EAAE,CAAC;QACvB,OAAO,EAAE,cAAc,EAAE,CAAC;KAC3B,CAAC,CAAC;IACH,MAAM,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9C,OAAO,CAAC,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9B,GAAG,UAAU,CAsCb"}