@ashlr/hub 3.4.0 → 3.5.1

package/dist/core/local-eval/trace.js

@@ -0,0 +1,309 @@
+/**
+ * Watching the wire, so a trial that produces no result JSON is still legible.
+ *
+ * WHY THIS EXISTS. The agent CLI emits its result JSON once, at the end. A
+ * trial killed at the wall-clock budget therefore records zero tokens, zero
+ * turns and an empty transcript — the harness could say a trial hung, and
+ * nothing whatsoever about WHERE. That is the same shape of useless outcome as
+ * the false-success problem this harness was built to catch: a verdict nobody
+ * can act on.
+ *
+ * The fix is to stop relying on the end of the run for evidence. This module
+ * inserts a proxy between the agent and whatever it was pointed at, and records
+ * every request, every byte of every response, and — the fact that turns out to
+ * matter — whether a stream ever reached its terminator.
+ *
+ * A PURE OBSERVER. It forwards bytes, headers and status untouched and adds no
+ * timeout of its own. It is deliberately NOT the normalising shim: it composes
+ * in front of whatever base URL the run already uses, so switching tracing on
+ * cannot change what the model is asked or what the agent receives. An
+ * instrument that alters the thing it measures is worse than no instrument.
+ *
+ * NO ASSUMPTIONS ABOUT BLAME. `diagnoseTimeout` reports what the wire showed
+ * and nothing more: a stream still delivering tokens when the clock ran out is
+ * reported as exactly that, not as "the model is too slow".
+ */
+import { createWriteStream } from 'node:fs';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { createServer, request as httpRequest } from 'node:http';
+import { join } from 'node:path';
+const DEFAULT_STALL_MS = 120_000;
+/** Records SSE event names out of a byte stream, tolerating split chunks. */
+class EventCounter {
+    counts;
+    tail = '';
+    constructor(counts) {
+        this.counts = counts;
+    }
+    push(chunk) {
+        this.tail += chunk.toString('utf8');
+        for (;;) {
+            const nl = this.tail.indexOf('\n');
+            if (nl < 0)
+                break;
+            const line = this.tail.slice(0, nl);
+            this.tail = this.tail.slice(nl + 1);
+            const match = /^event:\s*(\S+)/.exec(line);
+            if (match?.[1])
+                this.counts.set(match[1], (this.counts.get(match[1]) ?? 0) + 1);
+        }
+        // A pathological single line must not grow without bound.
+        if (this.tail.length > 64_000)
+            this.tail = this.tail.slice(-1_000);
+    }
+}
+/**
+ * Start a tracing proxy in front of `upstream`.
+ *
+ * Binds an ephemeral port, so any number of trials can trace at once without
+ * agreeing on port numbers — which matters because the harness runs trials
+ * concurrently by default.
+ */
+export async function startTrace(opts) {
+    const upstream = new URL(opts.upstream);
+    const stallMs = opts.stallMs ?? DEFAULT_STALL_MS;
+    await mkdir(opts.captureDir, { recursive: true });
+    const journal = join(opts.captureDir, 'wire.jsonl');
+    const finished = [];
+    const live = new Map();
+    const sockets = new Set();
+    /** Writes still in flight. `close()` waits on them so the capture is whole. */
+    const pending = new Set();
+    const track = (p) => {
+        const settled = p.catch(() => undefined);
+        pending.add(settled);
+        void settled.finally(() => pending.delete(settled));
+    };
+    let seq = 0;
+    // One stream, not repeated appends: concurrent `appendFile` calls have no
+    // ordering guarantee, and a journal whose lines interleave is not forensics.
+    // Errors are swallowed — a write failure must never take down the run this is
+    // only observing.
+    const log = createWriteStream(journal, { flags: 'a' });
+    log.on('error', () => undefined);
+    const note = (entry) => {
+        log.write(`${JSON.stringify({ at: new Date().toISOString(), ...entry })}\n`);
+    };
+    const settle = (state, how, upstreamComplete) => {
+        if (!live.delete(state.seq))
+            return;
+        const capture = state.capture;
+        state.capture = null;
+        if (capture) {
+            track(new Promise((resolve) => { capture.end(() => resolve()); }));
+        }
+        const record = {
+            seq: state.seq,
+            url: state.url,
+            status: state.status,
+            ended: how,
+            bytes: state.bytes,
+            events: Object.fromEntries(state.events),
+            sawTerminator: (state.events.get('message_stop') ?? 0) > 0,
+            upstreamComplete,
+            waitedForFirstByteMs: state.firstByteAt === null ? null : state.firstByteAt - state.startedAt,
+            idleAtEndMs: state.lastByteAt === null ? null : Date.now() - state.lastByteAt,
+            durationMs: Date.now() - state.startedAt,
+        };
+        finished.push(record);
+        note({ kind: 'stream-end', ...record });
+    };
+    const server = createServer((req, res) => {
+        const chunks = [];
+        req.on('data', (c) => chunks.push(c));
+        req.on('end', () => {
+            const index = seq;
+            seq += 1;
+            const tag = String(index).padStart(4, '0');
+            const body = Buffer.concat(chunks);
+            const state = {
+                seq: index, url: req.url ?? '', startedAt: Date.now(),
+                firstByteAt: null, lastByteAt: null, bytes: 0, status: null, events: new Map(),
+                capture: null,
+            };
+            live.set(index, state);
+            if (body.length > 0)
+                track(writeFile(join(opts.captureDir, `req-${tag}.json`), body));
+            note({ kind: 'request', seq: index, method: req.method, url: req.url, bytes: body.length });
+            const headers = { 'content-type': 'application/json' };
+            if (typeof req.headers.accept === 'string')
+                headers['accept'] = req.headers.accept;
+            if (body.length > 0)
+                headers['content-length'] = body.length;
+            const forwarded = httpRequest({
+                protocol: upstream.protocol,
+                hostname: upstream.hostname,
+                port: upstream.port,
+                path: req.url,
+                method: req.method,
+                headers,
+            }, (up) => {
+                state.status = up.statusCode ?? null;
+                note({ kind: 'response-head', seq: index, status: up.statusCode,
+                    waitedMs: Date.now() - state.startedAt });
+                res.writeHead(up.statusCode ?? 502, {
+                    'content-type': String(up.headers['content-type'] ?? 'application/json'),
+                });
+                const counter = new EventCounter(state.events);
+                state.capture = createWriteStream(join(opts.captureDir, `res-${tag}.sse`));
+                state.capture.on('error', () => undefined);
+                up.on('data', (chunk) => {
+                    const now = Date.now();
+                    if (state.firstByteAt === null)
+                        state.firstByteAt = now;
+                    state.lastByteAt = now;
+                    state.bytes += chunk.length;
+                    counter.push(chunk);
+                    state.capture?.write(chunk);
+                    res.write(chunk);
+                });
+                up.on('end', () => { settle(state, 'upstream-end', up.complete); res.end(); });
+                up.on('aborted', () => { settle(state, 'upstream-aborted', false); res.destroy(); });
+                up.on('error', (err) => {
+                    note({ kind: 'upstream-error', seq: index, error: String(err) });
+                    settle(state, 'upstream-error', false);
+                    res.destroy();
+                });
+            });
+            forwarded.on('error', (err) => {
+                note({ kind: 'connect-error', seq: index, error: String(err) });
+                settle(state, 'upstream-error', false);
+                if (!res.headersSent)
+                    res.writeHead(502, { 'content-type': 'application/json' });
+                res.end(JSON.stringify({ error: String(err) }));
+            });
+            // The agent hanging up first is itself evidence — it is how a client-side
+            // give-up is told apart from a server that stopped writing.
+            res.on('close', () => {
+                if (live.has(index)) {
+                    settle(state, 'client-closed', false);
+                    forwarded.destroy();
+                }
+            });
+            if (body.length > 0)
+                forwarded.end(body);
+            else
+                forwarded.end();
+        });
+    });
+    server.on('connection', (socket) => {
+        sockets.add(socket);
+        socket.on('close', () => sockets.delete(socket));
+    });
+    await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
+    const address = server.address();
+    const port = typeof address === 'object' && address ? address.port : 0;
+    return {
+        baseUrl: `http://127.0.0.1:${port}`,
+        snapshot: () => buildTrace({ finished, live, stallMs, captureDir: opts.captureDir }),
+        close: async () => {
+            for (const state of [...live.values()])
+                settle(state, 'client-closed', false);
+            for (const socket of sockets)
+                socket.destroy();
+            await new Promise((resolve) => server.close(() => resolve()));
+            // The capture is the evidence, so it is flushed before the handle is
+            // considered closed — a caller that reads it next must find it whole.
+            await Promise.allSettled([...pending]);
+            await new Promise((resolve) => log.end(() => resolve()));
+        },
+    };
+}
+function buildTrace(args) {
+    const now = Date.now();
+    const openStreams = [...args.live.values()].map((s) => ({
+        seq: s.seq,
+        url: s.url,
+        status: s.status,
+        bytes: s.bytes,
+        events: Object.fromEntries(s.events),
+        openForMs: now - s.startedAt,
+        sinceLastByteMs: s.lastByteAt === null ? null : now - s.lastByteAt,
+        waitedForFirstByteMs: s.firstByteAt === null ? null : s.firstByteAt - s.startedAt,
+    }));
+    return {
+        captureDir: args.captureDir,
+        requests: args.finished.length + openStreams.length,
+        // COPIED, not referenced. The runner reads the trace at the kill and then
+        // closes the proxy, which settles every in-flight stream — and a snapshot
+        // that aliased the live array would gain those entries afterwards, so the
+        // one stream the diagnosis is about would appear as both still-open and
+        // already-complete. A snapshot has to be a snapshot.
+        streams: [...args.finished],
+        openStreams,
+        stallMs: args.stallMs,
+    };
+}
+/**
+ * Name what the wire was doing when the clock ran out.
+ *
+ * READ IN ORDER. The first cause that fits wins, because these are not
+ * independent: a stream still open explains everything downstream of it, and
+ * reporting a stale earlier stream instead would point at the wrong thing.
+ */
+export function diagnoseTimeout(trace) {
+    const stalled = trace.openStreams.filter((s) => s.sinceLastByteMs === null || s.sinceLastByteMs >= trace.stallMs);
+    const flowing = trace.openStreams.filter((s) => s.sinceLastByteMs !== null && s.sinceLastByteMs < trace.stallMs);
+    if (flowing.length > 0) {
+        const worst = flowing.reduce((a, b) => (a.openForMs >= b.openForMs ? a : b));
+        return {
+            kind: 'generating-at-cutoff',
+            detail: `request #${worst.seq} was still streaming when the budget expired: `
+                + `${Math.round(worst.openForMs / 1000)}s open, ${worst.bytes} bytes, `
+                + `last token ${Math.round((worst.sinceLastByteMs ?? 0) / 1000)}s before the kill`,
+            seq: worst.seq,
+        };
+    }
+    if (stalled.length > 0) {
+        const worst = stalled.reduce((a, b) => (a.openForMs >= b.openForMs ? a : b));
+        const never = worst.sinceLastByteMs === null;
+        return {
+            kind: never ? 'no-first-token' : 'stream-stalled',
+            detail: never
+                ? `request #${worst.seq} was accepted${worst.status === null ? ' but never answered' : ` with status ${worst.status}`} `
+                    + `and produced no bytes in ${Math.round(worst.openForMs / 1000)}s`
+                : `request #${worst.seq} stopped producing bytes `
+                    + `${Math.round((worst.sinceLastByteMs ?? 0) / 1000)}s before the kill and was never closed `
+                    + `(${worst.bytes} bytes, events ${JSON.stringify(worst.events)})`,
+            seq: worst.seq,
+        };
+    }
+    // Only completion streams are evidence. The CLI also issues a connectivity
+    // probe and a token count against the same base URL, and both close without
+    // a `message_stop` because neither is an SSE turn. Counting those would
+    // report a missing terminator on every single timeout — a diagnosis that
+    // fires always is worth exactly as much as no diagnosis at all.
+    const turns = trace.streams.filter((s) => (s.events['message_start'] ?? 0) > 0);
+    // And only a turn the SERVER closed is evidence ABOUT the server. The CLI
+    // opens a warm-up completion and cancels it a second later; a cancelled turn
+    // has no terminator by construction, so reading one as a server fault would
+    // blame the runtime for the client's own choice.
+    const serverClosed = turns.filter((s) => s.ended === 'upstream-end');
+    const last = serverClosed[serverClosed.length - 1];
+    if (!last) {
+        return {
+            kind: 'no-request-reached-the-model',
+            detail: turns.length > 0
+                ? `${turns.length} turn(s) were opened and the agent abandoned every one before the server `
+                    + 'closed it — the hang is on the client side of the proxy'
+                : `the agent never sent a completion request (${trace.requests} non-streaming request(s) seen) `
+                    + '— the hang is on the client side of the proxy',
+            seq: null,
+        };
+    }
+    if (!last.sawTerminator) {
+        return {
+            kind: 'stream-ended-without-terminator',
+            detail: `request #${last.seq} closed (${last.ended}) after ${last.bytes} bytes without a message_stop; `
+                + 'the agent was left waiting on a turn the server considered finished',
+            seq: last.seq,
+        };
+    }
+    return {
+        kind: 'idle-between-turns',
+        detail: `every request completed — ${serverClosed.length} turn(s), the last one terminated cleanly `
+            + `${Math.round(last.durationMs / 1000)}s long — and the agent then stopped without sending another`,
+        seq: last.seq,
+    };
+}
+//# sourceMappingURL=trace.js.map

package/dist/core/local-eval/trace.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"trace.js","sourceRoot":"","sources":["../../../src/core/local-eval/trace.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,iBAAiB,EAAoB,MAAM,SAAS,CAAC;AAC9D,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,OAAO,IAAI,WAAW,EAA0D,MAAM,WAAW,CAAC;AACzH,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAkDjC,MAAM,gBAAgB,GAAG,OAAO,CAAC;AAEjC,6EAA6E;AAC7E,MAAM,YAAY;IAEa;IADrB,IAAI,GAAG,EAAE,CAAC;IAClB,YAA6B,MAA2B;QAA3B,WAAM,GAAN,MAAM,CAAqB;IAAG,CAAC;IAC5D,IAAI,CAAC,KAAa;QAChB,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACpC,SAAS,CAAC;YACR,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACnC,IAAI,EAAE,GAAG,CAAC;gBAAE,MAAM;YAClB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACpC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YACpC,MAAM,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3C,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC;gBAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAClF,CAAC;QACD,0DAA0D;QAC1D,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM;YAAE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC;IACrE,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAuB;IACtD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,gBAAgB,CAAC;IACjD,MAAM,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;IAEpD,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC3C,MAAM,OAAO,GAAG,IAAI,GAAG,EAA6B,CAAC;IACrD,+EAA+E;IAC/E,MAAM,OAAO,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC5C,MAAM,KAAK,GAAG,CAAI,CAAa,EAAQ,EAAE;QACvC,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACzC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACrB,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;IACtD,CAAC,CAAC;IACF,IAAI,GAAG,GAAG,CAAC,CAAC;IAEZ,0EAA0E;IAC1E,6EAA6E;IAC7E,8EAA8E;IAC9E,kBAAkB;IAClB,MAAM,GAAG,GAAG,iBAAiB,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;IACvD,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,CAAC,KAA8B,EAAQ,EAAE;QACpD,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,GAAG,KAAK,EAAE,CAAC,IAAI,CAAC,CAAC;IAC/E,CAAC,CAAC;IAEF,MAAM,MAAM,GAAG,CAAC,KAAiB,EAAE,GAA0B,EAAE,gBAAyB,EAAQ,EAAE;QAChG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;YAAE,OAAO;QACpC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;QAC9B,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC;QACrB,IAAI,OAAO,EAAE,CAAC;YACZ,KAAK,CAAC,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,CAAC;QACD,MAAM,MAAM,GAAiB;YAC3B,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,KAAK,EAAE,GAAG;YACV,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC;YACxC,aAAa,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC;YAC1D,gBAAgB;YAChB,oBAAoB,EAAE,KAAK,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,SAAS;YAC7F,WAAW,EAAE,KAAK,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,UAAU;YAC7E,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS;SACzC,CAAC;QACF,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtB,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAC1C,CAAC,CAAC;IAEF,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,GAAoB,EAAE,GAAmB,EAAE,EAAE;QACxE,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YACjB,MAAM,KAAK,GAAG,GAAG,CAAC;YAClB,GAAG,IAAI,CAAC,CAAC;YACT,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAC3C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACnC,MAAM,KAAK,GAAe;gBACxB,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;gBACrD,WAAW,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,GAAG,EAAE;gBAC9E,OAAO,EAAE,IAAI;aACd,CAAC;YACF,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAEvB,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;gBAAE,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,GAAG,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;YACtF,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAE5F,MAAM,OAAO,GAAoC,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC;YACxF,IAAI,OAAO,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,QAAQ;gBAAE,OAAO,CAAC,QAAQ,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC;YACnF,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;YAE7D,MAAM,SAAS,GAAG,WAAW,CAAC;gBAC5B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,IAAI,EAAE,GAAG,CAAC,GAAG;gBACb,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,OAAO;aACR,EAAE,CAAC,EAAE,EAAE,EAAE;gBACR,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,UAAU,IAAI,IAAI,CAAC;gBACrC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,UAAU;oBACxD,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;gBACjD,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,UAAU,IAAI,GAAG,EAAE;oBAClC,cAAc,EAAE,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,kBAAkB,CAAC;iBACzE,CAAC,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC/C,KAAK,CAAC,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC;gBAC3E,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBAC3C,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;oBAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBACvB,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;wBAAE,KAAK,CAAC,WAAW,GAAG,GAAG,CAAC;oBACxD,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC;oBACvB,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;oBAC5B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACpB,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;oBAC5B,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACnB,CAAC,CAAC,CAAC;gBACH,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/E,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC,KAAK,EAAE,kBAAkB,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBACrF,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;oBACrB,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACjE,MAAM,CAAC,KAAK,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;oBACvC,GAAG,CAAC,OAAO,EAAE,CAAC;gBAChB,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBAC5B,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAChE,MAAM,CAAC,KAAK,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;gBACvC,IAAI,CAAC,GAAG,CAAC,WAAW;oBAAE,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAC;gBACjF,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAClD,CAAC,CAAC,CAAC;YAEH,0EAA0E;YAC1E,4DAA4D;YAC5D,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACnB,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBACpB,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,CAAC,CAAC;oBACtC,SAAS,CAAC,OAAO,EAAE,CAAC;gBACtB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;;gBAAM,SAAS,CAAC,GAAG,EAAE,CAAC;QACjE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,MAAM,EAAE,EAAE;QACjC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACpB,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;IACjC,MAAM,IAAI,GAAG,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAEvE,OAAO;QACL,OAAO,EAAE,oBAAoB,IAAI,EAAE;QACnC,QAAQ,EAAE,GAAG,EAAE,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC;QACpF,KAAK,EAAE,KAAK,IAAI,EAAE;YAChB,KAAK,MAAM,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;gBAAE,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,CAAC,CAAC;YAC9E,KAAK,MAAM,MAAM,IAAI,OAAO;gBAAE,MAAM,CAAC,OAAO,EAAE,CAAC;YAC/C,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAE,MAAiB,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAChF,qEAAqE;YACrE,sEAAsE;YACtE,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;YACvC,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACjE,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,IAKnB;IACC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACtD,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC;QACpC,SAAS,EAAE,GAAG,GAAG,CAAC,CAAC,SAAS;QAC5B,eAAe,EAAE,CAAC,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU;QAClE,oBAAoB,EAAE,CAAC,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,SAAS;KAClF,CAAC,CAAC,CAAC;IACJ,OAAO;QACL,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM;QACnD,0EAA0E;QAC1E,0EAA0E;QAC1E,0EAA0E;QAC1E,wEAAwE;QACxE,qDAAqD;QACrD,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC3B,WAAW;QACX,OAAO,EAAE,IAAI,CAAC,OAAO;KACtB,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAAC,KAAiB;IAC/C,MAAM,OAAO,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CACtC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,KAAK,IAAI,IAAI,CAAC,CAAC,eAAe,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;IAC3E,MAAM,OAAO,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CACtC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,KAAK,IAAI,IAAI,CAAC,CAAC,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;IAE1E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7E,OAAO;YACL,IAAI,EAAE,sBAAsB;YAC5B,MAAM,EAAE,YAAY,KAAK,CAAC,GAAG,gDAAgD;kBACzE,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,WAAW,KAAK,CAAC,KAAK,UAAU;kBACrE,cAAc,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,eAAe,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB;YACpF,GAAG,EAAE,KAAK,CAAC,GAAG;SACf,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7E,MAAM,KAAK,GAAG,KAAK,CAAC,eAAe,KAAK,IAAI,CAAC;QAC7C,OAAO;YACL,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,gBAAgB;YACjD,MAAM,EAAE,KAAK;gBACX,CAAC,CAAC,YAAY,KAAK,CAAC,GAAG,gBAAgB,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,gBAAgB,KAAK,CAAC,MAAM,EAAE,GAAG;sBACpH,4BAA4B,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG;gBACrE,CAAC,CAAC,YAAY,KAAK,CAAC,GAAG,2BAA2B;sBAC9C,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,eAAe,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,yCAAyC;sBAC3F,IAAI,KAAK,CAAC,KAAK,kBAAkB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG;YACtE,GAAG,EAAE,KAAK,CAAC,GAAG;SACf,CAAC;IACJ,CAAC;IAED,2EAA2E;IAC3E,4EAA4E;IAC5E,wEAAwE;IACxE,yEAAyE;IACzE,gEAAgE;IAChE,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAChF,0EAA0E;IAC1E,6EAA6E;IAC7E,4EAA4E;IAC5E,iDAAiD;IACjD,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,cAAc,CAAC,CAAC;IACrE,MAAM,IAAI,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACnD,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO;YACL,IAAI,EAAE,8BAA8B;YACpC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC;gBACtB,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,2EAA2E;sBACxF,yDAAyD;gBAC7D,CAAC,CAAC,8CAA8C,KAAK,CAAC,QAAQ,kCAAkC;sBAC5F,+CAA+C;YACrD,GAAG,EAAE,IAAI;SACV,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;QACxB,OAAO;YACL,IAAI,EAAE,iCAAiC;YACvC,MAAM,EAAE,YAAY,IAAI,CAAC,GAAG,YAAY,IAAI,CAAC,KAAK,WAAW,IAAI,CAAC,KAAK,iCAAiC;kBACpG,qEAAqE;YACzE,GAAG,EAAE,IAAI,CAAC,GAAG;SACd,CAAC;IACJ,CAAC;IACD,OAAO;QACL,IAAI,EAAE,oBAAoB;QAC1B,MAAM,EAAE,6BAA6B,YAAY,CAAC,MAAM,4CAA4C;cAChG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,6DAA6D;QACtG,GAAG,EAAE,IAAI,CAAC,GAAG;KACd,CAAC;AACJ,CAAC"}

package/dist/core/local-eval/types.d.ts

@@ -0,0 +1,262 @@
+/**
+ * Types for the local-agent evaluation harness.
+ *
+ * WHY THIS EXISTS. Every number we have about the local seat so far is a
+ * handful of one-line fixes timed by hand. That is enough to show a prompt-cache
+ * bug was costing 23,301 reprocessed tokens a turn, and not nearly enough to
+ * answer the only question that matters when tuning: did this change make the
+ * agent BETTER at real work, or just faster at failing?
+ *
+ * Three commitments are encoded in these types, each one a lesson this
+ * repository paid for:
+ *
+ *   1. A verdict is an EXIT CODE, never a model's opinion. `TaskSpec.verify` is
+ *      a command; its status decides the trial. No model ever judges a model.
+ *   2. A result is a DISTRIBUTION, never a sample. `TaskOutcome` carries every
+ *      trial, because on this hardware turns vary by tens of seconds and one
+ *      run has already produced a wrong conclusion here.
+ *   3. A number without its CONFIGURATION cannot be compared to anything, so
+ *      `HarnessConfiguration` is captured from the live runtime at run time and
+ *      stored beside the results rather than written down by hand.
+ */
+/** What the task is asking of the agent, which decides how it is graded. */
+export type TaskExpectation = 
+/** The agent should change the tree, and `verify` proves the change correct. */
+'edit'
+/**
+ * The agent should decline or ask. `verify` proves the tree is still intact,
+ * and the closing message must not assert a change it did not make.
+ */
+ | 'refuse';
+/** A single evaluation task. */
+export interface TaskSpec {
+    readonly id: string;
+    /** Why this task earns a place in the set. Printed in the report. */
+    readonly why: string;
+    readonly expectation: TaskExpectation;
+    /** The instruction handed to the agent verbatim. */
+    readonly prompt: string;
+    /** Fixture contents, written relative to the trial's working directory. */
+    readonly files: Readonly<Record<string, string>>;
+    /**
+     * Source of the pass check, written OUTSIDE the agent's working directory.
+     *
+     * This placement is load-bearing, not tidiness. A checker sitting next to the
+     * code is itself editable, and "make the failing test pass" has an obvious
+     * cheat if the test is in reach. Claude Code confines its file tools to the
+     * working directory, so a checker one level above it cannot be rewritten by
+     * the agent under evaluation.
+     */
+    readonly check: string;
+    /**
+     * The pass check. Argv, not a shell string: no quoting to get wrong, and
+     * nothing a fixture file name could inject into a shell.
+     */
+    readonly verify: readonly string[];
+}
+/**
+ * How a trial failed. `pass` is included so a trial always carries exactly one
+ * verdict and callers never have to combine a boolean with an enum.
+ */
+export type FailureMode = 'pass'
+/** THE ONE THAT MATTERS: reported a change, the tree never moved. */
+ | 'claimed-change-none-made'
+/** Declined or stopped on a task that was plainly doable. */
+ | 'refused-doable-task'
+/** Complied with a request it should have questioned. */
+ | 'complied-with-bad-request'
+/** Ended before finishing: cancelled, max turns, or a non-completed stop. */
+ | 'stopped-early'
+/** Edited something, but the result is wrong. */
+ | 'wrong-edit'
+/** Ran past the wall-clock budget and was killed. */
+ | 'timeout'
+/** Hit the model's context limit. */
+ | 'context-exhausted'
+/** The CLI or the runtime errored — not the model's fault, and not a pass. */
+ | 'harness-error';
+/**
+ * One request/response exchange between the agent and the runtime, as seen on
+ * the wire by the tracing proxy.
+ *
+ * `sawTerminator` is the field this whole record exists for. An SSE turn that
+ * closes without `message_stop` leaves the agent waiting on a socket the server
+ * already considers finished, and that is indistinguishable from a slow model
+ * to everything downstream — including the CLI's own result JSON, which an
+ * aborted turn is documented to report as COMPLETED.
+ */
+export interface StreamRecord {
+    readonly seq: number;
+    readonly url: string;
+    readonly status: number | null;
+    /** Which side ended it, and how. */
+    readonly ended: 'upstream-end' | 'upstream-aborted' | 'upstream-error' | 'client-closed';
+    readonly bytes: number;
+    /** SSE event names to counts, e.g. `{ content_block_delta: 512 }`. */
+    readonly events: Readonly<Record<string, number>>;
+    /** Whether a `message_stop` event ever arrived. */
+    readonly sawTerminator: boolean;
+    /** Node's own view of whether the response body arrived in full. */
+    readonly upstreamComplete: boolean;
+    readonly waitedForFirstByteMs: number | null;
+    /** Silence between the last byte and the close. Null when no byte arrived. */
+    readonly idleAtEndMs: number | null;
+    readonly durationMs: number;
+}
+/** A stream that had not finished when the trace was read. */
+export interface OpenStreamRecord {
+    readonly seq: number;
+    readonly url: string;
+    readonly status: number | null;
+    readonly bytes: number;
+    readonly events: Readonly<Record<string, number>>;
+    readonly openForMs: number;
+    /** Null when not one byte has arrived yet. */
+    readonly sinceLastByteMs: number | null;
+    readonly waitedForFirstByteMs: number | null;
+}
+/** Everything the tracing proxy saw during one trial. */
+export interface TrialTrace {
+    /** Where the raw request bodies and response streams were written. */
+    readonly captureDir: string;
+    readonly requests: number;
+    readonly streams: readonly StreamRecord[];
+    readonly openStreams: readonly OpenStreamRecord[];
+    /** Silence longer than this is reported as a stall rather than as slowness. */
+    readonly stallMs: number;
+}
+/**
+ * WHY a trial ran out of clock.
+ *
+ * Each value names a different thing to go and fix, which is the entire point:
+ * "timeout" alone is the same non-answer as a pass rate with no failure modes.
+ */
+export type TimeoutDiagnosisKind = 
+/** Tokens were still arriving when the budget expired. The model was working. */
+'generating-at-cutoff'
+/** A stream went silent and was never closed. The socket outlived the turn. */
+ | 'stream-stalled'
+/** A request was accepted and never produced a single byte. */
+ | 'no-first-token'
+/** The last stream closed with no `message_stop`; the agent waited forever. */
+ | 'stream-ended-without-terminator'
+/** Every stream completed cleanly and the agent simply stopped. Client-side. */
+ | 'idle-between-turns'
+/** The agent never reached the runtime at all. */
+ | 'no-request-reached-the-model';
+export interface TimeoutDiagnosis {
+    readonly kind: TimeoutDiagnosisKind;
+    /** One sentence naming the evidence, safe to print in a report. */
+    readonly detail: string;
+    /** The request the verdict is about, when there is one. */
+    readonly seq: number | null;
+}
+/** Token counts for one trial, as reported by the agent CLI. */
+export interface TrialTokens {
+    readonly input: number;
+    readonly output: number;
+    readonly cacheRead: number;
+    readonly cacheCreation: number;
+}
+/** One attempt at one task. */
+export interface TrialResult {
+    readonly taskId: string;
+    readonly trial: number;
+    readonly mode: FailureMode;
+    readonly passed: boolean;
+    /** Wall clock for the agent turn, milliseconds. */
+    readonly wallMs: number;
+    readonly tokens: TrialTokens;
+    /** Exit status of the agent CLI. */
+    readonly agentExit: number | null;
+    /** Exit status of the `verify` command; null when it was never reached. */
+    readonly verifyExit: number | null;
+    /** Files the agent actually changed, counted from the git diff. */
+    readonly changedFiles: number | null;
+    /** The agent's closing message, as classified by completion-claims. */
+    readonly claim: string;
+    readonly integrity: string;
+    /** Number of assistant turns the CLI reported. */
+    readonly turns: number | null;
+    /** Short human-readable note, e.g. the first line of a crash. */
+    readonly note: string;
+    /**
+     * What the wire was doing when the clock ran out. Null for every trial that
+     * did not time out — and never null for one that did, which is the guarantee
+     * this field was added to make.
+     */
+    readonly timeoutDiagnosis: TimeoutDiagnosis | null;
+    /**
+     * The wire summary, present whenever tracing was on. Kept on the trial rather
+     * than only in a log file so a recorded baseline stays diagnosable after the
+     * temporary directory it ran in is gone.
+     */
+    readonly trace: TrialTrace | null;
+}
+/** Every trial for one task, plus the aggregate that answers "did this help?". */
+export interface TaskOutcome {
+    readonly taskId: string;
+    readonly why: string;
+    readonly expectation: TaskExpectation;
+    readonly trials: readonly TrialResult[];
+    readonly passes: number;
+    readonly total: number;
+    readonly passRate: number;
+    readonly medianWallMs: number;
+    readonly meanWallMs: number;
+    /** Failure modes seen, most frequent first. Empty when every trial passed. */
+    readonly modes: readonly {
+        readonly mode: FailureMode;
+        readonly count: number;
+    }[];
+}
+/**
+ * The configuration a result is only meaningful against.
+ *
+ * Read from the live runtime rather than accepted as an argument wherever
+ * possible: a hand-written record of what was running is exactly the thing that
+ * goes stale and silently invalidates a comparison.
+ */
+export interface HarnessConfiguration {
+    /** Resolved model reference, e.g. `qwen3.8:27b-q8_0`. */
+    readonly model: string;
+    /** The on-disk blob actually loaded, which is what `model` is derived from. */
+    readonly modelPath: string;
+    readonly quantization: string;
+    readonly slots: number;
+    /** Context per slot, which is the number that bounds a single agent. */
+    readonly contextPerSlot: number;
+    /** Total context across all slots, i.e. llama-server's `-c`. */
+    readonly contextTotal: number;
+    readonly samplingParams: Readonly<Record<string, unknown>>;
+    /** The base URL the agent was pointed at. */
+    readonly baseUrl: string;
+    /** Whether that base URL is the normalising proxy or llama-server directly. */
+    readonly proxy: 'on' | 'off';
+    /**
+     * Whether the tracing proxy was inserted in front of `baseUrl`.
+     *
+     * Recorded because it changes what a timed-out trial can say about itself,
+     * and a baseline that predates tracing must not be mistaken for one where
+     * tracing found nothing.
+     */
+    readonly tracing: 'on' | 'off';
+    /** What is actually serving the proxy port, when that could be determined. */
+    readonly proxyImplementation: string;
+    readonly agentCli: string;
+    readonly llamaServerArgv: readonly string[];
+    readonly capturedAt: string;
+}
+/** A complete harness run: the configuration, and what it scored. */
+export interface EvalReport {
+    readonly configuration: HarnessConfiguration;
+    readonly trialsPerTask: number;
+    readonly concurrency: number;
+    readonly outcomes: readonly TaskOutcome[];
+    readonly overallPassRate: number;
+    readonly totalPasses: number;
+    readonly totalTrials: number;
+    readonly startedAt: string;
+    readonly finishedAt: string;
+    readonly wallMs: number;
+}

package/dist/core/local-eval/types.js

@@ -0,0 +1,23 @@
+/**
+ * Types for the local-agent evaluation harness.
+ *
+ * WHY THIS EXISTS. Every number we have about the local seat so far is a
+ * handful of one-line fixes timed by hand. That is enough to show a prompt-cache
+ * bug was costing 23,301 reprocessed tokens a turn, and not nearly enough to
+ * answer the only question that matters when tuning: did this change make the
+ * agent BETTER at real work, or just faster at failing?
+ *
+ * Three commitments are encoded in these types, each one a lesson this
+ * repository paid for:
+ *
+ *   1. A verdict is an EXIT CODE, never a model's opinion. `TaskSpec.verify` is
+ *      a command; its status decides the trial. No model ever judges a model.
+ *   2. A result is a DISTRIBUTION, never a sample. `TaskOutcome` carries every
+ *      trial, because on this hardware turns vary by tens of seconds and one
+ *      run has already produced a wrong conclusion here.
+ *   3. A number without its CONFIGURATION cannot be compared to anything, so
+ *      `HarnessConfiguration` is captured from the live runtime at run time and
+ *      stored beside the results rather than written down by hand.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/core/local-eval/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/core/local-eval/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG"}