@ashlr/hub 3.4.0 → 3.5.1

package/dist/core/local-eval/configuration.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Capture the configuration a result is only meaningful against.
+ *
+ * "Four agents completed 4/4" is not a fact anyone can act on. "Four agents
+ * completed 4/4 on qwen3.8:27b-q8_0 at 4 slots x 65,536 context, temp 1.0,
+ * through the normalising proxy" is. The difference is whether the next person
+ * to change a setting can tell if they improved anything.
+ *
+ * READ, DO NOT ASK. Every field here is observed from the running system —
+ * llama-server's own `/props`, the process's real argv, Ollama's manifest —
+ * rather than accepted from a caller or a config file. A hand-maintained record
+ * of "what we were running" is the first thing to go stale, and a stale
+ * configuration silently invalidates every comparison made against it. The
+ * whole module is therefore best-effort and NEVER THROWS: an unknown field is
+ * recorded as unknown, which is honest, while a crashed capture would cost a
+ * whole benchmark run.
+ */
+import type { HarnessConfiguration } from './types.js';
+export interface CaptureOptions {
+    /** Base URL the agent will be pointed at, e.g. `http://127.0.0.1:8090`. */
+    readonly baseUrl: string;
+    /** Origin llama-server itself is on, used for `/props`. */
+    readonly upstreamOrigin: string;
+    readonly agentCli: string;
+    /** Whether the run inserted the tracing proxy in front of `baseUrl`. */
+    readonly tracing: boolean;
+}
+/**
+ * Observe the live runtime. Best effort throughout; never throws.
+ *
+ * NOTE ON `contextPerSlot`. llama-server's `-c` is the TOTAL context, divided
+ * across `--parallel` slots. The number that bounds any single agent is the
+ * quotient, and `/props.n_ctx` already reports it per slot — so it is taken from
+ * there rather than recomputed, and both are recorded because confusing the two
+ * is how a 262,144 headline number gets attached to a 65,536 reality.
+ */
+export declare function captureConfiguration(opts: CaptureOptions): Promise<HarnessConfiguration>;

package/dist/core/local-eval/configuration.js

@@ -0,0 +1,171 @@
+/**
+ * Capture the configuration a result is only meaningful against.
+ *
+ * "Four agents completed 4/4" is not a fact anyone can act on. "Four agents
+ * completed 4/4 on qwen3.8:27b-q8_0 at 4 slots x 65,536 context, temp 1.0,
+ * through the normalising proxy" is. The difference is whether the next person
+ * to change a setting can tell if they improved anything.
+ *
+ * READ, DO NOT ASK. Every field here is observed from the running system —
+ * llama-server's own `/props`, the process's real argv, Ollama's manifest —
+ * rather than accepted from a caller or a config file. A hand-maintained record
+ * of "what we were running" is the first thing to go stale, and a stale
+ * configuration silently invalidates every comparison made against it. The
+ * whole module is therefore best-effort and NEVER THROWS: an unknown field is
+ * recorded as unknown, which is honest, while a crashed capture would cost a
+ * whole benchmark run.
+ */
+import { execFile } from 'node:child_process';
+import { readdir, readFile, stat } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { promisify } from 'node:util';
+const execFileAsync = promisify(execFile);
+/** Sampling parameters worth recording. The rest of `/props` is noise here. */
+const SAMPLING_KEYS = [
+    'temperature',
+    'top_k',
+    'top_p',
+    'min_p',
+    'typical_p',
+    'repeat_penalty',
+    'repeat_last_n',
+    'presence_penalty',
+    'frequency_penalty',
+    'mirostat',
+    'seed',
+];
+const UNKNOWN = 'unknown';
+async function fetchJson(url, timeoutMs = 5_000) {
+    try {
+        const res = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });
+        if (!res.ok)
+            return null;
+        return (await res.json());
+    }
+    catch {
+        return null;
+    }
+}
+/** The argv of whatever is listening on `port`, via lsof then ps. */
+async function argvOnPort(port) {
+    try {
+        const { stdout } = await execFileAsync('lsof', ['-nP', `-iTCP:${port}`, '-sTCP:LISTEN', '-t']);
+        const pid = stdout.trim().split('\n')[0];
+        if (!pid)
+            return [];
+        const { stdout: cmd } = await execFileAsync('ps', ['-o', 'command=', '-p', pid]);
+        return cmd.trim().split(/\s+/).filter(Boolean);
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Reverse a content-addressed blob path back to the model reference a human
+ * would recognise.
+ *
+ * llama-server reports the blob it loaded, which is a sha256 and tells nobody
+ * anything. Ollama's manifest tree is the only mapping from that back to
+ * `qwen3.8:27b-q8_0`, so it is walked rather than the name being assumed from
+ * whatever someone last said in a status update.
+ */
+async function resolveModelRef(modelPath) {
+    const digest = /sha256-([0-9a-f]{64})/.exec(modelPath)?.[1];
+    if (!digest)
+        return UNKNOWN;
+    const root = join(process.env['OLLAMA_MODELS'] ?? join(homedir(), '.ollama', 'models'), 'manifests');
+    const walk = async (dir, depth = 0) => {
+        if (depth > 6)
+            return null;
+        let entries;
+        try {
+            entries = await readdir(dir, { withFileTypes: true });
+        }
+        catch {
+            return null;
+        }
+        for (const entry of entries) {
+            const full = join(dir, entry.name);
+            if (entry.isDirectory()) {
+                const found = await walk(full, depth + 1);
+                if (found)
+                    return found;
+                continue;
+            }
+            try {
+                if ((await stat(full)).size > 256_000)
+                    continue;
+                if (!(await readFile(full, 'utf8')).includes(digest))
+                    continue;
+            }
+            catch {
+                continue;
+            }
+            // .../manifests/<registry>/<namespace>/<model>/<tag>  ->  model:tag
+            const parts = full.slice(root.length + 1).split('/');
+            if (parts.length >= 2)
+                return `${parts[parts.length - 2]}:${parts[parts.length - 1]}`;
+            return parts.join('/');
+        }
+        return null;
+    };
+    return (await walk(root)) ?? UNKNOWN;
+}
+/**
+ * Observe the live runtime. Best effort throughout; never throws.
+ *
+ * NOTE ON `contextPerSlot`. llama-server's `-c` is the TOTAL context, divided
+ * across `--parallel` slots. The number that bounds any single agent is the
+ * quotient, and `/props.n_ctx` already reports it per slot — so it is taken from
+ * there rather than recomputed, and both are recorded because confusing the two
+ * is how a 262,144 headline number gets attached to a 65,536 reality.
+ */
+export async function captureConfiguration(opts) {
+    const props = await fetchJson(`${opts.upstreamOrigin}/props`);
+    const defaults = (props?.['default_generation_settings'] ?? {});
+    const params = (defaults['params'] ?? {});
+    const modelPath = typeof props?.['model_path'] === 'string' ? props['model_path'] : UNKNOWN;
+    const contextPerSlot = typeof defaults['n_ctx'] === 'number' ? defaults['n_ctx'] : 0;
+    const slots = typeof props?.['total_slots'] === 'number' ? props['total_slots'] : 0;
+    const sampling = {};
+    for (const key of SAMPLING_KEYS) {
+        if (key in params)
+            sampling[key] = params[key];
+    }
+    const upstreamPort = Number(new URL(opts.upstreamOrigin).port || 80);
+    const baseUrlPort = Number(new URL(opts.baseUrl).port || 80);
+    const llamaArgv = await argvOnPort(upstreamPort);
+    // `-c` is authoritative for the total; fall back to slots x per-slot.
+    const cFlag = llamaArgv.indexOf('-c');
+    const contextTotal = cFlag >= 0 && llamaArgv[cFlag + 1]
+        ? Number(llamaArgv[cFlag + 1])
+        : contextPerSlot * (slots || 1);
+    const proxyOn = baseUrlPort !== upstreamPort;
+    const proxyArgv = proxyOn ? await argvOnPort(baseUrlPort) : [];
+    let agentCli = UNKNOWN;
+    try {
+        const { stdout } = await execFileAsync(opts.agentCli, ['--version']);
+        agentCli = `${opts.agentCli} ${stdout.trim()}`;
+    }
+    catch {
+        agentCli = opts.agentCli;
+    }
+    return {
+        model: await resolveModelRef(modelPath),
+        modelPath,
+        quantization: typeof props?.['model_ftype'] === 'string' ? props['model_ftype'] : UNKNOWN,
+        slots,
+        contextPerSlot,
+        contextTotal: Number.isFinite(contextTotal) ? contextTotal : 0,
+        samplingParams: sampling,
+        baseUrl: opts.baseUrl,
+        proxy: proxyOn ? 'on' : 'off',
+        tracing: opts.tracing ? 'on' : 'off',
+        proxyImplementation: proxyOn ? (proxyArgv.join(' ') || UNKNOWN) : 'none (direct to llama-server)',
+        agentCli,
+        llamaServerArgv: llamaArgv,
+        capturedAt: new Date().toISOString(),
+    };
+}
+//# sourceMappingURL=configuration.js.map

package/dist/core/local-eval/configuration.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../../../src/core/local-eval/configuration.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAGtC,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAE1C,+EAA+E;AAC/E,MAAM,aAAa,GAAsB;IACvC,aAAa;IACb,OAAO;IACP,OAAO;IACP,OAAO;IACP,WAAW;IACX,gBAAgB;IAChB,eAAe;IACf,kBAAkB;IAClB,mBAAmB;IACnB,UAAU;IACV,MAAM;CACP,CAAC;AAEF,MAAM,OAAO,GAAG,SAAS,CAAC;AAE1B,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,SAAS,GAAG,KAAK;IACrD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACzE,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,OAAO,IAAI,CAAC;QACzB,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA4B,CAAC;IACvD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,qEAAqE;AACrE,KAAK,UAAU,UAAU,CAAC,IAAY;IACpC,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,SAAS,IAAI,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC,CAAC;QAC/F,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,CAAC;QACpB,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;QACjF,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,KAAK,UAAU,eAAe,CAAC,SAAiB;IAC9C,MAAM,MAAM,GAAG,uBAAuB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5D,IAAI,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC;IAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,WAAW,CAAC,CAAC;IAErG,MAAM,IAAI,GAAG,KAAK,EAAE,GAAW,EAAE,KAAK,GAAG,CAAC,EAA0B,EAAE;QACpE,IAAI,KAAK,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAC3B,IAAI,OAAO,CAAC;QACZ,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACnC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;gBAC1C,IAAI,KAAK;oBAAE,OAAO,KAAK,CAAC;gBACxB,SAAS;YACX,CAAC;YACD,IAAI,CAAC;gBACH,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,OAAO;oBAAE,SAAS;gBAChD,IAAI,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;oBAAE,SAAS;YACjE,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YACD,oEAAoE;YACpE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACrD,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC;gBAAE,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;YACtF,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC;AACvC,CAAC;AAYD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAoB;IAC7D,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,GAAG,IAAI,CAAC,cAAc,QAAQ,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,CAAC,KAAK,EAAE,CAAC,6BAA6B,CAAC,IAAI,EAAE,CAA4B,CAAC;IAC3F,MAAM,MAAM,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAA4B,CAAC;IAErE,MAAM,SAAS,GAAG,OAAO,KAAK,EAAE,CAAC,YAAY,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAC5F,MAAM,cAAc,GAAG,OAAO,QAAQ,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,MAAM,KAAK,GAAG,OAAO,KAAK,EAAE,CAAC,aAAa,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEpF,MAAM,QAAQ,GAA4B,EAAE,CAAC;IAC7C,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QAChC,IAAI,GAAG,IAAI,MAAM;YAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACjD,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IACrE,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC;IAEjD,sEAAsE;IACtE,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,MAAM,YAAY,GAAG,KAAK,IAAI,CAAC,IAAI,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC;QACrD,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAC9B,CAAC,CAAC,cAAc,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IAElC,MAAM,OAAO,GAAG,WAAW,KAAK,YAAY,CAAC;IAC7C,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAE/D,IAAI,QAAQ,GAAG,OAAO,CAAC;IACvB,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;QACrE,QAAQ,GAAG,GAAG,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;IACjD,CAAC;IAAC,MAAM,CAAC;QACP,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;IAC3B,CAAC;IAED,OAAO;QACL,KAAK,EAAE,MAAM,eAAe,CAAC,SAAS,CAAC;QACvC,SAAS;QACT,YAAY,EAAE,OAAO,KAAK,EAAE,CAAC,aAAa,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO;QACzF,KAAK;QACL,cAAc;QACd,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC9D,cAAc,EAAE,QAAQ;QACxB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK;QAC7B,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK;QACpC,mBAAmB,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,+BAA+B;QACjG,QAAQ;QACR,eAAe,EAAE,SAAS;QAC1B,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACrC,CAAC;AACJ,CAAC"}

package/dist/core/local-eval/index.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Local-agent evaluation harness.
+ *
+ * Answers one question with a number: did a change to the local-model harness
+ * make agents better, worse, or neither? See `types.ts` for the three
+ * commitments the design rests on.
+ */
+export * from './types.js';
+export { TASKS } from './tasks.js';
+export { classifyTrial, type TrialEvidence, type TrialVerdict } from './classify.js';
+export { captureConfiguration, type CaptureOptions } from './configuration.js';
+export { countChanges, parseAgentResult, runTrial, type RunTrialOptions } from './runner.js';
+export { buildReport, renderReport, summariseTask } from './report.js';
+export { diagnoseTimeout, startTrace, type StartTraceOptions, type TraceHandle } from './trace.js';
+export { parseArgs, pool } from './main.js';

package/dist/core/local-eval/index.js

@@ -0,0 +1,16 @@
+/**
+ * Local-agent evaluation harness.
+ *
+ * Answers one question with a number: did a change to the local-model harness
+ * make agents better, worse, or neither? See `types.ts` for the three
+ * commitments the design rests on.
+ */
+export * from './types.js';
+export { TASKS } from './tasks.js';
+export { classifyTrial } from './classify.js';
+export { captureConfiguration } from './configuration.js';
+export { countChanges, parseAgentResult, runTrial } from './runner.js';
+export { buildReport, renderReport, summariseTask } from './report.js';
+export { diagnoseTimeout, startTrace } from './trace.js';
+export { parseArgs, pool } from './main.js';
+//# sourceMappingURL=index.js.map

package/dist/core/local-eval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/local-eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,aAAa,EAAyC,MAAM,eAAe,CAAC;AACrF,OAAO,EAAE,oBAAoB,EAAuB,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,QAAQ,EAAwB,MAAM,aAAa,CAAC;AAC7F,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,UAAU,EAA4C,MAAM,YAAY,CAAC;AACnG,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC"}

package/dist/core/local-eval/main.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Entry point: run the task set N times and print a comparable number.
+ *
+ *   npx tsx src/core/local-eval/main.ts [--trials N] [--concurrency N]
+ *                                       [--base-url URL] [--upstream URL]
+ *                                       [--model REF] [--timeout-ms N]
+ *                                       [--task ID] [--out FILE] [--no-trace]
+ *
+ * CONCURRENCY DEFAULTS TO 2, NOT TO THE SLOT COUNT. The runtime is shared with
+ * whatever else is using this machine. Saturating all four slots would make the
+ * harness's own timings a measurement of contention, and would degrade everyone
+ * else's turns while it ran. Raise it deliberately, and only on an idle box —
+ * and note that timings taken at different concurrencies are not comparable.
+ */
+interface Args {
+    trials: number;
+    concurrency: number;
+    baseUrl: string;
+    upstream: string;
+    model: string;
+    agentCli: string;
+    timeoutMs: number;
+    taskFilter: string | null;
+    out: string | null;
+    /** Insert the tracing proxy. On by default; `--no-trace` turns it off. */
+    trace: boolean;
+}
+export declare function parseArgs(argv: readonly string[]): Args;
+/** Run `jobs` with at most `limit` in flight, preserving result order. */
+export declare function pool<T>(jobs: readonly (() => Promise<T>)[], limit: number): Promise<T[]>;
+export {};

package/dist/core/local-eval/main.js

@@ -0,0 +1,174 @@
+/**
+ * Entry point: run the task set N times and print a comparable number.
+ *
+ *   npx tsx src/core/local-eval/main.ts [--trials N] [--concurrency N]
+ *                                       [--base-url URL] [--upstream URL]
+ *                                       [--model REF] [--timeout-ms N]
+ *                                       [--task ID] [--out FILE] [--no-trace]
+ *
+ * CONCURRENCY DEFAULTS TO 2, NOT TO THE SLOT COUNT. The runtime is shared with
+ * whatever else is using this machine. Saturating all four slots would make the
+ * harness's own timings a measurement of contention, and would degrade everyone
+ * else's turns while it ran. Raise it deliberately, and only on an idle box —
+ * and note that timings taken at different concurrencies are not comparable.
+ */
+import { mkdir, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { captureConfiguration } from './configuration.js';
+import { DEFAULT_ANTHROPIC_PROXY_PORT, DEFAULT_LLAMA_HOST, DEFAULT_LLAMA_PORT, } from '../local-runtime/llama/config.js';
+import { buildReport, renderReport, summariseTask } from './report.js';
+import { runTrial } from './runner.js';
+import { TASKS } from './tasks.js';
+export function parseArgs(argv) {
+    const args = {
+        trials: 3,
+        concurrency: 2,
+        // These DEFAULT TO THE SHIPPING PORTS, derived from config.ts rather than
+        // written out, and that is the whole point. The previous default was
+        // `http://127.0.0.1:8090`, which is not a port anything in this repository
+        // ever binds — it was a hand-started scratch script outside the tree. A
+        // baseline measured against it therefore could not be reproduced from a
+        // clean checkout: the harness pointed at a port that, for anyone but the
+        // one machine the script was running on, had nothing behind it. Deriving
+        // both from the constants keeps the harness aimed at the lane the product
+        // actually serves.
+        baseUrl: process.env['ASHLR_EVAL_BASE_URL']
+            ?? `http://${DEFAULT_LLAMA_HOST}:${DEFAULT_ANTHROPIC_PROXY_PORT}`,
+        upstream: process.env['ASHLR_EVAL_UPSTREAM'] ?? `http://${DEFAULT_LLAMA_HOST}:${DEFAULT_LLAMA_PORT}`,
+        // llama-server serves whatever is loaded and ignores this, but the CLI
+        // stamps it into the result JSON, so it is the label the run is filed under.
+        model: process.env['ASHLR_EVAL_MODEL'] ?? 'qwen3.8',
+        agentCli: process.env['ASHLR_EVAL_AGENT'] ?? 'claude',
+        timeoutMs: 900_000,
+        taskFilter: null,
+        out: null,
+        trace: true,
+    };
+    for (let i = 0; i < argv.length; i += 1) {
+        const flag = argv[i];
+        const value = argv[i + 1];
+        switch (flag) {
+            case '--trials':
+                args.trials = Number(value);
+                i += 1;
+                break;
+            case '--concurrency':
+                args.concurrency = Number(value);
+                i += 1;
+                break;
+            case '--base-url':
+                args.baseUrl = String(value);
+                i += 1;
+                break;
+            case '--upstream':
+                args.upstream = String(value);
+                i += 1;
+                break;
+            case '--model':
+                args.model = String(value);
+                i += 1;
+                break;
+            case '--agent':
+                args.agentCli = String(value);
+                i += 1;
+                break;
+            case '--timeout-ms':
+                args.timeoutMs = Number(value);
+                i += 1;
+                break;
+            case '--task':
+                args.taskFilter = String(value);
+                i += 1;
+                break;
+            case '--out':
+                args.out = String(value);
+                i += 1;
+                break;
+            case '--no-trace':
+                args.trace = false;
+                break;
+            case '--trace':
+                args.trace = true;
+                break;
+            default: break;
+        }
+    }
+    return args;
+}
+/** Run `jobs` with at most `limit` in flight, preserving result order. */
+export async function pool(jobs, limit) {
+    const results = new Array(jobs.length);
+    let next = 0;
+    const workers = Array.from({ length: Math.max(1, Math.min(limit, jobs.length)) }, async () => {
+        for (;;) {
+            const index = next;
+            next += 1;
+            if (index >= jobs.length)
+                return;
+            results[index] = await jobs[index]();
+        }
+    });
+    await Promise.all(workers);
+    return results;
+}
+async function main() {
+    const args = parseArgs(process.argv.slice(2));
+    const tasks = args.taskFilter ? TASKS.filter((t) => t.id === args.taskFilter) : TASKS;
+    if (tasks.length === 0) {
+        console.error(`no task matched ${args.taskFilter}`);
+        process.exitCode = 2;
+        return;
+    }
+    const configuration = await captureConfiguration({
+        baseUrl: args.baseUrl,
+        upstreamOrigin: args.upstream,
+        agentCli: args.agentCli,
+        tracing: args.trace,
+    });
+    const root = join(tmpdir(), `ashlr-local-eval-${Date.now()}`);
+    await mkdir(root, { recursive: true });
+    console.error(`[local-eval] ${tasks.length} task(s) x ${args.trials} trial(s), `
+        + `concurrency ${args.concurrency}, artifacts in ${root}`);
+    const startedAt = Date.now();
+    const jobs = [];
+    for (const task of tasks) {
+        for (let trial = 1; trial <= args.trials; trial += 1) {
+            jobs.push(async () => {
+                const result = await runTrial({
+                    task,
+                    trial,
+                    trialDir: join(root, `${task.id}-${trial}`),
+                    baseUrl: args.baseUrl,
+                    model: args.model,
+                    agentCli: args.agentCli,
+                    timeoutMs: args.timeoutMs,
+                    trace: args.trace,
+                });
+                console.error(`[local-eval] ${task.id}#${trial} ${result.mode} `
+                    + `(${(result.wallMs / 1000).toFixed(0)}s)`
+                    + (result.timeoutDiagnosis ? ` — ${result.timeoutDiagnosis.kind}` : ''));
+                return result;
+            });
+        }
+    }
+    const all = await pool(jobs, args.concurrency);
+    const finishedAt = Date.now();
+    const outcomes = tasks.map((task) => summariseTask(task, all.filter((r) => r.taskId === task.id)));
+    const report = buildReport({
+        configuration, outcomes,
+        trialsPerTask: args.trials,
+        concurrency: args.concurrency,
+        startedAt, finishedAt,
+    });
+    const target = args.out ?? join(root, 'report.json');
+    await writeFile(target, JSON.stringify(report, null, 2), 'utf8');
+    console.log(renderReport(report));
+    console.error(`\n[local-eval] report written to ${target}`);
+}
+// Only run when invoked directly, so the module stays importable by tests.
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+    void main();
+}
+//# sourceMappingURL=main.js.map

package/dist/core/local-eval/main.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"main.js","sourceRoot":"","sources":["../../../src/core/local-eval/main.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EACL,4BAA4B,EAC5B,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACvE,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAiBnC,MAAM,UAAU,SAAS,CAAC,IAAuB;IAC/C,MAAM,IAAI,GAAS;QACjB,MAAM,EAAE,CAAC;QACT,WAAW,EAAE,CAAC;QACd,0EAA0E;QAC1E,qEAAqE;QACrE,2EAA2E;QAC3E,wEAAwE;QACxE,wEAAwE;QACxE,yEAAyE;QACzE,yEAAyE;QACzE,0EAA0E;QAC1E,mBAAmB;QACnB,OAAO,EACL,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC;eAC/B,UAAU,kBAAkB,IAAI,4BAA4B,EAAE;QACnE,QAAQ,EACN,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,IAAI,UAAU,kBAAkB,IAAI,kBAAkB,EAAE;QAC5F,uEAAuE;QACvE,6EAA6E;QAC7E,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,IAAI,SAAS;QACnD,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,IAAI,QAAQ;QACrD,SAAS,EAAE,OAAO;QAClB,UAAU,EAAE,IAAI;QAChB,GAAG,EAAE,IAAI;QACT,KAAK,EAAE,IAAI;KACZ,CAAC;IACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1B,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,UAAU;gBAAE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAAC,CAAC,IAAI,CAAC,CAAC;gBAAC,MAAM;YAC5D,KAAK,eAAe;gBAAE,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAAC,CAAC,IAAI,CAAC,CAAC;gBAAC,MAAM;YACtE,KAAK,YAAY;gBAAE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAAC,CAAC,IAAI,CAAC,CAAC;gBAAC,MAAM;YAC/D,KAAK,YAAY;gBAAE,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAAC,CAAC,IAAI,CAAC,CAAC;gBAAC,MAAM;YAChE,KAAK,SAAS;gBAAE,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAAC,CAAC,IAAI,CAAC,CAAC;gBAAC,MAAM;YAC1D,KAAK,SAAS;gBAAE,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAAC,CAAC,IAAI,CAAC,CAAC;gBAAC,MAAM;YAC7D,KAAK,cAAc;gBAAE,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAAC,CAAC,IAAI,CAAC,CAAC;gBAAC,MAAM;YACnE,KAAK,QAAQ;gBAAE,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAAC,CAAC,IAAI,CAAC,CAAC;gBAAC,MAAM;YAC9D,KAAK,OAAO;gBAAE,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAAC,CAAC,IAAI,CAAC,CAAC;gBAAC,MAAM;YACtD,KAAK,YAAY;gBAAE,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;gBAAC,MAAM;YAC7C,KAAK,SAAS;gBAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;gBAAC,MAAM;YACzC,OAAO,CAAC,CAAC,MAAM;QACjB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,0EAA0E;AAC1E,MAAM,CAAC,KAAK,UAAU,IAAI,CACxB,IAAmC,EACnC,KAAa;IAEb,MAAM,OAAO,GAAG,IAAI,KAAK,CAAI,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC3F,SAAS,CAAC;YACR,MAAM,KAAK,GAAG,IAAI,CAAC;YACnB,IAAI,IAAI,CAAC,CAAC;YACV,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM;gBAAE,OAAO;YACjC,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,KAAK,CAAE,EAAE,CAAC;QACxC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACtF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,KAAK,CAAC,mBAAmB,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACpD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,MAAM,aAAa,GAAG,MAAM,oBAAoB,CAAC;QAC/C,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,cAAc,EAAE,IAAI,CAAC,QAAQ;QAC7B,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,OAAO,EAAE,IAAI,CAAC,KAAK;KACpB,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,oBAAoB,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAC9D,MAAM,KAAK,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,OAAO,CAAC,KAAK,CAAC,gBAAgB,KAAK,CAAC,MAAM,cAAc,IAAI,CAAC,MAAM,aAAa;UAC5E,eAAe,IAAI,CAAC,WAAW,kBAAkB,IAAI,EAAE,CAAC,CAAC;IAE7D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAmC,EAAE,CAAC;IAChD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YACrD,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;gBACnB,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC;oBAC5B,IAAI;oBACJ,KAAK;oBACL,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;oBAC3C,OAAO,EAAE,IAAI,CAAC,OAAO;oBACrB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,KAAK,EAAE,IAAI,CAAC,KAAK;iBAClB,CAAC,CAAC;gBACH,OAAO,CAAC,KAAK,CAAC,gBAAgB,IAAI,CAAC,EAAE,IAAI,KAAK,IAAI,MAAM,CAAC,IAAI,GAAG;sBAC5D,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;sBACzC,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC3E,OAAO,MAAM,CAAC;YAChB,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE9B,MAAM,QAAQ,GAAkB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACjD,aAAa,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAEhE,MAAM,MAAM,GAAG,WAAW,CAAC;QACzB,aAAa,EAAE,QAAQ;QACvB,aAAa,EAAE,IAAI,CAAC,MAAM;QAC1B,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,SAAS,EAAE,UAAU;KACtB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IACrD,MAAM,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACjE,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;IAClC,OAAO,CAAC,KAAK,CAAC,oCAAoC,MAAM,EAAE,CAAC,CAAC;AAC9D,CAAC;AAED,2EAA2E;AAC3E,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/E,KAAK,IAAI,EAAE,CAAC;AACd,CAAC"}

package/dist/core/local-eval/report.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Turning trials into the number the harness exists to produce.
+ *
+ * MEDIAN, NOT JUST MEAN. Turn times on this hardware vary by tens of seconds,
+ * and a single slow trial drags a mean far enough to invent a regression that
+ * is not there. Both are reported, because a large gap between them is itself
+ * the signal that a run was contended and its timings should not be compared.
+ *
+ * FAILURE MODES ARE RANKED. The most common mode is printed first, because the
+ * question after "did this help?" is always "what is still broken?", and a
+ * ranked list answers it without opening a single log.
+ */
+import type { EvalReport, HarnessConfiguration, TaskOutcome, TaskSpec, TrialResult } from './types.js';
+/** Aggregate every trial of one task. */
+export declare function summariseTask(task: TaskSpec, trials: readonly TrialResult[]): TaskOutcome;
+export declare function buildReport(args: {
+    readonly configuration: HarnessConfiguration;
+    readonly outcomes: readonly TaskOutcome[];
+    readonly trialsPerTask: number;
+    readonly concurrency: number;
+    readonly startedAt: number;
+    readonly finishedAt: number;
+}): EvalReport;
+/** Render a report for a terminal. Configuration first — it frames everything. */
+export declare function renderReport(report: EvalReport): string;

package/dist/core/local-eval/report.js

@@ -0,0 +1,157 @@
+/**
+ * Turning trials into the number the harness exists to produce.
+ *
+ * MEDIAN, NOT JUST MEAN. Turn times on this hardware vary by tens of seconds,
+ * and a single slow trial drags a mean far enough to invent a regression that
+ * is not there. Both are reported, because a large gap between them is itself
+ * the signal that a run was contended and its timings should not be compared.
+ *
+ * FAILURE MODES ARE RANKED. The most common mode is printed first, because the
+ * question after "did this help?" is always "what is still broken?", and a
+ * ranked list answers it without opening a single log.
+ */
+function median(values) {
+    if (values.length === 0)
+        return 0;
+    const sorted = [...values].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    return sorted.length % 2 === 0 ? Math.round((sorted[mid - 1] + sorted[mid]) / 2) : sorted[mid];
+}
+function mean(values) {
+    if (values.length === 0)
+        return 0;
+    return Math.round(values.reduce((a, b) => a + b, 0) / values.length);
+}
+/** Aggregate every trial of one task. */
+export function summariseTask(task, trials) {
+    const passes = trials.filter((t) => t.passed).length;
+    const counts = new Map();
+    for (const trial of trials) {
+        if (trial.passed)
+            continue;
+        counts.set(trial.mode, (counts.get(trial.mode) ?? 0) + 1);
+    }
+    const wall = trials.map((t) => t.wallMs);
+    return {
+        taskId: task.id,
+        why: task.why,
+        expectation: task.expectation,
+        trials,
+        passes,
+        total: trials.length,
+        passRate: trials.length === 0 ? 0 : passes / trials.length,
+        medianWallMs: median(wall),
+        meanWallMs: mean(wall),
+        modes: [...counts.entries()]
+            .map(([mode, count]) => ({ mode, count }))
+            .sort((a, b) => b.count - a.count || a.mode.localeCompare(b.mode)),
+    };
+}
+export function buildReport(args) {
+    const totalTrials = args.outcomes.reduce((n, o) => n + o.total, 0);
+    const totalPasses = args.outcomes.reduce((n, o) => n + o.passes, 0);
+    return {
+        configuration: args.configuration,
+        trialsPerTask: args.trialsPerTask,
+        concurrency: args.concurrency,
+        outcomes: args.outcomes,
+        overallPassRate: totalTrials === 0 ? 0 : totalPasses / totalTrials,
+        totalPasses,
+        totalTrials,
+        startedAt: new Date(args.startedAt).toISOString(),
+        finishedAt: new Date(args.finishedAt).toISOString(),
+        wallMs: args.finishedAt - args.startedAt,
+    };
+}
+const pct = (n) => `${(n * 100).toFixed(0)}%`;
+const secs = (ms) => `${(ms / 1000).toFixed(0)}s`;
+/** Render a report for a terminal. Configuration first — it frames everything. */
+export function renderReport(report) {
+    const c = report.configuration;
+    const lines = [];
+    lines.push('CONFIGURATION');
+    lines.push(`  model            ${c.model} (${c.quantization})`);
+    lines.push(`  slots            ${c.slots}`);
+    lines.push(`  context/slot     ${c.contextPerSlot.toLocaleString()} (total ${c.contextTotal.toLocaleString()})`);
+    lines.push(`  sampling         ${JSON.stringify(c.samplingParams)}`);
+    // Baselines recorded before tracing existed have no such field, and printing
+    // `undefined` at them would read as a run that chose to turn it off.
+    const tracing = c.tracing ?? 'unknown (recorded before tracing existed)';
+    lines.push(`  base url         ${c.baseUrl}  (proxy ${c.proxy}, tracing ${tracing})`);
+    lines.push(`  proxy impl       ${c.proxyImplementation}`);
+    lines.push(`  agent cli        ${c.agentCli}`);
+    lines.push(`  concurrency      ${report.concurrency} trial(s) at a time`);
+    lines.push('');
+    lines.push(`RESULT  ${report.totalPasses}/${report.totalTrials} = ${pct(report.overallPassRate)} pass rate`
+        + `  (${report.trialsPerTask} trials/task, ${secs(report.wallMs)} wall)`);
+    lines.push('');
+    const pad = (s, n) => s.padEnd(n);
+    lines.push(`  ${pad('task', 24)} ${pad('pass', 8)} ${pad('median', 8)} ${pad('mean', 8)} failure modes`);
+    for (const o of report.outcomes) {
+        const modes = o.modes.length === 0
+            ? '-'
+            : o.modes.map((m) => `${m.mode} x${m.count}`).join(', ');
+        lines.push(`  ${pad(o.taskId, 24)} ${pad(`${o.passes}/${o.total}`, 8)} `
+            + `${pad(secs(o.medianWallMs), 8)} ${pad(secs(o.meanWallMs), 8)} ${modes}`);
+    }
+    lines.push('');
+    // Reported separately and never folded into the pass rate. An
+    // `unsupported-claim` reading is a prompt to go and read one transcript, not
+    // a verdict: the shared classifier fires on change verbs wherever they
+    // appear, including in a refusal that merely MENTIONS refactoring. Treat a
+    // flag on a passing trial as a question, and a flag on a failing one as the
+    // most likely explanation.
+    const flagged = report.outcomes.flatMap((o) => o.trials
+        .filter((t) => t.integrity === 'unsupported-claim')
+        .map((t) => `${o.taskId}#${t.trial} (${t.passed ? 'passed the check' : t.mode})`));
+    lines.push(`INTEGRITY FLAGS  ${flagged.length === 0 ? 'none' : flagged.join(', ')}`);
+    lines.push('  claimed a change while the tree did not move — read the transcript before believing either side');
+    lines.push('');
+    // A timeout used to be the end of the story: the CLI emits its result JSON
+    // once, at the end, so a killed trial recorded zero tokens, zero turns and no
+    // transcript. An outcome nobody can act on is the same failure as a false
+    // success, which is what this harness exists to catch — so every timeout now
+    // reports what the wire was doing when the clock ran out.
+    const timedOut = report.outcomes.flatMap((o) => o.trials.filter((t) => t.mode === 'timeout').map((t) => ({ task: o.taskId, trial: t })));
+    if (timedOut.length > 0) {
+        lines.push('TIMEOUTS, DIAGNOSED');
+        for (const { task, trial } of timedOut) {
+            const d = trial.timeoutDiagnosis;
+            lines.push(`  ${pad(`${task}#${trial.trial}`, 26)} `
+                + (d ? d.kind : 'UNDIAGNOSED — this run was not traced, so why it timed out is unrecoverable'));
+            if (d)
+                lines.push(`  ${' '.repeat(26)} ${d.detail}`);
+            if (trial.trace) {
+                // Turns the SERVER finished. The CLI's connectivity probe and token
+                // counts are not SSE streams at all, and a turn the agent hung up on
+                // is not evidence about the runtime — counting either would report a
+                // terminator failure on every timeout.
+                const turns = trial.trace.streams.filter((x) => (x.events['message_start'] ?? 0) > 0 && x.ended === 'upstream-end');
+                const unterminated = turns.filter((x) => !x.sawTerminator && x.ended === 'upstream-end').length;
+                lines.push(`  ${' '.repeat(26)} ${turns.length} completed turn(s), `
+                    + `${unterminated} ended without a terminator; raw capture in ${trial.trace.captureDir}`);
+                // Where the budget actually went. A single enormous turn and a long
+                // series of ordinary ones are different problems with different fixes,
+                // and the totals alone cannot tell them apart.
+                if (turns.length > 0) {
+                    const longest = turns.reduce((a, b) => (a.durationMs >= b.durationMs ? a : b));
+                    const spent = turns.reduce((n, t) => n + t.durationMs, 0);
+                    lines.push(`  ${' '.repeat(26)} longest completed turn ${secs(longest.durationMs)} `
+                        + `(#${longest.seq}), ${secs(spent)} across all completed turns`);
+                }
+            }
+        }
+        lines.push('');
+    }
+    lines.push('TOKENS PER TRIAL (input / output / cache-read)');
+    for (const o of report.outcomes) {
+        for (const t of o.trials) {
+            lines.push(`  ${pad(`${o.taskId}#${t.trial}`, 26)} `
+                + `${pad(String(t.tokens.input), 8)} ${pad(String(t.tokens.output), 8)} `
+                + `${pad(String(t.tokens.cacheRead), 10)} ${pad(t.mode, 26)} `
+                + `${t.changedFiles ?? '?'} file(s), ${t.turns ?? '?'} turns${t.note ? ` — ${t.note}` : ''}`);
+        }
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=report.js.map