@visorcraft/idlehands 0.9.1

package/dist/bench/compare.js

@@ -0,0 +1,239 @@
+#!/usr/bin/env node
+// NOTE: These utilities are intentionally duplicated from src/ to keep the
+// benchmark harness self-contained and free of production import dependencies.
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import os from 'node:os';
+import crypto from 'node:crypto';
+import { spawn } from 'node:child_process';
+import { loadConfig } from '../config.js';
+import { createSession } from '../agent.js';
+import { runOpenclaw } from './openclaw.js';
+function nowMs() {
+    return performance.now();
+}
+async function mkTempDir(prefix = 'idlehands-bench-') {
+    return await fs.mkdtemp(path.join(os.tmpdir(), prefix));
+}
+async function runShell(command, cwd, timeoutSec) {
+    return await new Promise((resolve, reject) => {
+        const shell = process.env.IDLEHANDS_SHELL || 'bash';
+        const child = spawn(shell, ['-lc', command], { cwd, stdio: ['ignore', 'pipe', 'pipe'] });
+        const out = [];
+        const err = [];
+        const t = setTimeout(() => child.kill('SIGKILL'), Math.max(1, timeoutSec) * 1000);
+        child.stdout.on('data', (d) => out.push(d));
+        child.stderr.on('data', (d) => err.push(d));
+        child.on('error', reject);
+        child.on('close', (code) => {
+            clearTimeout(t);
+            resolve({ rc: code ?? 0, out: Buffer.concat(out).toString('utf8'), err: Buffer.concat(err).toString('utf8') });
+        });
+    });
+}
+function randSessionId() {
+    return crypto.randomBytes(8).toString('hex');
+}
+async function setupWorkspace(c) {
+    if (c.workspace.kind === 'fixed')
+        return c.workspace.dir;
+    return await mkTempDir(c.workspace.prefix ?? `idlehands-${c.name}-`);
+}
+async function applySetup(c, workDir) {
+    if (!c.setup?.length)
+        return;
+    for (const cmd of c.setup) {
+        const r = await runShell(cmd, workDir, 30);
+        if (r.rc !== 0)
+            throw new Error(`setup failed rc=${r.rc}: ${cmd}\n${r.err}`);
+    }
+}
+async function checkSuccess(c, workDir, agentOutput) {
+    if (!c.success)
+        return { ok: true, reason: 'ok' };
+    if (c.success.type === 'equals') {
+        const out = agentOutput.trim();
+        const ok = out === c.success.value;
+        return { ok, reason: ok ? 'ok' : `expected ${JSON.stringify(c.success.value)}, got ${JSON.stringify(out)}` };
+    }
+    if (c.success.type === 'exec') {
+        const ex = await runShell(c.success.command, workDir, 60);
+        const wantRc = c.success.exitCode ?? 0;
+        let ok = ex.rc === wantRc;
+        let reason = ok ? 'ok' : `success check rc=${ex.rc} want=${wantRc}`;
+        if (ok && c.success.stdoutEquals !== undefined) {
+            ok = ex.out.trimEnd() === c.success.stdoutEquals;
+            if (!ok)
+                reason = `success stdout mismatch`;
+        }
+        if (ok && c.success.stdoutIncludes) {
+            ok = ex.out.includes(c.success.stdoutIncludes);
+            if (!ok)
+                reason = `success stdout missing substring`;
+        }
+        return { ok, reason };
+    }
+    return { ok: false, reason: 'unknown success type' };
+}
+async function runIdlehandsOnce(opts) {
+    const config = opts.cachedConfig
+        ? { ...opts.cachedConfig, dir: opts.workDir }
+        : (await loadConfig({
+            configPath: path.join(os.tmpdir(), 'idlehands-bench-config-does-not-exist.json'),
+            cli: {
+                endpoint: opts.endpoint,
+                dir: opts.workDir,
+                model: opts.model ?? '',
+                max_tokens: opts.maxTokens,
+                no_confirm: true,
+                dry_run: false,
+                verbose: false
+            }
+        })).config;
+    const initStart = nowMs();
+    const session = await createSession({ config });
+    const initMs = nowMs() - initStart;
+    const askStart = nowMs();
+    let ttfr = null;
+    let ttft = null;
+    const res = await session.ask(opts.instruction, {
+        onFirstDelta: () => {
+            if (ttfr == null)
+                ttfr = nowMs() - askStart;
+        },
+        onToken: () => {
+            if (ttft == null)
+                ttft = nowMs() - askStart;
+        }
+    });
+    const ttc = nowMs() - askStart;
+    return { initMs, ttfrMs: ttfr, ttftMs: ttft, ttcMs: ttc, exitCode: 0, out: res.text ?? '', turns: res.turns, toolCalls: res.toolCalls };
+}
+async function runEngineOnce(engine, c, workDir, endpoint, maxTokens, cachedConfig) {
+    if (engine === 'idlehands') {
+        return await runIdlehandsOnce({
+            workDir,
+            instruction: c.instruction,
+            endpoint,
+            maxTokens,
+            model: c.model,
+            cachedConfig
+        });
+    }
+    // openclaw (embedded local mode to keep it apples-to-apples)
+    const t0 = nowMs();
+    const r = await runOpenclaw({
+        workDir,
+        instruction: c.instruction,
+        timeoutSec: 180,
+        sessionId: `bench_${c.name}_${randSessionId()}`,
+        // Use embedded mode to avoid gateway pairing requirements.
+        // We still pin the model/provider via OPENCLAW_CONFIG_PATH.
+        local: true,
+        profile: 'idlehands-bench'
+    });
+    const initMs = 0;
+    const ttc = nowMs() - t0;
+    // openclaw emits JSON; extract first "text" field.
+    let out = '';
+    try {
+        const { extractFirstText } = await import('./json_extract.js');
+        const parsed = JSON.parse(r.stdout || r.stderr || '{}');
+        out = (extractFirstText(parsed) ?? '').trim();
+    }
+    catch {
+        out = (r.stdout || r.stderr || '').trim();
+    }
+    return { initMs, ttfrMs: r.ttfrMs, ttftMs: null, ttcMs: ttc, exitCode: r.exitCode, out, turns: 0, toolCalls: 0 };
+}
+async function main() {
+    const casePath = process.argv[2];
+    if (!casePath) {
+        console.error('Usage: compare <case.json>');
+        process.exit(2);
+    }
+    const raw = await fs.readFile(casePath, 'utf8');
+    const c = JSON.parse(raw);
+    const endpoint = process.env.IDLEHANDS_ENDPOINT || 'http://192.168.68.118:8080/v1';
+    const reps = c.repetitions ?? 5;
+    const maxTokens = c.max_tokens ?? 512;
+    const engines = c.engine === 'openclaw' ? ['openclaw'] : c.engine === 'both' ? ['idlehands', 'openclaw'] : ['idlehands'];
+    const results = [];
+    // Pre-load config once for idlehands runs (avoids redundant loadConfig + /v1/models per iteration)
+    let idlehandsCachedConfig = null;
+    if (engines.includes('idlehands')) {
+        const { config } = await loadConfig({
+            configPath: path.join(os.tmpdir(), 'idlehands-bench-config-does-not-exist.json'),
+            cli: {
+                endpoint,
+                dir: process.cwd(),
+                model: c.model ?? '',
+                max_tokens: maxTokens,
+                no_confirm: true,
+                dry_run: false,
+                verbose: false
+            }
+        });
+        idlehandsCachedConfig = config;
+    }
+    for (const engine of engines) {
+        for (let i = 0; i < reps; i++) {
+            const workDir = await setupWorkspace(c);
+            try {
+                console.error(`[bench] case=${c.name} engine=${engine} iter=${i + 1}/${reps} workdir=${workDir}`);
+                await applySetup(c, workDir);
+                const r = await runEngineOnce(engine, c, workDir, endpoint, maxTokens, engine === 'idlehands' ? idlehandsCachedConfig : undefined);
+                const check = await checkSuccess(c, workDir, r.out);
+                const ok = check.ok && r.exitCode === 0;
+                const reason = check.ok ? (r.exitCode === 0 ? 'ok' : `exitCode=${r.exitCode}`) : check.reason;
+                const ttcS = r.ttcMs != null ? (r.ttcMs / 1000).toFixed(2) : '?';
+                const ttfrS = r.ttfrMs != null ? (r.ttfrMs / 1000).toFixed(2) : '?';
+                console.error(`[bench] result case=${c.name} engine=${engine} iter=${i + 1}/${reps} ok=${ok} ttc_s=${ttcS} ttfr_s=${ttfrS} reason=${ok ? 'ok' : reason}`);
+                results.push({
+                    case: c.name,
+                    engine,
+                    iter: i + 1,
+                    ok,
+                    reason,
+                    init_ms: r.initMs,
+                    ttfr_ms: r.ttfrMs,
+                    ttft_ms: r.ttftMs,
+                    ttc_ms: r.ttcMs,
+                    exitCode: r.exitCode,
+                    turns: r.turns,
+                    toolCalls: r.toolCalls
+                });
+            }
+            catch (e) {
+                const reason = e?.message ?? String(e);
+                console.error(`[bench] result case=${c.name} engine=${engine} iter=${i + 1}/${reps} ok=false ttc_s=0 ttfr_s=? reason=${reason}`);
+                results.push({
+                    case: c.name,
+                    engine,
+                    iter: i + 1,
+                    ok: false,
+                    reason,
+                    init_ms: 0,
+                    ttfr_ms: null,
+                    ttft_ms: null,
+                    ttc_ms: 0,
+                    exitCode: null
+                });
+            }
+            finally {
+                if (c.workspace.kind === 'temp') {
+                    await fs.rm(workDir, { recursive: true, force: true }).catch(() => { });
+                }
+            }
+        }
+    }
+    const outPath = path.join(process.cwd(), 'bench', 'results', `${c.name}.compare.${Date.now()}.jsonl`);
+    await fs.mkdir(path.dirname(outPath), { recursive: true });
+    await fs.writeFile(outPath, results.map((r) => JSON.stringify(r)).join('\n') + '\n', 'utf8');
+    console.log(`Wrote: ${outPath}`);
+}
+main().catch((e) => {
+    console.error(e);
+    process.exit(1);
+});
+//# sourceMappingURL=compare.js.map

package/dist/bench/compare.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare.js","sourceRoot":"","sources":["../../src/bench/compare.ts"],"names":[],"mappings":";AACA,2EAA2E;AAC3E,+EAA+E;AAE/E,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE3C,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,SAAS,KAAK;IACZ,OAAO,WAAW,CAAC,GAAG,EAAE,CAAC;AAC3B,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAM,GAAG,kBAAkB;IAClD,OAAO,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;AAC1D,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,OAAe,EAAE,GAAW,EAAE,UAAkB;IACtE,OAAO,MAAM,IAAI,OAAO,CAA2C,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrF,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,MAAM,CAAC;QACpD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QACzF,MAAM,GAAG,GAAa,EAAE,CAAC;QACzB,MAAM,GAAG,GAAa,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC;QAClF,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACzB,YAAY,CAAC,CAAC,CAAC,CAAC;YAChB,OAAO,CAAC,EAAE,EAAE,EAAE,IAAI,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACjH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,aAAa;IACpB,OAAO,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;AAC/C,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,CAAY;IACxC,IAAI,CAAC,CAAC,SAAS,CAAC,IAAI,KAAK,OAAO;QAAE,OAAO,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC;IACzD,OAAO,MAAM,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,IAAI,aAAa,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;AACvE,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,CAAY,EAAE,OAAe;IACrD,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,MAAM;QAAE,OAAO;IAC7B,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;IAC/E,CAAC;AACH,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,CAAY,EAAE,OAAe,EAAE,WAAmB;IAC5E,IAAI,CAAC,CAAC,CAAC,OAAO;QAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAElD,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAChC,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,EAAE,GAAG,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;QACnC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;IAC/G,CAAC;IAED,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC9B,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;QACvC,IAAI,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,MAAM,CAAC;QAC1B,IAAI,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,oBAAoB,EAAE,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;QACpE,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YAC/C,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC;YACjD,IAAI,CAAC,EAAE;gBAAE,MAAM,GAAG,yBAAyB,CAAC;QAC9C,CAAC;QACD,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YACnC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YAC/C,IAAI,CAAC,EAAE;gBAAE,MAAM,GAAG,kCAAkC,CAAC;QACvD,CAAC;QACD,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC;IACxB,CAAC;IAED,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC;AACvD,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,IAO/B;IACC,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY;QAC9B,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,YAAY,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE;QAC7C,CAAC,CAAC,CAAC,MAAM,UAAU,CAAC;YAChB,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,4CAA4C,CAAC;YAChF,GAAG,EAAE;gBACH,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,GAAG,EAAE,IAAI,CAAC,OAAO;gBACjB,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,EAAE;gBACvB,UAAU,EAAE,IAAI,CAAC,SAAS;gBAC1B,UAAU,EAAE,IAAI;gBAChB,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,KAAK;aACR;SACT,CAAC,CAAC,CAAC,MAAM,CAAC;IAEf,MAAM,SAAS,GAAG,KAAK,EAAE,CAAC;IAC1B,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,KAAK,EAAE,GAAG,SAAS,CAAC;IAEnC,MAAM,QAAQ,GAAG,KAAK,EAAE,CAAC;IACzB,IAAI,IAAI,GAAkB,IAAI,CAAC;IAC/B,IAAI,IAAI,GAAkB,IAAI,CAAC;IAE/B,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE;QAC9C,YAAY,EAAE,GAAG,EAAE;YACjB,IAAI,IAAI,IAAI,IAAI;gBAAE,IAAI,GAAG,KAAK,EAAE,GAAG,QAAQ,CAAC;QAC9C,CAAC;QACD,OAAO,EAAE,GAAG,EAAE;YACZ,IAAI,IAAI,IAAI,IAAI;gBAAE,IAAI,GAAG,KAAK,EAAE,GAAG,QAAQ,CAAC;QAC9C,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,KAAK,EAAE,GAAG,QAAQ,CAAC;IAC/B,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,CAAC;AAC1I,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,MAAmB,EAAE,CAAY,EAAE,OAAe,EAAE,QAAgB,EAAE,SAAiB,EAAE,YAAkB;IACtI,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;QAC3B,OAAO,MAAM,gBAAgB,CAAC;YAC5B,OAAO;YACP,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,QAAQ;YACR,SAAS;YACT,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,YAAY;SACb,CAAC,CAAC;IACL,CAAC;IAED,6DAA6D;IAC7D,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,MAAM,CAAC,GAAG,MAAM,WAAW,CAAC;QAC1B,OAAO;QACP,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,UAAU,EAAE,GAAG;QACf,SAAS,EAAE,SAAS,CAAC,CAAC,IAAI,IAAI,aAAa,EAAE,EAAE;QAC/C,2DAA2D;QAC3D,4DAA4D;QAC5D,KAAK,EAAE,IAAI;QACX,OAAO,EAAE,iBAAiB;KAC3B,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,CAAC,CAAC;IACjB,MAAM,GAAG,GAAG,KAAK,EAAE,GAAG,EAAE,CAAC;IAEzB,mDAAmD;IACnD,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,IAAI,CAAC;QACH,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;QACxD,GAAG,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5C,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;AACnH,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAChD,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAc,CAAC;IAEvC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,+BAA+B,CAAC;IACnF,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC;IAChC,MAAM,SAAS,GAAG,CAAC,CAAC,UAAU,IAAI,GAAG,CAAC;IAEtC,MAAM,OAAO,GAAkB,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IAExI,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,mGAAmG;IACnG,IAAI,qBAAqB,GAAQ,IAAI,CAAC;IACtC,IAAI,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAClC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,UAAU,CAAC;YAClC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,4CAA4C,CAAC;YAChF,GAAG,EAAE;gBACH,QAAQ;gBACR,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE;gBAClB,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE;gBACpB,UAAU,EAAE,SAAS;gBACrB,UAAU,EAAE,IAAI;gBAChB,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,KAAK;aACR;SACT,CAAC,CAAC;QACH,qBAAqB,GAAG,MAAM,CAAC;IACjC,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,CAAC,CAAC,CAAC;YACxC,IAAI,CAAC;gBACH,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,IAAI,WAAW,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,IAAI,YAAY,OAAO,EAAE,CAAC,CAAC;gBAClG,MAAM,UAAU,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBAE7B,MAAM,CAAC,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;gBACnI,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;gBAEpD,MAAM,EAAE,GAAG,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC;gBACxC,MAAM,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC;gBAC9F,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;gBACjE,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;gBACpE,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC,IAAI,WAAW,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,IAAI,OAAO,EAAE,UAAU,IAAI,WAAW,KAAK,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;gBAE1J,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,MAAM;oBACN,IAAI,EAAE,CAAC,GAAG,CAAC;oBACX,EAAE;oBACF,MAAM;oBACN,OAAO,EAAE,CAAC,CAAC,MAAM;oBACjB,OAAO,EAAE,CAAC,CAAC,MAAM;oBACjB,OAAO,EAAE,CAAC,CAAC,MAAM;oBACjB,MAAM,EAAE,CAAC,CAAC,KAAK;oBACf,QAAQ,EAAE,CAAC,CAAC,QAAQ;oBACpB,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,SAAS,EAAE,CAAC,CAAC,SAAS;iBACvB,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBAChB,MAAM,MAAM,GAAG,CAAC,EAAE,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;gBACvC,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC,IAAI,WAAW,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,IAAI,qCAAqC,MAAM,EAAE,CAAC,CAAC;gBAEjI,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,MAAM;oBACN,IAAI,EAAE,CAAC,GAAG,CAAC;oBACX,EAAE,EAAE,KAAK;oBACT,MAAM;oBACN,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,IAAI;oBACb,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,CAAC;oBACT,QAAQ,EAAE,IAAI;iBACf,CAAC,CAAC;YACL,CAAC;oBAAS,CAAC;gBACT,IAAI,CAAC,CAAC,SAAS,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBAChC,MAAM,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,YAAY,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACtG,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,MAAM,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;IAC7F,OAAO,CAAC,GAAG,CAAC,UAAU,OAAO,EAAE,CAAC,CAAC;AACnC,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;IACjB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACjB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/bench/debug_hooks.js

@@ -0,0 +1,17 @@
+export function installBenchDebugHooks() {
+    const onUncaught = (e) => {
+        try {
+            console.error('[bench-debug] uncaughtException:', e);
+        }
+        catch { }
+    };
+    const onUnhandled = (e) => {
+        try {
+            console.error('[bench-debug] unhandledRejection:', e);
+        }
+        catch { }
+    };
+    process.on('uncaughtException', onUncaught);
+    process.on('unhandledRejection', onUnhandled);
+}
+//# sourceMappingURL=debug_hooks.js.map

package/dist/bench/debug_hooks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"debug_hooks.js","sourceRoot":"","sources":["../../src/bench/debug_hooks.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,sBAAsB;IACpC,MAAM,UAAU,GAAG,CAAC,CAAM,EAAE,EAAE;QAC5B,IAAI,CAAC;YACH,OAAO,CAAC,KAAK,CAAC,kCAAkC,EAAE,CAAC,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;IACZ,CAAC,CAAC;IACF,MAAM,WAAW,GAAG,CAAC,CAAM,EAAE,EAAE;QAC7B,IAAI,CAAC;YACH,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,CAAC,CAAC,CAAC;QACxD,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;IACZ,CAAC,CAAC;IAEF,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,UAAU,CAAC,CAAC;IAC5C,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,WAAW,CAAC,CAAC;AAChD,CAAC"}

package/dist/bench/json_extract.js

@@ -0,0 +1,22 @@
+export function extractFirstText(obj) {
+    const texts = [];
+    const walk = (x) => {
+        if (!x)
+            return;
+        if (typeof x === 'object') {
+            if (Array.isArray(x)) {
+                for (const v of x)
+                    walk(v);
+                return;
+            }
+            for (const [k, v] of Object.entries(x)) {
+                if (k === 'text' && typeof v === 'string')
+                    texts.push(v);
+                walk(v);
+            }
+        }
+    };
+    walk(obj);
+    return texts.length ? texts[0] : null;
+}
+//# sourceMappingURL=json_extract.js.map

package/dist/bench/json_extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json_extract.js","sourceRoot":"","sources":["../../src/bench/json_extract.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,gBAAgB,CAAC,GAAQ;IACvC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,CAAM,EAAE,EAAE;QACtB,IAAI,CAAC,CAAC;YAAE,OAAO;QACf,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrB,KAAK,MAAM,CAAC,IAAI,CAAC;oBAAE,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC3B,OAAO;YACT,CAAC;YACD,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvC,IAAI,CAAC,KAAK,MAAM,IAAI,OAAO,CAAC,KAAK,QAAQ;oBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACzD,IAAI,CAAC,CAAC,CAAC,CAAC;YACV,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACxC,CAAC"}

package/dist/bench/openclaw.js

@@ -0,0 +1,86 @@
+#!/usr/bin/env node
+// NOTE: These utilities are intentionally duplicated from src/ to keep the
+// benchmark harness self-contained and free of production import dependencies.
+import { spawn } from 'node:child_process';
+function nowMs() {
+    return performance.now();
+}
+export async function runOpenclaw(opts) {
+    const t0 = nowMs();
+    let ttfr = null;
+    const prefix = `Work in directory: ${opts.workDir}.\n` +
+        `Use absolute paths OR set cwd explicitly when using exec.\n` +
+        `Do not modify files outside that directory.\n\n`;
+    const args = [
+        'agent',
+        ...(opts.local ? ['--local'] : []),
+        '--agent',
+        'main',
+        '--channel',
+        'last',
+        '--thinking',
+        'off',
+        '--timeout',
+        String(opts.timeoutSec),
+        '--session-id',
+        opts.sessionId,
+        '--verbose',
+        'off',
+        '--json',
+        '--message',
+        prefix +
+            `IMPORTANT: for ANY command execution, you MUST run it as:\n` +
+            `  bash -lc 'cd ${opts.workDir} && <command>'\n` +
+            `Do NOT run commands without the 'cd ${opts.workDir} &&' prefix.\n\n` +
+            opts.instruction
+    ];
+    const binArgs = [];
+    if (opts.profile) {
+        binArgs.push('--profile', opts.profile);
+    }
+    binArgs.push(...args);
+    const child = spawn('openclaw', binArgs, {
+        stdio: ['ignore', 'pipe', 'pipe'],
+        env: {
+            ...process.env,
+            // Ensure the OpenClaw CLI uses the same pinned Halo model/provider.
+            // (We generate this config in bench to keep apples-to-apples with Idle Hands.)
+            OPENCLAW_CONFIG_PATH: process.env.OPENCLAW_CONFIG_PATH || '/home/user/.openclaw-idlehands-bench/openclaw.json'
+        }
+    });
+    const killTimer = setTimeout(() => {
+        try {
+            child.kill('SIGKILL');
+        }
+        catch {
+            // ignore
+        }
+    }, Math.max(1, opts.timeoutSec + 10) * 1000);
+    const out = [];
+    const err = [];
+    const mark = () => {
+        if (ttfr == null)
+            ttfr = nowMs() - t0;
+    };
+    child.stdout.on('data', (d) => {
+        mark();
+        out.push(d);
+    });
+    child.stderr.on('data', (d) => {
+        mark();
+        err.push(d);
+    });
+    const exitCode = await new Promise((resolve) => child.on('close', (code) => resolve(code ?? 0)));
+    clearTimeout(killTimer);
+    const t1 = nowMs();
+    const stdout = Buffer.concat(out).toString('utf8');
+    const stderr = Buffer.concat(err).toString('utf8');
+    return {
+        exitCode,
+        stdout,
+        stderr,
+        ttfrMs: ttfr,
+        ttcMs: t1 - t0
+    };
+}
+//# sourceMappingURL=openclaw.js.map

package/dist/bench/openclaw.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openclaw.js","sourceRoot":"","sources":["../../src/bench/openclaw.ts"],"names":[],"mappings":";AACA,2EAA2E;AAC3E,+EAA+E;AAE/E,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE3C,SAAS,KAAK;IACZ,OAAO,WAAW,CAAC,GAAG,EAAE,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAOjC;IACC,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,IAAI,IAAI,GAAkB,IAAI,CAAC;IAE/B,MAAM,MAAM,GAAG,sBAAsB,IAAI,CAAC,OAAO,KAAK;QACpD,6DAA6D;QAC7D,iDAAiD,CAAC;IAEpD,MAAM,IAAI,GAAG;QACX,OAAO;QACP,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAClC,SAAS;QACT,MAAM;QACN,WAAW;QACX,MAAM;QACN,YAAY;QACZ,KAAK;QACL,WAAW;QACX,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;QACvB,cAAc;QACd,IAAI,CAAC,SAAS;QACd,WAAW;QACX,KAAK;QACL,QAAQ;QACR,WAAW;QACX,MAAM;YACJ,6DAA6D;YAC7D,kBAAkB,IAAI,CAAC,OAAO,kBAAkB;YAChD,uCAAuC,IAAI,CAAC,OAAO,kBAAkB;YACrE,IAAI,CAAC,WAAW;KACnB,CAAC;IAEF,MAAM,OAAO,GAAG,EAAc,CAAC;IAC/B,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;IAEtB,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,EAAE,OAAO,EAAE;QACvC,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;QACjC,GAAG,EAAE;YACH,GAAG,OAAO,CAAC,GAAG;YACd,oEAAoE;YACpE,+EAA+E;YAC/E,oBAAoB,EAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,oDAAoD;SAC/G;KACF,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;QAChC,IAAI,CAAC;YACH,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;IACH,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;IAE7C,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,GAAG,GAAa,EAAE,CAAC;IAEzB,MAAM,IAAI,GAAG,GAAG,EAAE;QAChB,IAAI,IAAI,IAAI,IAAI;YAAE,IAAI,GAAG,KAAK,EAAE,GAAG,EAAE,CAAC;IACxC,CAAC,CAAC;IAEF,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE;QAC5B,IAAI,EAAE,CAAC;QACP,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACd,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE;QAC5B,IAAI,EAAE,CAAC;QACP,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACzG,YAAY,CAAC,SAAS,CAAC,CAAC;IACxB,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IAEnB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAEnD,OAAO;QACL,QAAQ;QACR,MAAM;QACN,MAAM;QACN,MAAM,EAAE,IAAI;QACZ,KAAK,EAAE,EAAE,GAAG,EAAE;KACf,CAAC;AACJ,CAAC"}

package/dist/bench/report.js

@@ -0,0 +1,116 @@
+#!/usr/bin/env node
+import fs from 'node:fs/promises';
+import path from 'node:path';
+function percentile(sorted, p) {
+    if (!sorted.length)
+        return NaN;
+    const idx = (sorted.length - 1) * p;
+    const lo = Math.floor(idx);
+    const hi = Math.ceil(idx);
+    if (lo === hi)
+        return sorted[lo];
+    const w = idx - lo;
+    return sorted[lo] * (1 - w) + sorted[hi] * w;
+}
+function median(sorted) {
+    return percentile(sorted, 0.5);
+}
+async function main() {
+    const resultsDir = path.join(process.cwd(), 'bench', 'results');
+    const ents = await fs.readdir(resultsDir, { withFileTypes: true }).catch(() => []);
+    const files = ents.filter((e) => e.isFile() && e.name.endsWith('.jsonl')).map((e) => path.join(resultsDir, e.name));
+    const rows = [];
+    for (const f of files) {
+        const raw = await fs.readFile(f, 'utf8');
+        for (const line of raw.split(/\r?\n/)) {
+            if (!line.trim())
+                continue;
+            try {
+                rows.push(JSON.parse(line));
+            }
+            catch {
+                // ignore
+            }
+        }
+    }
+    const byKey = new Map();
+    for (const r of rows) {
+        const key = `${r.case}::${r.engine}`;
+        const arr = byKey.get(key) ?? [];
+        arr.push(r);
+        byKey.set(key, arr);
+    }
+    const lines = [];
+    lines.push('# Idle Hands — Bench Report');
+    lines.push('');
+    lines.push(`Generated: ${new Date().toISOString()}`);
+    lines.push('');
+    for (const [key, arr] of [...byKey.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
+        const [caseName, engine] = key.split('::');
+        const oks = arr.filter((r) => r.ok);
+        const failCount = arr.length - oks.length;
+        const ttc = oks.map((r) => r.ttc_ms).filter((n) => Number.isFinite(n)).sort((a, b) => a - b);
+        const ttfr = oks
+            .map((r) => r.ttfr_ms)
+            .filter((n) => typeof n === 'number' && Number.isFinite(n))
+            .sort((a, b) => a - b);
+        const ttft = oks
+            .map((r) => r.ttft_ms)
+            .filter((n) => typeof n === 'number' && Number.isFinite(n))
+            .sort((a, b) => a - b);
+        lines.push(`## ${caseName} (${engine})`);
+        lines.push('');
+        lines.push(`Runs: ${arr.length}, ok: ${oks.length}, fail: ${failCount}`);
+        lines.push('');
+        const init = oks
+            .map((r) => (typeof r.init_ms === 'number' ? r.init_ms : NaN))
+            // In reuse_session mode we record init_ms=0 for iter>1; ignore those.
+            .filter((n) => Number.isFinite(n) && n > 0)
+            .sort((a, b) => a - b);
+        if (init.length) {
+            lines.push(`- INIT median: ${median(init).toFixed(1)} ms (session init: model pick + warmup)`);
+            lines.push(`- INIT p95:    ${percentile(init, 0.95).toFixed(1)} ms`);
+        }
+        if (ttc.length) {
+            lines.push(`- TTC median:  ${median(ttc).toFixed(1)} ms (ask: completion wall time)`);
+            lines.push(`- TTC p95:     ${percentile(ttc, 0.95).toFixed(1)} ms`);
+        }
+        if (ttfr.length) {
+            lines.push(`- TTFR median: ${median(ttfr).toFixed(1)} ms (ask: time-to-first-delta)`);
+            lines.push(`- TTFR p95:    ${percentile(ttfr, 0.95).toFixed(1)} ms`);
+        }
+        if (ttft.length) {
+            lines.push(`- TTFT median: ${median(ttft).toFixed(1)} ms (ask: time-to-first-content-token)`);
+            lines.push(`- TTFT p95:    ${percentile(ttft, 0.95).toFixed(1)} ms`);
+        }
+        if (!oks.length) {
+            lines.push('- No successful runs to compute latency statistics.');
+        }
+        // Flag warmup: iter=1 is typically slower (cold KV cache, model warmup)
+        const warmup = oks.find((r) => r.iter === 1);
+        const steady = oks.filter((r) => r.iter !== 1);
+        if (warmup && steady.length >= 2) {
+            const steadyTtc = steady.map((r) => r.ttc_ms).filter((n) => Number.isFinite(n)).sort((a, b) => a - b);
+            if (steadyTtc.length) {
+                const warmupPct = warmup.ttc_ms > 0 ? ((warmup.ttc_ms / median(steadyTtc) - 1) * 100).toFixed(0) : '?';
+                lines.push(`- Warmup: iter=1 TTC ${warmup.ttc_ms.toFixed(1)} ms (+${warmupPct}% vs steady median ${median(steadyTtc).toFixed(1)} ms)`);
+            }
+        }
+        const fails = arr.filter((r) => !r.ok).slice(0, 5);
+        if (fails.length) {
+            lines.push('');
+            lines.push('Top failures:');
+            for (const f of fails)
+                lines.push(`- iter ${f.iter}: ${f.reason}`);
+        }
+        lines.push('');
+    }
+    const outPath = path.join(process.cwd(), 'bench', 'REPORT.md');
+    await fs.writeFile(outPath, lines.join('\n') + '\n', 'utf8');
+    console.log(`Wrote: ${outPath}`);
+}
+main().catch((e) => {
+    console.error(e);
+    process.exit(1);
+});
+//# sourceMappingURL=report.js.map

package/dist/bench/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/bench/report.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAc7B,SAAS,UAAU,CAAC,MAAgB,EAAE,CAAS;IAC7C,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,OAAO,GAAG,CAAC;IAC/B,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,MAAM,CAAC,EAAE,CAAC,CAAC;IACjC,MAAM,CAAC,GAAG,GAAG,GAAG,EAAE,CAAC;IACnB,OAAO,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,MAAM,CAAC,MAAgB;IAC9B,OAAO,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AACjC,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAChE,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACnF,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAEpH,MAAM,IAAI,GAAU,EAAE,CAAC;IACvB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACzC,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YACtC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;gBAAE,SAAS;YAC3B,IAAI,CAAC;gBACH,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAC9B,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,GAAG,EAAiB,CAAC;IACvC,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACjC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACZ,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACtB,CAAC;IAED,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;IAC1C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IACrD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvF,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACpC,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;QAE1C,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7F,MAAM,IAAI,GAAG,GAAG;aACb,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;aAC1B,MAAM,CAAC,CAAC,CAAM,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;aAC5E,IAAI,CAAC,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAEzC,MAAM,IAAI,GAAG,GAAG;aACb,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;aACrB,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;aACvE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAEzB,KAAK,CAAC,IAAI,CAAC,MAAM,QAAQ,KAAK,MAAM,GAAG,CAAC,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,MAAM,SAAS,GAAG,CAAC,MAAM,WAAW,SAAS,EAAE,CAAC,CAAC;QACzE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,MAAM,IAAI,GAAG,GAAG;aACb,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAC9D,sEAAsE;aACrE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;aAC1C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAEzB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,KAAK,CAAC,IAAI,CAAC,kBAAkB,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,yCAAyC,CAAC,CAAC;YAC/F,KAAK,CAAC,IAAI,CAAC,kBAAkB,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACvE,CAAC;QAED,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,kBAAkB,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,iCAAiC,CAAC,CAAC;YACtF,KAAK,CAAC,IAAI,CAAC,kBAAkB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACtE,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,KAAK,CAAC,IAAI,CAAC,kBAAkB,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,gCAAgC,CAAC,CAAC;YACtF,KAAK,CAAC,IAAI,CAAC,kBAAkB,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACvE,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,KAAK,CAAC,IAAI,CAAC,kBAAkB,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,wCAAwC,CAAC,CAAC;YAC9F,KAAK,CAAC,IAAI,CAAC,kBAAkB,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACvE,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YAChB,KAAK,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;QACpE,CAAC;QAED,wEAAwE;QACxE,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC;QACpD,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACjC,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACtG,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;gBACrB,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;gBACvG,KAAK,CAAC,IAAI,CAAC,wBAAwB,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,SAAS,sBAAsB,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACzI,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACnD,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC5B,KAAK,MAAM,CAAC,IAAI,KAAK;gBAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QACrE,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC;IAC/D,MAAM,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,GAAG,CAAC,UAAU,OAAO,EAAE,CAAC,CAAC;AACnC,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;IACjB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACjB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}