offhands 0.1.17 → 0.1.19

package/dist/daemon.mjs

@@ -4776,8 +4776,8 @@ var require_main = __commonJS({
 });
 
 // ../daemon/src/index.ts
-import { resolve as resolve5 } from "node:path";
-import { existsSync as existsSync12 } from "node:fs";
+import { resolve as resolve6 } from "node:path";
+import { existsSync as existsSync13 } from "node:fs";
 
 // ../daemon/src/runners/claude-code.ts
 import { spawn } from "node:child_process";
@@ -4951,7 +4951,7 @@ var AsyncEventQueue = class {
     for (; ; ) {
       while (this.buffer.length > 0) yield this.buffer.shift();
       if (this.closed) return;
-      await new Promise((resolve6) => this.waiter = resolve6);
+      await new Promise((resolve7) => this.waiter = resolve7);
       this.waiter = null;
     }
   }
@@ -5038,10 +5038,10 @@ var ClaudeCodeRunner = class {
     return "claude";
   }
   async detect() {
-    return new Promise((resolve6) => {
+    return new Promise((resolve7) => {
       const p2 = spawn(this.resolveBin(), ["--version"], { stdio: "ignore" });
-      p2.on("error", () => resolve6(false));
-      p2.on("exit", (code) => resolve6(code === 0));
+      p2.on("error", () => resolve7(false));
+      p2.on("exit", (code) => resolve7(code === 0));
     });
   }
   start(run, callbacks) {
@@ -5132,10 +5132,10 @@ function extractSessionId(value) {
 // ../daemon/src/runners/stubs.ts
 import { spawn as spawn2 } from "node:child_process";
 function commandExists(bin) {
-  return new Promise((resolve6) => {
+  return new Promise((resolve7) => {
     const p2 = spawn2(bin, ["--version"], { stdio: "ignore", shell: process.platform === "win32" });
-    p2.on("error", () => resolve6(false));
-    p2.on("exit", (code) => resolve6(code === 0));
+    p2.on("error", () => resolve7(false));
+    p2.on("exit", (code) => resolve7(code === 0));
   });
 }
 function notImplemented(id) {
@@ -5276,10 +5276,10 @@ var CopilotCliRunner = class {
   async detect() {
     const cmd = this.resolveCommand();
     if (!cmd) return false;
-    return new Promise((resolve6) => {
+    return new Promise((resolve7) => {
       const p2 = spawn3(cmd[0], [...cmd.slice(1), "--version"], { stdio: "ignore" });
-      p2.on("error", () => resolve6(false));
-      p2.on("exit", (code) => resolve6(code === 0));
+      p2.on("error", () => resolve7(false));
+      p2.on("exit", (code) => resolve7(code === 0));
     });
   }
   loggedIn() {
@@ -5562,17 +5562,17 @@ var OpenCodeRunner = class {
     const cmd = this.resolveCommand();
     if (!cmd) return false;
     await this.fetchModels();
-    return new Promise((resolve6) => {
+    return new Promise((resolve7) => {
       const p2 = spawn4(cmd[0], [...cmd.slice(1), "--version"], { stdio: "ignore", shell: process.platform === "win32" });
-      p2.on("error", () => resolve6(false));
-      p2.on("exit", (code) => resolve6(code === 0));
+      p2.on("error", () => resolve7(false));
+      p2.on("exit", (code) => resolve7(code === 0));
     });
   }
   async fetchModels() {
     if (this.modelsCache.length > 0) return this.modelsCache;
     const cmd = this.resolveCommand();
     if (!cmd) return [];
-    return new Promise((resolve6) => {
+    return new Promise((resolve7) => {
       const p2 = spawn4(cmd[0], [...cmd.slice(1), "models"], {
         stdio: ["ignore", "pipe", "ignore"],
         shell: process.platform === "win32"
@@ -5593,15 +5593,15 @@ var OpenCodeRunner = class {
               writable: false,
               configurable: true
             });
-            resolve6(models);
+            resolve7(models);
           } catch {
-            resolve6([]);
+            resolve7([]);
           }
         } else {
-          resolve6([]);
+          resolve7([]);
         }
       });
-      p2.on("error", () => resolve6([]));
+      p2.on("error", () => resolve7([]));
     });
   }
   loggedIn() {
@@ -5882,8 +5882,8 @@ stderr: ${stderrTail}` : ""}`
       preview: truncate4(preview, 600),
       question: { text: first.question, options, multiSelect: first.multiple }
     }]);
-    const verdict = await new Promise((resolve6) => {
-      pendingQuestions.set(q2.id, (ok, answer) => resolve6({ ok, answer }));
+    const verdict = await new Promise((resolve7) => {
+      pendingQuestions.set(q2.id, (ok, answer) => resolve7({ ok, answer }));
     });
     emit([{
       type: "approval-result",
@@ -5996,14 +5996,14 @@ async function httpJson(url, method, body, expectNoContent = false) {
   return res.json();
 }
 function findFreePort() {
-  return new Promise((resolve6, reject) => {
+  return new Promise((resolve7, reject) => {
     const srv = createServer();
     srv.unref();
     srv.on("error", reject);
     srv.listen(0, "127.0.0.1", () => {
       const address = srv.address();
       const port2 = typeof address === "object" && address ? address.port : 0;
-      srv.close(() => resolve6(port2));
+      srv.close(() => resolve7(port2));
     });
   });
 }
@@ -6020,7 +6020,7 @@ async function waitForServerReady(port2, timeoutMs = 2e4) {
   throw new Error(`opencode serve did not become ready on port ${port2}`);
 }
 function sleep(ms) {
-  return new Promise((resolve6) => setTimeout(resolve6, ms));
+  return new Promise((resolve7) => setTimeout(resolve7, ms));
 }
 
 // ../daemon/src/session-manager.ts
@@ -6905,7 +6905,7 @@ async function buildContext(workspace, prompt, opts = {}) {
   const startedAt = Date.now();
   const maxChars = opts.maxChars ?? (Number(process.env.OFFHAND_INTEL_MAX_CHARS) || DEFAULT_MAX_CHARS);
   const work = gatherFindings(workspace, prompt);
-  const timeout = new Promise((resolve6) => setTimeout(() => resolve6("timeout"), OVERALL_TIMEOUT_MS));
+  const timeout = new Promise((resolve7) => setTimeout(() => resolve7("timeout"), OVERALL_TIMEOUT_MS));
   const result = await Promise.race([work, timeout]);
   const durationMs = Date.now() - startedAt;
   if (result === "timeout") {
@@ -12920,7 +12920,7 @@ var randomValuesStandard;
 var crypto;
 var randomValueNodeJS;
 var _Module = Module;
-Module.ready = new Promise(function(resolve6, reject) {
+Module.ready = new Promise(function(resolve7, reject) {
   var Module2 = _Module;
   Module2.onAbort = reject;
   Module2.print = function(what) {
@@ -12932,13 +12932,13 @@ Module.ready = new Promise(function(resolve6, reject) {
   Module2.onRuntimeInitialized = function() {
     try {
       Module2._crypto_secretbox_keybytes();
-      resolve6();
+      resolve7();
     } catch (err2) {
       reject(err2);
     }
   };
   Module2.useBackupModule = function() {
-    return new Promise(function(resolve7, reject2) {
+    return new Promise(function(resolve8, reject2) {
       var Module3 = {};
       Module3.onAbort = reject2;
       Module3.getRandomValue = _Module.getRandomValue;
@@ -12951,7 +12951,7 @@ Module.ready = new Promise(function(resolve6, reject) {
         Object.keys(Module3).forEach(function(k2) {
           _Module[k2] = Module3[k2];
         });
-        resolve7();
+        resolve8();
       };
       var Module3 = typeof Module3 != "undefined" ? Module3 : {};
       var ENVIRONMENT_IS_WEB2 = !!globalThis.window;
@@ -13011,13 +13011,13 @@ Module.ready = new Promise(function(resolve6, reject) {
           }
           readAsync2 = async (url) => {
             if (isFileURI2(url)) {
-              return new Promise((resolve8, reject3) => {
+              return new Promise((resolve9, reject3) => {
                 var xhr = new XMLHttpRequest();
                 xhr.open("GET", url, true);
                 xhr.responseType = "arraybuffer";
                 xhr.onload = () => {
                   if (xhr.status == 200 || xhr.status == 0 && xhr.response) {
-                    resolve8(xhr.response);
+                    resolve9(xhr.response);
                     return;
                   }
                   reject3(xhr.status);
@@ -39629,9 +39629,9 @@ Module.ready = new Promise(function(resolve6, reject) {
         }
         var info = getWasmImports2();
         if (Module3["instantiateWasm"]) {
-          return new Promise((resolve8, reject3) => {
+          return new Promise((resolve9, reject3) => {
             Module3["instantiateWasm"](info, (inst, mod) => {
-              resolve8(receiveInstance(inst, mod));
+              resolve9(receiveInstance(inst, mod));
             });
           });
         }
@@ -40132,13 +40132,13 @@ Module.ready = new Promise(function(resolve6, reject) {
       }
       readAsync = async (url) => {
         if (isFileURI(url)) {
-          return new Promise((resolve7, reject2) => {
+          return new Promise((resolve8, reject2) => {
             var xhr = new XMLHttpRequest();
             xhr.open("GET", url, true);
             xhr.responseType = "arraybuffer";
             xhr.onload = () => {
               if (xhr.status == 200 || xhr.status == 0 && xhr.response) {
-                resolve7(xhr.response);
+                resolve8(xhr.response);
                 return;
               }
               reject2(xhr.status);
@@ -40252,9 +40252,9 @@ Module.ready = new Promise(function(resolve6, reject) {
     }
     var info = getWasmImports();
     if (Module2["instantiateWasm"]) {
-      return new Promise((resolve7, reject2) => {
+      return new Promise((resolve8, reject2) => {
         Module2["instantiateWasm"](info, (inst, mod) => {
-          resolve7(receiveInstance(inst, mod));
+          resolve8(receiveInstance(inst, mod));
         });
       });
     }
@@ -44200,6 +44200,493 @@ function intelStatsCli(args2, out = console.log) {
   return 0;
 }
 
+// ../daemon/src/intelligence/ab.ts
+import { mkdtempSync, readFileSync as readFileSync6, rmSync as rmSync2, writeFileSync as writeFileSync5, existsSync as existsSync12 } from "node:fs";
+import { createServer as createServer3 } from "node:net";
+import { tmpdir as tmpdir4 } from "node:os";
+import { join as join13, resolve as resolve5 } from "node:path";
+var MIN_PAIRS_FOR_CLAIM = 10;
+var USAGE_GRACE_MS = 2e3;
+var median2 = (xs) => {
+  if (xs.length === 0) return null;
+  const s2 = [...xs].sort((a2, b2) => a2 - b2);
+  const mid = Math.floor(s2.length / 2);
+  return s2.length % 2 ? s2[mid] : (s2[mid - 1] + s2[mid]) / 2;
+};
+var mean = (xs) => xs.length ? xs.reduce((a2, b2) => a2 + b2, 0) / xs.length : null;
+function signTestP(a2, b2) {
+  const n3 = a2 + b2;
+  if (n3 === 0) return 1;
+  const k2 = Math.min(a2, b2);
+  let cumulative = 0;
+  let choose = 1;
+  for (let i2 = 0; i2 <= k2; i2++) {
+    cumulative += choose;
+    choose = choose * (n3 - i2) / (i2 + 1);
+  }
+  return Math.min(1, 2 * cumulative / 2 ** n3);
+}
+function signedRankP(diffs) {
+  const d2 = diffs.filter((x2) => x2 !== 0);
+  const n3 = d2.length;
+  if (n3 === 0) return null;
+  const order = d2.map((x2, i2) => ({ a: Math.abs(x2), i: i2 })).sort((p2, q2) => p2.a - q2.a);
+  const rank2 = new Array(n3);
+  for (let lo = 0; lo < n3; ) {
+    let hi = lo;
+    while (hi + 1 < n3 && order[hi + 1].a === order[lo].a) hi++;
+    const avgRank2 = lo + 1 + (hi + 1);
+    for (let k2 = lo; k2 <= hi; k2++) rank2[order[k2].i] = avgRank2;
+    lo = hi + 1;
+  }
+  const total2 = rank2.reduce((s2, r2) => s2 + r2, 0);
+  const observed2 = d2.reduce((s2, x2, i2) => s2 + (x2 > 0 ? rank2[i2] : 0), 0);
+  let counts = new Array(total2 + 1).fill(0);
+  counts[0] = 1;
+  for (const r2 of rank2) {
+    const next = counts.slice();
+    for (let s2 = 0; s2 + r2 <= total2; s2++) if (counts[s2]) next[s2 + r2] += counts[s2];
+    counts = next;
+  }
+  const dev = Math.abs(2 * observed2 - total2);
+  let extreme = 0;
+  for (let s2 = 0; s2 <= total2; s2++) if (Math.abs(2 * s2 - total2) >= dev) extreme += counts[s2];
+  return Math.min(1, extreme / 2 ** n3);
+}
+function recallOf(answer, expect) {
+  if (!expect || expect.length === 0) return null;
+  const hay = answer.toLowerCase().replace(/\\/g, "/");
+  const hits = expect.filter((e) => hay.includes(e.toLowerCase().replace(/\\/g, "/"))).length;
+  return hits / expect.length;
+}
+function summarize(name, pairs, pick) {
+  const both = pairs.map((p2) => ({ c: pick(p2.context), q: pick(p2.plain) })).filter((x2) => x2.c !== null && x2.q !== null);
+  const contextLower = both.filter((x2) => x2.c < x2.q).length;
+  const plainLower = both.filter((x2) => x2.c > x2.q).length;
+  return {
+    name,
+    n: both.length,
+    medianContext: median2(both.map((x2) => x2.c)),
+    medianPlain: median2(both.map((x2) => x2.q)),
+    medianDiff: median2(both.map((x2) => x2.c - x2.q)),
+    meanDiff: mean(both.map((x2) => x2.c - x2.q)),
+    contextLower,
+    plainLower,
+    ties: both.length - contextLower - plainLower,
+    p: both.length ? signTestP(contextLower, plainLower) : null,
+    pSignedRank: signedRankP(both.map((x2) => x2.c - x2.q))
+  };
+}
+function analyzePairs(pairs, prompts) {
+  const contextNotAttached = pairs.filter((p2) => p2.context.ok && !p2.context.contextAttached).length;
+  const errored = pairs.filter((p2) => !p2.context.ok && !p2.context.timedOut || !p2.plain.ok && !p2.plain.timedOut).length;
+  const usable = pairs.filter(
+    (p2) => (p2.context.ok || p2.context.timedOut) && (p2.plain.ok || p2.plain.timedOut) && p2.context.contextAttached
+  );
+  const timedOutRuns = usable.reduce((n3, p2) => n3 + (p2.context.timedOut ? 1 : 0) + (p2.plain.timedOut ? 1 : 0), 0);
+  const errorCounts = /* @__PURE__ */ new Map();
+  for (const p2 of pairs) {
+    for (const r2 of [p2.context, p2.plain]) {
+      if (r2.error) errorCounts.set(r2.error, (errorCounts.get(r2.error) ?? 0) + 1);
+    }
+  }
+  const errorMessages = [...errorCounts].map(([message, runs]) => ({ message, runs })).sort((a2, b2) => b2.runs - a2.runs);
+  const metrics = [
+    summarize("Tool calls", usable, (r2) => r2.toolCalls),
+    summarize("Duration (s)", usable, (r2) => r2.durationMs / 1e3),
+    summarize("Cost (USD)", usable, (r2) => r2.costUsd),
+    summarize("Context tokens", usable, (r2) => r2.contextTokens)
+  ];
+  const rec = usable.map((p2) => {
+    const expect = prompts[p2.promptIndex]?.expect;
+    return { c: recallOf(p2.context.answer, expect), q: recallOf(p2.plain.answer, expect) };
+  }).filter((x2) => x2.c !== null && x2.q !== null);
+  const contextHigher = rec.filter((x2) => x2.c > x2.q).length;
+  const plainHigher = rec.filter((x2) => x2.c < x2.q).length;
+  const quality = {
+    pairsWithGroundTruth: rec.length,
+    meanRecallContext: mean(rec.map((x2) => x2.c)),
+    meanRecallPlain: mean(rec.map((x2) => x2.q)),
+    contextHigher,
+    plainHigher,
+    ties: rec.length - contextHigher - plainHigher,
+    p: rec.length ? signTestP(contextHigher, plainHigher) : null
+  };
+  const conclusions = [];
+  const enough = usable.length >= MIN_PAIRS_FOR_CLAIM;
+  for (const m3 of metrics) {
+    if (m3.n === 0) continue;
+    const dir = (m3.meanDiff ?? 0) < 0 ? "lower" : "higher";
+    const ps = `signed-rank p=${m3.pSignedRank === null ? "\u2014" : m3.pSignedRank.toFixed(3)}, sign-test p=${m3.p === null ? "\u2014" : m3.p.toFixed(3)}`;
+    if (enough && m3.n >= MIN_PAIRS_FOR_CLAIM && m3.pSignedRank !== null && m3.pSignedRank < 0.05) {
+      conclusions.push(`${m3.name}: ${dir} with context (mean difference ${(m3.meanDiff ?? 0).toFixed(2)}; ${ps}; ${m3.contextLower} lower / ${m3.plainLower} higher / ${m3.ties} tied) \u2014 a measurable difference in this sample.`);
+    } else {
+      conclusions.push(`${m3.name}: no reliable difference detected (${m3.contextLower} lower with context, ${m3.plainLower} lower without, ${m3.ties} tied; ${ps}).`);
+    }
+  }
+  if (usable.length === 0 && errorMessages.length > 0) {
+    conclusions.unshift(`No run completed, so nothing can be concluded. Most common error: ${errorMessages[0].message}`);
+  }
+  if (!enough) {
+    conclusions.push(`Only ${usable.length} usable pair(s); at least ${MIN_PAIRS_FOR_CLAIM} are needed before any difference is reported as measurable.`);
+  }
+  if (quality.meanRecallContext !== null && quality.meanRecallPlain !== null && quality.meanRecallContext < quality.meanRecallPlain - 0.1) {
+    conclusions.push("Warning: answers WITH context were less complete against the expected files \u2014 any saving is not a win if the answer is worse.");
+  }
+  return {
+    pairsTotal: pairs.length,
+    pairsAnalysed: usable.length,
+    excluded: { contextNotAttached, errored },
+    errorMessages,
+    timedOutRuns,
+    metrics,
+    quality,
+    conclusions
+  };
+}
+var fmt = (x2, digits = 1) => x2 === null ? "\u2014" : x2.toFixed(digits);
+function formatAbReport(r2, meta) {
+  const L2 = [];
+  L2.push("Offhand Intelligence \u2014 paired A/B measurement");
+  L2.push("");
+  L2.push(`Workspace: ${meta.workspace}`);
+  L2.push(`Runner:    ${meta.runner}${meta.model ? ` (${meta.model})` : ""} \xB7 ${meta.prompts} prompt(s) \xD7 ${meta.repeats} repeat(s) = ${r2.pairsTotal} pair(s)`);
+  L2.push(
+    `Analysed:  ${r2.pairsAnalysed} of ${r2.pairsTotal} pair(s)` + (r2.excluded.contextNotAttached ? ` \xB7 ${r2.excluded.contextNotAttached} excluded (retrieval found nothing to attach)` : "") + (r2.excluded.errored ? ` \xB7 ${r2.excluded.errored} excluded (a run errored)` : "")
+  );
+  if (r2.timedOutRuns) L2.push(`Warning:   ${r2.timedOutRuns} run(s) hit the time limit \u2014 their numbers are lower bounds.`);
+  if (meta.pairsPlanned !== void 0 && r2.pairsTotal < meta.pairsPlanned) {
+    L2.push(`Stopped early: ran ${r2.pairsTotal} of ${meta.pairsPlanned} planned pair(s), because consecutive pairs failed entirely.`);
+  }
+  if (r2.errorMessages.length) {
+    L2.push("Errors:");
+    for (const m3 of r2.errorMessages.slice(0, 3)) L2.push(`  ${m3.runs} run(s): ${m3.message}`);
+  }
+  L2.push("");
+  L2.push("Metric              with ctx    plain     median diff  mean diff   ctx lower / plain lower / tie   signed-rank p   sign-test p");
+  for (const m3 of r2.metrics) {
+    const digits = m3.name.startsWith("Cost") ? 4 : m3.name.startsWith("Duration") ? 0 : 1;
+    L2.push(
+      `${m3.name.padEnd(19)} ${fmt(m3.medianContext, digits).padEnd(10)} ${fmt(m3.medianPlain, digits).padEnd(9)} ${fmt(m3.medianDiff, digits).padEnd(12)} ${fmt(m3.meanDiff, digits).padEnd(11)} ${`${m3.contextLower} / ${m3.plainLower} / ${m3.ties}`.padEnd(30)} ${(m3.pSignedRank === null ? "\u2014" : m3.pSignedRank.toFixed(3)).padEnd(15)} ${m3.p === null ? "\u2014" : m3.p.toFixed(3)}   (n=${m3.n})`
+    );
+  }
+  const q2 = r2.quality;
+  if (q2.pairsWithGroundTruth > 0) {
+    L2.push(
+      `${"Answer recall".padEnd(19)} ${fmt(q2.meanRecallContext, 2).padEnd(10)} ${fmt(q2.meanRecallPlain, 2).padEnd(9)} ${"".padEnd(12)} ${"".padEnd(11)} ${`${q2.contextHigher} / ${q2.plainHigher} / ${q2.ties}`.padEnd(30)} ${"".padEnd(15)} ${q2.p === null ? "\u2014" : q2.p.toFixed(3)}   (n=${q2.pairsWithGroundTruth}; higher is better)`
+    );
+  } else {
+    L2.push("Answer recall       not measured (no prompt listed expected files)");
+  }
+  L2.push("");
+  for (const c2 of r2.conclusions) L2.push(`\u2022 ${c2}`);
+  L2.push("");
+  L2.push(
+    "One model, one repository, this prompt set. Exact sign test and signed-rank test on paired runs; no savings figure is computed or implied, and none should be quoted from this."
+  );
+  return L2.join("\n");
+}
+function parsePromptFile(text) {
+  let raw;
+  try {
+    raw = JSON.parse(text);
+  } catch (e) {
+    throw new Error(`prompts file is not valid JSON: ${e instanceof Error ? e.message : String(e)}`);
+  }
+  if (!Array.isArray(raw) || raw.length === 0) throw new Error("prompts file must be a non-empty JSON array");
+  return raw.map((item, i2) => {
+    if (typeof item === "string" && item.trim()) return { prompt: item };
+    if (item && typeof item === "object" && typeof item.prompt === "string" && item.prompt.trim()) {
+      const expect = item.expect;
+      if (expect !== void 0 && !(Array.isArray(expect) && expect.every((e) => typeof e === "string"))) {
+        throw new Error(`prompt #${i2 + 1}: "expect" must be an array of strings`);
+      }
+      return { prompt: item.prompt, ...expect ? { expect } : {} };
+    }
+    throw new Error(`prompt #${i2 + 1} must be a non-empty string or { "prompt": "...", "expect": [...] }`);
+  });
+}
+function mulberry32(seed) {
+  let a2 = seed >>> 0;
+  return () => {
+    a2 = a2 + 1831565813 >>> 0;
+    let t2 = a2;
+    t2 = Math.imul(t2 ^ t2 >>> 15, t2 | 1);
+    t2 ^= t2 + Math.imul(t2 ^ t2 >>> 7, t2 | 61);
+    return ((t2 ^ t2 >>> 14) >>> 0) / 4294967296;
+  };
+}
+async function runAb(cfg, harness) {
+  const rng = mulberry32(cfg.seed ?? Date.now());
+  const total = cfg.prompts.length * cfg.repeats;
+  const abortAfter = cfg.abortAfterFailedPairs ?? 2;
+  const pairs = [];
+  let done = 0;
+  let consecutiveFailed = 0;
+  outer: for (let rep = 0; rep < cfg.repeats; rep++) {
+    for (let promptIndex = 0; promptIndex < cfg.prompts.length; promptIndex++) {
+      const order = rng() < 0.5 ? ["context", "plain"] : ["plain", "context"];
+      const prompt = cfg.prompts[promptIndex];
+      const results = {};
+      if (cfg.parallel) {
+        const [first, second] = await Promise.all(order.map((arm) => harness.runArm(arm, prompt)));
+        results[order[0]] = first;
+        results[order[1]] = second;
+      } else {
+        for (const arm of order) results[arm] = await harness.runArm(arm, prompt);
+      }
+      pairs.push({ promptIndex, rep, order, context: results.context, plain: results.plain });
+      done++;
+      cfg.onProgress?.(
+        `[${done}/${total}] prompt ${promptIndex + 1} rep ${rep + 1}: context ${results.context.toolCalls} tools/${(results.context.durationMs / 1e3).toFixed(0)}s${results.context.contextAttached ? "" : " (no context attached)"} \xB7 plain ${results.plain.toolCalls} tools/${(results.plain.durationMs / 1e3).toFixed(0)}s`
+      );
+      const failed = (r2) => !r2.ok && !r2.timedOut;
+      consecutiveFailed = failed(results.context) && failed(results.plain) ? consecutiveFailed + 1 : 0;
+      if (abortAfter > 0 && consecutiveFailed >= abortAfter) {
+        cfg.onProgress?.(
+          `Stopping early: ${consecutiveFailed} consecutive pairs failed entirely (${results.context.error ?? results.plain.error ?? "unknown error"}). More runs would not help \u2014 fix that first.`
+        );
+        break outer;
+      }
+    }
+  }
+  return pairs;
+}
+async function freePort() {
+  return new Promise((resolvePort, reject) => {
+    const srv = createServer3();
+    srv.once("error", reject);
+    srv.listen(0, "127.0.0.1", () => {
+      const addr = srv.address();
+      const port2 = typeof addr === "object" && addr ? addr.port : 0;
+      srv.close(() => resolvePort(port2));
+    });
+  });
+}
+async function createAbHarness(cfg) {
+  const dir = mkdtempSync(join13(tmpdir4(), "offhand-ab-"));
+  const store2 = new Store(dir);
+  store2.upsertWorkspace(cfg.workspace);
+  store2.setWorkspaceIntelligenceMode(cfg.workspace, "automatic");
+  const broker2 = new ApprovalBroker(3e4);
+  const port2 = cfg.serve ? await freePort() : 0;
+  const manager2 = new SessionManager(store2, cfg.makeRunners(broker2, port2));
+  await manager2.init();
+  broker2.policyProvider = () => manager2.currentPolicy();
+  if (cfg.serve) new LocalSessionServer(manager2, port2, broker2);
+  const send = (msg, reply) => manager2.handle(parseClientMessage(JSON.stringify(msg)), reply);
+  return {
+    runArm(arm, prompt) {
+      const session = store2.createSession(cfg.workspace, cfg.runnerId, cfg.model, `ab-${arm}`);
+      store2.updateSession(session.id, { permissionMode: "plan" });
+      const t0 = Date.now();
+      let toolCalls = 0;
+      let answer = "";
+      let costUsd = null;
+      let contextTokens = null;
+      let contextAttached = false;
+      let filesSelected = null;
+      let error;
+      let timedOut = false;
+      return new Promise((resolveArm) => {
+        let settled = false;
+        let detach = () => {
+        };
+        let timer;
+        let doneAt;
+        const finish = () => {
+          if (settled) return;
+          settled = true;
+          if (timer) clearTimeout(timer);
+          detach();
+          resolveArm({
+            arm,
+            ok: !error && !timedOut,
+            timedOut,
+            ...error ? { error } : {},
+            toolCalls,
+            // Measured to the agent's own `done`, not to when we stopped listening.
+            durationMs: (doneAt ?? Date.now()) - t0,
+            costUsd,
+            contextTokens,
+            contextAttached,
+            filesSelected,
+            answer: answer.slice(0, 2e4)
+          });
+        };
+        const sink = (m3) => {
+          if (m3.sessionId !== session.id) return;
+          if (m3.type === "receipt" && doneAt !== void 0) {
+            finish();
+            return;
+          }
+          if (m3.type === "intelligence-review") {
+            contextAttached = true;
+            filesSelected = m3.filesSelected;
+            send({ type: "intelligence-response", sessionId: session.id, reviewId: m3.reviewId, useContext: true }, sink);
+            return;
+          }
+          if (m3.type !== "run-event") return;
+          const e = m3.event;
+          if (e.type === "text") answer += e.chunk;
+          else if (e.type === "tool") {
+            if (e.name !== "auto-approved") toolCalls++;
+          } else if (e.type === "usage") {
+            if (typeof e.costUsd === "number") costUsd = e.costUsd;
+            if (typeof e.contextTokens === "number") contextTokens = e.contextTokens;
+          } else if (e.type === "approval") {
+            send({ type: "approval-response", approvalId: e.id, approve: false }, sink);
+          } else if (e.type === "error") {
+            error = String(e.message ?? "error");
+            finish();
+          } else if (e.type === "done") {
+            doneAt = Date.now();
+            setTimeout(finish, USAGE_GRACE_MS);
+          }
+        };
+        detach = manager2.attach(sink);
+        timer = setTimeout(() => {
+          timedOut = true;
+          send({ type: "cancel", sessionId: session.id }, sink);
+          setTimeout(finish, 3e3);
+        }, cfg.timeoutMs);
+        send({ type: arm === "context" ? "intelligence-prepare" : "prompt", sessionId: session.id, prompt: prompt.prompt }, sink);
+      });
+    },
+    close() {
+      try {
+        rmSync2(dir, { recursive: true, force: true });
+      } catch {
+      }
+    }
+  };
+}
+function flag(args2, name) {
+  const i2 = args2.indexOf(name);
+  return i2 >= 0 ? args2[i2 + 1] : void 0;
+}
+function intelAbMerge(args2, mergeAt, out) {
+  const files = args2.slice(mergeAt + 1).filter((a2) => !a2.startsWith("--"));
+  if (files.length === 0) {
+    out("usage: offhands intel-ab --merge <result.json> [<result.json> ...] [--json]");
+    return 1;
+  }
+  const prompts = [];
+  const pairs = [];
+  const workspaces = [];
+  let runner = "";
+  let model;
+  for (const f2 of files) {
+    let saved;
+    try {
+      saved = JSON.parse(readFileSync6(resolve5(f2), "utf8"));
+    } catch (e) {
+      out(`could not read ${f2}: ${e instanceof Error ? e.message : String(e)}`);
+      return 1;
+    }
+    if (!Array.isArray(saved?.prompts) || !Array.isArray(saved?.pairs)) {
+      out(`${f2} is not an intel-ab --out result (needs "prompts" and "pairs")`);
+      return 1;
+    }
+    const offset = prompts.length;
+    prompts.push(...saved.prompts);
+    for (const p2 of saved.pairs) pairs.push({ ...p2, promptIndex: p2.promptIndex + offset });
+    workspaces.push(String(saved.workspace ?? f2));
+    runner = runner || String(saved.runnerId ?? "");
+    model = model ?? saved.model;
+  }
+  const report = analyzePairs(pairs, prompts);
+  out(
+    args2.includes("--json") ? JSON.stringify(report, null, 2) : formatAbReport(report, {
+      workspace: workspaces.join("  +  "),
+      runner: runner || "unknown",
+      ...model ? { model } : {},
+      prompts: prompts.length,
+      repeats: Math.max(1, Math.round(pairs.length / Math.max(prompts.length, 1)))
+    })
+  );
+  return 0;
+}
+async function intelAbCli(args2, out = console.log, deps = {}) {
+  const mergeAt = args2.indexOf("--merge");
+  if (mergeAt >= 0) return intelAbMerge(args2, mergeAt, out);
+  const workspaceArg = flag(args2, "--workspace");
+  const promptsArg = flag(args2, "--prompts");
+  const runnerId = flag(args2, "--runner") ?? "claude-code";
+  const repeats = Number(flag(args2, "--repeats") ?? 2);
+  const timeoutS = Number(flag(args2, "--timeout") ?? 240);
+  const maxRuns = Number(flag(args2, "--max-runs") ?? 60);
+  const seedArg = flag(args2, "--seed");
+  const model = flag(args2, "--model") ?? (runnerId === "claude-code" ? "haiku" : void 0);
+  const outFile = flag(args2, "--out");
+  const json = args2.includes("--json");
+  const parallel = !args2.includes("--sequential");
+  if (!workspaceArg || !promptsArg) {
+    out("usage: offhands intel-ab --workspace <path> --prompts <file.json> [--runner claude-code|opencode] [--model m] [--repeats 2] [--timeout 240] [--sequential] [--max-runs 60] [--seed n] [--json] [--out file] [--dry-run] --yes");
+    return 1;
+  }
+  if (runnerId !== "claude-code" && runnerId !== "opencode") {
+    out(`unsupported runner "${runnerId}" \u2014 use claude-code or opencode`);
+    return 1;
+  }
+  if (!Number.isInteger(repeats) || repeats < 1 || !(timeoutS > 0)) {
+    out("--repeats must be a positive integer and --timeout a positive number of seconds");
+    return 1;
+  }
+  const workspace = resolve5(workspaceArg);
+  if (!existsSync12(workspace)) {
+    out(`workspace does not exist: ${workspace}`);
+    return 1;
+  }
+  let prompts;
+  try {
+    prompts = parsePromptFile(readFileSync6(resolve5(promptsArg), "utf8"));
+  } catch (e) {
+    out(`could not use the prompts file: ${e instanceof Error ? e.message : String(e)}`);
+    return 1;
+  }
+  const pairsPlanned = prompts.length * repeats;
+  const runsPlanned = pairsPlanned * 2;
+  if (runsPlanned > maxRuns) {
+    out(`that plan is ${runsPlanned} agent runs, above --max-runs ${maxRuns}. Lower --repeats or the prompt count, or raise --max-runs deliberately.`);
+    return 1;
+  }
+  out(
+    `Plan: ${prompts.length} prompt(s) \xD7 ${repeats} repeat(s) \xD7 2 arms = ${runsPlanned} real ${runnerId} runs${model ? ` (model ${model})` : ""} in read-only plan mode, ${parallel ? "each pair in parallel" : "sequential"}, up to ${timeoutS}s each.`
+  );
+  out(`Workspace: ${workspace}`);
+  if (args2.includes("--dry-run") || !args2.includes("--yes")) {
+    out("");
+    out(args2.includes("--dry-run") ? "Dry run \u2014 nothing was executed." : "This spends real agent tokens. Re-run with --yes to proceed (or --dry-run to only see the plan).");
+    return args2.includes("--dry-run") ? 0 : 2;
+  }
+  if (runnerId === "claude-code") {
+    delete process.env.CLAUDECODE;
+    delete process.env.CLAUDE_CODE_ENTRYPOINT;
+  }
+  const createHarness = deps.createHarness ?? createAbHarness;
+  const harness = await createHarness({
+    workspace,
+    runnerId,
+    ...model ? { model } : {},
+    timeoutMs: timeoutS * 1e3,
+    serve: !deps.createHarness,
+    makeRunners: (broker2, port2) => [
+      runnerId === "opencode" ? new OpenCodeRunner(broker2, `http://127.0.0.1:${port2}/status`) : new ClaudeCodeRunner(broker2, `http://127.0.0.1:${port2}/approval`)
+    ]
+  });
+  try {
+    const pairs = await runAb({ prompts, repeats, parallel, ...seedArg ? { seed: Number(seedArg) } : {}, onProgress: out }, harness);
+    const report = analyzePairs(pairs, prompts);
+    if (outFile) writeFileSync5(resolve5(outFile), JSON.stringify({ workspace, runnerId, model, prompts, pairs, report }, null, 2));
+    out("");
+    out(json ? JSON.stringify(report, null, 2) : formatAbReport(report, { workspace, runner: runnerId, ...model ? { model } : {}, prompts: prompts.length, repeats, pairsPlanned }));
+    return 0;
+  } finally {
+    harness.close();
+  }
+}
+
 // ../daemon/src/index.ts
 var args = process.argv.slice(2);
 if (args[0] === "drop") {
@@ -44220,14 +44707,17 @@ if (args[0] === "drop") {
 if (args[0] === "intel-stats") {
   process.exit(intelStatsCli(args.slice(1)));
 }
-function argValues(flag) {
+if (args[0] === "intel-ab") {
+  process.exit(await intelAbCli(args.slice(1)));
+}
+function argValues(flag2) {
   const out = [];
   for (let i2 = 0; i2 < args.length; i2++) {
-    if (args[i2] === flag && args[i2 + 1]) out.push(args[i2 + 1]);
+    if (args[i2] === flag2 && args[i2 + 1]) out.push(args[i2 + 1]);
   }
   return out;
 }
-var argValue = (flag) => argValues(flag)[0];
+var argValue = (flag2) => argValues(flag2)[0];
 var port = Number(argValue("--port") ?? 4317);
 var relayUrl = argValue("--relay")?.replace(/^ws(s?):\/\//, "http$1://");
 var forceRepair = args.includes("--repair");
@@ -44235,9 +44725,9 @@ var approvalTimeoutMs = Number(argValue("--approval-timeout") ?? 300) * 1e3;
 var webUrl = argValue("--web-url") ?? "https://offhand-web.onrender.com";
 var devUrl = argValue("--dev-url");
 var store = new Store();
-var wsArgs = argValues("--workspace").map((w2) => resolve5(w2));
+var wsArgs = argValues("--workspace").map((w2) => resolve6(w2));
 for (const w2 of wsArgs) {
-  if (!existsSync12(w2)) {
+  if (!existsSync13(w2)) {
     console.error(`workspace does not exist: ${w2}`);
     process.exit(1);
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "offhands",
-  "version": "0.1.17",
+  "version": "0.1.19",
   "description": "Phone → coding-agent relay. Daemon runs on your laptop, PWA on your phone. E2E encrypted.",
   "license": "UNLICENSED",
   "type": "module",
@@ -24,6 +24,6 @@
     "esbuild": "^0.24.0"
   },
   "dependencies": {
-    "offhands": "^0.1.16"
+    "offhands": "^0.1.17"
   }
 }