offhands 0.1.18 → 0.1.19

package/dist/daemon.mjs

@@ -44206,6 +44206,7 @@ import { createServer as createServer3 } from "node:net";
 import { tmpdir as tmpdir4 } from "node:os";
 import { join as join13, resolve as resolve5 } from "node:path";
 var MIN_PAIRS_FOR_CLAIM = 10;
+var USAGE_GRACE_MS = 2e3;
 var median2 = (xs) => {
   if (xs.length === 0) return null;
   const s2 = [...xs].sort((a2, b2) => a2 - b2);
@@ -44225,6 +44226,33 @@ function signTestP(a2, b2) {
   }
   return Math.min(1, 2 * cumulative / 2 ** n3);
 }
+function signedRankP(diffs) {
+  const d2 = diffs.filter((x2) => x2 !== 0);
+  const n3 = d2.length;
+  if (n3 === 0) return null;
+  const order = d2.map((x2, i2) => ({ a: Math.abs(x2), i: i2 })).sort((p2, q2) => p2.a - q2.a);
+  const rank2 = new Array(n3);
+  for (let lo = 0; lo < n3; ) {
+    let hi = lo;
+    while (hi + 1 < n3 && order[hi + 1].a === order[lo].a) hi++;
+    const avgRank2 = lo + 1 + (hi + 1);
+    for (let k2 = lo; k2 <= hi; k2++) rank2[order[k2].i] = avgRank2;
+    lo = hi + 1;
+  }
+  const total2 = rank2.reduce((s2, r2) => s2 + r2, 0);
+  const observed2 = d2.reduce((s2, x2, i2) => s2 + (x2 > 0 ? rank2[i2] : 0), 0);
+  let counts = new Array(total2 + 1).fill(0);
+  counts[0] = 1;
+  for (const r2 of rank2) {
+    const next = counts.slice();
+    for (let s2 = 0; s2 + r2 <= total2; s2++) if (counts[s2]) next[s2 + r2] += counts[s2];
+    counts = next;
+  }
+  const dev = Math.abs(2 * observed2 - total2);
+  let extreme = 0;
+  for (let s2 = 0; s2 <= total2; s2++) if (Math.abs(2 * s2 - total2) >= dev) extreme += counts[s2];
+  return Math.min(1, extreme / 2 ** n3);
+}
 function recallOf(answer, expect) {
   if (!expect || expect.length === 0) return null;
   const hay = answer.toLowerCase().replace(/\\/g, "/");
@@ -44241,10 +44269,12 @@ function summarize(name, pairs, pick) {
     medianContext: median2(both.map((x2) => x2.c)),
     medianPlain: median2(both.map((x2) => x2.q)),
     medianDiff: median2(both.map((x2) => x2.c - x2.q)),
+    meanDiff: mean(both.map((x2) => x2.c - x2.q)),
     contextLower,
     plainLower,
     ties: both.length - contextLower - plainLower,
-    p: both.length ? signTestP(contextLower, plainLower) : null
+    p: both.length ? signTestP(contextLower, plainLower) : null,
+    pSignedRank: signedRankP(both.map((x2) => x2.c - x2.q))
   };
 }
 function analyzePairs(pairs, prompts) {
@@ -44286,11 +44316,12 @@ function analyzePairs(pairs, prompts) {
   const enough = usable.length >= MIN_PAIRS_FOR_CLAIM;
   for (const m3 of metrics) {
     if (m3.n === 0) continue;
-    const dir = m3.contextLower > m3.plainLower ? "lower" : "higher";
-    if (enough && m3.n >= MIN_PAIRS_FOR_CLAIM && m3.p !== null && m3.p < 0.05 && m3.contextLower !== m3.plainLower) {
-      conclusions.push(`${m3.name}: ${dir} with context in ${Math.max(m3.contextLower, m3.plainLower)} of ${m3.n} pairs (p=${m3.p.toFixed(3)}) \u2014 a measurable difference in this sample.`);
+    const dir = (m3.meanDiff ?? 0) < 0 ? "lower" : "higher";
+    const ps = `signed-rank p=${m3.pSignedRank === null ? "\u2014" : m3.pSignedRank.toFixed(3)}, sign-test p=${m3.p === null ? "\u2014" : m3.p.toFixed(3)}`;
+    if (enough && m3.n >= MIN_PAIRS_FOR_CLAIM && m3.pSignedRank !== null && m3.pSignedRank < 0.05) {
+      conclusions.push(`${m3.name}: ${dir} with context (mean difference ${(m3.meanDiff ?? 0).toFixed(2)}; ${ps}; ${m3.contextLower} lower / ${m3.plainLower} higher / ${m3.ties} tied) \u2014 a measurable difference in this sample.`);
     } else {
-      conclusions.push(`${m3.name}: no reliable difference detected (${m3.contextLower} lower with context, ${m3.plainLower} lower without, ${m3.ties} tied; p=${m3.p === null ? "\u2014" : m3.p.toFixed(3)}).`);
+      conclusions.push(`${m3.name}: no reliable difference detected (${m3.contextLower} lower with context, ${m3.plainLower} lower without, ${m3.ties} tied; ${ps}).`);
     }
   }
   if (usable.length === 0 && errorMessages.length > 0) {
@@ -44332,17 +44363,17 @@ function formatAbReport(r2, meta) {
     for (const m3 of r2.errorMessages.slice(0, 3)) L2.push(`  ${m3.runs} run(s): ${m3.message}`);
   }
   L2.push("");
-  L2.push("Metric              with ctx    plain     median diff   ctx lower / plain lower / tie   sign-test p");
+  L2.push("Metric              with ctx    plain     median diff  mean diff   ctx lower / plain lower / tie   signed-rank p   sign-test p");
   for (const m3 of r2.metrics) {
     const digits = m3.name.startsWith("Cost") ? 4 : m3.name.startsWith("Duration") ? 0 : 1;
     L2.push(
-      `${m3.name.padEnd(19)} ${fmt(m3.medianContext, digits).padEnd(10)} ${fmt(m3.medianPlain, digits).padEnd(9)} ${fmt(m3.medianDiff, digits).padEnd(13)} ${`${m3.contextLower} / ${m3.plainLower} / ${m3.ties}`.padEnd(30)} ${m3.p === null ? "\u2014" : m3.p.toFixed(3)}   (n=${m3.n})`
+      `${m3.name.padEnd(19)} ${fmt(m3.medianContext, digits).padEnd(10)} ${fmt(m3.medianPlain, digits).padEnd(9)} ${fmt(m3.medianDiff, digits).padEnd(12)} ${fmt(m3.meanDiff, digits).padEnd(11)} ${`${m3.contextLower} / ${m3.plainLower} / ${m3.ties}`.padEnd(30)} ${(m3.pSignedRank === null ? "\u2014" : m3.pSignedRank.toFixed(3)).padEnd(15)} ${m3.p === null ? "\u2014" : m3.p.toFixed(3)}   (n=${m3.n})`
     );
   }
   const q2 = r2.quality;
   if (q2.pairsWithGroundTruth > 0) {
     L2.push(
-      `${"Answer recall".padEnd(19)} ${fmt(q2.meanRecallContext, 2).padEnd(10)} ${fmt(q2.meanRecallPlain, 2).padEnd(9)} ${"".padEnd(13)} ${`${q2.contextHigher} / ${q2.plainHigher} / ${q2.ties}`.padEnd(30)} ${q2.p === null ? "\u2014" : q2.p.toFixed(3)}   (n=${q2.pairsWithGroundTruth}; higher is better)`
+      `${"Answer recall".padEnd(19)} ${fmt(q2.meanRecallContext, 2).padEnd(10)} ${fmt(q2.meanRecallPlain, 2).padEnd(9)} ${"".padEnd(12)} ${"".padEnd(11)} ${`${q2.contextHigher} / ${q2.plainHigher} / ${q2.ties}`.padEnd(30)} ${"".padEnd(15)} ${q2.p === null ? "\u2014" : q2.p.toFixed(3)}   (n=${q2.pairsWithGroundTruth}; higher is better)`
     );
   } else {
     L2.push("Answer recall       not measured (no prompt listed expected files)");
@@ -44351,7 +44382,7 @@ function formatAbReport(r2, meta) {
   for (const c2 of r2.conclusions) L2.push(`\u2022 ${c2}`);
   L2.push("");
   L2.push(
-    "One model, one repository, this prompt set. A sign test on paired runs; no savings figure is computed or implied, and none should be quoted from this."
+    "One model, one repository, this prompt set. Exact sign test and signed-rank test on paired runs; no savings figure is computed or implied, and none should be quoted from this."
   );
   return L2.join("\n");
 }
@@ -44462,6 +44493,7 @@ async function createAbHarness(cfg) {
         let detach = () => {
         };
         let timer;
+        let doneAt;
         const finish = () => {
           if (settled) return;
           settled = true;
@@ -44473,7 +44505,8 @@ async function createAbHarness(cfg) {
             timedOut,
             ...error ? { error } : {},
             toolCalls,
-            durationMs: Date.now() - t0,
+            // Measured to the agent's own `done`, not to when we stopped listening.
+            durationMs: (doneAt ?? Date.now()) - t0,
             costUsd,
             contextTokens,
             contextAttached,
@@ -44483,6 +44516,10 @@ async function createAbHarness(cfg) {
         };
         const sink = (m3) => {
           if (m3.sessionId !== session.id) return;
+          if (m3.type === "receipt" && doneAt !== void 0) {
+            finish();
+            return;
+          }
           if (m3.type === "intelligence-review") {
             contextAttached = true;
             filesSelected = m3.filesSelected;
@@ -44502,7 +44539,10 @@ async function createAbHarness(cfg) {
           } else if (e.type === "error") {
             error = String(e.message ?? "error");
             finish();
-          } else if (e.type === "done") finish();
+          } else if (e.type === "done") {
+            doneAt = Date.now();
+            setTimeout(finish, USAGE_GRACE_MS);
+          }
         };
         detach = manager2.attach(sink);
         timer = setTimeout(() => {
@@ -44525,7 +44565,51 @@ function flag(args2, name) {
   const i2 = args2.indexOf(name);
   return i2 >= 0 ? args2[i2 + 1] : void 0;
 }
+function intelAbMerge(args2, mergeAt, out) {
+  const files = args2.slice(mergeAt + 1).filter((a2) => !a2.startsWith("--"));
+  if (files.length === 0) {
+    out("usage: offhands intel-ab --merge <result.json> [<result.json> ...] [--json]");
+    return 1;
+  }
+  const prompts = [];
+  const pairs = [];
+  const workspaces = [];
+  let runner = "";
+  let model;
+  for (const f2 of files) {
+    let saved;
+    try {
+      saved = JSON.parse(readFileSync6(resolve5(f2), "utf8"));
+    } catch (e) {
+      out(`could not read ${f2}: ${e instanceof Error ? e.message : String(e)}`);
+      return 1;
+    }
+    if (!Array.isArray(saved?.prompts) || !Array.isArray(saved?.pairs)) {
+      out(`${f2} is not an intel-ab --out result (needs "prompts" and "pairs")`);
+      return 1;
+    }
+    const offset = prompts.length;
+    prompts.push(...saved.prompts);
+    for (const p2 of saved.pairs) pairs.push({ ...p2, promptIndex: p2.promptIndex + offset });
+    workspaces.push(String(saved.workspace ?? f2));
+    runner = runner || String(saved.runnerId ?? "");
+    model = model ?? saved.model;
+  }
+  const report = analyzePairs(pairs, prompts);
+  out(
+    args2.includes("--json") ? JSON.stringify(report, null, 2) : formatAbReport(report, {
+      workspace: workspaces.join("  +  "),
+      runner: runner || "unknown",
+      ...model ? { model } : {},
+      prompts: prompts.length,
+      repeats: Math.max(1, Math.round(pairs.length / Math.max(prompts.length, 1)))
+    })
+  );
+  return 0;
+}
 async function intelAbCli(args2, out = console.log, deps = {}) {
+  const mergeAt = args2.indexOf("--merge");
+  if (mergeAt >= 0) return intelAbMerge(args2, mergeAt, out);
   const workspaceArg = flag(args2, "--workspace");
   const promptsArg = flag(args2, "--prompts");
   const runnerId = flag(args2, "--runner") ?? "claude-code";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "offhands",
-  "version": "0.1.18",
+  "version": "0.1.19",
   "description": "Phone → coding-agent relay. Daemon runs on your laptop, PWA on your phone. E2E encrypted.",
   "license": "UNLICENSED",
   "type": "module",