@staix/agent-hub 0.12.19 → 0.12.21

package/src/adapters/codex-appserver.ts

@@ -1,6 +1,7 @@
 import { spawn, type ChildProcess } from "node:child_process";
 import type { Server, ServerWebSocket } from "bun";
 import { renderDigest, replyAudience, replyParent, type Envelope, type PeerId } from "../hub/envelope.ts";
+import type { PermissionMode } from "../hub/permission-mode.ts";
 import { BasePeer } from "../hub/peers.ts";
 import { codexContext, type ContextReading } from "../hub/context-window.ts";
 import { peerChildEnv, stopOwnedProcess, trackGroup } from "../hub/child-process.ts";
@@ -8,6 +9,7 @@ import { peerChildEnv, stopOwnedProcess, trackGroup } from "../hub/child-process
 export interface CodexOptions {
   /** Port the TUI attaches to: `codex --enable tui_app_server --remote ws://127.0.0.1:<proxyPort>`. */
   proxyPort: number;
+  permissionMode?: PermissionMode;
   /** Port for the spawned `codex app-server`. Ignored when `upstreamUrl` is set. */
   appPort: number;
   /** Attach to an already running app-server instead of spawning one (tests). */
@@ -24,6 +26,8 @@ export interface CodexOptions {
   onContext?: (reading: ContextReading) => void;
   /** The native id of each turn as it starts, after the peer turned busy (issue #33: `ahub undo --context`). */
   onTurn?: (turnId: string) => void;
+  /** Each thread the TUI starts (`fresh`) or resumes, as it is adopted (#215: which turns belong to which thread). */
+  onThread?: (threadId: string, fresh: boolean) => void;
   /**
    * Each completed `fileChange`, `commandExecution`, `mcpToolCall` and `userMessage` item, for turn-free facts (issue
    * #108): the first three are boundaries, a `userMessage` is the readback of a steered fact. Codex emits
@@ -58,6 +62,10 @@ const TRACKED = new Set(["thread/start", "thread/resume"]);
  * TUI's, and their responses are swallowed.
  */
 export class CodexPeer extends BasePeer {
+  private permissionMode: PermissionMode;
+  private nativeApprovalPolicy: unknown;
+  private approvalOverridden = false;
+  private readonly restoringApprovals = new Set<string | number>();
   private proc: ChildProcess | undefined;
   private server: Server<Link> | undefined;
   private link: Link | undefined; // the connection that owns the current thread
@@ -94,6 +102,7 @@ export class CodexPeer extends BasePeer {
     private readonly opts: CodexOptions,
   ) {
     super(id, opts.watchdogMs);
+    this.permissionMode = opts.permissionMode ?? "ask";
   }
 
   recoveryMetadata(): Record<string, unknown> {
@@ -182,6 +191,7 @@ export class CodexPeer extends BasePeer {
       this.pending.set(id, {
         deliveryId, kind: "deliver",
         resolve: (result) => {
+          if (this.restoringApprovals.delete(id) && this.permissionMode === "ask") this.approvalOverridden = false;
           this.primed = true;
           this.injected = replyParent(envs);
           this.answering = [...envs];
@@ -194,13 +204,14 @@ export class CodexPeer extends BasePeer {
           resolve();
         },
         reject: (e) => {
+          this.restoringApprovals.delete(id);
           if (!this.activeTurns.size) this.setState("idle");
           if (deliveryId) this.delivery({ id: deliveryId, state: this.knownRejection(e) ? "failed_safe" : "needs_review", reason: e.message });
           reject(e);
         },
       });
       link.up.send(
-        JSON.stringify({ method: "turn/start", id, params: { threadId: this.threadId, input: [{ type: "text", text }] } }),
+        JSON.stringify({ method: "turn/start", id, params: this.applyPermissionPolicy({ threadId: this.threadId, input: [{ type: "text", text }] }, id) }),
       );
     });
   }
@@ -348,6 +359,7 @@ export class CodexPeer extends BasePeer {
     if (this.link !== link) return;
     clearInterval(this.usageTimer);
     this.link = undefined;
+    this.restoringApprovals.clear();
     this.threadId = "";
     this.contextEpoch++;
     this.activeTurns.clear();
@@ -370,8 +382,55 @@ export class CodexPeer extends BasePeer {
     this.lastAnswer = "";
   }
 
+  get proxyAttached(): boolean { return !!this.link && this.link.up.readyState === WebSocket.OPEN; }
+
+  getPermissionMode(): PermissionMode { return this.permissionMode; }
+
+  /** Reconcile the daemon's current choice after startup, before the returned proxy URL has a TUI. */
+  setStartupPermissionMode(mode: PermissionMode): void {
+    if (this.link || this.state !== "offline" || this.approvalOverridden) throw new Error("codex startup mode reconciliation requires a fresh unattached proxy");
+    this.permissionMode = mode;
+  }
+
+  /**
+   * The hub's choice was cleared while no TUI is attached: this proxy sends no override from now on. Returns false
+   * when an override it already sent cannot be undone on that thread (its native policy was never reported): a thread
+   * started or resumed through the proxy afterwards reports its own policy, and that one is left alone.
+   */
+  clearPermissionMode(): boolean {
+    const restorable = !this.approvalOverridden || this.nativeApprovalPolicy !== undefined;
+    this.permissionMode = "ask";
+    return restorable;
+  }
+
+  async setPermissionMode(mode: PermissionMode): Promise<void> {
+    if (!this.link || this.link.up.readyState !== WebSocket.OPEN) throw new Error("codex permission mode unavailable: attach the TUI through ahub codex first");
+    if (mode === "ask" && this.approvalOverridden && this.nativeApprovalPolicy === undefined) throw new Error("codex cannot restore ask: native approval policy unavailable; restart the Codex session");
+    this.permissionMode = mode;
+  }
+
+  private applyPermissionPolicy(params: Record<string, unknown>, id: string | number): Record<string, unknown> {
+    if (this.permissionMode !== "ask") {
+      this.approvalOverridden = true;
+      return { ...params, approvalPolicy: this.permissionMode === "never-ask" ? "never" : "on-request" };
+    }
+    if (this.permissionMode === "ask" && this.approvalOverridden) {
+      this.restoringApprovals.add(id);
+      return { ...params, approvalPolicy: this.nativeApprovalPolicy };
+    }
+    return params;
+  }
+
   private fromTui(link: Link, raw: string): void {
     const msg = parse(raw);
+    if (msg?.method === "turn/start" && msg.params && link === this.link) {
+      if (msg.params.approvalPolicy != null) this.nativeApprovalPolicy = msg.params.approvalPolicy;
+      if (this.permissionMode !== "ask" || this.approvalOverridden) {
+        // Native turn overrides persist: ask restores the captured native policy once on successful admission.
+        msg.params = this.applyPermissionPolicy(msg.params, msg.id);
+        raw = JSON.stringify(msg);
+      }
+    }
     if (msg?.id !== undefined && TRACKED.has(msg.method)) link.tracked.set(msg.id, msg.method);
     if (link.up.readyState === WebSocket.OPEN) link.up.send(raw);
     else link.backlog.push(raw);
@@ -389,9 +448,17 @@ export class CodexPeer extends BasePeer {
       else p?.resolve(msg.result);
       return; // ours: the TUI never asked for it
     }
+    if (msg.id !== undefined && !msg.method && this.restoringApprovals.delete(msg.id) && !msg.error && this.permissionMode === "ask") this.approvalOverridden = false;
     if (typeof msg.result?.userAgent === "string") this.version = /^[^/\s]+\/(\d+\.\d+\.\d+(?:-[\w.]+)?)/.exec(msg.result.userAgent)?.[1] ?? this.version;
     const tracked = msg.id !== undefined && !msg.method ? link.tracked.get(msg.id) : undefined;
-    if (tracked && link.tracked.delete(msg.id)) this.adopt(link, msg.result?.thread?.id, tracked === "thread/start");
+    if (tracked && link.tracked.delete(msg.id)) {
+      if (typeof msg.result?.thread?.id === "string") {
+        this.nativeApprovalPolicy = msg.result?.approvalPolicy;
+        this.approvalOverridden = false;
+        this.restoringApprovals.clear();
+        this.adopt(link, msg.result.thread.id, tracked === "thread/start");
+      }
+    }
     else if (msg.method) this.onNotification(link, msg.method, msg.params ?? {});
     link.tui.send(raw);
   }
@@ -406,6 +473,7 @@ export class CodexPeer extends BasePeer {
     this.activeTurns.clear();
     this.forgetTurn();
     this.opts.log?.(`[${this.id}] thread ${threadId}`);
+    this.opts.onThread?.(threadId, fresh);
     this.setState("idle");
     this.readUsage();
     clearInterval(this.usageTimer);

package/src/adapters/local-worker.ts

@@ -1,6 +1,7 @@
 import { HubRouteRuntime, type RouteEvent, type AdvisorEvent } from "../models/route/runtime.ts";
 import type { HubRoute } from "../models/route/config.ts";
 import type { RouteLabelEvent, RouteTurnOutcome } from "../models/route/labels.ts";
+import type { StaySwitchPolicy } from "../models/route/stage.ts";
 import type { ToolObservation } from "../models/route/signals.ts";
 import { randomUUID } from "node:crypto";
 import { renderDigest, replyAudience, replyParent, STANDING_INSTRUCTION, USER, type Envelope, type EnvelopeOpts, type PeerId } from "../hub/envelope.ts";
@@ -24,6 +25,8 @@ export interface LocalOptions {
   /** Model id sent straight to OmniRoute when the sidecar is absent, unhealthy or fails a call. */
   fixedModel: string;
   hubRoutes?: () => Record<string, HubRoute>;
+  /** routing.toml `stay_switch` for hub stage routes (#197). */
+  staySwitch?: () => StaySwitchPolicy | undefined;
   onRoute?: (event: RouteEvent) => void;
   onAdvisor?: (event: AdvisorEvent) => void;
   onRouteOutcome?: (event: RouteLabelEvent) => void;
@@ -107,6 +110,7 @@ export class LocalPeer extends BasePeer {
         return result;
       },
       onCampus: () => this.opts.omni.onCampus(),
+      staySwitch: () => this.opts.staySwitch?.(),
       onRoute: event => this.opts.onRoute?.(event),
       onAdvisor: event => this.opts.onAdvisor?.(event),
       onRouteOutcome: event => this.opts.onRouteOutcome?.(event),
@@ -260,7 +264,8 @@ export class LocalPeer extends BasePeer {
           const result: ChatMessage = { role: "tool", tool_call_id: call.id, content: output, is_error: toolResultFailed(name, output) };
           msgs.push(result); results.push(result);
           this.observeTool(call, output, policy, `${this.sessionId}.${turn}.${step}`);
-          capture?.observe({ tool: call.function.name, args: call.function.arguments, output, id: call.id, paths: touchedPaths(call.function.name, safeParse(call.function.arguments)) });
+          // Hub tool arguments/results are not capture evidence; Tasks owns its screened memory writes.
+          if (name !== "hub_send" && !TASK_TOOL_NAMES.has(name) && !CONDUCTOR_TOOL_NAMES.has(name)) capture?.observe({ tool: call.function.name, args: call.function.arguments, output, id: call.id, paths: touchedPaths(call.function.name, safeParse(call.function.arguments)) });
         }
       } finally { if (results.length) this.routes.observeResults(res.routeDecision, res.message, results); }
     }
@@ -272,13 +277,13 @@ export class LocalPeer extends BasePeer {
     return {
       cwd: this.opts.cwd,
       deny: this.opts.tools.deny,
-      permit: async (title) => {
+      permit: async (title, tool, _signal, path) => {
         if (turnSignal.aborted) return false;
         return new Promise<boolean>((resolve, reject) => {
           const finish = (allowed: boolean) => { turnSignal.removeEventListener("abort", onAbort); resolve(allowed); };
           const onAbort = () => finish(false);
           turnSignal.addEventListener("abort", onAbort, { once: true });
-          this.opts.tools.permit(title).then(finish, (error) => { turnSignal.removeEventListener("abort", onAbort); reject(error); });
+          this.opts.tools.permit(title, tool, turnSignal, path).then(finish, (error) => { turnSignal.removeEventListener("abort", onAbort); reject(error); });
         });
       },
       sandboxProfile: this.sandboxProfile,

package/src/adapters/pi.ts

@@ -15,6 +15,16 @@ import type { ExecutionBudgetDecision, ExecutionUnit } from "../hub/execution-bu
 export interface PiModelDescriptor { id: string; name?: string; contextWindow?: number; maxTokens?: number; reasoning?: boolean; }
 export interface PiToolSchema { name: string; description?: string; parameters: Record<string, unknown>; }
 export interface PiRelay { url: string; token: string; models: PiModelDescriptor[]; }
+export interface PiExit {
+  cause: "process_exit" | "spawn_error" | "session_shutdown" | "owner_lost" | "owner_stopped";
+  code: number | null;
+  signal: NodeJS.Signals | null;
+  expected: boolean;
+  started: boolean;
+  turnActive: boolean;
+  toolActive: boolean;
+  lastToolName?: string;
+}
 export interface PiOptions {
   cwd: string; stateDir: string; cmd?: string[]; mode: "headless" | "tui"; backend: "auto" | "dgx" | "mlx"; sessionFile?: string; sessionId?: string;
   model?: string;
@@ -28,6 +38,8 @@ export interface PiOptions {
   /** `ceiling` is the validated, session/turn-bound tool-step ceiling signal (#179), present only when the trusted extension emitted one for this turn. */
   onTurnFailure?: (envs: Envelope[], reason: string, ceiling?: PiToolStepCeiling) => Promise<void>;
   watchdogMs?: number; log?: (line: string) => void;
+  /** Once per owner, after it is offline; a TUI owner may have unknown OS exit status. */
+  onExit?: (exit: PiExit) => void;
   /** How long stop() waits for a graceful TUI owner exit before verified teardown. Tests shrink this. */
   stopGraceMs?: number;
 }
@@ -74,6 +86,10 @@ export class PiPeer extends BasePeer {
   private ownerClaimed = false;
   private ownerToken = "";
   private stopping = true;
+  private exitReported = false;
+  private started = false;
+  private lastToolName?: string;
+  private shutdownExit?: PiExit;
   private ownerPid?: number;
   private ownerSignature?: string;
   private ownerMonitor?: ReturnType<typeof setInterval>;
@@ -179,6 +195,8 @@ export class PiPeer extends BasePeer {
       if (reservation.deadlineAt !== undefined) idleBashTimer = setTimeout(() => idleBashAbort.abort(), Math.max(0, reservation.deadlineAt - Date.now()));
     }
     this.noteActivity();
+    const name = String(body.name);
+    this.lastToolName = this.opts.tools.some((tool) => tool.name === name) && /^[a-zA-Z0-9_.-]{1,128}$/.test(name) ? name : "unknown";
     this.activeTools++;
     if (this.state === "idle") this.setState("busy");
     if (this.state === "busy") this.touch();
@@ -230,6 +248,7 @@ export class PiPeer extends BasePeer {
 
   private async startImpl(): Promise<void> {
     this.stopping = false;
+    this.exitReported = false; this.started = false; this.lastToolName = undefined; this.shutdownExit = undefined;
     mkdirSync(this.opts.stateDir, { recursive: true });
     const sessions = join(this.opts.stateDir, "pi-sessions");
     mkdirSync(sessions, { recursive: true, mode: 0o700 });
@@ -275,8 +294,17 @@ export class PiPeer extends BasePeer {
     trackGroup(this.proc);
     this.proc.stdout.on("data", (chunk) => this.onOutput(String(chunk)));
     this.proc.stderr.on("data", (chunk) => this.opts.log?.(`[${this.id}] ${String(chunk).trimEnd()}`));
-    this.proc.on("error", (error) => this.fail(error));
-    this.proc.on("exit", (code) => { if (code !== 0) this.fail(new Error(`pi exited with code ${code}`)); else this.setState("offline"); });
+    const proc = this.proc;
+    proc.on("error", (error) => {
+      const exit = this.exitMetadata("spawn_error");
+      this.fail(error);
+      this.reportExit(exit);
+    });
+    proc.on("exit", (code, signal) => {
+      const exit = { ...(this.shutdownExit ?? this.exitMetadata("process_exit")), code, signal };
+      this.fail(new Error(`pi exited with ${signal ? `signal ${signal}` : `code ${code}`}`));
+      this.reportExit(exit);
+    });
     const state = await this.waitRpc("get_state", 15_000);
     if (state.success !== true || typeof state.data?.sessionId !== "string" || typeof state.data?.sessionFile !== "string" || !state.data.sessionId || !state.data.sessionFile) throw new Error("Pi get_state did not prove session identity");
     if ((this.opts.sessionId && state.data.sessionId !== this.opts.sessionId) || (this.opts.sessionFile && state.data.sessionFile !== this.opts.sessionFile)) {
@@ -285,10 +313,13 @@ export class PiPeer extends BasePeer {
     this.sessionId = state.data.sessionId; this.sessionFile = state.data.sessionFile;
     this.persistedOnce = existsSync(this.sessionFile);
     if ((this.opts.sessionId && this.sessionId !== this.opts.sessionId) || (this.opts.sessionFile && this.sessionFile !== this.opts.sessionFile)) throw new Error("Pi startup session identity mismatch");
+    this.started = true;
     this.setState("idle");
   }
 
   async stop(): Promise<void> {
+    const tuiExit = this.opts.mode === "tui" && this.ownerClaimed && this.ownerPid && this.ownerSignature
+      ? { ...this.exitMetadata("owner_stopped"), expected: true } : undefined;
     for (const id of this.activeDeliveryIds) this.delivery({ id, state: "needs_review", reason: "Pi session stopped before settlement" });
     this.activeDeliveryIds.clear();
     this.stopping = true;
@@ -329,6 +360,7 @@ export class PiPeer extends BasePeer {
     this.pending.clear();
     this.server?.stop(true); this.server = undefined; this.setState("offline");
     if (failed) throw failed;
+    if (tuiExit) this.reportExit(tuiExit); // the verified owner is gone; never invent an exit for an unclaimed launch
   }
 
   /**
@@ -416,9 +448,15 @@ export class PiPeer extends BasePeer {
       this.sessionId = String(event.sessionId ?? ""); this.sessionFile = String(event.sessionFile ?? "");
       this.ownerPid = Number.isInteger(event.pid) ? event.pid : undefined; this.ownerSignature = typeof event.signature === "string" ? event.signature : undefined;
       if ((this.opts.sessionId && this.sessionId !== this.opts.sessionId) || (this.opts.sessionFile && this.sessionFile !== this.opts.sessionFile)) { this.opts.log?.(`[${this.id}] Pi session identity mismatch`); return; }
-      this.startOwnerMonitor(); if (this.opts.mode === "tui") this.setState("idle");
+      this.startOwnerMonitor(); if (this.opts.mode === "tui") { this.started = true; this.setState("idle"); }
+    }
+    if (event.type === "session_shutdown") {
+      const exit = this.exitMetadata("session_shutdown");
+      this.shutdownExit ??= exit; // headless waits for the OS exit, retaining the pre-cleanup turn and stop facts
+      this.stopping = true; this.ownerClaimed = false; this.clearOwnerMonitor(); this.resolveTuiExit?.(); this.resolveTuiExit = undefined;
+      this.fail(new Error("Pi session shut down before settlement"));
+      if (this.opts.mode === "tui") this.reportExit(exit);
     }
-    if (event.type === "session_shutdown") { this.stopping = true; this.ownerClaimed = false; this.clearOwnerMonitor(); this.resolveTuiExit?.(); this.resolveTuiExit = undefined; this.fail(new Error("Pi session shut down before settlement")); }
     if (event.type === "agent_start") {
       const generation = Number.isSafeInteger(event.generation) ? event.generation : this.budgetGeneration + 1;
       if (generation <= this.budgetGeneration) return;
@@ -507,6 +545,20 @@ export class PiPeer extends BasePeer {
     try { return await Promise.race([this.sendRpc({ type: command }), new Promise<never>((_, reject) => { timer = setTimeout(() => reject(new Error(`Pi ${command} timed out`)), timeout); })]); }
     finally { clearTimeout(timer); }
   }
+  private exitMetadata(cause: PiExit["cause"]): PiExit {
+    return { cause, code: null, signal: null, expected: this.stopping, started: this.started,
+      turnActive: this.agentRunning || this.activeEnvs.length > 0, toolActive: this.activeTools > 0,
+      ...(this.lastToolName ? { lastToolName: this.lastToolName } : {}) };
+  }
+
+  private reportExit(exit: PiExit): void {
+    if (this.exitReported) return;
+    this.exitReported = true;
+    const line = `[${this.id}] Pi exit: cause=${exit.cause} code=${exit.code ?? "unknown"} signal=${exit.signal ?? "unknown"} expected=${exit.expected} started=${exit.started} turnActive=${exit.turnActive} toolActive=${exit.toolActive} lastTool=${exit.lastToolName ?? "none"}`;
+    try { this.opts.log?.(line); } catch { /* reporting must not prevent owner settlement */ }
+    try { this.opts.onExit?.(exit); } catch { /* the owner is already offline */ }
+  }
+
   private fail(error: Error): void {
     for (const pending of this.pending.values()) pending.reject(error);
     this.pending.clear();
@@ -525,11 +577,13 @@ export class PiPeer extends BasePeer {
     if (this.opts.mode !== "tui" || !this.ownerPid || !this.ownerSignature) return;
     this.ownerMonitor = setInterval(() => {
       if (!this.ownerPid || ownerStillAlive(this.ownerPid, this.ownerSignature)) return;
+      const exit = this.exitMetadata("owner_lost");
       this.stopping = true; this.ownerClaimed = false; this.clearOwnerMonitor();
       this.resolveTuiExit?.(); this.resolveTuiExit = undefined;
       for (const pending of this.tuiCommands.values()) pending.reject(new Error("Pi owner exited"));
       this.tuiCommands.clear(); this.tuiQueue = [];
       this.fail(new Error("Pi owner process exited without session_shutdown"));
+      this.reportExit(exit);
     }, 500);
     this.ownerMonitor.unref?.();
   }

package/src/cli/bench.ts

@@ -0,0 +1,287 @@
+import { spawn } from "node:child_process";
+import { createHash, randomUUID } from "node:crypto";
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { childEnv, stopOwnedProcess, trackGroup } from "../hub/child-process.ts";
+import { ControlClient } from "../hub/control-client.ts";
+import { readEvents } from "../hub/events.ts";
+import { taskRecords } from "../hub/research.ts";
+import { appendBench, BENCH_SCHEMA, parseSuite, suiteHash, type Attempt, type AttemptError, type AttemptMetrics, type Fingerprint, type Outcome, type SuiteTask } from "../hub/bench.ts";
+import { hubHome, realPath } from "../hub/project.ts";
+import { processSignature } from "../pi/process-signature.ts";
+import { VERSION } from "../version.ts";
+
+/** What the runner needs from the world, so tests can drive it without a terminal. */
+export interface BenchIo {
+  /**
+   * A shell command in the project, in its own process group, bounded: when the bound passes or `stopped()` turns true
+   * the group is stopped, and whatever the command left running in its group is stopped when it exits. Output is never
+   * kept. A command that cannot be started reads as a failure (code -1).
+   */
+  sh(command: string, cwd: string, timeoutMs: number, stopped?: () => boolean): Promise<{ code: number | null; timedOut: boolean; interrupted: boolean }>;
+  connect(): Promise<ControlClient>;
+  now(): number;
+  sleep(ms: number): Promise<void>;
+  log(line: string): void;
+}
+export interface BenchOptions { cwd: string; stateDir: string; suitePath: string; arm: string; repeat: number; only?: string[]; home?: string; pollMs?: number; verifyTimeoutMs?: number; setupTimeoutMs?: number; settleMs?: number; stopped?: () => boolean }
+
+const fail = (message: string): never => { throw new Error(message); };
+
+/** Commands git runs without a shell; the arguments are ours, never suite text. */
+async function git(cwd: string, args: string[]): Promise<{ code: number; out: string }> {
+  const p = Bun.spawn(["git", ...args], { cwd, stdout: "pipe", stderr: "ignore" });
+  const [code, out] = await Promise.all([p.exited, new Response(p.stdout).text()]);
+  return { code, out: out.trim() };
+}
+
+export const defaultIo = (stateDir: string, cwd: string): BenchIo => ({
+  sh: async (command, dir, timeoutMs, stopped) => {
+    const p = spawn("sh", ["-c", command], { cwd: dir, detached: true, stdio: "ignore", env: childEnv() });
+    trackGroup(p);
+    let code: number | null | undefined;
+    const exited = new Promise<void>((resolve) => {
+      p.once("exit", (c) => { code = c; resolve(); });
+      p.once("error", () => { code = -1; resolve(); }); // it never started (a missing directory, no shell)
+    });
+    const end = Date.now() + timeoutMs;
+    let timedOut = false, interrupted = false;
+    while (code === undefined) {
+      if (Date.now() >= end) { timedOut = true; break; }
+      if (stopped?.()) { interrupted = true; break; }
+      await Promise.race([exited, Bun.sleep(Math.min(200, Math.max(1, end - Date.now())))]);
+    }
+    // The whole group, also what a finished command left behind in it: nothing may run on into the next attempt's tree.
+    await stopOwnedProcess(p, { group: true, killMs: 500 }).catch((error: Error) => {
+      // ponytail: only when the stop lists the members the exited leader left (they hold the group id, so it is still
+      // ours) is the group killed outright; any other refusal (not provably ours, an unreadable process table) leaves
+      // it alone and says so. A leader start-time record at spawn would let stopOwnedProcess sweep it itself.
+      if (/still has members \(/.test(error.message)) try { process.kill(-p.pid!, "SIGKILL"); } catch { /* gone */ }
+      else console.error(`  bench: a command's process group could not be stopped and may still run in the tree (${error.message})`);
+    });
+    return { code: timedOut || interrupted ? null : code ?? null, timedOut, interrupted };
+  },
+  connect: () => ControlClient.connect(stateDir, { role: "console", projectRoot: cwd }),
+  now: () => Date.now(),
+  sleep: (ms) => Bun.sleep(ms),
+  log: (line) => console.log(line),
+});
+
+const TREE = ["status", "--porcelain", "--untracked-files=all", "--", ".", ":(exclude,icase).agenthub"];
+
+/**
+ * The preconditions a run checks before it changes anything; each refusal says why. Returns the branch (or commit) to
+ * return to. `pinned` receives each suite ref's commit: a reset checks out that commit, never the name, which an
+ * agent's own commit could move. `named` receives the full ref name of each one given as a plain branch or tag.
+ */
+export async function benchPreflight(cwd: string, enabled: boolean, refs: string[] = [], pinned: Map<string, string> = new Map(), named: Map<string, string> = new Map()): Promise<string> {
+  if (!enabled) throw new Error('benchmarks are off for this project: set "bench": { "enabled": true } in .agenthub/config.json of a project kept for benchmarks (each attempt resets its work tree)');
+  const top = await git(cwd, ["rev-parse", "--show-toplevel"]);
+  if (top.code !== 0) throw new Error("a benchmark project must be a git repository");
+  // A checkout and a clean act on the whole repository: a project in a subdirectory would reset what lies outside it.
+  if (realPath(top.out) !== realPath(cwd)) throw new Error(`a benchmark project must be the root of its git repository (${top.out} is), since a reset acts on the whole repository`);
+  // The hub's own files must not change under it: a checkout would replace tracked ones, or write them from a ref.
+  // Matched without case, since a case-insensitive disk opens `.AGENTHUB/config.json` as `.agenthub/config.json`.
+  const tracked = await git(cwd, ["ls-files", "--", ":(icase).agenthub"]);
+  if (tracked.code !== 0 || tracked.out) throw new Error("this repository tracks files under .agenthub/, which a reset would replace; in a bench project untrack them (git rm -r --cached .agenthub, then ignore it) and commit");
+  for (const ref of refs) {
+    const commit = await git(cwd, ["rev-parse", "--verify", "--quiet", "--end-of-options", `${ref}^{commit}`]);
+    if (commit.code !== 0 || !/^[0-9a-f]{40,64}$/.test(commit.out)) throw new Error(`suite ref ${ref} is not a commit in this repository`);
+    const names = await git(cwd, ["ls-tree", "--name-only", commit.out]);
+    if (names.code !== 0 || names.out.split("\n").some((name) => name.toLowerCase() === ".agenthub")) {
+      throw new Error(`suite ref ${ref} tracks files under .agenthub/, which a reset to it would write over the hub's own; use a commit made after .agenthub was untracked, or rebuild the ref without that directory`);
+    }
+    pinned.set(ref, commit.out);
+    // A plain branch or tag name has a full ref name; a hash, `HEAD~1` or `main^{commit}` has none and is not tracked.
+    // (A name that is both a branch and a tag has none either; it is pinned to what git resolves it to, the tag.)
+    // Tracked only when that name is what was pinned: a branch someone named like a hash is not the hash.
+    const full = await git(cwd, ["rev-parse", "--symbolic-full-name", "--verify", "--quiet", "--end-of-options", ref]);
+    if (full.code === 0 && full.out.startsWith("refs/") && (await git(cwd, ["rev-parse", "--verify", "--quiet", "--end-of-options", `${full.out}^{commit}`])).out === commit.out) named.set(ref, full.out);
+  }
+  const dirty = await git(cwd, TREE);
+  if (dirty.code !== 0 || dirty.out) throw new Error("the work tree has changes; commit or remove them first (a run resets the tree between attempts)");
+  // The bare branch name, never `--short`: a branch that shares its name with a tag would come back as `heads/<name>`.
+  const branch = await git(cwd, ["symbolic-ref", "--quiet", "HEAD"]);
+  return branch.code === 0 && branch.out.startsWith("refs/heads/") ? branch.out.slice("refs/heads/".length) : (await git(cwd, ["rev-parse", "HEAD"])).out;
+}
+
+/**
+ * Reset the tree to `target` (a pinned commit, or the branch to return to) and check that nothing is left; each step
+ * that fails says so. A pinned commit is checked out as `<hash>^{commit}` and HEAD is compared with it afterwards, so a
+ * branch someone named like the hash cannot stand in for it. `-ff` also removes a nested repository an agent created,
+ * and only the root `.agenthub` is kept.
+ */
+export async function reset(cwd: string, target: string, detach: boolean, log: (line: string) => void): Promise<boolean> {
+  // A branch is named before `--`, so a path of the same name cannot be meant; a commit is peeled, so no branch can.
+  if ((await git(cwd, ["checkout", "--quiet", "--force", ...(detach ? ["--detach", `${target}^{commit}`] : [target, "--"])])).code !== 0) { log(`  the reset could not check out ${target}`); return false; }
+  if (detach && (await git(cwd, ["rev-parse", "HEAD"])).out !== target) { log(`  the reset did not land on ${target}: something else in the repository carries that name`); return false; }
+  if (!detach && (await git(cwd, ["symbolic-ref", "--quiet", "HEAD"])).out !== `refs/heads/${target}`) { log(`  the reset did not return to branch ${target}`); return false; }
+  if ((await git(cwd, ["clean", "-ffdxq", "-e", "/.agenthub"])).code !== 0) { log("  the reset could not clean the tree"); return false; }
+  const left = await git(cwd, TREE);
+  if (left.code === 0 && !left.out) return true;
+  const paths = left.out.split("\n").filter(Boolean);
+  log(`  the reset left ${paths.length || "unreadable"} path${paths.length === 1 ? "" : "s"} behind${paths.length ? `: ${paths.slice(0, 5).join(", ")}${paths.length > 5 ? ", ..." : ""}` : ""}`);
+  return false;
+}
+
+/**
+ * The #247 measures of the attempt's task, the one with this id proposed last (an events file may keep an earlier task
+ * with the same id). The turn that approved it ends after the approval and its usage arrives later still, so this waits
+ * while a turn started since the task's proposal has not ended and its peer is still busy (`busy`, the hub's live
+ * view: a turn the hub closed without an end event does not hold the wait), at most `settleMs`, then one poll more.
+ * Unapproved tasks have no record, and a wait that was interrupted, could not read the hub, or ran out with the turn
+ * still open gives none either: partial measures would read as complete.
+ */
+export async function metricsOf(opts: Pick<BenchOptions, "stateDir" | "settleMs" | "pollMs" | "stopped">, io: Pick<BenchIo, "now" | "sleep" | "log">, projectId: string, task: number,
+    busy: () => Promise<Set<string> | undefined> = async () => undefined): Promise<AttemptMetrics | null> {
+  const events = () => readEvents(join(opts.stateDir, "events.jsonl"));
+  const last = (all: ReturnType<typeof events>, event: boolean, value: string) =>
+    all.flatMap((e) => e.type === "task" && e.id === task && (event ? e.event : e.state) === value ? [e.at] : []).at(-1);
+  const until = io.now() + (opts.settleMs ?? 600_000);
+  let said = false;
+  for (;;) {
+    const all = events(), proposed = last(all, true, "proposed"), at = last(all, false, "approved");
+    if (!at || !proposed || opts.stopped?.()) break;
+    const ended = new Set(all.flatMap((e) => e.type === "turn_end" ? [`${e.peer}\0${e.turn}`] : []));
+    const open = new Set(all.flatMap((e) => e.type === "turn_start" && e.at >= proposed && e.at <= at && !ended.has(`${e.peer}\0${e.turn}`) ? [e.peer] : []));
+    // `undefined`: no live view was offered, so every open turn is waited for. A view that cannot be read while a
+    // turn is open leaves the measures unknown: partial ones would read as complete.
+    const working = await busy().then((w) => w, () => null);
+    if (working === null && open.size) { io.log(`  the hub's status could not be read while task #${task}'s approving turn was open; its measures are not recorded`); return null; }
+    const waiting = [...open].filter((peer) => !working || working.has(peer));
+    if (!waiting.length) break;
+    if (io.now() >= until) { io.log(`  ${waiting.join(", ")} still had the turn that approved task #${task} open after the wait; its measures are not recorded`); return null; }
+    if (!said) { said = true; io.log(`  waiting for ${waiting.join(", ")} to end the turn that approved task #${task}, so its cost is counted`); }
+    await io.sleep(opts.pollMs ?? 2000);
+  }
+  if (opts.stopped?.()) return null; // cut short: partial measures would read as complete
+  await io.sleep(opts.pollMs ?? 2000); // the last usage of a turn lands a moment after its end
+  if (opts.stopped?.()) return null;
+  const all = events();
+  const proposed = last(all, true, "proposed");
+  const r = taskRecords(all, projectId, { version: VERSION, source: "live" }).find((x) => x.task === task && x.createdAt === proposed);
+  return r ? { tokens: r.tokens.total, wallMs: r.wallMs, activeMs: r.activeMs, reviewRounds: r.reviewRounds, changesRequested: r.changesRequested,
+    checkFailed: r.checkFailed, firstPass: r.firstPass, turns: r.turns, filesChanged: r.filesChanged, models: r.models } : null;
+}
+
+/**
+ * #251: run a suite against the peers attached to this project's hub, one attempt at a time. Each attempt resets the
+ * tree to the task's pinned commit, runs its setup, proposes the task as the console does, waits until it is approved
+ * or its time is up, then runs its verify command in the tree the agents left. Returns the run id.
+ * ponytail: a task that times out stops the run, since an agent may still be working in the tree the next attempt
+ * would reset; withdrawing an open task (a board state for it) would let the run continue.
+ */
+export async function runBench(opts: BenchOptions, io: BenchIo): Promise<string> {
+  const suiteText = readFileSync(opts.suitePath, "utf8");
+  const suite = parseSuite(suiteText);
+  // --tasks also sets the order, so a person can randomize it per run (docs/bench.md).
+  const tasks = opts.only?.length ? opts.only.map((id) => suite.tasks.find((t) => t.id === id) ?? fail(`--tasks names ${id}, which the suite does not have`)) : suite.tasks;
+  const pinned = new Map<string, string>(), named = new Map<string, string>();
+  const original = await benchPreflight(opts.cwd, true, [...new Set(tasks.map((t) => t.ref))], pinned, named);
+  const onBranch = (await git(opts.cwd, ["symbolic-ref", "--quiet", "HEAD"])).code === 0;
+  const hub = await io.connect();
+  const run = `${new Date(io.now()).toISOString().slice(0, 10).replace(/-/g, "")}-${randomUUID().slice(0, 8)}`;
+  const home = opts.home ?? hubHome();
+  try {
+    const answered = await hub.request({ t: "status" }, 5000);
+    if (!answered.status) throw new Error(`the hub did not answer its status: ${answered.error ?? "no reply"}`);
+    const status = answered.status as { projectId: string; version: string; peers: Record<string, { state: string; requestedModel?: string; permissionMode?: string }> };
+    const routing = join(opts.cwd, ".agenthub", "routing.toml");
+    const fingerprint: Fingerprint = {
+      // ponytail: the hub's status reports a requested model for Pi only and permission modes once #242 lands; the rest
+      // are recorded as absent. Reading each peer's launch settings would fill them in.
+      // The status lists every peer the hub has known; an arm is the ones attached now.
+      peers: Object.entries(status.peers ?? {}).filter(([, p]) => p.state !== "offline")
+        .map(([id, p]) => ({ id, ...(p.requestedModel ? { model: p.requestedModel } : {}), ...(p.permissionMode ? { permissionMode: p.permissionMode } : {}) })).sort((a, b) => a.id.localeCompare(b.id)),
+      routingHash: existsSync(routing) ? createHash("sha256").update(readFileSync(routing)).digest("hex") : null,
+      hubVersion: status.version,
+    };
+    appendBench({ schema: BENCH_SCHEMA, kind: "run", run, suite: suite.name, suiteHash: suiteHash(suiteText), arm: opts.arm, fingerprint, version: VERSION,
+      startedAt: new Date(io.now()).toISOString(), tasks: tasks.length, repeats: opts.repeat, order: tasks.map((t) => t.id), runner: { pid: process.pid, signature: processSignature(process.pid) ?? null } }, home);
+    io.log(`bench run ${run}: ${suite.name} [${opts.arm}], ${tasks.length} task${tasks.length === 1 ? "" : "s"} x ${opts.repeat}`);
+    let stopped: "timeout" | "interrupted" | "error" | undefined;
+    outer: for (let repeat = 1; repeat <= opts.repeat; repeat++) {
+      for (const task of tasks) {
+        if (opts.stopped?.()) { stopped = "interrupted"; break outer; }
+        const { attempt, open } = await runAttempt(task, pinned.get(task.ref)!, repeat, run, opts, io, hub, status.projectId);
+        appendBench(attempt, home);
+        io.log(`  ${task.id} #${repeat}: ${attempt.outcome}${attempt.error ? ` (${attempt.error})` : ""}`);
+        if (attempt.outcome === "timeout") stopped = "timeout";
+        else if (attempt.error === "interrupted") stopped = "interrupted";
+        else if (attempt.error === "hub stopped" || attempt.error === "reset failed") stopped = "error";
+        if (!stopped) continue;
+        if (open) io.log(`  task #${attempt.hubTask} is still open on the board; settle it before the next run`);
+        break outer;
+      }
+    }
+    // Back where it started: on its branch, or at its commit. A branch or tag the suite names that moved or went
+    // away during the run names different work next time (refs given as a hash or relative to another are not tracked).
+    if (!stopped && !(await reset(opts.cwd, original, !onBranch, io.log))) io.log(`  the tree was not returned to ${original}; check it before the next run`);
+    for (const [ref, full] of named) {
+      const commit = pinned.get(ref)!;
+      const now = await git(opts.cwd, ["rev-parse", "--verify", "--quiet", "--end-of-options", `${full}^{commit}`]);
+      // `rev-parse --verify --quiet` exits 1 for a name that no longer resolves; anything else is git not answering.
+      if (now.code === 1) io.log(`  suite ref ${ref} was deleted during the run; this run used ${commit.slice(0, 12)}`);
+      else if (now.code !== 0) io.log(`  suite ref ${ref} could not be checked after the run`);
+      else if (now.out !== commit) io.log(`  suite ref ${ref} moved during the run (${commit.slice(0, 12)} to ${now.out.slice(0, 12)}); this run used ${commit.slice(0, 12)}`);
+    }
+    appendBench({ schema: BENCH_SCHEMA, kind: "end", run, endedAt: new Date(io.now()).toISOString(), ...(stopped ? { stopped } : {}) }, home);
+    return run;
+  } finally { hub.close(); }
+}
+
+/** One attempt, and whether its hub task is still open on the board when it ends. */
+async function runAttempt(task: SuiteTask, commit: string, repeat: number, run: string, opts: BenchOptions, io: BenchIo, hub: ControlClient, projectId: string): Promise<{ attempt: Attempt; open: boolean }> {
+  const started = io.now();
+  // The attempt ends when its outcome is known: the wait for its measures is not part of its time.
+  const done = (outcome: Outcome, extra: { error?: AttemptError; verifyExit?: number | null; hubTask?: number | null; metrics?: AttemptMetrics | null; ended?: number; open?: boolean } = {}) => {
+    const ended = extra.ended ?? io.now();
+    const attempt: Attempt = { schema: BENCH_SCHEMA, kind: "attempt", run, task: task.id, repeat, outcome, ...(extra.error ? { error: extra.error } : {}), verifyExit: extra.verifyExit ?? null,
+      hubTask: extra.hubTask ?? null, startedAt: new Date(started).toISOString(), endedAt: new Date(ended).toISOString(), ms: ended - started, metrics: extra.metrics ?? null };
+    return { attempt, open: extra.open ?? false };
+  };
+  if (!(await reset(opts.cwd, commit, true, io.log))) return done("error", { error: "reset failed" });
+  if (task.setup) {
+    const setup = await io.sh(task.setup, opts.cwd, opts.setupTimeoutMs ?? 300_000, opts.stopped);
+    if (setup.interrupted) return done("error", { error: "interrupted" });
+    if (setup.timedOut) return done("error", { error: "setup timeout" });
+    if (setup.code !== 0) return done("error", { error: "setup failed" });
+  }
+  let id: number;
+  try {
+    const reply = await hub.request({ t: "task", op: "hub_task_propose", args: { title: task.title, ...(task.detail ? { detail: task.detail } : {}), class: task.class ?? "implement", ...(task.owner ? { owner: task.owner } : {}) } }, 30_000);
+    // A request never rejects: a hub that is gone, or silent, answers `ok: false` with one of these.
+    if (!reply.ok && /hub connection (is not open|closed)|hub is stopping|no answer from the hub/.test(String(reply.error))) {
+      io.log(`  ${task.id}: ${reply.error}; if the hub is still up, check its board for a task it may have created`);
+      return done("error", { error: "hub stopped" });
+    }
+    const match = reply.ok ? /^task #(\d+)/.exec(String(reply.text ?? "")) : null;
+    if (!match) { io.log(`  ${task.id}: the hub refused the task: ${reply.error ?? "no task id in its reply"}`); return done("error", { error: "propose refused" }); }
+    id = Number(match[1]);
+  } catch { return done("error", { error: "hub stopped" }); }
+  const deadline = started + task.timeout_s * 1000;
+  for (;;) {
+    let state: string | undefined;
+    try {
+      const list = await hub.request({ t: "task", op: "hub_task_list", args: {} }, 15_000);
+      if (!list.ok) return done("error", { error: "hub stopped", hubTask: id, open: true });
+      state = (JSON.parse(String(list.text)) as { id: number; state: string }[]).find((t) => t.id === id)?.state;
+    } catch { return done("error", { error: "hub stopped", hubTask: id, open: true }); }
+    if (state === "approved") break;
+    if (opts.stopped?.()) return done("error", { error: "interrupted", hubTask: id, open: true });
+    if (io.now() >= deadline) return done("timeout", { hubTask: id, open: true });
+    await io.sleep(opts.pollMs ?? 2000);
+  }
+  const verify = await io.sh(task.verify, opts.cwd, opts.verifyTimeoutMs ?? 600_000, opts.stopped);
+  const ended = io.now();
+  if (verify.interrupted) return done("error", { error: "interrupted", hubTask: id, ended });
+  const busy = async () => {
+    const reply = await hub.request({ t: "status" }, 5000);
+    if (!reply.status) throw new Error("no status");
+    const peers = (reply.status as { peers?: Record<string, { state: string }> }).peers ?? {};
+    return new Set(Object.entries(peers).flatMap(([peer, p]) => p.state === "busy" || p.state === "paused" ? [peer] : [])); // a paused peer's turn still runs
+  };
+  const metrics = await metricsOf(opts, io, projectId, id, busy);
+  if (verify.timedOut) return done("fail", { error: "verify timeout", hubTask: id, metrics, ended });
+  return done(verify.code === 0 ? "pass" : "fail", { verifyExit: verify.code, hubTask: id, metrics, ended });
+}