@phi-code-admin/phi-code 0.93.2 → 0.95.0

package/CHANGELOG.md

@@ -1,5 +1,52 @@
 # Changelog
 
+## [0.95.0] - 2026-07-12
+
+### Fixed — the 6h-drift class of failures (measured on seaborn-2848)
+
+sandbox_run executed commands with spawnSync, which blocks Node's event loop:
+no JS timer (session timeouts, phase timeouts, harness watchdogs) can fire while
+a command runs. Back-to-back long executions drifted a 25-minute cap to 6h18.
+Four layered defences, per the post-mortem:
+
+- **Root fix:** execution.ts gains runCommandAsync/runArgvAsync (spawn-based,
+  event loop stays alive, the timeout kills the whole process TREE via
+  taskkill /T on Windows). Sandbox gains execAsync (all three backends);
+  sandbox_run now awaits execAsync. Regression test proves a timer fires DURING
+  a child run.
+- **Cumulative execution budget per orchestration** (default 20 min, env
+  PHI_SANDBOX_BUDGET_MS): beyond it, sandbox_run refuses with BUDGET_EXHAUSTED
+  and instructs the agent to conclude honestly with the evidence it has.
+- **Per-call cap tightenable for batch harnesses** (env PHI_SANDBOX_MAX_TIMEOUT_S,
+  default 1800s; the SWE-bench harness now sets 180s).
+- **External watchdog in the harness driver** (timeout -k 30 4500): an internal
+  guard can never protect against itself; a hard kill from another process can.
+
++8 tests (async twins incl. event-loop-liveness regression, budget refusal).
+
+## [0.94.0] - 2026-07-12
+
+### Fixed — three defects the run telemetry caught live
+
+The .phi/runs.jsonl telemetry (0.93.0) paid for itself on its first outing: it
+showed /fix shots being killed mid-work by the flat 10-min phase cap (601s run,
+empty phases, lost REPRO-CMD handoff), oracles going blind after those aborts,
+and escalations burning ~10 min re-confirming a reproduction the driver itself
+had just executed.
+
+- **Per-phase timeout** (OrchestratorPhase.timeoutMs): the /fix single shot now
+  gets 25 min (measured: real shots run 8-18 min). The FIRST phase of a generic
+  run also gains the standard retry-on-fallback before being skipped (it was
+  skip-direct).
+- **Oracle fallback reproduction**: when the shot's REPRO-CMD handoff was lost
+  but the conventional repro_issue.py exists, the oracle runs it instead of
+  finishing UNVERIFIED blind.
+- **Escalation skips REPRODUCE**: the oracle just executed the red reproduction;
+  /fix now goes straight to LOCALIZE -> FIX -> VERIFY with the red run seeded,
+  saving ~10 min under tight budgets.
+
+Full suite green (1472).
+
 ## [0.93.2] - 2026-07-12
 
 ### Fixed

package/extensions/phi/orchestrator.ts

@@ -307,6 +307,11 @@ export default function orchestratorExtension(pi: ExtensionAPI) {
 		// 0..4 for the five base phases (explore..review); undefined for synthetic
 		// phases (the review fix + re-review). Used to checkpoint resume position.
 		baseIndex?: number;
+		// Per-phase timeout override (generic driver only). Defaults to
+		// MAX_PHASE_DURATION_MS. Measured need: a /fix single shot legitimately
+		// runs 8–18 min — the flat 10 min cap killed it mid-work and lost its
+		// REPRO-CMD handoff (telemetry: 601s run, phases empty, UNVERIFIED).
+		timeoutMs?: number;
 	}
 
 	let phaseQueue: OrchestratorPhase[] = [];
@@ -333,6 +338,14 @@ export default function orchestratorExtension(pi: ExtensionAPI) {
 	// invocation count (reset at orchestration start, flushed at finish).
 	let runPhaseRecords: PhaseRecord[] = [];
 	let sandboxExecCount = 0;
+	// Cumulative sandbox execution time this orchestration (drift guard #2) and
+	// its budget; per-call cap (#4) is tightenable for batch harnesses via env.
+	let sandboxExecMs = 0;
+	const SANDBOX_BUDGET_MS = Math.max(
+		1_000,
+		Number(process.env.PHI_SANDBOX_BUDGET_MS ?? 20 * 60 * 1000) || 20 * 60 * 1000,
+	);
+	const SANDBOX_MAX_TIMEOUT_S = Math.max(30, Number(process.env.PHI_SANDBOX_MAX_TIMEOUT_S ?? 1800) || 1800);
 
 	/** Run a git command in cwd; returns stdout ("" on any failure). */
 	function gitIn(cwd: string, args: string): string {
@@ -466,11 +479,29 @@ export default function orchestratorExtension(pi: ExtensionAPI) {
 		async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
 			const p = params as { command: string; timeoutSeconds?: number };
 			const cwd = ctx?.cwd || process.cwd();
+			// Guard #2 against runtime drift: a cumulative execution budget per
+			// orchestration. Even with async runs, an agent chaining long commands
+			// can eat a whole session — beyond the budget it must conclude with
+			// what it has. (Measured: back-to-back long runs drifted a 25-min cap
+			// to 6h18 before the async fix.)
+			if (orchestrationActive && sandboxExecMs >= SANDBOX_BUDGET_MS) {
+				return {
+					content: [
+						{
+							type: "text",
+							text: `SANDBOX BUDGET EXHAUSTED — ${Math.round(sandboxExecMs / 60000)} min of cumulative execution used this run (budget ${Math.round(SANDBOX_BUDGET_MS / 60000)} min). No further runs this orchestration: conclude with the evidence you already have, honestly (BLOCKED/PARTIAL if unverified).`,
+						},
+					],
+					details: { verdict: "BUDGET_EXHAUSTED", budgetMs: SANDBOX_BUDGET_MS, usedMs: sandboxExecMs },
+				};
+			}
 			if (orchestrationActive) sandboxExecCount++;
 			const sandbox = getSessionSandbox(cwd);
-			const result = sandbox.exec(p.command, {
-				timeoutMs: Math.max(1, Math.min(1800, p.timeoutSeconds ?? 300)) * 1000,
+			// Guard #4: per-call cap, tightenable for batch harnesses via env.
+			const result = await sandbox.execAsync(p.command, {
+				timeoutMs: Math.max(1, Math.min(SANDBOX_MAX_TIMEOUT_S, p.timeoutSeconds ?? 300)) * 1000,
 			});
+			if (orchestrationActive) sandboxExecMs += result.durationMs;
 			const verdict = !sandbox.available()
 				? "UNAVAILABLE"
 				: result.timedOut
@@ -1360,8 +1391,18 @@ Tag the note with relevant keywords for vector search.
 		fixContext.oracleRan = true;
 		const cwd = ctx.cwd || process.cwd();
 		const sandbox = getSessionSandbox(cwd);
-		const reproCmd =
+		let reproCmd =
 			fixContext.state.failingTest?.trim() || fixContext.state.reproCommand?.trim() || fixContext.reproFromShot;
+		// Conventional fallback: the shot is instructed to write `repro_issue.py`.
+		// If its handoff was lost (phase timeout/abort) but the file exists, the
+		// oracle can still run it instead of going blind (telemetry-measured gap).
+		if (!reproCmd && existsSync(join(cwd, "repro_issue.py"))) {
+			reproCmd = "python repro_issue.py";
+			ctx.ui.notify(
+				`\n🧷 Oracle fallback: found \`repro_issue.py\` — using \`${reproCmd}\` as the reproduction.`,
+				"info",
+			);
+		}
 		const suiteCmd = sandbox.recipe.test?.trim();
 
 		// A shot that changed NOTHING has nothing to verify — "UNVERIFIED" would be
@@ -1409,13 +1450,16 @@ Tag the note with relevant keywords for vector search.
 			);
 			return true;
 		}
-		// escalate: the pipeline inherits the RED run as its concrete failing state.
+		// escalate: the pipeline inherits the RED run as its concrete failing
+		// state. REPRODUCE is SKIPPED — the oracle just executed the reproduction
+		// and it is red; re-confirming it would burn ~10 min re-running what the
+		// driver already ran (measured: escalations were dying on outer budgets).
 		ctx.ui.notify(
-			`\n🔺 **/fix escalating to the full /debug pipeline** — ${decision.diagnostic}. The single shot was not enough; the pipeline inherits the red run as its reproduction.`,
+			`\n🔺 **/fix escalating** — ${decision.diagnostic}. The reproduction is already established (the oracle just ran it); going straight to LOCALIZE → FIX → VERIFY.`,
 			"warning",
 		);
-		phaseQueue.push(...buildDebugPhases(decision.failing));
-		return false; // fall through: sendNextGenericPhase will dispatch REPRODUCE
+		phaseQueue.push(...buildDebugPhases(decision.failing).slice(1));
+		return false; // fall through: sendNextGenericPhase dispatches LOCALIZE
 	}
 
 	// Multi-candidate arbitration — the oracle disposes. Apply each captured
@@ -1522,6 +1566,7 @@ Tag the note with relevant keywords for vector search.
 			if (warning) ctx.ui.notify(`\n⚠️ ${warning}`, "warning");
 			setTimeout(() => pi.sendUserMessage(phase.instruction, { deliverAs: "followUp" }), 500);
 			if (phaseTimeoutId) clearTimeout(phaseTimeoutId);
+			const phaseBudget = phase.timeoutMs ?? MAX_PHASE_DURATION_MS;
 			phaseTimeoutId = setTimeout(() => {
 				if (orchestrationActive && phasePending) {
 					phasePending = false;
@@ -1536,7 +1581,7 @@ Tag the note with relevant keywords for vector search.
 						phase.useFallback = true;
 						phaseQueue.unshift(phase);
 						ctx.ui.notify(
-							`\n⏰ **Phase timed out** (${MAX_PHASE_DURATION_MS / 60000} min) — retrying once on the fallback model.`,
+							`\n⏰ **Phase timed out** (${Math.round(phaseBudget / 60000)} min) — retrying once on the fallback model.`,
 							"warning",
 						);
 					} else {
@@ -1545,7 +1590,7 @@ Tag the note with relevant keywords for vector search.
 					}
 					sendNextGenericPhase(ctx);
 				}
-			}, MAX_PHASE_DURATION_MS);
+			}, phaseBudget);
 		});
 	}
 
@@ -1719,6 +1764,7 @@ Tag the note with relevant keywords for vector search.
 		if (mode !== "debug") candidateContext = null;
 		runPhaseRecords = [];
 		sandboxExecCount = 0;
+		sandboxExecMs = 0;
 		setOrchestrationActive(true);
 		phasePending = true;
 		originalModel = ctx.model || null;
@@ -1743,6 +1789,7 @@ Tag the note with relevant keywords for vector search.
 			ctx.ui.notify(`\n${first.label} → \`${modelId}\``, "info");
 			if (warning) ctx.ui.notify(`\n⚠️ ${warning}`, "warning");
 			if (phaseTimeoutId) clearTimeout(phaseTimeoutId);
+			const firstBudget = first.timeoutMs ?? MAX_PHASE_DURATION_MS;
 			phaseTimeoutId = setTimeout(() => {
 				if (orchestrationActive && phasePending) {
 					phasePending = false;
@@ -1752,11 +1799,24 @@ Tag the note with relevant keywords for vector search.
 					} catch {
 						/* best effort */
 					}
-					skippedPhases++;
-					ctx.ui.notify(`\n⏰ **First phase timed out** — skipping.`, "warning");
+					// Same policy as every other phase: one retry on the fallback
+					// model before skipping (the old skip-direct lost the whole run
+					// when the very first phase was slow).
+					if (!first.retried) {
+						first.retried = true;
+						first.useFallback = true;
+						phaseQueue.unshift(first);
+						ctx.ui.notify(
+							`\n⏰ **First phase timed out** (${Math.round(firstBudget / 60000)} min) — retrying once on the fallback model.`,
+							"warning",
+						);
+					} else {
+						skippedPhases++;
+						ctx.ui.notify(`\n⏰ **First phase timed out again** — skipping.`, "warning");
+					}
 					sendNextGenericPhase(ctx);
 				}
-			}, MAX_PHASE_DURATION_MS);
+			}, firstBudget);
 			setTimeout(() => pi.sendUserMessage(first.instruction, { deliverAs: "followUp" }), 200);
 		});
 	}
@@ -2319,6 +2379,9 @@ With no runnable check at all, the result is honestly labelled UNVERIFIED.`,
 			await ensurePlansDir();
 			fixContext = { state, oracleRan: false };
 			const shot = genericPhase("shot", "🎯 Phase 1 — SINGLE SHOT", "code", "code", singleShotInstruction(state));
+			// A real single shot legitimately runs 8–18 min (measured on the
+			// baselines); the flat 10 min phase cap was killing it mid-work.
+			shot.timeoutMs = 25 * 60 * 1000;
 			startGenericOrchestration(
 				"fix",
 				[shot],

package/extensions/phi/providers/execution.ts

@@ -8,10 +8,102 @@
  * itself is exercised with trivial real commands.
  */
 
-import { spawnSync } from "node:child_process";
+import { spawn, spawnSync } from "node:child_process";
 
 const DEFAULT_MAX_BUFFER = 32 * 1024 * 1024;
 
+/**
+ * Kill a spawned process AND its children. On Windows, child.kill() only hits
+ * the direct child (a shell) — the actual workload (docker, pytest…) survives;
+ * taskkill /T fells the whole tree.
+ */
+function killTree(pid: number | undefined): void {
+	if (!pid) return;
+	try {
+		if (process.platform === "win32") {
+			spawnSync("taskkill", ["/PID", String(pid), "/T", "/F"], { windowsHide: true });
+		} else {
+			process.kill(-pid, "SIGKILL");
+		}
+	} catch {
+		/* best effort */
+	}
+}
+
+/**
+ * ASYNC command run — the drift fix. spawnSync blocks Node's event loop, so no
+ * JS timer (session timeouts, phase timeouts, a harness Promise.race) can fire
+ * while a command runs; long back-to-back runs measured a 25-minute cap
+ * drifting to 6h18. spawn keeps the loop alive: every watchdog fires on time,
+ * and the timeout here kills the whole process tree itself. Same contract as
+ * runCommand: never throws, everything comes back as data.
+ */
+export function runCommandAsync(command: string, options: RunOptions = {}): Promise<CommandResult> {
+	return spawnAsync(command, undefined, options, command);
+}
+
+/** ASYNC no-shell argv run — twin of runArgv (used for docker invocations). */
+export function runArgvAsync(
+	file: string,
+	args: string[],
+	options: RunOptions & { label?: string } = {},
+): Promise<CommandResult> {
+	return spawnAsync(file, args, options, options.label ?? `${file} ${args.join(" ")}`.trim());
+}
+
+function spawnAsync(
+	fileOrCommand: string,
+	args: string[] | undefined,
+	options: RunOptions,
+	label: string,
+): Promise<CommandResult> {
+	const start = Date.now();
+	const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+	return new Promise((resolve) => {
+		let stdout = "";
+		let stderr = "";
+		let timedOut = false;
+		let settled = false;
+		const child = args
+			? spawn(fileOrCommand, args, {
+					cwd: options.cwd,
+					env: options.env ?? process.env,
+					shell: false,
+					windowsHide: true,
+				})
+			: spawn(fileOrCommand, { cwd: options.cwd, env: options.env ?? process.env, shell: true, windowsHide: true });
+		const timer = setTimeout(() => {
+			timedOut = true;
+			killTree(child.pid);
+		}, timeoutMs);
+		const cap = (s: string, d: unknown) => {
+			const next = s + String(d);
+			return next.length > DEFAULT_MAX_BUFFER ? next.slice(-DEFAULT_MAX_BUFFER) : next;
+		};
+		child.stdout?.on("data", (d) => {
+			stdout = cap(stdout, d);
+		});
+		child.stderr?.on("data", (d) => {
+			stderr = cap(stderr, d);
+		});
+		const finish = (exitCode: number | null, extraErr?: string) => {
+			if (settled) return;
+			settled = true;
+			clearTimeout(timer);
+			resolve({
+				command: label,
+				exitCode: timedOut ? null : exitCode,
+				stdout,
+				stderr: extraErr ? `${extraErr}\n${stderr}` : stderr,
+				durationMs: Date.now() - start,
+				timedOut,
+			});
+		};
+		child.on("error", (err) => finish(null, err.message));
+		child.on("close", (code) => finish(code));
+	});
+}
+
 export interface CommandResult {
 	command: string;
 	exitCode: number | null;

package/extensions/phi/providers/sandbox.ts

@@ -18,7 +18,9 @@ import {
 	passed,
 	type RunOptions,
 	runArgv as realRunArgv,
+	runArgvAsync as realRunArgvAsync,
 	runCommand as realRunCommand,
+	runCommandAsync as realRunCommandAsync,
 } from "./execution.js";
 import {
 	applyConfig,
@@ -45,19 +47,34 @@ export interface Sandbox {
 	readonly reason: string;
 	describe(): string;
 	available(): boolean;
-	/** Run a command in the environment. Never throws (see execution.ts). */
+	/** Run a command in the environment. Never throws (see execution.js). */
 	exec(command: string, options?: RunOptions): CommandResult;
+	/**
+	 * ASYNC twin of exec — non-blocking (the drift fix): the event loop stays
+	 * alive during the run, so session/phase timers fire on time and the run's
+	 * own timeout kills the process tree. Agent-facing paths (sandbox_run) MUST
+	 * use this; sync exec remains for bounded driver-internal steps.
+	 */
+	execAsync(command: string, options?: RunOptions): Promise<CommandResult>;
 	/** Build the image / install deps. Idempotent, best-effort. */
 	prepare(): PrepareResult;
 }
 
 type RunCommandFn = (command: string, options?: RunOptions) => CommandResult;
 type RunArgvFn = (file: string, args: string[], options?: RunOptions & { label?: string }) => CommandResult;
+type RunCommandAsyncFn = (command: string, options?: RunOptions) => Promise<CommandResult>;
+type RunArgvAsyncFn = (
+	file: string,
+	args: string[],
+	options?: RunOptions & { label?: string },
+) => Promise<CommandResult>;
 
 /** Injectable seams — real fs/spawn by default, fakes in tests. */
 export interface SandboxDeps {
 	runCommand?: RunCommandFn;
 	runArgv?: RunArgvFn;
+	runCommandAsync?: RunCommandAsyncFn;
+	runArgvAsync?: RunArgvAsyncFn;
 	listFiles?: (cwd: string) => string[];
 	readConfig?: (cwd: string) => SandboxConfig | undefined;
 	/** Force docker availability in tests; otherwise probed via `docker version`. */
@@ -112,6 +129,8 @@ export function resolveSandbox(opts: ResolveOptions): Sandbox {
 	const deps = opts.deps ?? {};
 	const rc = deps.runCommand ?? realRunCommand;
 	const ra = deps.runArgv ?? realRunArgv;
+	const rcAsync = deps.runCommandAsync ?? realRunCommandAsync;
+	const raAsync = deps.runArgvAsync ?? realRunArgvAsync;
 	const files = (deps.listFiles ?? listProjectFiles)(opts.cwd);
 	const config = (deps.readConfig ?? readSandboxConfig)(opts.cwd);
 	const tc = detectToolchain(files);
@@ -128,8 +147,8 @@ export function resolveSandbox(opts: ResolveOptions): Sandbox {
 	// Effective image can change after prepare() builds a project Dockerfile.
 	const state = { image: recipe.image };
 
-	const dockerExec = (command: string, options?: RunOptions): CommandResult => {
-		const args = buildDockerRunArgs({
+	const dockerArgsFor = (command: string) =>
+		buildDockerRunArgs({
 			image: state.image,
 			command,
 			mountSource: opts.cwd,
@@ -139,12 +158,18 @@ export function resolveSandbox(opts: ResolveOptions): Sandbox {
 			memory: recipe.memory,
 			cpus: recipe.cpus,
 		});
-		return ra("docker", args, {
+	const dockerExec = (command: string, options?: RunOptions): CommandResult =>
+		ra("docker", dockerArgsFor(command), {
+			cwd: opts.cwd,
+			timeoutMs: options?.timeoutMs,
+			label: `docker[${state.image}] ${command}`,
+		});
+	const dockerExecAsync = (command: string, options?: RunOptions): Promise<CommandResult> =>
+		raAsync("docker", dockerArgsFor(command), {
 			cwd: opts.cwd,
 			timeoutMs: options?.timeoutMs,
 			label: `docker[${state.image}] ${command}`,
 		});
-	};
 
 	const base = { backend: decision.backend, recipe, reason: decision.reason };
 
@@ -154,6 +179,7 @@ export function resolveSandbox(opts: ResolveOptions): Sandbox {
 			describe: () => `unavailable — ${decision.reason}`,
 			available: () => false,
 			exec: (command) => unavailableResult(command),
+			execAsync: async (command) => unavailableResult(command),
 			prepare: () => ({ ok: false, backend: "unavailable", detail: decision.reason }),
 		};
 	}
@@ -164,6 +190,7 @@ export function resolveSandbox(opts: ResolveOptions): Sandbox {
 			describe: () => `local host (${opts.cwd})`,
 			available: () => true,
 			exec: (command, options) => rc(command, { cwd: opts.cwd, ...options }),
+			execAsync: (command, options) => rcAsync(command, { cwd: opts.cwd, ...options }),
 			// Deliberately no host-side dependency install — running `npm install`
 			// etc. on the user's host is intrusive; local is best-effort as-is.
 			prepare: () => ({ ok: true, backend: "local", detail: "local host — no preparation performed" }),
@@ -176,6 +203,7 @@ export function resolveSandbox(opts: ResolveOptions): Sandbox {
 		describe: () => `docker (${state.image})`,
 		available: () => true,
 		exec: dockerExec,
+		execAsync: dockerExecAsync,
 		prepare: () => {
 			if (recipe.source === "dockerfile") {
 				const tag = imageTagFor(recipe);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@phi-code-admin/phi-code",
-	"version": "0.93.2",
+	"version": "0.95.0",
 	"description": "Coding agent CLI with persistent memory, sub-agents, intelligent routing, and orchestration",
 	"type": "module",
 	"piConfig": {