@phi-code-admin/phi-code 0.94.0 → 0.95.0

package/CHANGELOG.md

@@ -1,5 +1,29 @@
 # Changelog
 
+## [0.95.0] - 2026-07-12
+
+### Fixed — the 6h-drift class of failures (measured on seaborn-2848)
+
+sandbox_run executed commands with spawnSync, which blocks Node's event loop:
+no JS timer (session timeouts, phase timeouts, harness watchdogs) can fire while
+a command runs. Back-to-back long executions drifted a 25-minute cap to 6h18.
+Four layered defences, per the post-mortem:
+
+- **Root fix:** execution.ts gains runCommandAsync/runArgvAsync (spawn-based,
+  event loop stays alive, the timeout kills the whole process TREE via
+  taskkill /T on Windows). Sandbox gains execAsync (all three backends);
+  sandbox_run now awaits execAsync. Regression test proves a timer fires DURING
+  a child run.
+- **Cumulative execution budget per orchestration** (default 20 min, env
+  PHI_SANDBOX_BUDGET_MS): beyond it, sandbox_run refuses with BUDGET_EXHAUSTED
+  and instructs the agent to conclude honestly with the evidence it has.
+- **Per-call cap tightenable for batch harnesses** (env PHI_SANDBOX_MAX_TIMEOUT_S,
+  default 1800s; the SWE-bench harness now sets 180s).
+- **External watchdog in the harness driver** (timeout -k 30 4500): an internal
+  guard can never protect against itself; a hard kill from another process can.
+
++8 tests (async twins incl. event-loop-liveness regression, budget refusal).
+
 ## [0.94.0] - 2026-07-12
 
 ### Fixed — three defects the run telemetry caught live

package/extensions/phi/orchestrator.ts

@@ -338,6 +338,14 @@ export default function orchestratorExtension(pi: ExtensionAPI) {
 	// invocation count (reset at orchestration start, flushed at finish).
 	let runPhaseRecords: PhaseRecord[] = [];
 	let sandboxExecCount = 0;
+	// Cumulative sandbox execution time this orchestration (drift guard #2) and
+	// its budget; per-call cap (#4) is tightenable for batch harnesses via env.
+	let sandboxExecMs = 0;
+	const SANDBOX_BUDGET_MS = Math.max(
+		1_000,
+		Number(process.env.PHI_SANDBOX_BUDGET_MS ?? 20 * 60 * 1000) || 20 * 60 * 1000,
+	);
+	const SANDBOX_MAX_TIMEOUT_S = Math.max(30, Number(process.env.PHI_SANDBOX_MAX_TIMEOUT_S ?? 1800) || 1800);
 
 	/** Run a git command in cwd; returns stdout ("" on any failure). */
 	function gitIn(cwd: string, args: string): string {
@@ -471,11 +479,29 @@ export default function orchestratorExtension(pi: ExtensionAPI) {
 		async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
 			const p = params as { command: string; timeoutSeconds?: number };
 			const cwd = ctx?.cwd || process.cwd();
+			// Guard #2 against runtime drift: a cumulative execution budget per
+			// orchestration. Even with async runs, an agent chaining long commands
+			// can eat a whole session — beyond the budget it must conclude with
+			// what it has. (Measured: back-to-back long runs drifted a 25-min cap
+			// to 6h18 before the async fix.)
+			if (orchestrationActive && sandboxExecMs >= SANDBOX_BUDGET_MS) {
+				return {
+					content: [
+						{
+							type: "text",
+							text: `SANDBOX BUDGET EXHAUSTED — ${Math.round(sandboxExecMs / 60000)} min of cumulative execution used this run (budget ${Math.round(SANDBOX_BUDGET_MS / 60000)} min). No further runs this orchestration: conclude with the evidence you already have, honestly (BLOCKED/PARTIAL if unverified).`,
+						},
+					],
+					details: { verdict: "BUDGET_EXHAUSTED", budgetMs: SANDBOX_BUDGET_MS, usedMs: sandboxExecMs },
+				};
+			}
 			if (orchestrationActive) sandboxExecCount++;
 			const sandbox = getSessionSandbox(cwd);
-			const result = sandbox.exec(p.command, {
-				timeoutMs: Math.max(1, Math.min(1800, p.timeoutSeconds ?? 300)) * 1000,
+			// Guard #4: per-call cap, tightenable for batch harnesses via env.
+			const result = await sandbox.execAsync(p.command, {
+				timeoutMs: Math.max(1, Math.min(SANDBOX_MAX_TIMEOUT_S, p.timeoutSeconds ?? 300)) * 1000,
 			});
+			if (orchestrationActive) sandboxExecMs += result.durationMs;
 			const verdict = !sandbox.available()
 				? "UNAVAILABLE"
 				: result.timedOut
@@ -1738,6 +1764,7 @@ Tag the note with relevant keywords for vector search.
 		if (mode !== "debug") candidateContext = null;
 		runPhaseRecords = [];
 		sandboxExecCount = 0;
+		sandboxExecMs = 0;
 		setOrchestrationActive(true);
 		phasePending = true;
 		originalModel = ctx.model || null;

package/extensions/phi/providers/execution.ts

@@ -8,10 +8,102 @@
  * itself is exercised with trivial real commands.
  */
 
-import { spawnSync } from "node:child_process";
+import { spawn, spawnSync } from "node:child_process";
 
 const DEFAULT_MAX_BUFFER = 32 * 1024 * 1024;
 
+/**
+ * Kill a spawned process AND its children. On Windows, child.kill() only hits
+ * the direct child (a shell) — the actual workload (docker, pytest…) survives;
+ * taskkill /T fells the whole tree.
+ */
+function killTree(pid: number | undefined): void {
+	if (!pid) return;
+	try {
+		if (process.platform === "win32") {
+			spawnSync("taskkill", ["/PID", String(pid), "/T", "/F"], { windowsHide: true });
+		} else {
+			process.kill(-pid, "SIGKILL");
+		}
+	} catch {
+		/* best effort */
+	}
+}
+
+/**
+ * ASYNC command run — the drift fix. spawnSync blocks Node's event loop, so no
+ * JS timer (session timeouts, phase timeouts, a harness Promise.race) can fire
+ * while a command runs; long back-to-back runs measured a 25-minute cap
+ * drifting to 6h18. spawn keeps the loop alive: every watchdog fires on time,
+ * and the timeout here kills the whole process tree itself. Same contract as
+ * runCommand: never throws, everything comes back as data.
+ */
+export function runCommandAsync(command: string, options: RunOptions = {}): Promise<CommandResult> {
+	return spawnAsync(command, undefined, options, command);
+}
+
+/** ASYNC no-shell argv run — twin of runArgv (used for docker invocations). */
+export function runArgvAsync(
+	file: string,
+	args: string[],
+	options: RunOptions & { label?: string } = {},
+): Promise<CommandResult> {
+	return spawnAsync(file, args, options, options.label ?? `${file} ${args.join(" ")}`.trim());
+}
+
+function spawnAsync(
+	fileOrCommand: string,
+	args: string[] | undefined,
+	options: RunOptions,
+	label: string,
+): Promise<CommandResult> {
+	const start = Date.now();
+	const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+	return new Promise((resolve) => {
+		let stdout = "";
+		let stderr = "";
+		let timedOut = false;
+		let settled = false;
+		const child = args
+			? spawn(fileOrCommand, args, {
+					cwd: options.cwd,
+					env: options.env ?? process.env,
+					shell: false,
+					windowsHide: true,
+				})
+			: spawn(fileOrCommand, { cwd: options.cwd, env: options.env ?? process.env, shell: true, windowsHide: true });
+		const timer = setTimeout(() => {
+			timedOut = true;
+			killTree(child.pid);
+		}, timeoutMs);
+		const cap = (s: string, d: unknown) => {
+			const next = s + String(d);
+			return next.length > DEFAULT_MAX_BUFFER ? next.slice(-DEFAULT_MAX_BUFFER) : next;
+		};
+		child.stdout?.on("data", (d) => {
+			stdout = cap(stdout, d);
+		});
+		child.stderr?.on("data", (d) => {
+			stderr = cap(stderr, d);
+		});
+		const finish = (exitCode: number | null, extraErr?: string) => {
+			if (settled) return;
+			settled = true;
+			clearTimeout(timer);
+			resolve({
+				command: label,
+				exitCode: timedOut ? null : exitCode,
+				stdout,
+				stderr: extraErr ? `${extraErr}\n${stderr}` : stderr,
+				durationMs: Date.now() - start,
+				timedOut,
+			});
+		};
+		child.on("error", (err) => finish(null, err.message));
+		child.on("close", (code) => finish(code));
+	});
+}
+
 export interface CommandResult {
 	command: string;
 	exitCode: number | null;

package/extensions/phi/providers/sandbox.ts

@@ -18,7 +18,9 @@ import {
 	passed,
 	type RunOptions,
 	runArgv as realRunArgv,
+	runArgvAsync as realRunArgvAsync,
 	runCommand as realRunCommand,
+	runCommandAsync as realRunCommandAsync,
 } from "./execution.js";
 import {
 	applyConfig,
@@ -45,19 +47,34 @@ export interface Sandbox {
 	readonly reason: string;
 	describe(): string;
 	available(): boolean;
-	/** Run a command in the environment. Never throws (see execution.ts). */
+	/** Run a command in the environment. Never throws (see execution.js). */
 	exec(command: string, options?: RunOptions): CommandResult;
+	/**
+	 * ASYNC twin of exec — non-blocking (the drift fix): the event loop stays
+	 * alive during the run, so session/phase timers fire on time and the run's
+	 * own timeout kills the process tree. Agent-facing paths (sandbox_run) MUST
+	 * use this; sync exec remains for bounded driver-internal steps.
+	 */
+	execAsync(command: string, options?: RunOptions): Promise<CommandResult>;
 	/** Build the image / install deps. Idempotent, best-effort. */
 	prepare(): PrepareResult;
 }
 
 type RunCommandFn = (command: string, options?: RunOptions) => CommandResult;
 type RunArgvFn = (file: string, args: string[], options?: RunOptions & { label?: string }) => CommandResult;
+type RunCommandAsyncFn = (command: string, options?: RunOptions) => Promise<CommandResult>;
+type RunArgvAsyncFn = (
+	file: string,
+	args: string[],
+	options?: RunOptions & { label?: string },
+) => Promise<CommandResult>;
 
 /** Injectable seams — real fs/spawn by default, fakes in tests. */
 export interface SandboxDeps {
 	runCommand?: RunCommandFn;
 	runArgv?: RunArgvFn;
+	runCommandAsync?: RunCommandAsyncFn;
+	runArgvAsync?: RunArgvAsyncFn;
 	listFiles?: (cwd: string) => string[];
 	readConfig?: (cwd: string) => SandboxConfig | undefined;
 	/** Force docker availability in tests; otherwise probed via `docker version`. */
@@ -112,6 +129,8 @@ export function resolveSandbox(opts: ResolveOptions): Sandbox {
 	const deps = opts.deps ?? {};
 	const rc = deps.runCommand ?? realRunCommand;
 	const ra = deps.runArgv ?? realRunArgv;
+	const rcAsync = deps.runCommandAsync ?? realRunCommandAsync;
+	const raAsync = deps.runArgvAsync ?? realRunArgvAsync;
 	const files = (deps.listFiles ?? listProjectFiles)(opts.cwd);
 	const config = (deps.readConfig ?? readSandboxConfig)(opts.cwd);
 	const tc = detectToolchain(files);
@@ -128,8 +147,8 @@ export function resolveSandbox(opts: ResolveOptions): Sandbox {
 	// Effective image can change after prepare() builds a project Dockerfile.
 	const state = { image: recipe.image };
 
-	const dockerExec = (command: string, options?: RunOptions): CommandResult => {
-		const args = buildDockerRunArgs({
+	const dockerArgsFor = (command: string) =>
+		buildDockerRunArgs({
 			image: state.image,
 			command,
 			mountSource: opts.cwd,
@@ -139,12 +158,18 @@ export function resolveSandbox(opts: ResolveOptions): Sandbox {
 			memory: recipe.memory,
 			cpus: recipe.cpus,
 		});
-		return ra("docker", args, {
+	const dockerExec = (command: string, options?: RunOptions): CommandResult =>
+		ra("docker", dockerArgsFor(command), {
+			cwd: opts.cwd,
+			timeoutMs: options?.timeoutMs,
+			label: `docker[${state.image}] ${command}`,
+		});
+	const dockerExecAsync = (command: string, options?: RunOptions): Promise<CommandResult> =>
+		raAsync("docker", dockerArgsFor(command), {
 			cwd: opts.cwd,
 			timeoutMs: options?.timeoutMs,
 			label: `docker[${state.image}] ${command}`,
 		});
-	};
 
 	const base = { backend: decision.backend, recipe, reason: decision.reason };
 
@@ -154,6 +179,7 @@ export function resolveSandbox(opts: ResolveOptions): Sandbox {
 			describe: () => `unavailable — ${decision.reason}`,
 			available: () => false,
 			exec: (command) => unavailableResult(command),
+			execAsync: async (command) => unavailableResult(command),
 			prepare: () => ({ ok: false, backend: "unavailable", detail: decision.reason }),
 		};
 	}
@@ -164,6 +190,7 @@ export function resolveSandbox(opts: ResolveOptions): Sandbox {
 			describe: () => `local host (${opts.cwd})`,
 			available: () => true,
 			exec: (command, options) => rc(command, { cwd: opts.cwd, ...options }),
+			execAsync: (command, options) => rcAsync(command, { cwd: opts.cwd, ...options }),
 			// Deliberately no host-side dependency install — running `npm install`
 			// etc. on the user's host is intrusive; local is best-effort as-is.
 			prepare: () => ({ ok: true, backend: "local", detail: "local host — no preparation performed" }),
@@ -176,6 +203,7 @@ export function resolveSandbox(opts: ResolveOptions): Sandbox {
 		describe: () => `docker (${state.image})`,
 		available: () => true,
 		exec: dockerExec,
+		execAsync: dockerExecAsync,
 		prepare: () => {
 			if (recipe.source === "dockerfile") {
 				const tag = imageTagFor(recipe);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@phi-code-admin/phi-code",
-	"version": "0.94.0",
+	"version": "0.95.0",
 	"description": "Coding agent CLI with persistent memory, sub-agents, intelligent routing, and orchestration",
 	"type": "module",
 	"piConfig": {