@gajae-code/coding-agent 0.12.19 → 0.12.21

package/src/sdk/broker/ensure.ts

@@ -1,4 +1,8 @@
 import { type ChildProcess, spawn } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import type { FileHandle } from "node:fs/promises";
+import * as fs from "node:fs/promises";
+import path from "node:path";
 import { type BrokerDiscovery, brokerProcessIncarnation, readBrokerDiscovery } from "./discovery";
 import { resolveSdkInternalSpawnCommand, type SdkInternalSpawnCommand } from "./runtime";
 export interface EnsureBrokerSettings {
@@ -18,6 +22,57 @@ const FIXTURE_DISCOVERY_TIMEOUT_MS = 30_000;
 // owned-process teardown convention (SIGTERM -> grace -> SIGKILL -> hard cap).
 const REAP_GRACEFUL_MS = 2_000;
 const REAP_SIGKILL_CAP_MS = 2_000;
+
+/**
+ * Tail of the detached broker's stderr folded into a discovery failure.
+ *
+ * The broker used to spawn with `stdio: "ignore"`, so a broker that exited
+ * cleanly told the caller nothing beyond `code=0` (#3963). Its stderr goes to a
+ * file instead of a pipe because the child is detached and outlives this
+ * process: a pipe would break under it the moment the parent exits.
+ */
+export const BROKER_SPAWN_LOG_TAIL_BYTES = 4_096;
+
+export interface BrokerSpawnLog {
+	path: string;
+	handle: FileHandle;
+}
+
+function brokerSpawnLogPath(agentDir: string): string {
+	return path.join(agentDir, "sdk", `broker-spawn.${randomUUID()}.log`);
+}
+
+/** Opens an isolated, bounded-lifetime diagnostic sink for one broker spawn. */
+export async function openBrokerSpawnLog(agentDir: string): Promise<BrokerSpawnLog | undefined> {
+	try {
+		await fs.mkdir(path.join(agentDir, "sdk"), { recursive: true, mode: 0o700 });
+		const spawnLogPath = brokerSpawnLogPath(agentDir);
+		return { path: spawnLogPath, handle: await fs.open(spawnLogPath, "w", 0o600) };
+	} catch {
+		// Diagnostics are never allowed to block a broker spawn.
+		return undefined;
+	}
+}
+
+export async function readBrokerSpawnLogTail(spawnLogPath: string): Promise<string> {
+	try {
+		const file = Bun.file(spawnLogPath);
+		const size = file.size;
+		if (!Number.isFinite(size) || size <= 0) return "";
+		const tail = size > BROKER_SPAWN_LOG_TAIL_BYTES ? file.slice(size - BROKER_SPAWN_LOG_TAIL_BYTES) : file;
+		return (await tail.text()).trim();
+	} catch {
+		return "";
+	}
+}
+
+async function removeBrokerSpawnLog(spawnLogPath: string): Promise<void> {
+	try {
+		await fs.unlink(spawnLogPath);
+	} catch {
+		// Diagnostics are best-effort and must not affect broker ownership.
+	}
+}
 export interface FixtureBrokerLease {
 	/** Backward-compatible fixture cleanup alias for exact child termination. */
 	close(): Promise<void>;
@@ -245,74 +300,89 @@ async function ensureBrokerOnce(settings: EnsureBrokerSettings, initiator: Ensur
 	}
 
 	const command = resolveSdkInternalSpawnCommand("broker-internal");
-	const child = spawn(command.file, [...command.args, "--agent-dir", settings.agentDir], {
-		detached: true,
-		stdio: "ignore",
-		env: brokerSpawnEnvironment(command, settings.env),
-		...(command.kind === "bun-source" ? { cwd: command.cwd } : {}),
-	});
-	child.unref();
-	let spawnError: Error | undefined;
-	child.once("error", error => {
-		spawnError = error;
-	});
-	const owner = registerBrokerOwner(settings.agentDir, child);
-	const discoveryTimeoutMs = initiator === "fixture-lease" ? FIXTURE_DISCOVERY_TIMEOUT_MS : DISCOVERY_TIMEOUT_MS;
-	const deadline = Date.now() + discoveryTimeoutMs;
-	let discoveryError: unknown;
-	while (Date.now() < deadline) {
-		if (spawnError || child.exitCode !== null || child.signalCode !== null) break;
-		try {
-			const discovered = await readBrokerDiscovery(settings.agentDir, settings.heartbeatTtlMs);
-			if (discovered) {
-				if (owner.markReady(discovered)) {
-					return initiator === "fixture-lease"
-						? { kind: "local-started-fixture", discovery: discovered, owner, child }
-						: { kind: "local-started-discovery", discovery: discovered };
+	const spawnLog = await openBrokerSpawnLog(settings.agentDir);
+	try {
+		const child = spawn(command.file, [...command.args, "--agent-dir", settings.agentDir], {
+			detached: true,
+			stdio: ["ignore", "ignore", spawnLog ? spawnLog.handle.fd : "ignore"],
+			env: brokerSpawnEnvironment(command, settings.env),
+			...(command.kind === "bun-source" ? { cwd: command.cwd } : {}),
+		});
+		// The child holds its own duplicate of the descriptor; this one is done.
+		await spawnLog?.handle.close();
+		child.unref();
+		let spawnError: Error | undefined;
+		child.once("error", error => {
+			spawnError = error;
+		});
+		const owner = registerBrokerOwner(settings.agentDir, child);
+		const discoveryTimeoutMs = initiator === "fixture-lease" ? FIXTURE_DISCOVERY_TIMEOUT_MS : DISCOVERY_TIMEOUT_MS;
+		const deadline = Date.now() + discoveryTimeoutMs;
+		let discoveryError: unknown;
+		while (Date.now() < deadline) {
+			if (spawnError || child.exitCode !== null || child.signalCode !== null) break;
+			try {
+				const discovered = await readBrokerDiscovery(settings.agentDir, settings.heartbeatTtlMs);
+				if (discovered) {
+					if (owner.markReady(discovered)) {
+						return initiator === "fixture-lease"
+							? { kind: "local-started-fixture", discovery: discovered, owner, child }
+							: { kind: "local-started-discovery", discovery: discovered };
+					}
+					await owner.stop();
+					return { kind: "external-discovery", discovery: discovered };
 				}
-				await owner.stop();
-				return { kind: "external-discovery", discovery: discovered };
+			} catch (error) {
+				discoveryError = error;
 			}
-		} catch (error) {
-			discoveryError = error;
+			await sleep(50);
 		}
-		await sleep(50);
-	}
-	const exitedBeforeDiscovery = child.exitCode !== null || child.signalCode !== null;
-	if (exitedBeforeDiscovery && child.exitCode === 0) {
-		// A clean exit means another broker won the ownership lock (two ACP
-		// processes racing a cold broker state, e.g. a provider probe and an
-		// agent launch). The winner may publish its discovery right after our
-		// last poll; reuse it instead of failing the caller. Transient discovery
-		// read failures fall through to the common cleanup + failure path below.
-		try {
-			for (let retry = 0; retry < 20; retry++) {
-				const winner = await readBrokerDiscovery(settings.agentDir, settings.heartbeatTtlMs);
-				if (winner) {
-					await owner.stop();
-					return { kind: "external-discovery", discovery: winner };
+		const exitedBeforeDiscovery = child.exitCode !== null || child.signalCode !== null;
+		if (exitedBeforeDiscovery && child.exitCode === 0) {
+			// A clean exit means another broker won the ownership lock (two ACP
+			// processes racing a cold broker state, e.g. a provider probe and an
+			// agent launch). The winner may publish its discovery right after our
+			// last poll; reuse it instead of failing the caller. Transient discovery
+			// read failures fall through to the common cleanup + failure path below.
+			try {
+				for (let retry = 0; retry < 20; retry++) {
+					const winner = await readBrokerDiscovery(settings.agentDir, settings.heartbeatTtlMs);
+					if (winner) {
+						await owner.stop();
+						return { kind: "external-discovery", discovery: winner };
+					}
+					await sleep(50);
 				}
-				await sleep(50);
+			} catch {
+				// fall through to cleanup + failure
 			}
-		} catch {
-			// fall through to cleanup + failure
 		}
+		const spawnLogTail = exitedBeforeDiscovery && spawnLog ? await readBrokerSpawnLogTail(spawnLog.path) : "";
+		const failure = spawnError
+			? new Error(`Failed to spawn detached SDK broker: ${spawnError.message}`)
+			: exitedBeforeDiscovery
+				? new Error(
+						`Detached SDK broker exited before discovery (code=${child.exitCode}, signal=${child.signalCode}).${
+							spawnLogTail
+								? ` Broker stderr: ${spawnLogTail}`
+								: " The broker wrote no stderr; see the gjc log for its lock-contention diagnostic."
+						}`,
+					)
+				: discoveryError
+					? discoveryError
+					: new Error("Timed out waiting for detached SDK broker discovery.");
+		try {
+			await owner.stop();
+		} catch (cleanupError) {
+			throw new AggregateError(
+				[failure, cleanupError],
+				"SDK broker discovery and spawned broker cleanup both failed.",
+			);
+		}
+		throw failure;
+	} finally {
+		if (spawnLog) await removeBrokerSpawnLog(spawnLog.path);
 	}
-	const failure = spawnError
-		? new Error(`Failed to spawn detached SDK broker: ${spawnError.message}`)
-		: exitedBeforeDiscovery
-			? new Error(
-					`Detached SDK broker exited before discovery (code=${child.exitCode}, signal=${child.signalCode}).`,
-				)
-			: discoveryError
-				? discoveryError
-				: new Error("Timed out waiting for detached SDK broker discovery.");
-	try {
-		await owner.stop();
-	} catch (cleanupError) {
-		throw new AggregateError([failure, cleanupError], "SDK broker discovery and spawned broker cleanup both failed.");
-	}
-	throw failure;
 }
 
 function startEnsure(settings: EnsureBrokerSettings, initiator: EnsureInitiator): EnsureInFlight {

package/src/sdk/broker/index.ts

@@ -5,4 +5,5 @@ export * from "./identity";
 export * from "./lifecycle";
 export * from "./lifecycle-ledger";
 export * from "./session-index";
+export * from "./startup-budget";
 export * from "./transport";

package/src/sdk/broker/lifecycle-ledger.ts

@@ -99,6 +99,11 @@ export interface LifecycleLedgerLimits {
 	maxBytes?: number;
 	maxLineBytes?: number;
 	maxRows?: number;
+	/**
+	 * Cap for the `.corrupt` quarantine sidecar. Reaching it rotates one
+	 * generation aside instead of appending forever.
+	 */
+	maxCorruptBytes?: number;
 }
 function canonicalCleanupSessionId(value: unknown): value is string {
 	return typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(value);
@@ -108,7 +113,11 @@ const DEFAULT_LIFECYCLE_LEDGER_LIMITS: Required<LifecycleLedgerLimits> = {
 	maxBytes: 64 * 1024 * 1024,
 	maxLineBytes: 8 * 1024 * 1024,
 	maxRows: 10_000,
+	maxCorruptBytes: 8 * 1024 * 1024,
 };
+
+/** Appended when a quarantined row is clipped to the sidecar cap. */
+const LIFECYCLE_QUARANTINE_TRUNCATION_MARKER = "\n[gjc: quarantined row truncated at the corrupt-ledger cap]";
 const MAX_LIFECYCLE_LEDGER_JSON_DEPTH = 64;
 const MAX_LIFECYCLE_LEDGER_JSON_FIELDS = 1024;
 
@@ -251,6 +260,7 @@ export class LifecycleLedger {
 			maxBytes: limits.maxBytes ?? DEFAULT_LIFECYCLE_LEDGER_LIMITS.maxBytes,
 			maxLineBytes: limits.maxLineBytes ?? DEFAULT_LIFECYCLE_LEDGER_LIMITS.maxLineBytes,
 			maxRows: limits.maxRows ?? DEFAULT_LIFECYCLE_LEDGER_LIMITS.maxRows,
+			maxCorruptBytes: limits.maxCorruptBytes ?? DEFAULT_LIFECYCLE_LEDGER_LIMITS.maxCorruptBytes,
 		};
 	}
 	async open(): Promise<this> {
@@ -353,12 +363,16 @@ export class LifecycleLedger {
 		return this;
 	}
 	/**
-	 * Reads terminal proof for one request without recovering or changing the ledger.
+	 * Reads the durable terminal record for one request without recovering or changing the ledger.
 	 *
 	 * This intentionally accepts an unrelated unterminated final write: concurrent
 	 * appenders may have started a different row after this request's synced terminal
 	 * row. Complete rows are still decoded and validated strictly, and any incomplete
 	 * tail that identifies this request withholds proof.
+	 *
+	 * `terminal_uncertain` is a durable terminal record too: an operation that concluded
+	 * uncertainly persists that conclusion, and refusing to read it back would report a
+	 * synced row as unpersisted and replace its true reason with a generic one.
 	 */
 	async readTerminal(identity: string, requestHash: string): Promise<LifecycleLedgerEntry | undefined> {
 		const source = await this.#readBoundedSourceReadOnly();
@@ -403,12 +417,20 @@ export class LifecycleLedger {
 			const identityMarker = Buffer.from(`"identity":${JSON.stringify(identity)}`);
 			if (tail.includes(identityMarker)) return undefined;
 		}
-		return latest && terminal(latest.state) ? latest : undefined;
+		return latest && final(latest.state) ? latest : undefined;
 	}
 
+	/**
+	 * A recovery pass appends `terminal_uncertain` for every row it finds mid-flight, so a
+	 * long-running operation can have that marker interleaved before its own terminal row
+	 * lands. The marker records unresolved work, not a proven outcome: the owner that ran
+	 * the effects stays authoritative and may still resolve it. A proven terminal row is
+	 * immutable and admits no successor.
+	 */
 	#isValidHistoryContinuation(previous: LifecycleLedgerEntry | undefined, next: LifecycleLedgerEntry): boolean {
 		if (!previous) return next.state === "accepted";
-		if (previous.requestHash !== next.requestHash || final(previous.state)) return false;
+		if (previous.requestHash !== next.requestHash || terminal(previous.state)) return false;
+		if (previous.state === "terminal_uncertain") return final(next.state);
 		if (previous.state === "accepted") return true;
 		return next.state === "effect_started" || next.state === "awaiting_ready" || final(next.state);
 	}
@@ -533,11 +555,43 @@ export class LifecycleLedger {
 			await h.close();
 		}
 	}
+	/**
+	 * Rotate the quarantine sidecar aside once it would exceed its cap.
+	 *
+	 * Quarantine is diagnostic, not durable state: an unreadable ledger
+	 * re-quarantines its whole contents on every open, so a plain append grows
+	 * the sidecar without bound (a 126 MB `.corrupt` next to a 6 MB live ledger,
+	 * #3963). One retained generation keeps the newest evidence while bounding
+	 * the pair at twice the cap.
+	 */
+	async #rotateCorruptWhenFull(incomingBytes: number): Promise<void> {
+		let size: number;
+		try {
+			const stat = await fs.lstat(this.#corruptFile);
+			// A non-regular quarantine path is never rotated; the append guard rejects it.
+			if (!stat.isFile()) return;
+			size = stat.size;
+		} catch (error) {
+			if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
+			throw error;
+		}
+		if (size + incomingBytes <= this.#limits.maxCorruptBytes) return;
+		await fs.rename(this.#corruptFile, `${this.#corruptFile}.1`);
+	}
+
 	async #quarantine(line: string | Uint8Array): Promise<void> {
+		const payload =
+			typeof line === "string" ? Buffer.from(line) : Buffer.from(line.buffer, line.byteOffset, line.byteLength);
+		// A single quarantined row can be as large as the whole ledger, so the
+		// write itself is capped too; otherwise one oversized row would blow past
+		// the rotation bound it just triggered.
+		const truncated = payload.length > this.#limits.maxCorruptBytes;
+		const retained = truncated ? payload.subarray(0, this.#limits.maxCorruptBytes) : payload;
+		await this.#rotateCorruptWhenFull(retained.length + 1);
 		const h = await this.#openAppendRegular(this.#corruptFile);
 		try {
-			await h.writeFile(line);
-			await h.writeFile("\n");
+			await h.writeFile(retained);
+			await h.writeFile(truncated ? `${LIFECYCLE_QUARANTINE_TRUNCATION_MARKER}\n` : "\n");
 			await h.sync();
 		} finally {
 			await h.close();