@gajae-code/coding-agent 0.16.4 → 0.16.6

package/src/runtime-mcp/manager.ts

@@ -382,6 +382,11 @@ export class MCPManager {
 		pending?.delete(error);
 		if (pending?.size === 0) this.#pendingAcquireCleanups.delete(name);
 	}
+	#pruneSettledPendingAcquireCleanups(name: string): void {
+		for (const error of this.#pendingAcquireCleanups.get(name) ?? []) {
+			if (!this.#pool.hasPendingAcquireCleanup(error)) this.#removePendingAcquireCleanup(name, error);
+		}
+	}
 
 	#retainPendingAcquireCleanup(name: string, error: MCPPoolAcquireAbortError): void {
 		if (error.cleanupGeneration === undefined || !error.cleanup || !this.#pool.hasPendingAcquireCleanup(error))
@@ -393,7 +398,11 @@ export class MCPManager {
 		}
 		if (pending.has(error)) return;
 		pending.add(error);
-		void error.cleanup.catch(() => {});
+		void error.cleanup
+			.finally(() => {
+				if (!this.#pool.hasPendingAcquireCleanup(error)) this.#removePendingAcquireCleanup(name, error);
+			})
+			.catch(() => {});
 	}
 
 	#retainConnectionCleanupFailure(name: string, error: unknown): MCPConnectionCleanupFailure | undefined {
@@ -518,6 +527,7 @@ export class MCPManager {
 			onRequest: (method, params) => this.#handleServerRequest(method, params),
 			onCleanupPending: error => this.#retainPendingAcquireCleanup(name, error),
 		});
+		this.#pruneSettledPendingAcquireCleanups(name);
 		if (trackManagerLease) this.#leaseByConnection.set(lease.connection, lease);
 		if (!this.#toolsOnly) lease.updateRoots(this.#getRoots().roots);
 		return lease;
@@ -1040,6 +1050,10 @@ export class MCPManager {
 		const connectionTasks: ConnectionTask[] = [];
 
 		for (const [name, config] of Object.entries(configs)) {
+			// Another manager sharing the pool may have retried and completed an
+			// abandoned-acquire cleanup. Retire this manager's observation before
+			// applying its reconnect fence; pool ownership remains authoritative.
+			this.#pruneSettledPendingAcquireCleanups(name);
 			const pendingAcquire = this.#pendingAcquireCleanups.get(name);
 			const pendingCleanup = this.#pendingConnectionCleanups.get(name);
 			if (pendingCleanup || pendingAcquire) {
@@ -1781,7 +1795,11 @@ export class MCPManager {
 
 		let closeError: unknown;
 		try {
-			await this.#closePendingConnectionCleanup(name);
+			// A transport may ignore its abort signal while the initial connection
+			// is still opening. The pool owns that abandoned flight and closes any
+			// late connection, so disconnect must not wait for the uncooperative
+			// opener. Already-materialized cleanup failures are still retried here.
+			await this.#closePendingConnectionCleanup(name, false);
 		} catch (error) {
 			closeError = error;
 		}

package/src/runtime-mcp/pool.test.ts

@@ -462,7 +462,7 @@ describe("MCPConnectionPool", () => {
 		expect(recoveryTransports).toHaveLength(2);
 	});
 
-	test("manager disconnect joins and retries pool-owned late cleanup without spawning", async () => {
+	test("manager disconnect preserves late cleanup failure until pool retry clears every manager fence", async () => {
 		const firstFlight = Promise.withResolvers<MCPServerConnection>();
 		const firstStarted = Promise.withResolvers<void>();
 		const cleanupTransport = new FakeTransport();
@@ -481,7 +481,7 @@ describe("MCPConnectionPool", () => {
 					firstStarted.resolve();
 					return firstFlight.promise;
 				}
-				return connection(name, cfg, new FakeTransport());
+				return connection(name, cfg, new ToolListTransport());
 			},
 		});
 		const manager = new MCPManager(".", null, { pool, sessionId: "disconnect-cleanup" });
@@ -492,16 +492,19 @@ describe("MCPConnectionPool", () => {
 		});
 		await firstStarted.promise;
 		controller.abort(new Error("abort before deferred cleanup failure"));
-		await expect(operation).rejects.toBeInstanceOf(MCPPoolAcquireAbortError);
+		const abortedAcquire = await operation.catch(error => error);
+		expect(abortedAcquire).toBeInstanceOf(MCPPoolAcquireAbortError);
+		if (!(abortedAcquire instanceof MCPPoolAcquireAbortError)) throw new Error("Expected aborted acquisition");
 		expect(manager.pendingConnectionCleanupCountForTests).toBe(1);
 
-		const disconnect = manager.disconnectServer("server");
+		await manager.disconnectServer("server");
 		const fenced = await manager.connectServers({ server: serverConfig }, {});
 		expect(fenced.errors.has("server")).toBe(true);
 		expect(opens).toBe(1);
 		const primaryFailure = new Error("late connection cleanup failed");
 		firstFlight.reject(new MCPConnectionCleanupFailure(primaryFailure, cleanupTransport));
-		const disconnectError = await disconnect.catch(error => error);
+		await expect(abortedAcquire.cleanup).rejects.toBeInstanceOf(MCPConnectionCleanupFailure);
+		const disconnectError = await manager.disconnectServer("server").catch(error => error);
 		expect(disconnectError).toBeInstanceOf(MCPConnectionCleanupFailure);
 		if (!(disconnectError instanceof MCPConnectionCleanupFailure)) {
 			throw new Error("Expected retained disconnect cleanup failure");
@@ -513,14 +516,20 @@ describe("MCPConnectionPool", () => {
 		expect(manager.pendingConnectionCleanupCountForTests).toBe(1);
 		expect(pool.pendingAcquireCleanupCountForTests).toBe(1);
 
-		await manager.disconnectServer("server");
-		expect(cleanupAttempts).toBe(2);
-		expect(manager.pendingConnectionCleanupCountForTests).toBe(0);
-		expect(pool.pendingAcquireCleanupCountForTests).toBe(0);
-		await manager.withPreparedLease("server", serverConfig, async lease => {
+		const recoveryManager = new MCPManager(".", null, { pool, sessionId: "disconnect-cleanup" });
+		await recoveryManager.withPreparedLease("server", serverConfig, async lease => {
 			expect(lease.serverName).toBe("server");
 		});
-		expect(opens).toBe(2);
+		expect(cleanupAttempts).toBe(2);
+		expect(pool.pendingAcquireCleanupCountForTests).toBe(0);
+		expect(manager.pendingConnectionCleanupCountForTests).toBe(1);
+
+		const recovered = await manager.connectServers({ server: serverConfig }, {});
+		expect(recovered.errors.size).toBe(0);
+		expect(recovered.connectedServers).toEqual(["server"]);
+		expect(manager.pendingConnectionCleanupCountForTests).toBe(0);
+		expect(opens).toBe(3);
+		await recoveryManager.disconnectAll();
 		await manager.disconnectAll();
 	});
 

package/src/sdk/broker/broker.ts

@@ -41,7 +41,9 @@ import {
 	executeLifecycle,
 	isCanonicalSessionId,
 	prepareSpawnChildHostLaunch,
+	readSessionLifecycleFailure,
 	validateBrokerModelPresetSync,
+	writeEffectMarker,
 } from "./lifecycle";
 import {
 	type LifecycleDurableEffectsReceipt,
@@ -724,6 +726,7 @@ function credentialFreeLifecycleResponse(value: unknown): unknown {
 type LifecycleReplayEndpoint = {
 	endpoint: Record<string, unknown>;
 	endpointGeneration: number;
+	endpointIncarnation: string;
 	pid: number;
 	endpointMtimeMs: number;
 };
@@ -1588,12 +1591,34 @@ export class Broker {
 					).claim;
 					return spawnFailureError(failure);
 				}
+				const { pid, processIncarnation: incarnation } = launched.proof;
+				if (pid === undefined || incarnation === undefined) {
+					const failure: SpawnSubstrateFailure = {
+						substrateKind: launched.proof.substrateKind,
+						code: "substrate_proof_failed",
+						message: "session.spawn substrate lacks lifecycle process authority",
+					};
+					await this.#releaseUnownedSubstrate(provider, launchedProof);
+					launchedProof = undefined;
+					current = (
+						await store.persistTransition(lifecycleIdentity, {
+							claimId: current.claimId,
+							from: "substrate_starting",
+							to: "pre_send_rejected",
+							failure,
+						})
+					).claim;
+					return spawnFailureError(failure);
+				}
+				const marker = { pid, incarnation, effectMarker: prep.effectMarker };
+				await writeEffectMarker(prep.stateRoot, prep.childId, marker);
 				const registration = await this.#spawnPromptLayer.awaitRegistration({
 					childId: prep.childId,
 					cwd: prep.cwd,
 					stateRoot: prep.stateRoot,
 				});
 				if (!registration.ok) {
+					const startupFailure = await readSessionLifecycleFailure(prep.stateRoot, prep.childId, marker);
 					await this.#releaseUnownedSubstrate(provider, launchedProof);
 					launchedProof = undefined;
 					current = (
@@ -1603,7 +1628,11 @@ export class Broker {
 							to: "uncertain",
 						})
 					).claim;
-					return error("terminal_uncertain", "session.spawn child registration is uncertain");
+					return error(
+						"terminal_uncertain",
+						"session.spawn child registration is uncertain" +
+							(startupFailure ? ` (${startupFailure.phase}/${startupFailure.reason})` : ""),
+					);
 				}
 				pinnedRegistration = registration.registration;
 				const now = Date.now();
@@ -2404,26 +2433,35 @@ export class Broker {
 		)
 			return;
 
-		// Snapshot validation and the deterministic instance suffix protect successor locks.
-		const tombstone = path.join(
-			path.dirname(this.#lock),
-			`.broker.lock.stale-${createHash("sha256").update(snapshot.lockIdentity).digest("hex")}`,
-		);
-		try {
-			await fs.rename(this.#lock, tombstone);
-		} catch (e) {
-			const code = (e as NodeJS.ErrnoException).code;
-			if (["ENOENT", "EEXIST", "ENOTEMPTY", "EISDIR", "ENOTDIR"].includes(code ?? "")) return;
-			if (code === "EPERM") {
-				try {
-					await fs.lstat(tombstone);
-					return;
-				} catch (statError) {
-					if ((statError as NodeJS.ErrnoException).code !== "ENOENT") throw statError;
+		// Give every reclaimed generation a fresh quarantine name. Older brokers
+		// used only the lock inode hash; if that tombstone survived a crash, the
+		// deterministic name collided forever and left the dead canonical lock in
+		// place. The random suffix preserves no-replace rename semantics without
+		// letting retained cleanup debris block takeover.
+		for (let attempt = 0; attempt < 8; attempt++) {
+			const tombstone = path.join(
+				path.dirname(this.#lock),
+				`.broker.lock.stale-${createHash("sha256").update(snapshot.lockIdentity).digest("hex")}-${randomBytes(8).toString("hex")}`,
+			);
+			try {
+				await fs.rename(this.#lock, tombstone);
+				return;
+			} catch (e) {
+				const code = (e as NodeJS.ErrnoException).code;
+				if (code === "EEXIST" || code === "ENOTEMPTY") continue;
+				if (["ENOENT", "EISDIR", "ENOTDIR"].includes(code ?? "")) return;
+				if (code === "EPERM") {
+					try {
+						await fs.lstat(tombstone);
+						continue;
+					} catch (statError) {
+						if ((statError as NodeJS.ErrnoException).code !== "ENOENT") throw statError;
+					}
 				}
+				throw e;
 			}
-			throw e;
 		}
+		throw new Error(`Broker lock quarantine namespace is saturated for ${this.#lock}`);
 	}
 	async #releaseOwnedLock(): Promise<void> {
 		try {
@@ -2549,6 +2587,11 @@ export class Broker {
 				startedAt: now,
 				heartbeatAt: now,
 			};
+			// Readiness must not be externally visible until the initial session
+			// checkpoint settles. The bootstrap watchdog owns this pre-publication
+			// interval; publishing first allowed it to kill an endpoint already handed
+			// to callers when a legitimate index-lock wait outlived the fence.
+			await this.#checkpointSessionHeartbeats();
 			this.#publication = await publishBrokerDiscovery(this.settings.agentDir, this.discovery);
 			this.#publicationState = "healthy-owned";
 			this.#publishedAt = process.hrtime.bigint();
@@ -2560,7 +2603,6 @@ export class Broker {
 				void this.#watchPublication();
 				void this.#reapSpawnOrphans();
 			}, cadenceMs);
-			await this.#checkpointSessionHeartbeats();
 			return this.discovery;
 		} catch (error) {
 			await this.#transport?.stop();
@@ -2862,6 +2904,8 @@ export class Broker {
 			endpointMtimeMs <= 0
 		)
 			return error("endpoint_stale", "session endpoint authority is incomplete");
+		const currentIncarnation = endpointIncarnation(record, sessionId);
+		if (!currentIncarnation) return error("endpoint_stale", "session endpoint incarnation is unavailable");
 		const endpoint = await this.#readEndpoint(record, {});
 		if (!endpoint.ok) return endpoint;
 		if (endpoint.result === null || typeof endpoint.result !== "object" || Array.isArray(endpoint.result))
@@ -2869,6 +2913,7 @@ export class Broker {
 		return {
 			endpoint: endpoint.result as Record<string, unknown>,
 			endpointGeneration: record.endpointGeneration,
+			endpointIncarnation: currentIncarnation,
 			pid: record.pid,
 			endpointMtimeMs,
 		};
@@ -3041,16 +3086,21 @@ export class Broker {
 			if (this.#chains.get(target) === chain) this.#chains.delete(target);
 		}
 	}
-	#storeSessionListCursor(cursor: SessionListCursor, replacingToken?: string): string | BrokerResponse {
+	#storeSessionListCursor(cursor: SessionListCursor, replacingToken?: string): string {
 		const now = Date.now();
 		for (const [token, stored] of this.#sessionListCursors) {
 			if (stored.expiresAt <= now) this.#sessionListCursors.delete(token);
 		}
-		const replacing = replacingToken !== undefined && this.#sessionListCursors.has(replacingToken);
-		if (!replacing && this.#sessionListCursors.size >= SESSION_LIST_MAX_CURSORS)
-			return error("invalid_input", "session.list cursor capacity is exhausted");
-		const token = randomBytes(24).toString("base64url");
 		if (replacingToken !== undefined) this.#sessionListCursors.delete(replacingToken);
+		// Pagination cursors are a paging convenience, not durable state (#5370).
+		// Evict the oldest cursor when the budget is full so abandoned or partial
+		// paginations degrade gracefully instead of failing unrelated session ops.
+		while (this.#sessionListCursors.size >= SESSION_LIST_MAX_CURSORS) {
+			const oldest = this.#sessionListCursors.keys().next();
+			if (oldest.done) break;
+			this.#sessionListCursors.delete(oldest.value);
+		}
+		const token = randomBytes(24).toString("base64url");
 		this.#sessionListCursors.set(token, cursor);
 		return token;
 	}
@@ -3121,7 +3171,6 @@ export class Broker {
 						{ ...snapshot, offset, expiresAt: Date.now() + SESSION_LIST_CURSOR_TTL_MS },
 						typeof cursor === "string" ? cursor : undefined,
 					);
-		if (isBrokerResponse(continuationCursor)) return continuationCursor;
 		return {
 			ok: true,
 			result: {
@@ -3326,15 +3375,32 @@ export class Broker {
 						(operation === "session.create" || operation === "session.fork" || operation === "session.resume") &&
 						typeof (replay.result as { sessionId?: unknown } | undefined)?.sessionId === "string"
 					) {
-						const refreshed = await this.#readLifecycleReplayEndpoint(
-							(replay.result as { sessionId: string }).sessionId,
-						);
+						const replayResult = objectRecord(replay.result);
+						const replaySessionId = (replay.result as { sessionId: string }).sessionId;
+						const replayIncarnation =
+							typeof replayResult?.endpointIncarnation === "string" &&
+							/^[a-f0-9]{64}$/.test(replayResult.endpointIncarnation)
+								? replayResult.endpointIncarnation
+								: endpointIncarnation(
+										{
+											endpointGeneration: replayResult?.endpointGeneration as number,
+											endpointMtimeMs: replayResult?.endpointMtimeMs as number,
+											pid: replayResult?.pid as number,
+										},
+										replaySessionId,
+									);
+						if (!replayIncarnation)
+							return error("endpoint_stale", "lifecycle replay lacks original endpoint authority");
+						const refreshed = await this.#readLifecycleReplayEndpoint(replaySessionId);
 						if (isBrokerResponse(refreshed)) return refreshed;
+						if (refreshed.endpointIncarnation !== replayIncarnation)
+							return error("endpoint_stale", "lifecycle replay target was replaced");
 						return {
 							ok: true,
 							result: {
 								...(replay.result as Record<string, unknown>),
 								endpointGeneration: refreshed.endpointGeneration,
+								endpointIncarnation: refreshed.endpointIncarnation,
 								pid: refreshed.pid,
 								endpointMtimeMs: refreshed.endpointMtimeMs,
 								endpoint: refreshed.endpoint,
@@ -3347,6 +3413,12 @@ export class Broker {
 				const response = outcome.response;
 				const storedResponse = credentialFreeLifecycleResponse(response) as BrokerResponse;
 				await this.ledger.transition(identity, lifecycleResponseState(response), {
+					...(operation === "session.delete" &&
+					typeof input.sessionId === "string" &&
+					lifecycleResponseState(response) === "terminal_uncertain" &&
+					!pendingCleanupSessionId(response)
+						? { intendedSessionId: input.sessionId }
+						: {}),
 					response: storedResponse,
 					responseDigest: createHash("sha256").update(canonicalJson(storedResponse)).digest("hex"),
 					...(outcome.durableEffects ? { durableEffects: outcome.durableEffects } : {}),
@@ -3366,6 +3438,12 @@ export class Broker {
 				const storedResponse = credentialFreeLifecycleResponse(response) as BrokerResponse;
 				await this.ledger.transition(identity, lifecycleResponseState(response), {
 					...(pendingCleanupSessionId(response) ? { intendedSessionId: pendingCleanupSessionId(response) } : {}),
+					...(operation === "session.delete" &&
+					typeof input.sessionId === "string" &&
+					lifecycleResponseState(response) === "terminal_uncertain" &&
+					!pendingCleanupSessionId(response)
+						? { intendedSessionId: input.sessionId }
+						: {}),
 					response: storedResponse,
 					responseDigest: createHash("sha256").update(canonicalJson(storedResponse)).digest("hex"),
 					...(outcome.durableEffects ? { durableEffects: outcome.durableEffects } : {}),
@@ -3377,8 +3455,21 @@ export class Broker {
 			const outcome = await executeLifecycle(this, operation, input, identity);
 			const response = outcome.response;
 			const storedResponse = credentialFreeLifecycleResponse(response) as BrokerResponse;
+			// Record the refusal's own target session so the fence it may leave
+			// is scoped to that session. Without this, a session.delete refusal
+			// for X persists as an unbound terminal_uncertain row and fences
+			// every later delete for unrelated sessions (#5364). A refusal that
+			// truly cannot name its target fences nothing (see
+			// hasUncertainCleanupForSession).
+			const refusalSessionId =
+				operation === "session.delete" && typeof input.sessionId === "string" ? input.sessionId : undefined;
 			await this.ledger.transition(identity, lifecycleResponseState(response), {
 				...(pendingCleanupSessionId(response) ? { intendedSessionId: pendingCleanupSessionId(response) } : {}),
+				...(refusalSessionId !== undefined &&
+				lifecycleResponseState(response) === "terminal_uncertain" &&
+				!pendingCleanupSessionId(response)
+					? { intendedSessionId: refusalSessionId }
+					: {}),
 				resultSessionId:
 					response.ok && typeof (response.result as { sessionId?: unknown } | undefined)?.sessionId === "string"
 						? (response.result as { sessionId: string }).sessionId