@namzu/sandbox 7.0.0 → 7.1.0

package/src/backends/aci-standby-pool/index.ts

@@ -55,6 +55,7 @@ import type {
 import { asSandboxId } from '@namzu/sdk'
 
 import type { SandboxBackend, SandboxBackendOptions } from '../../index.js'
+import { HttpWorkerClient } from '../http-worker-client.js'
 import {
 	OperationDeadline,
 	OperationDeadlineExpired,
@@ -62,6 +63,7 @@ import {
 	resolveReadinessOptions,
 	runFailureCleanup,
 } from '../readiness.js'
+import { RemoteCancellationUnknownError } from '../remote-execution-controller.js'
 
 /**
  * Authentication callback. Caller returns a fresh Azure Resource
@@ -301,6 +303,7 @@ async function armCall<T>(
 	getToken: ArmTokenProvider,
 	body?: unknown,
 	signal?: AbortSignal,
+	acceptedStatuses: readonly number[] = [],
 ): Promise<T | undefined> {
 	signal?.throwIfAborted()
 	const token = await getToken()
@@ -319,6 +322,11 @@ async function armCall<T>(
 	const res = await fetch(url, init)
 	signal?.throwIfAborted()
 	if (!res.ok) {
+		if (acceptedStatuses.includes(res.status)) {
+			await res.text()
+			signal?.throwIfAborted()
+			return undefined
+		}
 		const text = await res.text()
 		signal?.throwIfAborted()
 		throw new Error(`ARM ${method} ${url} → ${res.status}: ${text}`)
@@ -517,13 +525,92 @@ async function spawnAciSandbox(
 			readiness.pollIntervalMs,
 		)
 
-		let status: SandboxStatus = 'ready'
+		type Lifecycle = 'active' | 'retiring' | 'destroyed'
+		let activeExecutions = 0
+		let lifecycle: Lifecycle = 'active'
+		let retirementPromise:
+			| Promise<{ readonly accepted: boolean; readonly error?: Error }>
+			| undefined
+		let teardownPromise: Promise<void> | undefined
+		let teardownComplete = false
 		const rootDir = config.layout.outputs.containerPath
+		const workerClient = new HttpWorkerClient(baseUrl)
+		const assertActive = (): void => {
+			if (lifecycle !== 'active') {
+				throw new Error(`Sandbox ${id} is ${lifecycle}; no new worker operation can be admitted`)
+			}
+		}
+		const teardownSandbox = (signal?: AbortSignal): Promise<void> => {
+			lifecycle = 'retiring'
+			if (teardownComplete) return Promise.resolve()
+			if (teardownPromise) return teardownPromise
+			const attempt = armCall(
+				armUrl,
+				'DELETE',
+				config.getArmToken,
+				undefined,
+				signal,
+				[404, 410],
+			).then(() => undefined)
+			const shared = attempt.then(
+				() => {
+					teardownComplete = true
+					lifecycle = 'destroyed'
+				},
+				(error: unknown) => {
+					if (teardownPromise === shared) teardownPromise = undefined
+					throw error
+				},
+			)
+			teardownPromise = shared
+			return shared
+		}
+		const retire = (): Promise<{
+			readonly accepted: boolean
+			readonly error?: Error
+		}> => {
+			lifecycle = 'retiring'
+			if (retirementPromise) return retirementPromise
+			const deadline = new OperationDeadline(5_000, `ACI sandbox ${id} retirement`)
+			retirementPromise = deadline
+				.run(async (signal) => {
+					const joinedExistingAttempt = teardownPromise !== undefined
+					try {
+						await teardownSandbox(signal)
+					} catch (error) {
+						if (!joinedExistingAttempt || signal.aborted) throw error
+						await teardownSandbox(signal)
+					}
+				})
+				.then(() => {
+					return { accepted: true as const }
+				})
+				.catch((error: unknown) => ({
+					accepted: false as const,
+					error: error instanceof Error ? error : new Error(String(error)),
+				}))
+			return retirementPromise
+		}
+		const runExecution = async <T>(operation: () => Promise<T>): Promise<T> => {
+			assertActive()
+			activeExecutions += 1
+			try {
+				return await operation()
+			} catch (error) {
+				if (error instanceof RemoteCancellationUnknownError) {
+					error.retirement = await retire()
+				}
+				throw error
+			} finally {
+				activeExecutions = Math.max(0, activeExecutions - 1)
+			}
+		}
 
 		return {
 			id,
 			get status(): SandboxStatus {
-				return status
+				if (lifecycle !== 'active') return 'destroyed'
+				return activeExecutions > 0 ? 'busy' : 'ready'
 			},
 			rootDir,
 			environment: detectEnvironment(),
@@ -533,15 +620,11 @@ async function spawnAciSandbox(
 				argv?: string[],
 				opts?: SandboxExecOptions,
 			): Promise<SandboxExecResult> {
-				status = 'busy'
-				try {
-					return await execViaWorker(baseUrl, command, argv, opts)
-				} finally {
-					status = 'ready'
-				}
+				return await runExecution(async () => await workerClient.exec(command, argv, opts))
 			},
 
 			async writeFile(path: string, content: string | Buffer): Promise<void> {
+				assertActive()
 				const buf = Buffer.isBuffer(content) ? content : Buffer.from(content, 'utf8')
 				const res = await fetch(`${baseUrl}/write-file`, {
 					method: 'POST',
@@ -558,6 +641,7 @@ async function spawnAciSandbox(
 			},
 
 			async readFile(path: string): Promise<Buffer> {
+				assertActive()
 				const res = await fetch(`${baseUrl}/read-file`, {
 					method: 'POST',
 					headers: { 'content-type': 'application/json' },
@@ -578,11 +662,15 @@ async function spawnAciSandbox(
 			},
 
 			async listFiles(rootPath: string): Promise<readonly SandboxFileEntry[]> {
-				return await listFilesViaWorker(baseUrl, rootPath)
+				return await runExecution(async () => await listFilesViaWorker(workerClient, rootPath))
 			},
 
 			async destroy(options?: SandboxDestroyOptions): Promise<void> {
-				status = 'destroyed'
+				if (retirementPromise) {
+					const observation = await retirementPromise
+					if (observation.accepted) return
+					retirementPromise = undefined
+				}
 				// ARM DELETE — let failures propagate. The Vandal-side
 				// lifecycle wraps this in its own try/catch with logging,
 				// so a silently swallowed error here means orphan ACI
@@ -591,7 +679,7 @@ async function spawnAciSandbox(
 				// the WARM side topped up; that has nothing to do with
 				// cleaning up a CLAIMED instance, which is exclusively the
 				// claimer's responsibility.
-				await armCall(armUrl, 'DELETE', config.getArmToken, undefined, options?.signal)
+				await teardownSandbox(options?.signal)
 			},
 		}
 	} catch (err) {
@@ -659,80 +747,6 @@ async function waitForWorkerReady(
 	throw new Error(`aci-standby-pool: worker /healthz never responded (${timeoutMs}ms)`)
 }
 
-async function execViaWorker(
-	baseUrl: string,
-	command: string,
-	argv: string[] | undefined,
-	opts: SandboxExecOptions | undefined,
-): Promise<SandboxExecResult> {
-	const start = Date.now()
-	const res = await fetch(`${baseUrl}/execute`, {
-		method: 'POST',
-		headers: { 'content-type': 'application/json' },
-		body: JSON.stringify({
-			command,
-			args: argv ?? [],
-			cwd: opts?.cwd,
-			env: opts?.env,
-			timeoutMs: opts?.timeout,
-		}),
-	})
-	if (!res.ok || !res.body) {
-		throw new Error(`execute failed: HTTP ${res.status} ${await res.text()}`)
-	}
-
-	let stdout = ''
-	let stderr = ''
-	let exitCode = -1
-	let timedOut = false
-
-	const decoder = new TextDecoder()
-	const reader = res.body.getReader()
-	let buffered = ''
-	for (;;) {
-		const { value, done } = await reader.read()
-		if (done) break
-		buffered += decoder.decode(value, { stream: true })
-		let newlineIdx = buffered.indexOf('\n')
-		while (newlineIdx !== -1) {
-			const line = buffered.slice(0, newlineIdx).trim()
-			buffered = buffered.slice(newlineIdx + 1)
-			if (line) {
-				try {
-					const event = JSON.parse(line) as
-						| { type: 'stdout_delta'; data: string }
-						| { type: 'stderr_delta'; data: string }
-						| { type: 'result'; exitCode: number; timedOut: boolean }
-						| { type: 'error'; error: string }
-					if (event.type === 'stdout_delta') {
-						stdout += event.data
-						opts?.onOutput?.({ stream: 'stdout', data: event.data })
-					} else if (event.type === 'stderr_delta') {
-						stderr += event.data
-						opts?.onOutput?.({ stream: 'stderr', data: event.data })
-					} else if (event.type === 'result') {
-						exitCode = event.exitCode
-						timedOut = event.timedOut
-					} else if (event.type === 'error') {
-						throw new Error(event.error)
-					}
-				} catch (err) {
-					if (!(err instanceof SyntaxError)) throw err
-				}
-			}
-			newlineIdx = buffered.indexOf('\n')
-		}
-	}
-
-	return {
-		stdout,
-		stderr,
-		exitCode,
-		timedOut,
-		durationMs: Date.now() - start,
-	} as SandboxExecResult
-}
-
 /**
  * Recursively list regular files under `rootPath` by shelling out to
  * the worker's `find` (GNU find on the Debian-based reference image).
@@ -742,11 +756,10 @@ async function execViaWorker(
  * have produced anything in `rootPath` yet.
  */
 async function listFilesViaWorker(
-	baseUrl: string,
+	workerClient: HttpWorkerClient,
 	rootPath: string,
 ): Promise<readonly SandboxFileEntry[]> {
-	const result = await execViaWorker(
-		baseUrl,
+	const result = await workerClient.exec(
 		'find',
 		[rootPath, '-type', 'f', '-printf', '%p\t%s\n'],
 		undefined,

package/src/backends/docker/index.ts

@@ -57,6 +57,7 @@ import {
 	type SandboxBackend,
 	type SandboxBackendOptions,
 } from '../../index.js'
+import { HttpWorkerClient } from '../http-worker-client.js'
 import {
 	OperationDeadline,
 	OperationDeadlineExpired,
@@ -64,6 +65,7 @@ import {
 	resolveReadinessOptions,
 	runFailureCleanup,
 } from '../readiness.js'
+import { RemoteCancellationUnknownError } from '../remote-execution-controller.js'
 
 /**
  * Backend-specific tuning. Most hosts use the defaults; advanced
@@ -599,12 +601,96 @@ async function spawnDockerSandbox(
 		throw err
 	}
 
-	let status: SandboxStatus = 'ready'
+	type Lifecycle = 'active' | 'retiring' | 'destroyed'
+	let activeExecutions = 0
+	let lifecycle: Lifecycle = 'active'
+	let retirementPromise: Promise<{ readonly accepted: boolean; readonly error?: Error }> | undefined
+	let teardownPromise: Promise<void> | undefined
+	let teardownComplete = false
+	const workerClient = new HttpWorkerClient(baseUrl)
+	const assertActive = (): void => {
+		if (lifecycle !== 'active') {
+			throw new Error(`Sandbox ${id} is ${lifecycle}; no new worker operation can be admitted`)
+		}
+	}
+	const teardownSandbox = (signal?: AbortSignal): Promise<void> => {
+		lifecycle = 'retiring'
+		if (teardownComplete) return Promise.resolve()
+		if (teardownPromise) return teardownPromise
+		const attempt = (async () => {
+			let teardownError: unknown
+			try {
+				await runOnce(docker, ['rm', '-f', containerName], signal)
+			} catch (error) {
+				teardownError = error
+			} finally {
+				try {
+					await egressProxy?.close()
+				} catch (error) {
+					teardownError ??= error
+				}
+			}
+			if (teardownError !== undefined) throw teardownError
+		})()
+		const shared = attempt.then(
+			() => {
+				teardownComplete = true
+				lifecycle = 'destroyed'
+			},
+			(error: unknown) => {
+				if (teardownPromise === shared) teardownPromise = undefined
+				throw error
+			},
+		)
+		teardownPromise = shared
+		return shared
+	}
+	const retire = (): Promise<{
+		readonly accepted: boolean
+		readonly error?: Error
+	}> => {
+		lifecycle = 'retiring'
+		if (retirementPromise) return retirementPromise
+		const deadline = new OperationDeadline(5_000, `docker sandbox ${id} retirement`)
+		retirementPromise = deadline
+			.run(async (signal) => {
+				const joinedExistingAttempt = teardownPromise !== undefined
+				try {
+					await teardownSandbox(signal)
+				} catch (error) {
+					if (!joinedExistingAttempt || signal.aborted) throw error
+					await teardownSandbox(signal)
+				}
+			})
+			.then(() => {
+				return { accepted: true as const }
+			})
+			.catch((error: unknown) => ({
+				accepted: false as const,
+				error: error instanceof Error ? error : new Error(String(error)),
+			}))
+		return retirementPromise
+	}
+	const runExecution = async <T>(operation: () => Promise<T>): Promise<T> => {
+		assertActive()
+		activeExecutions += 1
+		try {
+			return await operation()
+		} catch (error) {
+			if (error instanceof RemoteCancellationUnknownError) {
+				error.retirement = await retire()
+			}
+			throw error
+		} finally {
+			activeExecutions = Math.max(0, activeExecutions - 1)
+		}
+	}
 
 	return {
 		id,
 		get status(): SandboxStatus {
-			return status
+			if (lifecycle !== 'active') return 'destroyed'
+			return activeExecutions > 0 ? 'busy' : 'ready'
 		},
 		rootDir,
 		environment: detectEnvironment(),
@@ -614,15 +700,11 @@ async function spawnDockerSandbox(
 			argv?: string[],
 			opts?: SandboxExecOptions,
 		): Promise<SandboxExecResult> {
-			status = 'busy'
-			try {
-				return await execViaWorker(baseUrl, command, argv, opts)
-			} finally {
-				status = 'ready'
-			}
+			return await runExecution(async () => await workerClient.exec(command, argv, opts))
 		},
 
 		async setNetworkPolicy(policy): Promise<void> {
+			assertActive()
 			// Enforceable only through the egress proxy. Without one the
 			// container's network was fixed at creation — `--network none`
 			// or not — and there is nothing to narrow: accepting the policy
@@ -641,6 +723,7 @@ async function spawnDockerSandbox(
 		},
 
 		async writeFile(path: string, content: string | Buffer): Promise<void> {
+			assertActive()
 			const buf = Buffer.isBuffer(content) ? content : Buffer.from(content, 'utf8')
 			let res: Response
 			try {
@@ -672,6 +755,7 @@ async function spawnDockerSandbox(
 		},
 
 		async readFile(path: string): Promise<Buffer> {
+			assertActive()
 			const res = await fetch(`${baseUrl}/read-file`, {
 				method: 'POST',
 				headers: { 'content-type': 'application/json' },
@@ -692,17 +776,16 @@ async function spawnDockerSandbox(
 		},
 
 		async listFiles(rootPath: string): Promise<readonly SandboxFileEntry[]> {
-			return await listFilesViaWorker(baseUrl, rootPath)
+			return await runExecution(async () => await listFilesViaWorker(workerClient, rootPath))
 		},
 
 		async destroy(options?: SandboxDestroyOptions): Promise<void> {
-			status = 'destroyed'
-			await runOnceQuiet(docker, ['rm', '-f', containerName], options?.signal)
-			// The proxy holds real credentials and a live allowlist. Leaving
-			// it listening after the sandbox it was filtering for is gone
-			// means a loopback port that still stamps a token onto anything
-			// that asks — outliving the only thing that justified it.
-			await egressProxy?.close()
+			if (retirementPromise) {
+				const observation = await retirementPromise
+				if (observation.accepted) return
+				retirementPromise = undefined
+			}
+			await teardownSandbox(options?.signal)
 			// Backend never allocates host paths — every bind source
 			// comes from the consumer-supplied layout. Container
 			// teardown is sufficient; the consumer's own lifecycle
@@ -746,119 +829,6 @@ async function readMappedPort(
 	return port
 }
 
-async function execViaWorker(
-	baseUrl: string,
-	command: string,
-	argv: string[] | undefined,
-	opts: SandboxExecOptions | undefined,
-): Promise<SandboxExecResult> {
-	// `opts.signal` is deliberately NOT passed to `fetch`. It would abort the
-	// request and leave the worker running the command — abandoning the wait
-	// while the process lives on is the exact failure
-	// `SandboxExecOptions.signal` exists to prevent, and wiring it here would
-	// make the option look honoured while delivering that failure. Honouring
-	// it needs a cancel endpoint on the worker.
-	const start = Date.now()
-	let res: Response
-	try {
-		res = await fetch(`${baseUrl}/execute`, {
-			method: 'POST',
-			headers: { 'content-type': 'application/json' },
-			body: JSON.stringify({
-				command,
-				args: argv ?? [],
-				cwd: opts?.cwd,
-				env: opts?.env,
-				timeoutMs: opts?.timeout,
-			}),
-		})
-	} catch (err) {
-		// Surface the underlying transport error (DNS, ECONNREFUSED,
-		// socket-hangup, …) instead of the generic "fetch failed" the
-		// undici client throws. Without `cause`, ops cannot tell whether
-		// the worker died, the bridge dropped, or something else.
-		const cause = err instanceof Error ? err.cause : undefined
-		const causeMsg =
-			cause instanceof Error
-				? `${cause.message}${(cause as Error & { code?: string }).code ? ` (${(cause as Error & { code?: string }).code})` : ''}`
-				: cause
-					? String(cause)
-					: 'unknown'
-		throw withHint(
-			new Error(
-				`namzu-sandbox /execute fetch failed (baseUrl=${baseUrl}): ${err instanceof Error ? err.message : String(err)} — cause: ${causeMsg}`,
-				{ cause: err },
-			),
-			'The container was reachable when it started, so it has most likely exited or been killed since — an out-of-memory kill under `memoryLimitMb` is the common cause. Check the container logs and its exit code.',
-		)
-	}
-	if (!res.ok || !res.body) {
-		throw new Error(`execute failed: HTTP ${res.status} ${await res.text()}`)
-	}
-
-	let stdout = ''
-	let stderr = ''
-	let exitCode = -1
-	let timedOut = false
-	let signal: string | undefined
-
-	const decoder = new TextDecoder()
-	const reader = res.body.getReader()
-	let buffered = ''
-	for (;;) {
-		const { value, done } = await reader.read()
-		if (done) break
-		buffered += decoder.decode(value, { stream: true })
-		let newlineIdx = buffered.indexOf('\n')
-		while (newlineIdx !== -1) {
-			const line = buffered.slice(0, newlineIdx).trim()
-			buffered = buffered.slice(newlineIdx + 1)
-			if (line) {
-				try {
-					const event = JSON.parse(line) as
-						| { type: 'stdout_delta'; data: string }
-						| { type: 'stderr_delta'; data: string }
-						| {
-								type: 'result'
-								exitCode: number
-								timedOut: boolean
-								durationMs: number
-						  }
-						| { type: 'error'; error: string }
-					if (event.type === 'stdout_delta') {
-						stdout += event.data
-						opts?.onOutput?.({ stream: 'stdout', data: event.data })
-					} else if (event.type === 'stderr_delta') {
-						stderr += event.data
-						opts?.onOutput?.({ stream: 'stderr', data: event.data })
-					} else if (event.type === 'result') {
-						exitCode = event.exitCode
-						timedOut = event.timedOut
-					} else if (event.type === 'error') {
-						throw new Error(event.error)
-					}
-				} catch (err) {
-					if (err instanceof SyntaxError) {
-						// Ignore malformed lines from the worker.
-					} else {
-						throw err
-					}
-				}
-			}
-			newlineIdx = buffered.indexOf('\n')
-		}
-	}
-
-	return {
-		exitCode,
-		stdout,
-		stderr,
-		...(signal ? { signal } : {}),
-		timedOut,
-		durationMs: Date.now() - start,
-	}
-}
-
 /**
  * Recursively list regular files under `rootPath` by shelling out to
  * the worker's `find` (GNU find on the Debian-based reference image).
@@ -868,11 +838,10 @@ async function execViaWorker(
  * have produced anything in `rootPath` yet.
  */
 async function listFilesViaWorker(
-	baseUrl: string,
+	workerClient: HttpWorkerClient,
 	rootPath: string,
 ): Promise<readonly SandboxFileEntry[]> {
-	const result = await execViaWorker(
-		baseUrl,
+	const result = await workerClient.exec(
 		'find',
 		[rootPath, '-type', 'f', '-printf', '%p\t%s\n'],
 		undefined,