@namzu/sandbox 6.1.0 → 7.1.0

package/src/backends/docker/index.ts

@@ -34,6 +34,7 @@ import {
 	SANDBOX_DEFAULT_TRANSCRIPTS_PATH,
 	SANDBOX_DEFAULT_UPLOADS_PATH,
 	type Sandbox,
+	type SandboxDestroyOptions,
 	type SandboxEnvironment,
 	type SandboxExecOptions,
 	type SandboxExecResult,
@@ -56,6 +57,15 @@ import {
 	type SandboxBackend,
 	type SandboxBackendOptions,
 } from '../../index.js'
+import { HttpWorkerClient } from '../http-worker-client.js'
+import {
+	OperationDeadline,
+	OperationDeadlineExpired,
+	probeHttpHealth,
+	resolveReadinessOptions,
+	runFailureCleanup,
+} from '../readiness.js'
+import { RemoteCancellationUnknownError } from '../remote-execution-controller.js'
 
 /**
  * Backend-specific tuning. Most hosts use the defaults; advanced
@@ -173,11 +183,20 @@ const WORKER_PORT_INSIDE_CONTAINER = 2024
  * `create()` call.
  */
 export function buildDockerBackend(config: DockerBackendInternalConfig): SandboxBackend {
+	const readiness = resolveReadinessOptions(
+		'docker',
+		config.readyTimeoutMs,
+		config.readyPollIntervalMs,
+		{
+			timeoutMs: DEFAULT_READY_TIMEOUT_MS,
+			pollIntervalMs: DEFAULT_READY_POLL_MS,
+		},
+	)
 	return {
 		tier: 'container',
 		name: 'docker',
 		async create(options: SandboxBackendOptions) {
-			return await spawnDockerSandbox(config, options)
+			return await spawnDockerSandbox(config, options, readiness)
 		},
 	}
 }
@@ -375,7 +394,9 @@ const PROXY_HOST_ALIAS = 'namzu-egress'
 async function spawnDockerSandbox(
 	config: DockerBackendInternalConfig,
 	options: SandboxBackendOptions,
+	readiness: { readonly timeoutMs: number; readonly pollIntervalMs: number },
 ): Promise<Sandbox> {
+	options.signal?.throwIfAborted()
 	const resolvedLayout = config.layout
 	const id = generateSandboxId()
 	const docker = config.dockerBinary ?? DEFAULT_DOCKER_BINARY
@@ -387,7 +408,13 @@ async function spawnDockerSandbox(
 	let egressProxy: RunningEgressProxy | undefined
 	if (needsEgressProxy(options.egress) && options.egress) {
 		const policy = options.egress
-		egressProxy = await new EgressProxy(egressProxyOptions(config, policy)).listen()
+		try {
+			egressProxy = await new EgressProxy(egressProxyOptions(config, policy)).listen()
+			options.signal?.throwIfAborted()
+		} catch (error) {
+			await egressProxy?.close().catch(() => undefined)
+			throw error
+		}
 	}
 
 	const hostReachability = config.hostReachability ?? 'host-port'
@@ -407,7 +434,7 @@ async function spawnDockerSandbox(
 			network,
 			hostReachability,
 			options.egress,
-			await inspectNetworkInternalFlag(docker, network),
+			await inspectNetworkInternalFlag(docker, network, options.signal),
 		)
 	} catch (err) {
 		// The allowlist kinds start a proxy above, and this is outside the
@@ -425,12 +452,13 @@ async function spawnDockerSandbox(
 	// the EACCES bug in sibling-container setups (the consumer owns
 	// the host filesystem, the spawned backend can't reach it from
 	// inside its own container's mount namespace). Clean break.
-	let containerStarted = false
-
-	async function cleanupOnFailure() {
-		if (containerStarted) {
-			await runOnceQuiet(docker, ['rm', '-f', containerName])
-		}
+	async function cleanupOnFailure(signal: AbortSignal) {
+		// The name is known before `docker run`. Remove by name even when the
+		// client process was interrupted before it reported success: the daemon
+		// may already have committed the container. This is best-effort
+		// reconciliation; an external daemon that commits after this delete still
+		// needs its ordinary label/name reaper.
+		const removeContainer = runOnceQuiet(docker, ['rm', '-f', containerName], signal)
 		// The proxy starts BEFORE the container and its only other close is
 		// in `destroy()`, which a create that never returned can never
 		// reach. So every failure between the two — a daemon that is down, a
@@ -440,8 +468,11 @@ async function spawnDockerSandbox(
 		// event-loop handle, and a retry loop left one per attempt. That is
 		// exactly the invariant this file states where the proxy is started:
 		// it must not outlive the thing it was filtering for.
-		await egressProxy?.close().catch(() => undefined)
+		// Start both teardown arms before awaiting either. A stuck runtime must
+		// not prevent the proxy from releasing its credential-bearing listener.
+		const closeProxy = egressProxy?.close().catch(() => undefined) ?? Promise.resolve()
 		egressProxy = undefined
+		await Promise.all([removeContainer, closeProxy])
 	}
 
 	let hostPort: number
@@ -543,16 +574,15 @@ async function spawnDockerSandbox(
 
 		args.push(config.image)
 
-		await runOnce(docker, args)
-		containerStarted = true
-
+		await runOnce(docker, args, options.signal)
 		if (hostReachability === 'host-port') {
-			hostPort = await readMappedPort(docker, containerName)
+			hostPort = await readMappedPort(docker, containerName, options.signal)
 			baseUrl = `http://127.0.0.1:${hostPort}`
 			await waitForWorkerReady(
 				baseUrl,
-				config.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS,
-				config.readyPollIntervalMs ?? DEFAULT_READY_POLL_MS,
+				readiness.timeoutMs,
+				readiness.pollIntervalMs,
+				options.signal,
 			)
 		} else {
 			// container-network: connect by container DNS name on the
@@ -561,21 +591,106 @@ async function spawnDockerSandbox(
 			baseUrl = `http://${containerName}:${WORKER_PORT_INSIDE_CONTAINER}`
 			await waitForWorkerReady(
 				baseUrl,
-				config.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS,
-				config.readyPollIntervalMs ?? DEFAULT_READY_POLL_MS,
+				readiness.timeoutMs,
+				readiness.pollIntervalMs,
+				options.signal,
 			)
 		}
 	} catch (err) {
-		await cleanupOnFailure()
+		await runFailureCleanup(cleanupOnFailure)
 		throw err
 	}
 
-	let status: SandboxStatus = 'ready'
+	type Lifecycle = 'active' | 'retiring' | 'destroyed'
+	let activeExecutions = 0
+	let lifecycle: Lifecycle = 'active'
+	let retirementPromise: Promise<{ readonly accepted: boolean; readonly error?: Error }> | undefined
+	let teardownPromise: Promise<void> | undefined
+	let teardownComplete = false
+	const workerClient = new HttpWorkerClient(baseUrl)
+	const assertActive = (): void => {
+		if (lifecycle !== 'active') {
+			throw new Error(`Sandbox ${id} is ${lifecycle}; no new worker operation can be admitted`)
+		}
+	}
+	const teardownSandbox = (signal?: AbortSignal): Promise<void> => {
+		lifecycle = 'retiring'
+		if (teardownComplete) return Promise.resolve()
+		if (teardownPromise) return teardownPromise
+		const attempt = (async () => {
+			let teardownError: unknown
+			try {
+				await runOnce(docker, ['rm', '-f', containerName], signal)
+			} catch (error) {
+				teardownError = error
+			} finally {
+				try {
+					await egressProxy?.close()
+				} catch (error) {
+					teardownError ??= error
+				}
+			}
+			if (teardownError !== undefined) throw teardownError
+		})()
+		const shared = attempt.then(
+			() => {
+				teardownComplete = true
+				lifecycle = 'destroyed'
+			},
+			(error: unknown) => {
+				if (teardownPromise === shared) teardownPromise = undefined
+				throw error
+			},
+		)
+		teardownPromise = shared
+		return shared
+	}
+	const retire = (): Promise<{
+		readonly accepted: boolean
+		readonly error?: Error
+	}> => {
+		lifecycle = 'retiring'
+		if (retirementPromise) return retirementPromise
+		const deadline = new OperationDeadline(5_000, `docker sandbox ${id} retirement`)
+		retirementPromise = deadline
+			.run(async (signal) => {
+				const joinedExistingAttempt = teardownPromise !== undefined
+				try {
+					await teardownSandbox(signal)
+				} catch (error) {
+					if (!joinedExistingAttempt || signal.aborted) throw error
+					await teardownSandbox(signal)
+				}
+			})
+			.then(() => {
+				return { accepted: true as const }
+			})
+			.catch((error: unknown) => ({
+				accepted: false as const,
+				error: error instanceof Error ? error : new Error(String(error)),
+			}))
+		return retirementPromise
+	}
+	const runExecution = async <T>(operation: () => Promise<T>): Promise<T> => {
+		assertActive()
+		activeExecutions += 1
+		try {
+			return await operation()
+		} catch (error) {
+			if (error instanceof RemoteCancellationUnknownError) {
+				error.retirement = await retire()
+			}
+			throw error
+		} finally {
+			activeExecutions = Math.max(0, activeExecutions - 1)
+		}
+	}
 
 	return {
 		id,
 		get status(): SandboxStatus {
-			return status
+			if (lifecycle !== 'active') return 'destroyed'
+			return activeExecutions > 0 ? 'busy' : 'ready'
 		},
 		rootDir,
 		environment: detectEnvironment(),
@@ -585,15 +700,11 @@ async function spawnDockerSandbox(
 			argv?: string[],
 			opts?: SandboxExecOptions,
 		): Promise<SandboxExecResult> {
-			status = 'busy'
-			try {
-				return await execViaWorker(baseUrl, command, argv, opts)
-			} finally {
-				status = 'ready'
-			}
+			return await runExecution(async () => await workerClient.exec(command, argv, opts))
 		},
 
 		async setNetworkPolicy(policy): Promise<void> {
+			assertActive()
 			// Enforceable only through the egress proxy. Without one the
 			// container's network was fixed at creation — `--network none`
 			// or not — and there is nothing to narrow: accepting the policy
@@ -612,6 +723,7 @@ async function spawnDockerSandbox(
 		},
 
 		async writeFile(path: string, content: string | Buffer): Promise<void> {
+			assertActive()
 			const buf = Buffer.isBuffer(content) ? content : Buffer.from(content, 'utf8')
 			let res: Response
 			try {
@@ -643,6 +755,7 @@ async function spawnDockerSandbox(
 		},
 
 		async readFile(path: string): Promise<Buffer> {
+			assertActive()
 			const res = await fetch(`${baseUrl}/read-file`, {
 				method: 'POST',
 				headers: { 'content-type': 'application/json' },
@@ -651,7 +764,11 @@ async function spawnDockerSandbox(
 			if (!res.ok) {
 				throw new Error(`read-file failed: HTTP ${res.status} ${await res.text()}`)
 			}
-			const json = (await res.json()) as { ok: boolean; content?: string; error?: string }
+			const json = (await res.json()) as {
+				ok: boolean
+				content?: string
+				error?: string
+			}
 			if (!json.ok || typeof json.content !== 'string') {
 				throw new Error(json.error ?? 'read-file: no content')
 			}
@@ -659,17 +776,16 @@ async function spawnDockerSandbox(
 		},
 
 		async listFiles(rootPath: string): Promise<readonly SandboxFileEntry[]> {
-			return await listFilesViaWorker(baseUrl, rootPath)
+			return await runExecution(async () => await listFilesViaWorker(workerClient, rootPath))
 		},
 
-		async destroy(): Promise<void> {
-			status = 'destroyed'
-			await runOnceQuiet(docker, ['rm', '-f', containerName])
-			// The proxy holds real credentials and a live allowlist. Leaving
-			// it listening after the sandbox it was filtering for is gone
-			// means a loopback port that still stamps a token onto anything
-			// that asks — outliving the only thing that justified it.
-			await egressProxy?.close()
+		async destroy(options?: SandboxDestroyOptions): Promise<void> {
+			if (retirementPromise) {
+				const observation = await retirementPromise
+				if (observation.accepted) return
+				retirementPromise = undefined
+			}
+			await teardownSandbox(options?.signal)
 			// Backend never allocates host paths — every bind source
 			// comes from the consumer-supplied layout. Container
 			// teardown is sufficient; the consumer's own lifecycle
@@ -686,13 +802,21 @@ async function spawnDockerSandbox(
  * process could grab it in the meantime). Letting Docker
  * allocate and reading the mapping back is race-free.
  */
-async function readMappedPort(docker: string, containerName: string): Promise<number> {
-	const inspectOutput = await runOnce(docker, [
-		'inspect',
-		'--format',
-		`{{(index (index .NetworkSettings.Ports "${WORKER_PORT_INSIDE_CONTAINER}/tcp") 0).HostPort}}`,
-		containerName,
-	])
+async function readMappedPort(
+	docker: string,
+	containerName: string,
+	signal?: AbortSignal,
+): Promise<number> {
+	const inspectOutput = await runOnce(
+		docker,
+		[
+			'inspect',
+			'--format',
+			`{{(index (index .NetworkSettings.Ports "${WORKER_PORT_INSIDE_CONTAINER}/tcp") 0).HostPort}}`,
+			containerName,
+		],
+		signal,
+	)
 	const port = Number(inspectOutput.trim())
 	if (!Number.isInteger(port) || port <= 0 || port > 65535) {
 		throw withHint(
@@ -705,119 +829,6 @@ async function readMappedPort(docker: string, containerName: string): Promise<nu
 	return port
 }
 
-async function execViaWorker(
-	baseUrl: string,
-	command: string,
-	argv: string[] | undefined,
-	opts: SandboxExecOptions | undefined,
-): Promise<SandboxExecResult> {
-	// `opts.signal` is deliberately NOT passed to `fetch`. It would abort the
-	// request and leave the worker running the command — abandoning the wait
-	// while the process lives on is the exact failure
-	// `SandboxExecOptions.signal` exists to prevent, and wiring it here would
-	// make the option look honoured while delivering that failure. Honouring
-	// it needs a cancel endpoint on the worker.
-	const start = Date.now()
-	let res: Response
-	try {
-		res = await fetch(`${baseUrl}/execute`, {
-			method: 'POST',
-			headers: { 'content-type': 'application/json' },
-			body: JSON.stringify({
-				command,
-				args: argv ?? [],
-				cwd: opts?.cwd,
-				env: opts?.env,
-				timeoutMs: opts?.timeout,
-			}),
-		})
-	} catch (err) {
-		// Surface the underlying transport error (DNS, ECONNREFUSED,
-		// socket-hangup, …) instead of the generic "fetch failed" the
-		// undici client throws. Without `cause`, ops cannot tell whether
-		// the worker died, the bridge dropped, or something else.
-		const cause = err instanceof Error ? err.cause : undefined
-		const causeMsg =
-			cause instanceof Error
-				? `${cause.message}${(cause as Error & { code?: string }).code ? ` (${(cause as Error & { code?: string }).code})` : ''}`
-				: cause
-					? String(cause)
-					: 'unknown'
-		throw withHint(
-			new Error(
-				`namzu-sandbox /execute fetch failed (baseUrl=${baseUrl}): ${err instanceof Error ? err.message : String(err)} — cause: ${causeMsg}`,
-				{ cause: err },
-			),
-			'The container was reachable when it started, so it has most likely exited or been killed since — an out-of-memory kill under `memoryLimitMb` is the common cause. Check the container logs and its exit code.',
-		)
-	}
-	if (!res.ok || !res.body) {
-		throw new Error(`execute failed: HTTP ${res.status} ${await res.text()}`)
-	}
-
-	let stdout = ''
-	let stderr = ''
-	let exitCode = -1
-	let timedOut = false
-	let signal: string | undefined
-
-	const decoder = new TextDecoder()
-	const reader = res.body.getReader()
-	let buffered = ''
-	for (;;) {
-		const { value, done } = await reader.read()
-		if (done) break
-		buffered += decoder.decode(value, { stream: true })
-		let newlineIdx = buffered.indexOf('\n')
-		while (newlineIdx !== -1) {
-			const line = buffered.slice(0, newlineIdx).trim()
-			buffered = buffered.slice(newlineIdx + 1)
-			if (line) {
-				try {
-					const event = JSON.parse(line) as
-						| { type: 'stdout_delta'; data: string }
-						| { type: 'stderr_delta'; data: string }
-						| {
-								type: 'result'
-								exitCode: number
-								timedOut: boolean
-								durationMs: number
-						  }
-						| { type: 'error'; error: string }
-					if (event.type === 'stdout_delta') {
-						stdout += event.data
-						opts?.onOutput?.({ stream: 'stdout', data: event.data })
-					} else if (event.type === 'stderr_delta') {
-						stderr += event.data
-						opts?.onOutput?.({ stream: 'stderr', data: event.data })
-					} else if (event.type === 'result') {
-						exitCode = event.exitCode
-						timedOut = event.timedOut
-					} else if (event.type === 'error') {
-						throw new Error(event.error)
-					}
-				} catch (err) {
-					if (err instanceof SyntaxError) {
-						// Ignore malformed lines from the worker.
-					} else {
-						throw err
-					}
-				}
-			}
-			newlineIdx = buffered.indexOf('\n')
-		}
-	}
-
-	return {
-		exitCode,
-		stdout,
-		stderr,
-		...(signal ? { signal } : {}),
-		timedOut,
-		durationMs: Date.now() - start,
-	}
-}
-
 /**
  * Recursively list regular files under `rootPath` by shelling out to
  * the worker's `find` (GNU find on the Debian-based reference image).
@@ -827,11 +838,10 @@ async function execViaWorker(
  * have produced anything in `rootPath` yet.
  */
 async function listFilesViaWorker(
-	baseUrl: string,
+	workerClient: HttpWorkerClient,
 	rootPath: string,
 ): Promise<readonly SandboxFileEntry[]> {
-	const result = await execViaWorker(
-		baseUrl,
+	const result = await workerClient.exec(
 		'find',
 		[rootPath, '-type', 'f', '-printf', '%p\t%s\n'],
 		undefined,
@@ -883,18 +893,25 @@ async function waitForWorkerReady(
 	baseUrl: string,
 	timeoutMs: number,
 	pollMs: number,
+	signal?: AbortSignal,
 ): Promise<void> {
-	const deadline = Date.now() + timeoutMs
+	const deadline = new OperationDeadline(timeoutMs, 'docker worker readiness', signal)
 	let lastError: unknown
-	while (Date.now() < deadline) {
+	while (deadline.remainingMs() > 0) {
 		try {
-			const res = await fetch(`${baseUrl}/healthz`)
-			if (res.ok) return
-			lastError = new Error(`healthz HTTP ${res.status}`)
+			const result = await deadline.run((signal) => probeHttpHealth(`${baseUrl}/healthz`, signal))
+			if (result.ok) return
+			lastError = new Error(`healthz HTTP ${result.status}`)
 		} catch (err) {
 			lastError = err
+			if (err instanceof OperationDeadlineExpired) break
+		}
+		try {
+			await deadline.delay(pollMs)
+		} catch (err) {
+			if (err instanceof OperationDeadlineExpired) break
+			throw err
 		}
-		await new Promise((resolve) => setTimeout(resolve, pollMs))
 	}
 	// A hint attached at the throw site, where the cause is actually known.
 	// The container runtime's own message says a request failed; it cannot
@@ -910,22 +927,40 @@ async function waitForWorkerReady(
 	)
 }
 
-function runOnce(binary: string, args: string[]): Promise<string> {
+function runOnce(binary: string, args: string[], signal?: AbortSignal): Promise<string> {
 	return new Promise((resolve, reject) => {
+		signal?.throwIfAborted()
 		const child = spawn(binary, args, { stdio: ['ignore', 'pipe', 'pipe'] })
 		let stdout = ''
 		let stderr = ''
+		let settled = false
+		const finish = (error?: unknown, value?: string) => {
+			if (settled) return
+			settled = true
+			signal?.removeEventListener('abort', abort)
+			child.removeAllListeners('error')
+			child.removeAllListeners('close')
+			if (error !== undefined) reject(error)
+			else resolve(value ?? '')
+		}
+		const abort = () => {
+			child.kill('SIGKILL')
+			child.unref()
+			finish(signal?.reason ?? new Error('operation aborted'))
+		}
 		child.stdout.on('data', (chunk: Buffer) => {
 			stdout += chunk.toString('utf8')
 		})
 		child.stderr.on('data', (chunk: Buffer) => {
 			stderr += chunk.toString('utf8')
 		})
-		child.on('error', reject)
+		child.on('error', (error) => finish(error))
 		child.on('close', (code) => {
-			if (code === 0) resolve(stdout.trim())
-			else reject(new Error(`${binary} ${args.join(' ')} exited ${code}: ${stderr.trim()}`))
+			if (code === 0) finish(undefined, stdout.trim())
+			else finish(new Error(`${binary} ${args.join(' ')} exited ${code}: ${stderr.trim()}`))
 		})
+		if (signal?.aborted) abort()
+		else signal?.addEventListener('abort', abort, { once: true })
 	})
 }
 
@@ -938,19 +973,44 @@ function runOnce(binary: string, args: string[]): Promise<string> {
  * a boundary" — instead of surfacing a docker CLI error that says nothing
  * about the egress policy that prompted the lookup.
  */
-async function inspectNetworkInternalFlag(docker: string, network: string): Promise<string> {
+async function inspectNetworkInternalFlag(
+	docker: string,
+	network: string,
+	signal?: AbortSignal,
+): Promise<string> {
 	try {
-		return await runOnce(docker, ['network', 'inspect', '--format', '{{.Internal}}', network])
+		return await runOnce(
+			docker,
+			['network', 'inspect', '--format', '{{.Internal}}', network],
+			signal,
+		)
 	} catch {
+		signal?.throwIfAborted()
 		return ''
 	}
 }
 
-function runOnceQuiet(binary: string, args: string[]): Promise<void> {
+function runOnceQuiet(binary: string, args: string[], signal?: AbortSignal): Promise<void> {
 	return new Promise((resolve) => {
 		const child = spawn(binary, args, { stdio: 'ignore' })
-		child.on('error', () => resolve())
-		child.on('close', () => resolve())
+		let settled = false
+		const finish = () => {
+			if (settled) return
+			settled = true
+			signal?.removeEventListener('abort', abort)
+			child.removeListener('error', finish)
+			child.removeListener('close', finish)
+			resolve()
+		}
+		const abort = () => {
+			child.kill('SIGKILL')
+			child.unref()
+			finish()
+		}
+		child.on('error', finish)
+		child.on('close', finish)
+		if (signal?.aborted) abort()
+		else signal?.addEventListener('abort', abort, { once: true })
 	})
 }