@namzu/sandbox 6.1.0 → 7.0.0

package/src/backends/docker/index.ts

@@ -34,6 +34,7 @@ import {
 	SANDBOX_DEFAULT_TRANSCRIPTS_PATH,
 	SANDBOX_DEFAULT_UPLOADS_PATH,
 	type Sandbox,
+	type SandboxDestroyOptions,
 	type SandboxEnvironment,
 	type SandboxExecOptions,
 	type SandboxExecResult,
@@ -56,6 +57,13 @@ import {
 	type SandboxBackend,
 	type SandboxBackendOptions,
 } from '../../index.js'
+import {
+	OperationDeadline,
+	OperationDeadlineExpired,
+	probeHttpHealth,
+	resolveReadinessOptions,
+	runFailureCleanup,
+} from '../readiness.js'
 
 /**
  * Backend-specific tuning. Most hosts use the defaults; advanced
@@ -173,11 +181,20 @@ const WORKER_PORT_INSIDE_CONTAINER = 2024
  * `create()` call.
  */
 export function buildDockerBackend(config: DockerBackendInternalConfig): SandboxBackend {
+	const readiness = resolveReadinessOptions(
+		'docker',
+		config.readyTimeoutMs,
+		config.readyPollIntervalMs,
+		{
+			timeoutMs: DEFAULT_READY_TIMEOUT_MS,
+			pollIntervalMs: DEFAULT_READY_POLL_MS,
+		},
+	)
 	return {
 		tier: 'container',
 		name: 'docker',
 		async create(options: SandboxBackendOptions) {
-			return await spawnDockerSandbox(config, options)
+			return await spawnDockerSandbox(config, options, readiness)
 		},
 	}
 }
@@ -375,7 +392,9 @@ const PROXY_HOST_ALIAS = 'namzu-egress'
 async function spawnDockerSandbox(
 	config: DockerBackendInternalConfig,
 	options: SandboxBackendOptions,
+	readiness: { readonly timeoutMs: number; readonly pollIntervalMs: number },
 ): Promise<Sandbox> {
+	options.signal?.throwIfAborted()
 	const resolvedLayout = config.layout
 	const id = generateSandboxId()
 	const docker = config.dockerBinary ?? DEFAULT_DOCKER_BINARY
@@ -387,7 +406,13 @@ async function spawnDockerSandbox(
 	let egressProxy: RunningEgressProxy | undefined
 	if (needsEgressProxy(options.egress) && options.egress) {
 		const policy = options.egress
-		egressProxy = await new EgressProxy(egressProxyOptions(config, policy)).listen()
+		try {
+			egressProxy = await new EgressProxy(egressProxyOptions(config, policy)).listen()
+			options.signal?.throwIfAborted()
+		} catch (error) {
+			await egressProxy?.close().catch(() => undefined)
+			throw error
+		}
 	}
 
 	const hostReachability = config.hostReachability ?? 'host-port'
@@ -407,7 +432,7 @@ async function spawnDockerSandbox(
 			network,
 			hostReachability,
 			options.egress,
-			await inspectNetworkInternalFlag(docker, network),
+			await inspectNetworkInternalFlag(docker, network, options.signal),
 		)
 	} catch (err) {
 		// The allowlist kinds start a proxy above, and this is outside the
@@ -425,12 +450,13 @@ async function spawnDockerSandbox(
 	// the EACCES bug in sibling-container setups (the consumer owns
 	// the host filesystem, the spawned backend can't reach it from
 	// inside its own container's mount namespace). Clean break.
-	let containerStarted = false
-
-	async function cleanupOnFailure() {
-		if (containerStarted) {
-			await runOnceQuiet(docker, ['rm', '-f', containerName])
-		}
+	async function cleanupOnFailure(signal: AbortSignal) {
+		// The name is known before `docker run`. Remove by name even when the
+		// client process was interrupted before it reported success: the daemon
+		// may already have committed the container. This is best-effort
+		// reconciliation; an external daemon that commits after this delete still
+		// needs its ordinary label/name reaper.
+		const removeContainer = runOnceQuiet(docker, ['rm', '-f', containerName], signal)
 		// The proxy starts BEFORE the container and its only other close is
 		// in `destroy()`, which a create that never returned can never
 		// reach. So every failure between the two — a daemon that is down, a
@@ -440,8 +466,11 @@ async function spawnDockerSandbox(
 		// event-loop handle, and a retry loop left one per attempt. That is
 		// exactly the invariant this file states where the proxy is started:
 		// it must not outlive the thing it was filtering for.
-		await egressProxy?.close().catch(() => undefined)
+		// Start both teardown arms before awaiting either. A stuck runtime must
+		// not prevent the proxy from releasing its credential-bearing listener.
+		const closeProxy = egressProxy?.close().catch(() => undefined) ?? Promise.resolve()
 		egressProxy = undefined
+		await Promise.all([removeContainer, closeProxy])
 	}
 
 	let hostPort: number
@@ -543,16 +572,15 @@ async function spawnDockerSandbox(
 
 		args.push(config.image)
 
-		await runOnce(docker, args)
-		containerStarted = true
-
+		await runOnce(docker, args, options.signal)
 		if (hostReachability === 'host-port') {
-			hostPort = await readMappedPort(docker, containerName)
+			hostPort = await readMappedPort(docker, containerName, options.signal)
 			baseUrl = `http://127.0.0.1:${hostPort}`
 			await waitForWorkerReady(
 				baseUrl,
-				config.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS,
-				config.readyPollIntervalMs ?? DEFAULT_READY_POLL_MS,
+				readiness.timeoutMs,
+				readiness.pollIntervalMs,
+				options.signal,
 			)
 		} else {
 			// container-network: connect by container DNS name on the
@@ -561,12 +589,13 @@ async function spawnDockerSandbox(
 			baseUrl = `http://${containerName}:${WORKER_PORT_INSIDE_CONTAINER}`
 			await waitForWorkerReady(
 				baseUrl,
-				config.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS,
-				config.readyPollIntervalMs ?? DEFAULT_READY_POLL_MS,
+				readiness.timeoutMs,
+				readiness.pollIntervalMs,
+				options.signal,
 			)
 		}
 	} catch (err) {
-		await cleanupOnFailure()
+		await runFailureCleanup(cleanupOnFailure)
 		throw err
 	}
 
@@ -651,7 +680,11 @@ async function spawnDockerSandbox(
 			if (!res.ok) {
 				throw new Error(`read-file failed: HTTP ${res.status} ${await res.text()}`)
 			}
-			const json = (await res.json()) as { ok: boolean; content?: string; error?: string }
+			const json = (await res.json()) as {
+				ok: boolean
+				content?: string
+				error?: string
+			}
 			if (!json.ok || typeof json.content !== 'string') {
 				throw new Error(json.error ?? 'read-file: no content')
 			}
@@ -662,9 +695,9 @@ async function spawnDockerSandbox(
 			return await listFilesViaWorker(baseUrl, rootPath)
 		},
 
-		async destroy(): Promise<void> {
+		async destroy(options?: SandboxDestroyOptions): Promise<void> {
 			status = 'destroyed'
-			await runOnceQuiet(docker, ['rm', '-f', containerName])
+			await runOnceQuiet(docker, ['rm', '-f', containerName], options?.signal)
 			// The proxy holds real credentials and a live allowlist. Leaving
 			// it listening after the sandbox it was filtering for is gone
 			// means a loopback port that still stamps a token onto anything
@@ -686,13 +719,21 @@ async function spawnDockerSandbox(
  * process could grab it in the meantime). Letting Docker
  * allocate and reading the mapping back is race-free.
  */
-async function readMappedPort(docker: string, containerName: string): Promise<number> {
-	const inspectOutput = await runOnce(docker, [
-		'inspect',
-		'--format',
-		`{{(index (index .NetworkSettings.Ports "${WORKER_PORT_INSIDE_CONTAINER}/tcp") 0).HostPort}}`,
-		containerName,
-	])
+async function readMappedPort(
+	docker: string,
+	containerName: string,
+	signal?: AbortSignal,
+): Promise<number> {
+	const inspectOutput = await runOnce(
+		docker,
+		[
+			'inspect',
+			'--format',
+			`{{(index (index .NetworkSettings.Ports "${WORKER_PORT_INSIDE_CONTAINER}/tcp") 0).HostPort}}`,
+			containerName,
+		],
+		signal,
+	)
 	const port = Number(inspectOutput.trim())
 	if (!Number.isInteger(port) || port <= 0 || port > 65535) {
 		throw withHint(
@@ -883,18 +924,25 @@ async function waitForWorkerReady(
 	baseUrl: string,
 	timeoutMs: number,
 	pollMs: number,
+	signal?: AbortSignal,
 ): Promise<void> {
-	const deadline = Date.now() + timeoutMs
+	const deadline = new OperationDeadline(timeoutMs, 'docker worker readiness', signal)
 	let lastError: unknown
-	while (Date.now() < deadline) {
+	while (deadline.remainingMs() > 0) {
 		try {
-			const res = await fetch(`${baseUrl}/healthz`)
-			if (res.ok) return
-			lastError = new Error(`healthz HTTP ${res.status}`)
+			const result = await deadline.run((signal) => probeHttpHealth(`${baseUrl}/healthz`, signal))
+			if (result.ok) return
+			lastError = new Error(`healthz HTTP ${result.status}`)
 		} catch (err) {
 			lastError = err
+			if (err instanceof OperationDeadlineExpired) break
+		}
+		try {
+			await deadline.delay(pollMs)
+		} catch (err) {
+			if (err instanceof OperationDeadlineExpired) break
+			throw err
 		}
-		await new Promise((resolve) => setTimeout(resolve, pollMs))
 	}
 	// A hint attached at the throw site, where the cause is actually known.
 	// The container runtime's own message says a request failed; it cannot
@@ -910,22 +958,40 @@ async function waitForWorkerReady(
 	)
 }
 
-function runOnce(binary: string, args: string[]): Promise<string> {
+function runOnce(binary: string, args: string[], signal?: AbortSignal): Promise<string> {
 	return new Promise((resolve, reject) => {
+		signal?.throwIfAborted()
 		const child = spawn(binary, args, { stdio: ['ignore', 'pipe', 'pipe'] })
 		let stdout = ''
 		let stderr = ''
+		let settled = false
+		const finish = (error?: unknown, value?: string) => {
+			if (settled) return
+			settled = true
+			signal?.removeEventListener('abort', abort)
+			child.removeAllListeners('error')
+			child.removeAllListeners('close')
+			if (error !== undefined) reject(error)
+			else resolve(value ?? '')
+		}
+		const abort = () => {
+			child.kill('SIGKILL')
+			child.unref()
+			finish(signal?.reason ?? new Error('operation aborted'))
+		}
 		child.stdout.on('data', (chunk: Buffer) => {
 			stdout += chunk.toString('utf8')
 		})
 		child.stderr.on('data', (chunk: Buffer) => {
 			stderr += chunk.toString('utf8')
 		})
-		child.on('error', reject)
+		child.on('error', (error) => finish(error))
 		child.on('close', (code) => {
-			if (code === 0) resolve(stdout.trim())
-			else reject(new Error(`${binary} ${args.join(' ')} exited ${code}: ${stderr.trim()}`))
+			if (code === 0) finish(undefined, stdout.trim())
+			else finish(new Error(`${binary} ${args.join(' ')} exited ${code}: ${stderr.trim()}`))
 		})
+		if (signal?.aborted) abort()
+		else signal?.addEventListener('abort', abort, { once: true })
 	})
 }
 
@@ -938,19 +1004,44 @@ function runOnce(binary: string, args: string[]): Promise<string> {
  * a boundary" — instead of surfacing a docker CLI error that says nothing
  * about the egress policy that prompted the lookup.
  */
-async function inspectNetworkInternalFlag(docker: string, network: string): Promise<string> {
+async function inspectNetworkInternalFlag(
+	docker: string,
+	network: string,
+	signal?: AbortSignal,
+): Promise<string> {
 	try {
-		return await runOnce(docker, ['network', 'inspect', '--format', '{{.Internal}}', network])
+		return await runOnce(
+			docker,
+			['network', 'inspect', '--format', '{{.Internal}}', network],
+			signal,
+		)
 	} catch {
+		signal?.throwIfAborted()
 		return ''
 	}
 }
 
-function runOnceQuiet(binary: string, args: string[]): Promise<void> {
+function runOnceQuiet(binary: string, args: string[], signal?: AbortSignal): Promise<void> {
 	return new Promise((resolve) => {
 		const child = spawn(binary, args, { stdio: 'ignore' })
-		child.on('error', () => resolve())
-		child.on('close', () => resolve())
+		let settled = false
+		const finish = () => {
+			if (settled) return
+			settled = true
+			signal?.removeEventListener('abort', abort)
+			child.removeListener('error', finish)
+			child.removeListener('close', finish)
+			resolve()
+		}
+		const abort = () => {
+			child.kill('SIGKILL')
+			child.unref()
+			finish()
+		}
+		child.on('error', finish)
+		child.on('close', finish)
+		if (signal?.aborted) abort()
+		else signal?.addEventListener('abort', abort, { once: true })
 	})
 }
 

package/src/backends/firecracker/index.ts

@@ -48,6 +48,7 @@ import https from 'node:https'
 
 import type {
 	Sandbox,
+	SandboxDestroyOptions,
 	SandboxEnvironment,
 	SandboxExecOptions,
 	SandboxExecResult,
@@ -57,6 +58,7 @@ import type {
 } from '@namzu/sdk'
 
 import type { AgentSnapshotRef, SandboxBackend, SandboxBackendOptions } from '../../index.js'
+import { resolveReadinessOptions, runFailureCleanup } from '../readiness.js'
 import type {
 	MtlsClientMaterial,
 	SandboxAgentHandle,
@@ -148,11 +150,20 @@ const DEFAULT_READY_POLL_MS = 250
  * on the first `create()`.
  */
 export function buildFirecrackerBackend(config: FirecrackerBackendInternalConfig): SandboxBackend {
+	const readiness = resolveReadinessOptions(
+		'firecracker',
+		config.readyTimeoutMs,
+		config.readyPollIntervalMs,
+		{
+			timeoutMs: DEFAULT_READY_TIMEOUT_MS,
+			pollIntervalMs: DEFAULT_READY_POLL_MS,
+		},
+	)
 	return {
 		tier: 'microvm',
 		name: 'firecracker',
 		async create(options: SandboxBackendOptions): Promise<Sandbox> {
-			return await spawnFirecrackerSandbox(config, options)
+			return await spawnFirecrackerSandbox(config, options, readiness)
 		},
 	}
 }
@@ -208,8 +219,11 @@ async function orchestratorCall<T>(
 	getToken: OrchestratorTokenProvider,
 	body?: unknown,
 	mtls?: MtlsClientMaterial,
+	signal?: AbortSignal,
 ): Promise<T | undefined> {
+	signal?.throwIfAborted()
 	const token = await getToken()
+	signal?.throwIfAborted()
 	const url = `${stripTrailingSlashes(endpoint)}${pathSuffix}`
 	const payload = body !== undefined ? JSON.stringify(body) : undefined
 	const headers: Record<string, string> = {
@@ -226,8 +240,8 @@ async function orchestratorCall<T>(
 		// than fetch+undici-dispatcher because the package declares no undici
 		// dependency — node:https is always importable and needs nothing added.
 		res = mtls
-			? await httpsOrchestratorRequest(url, method, headers, payload, mtls)
-			: await fetchOrchestratorRequest(url, method, headers, payload)
+			? await httpsOrchestratorRequest(url, method, headers, payload, mtls, signal)
+			: await fetchOrchestratorRequest(url, method, headers, payload, signal)
 	} catch (err) {
 		const cause = err instanceof Error ? err.cause : undefined
 		throw new Error(
@@ -238,12 +252,16 @@ async function orchestratorCall<T>(
 		)
 	}
 	if (res.status < 200 || res.status >= 300) {
-		throw new Error(
-			`firecracker orchestrator ${method} ${url} → ${res.status}: ${await res.text()}`,
-		)
+		const text = await res.text()
+		signal?.throwIfAborted()
+		throw new Error(`firecracker orchestrator ${method} ${url} → ${res.status}: ${text}`)
 	}
 	if (res.status === 204) return undefined
-	if (res.contentType.includes('application/json')) return JSON.parse(await res.text()) as T
+	if (res.contentType.includes('application/json')) {
+		const text = await res.text()
+		signal?.throwIfAborted()
+		return JSON.parse(text) as T
+	}
 	return undefined
 }
 
@@ -253,9 +271,11 @@ async function fetchOrchestratorRequest(
 	method: 'POST' | 'DELETE',
 	headers: Record<string, string>,
 	payload: string | undefined,
+	signal?: AbortSignal,
 ): Promise<OrchestratorRawResponse> {
 	const init: RequestInit = { method, headers }
 	if (payload !== undefined) init.body = payload
+	if (signal !== undefined) init.signal = signal
 	const res = await fetch(url, init)
 	return {
 		status: res.status,
@@ -276,6 +296,7 @@ function httpsOrchestratorRequest(
 	headers: Record<string, string>,
 	payload: string | undefined,
 	mtls: MtlsClientMaterial,
+	signal?: AbortSignal,
 ): Promise<OrchestratorRawResponse> {
 	const target = new URL(url)
 	return new Promise<OrchestratorRawResponse>((resolve, reject) => {
@@ -293,6 +314,7 @@ function httpsOrchestratorRequest(
 				rejectUnauthorized: true,
 				minVersion: 'TLSv1.3',
 				...(mtls.servername !== undefined ? { servername: mtls.servername } : {}),
+				...(signal !== undefined ? { signal } : {}),
 			},
 			(res) => {
 				const chunks: Buffer[] = []
@@ -321,9 +343,12 @@ function httpsOrchestratorRequest(
 async function spawnFirecrackerSandbox(
 	config: FirecrackerBackendInternalConfig,
 	options: SandboxBackendOptions,
+	readiness: { readonly timeoutMs: number; readonly pollIntervalMs: number },
 ): Promise<Sandbox> {
+	options.signal?.throwIfAborted()
 	const endpoint = config.orchestratorEndpoint
 	const egressAllowlist = await resolveEgressAllowlist(options)
+	options.signal?.throwIfAborted()
 	const createBody: OrchestratorCreateRequest = {
 		...(config.template !== undefined ? { template: config.template } : {}),
 		...(config.agentSnapshot !== undefined ? { agentSnapshot: config.agentSnapshot } : {}),
@@ -345,11 +370,17 @@ async function spawnFirecrackerSandbox(
 			config.getToken,
 			createBody,
 			config.controlPlaneMtls,
+			options.signal,
 		)
 	} catch (err) {
+		const interrupted = options.signal?.aborted
 		throw new Error(
 			`firecracker: failed to create microVM sandbox — ${
 				err instanceof Error ? err.message : String(err)
+			}${
+				interrupted
+					? '; allocation outcome is unresolved because the orchestrator did not return a sandboxId — the deployment fleet reaper must reconcile it'
+					: ''
 			}`,
 			{ cause: err },
 		)
@@ -380,7 +411,7 @@ async function spawnFirecrackerSandbox(
 	)
 	const transport = new VsockAgentTransport(handle, config.transport ?? {})
 
-	const destroy = async (): Promise<void> => {
+	const destroy = async (signal?: AbortSignal): Promise<void> => {
 		await orchestratorCall(
 			endpoint,
 			`/sandboxes/${encodeURIComponent(id)}:delete`,
@@ -388,28 +419,25 @@ async function spawnFirecrackerSandbox(
 			config.getToken,
 			undefined,
 			config.controlPlaneMtls,
+			signal,
 		)
 	}
 
 	try {
+		options.signal?.throwIfAborted()
 		// Readiness fence: the orchestrator's resume returns BEFORE the
 		// guest agent has reseeded entropy and re-listened on vsock. Stop
 		// the clock on the agent's healthz, exactly as the HTTP backends
 		// wait on `/healthz` — never on the orchestrator's 2xx, which
 		// fires before the guest runs (§5 clock semantics).
-		await transport.waitForReady(
-			config.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS,
-			config.readyPollIntervalMs ?? DEFAULT_READY_POLL_MS,
-		)
+		await transport.waitForReady(readiness.timeoutMs, readiness.pollIntervalMs, options.signal)
 	} catch (err) {
 		// Best-effort orchestrator teardown so a readiness failure does not
-		// orphan a microVM/netns/UFFD handler (the reaper backstops, but
-		// surfacing the delete failure keeps the leak observable).
-		try {
-			await destroy()
-		} catch {
-			// Preserve the readiness error as primary.
-		}
+		// orphan a microVM/netns/UFFD handler. The fleet reaper backstops a
+		// failed delete; this call gets a separate short grace so cleanup
+		// cannot turn the captured readiness error back into an unbounded
+		// create operation.
+		await runFailureCleanup(async (signal) => destroy(signal))
 		throw err
 	}
 
@@ -482,13 +510,13 @@ async function spawnFirecrackerSandbox(
 			return entries
 		},
 
-		async destroy(): Promise<void> {
+		async destroy(options?: SandboxDestroyOptions): Promise<void> {
 			status = 'destroyed'
 			// Let the orchestrator DELETE failure propagate — the
 			// Vandal-side lifecycle wraps this with logging, and a
 			// swallowed error here means orphaned microVMs (and their
 			// netns / UFFD handlers) pile up with no observability handle.
-			await destroy()
+			await destroy(options?.signal)
 		},
 	}
 }