@namzu/sandbox 6.1.0 → 7.1.0

package/src/backends/firecracker/index.ts

@@ -48,6 +48,7 @@ import https from 'node:https'
 
 import type {
 	Sandbox,
+	SandboxDestroyOptions,
 	SandboxEnvironment,
 	SandboxExecOptions,
 	SandboxExecResult,
@@ -57,6 +58,8 @@ import type {
 } from '@namzu/sdk'
 
 import type { AgentSnapshotRef, SandboxBackend, SandboxBackendOptions } from '../../index.js'
+import { OperationDeadline, resolveReadinessOptions, runFailureCleanup } from '../readiness.js'
+import { RemoteCancellationUnknownError } from '../remote-execution-controller.js'
 import type {
 	MtlsClientMaterial,
 	SandboxAgentHandle,
@@ -141,6 +144,7 @@ export interface FirecrackerBackendInternalConfig {
 const DEFAULT_AGENT_VSOCK_PORT = 1024
 const DEFAULT_READY_TIMEOUT_MS = 60_000
 const DEFAULT_READY_POLL_MS = 250
+const RETIREMENT_TIMEOUT_MS = 5_000
 
 /**
  * Build a {@link SandboxBackend} backed by the owned Firecracker
@@ -148,11 +152,20 @@ const DEFAULT_READY_POLL_MS = 250
  * on the first `create()`.
  */
 export function buildFirecrackerBackend(config: FirecrackerBackendInternalConfig): SandboxBackend {
+	const readiness = resolveReadinessOptions(
+		'firecracker',
+		config.readyTimeoutMs,
+		config.readyPollIntervalMs,
+		{
+			timeoutMs: DEFAULT_READY_TIMEOUT_MS,
+			pollIntervalMs: DEFAULT_READY_POLL_MS,
+		},
+	)
 	return {
 		tier: 'microvm',
 		name: 'firecracker',
 		async create(options: SandboxBackendOptions): Promise<Sandbox> {
-			return await spawnFirecrackerSandbox(config, options)
+			return await spawnFirecrackerSandbox(config, options, readiness)
 		},
 	}
 }
@@ -208,8 +221,12 @@ async function orchestratorCall<T>(
 	getToken: OrchestratorTokenProvider,
 	body?: unknown,
 	mtls?: MtlsClientMaterial,
+	signal?: AbortSignal,
+	acceptedStatuses: readonly number[] = [],
 ): Promise<T | undefined> {
+	signal?.throwIfAborted()
 	const token = await getToken()
+	signal?.throwIfAborted()
 	const url = `${stripTrailingSlashes(endpoint)}${pathSuffix}`
 	const payload = body !== undefined ? JSON.stringify(body) : undefined
 	const headers: Record<string, string> = {
@@ -226,8 +243,8 @@ async function orchestratorCall<T>(
 		// than fetch+undici-dispatcher because the package declares no undici
 		// dependency — node:https is always importable and needs nothing added.
 		res = mtls
-			? await httpsOrchestratorRequest(url, method, headers, payload, mtls)
-			: await fetchOrchestratorRequest(url, method, headers, payload)
+			? await httpsOrchestratorRequest(url, method, headers, payload, mtls, signal)
+			: await fetchOrchestratorRequest(url, method, headers, payload, signal)
 	} catch (err) {
 		const cause = err instanceof Error ? err.cause : undefined
 		throw new Error(
@@ -238,12 +255,21 @@ async function orchestratorCall<T>(
 		)
 	}
 	if (res.status < 200 || res.status >= 300) {
-		throw new Error(
-			`firecracker orchestrator ${method} ${url} → ${res.status}: ${await res.text()}`,
-		)
+		if (acceptedStatuses.includes(res.status)) {
+			await res.text()
+			signal?.throwIfAborted()
+			return undefined
+		}
+		const text = await res.text()
+		signal?.throwIfAborted()
+		throw new Error(`firecracker orchestrator ${method} ${url} → ${res.status}: ${text}`)
 	}
 	if (res.status === 204) return undefined
-	if (res.contentType.includes('application/json')) return JSON.parse(await res.text()) as T
+	if (res.contentType.includes('application/json')) {
+		const text = await res.text()
+		signal?.throwIfAborted()
+		return JSON.parse(text) as T
+	}
 	return undefined
 }
 
@@ -253,9 +279,11 @@ async function fetchOrchestratorRequest(
 	method: 'POST' | 'DELETE',
 	headers: Record<string, string>,
 	payload: string | undefined,
+	signal?: AbortSignal,
 ): Promise<OrchestratorRawResponse> {
 	const init: RequestInit = { method, headers }
 	if (payload !== undefined) init.body = payload
+	if (signal !== undefined) init.signal = signal
 	const res = await fetch(url, init)
 	return {
 		status: res.status,
@@ -276,6 +304,7 @@ function httpsOrchestratorRequest(
 	headers: Record<string, string>,
 	payload: string | undefined,
 	mtls: MtlsClientMaterial,
+	signal?: AbortSignal,
 ): Promise<OrchestratorRawResponse> {
 	const target = new URL(url)
 	return new Promise<OrchestratorRawResponse>((resolve, reject) => {
@@ -293,6 +322,7 @@ function httpsOrchestratorRequest(
 				rejectUnauthorized: true,
 				minVersion: 'TLSv1.3',
 				...(mtls.servername !== undefined ? { servername: mtls.servername } : {}),
+				...(signal !== undefined ? { signal } : {}),
 			},
 			(res) => {
 				const chunks: Buffer[] = []
@@ -321,9 +351,12 @@ function httpsOrchestratorRequest(
 async function spawnFirecrackerSandbox(
 	config: FirecrackerBackendInternalConfig,
 	options: SandboxBackendOptions,
+	readiness: { readonly timeoutMs: number; readonly pollIntervalMs: number },
 ): Promise<Sandbox> {
+	options.signal?.throwIfAborted()
 	const endpoint = config.orchestratorEndpoint
 	const egressAllowlist = await resolveEgressAllowlist(options)
+	options.signal?.throwIfAborted()
 	const createBody: OrchestratorCreateRequest = {
 		...(config.template !== undefined ? { template: config.template } : {}),
 		...(config.agentSnapshot !== undefined ? { agentSnapshot: config.agentSnapshot } : {}),
@@ -345,11 +378,17 @@ async function spawnFirecrackerSandbox(
 			config.getToken,
 			createBody,
 			config.controlPlaneMtls,
+			options.signal,
 		)
 	} catch (err) {
+		const interrupted = options.signal?.aborted
 		throw new Error(
 			`firecracker: failed to create microVM sandbox — ${
 				err instanceof Error ? err.message : String(err)
+			}${
+				interrupted
+					? '; allocation outcome is unresolved because the orchestrator did not return a sandboxId — the deployment fleet reaper must reconcile it'
+					: ''
 			}`,
 			{ cause: err },
 		)
@@ -380,7 +419,10 @@ async function spawnFirecrackerSandbox(
 	)
 	const transport = new VsockAgentTransport(handle, config.transport ?? {})
 
-	const destroy = async (): Promise<void> => {
+	const destroy = async (
+		signal?: AbortSignal,
+		acceptedStatuses: readonly number[] = [],
+	): Promise<void> => {
 		await orchestratorCall(
 			endpoint,
 			`/sandboxes/${encodeURIComponent(id)}:delete`,
@@ -388,37 +430,111 @@ async function spawnFirecrackerSandbox(
 			config.getToken,
 			undefined,
 			config.controlPlaneMtls,
+			signal,
+			acceptedStatuses,
 		)
 	}
 
 	try {
+		options.signal?.throwIfAborted()
 		// Readiness fence: the orchestrator's resume returns BEFORE the
 		// guest agent has reseeded entropy and re-listened on vsock. Stop
 		// the clock on the agent's healthz, exactly as the HTTP backends
 		// wait on `/healthz` — never on the orchestrator's 2xx, which
 		// fires before the guest runs (§5 clock semantics).
-		await transport.waitForReady(
-			config.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS,
-			config.readyPollIntervalMs ?? DEFAULT_READY_POLL_MS,
-		)
+		await transport.waitForReady(readiness.timeoutMs, readiness.pollIntervalMs, options.signal)
 	} catch (err) {
 		// Best-effort orchestrator teardown so a readiness failure does not
-		// orphan a microVM/netns/UFFD handler (the reaper backstops, but
-		// surfacing the delete failure keeps the leak observable).
-		try {
-			await destroy()
-		} catch {
-			// Preserve the readiness error as primary.
-		}
+		// orphan a microVM/netns/UFFD handler. The fleet reaper backstops a
+		// failed delete; this call gets a separate short grace so cleanup
+		// cannot turn the captured readiness error back into an unbounded
+		// create operation.
+		await runFailureCleanup(async (signal) => destroy(signal))
 		throw err
 	}
 
-	let status: SandboxStatus = 'ready'
+	type Lifecycle = 'active' | 'retiring' | 'destroyed'
+	let activeExecutions = 0
+	let lifecycle: Lifecycle = 'active'
+	let retirementPromise: Promise<{ readonly accepted: boolean; readonly error?: Error }> | undefined
+	let teardownPromise: Promise<void> | undefined
+	let teardownComplete = false
+
+	const assertActive = (): void => {
+		if (lifecycle !== 'active') {
+			throw new Error(`Sandbox ${id} is ${lifecycle}; no new guest operation can be admitted`)
+		}
+	}
+
+	const teardownSandbox = (signal?: AbortSignal): Promise<void> => {
+		lifecycle = 'retiring'
+		if (teardownComplete) return Promise.resolve()
+		if (teardownPromise) return teardownPromise
+		const attempt = destroy(signal, [404, 410])
+		const shared = attempt.then(
+			() => {
+				teardownComplete = true
+				lifecycle = 'destroyed'
+			},
+			(error: unknown) => {
+				if (teardownPromise === shared) teardownPromise = undefined
+				throw error
+			},
+		)
+		teardownPromise = shared
+		return shared
+	}
+	const retire = (): Promise<{ readonly accepted: boolean; readonly error?: Error }> => {
+		lifecycle = 'retiring'
+		if (retirementPromise) return retirementPromise
+		const deadline = new OperationDeadline(
+			RETIREMENT_TIMEOUT_MS,
+			`firecracker sandbox ${id} retirement`,
+		)
+		retirementPromise = deadline
+			.run(async (signal) => {
+				const joinedExistingAttempt = teardownPromise !== undefined
+				try {
+					await teardownSandbox(signal)
+				} catch (error) {
+					if (!joinedExistingAttempt || signal.aborted) throw error
+					await teardownSandbox(signal)
+				}
+			})
+			.then(() => {
+				return { accepted: true as const }
+			})
+			.catch((error: unknown) => {
+				const normalized = error instanceof Error ? error : new Error(String(error))
+				return { accepted: false as const, error: normalized }
+			})
+		return retirementPromise
+	}
+
+	const classifyExecutionFailure = async (error: unknown): Promise<never> => {
+		if (error instanceof RemoteCancellationUnknownError) {
+			error.retirement = await retire()
+		}
+		throw error
+	}
+
+	const runExecution = async <T>(operation: () => Promise<T>): Promise<T> => {
+		assertActive()
+		activeExecutions += 1
+		try {
+			return await operation()
+		} catch (error) {
+			return await classifyExecutionFailure(error)
+		} finally {
+			activeExecutions = Math.max(0, activeExecutions - 1)
+		}
+	}
 
 	return {
 		id,
 		get status(): SandboxStatus {
-			return status
+			if (lifecycle !== 'active') return 'destroyed'
+			return activeExecutions > 0 ? 'busy' : 'ready'
 		},
 		rootDir,
 		environment: detectEnvironment(),
@@ -428,67 +544,51 @@ async function spawnFirecrackerSandbox(
 			argv?: string[],
 			opts?: SandboxExecOptions,
 		): Promise<SandboxExecResult> {
-			// `opts.signal` is deliberately not forwarded, and the reason is
-			// worth writing down because the obvious "fix" is worse than the
-			// gap. There is no cancel op on this wire — the guest agent takes
-			// an execute frame and answers when the command is done. Aborting
-			// the socket here would abandon the WAIT while the process keeps
-			// running inside the microVM, which is verbatim the failure
-			// `SandboxExecOptions.signal` exists to prevent, except it would
-			// then look honoured. Honouring it means a cancel op in the guest
-			// protocol; until then, ignoring it is the truthful behaviour the
-			// option's own contract allows.
-			status = 'busy'
-			try {
-				return await transport.execute({
-					command,
-					args: argv ?? [],
-					...(opts?.cwd !== undefined ? { cwd: opts.cwd } : {}),
-					...(opts?.env !== undefined ? { env: opts.env } : {}),
-					...(opts?.timeout !== undefined ? { timeoutMs: opts.timeout } : {}),
-				})
-			} finally {
-				status = 'ready'
-			}
+			return await runExecution(async () => await transport.exec(command, argv, opts))
 		},
 
 		async writeFile(path: string, content: string | Buffer): Promise<void> {
+			assertActive()
 			const buf = Buffer.isBuffer(content) ? content : Buffer.from(content, 'utf8')
 			await transport.writeFile(path, buf)
 		},
 
 		async readFile(path: string): Promise<Buffer> {
+			assertActive()
 			return await transport.readFile(path)
 		},
 
 		async listFiles(rootPath: string): Promise<readonly SandboxFileEntry[]> {
-			// Same wire as docker/aci: `find -printf '%p\t%s\n'`, parse
-			// line-by-line, map a non-zero exit (missing root) to "empty".
-			const result = await transport.execute({
-				command: 'find',
-				args: [rootPath, '-type', 'f', '-printf', '%p\t%s\n'],
+			return await runExecution(async () => {
+				// Same wire as docker/aci: `find -printf '%p\t%s\n'`, parse
+				// line-by-line, map a non-zero exit (missing root) to "empty".
+				const result = await transport.exec('find', [rootPath, '-type', 'f', '-printf', '%p\t%s\n'])
+				if (result.exitCode !== 0) return []
+				const entries: SandboxFileEntry[] = []
+				for (const rawLine of result.stdout.split('\n')) {
+					if (!rawLine) continue
+					const tab = rawLine.indexOf('\t')
+					if (tab < 0) continue
+					const filePath = rawLine.slice(0, tab)
+					const size = Number.parseInt(rawLine.slice(tab + 1), 10)
+					if (!filePath || !Number.isFinite(size)) continue
+					entries.push({ path: filePath, size })
+				}
+				return entries
 			})
-			if (result.exitCode !== 0) return []
-			const entries: SandboxFileEntry[] = []
-			for (const rawLine of result.stdout.split('\n')) {
-				if (!rawLine) continue
-				const tab = rawLine.indexOf('\t')
-				if (tab < 0) continue
-				const filePath = rawLine.slice(0, tab)
-				const size = Number.parseInt(rawLine.slice(tab + 1), 10)
-				if (!filePath || !Number.isFinite(size)) continue
-				entries.push({ path: filePath, size })
-			}
-			return entries
 		},
 
-		async destroy(): Promise<void> {
-			status = 'destroyed'
+		async destroy(options?: SandboxDestroyOptions): Promise<void> {
+			if (retirementPromise) {
+				const observation = await retirementPromise
+				if (observation.accepted) return
+				retirementPromise = undefined
+			}
 			// Let the orchestrator DELETE failure propagate — the
 			// Vandal-side lifecycle wraps this with logging, and a
 			// swallowed error here means orphaned microVMs (and their
 			// netns / UFFD handlers) pile up with no observability handle.
-			await destroy()
+			await teardownSandbox(options?.signal)
 		},
 	}
 }

package/src/backends/firecracker/protocol.ts

@@ -1,5 +1,5 @@
 /**
- * The ONE wire contract, shared across both transports.
+ * The execution-data codec shared by both transports.
  *
  * The docker (`backends/docker/`) and ACI (`backends/aci-standby-pool/`)
  * backends speak this contract over **HTTP**: a streaming NDJSON
@@ -9,23 +9,15 @@
  * shapes** — only the transport changes from HTTP-over-TCP to
  * framed-over-vsock (see `transport.ts`).
  *
- * This module is the single source of truth for the message shapes
- * and the streaming exec-line parser, so the two transports cannot
- * drift. It deliberately carries NO transport concern (no `fetch`, no
- * socket) — it is pure codec. The HTTP backends keep their own inline
- * copies (they predate this module and stay UNTOUCHED per the P0
- * scope); this module mirrors those shapes verbatim and is the
- * canonical definition the vsock path consumes.
- *
- * Why mirror rather than refactor docker/aci onto it: the P0 scope is
- * "do NOT touch docker or aci". Centralising the shapes here, with the
- * inline docker copy pinned by `backends/docker/__tests__` and this
- * module pinned by `backends/firecracker/__tests__`, keeps both honest
- * without editing the frozen HTTP path. A later cleanup pass can fold
- * docker/aci onto this module once it is no longer release-frozen.
+ * This module remains the pure codec consumed by the framed guest transport.
+ * The two HTTP backends share one strict HTTP client, and both transports put
+ * the same reserve-before-admission + idempotent-cancel state machine around
+ * these data events. HTTP uses endpoints; the framed guest uses dedicated ops.
  */
 
-import type { SandboxExecResult } from '@namzu/sdk'
+import type { SandboxExecOptions, SandboxExecResult } from '@namzu/sdk'
+
+import { RemoteCommandError, RemoteProtocolError } from '../remote-execution-controller.js'
 
 // ---------------------------------------------------------------------------
 // Exec — request + the NDJSON event shapes (verbatim from worker/server.js)
@@ -38,6 +30,7 @@ import type { SandboxExecResult } from '@namzu/sdk'
  * `SandboxExecOptions.timeout`.
  */
 export interface ExecRequest {
+	readonly executionId?: string
 	readonly command: string
 	readonly args?: readonly string[]
 	readonly cwd?: string
@@ -62,7 +55,8 @@ export type ExecEvent =
 			readonly type: 'result'
 			readonly exitCode: number
 			readonly timedOut: boolean
-			readonly durationMs?: number
+			readonly durationMs: number
+			readonly signal?: string
 			readonly stdoutTruncated?: boolean
 			readonly stderrTruncated?: boolean
 	  }
@@ -113,9 +107,8 @@ export interface ReadFileResponse {
  * terminal `result`, and **throw** on an `error` event.
  *
  * Transport-agnostic: feed it whole parsed {@link ExecEvent}s (the
- * transport owns newline-framing → JSON.parse → here). Malformed lines
- * are dropped by the transport's `JSON.parse` guard before they reach
- * this accumulator, matching the docker loop's `SyntaxError` swallow.
+ * transport owns framing → strict JSON validation → here). Malformed or
+ * trailing events are protocol failures rather than silently discarded data.
  */
 export class ExecResultAccumulator {
 	private stdout = ''
@@ -123,11 +116,16 @@ export class ExecResultAccumulator {
 	private exitCode = -1
 	private timedOut = false
 	private signal: string | undefined
+	private durationMs: number | undefined
+	private stdoutTruncated: boolean | undefined
+	private stderrTruncated: boolean | undefined
 	private settled = false
 	private readonly start: number
+	private readonly onOutput: SandboxExecOptions['onOutput']
 
-	constructor(start: number = Date.now()) {
+	constructor(start: number = Date.now(), onOutput?: SandboxExecOptions['onOutput']) {
 		this.start = start
+		this.onOutput = onOutput
 	}
 
 	/**
@@ -137,55 +135,91 @@ export class ExecResultAccumulator {
 	 * docker loop uses (`throw new Error(event.error)`).
 	 */
 	push(event: ExecEvent): boolean {
+		if (this.settled) {
+			throw new RemoteProtocolError('exec stream emitted data after its terminal event')
+		}
 		if (event.type === 'stdout_delta') {
 			this.stdout += event.data
+			this.onOutput?.({ stream: 'stdout', data: event.data })
 			return false
 		}
 		if (event.type === 'stderr_delta') {
 			this.stderr += event.data
+			this.onOutput?.({ stream: 'stderr', data: event.data })
 			return false
 		}
 		if (event.type === 'result') {
 			this.exitCode = event.exitCode
 			this.timedOut = event.timedOut
+			this.durationMs = event.durationMs
+			this.signal = event.signal
+			this.stdoutTruncated = event.stdoutTruncated
+			this.stderrTruncated = event.stderrTruncated
 			this.settled = true
 			return true
 		}
 		// event.type === 'error'
-		throw new Error(event.error)
+		throw new RemoteCommandError(event.error)
 	}
 
 	get done(): boolean {
 		return this.settled
 	}
 
-	/** Build the SDK-shaped result. `durationMs` measured host-side. */
+	/** Build the SDK-shaped result from the guest's terminal metadata. */
 	finish(): SandboxExecResult {
+		if (!this.settled || this.durationMs === undefined) {
+			throw new RemoteProtocolError('exec stream ended without exactly one result event')
+		}
 		return {
 			exitCode: this.exitCode,
 			stdout: this.stdout,
 			stderr: this.stderr,
 			...(this.signal ? { signal: this.signal } : {}),
 			timedOut: this.timedOut,
-			durationMs: Date.now() - this.start,
+			durationMs: this.durationMs ?? Date.now() - this.start,
+			...(this.stdoutTruncated !== undefined ? { stdoutTruncated: this.stdoutTruncated } : {}),
+			...(this.stderrTruncated !== undefined ? { stderrTruncated: this.stderrTruncated } : {}),
 		}
 	}
 }
 
 /**
- * Parse a single NDJSON line into an {@link ExecEvent}, or `undefined`
- * if the line is blank or not valid JSON (the docker loop's
- * `SyntaxError` swallow). Non-`SyntaxError` problems are impossible
- * here because we only `JSON.parse`; structural validation is by the
- * `type` discriminator at the call site.
+ * Parse and structurally validate a single NDJSON event. Blank padding is
+ * ignored; malformed JSON and unknown/partial event shapes are refused.
  */
 export function parseExecLine(line: string): ExecEvent | undefined {
 	const trimmed = line.trim()
 	if (!trimmed) return undefined
+	let parsed: unknown
 	try {
-		return JSON.parse(trimmed) as ExecEvent
-	} catch (err) {
-		if (err instanceof SyntaxError) return undefined
-		throw err
+		parsed = JSON.parse(trimmed)
+	} catch (error) {
+		throw new RemoteProtocolError(
+			`agent emitted malformed NDJSON: ${error instanceof Error ? error.message : String(error)}`,
+		)
+	}
+	if (!parsed || typeof parsed !== 'object') {
+		throw new RemoteProtocolError('agent emitted an event without an object body')
+	}
+	const event = parsed as Record<string, unknown>
+	if (
+		(event.type === 'stdout_delta' || event.type === 'stderr_delta') &&
+		typeof event.data === 'string'
+	) {
+		return event as ExecEvent
+	}
+	if (event.type === 'error' && typeof event.error === 'string') return event as ExecEvent
+	if (
+		event.type === 'result' &&
+		Number.isFinite(event.exitCode) &&
+		typeof event.timedOut === 'boolean' &&
+		Number.isFinite(event.durationMs) &&
+		(event.signal === undefined || typeof event.signal === 'string') &&
+		(event.stdoutTruncated === undefined || typeof event.stdoutTruncated === 'boolean') &&
+		(event.stderrTruncated === undefined || typeof event.stderrTruncated === 'boolean')
+	) {
+		return event as ExecEvent
 	}
+	throw new RemoteProtocolError(`agent emitted an invalid ${String(event.type)} event`)
 }