@namzu/sandbox 7.0.0 → 7.2.0

package/src/backends/firecracker/index.ts

@@ -58,7 +58,8 @@ import type {
 } from '@namzu/sdk'
 
 import type { AgentSnapshotRef, SandboxBackend, SandboxBackendOptions } from '../../index.js'
-import { resolveReadinessOptions, runFailureCleanup } from '../readiness.js'
+import { OperationDeadline, resolveReadinessOptions, runFailureCleanup } from '../readiness.js'
+import { RemoteCancellationUnknownError } from '../remote-execution-controller.js'
 import type {
 	MtlsClientMaterial,
 	SandboxAgentHandle,
@@ -143,6 +144,7 @@ export interface FirecrackerBackendInternalConfig {
 const DEFAULT_AGENT_VSOCK_PORT = 1024
 const DEFAULT_READY_TIMEOUT_MS = 60_000
 const DEFAULT_READY_POLL_MS = 250
+const RETIREMENT_TIMEOUT_MS = 5_000
 
 /**
  * Build a {@link SandboxBackend} backed by the owned Firecracker
@@ -220,6 +222,7 @@ async function orchestratorCall<T>(
 	body?: unknown,
 	mtls?: MtlsClientMaterial,
 	signal?: AbortSignal,
+	acceptedStatuses: readonly number[] = [],
 ): Promise<T | undefined> {
 	signal?.throwIfAborted()
 	const token = await getToken()
@@ -252,6 +255,11 @@ async function orchestratorCall<T>(
 		)
 	}
 	if (res.status < 200 || res.status >= 300) {
+		if (acceptedStatuses.includes(res.status)) {
+			await res.text()
+			signal?.throwIfAborted()
+			return undefined
+		}
 		const text = await res.text()
 		signal?.throwIfAborted()
 		throw new Error(`firecracker orchestrator ${method} ${url} → ${res.status}: ${text}`)
@@ -411,7 +419,10 @@ async function spawnFirecrackerSandbox(
 	)
 	const transport = new VsockAgentTransport(handle, config.transport ?? {})
 
-	const destroy = async (signal?: AbortSignal): Promise<void> => {
+	const destroy = async (
+		signal?: AbortSignal,
+		acceptedStatuses: readonly number[] = [],
+	): Promise<void> => {
 		await orchestratorCall(
 			endpoint,
 			`/sandboxes/${encodeURIComponent(id)}:delete`,
@@ -420,6 +431,7 @@ async function spawnFirecrackerSandbox(
 			undefined,
 			config.controlPlaneMtls,
 			signal,
+			acceptedStatuses,
 		)
 	}
 
@@ -441,12 +453,88 @@ async function spawnFirecrackerSandbox(
 		throw err
 	}
 
-	let status: SandboxStatus = 'ready'
+	type Lifecycle = 'active' | 'retiring' | 'destroyed'
+	let activeExecutions = 0
+	let lifecycle: Lifecycle = 'active'
+	let retirementPromise: Promise<{ readonly accepted: boolean; readonly error?: Error }> | undefined
+	let teardownPromise: Promise<void> | undefined
+	let teardownComplete = false
+
+	const assertActive = (): void => {
+		if (lifecycle !== 'active') {
+			throw new Error(`Sandbox ${id} is ${lifecycle}; no new guest operation can be admitted`)
+		}
+	}
+
+	const teardownSandbox = (signal?: AbortSignal): Promise<void> => {
+		lifecycle = 'retiring'
+		if (teardownComplete) return Promise.resolve()
+		if (teardownPromise) return teardownPromise
+		const attempt = destroy(signal, [404, 410])
+		const shared = attempt.then(
+			() => {
+				teardownComplete = true
+				lifecycle = 'destroyed'
+			},
+			(error: unknown) => {
+				if (teardownPromise === shared) teardownPromise = undefined
+				throw error
+			},
+		)
+		teardownPromise = shared
+		return shared
+	}
+	const retire = (): Promise<{ readonly accepted: boolean; readonly error?: Error }> => {
+		lifecycle = 'retiring'
+		if (retirementPromise) return retirementPromise
+		const deadline = new OperationDeadline(
+			RETIREMENT_TIMEOUT_MS,
+			`firecracker sandbox ${id} retirement`,
+		)
+		retirementPromise = deadline
+			.run(async (signal) => {
+				const joinedExistingAttempt = teardownPromise !== undefined
+				try {
+					await teardownSandbox(signal)
+				} catch (error) {
+					if (!joinedExistingAttempt || signal.aborted) throw error
+					await teardownSandbox(signal)
+				}
+			})
+			.then(() => {
+				return { accepted: true as const }
+			})
+			.catch((error: unknown) => {
+				const normalized = error instanceof Error ? error : new Error(String(error))
+				return { accepted: false as const, error: normalized }
+			})
+		return retirementPromise
+	}
+
+	const classifyExecutionFailure = async (error: unknown): Promise<never> => {
+		if (error instanceof RemoteCancellationUnknownError) {
+			error.retirement = await retire()
+		}
+		throw error
+	}
+
+	const runExecution = async <T>(operation: () => Promise<T>): Promise<T> => {
+		assertActive()
+		activeExecutions += 1
+		try {
+			return await operation()
+		} catch (error) {
+			return await classifyExecutionFailure(error)
+		} finally {
+			activeExecutions = Math.max(0, activeExecutions - 1)
+		}
+	}
 
 	return {
 		id,
 		get status(): SandboxStatus {
-			return status
+			if (lifecycle !== 'active') return 'destroyed'
+			return activeExecutions > 0 ? 'busy' : 'ready'
 		},
 		rootDir,
 		environment: detectEnvironment(),
@@ -456,67 +544,51 @@ async function spawnFirecrackerSandbox(
 			argv?: string[],
 			opts?: SandboxExecOptions,
 		): Promise<SandboxExecResult> {
-			// `opts.signal` is deliberately not forwarded, and the reason is
-			// worth writing down because the obvious "fix" is worse than the
-			// gap. There is no cancel op on this wire — the guest agent takes
-			// an execute frame and answers when the command is done. Aborting
-			// the socket here would abandon the WAIT while the process keeps
-			// running inside the microVM, which is verbatim the failure
-			// `SandboxExecOptions.signal` exists to prevent, except it would
-			// then look honoured. Honouring it means a cancel op in the guest
-			// protocol; until then, ignoring it is the truthful behaviour the
-			// option's own contract allows.
-			status = 'busy'
-			try {
-				return await transport.execute({
-					command,
-					args: argv ?? [],
-					...(opts?.cwd !== undefined ? { cwd: opts.cwd } : {}),
-					...(opts?.env !== undefined ? { env: opts.env } : {}),
-					...(opts?.timeout !== undefined ? { timeoutMs: opts.timeout } : {}),
-				})
-			} finally {
-				status = 'ready'
-			}
+			return await runExecution(async () => await transport.exec(command, argv, opts))
 		},
 
 		async writeFile(path: string, content: string | Buffer): Promise<void> {
+			assertActive()
 			const buf = Buffer.isBuffer(content) ? content : Buffer.from(content, 'utf8')
 			await transport.writeFile(path, buf)
 		},
 
 		async readFile(path: string): Promise<Buffer> {
+			assertActive()
 			return await transport.readFile(path)
 		},
 
 		async listFiles(rootPath: string): Promise<readonly SandboxFileEntry[]> {
-			// Same wire as docker/aci: `find -printf '%p\t%s\n'`, parse
-			// line-by-line, map a non-zero exit (missing root) to "empty".
-			const result = await transport.execute({
-				command: 'find',
-				args: [rootPath, '-type', 'f', '-printf', '%p\t%s\n'],
+			return await runExecution(async () => {
+				// Same wire as docker/aci: `find -printf '%p\t%s\n'`, parse
+				// line-by-line, map a non-zero exit (missing root) to "empty".
+				const result = await transport.exec('find', [rootPath, '-type', 'f', '-printf', '%p\t%s\n'])
+				if (result.exitCode !== 0) return []
+				const entries: SandboxFileEntry[] = []
+				for (const rawLine of result.stdout.split('\n')) {
+					if (!rawLine) continue
+					const tab = rawLine.indexOf('\t')
+					if (tab < 0) continue
+					const filePath = rawLine.slice(0, tab)
+					const size = Number.parseInt(rawLine.slice(tab + 1), 10)
+					if (!filePath || !Number.isFinite(size)) continue
+					entries.push({ path: filePath, size })
+				}
+				return entries
 			})
-			if (result.exitCode !== 0) return []
-			const entries: SandboxFileEntry[] = []
-			for (const rawLine of result.stdout.split('\n')) {
-				if (!rawLine) continue
-				const tab = rawLine.indexOf('\t')
-				if (tab < 0) continue
-				const filePath = rawLine.slice(0, tab)
-				const size = Number.parseInt(rawLine.slice(tab + 1), 10)
-				if (!filePath || !Number.isFinite(size)) continue
-				entries.push({ path: filePath, size })
-			}
-			return entries
 		},
 
 		async destroy(options?: SandboxDestroyOptions): Promise<void> {
-			status = 'destroyed'
+			if (retirementPromise) {
+				const observation = await retirementPromise
+				if (observation.accepted) return
+				retirementPromise = undefined
+			}
 			// Let the orchestrator DELETE failure propagate — the
 			// Vandal-side lifecycle wraps this with logging, and a
 			// swallowed error here means orphaned microVMs (and their
 			// netns / UFFD handlers) pile up with no observability handle.
-			await destroy(options?.signal)
+			await teardownSandbox(options?.signal)
 		},
 	}
 }

package/src/backends/firecracker/protocol.ts

@@ -1,5 +1,5 @@
 /**
- * The ONE wire contract, shared across both transports.
+ * The execution-data codec shared by both transports.
  *
  * The docker (`backends/docker/`) and ACI (`backends/aci-standby-pool/`)
  * backends speak this contract over **HTTP**: a streaming NDJSON
@@ -9,23 +9,15 @@
  * shapes** — only the transport changes from HTTP-over-TCP to
  * framed-over-vsock (see `transport.ts`).
  *
- * This module is the single source of truth for the message shapes
- * and the streaming exec-line parser, so the two transports cannot
- * drift. It deliberately carries NO transport concern (no `fetch`, no
- * socket) — it is pure codec. The HTTP backends keep their own inline
- * copies (they predate this module and stay UNTOUCHED per the P0
- * scope); this module mirrors those shapes verbatim and is the
- * canonical definition the vsock path consumes.
- *
- * Why mirror rather than refactor docker/aci onto it: the P0 scope is
- * "do NOT touch docker or aci". Centralising the shapes here, with the
- * inline docker copy pinned by `backends/docker/__tests__` and this
- * module pinned by `backends/firecracker/__tests__`, keeps both honest
- * without editing the frozen HTTP path. A later cleanup pass can fold
- * docker/aci onto this module once it is no longer release-frozen.
+ * This module remains the pure codec consumed by the framed guest transport.
+ * The two HTTP backends share one strict HTTP client, and both transports put
+ * the same reserve-before-admission + idempotent-cancel state machine around
+ * these data events. HTTP uses endpoints; the framed guest uses dedicated ops.
  */
 
-import type { SandboxExecResult } from '@namzu/sdk'
+import type { SandboxExecOptions, SandboxExecResult } from '@namzu/sdk'
+
+import { RemoteCommandError, RemoteProtocolError } from '../remote-execution-controller.js'
 
 // ---------------------------------------------------------------------------
 // Exec — request + the NDJSON event shapes (verbatim from worker/server.js)
@@ -38,6 +30,7 @@ import type { SandboxExecResult } from '@namzu/sdk'
  * `SandboxExecOptions.timeout`.
  */
 export interface ExecRequest {
+	readonly executionId?: string
 	readonly command: string
 	readonly args?: readonly string[]
 	readonly cwd?: string
@@ -62,7 +55,8 @@ export type ExecEvent =
 			readonly type: 'result'
 			readonly exitCode: number
 			readonly timedOut: boolean
-			readonly durationMs?: number
+			readonly durationMs: number
+			readonly signal?: string
 			readonly stdoutTruncated?: boolean
 			readonly stderrTruncated?: boolean
 	  }
@@ -113,9 +107,8 @@ export interface ReadFileResponse {
  * terminal `result`, and **throw** on an `error` event.
  *
  * Transport-agnostic: feed it whole parsed {@link ExecEvent}s (the
- * transport owns newline-framing → JSON.parse → here). Malformed lines
- * are dropped by the transport's `JSON.parse` guard before they reach
- * this accumulator, matching the docker loop's `SyntaxError` swallow.
+ * transport owns framing → strict JSON validation → here). Malformed or
+ * trailing events are protocol failures rather than silently discarded data.
  */
 export class ExecResultAccumulator {
 	private stdout = ''
@@ -123,11 +116,16 @@ export class ExecResultAccumulator {
 	private exitCode = -1
 	private timedOut = false
 	private signal: string | undefined
+	private durationMs: number | undefined
+	private stdoutTruncated: boolean | undefined
+	private stderrTruncated: boolean | undefined
 	private settled = false
 	private readonly start: number
+	private readonly onOutput: SandboxExecOptions['onOutput']
 
-	constructor(start: number = Date.now()) {
+	constructor(start: number = Date.now(), onOutput?: SandboxExecOptions['onOutput']) {
 		this.start = start
+		this.onOutput = onOutput
 	}
 
 	/**
@@ -137,55 +135,91 @@ export class ExecResultAccumulator {
 	 * docker loop uses (`throw new Error(event.error)`).
 	 */
 	push(event: ExecEvent): boolean {
+		if (this.settled) {
+			throw new RemoteProtocolError('exec stream emitted data after its terminal event')
+		}
 		if (event.type === 'stdout_delta') {
 			this.stdout += event.data
+			this.onOutput?.({ stream: 'stdout', data: event.data })
 			return false
 		}
 		if (event.type === 'stderr_delta') {
 			this.stderr += event.data
+			this.onOutput?.({ stream: 'stderr', data: event.data })
 			return false
 		}
 		if (event.type === 'result') {
 			this.exitCode = event.exitCode
 			this.timedOut = event.timedOut
+			this.durationMs = event.durationMs
+			this.signal = event.signal
+			this.stdoutTruncated = event.stdoutTruncated
+			this.stderrTruncated = event.stderrTruncated
 			this.settled = true
 			return true
 		}
 		// event.type === 'error'
-		throw new Error(event.error)
+		throw new RemoteCommandError(event.error)
 	}
 
 	get done(): boolean {
 		return this.settled
 	}
 
-	/** Build the SDK-shaped result. `durationMs` measured host-side. */
+	/** Build the SDK-shaped result from the guest's terminal metadata. */
 	finish(): SandboxExecResult {
+		if (!this.settled || this.durationMs === undefined) {
+			throw new RemoteProtocolError('exec stream ended without exactly one result event')
+		}
 		return {
 			exitCode: this.exitCode,
 			stdout: this.stdout,
 			stderr: this.stderr,
 			...(this.signal ? { signal: this.signal } : {}),
 			timedOut: this.timedOut,
-			durationMs: Date.now() - this.start,
+			durationMs: this.durationMs ?? Date.now() - this.start,
+			...(this.stdoutTruncated !== undefined ? { stdoutTruncated: this.stdoutTruncated } : {}),
+			...(this.stderrTruncated !== undefined ? { stderrTruncated: this.stderrTruncated } : {}),
 		}
 	}
 }
 
 /**
- * Parse a single NDJSON line into an {@link ExecEvent}, or `undefined`
- * if the line is blank or not valid JSON (the docker loop's
- * `SyntaxError` swallow). Non-`SyntaxError` problems are impossible
- * here because we only `JSON.parse`; structural validation is by the
- * `type` discriminator at the call site.
+ * Parse and structurally validate a single NDJSON event. Blank padding is
+ * ignored; malformed JSON and unknown/partial event shapes are refused.
  */
 export function parseExecLine(line: string): ExecEvent | undefined {
 	const trimmed = line.trim()
 	if (!trimmed) return undefined
+	let parsed: unknown
 	try {
-		return JSON.parse(trimmed) as ExecEvent
-	} catch (err) {
-		if (err instanceof SyntaxError) return undefined
-		throw err
+		parsed = JSON.parse(trimmed)
+	} catch (error) {
+		throw new RemoteProtocolError(
+			`agent emitted malformed NDJSON: ${error instanceof Error ? error.message : String(error)}`,
+		)
+	}
+	if (!parsed || typeof parsed !== 'object') {
+		throw new RemoteProtocolError('agent emitted an event without an object body')
+	}
+	const event = parsed as Record<string, unknown>
+	if (
+		(event.type === 'stdout_delta' || event.type === 'stderr_delta') &&
+		typeof event.data === 'string'
+	) {
+		return event as ExecEvent
+	}
+	if (event.type === 'error' && typeof event.error === 'string') return event as ExecEvent
+	if (
+		event.type === 'result' &&
+		Number.isFinite(event.exitCode) &&
+		typeof event.timedOut === 'boolean' &&
+		Number.isFinite(event.durationMs) &&
+		(event.signal === undefined || typeof event.signal === 'string') &&
+		(event.stdoutTruncated === undefined || typeof event.stdoutTruncated === 'boolean') &&
+		(event.stderrTruncated === undefined || typeof event.stderrTruncated === 'boolean')
+	) {
+		return event as ExecEvent
 	}
+	throw new RemoteProtocolError(`agent emitted an invalid ${String(event.type)} event`)
 }