@namzu/sandbox 6.1.0 → 7.0.0

package/src/backends/firecracker/transport.ts

@@ -57,6 +57,7 @@ import net from 'node:net'
 import tls from 'node:tls'
 
 import type { SandboxExecResult } from '@namzu/sdk'
+import { OperationDeadline, OperationDeadlineExpired } from '../readiness.js'
 import {
 	type ExecRequest,
 	ExecResultAccumulator,
@@ -256,16 +257,18 @@ export class VsockAgentTransport {
 	 * failures (ECONNREFUSED while the agent re-listens after a resume,
 	 * a dropped CONNECT ack) until the budget is exhausted.
 	 */
-	private async dial(): Promise<net.Socket> {
+	private async dial(signal?: AbortSignal): Promise<net.Socket> {
 		const deadline = Date.now() + this.connectRetryBudgetMs
 		let lastErr: unknown
 		for (;;) {
+			signal?.throwIfAborted()
 			try {
-				return await this.connectOnce()
+				return await this.connectOnce(signal)
 			} catch (err) {
+				if (signal?.aborted) throw signal.reason
 				lastErr = err
 				if (Date.now() >= deadline) break
-				await delay(this.connectRetryIntervalMs)
+				await delay(this.connectRetryIntervalMs, signal)
 			}
 		}
 		throw new Error(
@@ -276,36 +279,41 @@ export class VsockAgentTransport {
 		)
 	}
 
-	private connectOnce(): Promise<net.Socket> {
+	private connectOnce(signal?: AbortSignal): Promise<net.Socket> {
 		const handle = this.handle
-		if (handle.kind === 'mtls') return this.connectOnceMtls(handle)
+		if (handle.kind === 'mtls') return this.connectOnceMtls(handle, signal)
 		return new Promise<net.Socket>((resolve, reject) => {
 			const path = handle.kind === 'unix' ? handle.path : handle.udsPath
 			const socket = net.connect({ path })
 			let settled = false
-			const timer = setTimeout(() => {
-				if (settled) return
-				settled = true
-				socket.destroy()
-				reject(new Error(`connect/handshake timed out after ${this.connectTimeoutMs}ms`))
-			}, this.connectTimeoutMs)
-			timer.unref()
-
 			const fail = (err: Error) => {
 				if (settled) return
 				settled = true
 				clearTimeout(timer)
+				signal?.removeEventListener('abort', abort)
 				socket.destroy()
 				reject(err)
 			}
+			const abort = () => fail(signalError(signal))
+			const timer = setTimeout(
+				() => fail(new Error(`connect/handshake timed out after ${this.connectTimeoutMs}ms`)),
+				this.connectTimeoutMs,
+			)
+			timer.unref()
 
 			socket.once('error', fail)
+			if (signal?.aborted) {
+				abort()
+				return
+			}
+			signal?.addEventListener('abort', abort, { once: true })
 
 			socket.once('connect', () => {
 				if (handle.kind === 'unix') {
 					if (settled) return
 					settled = true
 					clearTimeout(timer)
+					signal?.removeEventListener('abort', abort)
 					socket.removeListener('error', fail)
 					resolve(socket)
 					return
@@ -327,6 +335,7 @@ export class VsockAgentTransport {
 					if (settled) return
 					settled = true
 					clearTimeout(timer)
+					signal?.removeEventListener('abort', abort)
 					socket.removeListener('error', fail)
 					// Any bytes the ackReader over-read after the ack line are
 					// application framing; replay them into the caller.
@@ -363,6 +372,7 @@ export class VsockAgentTransport {
 	 */
 	private connectOnceMtls(
 		handle: Extract<SandboxAgentHandle, { kind: 'mtls' }>,
+		signal?: AbortSignal,
 	): Promise<net.Socket> {
 		return new Promise<net.Socket>((resolve, reject) => {
 			const socket = tls.connect({
@@ -376,23 +386,27 @@ export class VsockAgentTransport {
 				minVersion: 'TLSv1.3',
 			})
 			let settled = false
-			const timer = setTimeout(() => {
-				if (settled) return
-				settled = true
-				socket.destroy()
-				reject(new Error(`connect/handshake timed out after ${this.connectTimeoutMs}ms`))
-			}, this.connectTimeoutMs)
-			timer.unref()
-
 			const fail = (err: Error) => {
 				if (settled) return
 				settled = true
 				clearTimeout(timer)
+				signal?.removeEventListener('abort', abort)
 				socket.destroy()
 				reject(err)
 			}
+			const abort = () => fail(signalError(signal))
+			const timer = setTimeout(
+				() => fail(new Error(`connect/handshake timed out after ${this.connectTimeoutMs}ms`)),
+				this.connectTimeoutMs,
+			)
+			timer.unref()
 
 			socket.once('error', fail)
+			if (signal?.aborted) {
+				abort()
+				return
+			}
+			signal?.addEventListener('abort', abort, { once: true })
 
 			// `secureConnect` fires only after the cert chain is verified
 			// (rejectUnauthorized rejects a bad/missing-CA server via 'error'
@@ -411,6 +425,7 @@ export class VsockAgentTransport {
 				}
 				settled = true
 				clearTimeout(timer)
+				signal?.removeEventListener('abort', abort)
 				socket.removeListener('error', fail)
 				// Routing preamble — the host-relay analogue of the vsock
 				// `CONNECT <port>` line. The relay consumes it, resolves the
@@ -427,25 +442,24 @@ export class VsockAgentTransport {
 	 * healthz). Applies the read-idle timeout so a post-resume hung read
 	 * is torn down rather than wedging the caller.
 	 */
-	async request<T>(req: AgentRequest): Promise<T> {
-		const socket = await this.dial()
+	async request<T>(req: AgentRequest, signal?: AbortSignal): Promise<T> {
+		const socket = await this.dial(signal)
 		return await new Promise<T>((resolve, reject) => {
 			const reader = new FrameReader()
 			let settled = false
-			const idle = new IdleTimer(this.readIdleTimeoutMs, () => {
-				if (settled) return
-				settled = true
-				socket.destroy()
-				reject(new Error(`vsock transport: read idle timeout after ${this.readIdleTimeoutMs}ms`))
-			})
 			const finish = (err: Error | null, value?: T) => {
 				if (settled) return
 				settled = true
 				idle.clear()
+				signal?.removeEventListener('abort', abort)
 				socket.destroy()
 				if (err) reject(err)
 				else resolve(value as T)
 			}
+			const abort = () => finish(signalError(signal))
+			const idle = new IdleTimer(this.readIdleTimeoutMs, () =>
+				finish(new Error(`vsock transport: read idle timeout after ${this.readIdleTimeoutMs}ms`)),
+			)
 			socket.on('data', (chunk: Buffer) => {
 				idle.bump()
 				let frames: string[]
@@ -466,6 +480,11 @@ export class VsockAgentTransport {
 			})
 			socket.once('error', (err) => finish(err))
 			socket.once('close', () => finish(new Error('vsock transport: socket closed before reply')))
+			if (signal?.aborted) {
+				abort()
+				return
+			}
+			signal?.addEventListener('abort', abort, { once: true })
 			idle.bump()
 			socket.write(frame(JSON.stringify(req)))
 		})
@@ -547,11 +566,12 @@ export class VsockAgentTransport {
 	}
 
 	/** Liveness probe. Returns true on an `{ ok: true }` healthz reply. */
-	async healthz(): Promise<boolean> {
+	async healthz(signal?: AbortSignal): Promise<boolean> {
 		try {
-			const res = await this.request<{ ok?: boolean }>({ op: 'healthz' })
+			const res = await this.request<{ ok?: boolean }>({ op: 'healthz' }, signal)
 			return res.ok === true
 		} catch {
+			if (signal?.aborted) throw signal.reason
 			return false
 		}
 	}
@@ -562,17 +582,27 @@ export class VsockAgentTransport {
 	 * (which already carries connect retry) — used by the backend's
 	 * post-create readiness fence.
 	 */
-	async waitForReady(timeoutMs: number, pollIntervalMs: number): Promise<void> {
-		const deadline = Date.now() + timeoutMs
+	async waitForReady(
+		timeoutMs: number,
+		pollIntervalMs: number,
+		signal?: AbortSignal,
+	): Promise<void> {
+		const deadline = new OperationDeadline(timeoutMs, 'firecracker agent readiness', signal)
 		let lastErr: unknown
-		while (Date.now() < deadline) {
+		while (deadline.remainingMs() > 0) {
 			try {
-				if (await this.healthz()) return
+				if (await deadline.run((signal) => this.healthz(signal))) return
 				lastErr = new Error('healthz returned not-ok')
 			} catch (err) {
 				lastErr = err
+				if (err instanceof OperationDeadlineExpired) break
+			}
+			try {
+				await deadline.delay(pollIntervalMs)
+			} catch (err) {
+				if (err instanceof OperationDeadlineExpired) break
+				throw err
 			}
-			await delay(pollIntervalMs)
 		}
 		throw new Error(
 			`vsock transport: agent did not become ready within ${timeoutMs}ms: ${
@@ -647,8 +677,27 @@ class IdleTimer {
 	}
 }
 
-function delay(ms: number): Promise<void> {
-	return new Promise((resolve) => setTimeout(resolve, ms))
+function delay(ms: number, signal?: AbortSignal): Promise<void> {
+	return new Promise((resolve, reject) => {
+		if (signal?.aborted) {
+			reject(signalError(signal))
+			return
+		}
+		const finish = (err?: unknown) => {
+			clearTimeout(timer)
+			signal?.removeEventListener('abort', abort)
+			if (err === undefined) resolve()
+			else reject(err)
+		}
+		const abort = () => finish(signalError(signal))
+		const timer = setTimeout(() => finish(), ms)
+		signal?.addEventListener('abort', abort, { once: true })
+	})
+}
+
+function signalError(signal: AbortSignal | undefined): Error {
+	if (signal?.reason instanceof Error) return signal.reason
+	return new Error(signal?.reason === undefined ? 'operation aborted' : String(signal.reason))
 }
 
 function describeHandle(handle: SandboxAgentHandle): string {

package/src/backends/readiness.ts

@@ -0,0 +1,173 @@
+/**
+ * One owner for backend readiness clocks.
+ *
+ * A timestamp checked between retries is not a deadline when the thing inside
+ * a retry can remain pending. This owner races every foreign promise against
+ * the remaining operation budget and supplies a private signal so cooperative
+ * transports can release their sockets too.
+ */
+
+const MAX_NODE_TIMER_MS = 2_147_483_647
+
+/**
+ * Readiness failure teardown is deliberately a second, short budget. The
+ * health deadline has already expired, so pretending cleanup fits inside it
+ * either skips cleanup entirely or keeps `create()` pending without a bound.
+ */
+export const FAILURE_CLEANUP_GRACE_MS = 1_000
+
+export interface ReadinessOptions {
+	readonly timeoutMs: number
+	readonly pollIntervalMs: number
+}
+
+export class OperationDeadlineExpired extends Error {
+	constructor(readonly label: string) {
+		super(`${label} deadline expired`)
+		this.name = 'OperationDeadlineExpired'
+	}
+}
+
+function assertTimerValue(field: string, value: number): void {
+	if (!Number.isSafeInteger(value) || value <= 0 || value > MAX_NODE_TIMER_MS) {
+		throw new RangeError(
+			`${field} must be a positive safe integer no greater than ${MAX_NODE_TIMER_MS}; received ${String(value)}`,
+		)
+	}
+}
+
+export function resolveReadinessOptions(
+	owner: string,
+	timeoutMs: number | undefined,
+	pollIntervalMs: number | undefined,
+	defaults: ReadinessOptions,
+): ReadinessOptions {
+	const resolved = {
+		timeoutMs: timeoutMs ?? defaults.timeoutMs,
+		pollIntervalMs: pollIntervalMs ?? defaults.pollIntervalMs,
+	}
+	assertTimerValue(`${owner}.readyTimeoutMs`, resolved.timeoutMs)
+	assertTimerValue(`${owner}.readyPollIntervalMs`, resolved.pollIntervalMs)
+	return resolved
+}
+
+export class OperationDeadline {
+	private readonly expiresAt: number
+
+	constructor(
+		timeoutMs: number,
+		readonly label: string,
+		private readonly callerSignal?: AbortSignal,
+	) {
+		assertTimerValue(`${label} timeout`, timeoutMs)
+		this.expiresAt = performance.now() + timeoutMs
+	}
+
+	remainingMs(): number {
+		return Math.max(0, this.expiresAt - performance.now())
+	}
+
+	async run<T>(operation: (signal: AbortSignal) => Promise<T>): Promise<T> {
+		this.callerSignal?.throwIfAborted()
+		const remaining = this.remainingMs()
+		if (remaining <= 0) throw new OperationDeadlineExpired(this.label)
+
+		const controller = new AbortController()
+		const expired = new OperationDeadlineExpired(this.label)
+		let timer: ReturnType<typeof setTimeout> | undefined
+		let onCallerAbort: (() => void) | undefined
+		const cancelled = this.callerSignal
+			? new Promise<never>((_resolve, reject) => {
+					onCallerAbort = () => {
+						const reason = this.callerSignal?.reason
+						controller.abort(reason)
+						reject(reason)
+					}
+					this.callerSignal?.addEventListener('abort', onCallerAbort, {
+						once: true,
+					})
+				})
+			: undefined
+		const expiry = new Promise<never>((_resolve, reject) => {
+			timer = setTimeout(() => {
+				controller.abort(expired)
+				reject(expired)
+			}, remaining)
+		})
+		const pending = Promise.resolve().then(() => operation(controller.signal))
+
+		try {
+			const value = await Promise.race(cancelled ? [pending, expiry, cancelled] : [pending, expiry])
+			// Publication fence: a foreign promise that settles at the boundary
+			// does not get to turn an already-expired attempt into readiness.
+			if (this.callerSignal?.aborted) throw this.callerSignal.reason
+			if (controller.signal.aborted || this.remainingMs() <= 0) {
+				controller.abort(expired)
+				throw expired
+			}
+			return value
+		} finally {
+			if (timer !== undefined) clearTimeout(timer)
+			if (onCallerAbort) this.callerSignal?.removeEventListener('abort', onCallerAbort)
+		}
+	}
+
+	async delay(maximumMs: number): Promise<void> {
+		await this.run(
+			(signal) =>
+				new Promise<void>((resolve, reject) => {
+					if (signal.aborted) {
+						reject(signal.reason)
+						return
+					}
+					const finish = (err?: unknown) => {
+						clearTimeout(timer)
+						signal.removeEventListener('abort', abort)
+						if (err === undefined) resolve()
+						else reject(err)
+					}
+					const abort = () => {
+						finish(signal.reason)
+					}
+					const timer = setTimeout(() => finish(), maximumMs)
+					signal.addEventListener('abort', abort, { once: true })
+				}),
+		)
+	}
+}
+
+export async function probeHttpHealth(
+	url: string,
+	signal: AbortSignal,
+): Promise<{ readonly ok: boolean; readonly status: number }> {
+	signal.throwIfAborted()
+	const response = await fetch(url, { signal })
+	const result = { ok: response.ok, status: response.status }
+	try {
+		await response.body?.cancel()
+	} catch {
+		// The status is already known. A body cancellation failure must not
+		// hide it; the owning deadline still aborts the transport if needed.
+	}
+	signal.throwIfAborted()
+	return result
+}
+
+/**
+ * Failure cleanup must never replace or indefinitely delay the primary
+ * readiness error. Cooperative cleanup observes the signal; an implementation
+ * that ignores it is still detached from the caller by the independent race.
+ */
+export async function runFailureCleanup(
+	cleanup: (signal: AbortSignal) => Promise<void>,
+	graceMs = FAILURE_CLEANUP_GRACE_MS,
+): Promise<void> {
+	const deadline = new OperationDeadline(graceMs, 'sandbox failure cleanup')
+	try {
+		await deadline.run(cleanup)
+	} catch {
+		// The readiness failure remains primary. Promise.race installed a
+		// rejection observer on the losing cleanup promise, so a late failure
+		// cannot become an unhandled rejection.
+	}
+}

package/src/egress/proxy.ts

@@ -121,6 +121,11 @@ export class EgressProxy {
 		const server = createServer((req, res) => {
 			void this.handleRequest(req, res)
 		})
+		const connections = new Set<Duplex>()
+		server.on('connection', (socket) => {
+			connections.add(socket)
+			socket.once('close', () => connections.delete(socket))
+		})
 		server.on('connect', (req, socket, head) => {
 			void this.handleConnect(req, socket, head)
 		})
@@ -144,6 +149,7 @@ export class EgressProxy {
 		// process runs out. Found by a test that hung rather than failed,
 		// which is the shape this failure takes in production too.
 		this.selfPort = boundPort
+		let closePromise: Promise<void> | undefined
 
 		return {
 			port: boundPort,
@@ -151,10 +157,16 @@ export class EgressProxy {
 			setAllowedHosts: (resolve) => {
 				this.resolveAllowed = resolve
 			},
-			close: () =>
-				new Promise<void>((resolve) => {
+			close: () => {
+				closePromise ??= new Promise<void>((resolve) => {
+					// `server.close()` waits for existing connections. A readiness
+					// failure has no returned Sandbox handle through which those
+					// sockets can later be retired, so teardown owns them here.
+					for (const socket of connections) socket.destroy()
 					server.close(() => resolve())
-				}),
+				})
+				return closePromise
+			},
 		}
 	}
 
@@ -346,6 +358,9 @@ export class EgressProxy {
 		socket.on('error', () => {
 			upstream.destroy()
 		})
+		socket.once('close', () => {
+			upstream.destroy()
+		})
 	}
 
 	/** Whether a target names this proxy. See the loop guard in `listen`. */

package/src/index.ts

@@ -155,7 +155,9 @@ export interface ACIStandbyPoolBackendConfig {
 	 */
 	readonly getArmToken: () => Promise<string>
 	readonly subnetId?: string
+	/** Delay between IP / health probes. Default 500ms. */
 	readonly readyPollIntervalMs?: number
+	/** Total deadline across IP publication and worker health. Default 60000ms. */
 	readonly readyTimeoutMs?: number
 	readonly workerPort?: number
 	readonly armApiVersion?: string
@@ -218,6 +220,18 @@ export interface ContainerBackendConfig {
 	 * which is checked against this network rather than trusted.
 	 */
 	readonly network?: 'none' | 'bridge' | string
+	/**
+	 * Maximum time spent waiting for the container worker's `/healthz`
+	 * readiness probe. Must be a positive integer within Node's timer range.
+	 * Default 30000ms.
+	 */
+	readonly readyTimeoutMs?: number
+	/**
+	 * Delay between worker readiness probes. Must be a positive integer within
+	 * Node's timer range; the final delay is capped by `readyTimeoutMs`.
+	 * Default 100ms.
+	 */
+	readonly readyPollIntervalMs?: number
 	/**
 	 * Allowlisted hosts permitted to resolve to an inward address anyway.
 	 *
@@ -297,7 +311,9 @@ export type MicroVMBackendConfig = {
 	readonly agentSnapshot?: AgentSnapshotRef
 	/** Fixed guest AF_VSOCK port the in-VM agent listens on. */
 	readonly agentVsockPort?: number
+	/** Total guest-agent health deadline after the orchestrator claim. Default 60000ms. */
 	readonly readyTimeoutMs?: number
+	/** Delay between guest-agent health probes. Default 250ms. */
 	readonly readyPollIntervalMs?: number
 	/**
 	 * NETWORK-mode mTLS client material (ses_051 P4 client-proxy
@@ -453,6 +469,8 @@ export interface SandboxBackend {
  * shape.
  */
 export interface SandboxBackendOptions {
+	/** Run authority for allocation/readiness. See `SandboxCreateConfig.signal`. */
+	readonly signal?: AbortSignal
 	readonly workingDirectory: string
 	readonly egress?: EgressPolicy
 	readonly timeoutMs?: number
@@ -531,6 +549,7 @@ export function createSandboxProvider(config: SandboxProviderConfig): SandboxPro
 		environment: 'basic',
 		async create(perCall?: SandboxCreateConfig): Promise<Sandbox> {
 			return await backend.create({
+				...(perCall?.signal !== undefined ? { signal: perCall.signal } : {}),
 				workingDirectory: perCall?.workingDirectory ?? '/workspace',
 				...(config.defaultEgress !== undefined ? { egress: config.defaultEgress } : {}),
 				...(perCall?.timeoutMs !== undefined
@@ -569,6 +588,10 @@ function pickBackend(config: SandboxProviderConfig): SandboxBackend {
 		return buildDockerBackend({
 			image: backend.image,
 			layout: resolved,
+			...(backend.readyTimeoutMs !== undefined ? { readyTimeoutMs: backend.readyTimeoutMs } : {}),
+			...(backend.readyPollIntervalMs !== undefined
+				? { readyPollIntervalMs: backend.readyPollIntervalMs }
+				: {}),
 			...(backend.hostReachability !== undefined
 				? { hostReachability: backend.hostReachability }
 				: {}),
@@ -620,6 +643,10 @@ function pickBackend(config: SandboxProviderConfig): SandboxBackend {
 			image: backend.image,
 			layout: resolved,
 			runtime: 'runsc',
+			...(backend.readyTimeoutMs !== undefined ? { readyTimeoutMs: backend.readyTimeoutMs } : {}),
+			...(backend.readyPollIntervalMs !== undefined
+				? { readyPollIntervalMs: backend.readyPollIntervalMs }
+				: {}),
 			...(backend.hostReachability !== undefined
 				? { hostReachability: backend.hostReachability }
 				: {}),