@namzu/sandbox 6.1.0 → 7.1.0

package/src/backends/aci-standby-pool/index.ts

@@ -44,6 +44,7 @@ import type {
 	ContainerSandboxMountSource,
 	ResolvedContainerSandboxLayout,
 	Sandbox,
+	SandboxDestroyOptions,
 	SandboxEnvironment,
 	SandboxExecOptions,
 	SandboxExecResult,
@@ -54,6 +55,15 @@ import type {
 import { asSandboxId } from '@namzu/sdk'
 
 import type { SandboxBackend, SandboxBackendOptions } from '../../index.js'
+import { HttpWorkerClient } from '../http-worker-client.js'
+import {
+	OperationDeadline,
+	OperationDeadlineExpired,
+	probeHttpHealth,
+	resolveReadinessOptions,
+	runFailureCleanup,
+} from '../readiness.js'
+import { RemoteCancellationUnknownError } from '../remote-execution-controller.js'
 
 /**
  * Authentication callback. Caller returns a fresh Azure Resource
@@ -147,11 +157,20 @@ const DEFAULT_CONTAINER_NAME_PREFIX = 'namzu-task'
 export function buildAciStandbyPoolBackend(
 	config: ACIStandbyPoolBackendInternalConfig,
 ): SandboxBackend {
+	const readiness = resolveReadinessOptions(
+		'aci-standby-pool',
+		config.readyTimeoutMs,
+		config.readyPollIntervalMs,
+		{
+			timeoutMs: DEFAULT_READY_TIMEOUT_MS,
+			pollIntervalMs: DEFAULT_READY_POLL_MS,
+		},
+	)
 	return {
 		tier: 'container',
 		name: 'aci-standby-pool',
 		async create(options: SandboxBackendOptions): Promise<Sandbox> {
-			return await spawnAciSandbox(config, options)
+			return await spawnAciSandbox(config, options, readiness)
 		},
 	}
 }
@@ -219,7 +238,10 @@ function buildAzureFileVolumesFromLayout(layout: ResolvedContainerSandboxLayout)
 	let counter = 0
 
 	function add(
-		mount: { readonly source: ContainerSandboxMountSource; readonly containerPath: string },
+		mount: {
+			readonly source: ContainerSandboxMountSource
+			readonly containerPath: string
+		},
 		label: string,
 		readOnly: boolean,
 	): void {
@@ -280,8 +302,12 @@ async function armCall<T>(
 	method: 'GET' | 'PUT' | 'DELETE',
 	getToken: ArmTokenProvider,
 	body?: unknown,
+	signal?: AbortSignal,
+	acceptedStatuses: readonly number[] = [],
 ): Promise<T | undefined> {
+	signal?.throwIfAborted()
 	const token = await getToken()
+	signal?.throwIfAborted()
 	const init: RequestInit = {
 		method,
 		headers: {
@@ -292,15 +318,25 @@ async function armCall<T>(
 	if (body !== undefined) {
 		init.body = JSON.stringify(body)
 	}
+	if (signal !== undefined) init.signal = signal
 	const res = await fetch(url, init)
+	signal?.throwIfAborted()
 	if (!res.ok) {
+		if (acceptedStatuses.includes(res.status)) {
+			await res.text()
+			signal?.throwIfAborted()
+			return undefined
+		}
 		const text = await res.text()
+		signal?.throwIfAborted()
 		throw new Error(`ARM ${method} ${url} → ${res.status}: ${text}`)
 	}
 	if (res.status === 204 || res.status === 202) return undefined
 	const ct = res.headers.get('content-type') ?? ''
 	if (ct.includes('application/json')) {
-		return (await res.json()) as T
+		const json = (await res.json()) as T
+		signal?.throwIfAborted()
+		return json
 	}
 	return undefined
 }
@@ -389,7 +425,9 @@ export function assertNotPubliclyAddressed(config: {
 async function spawnAciSandbox(
 	config: ACIStandbyPoolBackendInternalConfig,
 	options: SandboxBackendOptions,
+	readiness: { readonly timeoutMs: number; readonly pollIntervalMs: number },
 ): Promise<Sandbox> {
+	options.signal?.throwIfAborted()
 	assertEnforceable(options)
 	assertNotPubliclyAddressed(config)
 	const id = generateSandboxId()
@@ -439,8 +477,23 @@ async function spawnAciSandbox(
 
 	let claimed: ArmContainerGroup | undefined
 	try {
-		claimed = await armCall<ArmContainerGroup>(armUrl, 'PUT', config.getArmToken, body)
+		claimed = await armCall<ArmContainerGroup>(
+			armUrl,
+			'PUT',
+			config.getArmToken,
+			body,
+			options.signal,
+		)
 	} catch (err) {
+		// The resource name is client-owned even when ARM never returns the
+		// claim response. Best-effort reconciliation is therefore possible.
+		// ARM still owns the create/delete ordering semantics; a resource that
+		// commits after this DELETE needs the deployment's normal resource reaper.
+		if (options.signal?.aborted) {
+			await runFailureCleanup(async (signal) => {
+				await armCall(armUrl, 'DELETE', config.getArmToken, undefined, signal)
+			})
+		}
 		throw new Error(
 			`aci-standby-pool: failed to claim from pool — ${err instanceof Error ? err.message : String(err)}`,
 		)
@@ -448,30 +501,116 @@ async function spawnAciSandbox(
 
 	const initialIp = claimed?.properties?.ipAddress?.ip
 	let ip = initialIp
+	const readinessDeadline = new OperationDeadline(
+		readiness.timeoutMs,
+		'aci-standby-pool readiness',
+		options.signal,
+	)
 	try {
 		if (!ip) {
 			ip = await pollForRunningIp(
 				armUrl,
 				config.getArmToken,
-				config.readyPollIntervalMs ?? DEFAULT_READY_POLL_MS,
-				config.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS,
+				readinessDeadline,
+				readiness.pollIntervalMs,
+				readiness.timeoutMs,
 			)
 		}
 
 		const baseUrl = `http://${ip}:${workerPort}`
 		await waitForWorkerReady(
 			baseUrl,
-			config.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS,
-			config.readyPollIntervalMs ?? DEFAULT_READY_POLL_MS,
+			readinessDeadline,
+			readiness.timeoutMs,
+			readiness.pollIntervalMs,
 		)
 
-		let status: SandboxStatus = 'ready'
+		type Lifecycle = 'active' | 'retiring' | 'destroyed'
+		let activeExecutions = 0
+		let lifecycle: Lifecycle = 'active'
+		let retirementPromise:
+			| Promise<{ readonly accepted: boolean; readonly error?: Error }>
+			| undefined
+		let teardownPromise: Promise<void> | undefined
+		let teardownComplete = false
 		const rootDir = config.layout.outputs.containerPath
+		const workerClient = new HttpWorkerClient(baseUrl)
+		const assertActive = (): void => {
+			if (lifecycle !== 'active') {
+				throw new Error(`Sandbox ${id} is ${lifecycle}; no new worker operation can be admitted`)
+			}
+		}
+		const teardownSandbox = (signal?: AbortSignal): Promise<void> => {
+			lifecycle = 'retiring'
+			if (teardownComplete) return Promise.resolve()
+			if (teardownPromise) return teardownPromise
+			const attempt = armCall(
+				armUrl,
+				'DELETE',
+				config.getArmToken,
+				undefined,
+				signal,
+				[404, 410],
+			).then(() => undefined)
+			const shared = attempt.then(
+				() => {
+					teardownComplete = true
+					lifecycle = 'destroyed'
+				},
+				(error: unknown) => {
+					if (teardownPromise === shared) teardownPromise = undefined
+					throw error
+				},
+			)
+			teardownPromise = shared
+			return shared
+		}
+		const retire = (): Promise<{
+			readonly accepted: boolean
+			readonly error?: Error
+		}> => {
+			lifecycle = 'retiring'
+			if (retirementPromise) return retirementPromise
+			const deadline = new OperationDeadline(5_000, `ACI sandbox ${id} retirement`)
+			retirementPromise = deadline
+				.run(async (signal) => {
+					const joinedExistingAttempt = teardownPromise !== undefined
+					try {
+						await teardownSandbox(signal)
+					} catch (error) {
+						if (!joinedExistingAttempt || signal.aborted) throw error
+						await teardownSandbox(signal)
+					}
+				})
+				.then(() => {
+					return { accepted: true as const }
+				})
+				.catch((error: unknown) => ({
+					accepted: false as const,
+					error: error instanceof Error ? error : new Error(String(error)),
+				}))
+			return retirementPromise
+		}
+		const runExecution = async <T>(operation: () => Promise<T>): Promise<T> => {
+			assertActive()
+			activeExecutions += 1
+			try {
+				return await operation()
+			} catch (error) {
+				if (error instanceof RemoteCancellationUnknownError) {
+					error.retirement = await retire()
+				}
+				throw error
+			} finally {
+				activeExecutions = Math.max(0, activeExecutions - 1)
+			}
+		}
 
 		return {
 			id,
 			get status(): SandboxStatus {
-				return status
+				if (lifecycle !== 'active') return 'destroyed'
+				return activeExecutions > 0 ? 'busy' : 'ready'
 			},
 			rootDir,
 			environment: detectEnvironment(),
@@ -481,15 +620,11 @@ async function spawnAciSandbox(
 				argv?: string[],
 				opts?: SandboxExecOptions,
 			): Promise<SandboxExecResult> {
-				status = 'busy'
-				try {
-					return await execViaWorker(baseUrl, command, argv, opts)
-				} finally {
-					status = 'ready'
-				}
+				return await runExecution(async () => await workerClient.exec(command, argv, opts))
 			},
 
 			async writeFile(path: string, content: string | Buffer): Promise<void> {
+				assertActive()
 				const buf = Buffer.isBuffer(content) ? content : Buffer.from(content, 'utf8')
 				const res = await fetch(`${baseUrl}/write-file`, {
 					method: 'POST',
@@ -506,6 +641,7 @@ async function spawnAciSandbox(
 			},
 
 			async readFile(path: string): Promise<Buffer> {
+				assertActive()
 				const res = await fetch(`${baseUrl}/read-file`, {
 					method: 'POST',
 					headers: { 'content-type': 'application/json' },
@@ -514,7 +650,11 @@ async function spawnAciSandbox(
 				if (!res.ok) {
 					throw new Error(`read-file failed: HTTP ${res.status} ${await res.text()}`)
 				}
-				const json = (await res.json()) as { ok: boolean; content?: string; error?: string }
+				const json = (await res.json()) as {
+					ok: boolean
+					content?: string
+					error?: string
+				}
 				if (!json.ok || typeof json.content !== 'string') {
 					throw new Error(json.error ?? 'read-file: no content')
 				}
@@ -522,11 +662,15 @@ async function spawnAciSandbox(
 			},
 
 			async listFiles(rootPath: string): Promise<readonly SandboxFileEntry[]> {
-				return await listFilesViaWorker(baseUrl, rootPath)
+				return await runExecution(async () => await listFilesViaWorker(workerClient, rootPath))
 			},
 
-			async destroy(): Promise<void> {
-				status = 'destroyed'
+			async destroy(options?: SandboxDestroyOptions): Promise<void> {
+				if (retirementPromise) {
+					const observation = await retirementPromise
+					if (observation.accepted) return
+					retirementPromise = undefined
+				}
 				// ARM DELETE — let failures propagate. The Vandal-side
 				// lifecycle wraps this in its own try/catch with logging,
 				// so a silently swallowed error here means orphan ACI
@@ -535,17 +679,13 @@ async function spawnAciSandbox(
 				// the WARM side topped up; that has nothing to do with
 				// cleaning up a CLAIMED instance, which is exclusively the
 				// claimer's responsibility.
-				await armCall(armUrl, 'DELETE', config.getArmToken)
+				await teardownSandbox(options?.signal)
 			},
 		}
 	} catch (err) {
-		try {
-			await armCall(armUrl, 'DELETE', config.getArmToken)
-		} catch {
-			// Preserve the readiness failure as the primary error; the
-			// caller cannot use a Sandbox handle yet, so best-effort ARM
-			// cleanup is the only reliable orphan-prevention hook here.
-		}
+		await runFailureCleanup(async (signal) => {
+			await armCall(armUrl, 'DELETE', config.getArmToken, undefined, signal)
+		})
 		throw err
 	}
 }
@@ -553,115 +693,58 @@ async function spawnAciSandbox(
 async function pollForRunningIp(
 	armUrl: string,
 	getToken: ArmTokenProvider,
+	deadline: OperationDeadline,
 	pollIntervalMs: number,
 	timeoutMs: number,
 ): Promise<string> {
-	const deadline = Date.now() + timeoutMs
-	while (Date.now() < deadline) {
-		const cg = await armCall<ArmContainerGroup>(armUrl, 'GET', getToken)
+	while (deadline.remainingMs() > 0) {
+		let cg: ArmContainerGroup | undefined
+		try {
+			cg = await deadline.run((signal) =>
+				armCall<ArmContainerGroup>(armUrl, 'GET', getToken, undefined, signal),
+			)
+		} catch (err) {
+			if (err instanceof OperationDeadlineExpired) break
+			throw err
+		}
 		const state = cg?.properties?.provisioningState
 		const ip = cg?.properties?.ipAddress?.ip
 		if (state === 'Succeeded' && ip) return ip
 		if (state === 'Failed') {
 			throw new Error('aci-standby-pool: container group provisioning failed')
 		}
-		await new Promise((r) => setTimeout(r, pollIntervalMs))
+		try {
+			await deadline.delay(pollIntervalMs)
+		} catch (err) {
+			if (err instanceof OperationDeadlineExpired) break
+			throw err
+		}
 	}
 	throw new Error(`aci-standby-pool: timed out waiting for container group IP (${timeoutMs}ms)`)
 }
 
 async function waitForWorkerReady(
 	baseUrl: string,
+	deadline: OperationDeadline,
 	timeoutMs: number,
 	pollIntervalMs: number,
 ): Promise<void> {
-	const deadline = Date.now() + timeoutMs
-	while (Date.now() < deadline) {
+	while (deadline.remainingMs() > 0) {
 		try {
-			const res = await fetch(`${baseUrl}/healthz`, {
-				signal: AbortSignal.timeout(2000),
-			})
-			if (res.ok) return
-		} catch {
+			const result = await deadline.run((signal) => probeHttpHealth(`${baseUrl}/healthz`, signal))
+			if (result.ok) return
+		} catch (err) {
+			if (err instanceof OperationDeadlineExpired) break
 			// Network not ready yet, try again.
 		}
-		await new Promise((r) => setTimeout(r, pollIntervalMs))
-	}
-	throw new Error(`aci-standby-pool: worker /healthz never responded (${timeoutMs}ms)`)
-}
-
-async function execViaWorker(
-	baseUrl: string,
-	command: string,
-	argv: string[] | undefined,
-	opts: SandboxExecOptions | undefined,
-): Promise<SandboxExecResult> {
-	const start = Date.now()
-	const res = await fetch(`${baseUrl}/execute`, {
-		method: 'POST',
-		headers: { 'content-type': 'application/json' },
-		body: JSON.stringify({
-			command,
-			args: argv ?? [],
-			cwd: opts?.cwd,
-			env: opts?.env,
-			timeoutMs: opts?.timeout,
-		}),
-	})
-	if (!res.ok || !res.body) {
-		throw new Error(`execute failed: HTTP ${res.status} ${await res.text()}`)
-	}
-
-	let stdout = ''
-	let stderr = ''
-	let exitCode = -1
-	let timedOut = false
-
-	const decoder = new TextDecoder()
-	const reader = res.body.getReader()
-	let buffered = ''
-	for (;;) {
-		const { value, done } = await reader.read()
-		if (done) break
-		buffered += decoder.decode(value, { stream: true })
-		let newlineIdx = buffered.indexOf('\n')
-		while (newlineIdx !== -1) {
-			const line = buffered.slice(0, newlineIdx).trim()
-			buffered = buffered.slice(newlineIdx + 1)
-			if (line) {
-				try {
-					const event = JSON.parse(line) as
-						| { type: 'stdout_delta'; data: string }
-						| { type: 'stderr_delta'; data: string }
-						| { type: 'result'; exitCode: number; timedOut: boolean }
-						| { type: 'error'; error: string }
-					if (event.type === 'stdout_delta') {
-						stdout += event.data
-						opts?.onOutput?.({ stream: 'stdout', data: event.data })
-					} else if (event.type === 'stderr_delta') {
-						stderr += event.data
-						opts?.onOutput?.({ stream: 'stderr', data: event.data })
-					} else if (event.type === 'result') {
-						exitCode = event.exitCode
-						timedOut = event.timedOut
-					} else if (event.type === 'error') {
-						throw new Error(event.error)
-					}
-				} catch (err) {
-					if (!(err instanceof SyntaxError)) throw err
-				}
-			}
-			newlineIdx = buffered.indexOf('\n')
+		try {
+			await deadline.delay(pollIntervalMs)
+		} catch (err) {
+			if (err instanceof OperationDeadlineExpired) break
+			throw err
 		}
 	}
-
-	return {
-		stdout,
-		stderr,
-		exitCode,
-		timedOut,
-		durationMs: Date.now() - start,
-	} as SandboxExecResult
+	throw new Error(`aci-standby-pool: worker /healthz never responded (${timeoutMs}ms)`)
 }
 
 /**
@@ -673,11 +756,10 @@ async function execViaWorker(
  * have produced anything in `rootPath` yet.
  */
 async function listFilesViaWorker(
-	baseUrl: string,
+	workerClient: HttpWorkerClient,
 	rootPath: string,
 ): Promise<readonly SandboxFileEntry[]> {
-	const result = await execViaWorker(
-		baseUrl,
+	const result = await workerClient.exec(
 		'find',
 		[rootPath, '-type', 'f', '-printf', '%p\t%s\n'],
 		undefined,