@namzu/sandbox 13.0.0 → 14.0.0

package/src/backends/kubernetes/k8s-client.ts

@@ -0,0 +1,352 @@
+/**
+ * Minimal Kubernetes API client: bare fetch/https transport plus in-cluster
+ * ServiceAccount bootstrap.
+ *
+ * Sibling of the ACI and Firecracker control-plane clients: same
+ * dependency-free shape, different API. No `@kubernetes/client-node`, no
+ * kubeconfig parsing, no exec-credential-plugin invocation, no YAML anywhere
+ * — `@namzu/sandbox` still declares zero runtime `dependencies` after this
+ * module.
+ *
+ * ## Auth / SDK-dependency boundary
+ * Two access sources, neither needing a YAML parser:
+ *  - `{ inCluster: true }` reads the projected ServiceAccount volume
+ *    (`token`, `ca.crt`, `namespace`) plus `KUBERNETES_SERVICE_HOST` /
+ *    `KUBERNETES_SERVICE_PORT` — pure file + env reads, the same boundary
+ *    `kubectl` itself uses inside a pod. The token file is re-read on EVERY
+ *    request because kubelet rotates projected tokens under the pod; caching
+ *    it would produce an intermittent 401 hours into a long-lived host
+ *    process.
+ *  - `{ server, ca?, getToken }` supplied by the caller mirrors ACI's
+ *    `ArmTokenProvider` and Firecracker's `OrchestratorTokenProvider`
+ *    (`../aci-standby-pool/index.ts`, `../firecracker/index.ts`). Kubeconfig
+ *    parsing, context merging and exec-credential-plugin invocation
+ *    (kubelogin, gcloud, aws-iam-authenticator, ...) all stay OUTSIDE this
+ *    package; the caller resolves a bearer token however it likes.
+ *
+ * ## Transport
+ * Default path is bare global `fetch`. When a custom CA is present — always
+ * true in-cluster, since the cluster CA is never in the process's system
+ * trust store — the call goes through `node:https.request` with that `ca`
+ * and `rejectUnauthorized: true` instead, exactly as
+ * `../firecracker/index.ts` splits `fetchOrchestratorRequest` from
+ * `httpsOrchestratorRequest` ("the package declares no undici dependency").
+ *
+ * ## Status mapping
+ * 404/410 become an `AlreadyGoneError` sentinel a teardown path can treat as
+ * success (ACI's `armCall(..., [404, 410])` accepts the same pair for
+ * exactly that reason — see `../aci-standby-pool/index.ts`). 409 becomes a
+ * `ConflictError` an adoption/retry path can catch. 401/403 become a
+ * `CredentialError` naming the attempted verb and resource — never the
+ * token, which this module never logs or embeds in any thrown message.
+ *
+ * ## Not in v1
+ * No watch, no informers, no resourceVersion/bookmark tracking. Readiness is
+ * polled by the caller with `../readiness.js`'s `OperationDeadline`, exactly
+ * as ACI polls `provisioningState`.
+ */
+
+import { readFileSync } from 'node:fs'
+import https from 'node:https'
+
+/** The only verbs anything in this backend needs to send. */
+export type KubernetesHttpMethod = 'GET' | 'POST' | 'PATCH' | 'DELETE'
+
+/**
+ * Authentication callback. Caller returns a fresh bearer token. Invoked on
+ * every request so a long-running host survives token rotation — the same
+ * contract as ACI's `ArmTokenProvider` and Firecracker's
+ * `OrchestratorTokenProvider`.
+ */
+export type KubernetesTokenProvider = () => Promise<string>
+
+/**
+ * In-cluster bootstrap. The token, CA and namespace come off the projected
+ * ServiceAccount volume; the API server address comes off the env vars the
+ * kubelet always sets for the pod's default-namespace Service.
+ */
+export interface InClusterKubernetesAccess {
+	readonly inCluster: true
+	/**
+	 * Overridable so tests never touch the real projected-volume path.
+	 * Defaults to `/var/run/secrets/kubernetes.io/serviceaccount`.
+	 */
+	readonly serviceAccountDir?: string
+}
+
+/**
+ * Caller-supplied server + credential (see {@link KubernetesTokenProvider}).
+ * `namespace` is required here because, unlike the in-cluster path, there is
+ * no ServiceAccount file to read it from — the caller must say which
+ * namespace this client operates in.
+ */
+export interface ExplicitKubernetesAccess {
+	readonly inCluster?: false
+	readonly server: string
+	readonly namespace: string
+	/** Custom cluster CA. Present → the client dials over `node:https`. */
+	readonly ca?: string | Buffer
+	readonly getToken: KubernetesTokenProvider
+}
+
+export type KubernetesAccess = InClusterKubernetesAccess | ExplicitKubernetesAccess
+
+export interface KubernetesClient {
+	/**
+	 * `path` is the API-server path (e.g.
+	 * `/apis/agents.x-k8s.io/v1/namespaces/ns/sandboxclaims/id`), not a full
+	 * URL. Returns the parsed JSON body, or `undefined` for a 204 or an empty
+	 * body. Rejects with {@link KubernetesAlreadyGoneError},
+	 * {@link KubernetesConflictError} or {@link KubernetesCredentialError} for
+	 * the status codes each names; any other non-2xx status rejects with a
+	 * plain `Error`.
+	 */
+	request<T>(
+		method: KubernetesHttpMethod,
+		path: string,
+		body?: unknown,
+		signal?: AbortSignal,
+	): Promise<T | undefined>
+	/** From the ServiceAccount file in-cluster, from config otherwise. */
+	namespace(): string
+}
+
+const DEFAULT_SERVICE_ACCOUNT_DIR = '/var/run/secrets/kubernetes.io/serviceaccount'
+
+/** 404/410 → this. A teardown path treats it as "already achieved". */
+export class KubernetesAlreadyGoneError extends Error {
+	constructor(
+		readonly method: KubernetesHttpMethod,
+		readonly resource: string,
+		readonly status: number,
+	) {
+		super(`kubernetes ${method} ${resource} -> ${status}: already gone`)
+		this.name = 'KubernetesAlreadyGoneError'
+	}
+}
+
+/** 409 → this. An adoption/retry path can catch it and re-read + retry. */
+export class KubernetesConflictError extends Error {
+	constructor(
+		readonly method: KubernetesHttpMethod,
+		readonly resource: string,
+	) {
+		super(`kubernetes ${method} ${resource} -> 409: conflict`)
+		this.name = 'KubernetesConflictError'
+	}
+}
+
+/**
+ * 401/403 → this. Names the attempted verb and resource only — never the
+ * token, and never the response body (the API server does not echo the
+ * token back, but the body is untrusted content this module has no reason
+ * to repeat into a thrown message).
+ */
+export class KubernetesCredentialError extends Error {
+	constructor(
+		readonly verb: KubernetesHttpMethod,
+		readonly resource: string,
+		readonly status: number,
+	) {
+		super(`kubernetes API refused ${verb} ${resource}: ${status}`)
+		this.name = 'KubernetesCredentialError'
+	}
+}
+
+function stripTrailingSlash(url: string): string {
+	return url.endsWith('/') ? url.slice(0, -1) : url
+}
+
+interface ResolvedAccess {
+	readonly baseUrl: string
+	readonly ca?: string | Buffer
+	readonly namespace: string
+	readonly getToken: KubernetesTokenProvider
+}
+
+/**
+ * Reads `ca.crt` and `namespace` once (they do not rotate for the pod's
+ * lifetime); returns a `getToken` closure that re-reads `token` on every
+ * call, because kubelet DOES rotate the projected token under a live pod.
+ */
+function resolveInCluster(access: InClusterKubernetesAccess): ResolvedAccess {
+	const dir = access.serviceAccountDir ?? DEFAULT_SERVICE_ACCOUNT_DIR
+	const host = process.env.KUBERNETES_SERVICE_HOST
+	const port = process.env.KUBERNETES_SERVICE_PORT
+	if (host === undefined || host === '' || port === undefined || port === '') {
+		throw new Error(
+			'kubernetes: in-cluster access requires KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT',
+		)
+	}
+	const ca = readFileSync(`${dir}/ca.crt`)
+	const namespace = readFileSync(`${dir}/namespace`, 'utf8').trim()
+	return {
+		baseUrl: `https://${host}:${port}`,
+		ca,
+		namespace,
+		getToken: async () => readFileSync(`${dir}/token`, 'utf8').trim(),
+	}
+}
+
+function resolveExplicit(access: ExplicitKubernetesAccess): ResolvedAccess {
+	return {
+		baseUrl: stripTrailingSlash(access.server),
+		ca: access.ca,
+		namespace: access.namespace,
+		getToken: access.getToken,
+	}
+}
+
+/** A transport-agnostic view of the API-server response the mapping needs. */
+interface RawResponse {
+	readonly status: number
+	readonly contentType: string
+	readonly text: () => Promise<string>
+}
+
+/** The plain-`fetch` path — the DEFAULT, used whenever no custom CA is set. */
+async function fetchRequest(
+	url: string,
+	method: KubernetesHttpMethod,
+	headers: Record<string, string>,
+	payload: string | undefined,
+	signal?: AbortSignal,
+): Promise<RawResponse> {
+	const init: RequestInit = { method, headers }
+	if (payload !== undefined) init.body = payload
+	if (signal !== undefined) init.signal = signal
+	const res = await fetch(url, init)
+	return {
+		status: res.status,
+		contentType: res.headers.get('content-type') ?? '',
+		text: () => res.text(),
+	}
+}
+
+/**
+ * The custom-CA path: a `node:https` request presenting the injected `ca`
+ * and verifying the API server's cert against it (`rejectUnauthorized:
+ * true`). `node:https` is used rather than fetch+a custom dispatcher because
+ * the package declares no undici dependency — `node:https` is always
+ * importable and needs nothing added (mirrors
+ * `../firecracker/index.ts`'s `httpsOrchestratorRequest`).
+ */
+function httpsRequest(
+	url: string,
+	method: KubernetesHttpMethod,
+	headers: Record<string, string>,
+	payload: string | undefined,
+	ca: string | Buffer,
+	signal?: AbortSignal,
+): Promise<RawResponse> {
+	const target = new URL(url)
+	return new Promise<RawResponse>((resolve, reject) => {
+		const req = https.request(
+			{
+				protocol: target.protocol,
+				hostname: target.hostname,
+				port: target.port !== '' ? Number(target.port) : 443,
+				path: `${target.pathname}${target.search}`,
+				method,
+				headers,
+				ca,
+				rejectUnauthorized: true,
+				...(signal !== undefined ? { signal } : {}),
+			},
+			(res) => {
+				const chunks: Buffer[] = []
+				res.on('data', (chunk: Buffer) => chunks.push(chunk))
+				res.on('end', () => {
+					const ct = res.headers['content-type']
+					resolve({
+						status: res.statusCode ?? 0,
+						contentType: Array.isArray(ct) ? (ct[0] ?? '') : (ct ?? ''),
+						text: async () => Buffer.concat(chunks).toString('utf8'),
+					})
+				})
+				res.on('error', reject)
+			},
+		)
+		req.on('error', reject)
+		if (payload !== undefined) req.write(payload)
+		req.end()
+	})
+}
+
+/**
+ * `PATCH` always carries a JSON MERGE patch body (RFC 7386) — never
+ * server-side apply, never YAML. `POST` carries a plain JSON create body.
+ * `GET`/`DELETE` normally carry no body; if a caller ever does pass one to
+ * `DELETE` it is sent as plain JSON.
+ */
+function contentTypeFor(method: KubernetesHttpMethod): string {
+	return method === 'PATCH' ? 'application/merge-patch+json' : 'application/json'
+}
+
+export function createKubernetesClient(access: KubernetesAccess): KubernetesClient {
+	const resolved = access.inCluster === true ? resolveInCluster(access) : resolveExplicit(access)
+
+	async function request<T>(
+		method: KubernetesHttpMethod,
+		path: string,
+		body?: unknown,
+		signal?: AbortSignal,
+	): Promise<T | undefined> {
+		signal?.throwIfAborted()
+		const token = await resolved.getToken()
+		signal?.throwIfAborted()
+		const url = `${resolved.baseUrl}${path}`
+		const payload = body !== undefined ? JSON.stringify(body) : undefined
+		const headers: Record<string, string> = {
+			Authorization: `Bearer ${token}`,
+			Accept: 'application/json',
+		}
+		if (payload !== undefined) headers['content-type'] = contentTypeFor(method)
+
+		let res: RawResponse
+		try {
+			res =
+				resolved.ca !== undefined
+					? await httpsRequest(url, method, headers, payload, resolved.ca, signal)
+					: await fetchRequest(url, method, headers, payload, signal)
+		} catch (err) {
+			throw new Error(
+				`kubernetes ${method} ${path} failed: ${err instanceof Error ? err.message : String(err)}`,
+				{ cause: err },
+			)
+		}
+
+		if (res.status === 401 || res.status === 403) {
+			await res.text()
+			signal?.throwIfAborted()
+			throw new KubernetesCredentialError(method, path, res.status)
+		}
+		if (res.status === 404 || res.status === 410) {
+			await res.text()
+			signal?.throwIfAborted()
+			throw new KubernetesAlreadyGoneError(method, path, res.status)
+		}
+		if (res.status === 409) {
+			await res.text()
+			signal?.throwIfAborted()
+			throw new KubernetesConflictError(method, path)
+		}
+		if (res.status < 200 || res.status >= 300) {
+			const text = await res.text()
+			signal?.throwIfAborted()
+			throw new Error(`kubernetes ${method} ${path} -> ${res.status}: ${text}`)
+		}
+		if (res.status === 204) return undefined
+		if (res.contentType.includes('application/json')) {
+			const text = await res.text()
+			signal?.throwIfAborted()
+			return text.length > 0 ? (JSON.parse(text) as T) : undefined
+		}
+		return undefined
+	}
+
+	return {
+		request,
+		namespace: () => resolved.namespace,
+	}
+}

package/src/backends/kubernetes/lease.ts

@@ -0,0 +1,198 @@
+/**
+ * Lease renewal for an acquired kubernetes sandbox.
+ *
+ * Acquire stamps an ABSOLUTE `shutdownTime` (now + `claimTtlSeconds`) plus
+ * `shutdownPolicy: Delete` onto every object it creates. That is the leak
+ * guard: a host that dies mid-run costs the cluster one expiry rather than a
+ * sandbox that lives forever. It is also, unrenewed, a deadline on the RUN —
+ * a session that outlives the TTL has its pod deleted underneath it, mid
+ * command, with no error the host can attribute to anything.
+ *
+ * The two requirements are not in tension, they just need a second half:
+ * a live handle renews its own lease, and a dead host stops renewing. So
+ * this timer is owned by the Sandbox handle, and `destroy()` stops it.
+ *
+ * ## The shape of the timer
+ *
+ * One `setTimeout` chained per tick, never `setInterval`: a renewal PATCH
+ * that takes longer than the interval must not queue a second one behind
+ * it. The interval is HALF the TTL, so a single failed tick still leaves a
+ * whole half-TTL of headroom for the next one to succeed, and it is
+ * jittered ±10% so a hundred handles acquired in the same second do not
+ * PATCH the API server in the same millisecond forever after.
+ *
+ * The timer is `unref`'d: a host process that has finished its work should
+ * exit, not linger because a sandbox handle is still counting. A handle
+ * nobody destroyed then expires on the cluster's clock exactly as an
+ * abandoned one does, which is the behaviour the TTL exists for.
+ *
+ * ## Every tick is bounded
+ *
+ * A renewal that FAILS is survivable — it is reported and retried with half
+ * a TTL of headroom. A renewal that HANGS is not: the next tick is scheduled
+ * only after the current one settles, so a PATCH that never answers parks
+ * the loop forever, reports nothing, and lets the lease expire in silence —
+ * precisely the defect this file exists to close, moved onto the failure
+ * path. An API server that accepts a connection and then never responds is
+ * an ordinary cluster event, so each PATCH runs under its own deadline: it
+ * aborts the request through the signal the client already takes, and an
+ * expiry is then just another reported failure that retries on the next
+ * tick.
+ *
+ * ## What each outcome means
+ *
+ *  - Success → the object's expiry moves a full TTL into the future.
+ *  - Any error, a tick that ran out of time included → reported to
+ *    `onRenewalError` and RETRIED on the next tick. A transient API blip
+ *    must not tear down a working sandbox, and there is still half a TTL of
+ *    headroom.
+ *  - Already gone (404/410) → the object this handle owns no longer exists.
+ *    Nothing will bring it back, so the loop stops and the handle is marked
+ *    gone; every later call fails with a named error instead of dialing an
+ *    address whose pod the controller has already deleted.
+ */
+
+import { OperationDeadline } from '../readiness.js'
+import { KubernetesAlreadyGoneError } from './k8s-client.js'
+
+/** How far each tick pushes the expiry, and how often ticks happen. */
+export interface LeaseRenewalOptions {
+	/** The same TTL acquire stamped. Each tick sets `now + ttlSeconds`. */
+	readonly ttlSeconds: number
+	/**
+	 * Send the merge patch. Rejects with whatever the client rejects with.
+	 *
+	 * The `signal` is the tick's own deadline and IS passed on every call —
+	 * an implementation that drops it still gets abandoned on time, but its
+	 * socket then stays open until the peer or the OS closes it.
+	 */
+	readonly renew: (shutdownTime: string, signal?: AbortSignal) => Promise<void>
+	/** Called once, when the renewed object turns out to be gone. */
+	readonly onGone: () => void
+	/**
+	 * Every renewal failure that is not "already gone". `@namzu/sandbox` has
+	 * no logger of its own and reads none from module scope, so a diagnostic
+	 * this package cannot print is handed to the caller that can.
+	 */
+	readonly onRenewalError?: (error: unknown) => void
+	/**
+	 * Base interval between ticks. Defaults to half the TTL. Present so a
+	 * test can drive many ticks in a few milliseconds without pretending a
+	 * sub-second TTL is a realistic configuration.
+	 */
+	readonly intervalMs?: number
+	/**
+	 * How long ONE renewal PATCH may take before it is abandoned and retried.
+	 * Defaults to a quarter of the interval, capped at
+	 * {@link MAX_RENEWAL_TIMEOUT_MS} — a fraction rather than the whole
+	 * interval so that a stalled API server still leaves the loop several
+	 * attempts inside the half-TTL of headroom.
+	 */
+	readonly patchTimeoutMs?: number
+	/** Deterministic jitter for tests. Defaults to `Math.random`. */
+	readonly random?: () => number
+}
+
+/**
+ * The ceiling on one renewal PATCH. A write to the API server that has not
+ * answered in half a minute is not going to; at the one-hour default TTL the
+ * derived quarter-interval would otherwise be 7.5 minutes of silence.
+ */
+const MAX_RENEWAL_TIMEOUT_MS = 30_000
+
+/** ±10%: enough to spread a synchronised fleet, far too little to matter
+ * against a half-TTL of headroom. */
+const JITTER_FRACTION = 0.1
+
+export function jitteredInterval(baseMs: number, random: () => number): number {
+	const factor = 1 - JITTER_FRACTION + random() * (2 * JITTER_FRACTION)
+	// Never zero, whatever a caller passes: a zero-delay chain would spin.
+	return Math.max(1, Math.round(baseMs * factor))
+}
+
+/**
+ * The renewal loop. Start it when the handle is handed out, stop it on
+ * `destroy()`. Both are idempotent.
+ */
+export class KubernetesLeaseRenewal {
+	private timer: ReturnType<typeof setTimeout> | undefined
+	private stopped = false
+	private started = false
+	private readonly baseIntervalMs: number
+	private readonly patchTimeoutMs: number
+	private readonly random: () => number
+
+	constructor(private readonly options: LeaseRenewalOptions) {
+		this.baseIntervalMs = options.intervalMs ?? Math.max(1, (options.ttlSeconds * 1_000) / 2)
+		this.patchTimeoutMs =
+			options.patchTimeoutMs ??
+			Math.max(1, Math.min(MAX_RENEWAL_TIMEOUT_MS, Math.round(this.baseIntervalMs / 4)))
+		this.random = options.random ?? Math.random
+	}
+
+	/**
+	 * The loop is alive: it has been started and not stopped. Deliberately
+	 * NOT "a timer is pending" — `tick()` clears the timer before it awaits,
+	 * so a liveness check written that way reads false for the whole duration
+	 * of an in-flight renewal and would quietly pass against a loop that had
+	 * parked forever inside one.
+	 */
+	get active(): boolean {
+		return !this.stopped && this.started
+	}
+
+	start(): void {
+		if (this.stopped || this.timer !== undefined) return
+		this.started = true
+		this.schedule()
+	}
+
+	stop(): void {
+		this.stopped = true
+		if (this.timer !== undefined) {
+			clearTimeout(this.timer)
+			this.timer = undefined
+		}
+	}
+
+	private schedule(): void {
+		if (this.stopped) return
+		const timer = setTimeout(
+			() => {
+				void this.tick()
+			},
+			jitteredInterval(this.baseIntervalMs, this.random),
+		)
+		// A pending renewal must never be the reason a host process stays up.
+		timer.unref?.()
+		this.timer = timer
+	}
+
+	/** Exposed for tests: one renewal attempt plus its scheduling decision. */
+	async tick(): Promise<void> {
+		this.timer = undefined
+		if (this.stopped) return
+		const shutdownTime = new Date(Date.now() + this.options.ttlSeconds * 1_000).toISOString()
+		try {
+			// On its own clock: the next tick is scheduled only once this one
+			// settles, so an unbounded PATCH that never answers would park the
+			// loop permanently and let the lease expire with nothing reported.
+			// The deadline aborts the request and hands the expiry to the same
+			// report-and-retry path every other failure takes.
+			await new OperationDeadline(this.patchTimeoutMs, 'kubernetes lease renewal').run(
+				async (signal) => await this.options.renew(shutdownTime, signal),
+			)
+		} catch (error) {
+			if (error instanceof KubernetesAlreadyGoneError) {
+				this.stop()
+				this.options.onGone()
+				return
+			}
+			// Everything else is transient until proven otherwise: report it
+			// and try again on the next tick, which is still half a TTL
+			// before anything expires.
+			this.options.onRenewalError?.(error)
+		}
+		this.schedule()
+	}
+}