@namzu/sandbox 13.0.0 → 15.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/CHANGELOG.md +1147 -0
  2. package/README.md +447 -0
  3. package/dist/backends/aci-standby-pool/index.d.ts.map +1 -1
  4. package/dist/backends/aci-standby-pool/index.js +13 -1
  5. package/dist/backends/aci-standby-pool/index.js.map +1 -1
  6. package/dist/backends/docker/index.d.ts.map +1 -1
  7. package/dist/backends/docker/index.js +19 -1
  8. package/dist/backends/docker/index.js.map +1 -1
  9. package/dist/backends/firecracker/index.d.ts.map +1 -1
  10. package/dist/backends/firecracker/index.js +12 -2
  11. package/dist/backends/firecracker/index.js.map +1 -1
  12. package/dist/backends/firecracker/protocol.d.ts +481 -8
  13. package/dist/backends/firecracker/protocol.d.ts.map +1 -1
  14. package/dist/backends/firecracker/protocol.js +136 -0
  15. package/dist/backends/firecracker/protocol.js.map +1 -1
  16. package/dist/backends/firecracker/transport.d.ts +642 -14
  17. package/dist/backends/firecracker/transport.d.ts.map +1 -1
  18. package/dist/backends/firecracker/transport.js +1307 -34
  19. package/dist/backends/firecracker/transport.js.map +1 -1
  20. package/dist/backends/kubernetes/egress-policy.d.ts +1296 -0
  21. package/dist/backends/kubernetes/egress-policy.d.ts.map +1 -0
  22. package/dist/backends/kubernetes/egress-policy.js +2458 -0
  23. package/dist/backends/kubernetes/egress-policy.js.map +1 -0
  24. package/dist/backends/kubernetes/identity.d.ts +193 -0
  25. package/dist/backends/kubernetes/identity.d.ts.map +1 -0
  26. package/dist/backends/kubernetes/identity.js +147 -0
  27. package/dist/backends/kubernetes/identity.js.map +1 -0
  28. package/dist/backends/kubernetes/index.d.ts +1019 -0
  29. package/dist/backends/kubernetes/index.d.ts.map +1 -0
  30. package/dist/backends/kubernetes/index.js +1756 -0
  31. package/dist/backends/kubernetes/index.js.map +1 -0
  32. package/dist/backends/kubernetes/ingress-policy.d.ts +375 -0
  33. package/dist/backends/kubernetes/ingress-policy.d.ts.map +1 -0
  34. package/dist/backends/kubernetes/ingress-policy.js +1050 -0
  35. package/dist/backends/kubernetes/ingress-policy.js.map +1 -0
  36. package/dist/backends/kubernetes/k8s-client.d.ts +334 -0
  37. package/dist/backends/kubernetes/k8s-client.d.ts.map +1 -0
  38. package/dist/backends/kubernetes/k8s-client.js +553 -0
  39. package/dist/backends/kubernetes/k8s-client.js.map +1 -0
  40. package/dist/backends/kubernetes/lease.d.ts +145 -0
  41. package/dist/backends/kubernetes/lease.d.ts.map +1 -0
  42. package/dist/backends/kubernetes/lease.js +201 -0
  43. package/dist/backends/kubernetes/lease.js.map +1 -0
  44. package/dist/backends/kubernetes/objects.d.ts +702 -0
  45. package/dist/backends/kubernetes/objects.d.ts.map +1 -0
  46. package/dist/backends/kubernetes/objects.js +518 -0
  47. package/dist/backends/kubernetes/objects.js.map +1 -0
  48. package/dist/backends/kubernetes/per-sandbox-policy.d.ts +219 -0
  49. package/dist/backends/kubernetes/per-sandbox-policy.d.ts.map +1 -0
  50. package/dist/backends/kubernetes/per-sandbox-policy.js +407 -0
  51. package/dist/backends/kubernetes/per-sandbox-policy.js.map +1 -0
  52. package/dist/backends/kubernetes/privilege-probe.d.ts +136 -0
  53. package/dist/backends/kubernetes/privilege-probe.d.ts.map +1 -0
  54. package/dist/backends/kubernetes/privilege-probe.js +185 -0
  55. package/dist/backends/kubernetes/privilege-probe.js.map +1 -0
  56. package/dist/backends/kubernetes/rbac.d.ts +153 -0
  57. package/dist/backends/kubernetes/rbac.d.ts.map +1 -0
  58. package/dist/backends/kubernetes/rbac.js +177 -0
  59. package/dist/backends/kubernetes/rbac.js.map +1 -0
  60. package/dist/backends/kubernetes/sandbox.d.ts +190 -0
  61. package/dist/backends/kubernetes/sandbox.d.ts.map +1 -0
  62. package/dist/backends/kubernetes/sandbox.js +433 -0
  63. package/dist/backends/kubernetes/sandbox.js.map +1 -0
  64. package/dist/backends/kubernetes/transport.d.ts +1048 -0
  65. package/dist/backends/kubernetes/transport.d.ts.map +1 -0
  66. package/dist/backends/kubernetes/transport.js +2093 -0
  67. package/dist/backends/kubernetes/transport.js.map +1 -0
  68. package/dist/backends/kubernetes/workspace.d.ts +1512 -0
  69. package/dist/backends/kubernetes/workspace.d.ts.map +1 -0
  70. package/dist/backends/kubernetes/workspace.js +3703 -0
  71. package/dist/backends/kubernetes/workspace.js.map +1 -0
  72. package/dist/backends/remote-execution-controller.d.ts +14 -0
  73. package/dist/backends/remote-execution-controller.d.ts.map +1 -1
  74. package/dist/backends/remote-execution-controller.js.map +1 -1
  75. package/dist/index.d.ts +350 -2
  76. package/dist/index.d.ts.map +1 -1
  77. package/dist/index.js +344 -34
  78. package/dist/index.js.map +1 -1
  79. package/dist/testing/sandbox-conformance.d.ts +227 -0
  80. package/dist/testing/sandbox-conformance.d.ts.map +1 -0
  81. package/dist/testing/sandbox-conformance.js +896 -0
  82. package/dist/testing/sandbox-conformance.js.map +1 -0
  83. package/package.json +5 -4
  84. package/src/backends/aci-standby-pool/index.ts +16 -1
  85. package/src/backends/docker/index.ts +22 -1
  86. package/src/backends/firecracker/index.ts +14 -2
  87. package/src/backends/firecracker/protocol.ts +541 -6
  88. package/src/backends/firecracker/transport.ts +1687 -64
  89. package/src/backends/kubernetes/egress-policy.ts +3448 -0
  90. package/src/backends/kubernetes/identity.ts +261 -0
  91. package/src/backends/kubernetes/index.ts +2670 -0
  92. package/src/backends/kubernetes/ingress-policy.ts +1344 -0
  93. package/src/backends/kubernetes/k8s-client.ts +742 -0
  94. package/src/backends/kubernetes/lease.ts +254 -0
  95. package/src/backends/kubernetes/objects.ts +983 -0
  96. package/src/backends/kubernetes/per-sandbox-policy.ts +542 -0
  97. package/src/backends/kubernetes/privilege-probe.ts +261 -0
  98. package/src/backends/kubernetes/rbac.ts +192 -0
  99. package/src/backends/kubernetes/sandbox.ts +593 -0
  100. package/src/backends/kubernetes/transport.ts +2895 -0
  101. package/src/backends/kubernetes/workspace.ts +5640 -0
  102. package/src/backends/remote-execution-controller.ts +14 -0
  103. package/src/index.ts +838 -35
  104. package/src/testing/sandbox-conformance.ts +1202 -0
@@ -0,0 +1,1756 @@
1
+ /**
2
+ * Kubernetes / agent-sandbox backend — acquire, readiness and teardown.
3
+ *
4
+ * Sibling of `firecracker/` and `aci-standby-pool/`: same
5
+ * {@link SandboxBackend} surface, same "the host supplies the credential
6
+ * callback and this package carries no cloud SDK" boundary, a different
7
+ * control plane. Here the control plane is the Kubernetes API server and the
8
+ * warm pool is a `SandboxWarmPool` reconciled by the agent-sandbox controller
9
+ * (kubernetes-sigs/agent-sandbox v1.0.2).
10
+ *
11
+ * Registered as `microvm` because {@link SandboxTier} names the strength of
12
+ * the boundary, not the orchestrator that starts it: a pod scheduled onto a
13
+ * Kata RuntimeClass runs in a hardware-virtualized guest. The tier also keeps
14
+ * the backend out of the container tier's mandatory
15
+ * `ContainerSandboxLayout`, which a remote-copy backend has no use for —
16
+ * Firecracker took the same exemption.
17
+ *
18
+ * ## Two acquire paths, and why they create different kinds
19
+ *
20
+ * - `warmPoolName` set → POST a `SandboxClaim` at the named pool. The
21
+ * controller binds an already-running Sandbox out of the pool, which is
22
+ * what makes a sub-second acquire possible at all.
23
+ * - `warmPoolName` unset → POST a `Sandbox` directly. This is necessitated
24
+ * rather than chosen: `SandboxClaimSpec.warmPoolRef` is a REQUIRED field,
25
+ * so a pool-less claim does not exist in the API.
26
+ *
27
+ * The claim this backend POSTs is PRISTINE: `warmPoolRef` and a lifecycle
28
+ * bound, nothing else. `spec.env` and `spec.volumeClaimTemplates` are
29
+ * available on the claim and are never set, because a claim carrying either
30
+ * is forced to cold-start instead of adopting a pool sandbox — the single
31
+ * most expensive mistake available on this path, and a silent one, since such
32
+ * a claim still works and only the latency shows it. Per-sandbox controls
33
+ * that would need those fields are refused by {@link assertEnforceable}
34
+ * rather than accepted and dropped.
35
+ *
36
+ * ## The bound sandbox is not named after the claim
37
+ *
38
+ * A pool sandbox keeps the generated name the pool gave it when the claim
39
+ * adopts it. The backend therefore reads the bound identity back out of
40
+ * `status.sandbox` and never derives it from the claim's own name. A test
41
+ * covers exactly that asymmetry.
42
+ *
43
+ * ## Credential
44
+ *
45
+ * The per-instance agent bind token is the backing pod's own
46
+ * `metadata.uid`: the host learns it from the API server after readiness, the
47
+ * guest learns it through the downward API (`NAMZU_AGENT_BIND_TOKEN`), and
48
+ * nothing has to be minted, stored or injected at claim time. One GET, no
49
+ * claim mutation, so the warm path stays pristine. A resumed pod is a new pod
50
+ * with a new uid, which is correct: it is a new instance.
51
+ *
52
+ * ## The lease
53
+ *
54
+ * The `shutdownTime` acquire stamps is the leak guard AND, unrenewed, a
55
+ * deadline on the run. The Sandbox handle owns a renewal loop that PATCHes
56
+ * it forward every half-TTL for as long as the handle is alive, and
57
+ * `destroy()` stops it — so a live run keeps its pod and a dead host still
58
+ * costs the cluster exactly one expiry. See `lease.ts`.
59
+ *
60
+ * ## The privilege probe
61
+ *
62
+ * `create()` does not resolve until the guest has reported — and this
63
+ * backend has checked — that it is deprivileged: all four capability masks
64
+ * zero, `NoNewPrivs: 1`, read out of `/proc/self/status` over the agent's
65
+ * `execute` op, on a clock of its own so a guest that goes quiet is refused
66
+ * rather than waited on. A refusal destroys the instance and rejects, so no
67
+ * handle to an under-hardened sandbox escapes. There is no off switch. See
68
+ * `privilege-probe.ts`.
69
+ *
70
+ * ## Not watch
71
+ *
72
+ * Readiness is polled against the shared {@link OperationDeadline}, exactly as
73
+ * ACI polls `provisioningState`. A watch would buy nothing on a path whose
74
+ * whole budget is under a second, and would cost resourceVersion tracking,
75
+ * bookmarks, 410-relist and reconnect backoff.
76
+ *
77
+ * ## Egress
78
+ *
79
+ * `config.egress` is optional and, when set, translated and VERIFIED — never
80
+ * created — by `egress-policy.ts`, in two steps. The NAMED object is GETted
81
+ * and compared to the translation exactly, once, lazily, on the first
82
+ * `create()`, so `buildKubernetesBackend` itself still contacts nothing.
83
+ * Then, because the API server UNIONS every policy selecting a pod, every
84
+ * `NetworkPolicy` in the namespace (and, under `engine: 'cilium'`, every
85
+ * `CiliumNetworkPolicy`) is enumerated against the pod's real labels and the
86
+ * create is refused when any of them lets out more than the translation does
87
+ * — a second policy widens egress however exactly the named one matches, and
88
+ * a `SandboxTemplate`'s own `networkPolicy` block becomes exactly such a
89
+ * policy. `egress.verify: 'named-object-only'` is the opt-out and restores
90
+ * the first step alone. Every Sandbox this file creates directly
91
+ * (`buildSandboxBody`) carries {@link sandboxTemplateLabel} on its
92
+ * podTemplate specifically so that translated policy's `podSelector` has
93
+ * something stable to match — see `objects.ts`'s doc comment on that label
94
+ * for why agent-sandbox's own controller-owned label does not cover this
95
+ * path.
96
+ *
97
+ * ## Ingress
98
+ *
99
+ * `config.ingress` is the opposite default: verification is ON unless a
100
+ * deployment says `'unverified'`. Before the POST for a direct Sandbox, and
101
+ * after the bind for a claimed one, `ingress-policy.ts` lists the namespace's
102
+ * policies and refuses unless one of them actually closes the agent port on
103
+ * the labels this pod carries. Nothing checked that before, while three
104
+ * pieces of shipped text said it was covered — see that module's own doc
105
+ * comment for what was measured.
106
+ */
107
+ import { generateSandboxId } from '@namzu/sdk';
108
+ import { OperationDeadline, OperationDeadlineExpired, resolveReadinessOptions, runFailureCleanup, } from '../readiness.js';
109
+ import { KubernetesPodLabelNotObservedError, KubernetesPodLabelsRejectedError, assertEgressPolicyIsEnforceable, assertEgressProfileIsUsable, assertPerSandboxEgressIsUsable, composeAdditionalPodLabels, defaultEgressPolicyName, egressProfileLabel, egressUnionVerificationEnabled, perSandboxEgressLabelKey, translateEgressPolicy, verifyEgressPolicyApplied, verifyEgressPolicyUnion, } from './egress-policy.js';
110
+ import { ingressVerificationEnabled, resolveIngressEngine, verifyIngressPolicyApplied, } from './ingress-policy.js';
111
+ import { KubernetesAlreadyGoneError, KubernetesApiError, KubernetesApiTimeoutError, KubernetesCredentialError, createKubernetesClient, } from './k8s-client.js';
112
+ import { READY_CONDITION, SANDBOX_API_GROUP, SANDBOX_API_VERSION, SANDBOX_EXTENSIONS_API_GROUP, claimCollectionPath, claimListPath, claimPath, isConditionTrue, isPodLive, podCollectionPath, podListPath, podPath, readPodIP, sandboxCollectionPath, sandboxPath, sandboxTemplateLabel, sandboxTemplatePath, warmPoolPath, } from './objects.js';
113
+ import { KubernetesOwnerUidMissingError, buildAdmissionFence, buildPerSandboxPolicySetter, } from './per-sandbox-policy.js';
114
+ import { privilegeProbeTimedOut, runPrivilegeProbe } from './privilege-probe.js';
115
+ import { buildKubernetesSandbox } from './sandbox.js';
116
+ import { KubernetesAgentTransport } from './transport.js';
117
+ /**
118
+ * Default {@link KubernetesBackendInternalConfig.streamHeartbeatMs} — 15 s,
119
+ * so a stream whose peer vanished without a FIN or an RST is given up on
120
+ * within 45 s rather than never.
121
+ *
122
+ * The Kubernetes backend opts IN here; `VsockTransportOptions.heartbeatMs`
123
+ * stays undefined by default, because that transport is shared with the
124
+ * Firecracker tier and a default there would force-close an existing
125
+ * consumer's quiet-but-alive terminal.
126
+ */
127
+ export const DEFAULT_STREAM_HEARTBEAT_MS = 15_000;
128
+ /**
129
+ * Validate the configured stream-heartbeat interval, or supply the default.
130
+ *
131
+ * `0` IS accepted here, unlike `apiRequestTimeoutMs`: the heartbeat is a new
132
+ * capability that a deployment behind a middlebox with its own idea about
133
+ * unexpected frames may want off, and turning it off restores exactly the
134
+ * behaviour every release before this one had. An unanswered API request has
135
+ * no such prior behaviour worth restoring.
136
+ */
137
+ export function resolveStreamHeartbeatMs(value) {
138
+ if (value === undefined)
139
+ return DEFAULT_STREAM_HEARTBEAT_MS;
140
+ if (!Number.isSafeInteger(value) || value < 0) {
141
+ throw new Error(`kubernetes: streamHeartbeatMs must be a non-negative integer, got ${JSON.stringify(value)}. Use 0 to send no heartbeats at all, which is how every release before this one behaved; the default is ${DEFAULT_STREAM_HEARTBEAT_MS}ms.`);
142
+ }
143
+ return value;
144
+ }
145
+ /**
146
+ * The client options every `createKubernetesClient` call in this backend is
147
+ * built with. One function so the five call sites — the provider, and each
148
+ * of the workspace verbs — cannot drift apart on which bounds they honour.
149
+ */
150
+ export function clientOptions(config) {
151
+ return { requestTimeoutMs: config.apiRequestTimeoutMs };
152
+ }
153
+ /**
154
+ * The same number the Firecracker guest agent listens on over vsock
155
+ * (`DEFAULT_AGENT_VSOCK_PORT`), so one agent has one port across both tiers
156
+ * and a manifest, a NetworkPolicy and a transport can all name it from
157
+ * memory. Unprivileged, and the sandbox pod is not sharing it with anything.
158
+ */
159
+ export const DEFAULT_AGENT_PORT = 1024;
160
+ /**
161
+ * Deliberately far below ACI's 500 ms. A pool bind lands in ~120 ms on a warm
162
+ * cluster, so a half-second poll would spend most of the sub-second acquire
163
+ * budget asleep; 50 ms costs a handful of cheap GETs and gives the measurement
164
+ * somewhere to land.
165
+ */
166
+ const DEFAULT_READY_POLL_MS = 50;
167
+ const DEFAULT_READY_TIMEOUT_MS = 60_000;
168
+ const DEFAULT_CLAIM_TTL_SECONDS = 3_600;
169
+ /**
170
+ * The ceiling on the privilege probe's own clock — see
171
+ * {@link resolveProbeTimeoutMs} for where the rest of the number comes from.
172
+ *
173
+ * The probe is one `cat` of a pseudo-file over an already-established path,
174
+ * so half a minute would already be generous and a quarter of one is plenty.
175
+ * The number matters because the alternative is not "a bit longer": with no
176
+ * clock of its own the probe falls back on the execution controller's generic
177
+ * defaults — a five-minute execution observation, then a cancel-confirm and a
178
+ * drain — so a guest that accepts the TCP connection and then stops answering
179
+ * would keep a 60 s `create()` pending for over six minutes.
180
+ */
181
+ const PRIVILEGE_PROBE_TIMEOUT_CAP_MS = 15_000;
182
+ /**
183
+ * How long the privilege probe may take, given the caller's readiness budget.
184
+ *
185
+ * `readyTimeoutMs` bounds the CONTROL plane and has usually expired by the
186
+ * time the probe starts, so the probe cannot share it — but it is still the
187
+ * number the caller chose to describe how long an acquire may take, so the
188
+ * probe is allowed exactly that much again and no more, capped. A caller who
189
+ * asked for a 500 ms acquire gets a 500 ms probe; one who asked for two
190
+ * minutes of cold start still gets {@link PRIVILEGE_PROBE_TIMEOUT_CAP_MS}.
191
+ * Deliberately not a separate config key: a knob whose only correct value is
192
+ * "long enough for one `cat`" is a knob that only ever gets set wrong.
193
+ */
194
+ export function resolveProbeTimeoutMs(readyTimeoutMs) {
195
+ return Math.min(readyTimeoutMs, PRIVILEGE_PROBE_TIMEOUT_CAP_MS);
196
+ }
197
+ /**
198
+ * The readiness bounds every path in this backend polls against — acquire,
199
+ * and `workspace.ts`'s create/suspend/resume. One function so the two cannot
200
+ * drift apart on defaults.
201
+ */
202
+ export function resolveKubernetesReadiness(config) {
203
+ return resolveReadinessOptions('kubernetes', config.readyTimeoutMs, config.readyPollIntervalMs, {
204
+ timeoutMs: DEFAULT_READY_TIMEOUT_MS,
205
+ pollIntervalMs: DEFAULT_READY_POLL_MS,
206
+ });
207
+ }
208
+ /**
209
+ * Per-sandbox controls this backend cannot apply, and therefore refuses.
210
+ *
211
+ * `env` is the load-bearing one. A SandboxClaim CAN carry `spec.env`, so this
212
+ * looks at first like a control that fits — but a claim that sets it is forced
213
+ * to cold-start instead of adopting a pool sandbox, which turns a 120 ms
214
+ * acquire into a full pod start. Accepting env here would buy a caller a
215
+ * feature and silently take the warm pool away, and the only symptom would be
216
+ * latency. The limits belong on the SandboxTemplate the pool is built from.
217
+ *
218
+ * `egress` is refused because this backend applies egress at the network
219
+ * layer, on the template's NetworkPolicy, which cannot be rewritten per
220
+ * running sandbox. The container backend's habit of emitting proxy
221
+ * environment variables as a substitute is not repeated here: a policy
222
+ * accepted and quietly not enforced is worse than one that is refused.
223
+ */
224
+ const UNSUPPORTED_PER_SANDBOX_CONTROLS = [
225
+ ['egress', 'network egress policy'],
226
+ ['memoryLimitMb', 'memory limit'],
227
+ ['maxProcesses', 'process limit'],
228
+ ['env', 'environment variables'],
229
+ ];
230
+ export function assertEnforceable(options) {
231
+ const unenforceable = UNSUPPORTED_PER_SANDBOX_CONTROLS.filter(([key]) => {
232
+ const value = options[key];
233
+ return value !== undefined && (key !== 'env' || Object.keys(value).length > 0);
234
+ });
235
+ if (unenforceable.length === 0)
236
+ return;
237
+ throw new Error(`The kubernetes sandbox backend cannot enforce per-sandbox ${unenforceable
238
+ .map(([, label]) => label)
239
+ .join(', ')}: a SandboxClaim carrying env or volumes is forced to cold-start instead of adopting a warm pool sandbox, and egress is a NetworkPolicy on the pool's SandboxTemplate rather than a per-sandbox setting. Set them on the SandboxTemplate the SandboxWarmPool is built from, or use a backend that applies them per sandbox. Refusing rather than accepting a control that would be silently dropped.`);
240
+ }
241
+ /**
242
+ * Refuse a runtime class the pool path cannot honour.
243
+ *
244
+ * A pooled sandbox is already running by the time a claim reaches it, under
245
+ * whatever RuntimeClass its SandboxTemplate named. `runtimeClassName` in this
246
+ * config would therefore be read, accepted and ignored — and the thing it
247
+ * selects is the VM boundary, which is the last control to lose quietly.
248
+ */
249
+ export function assertRuntimeClassIsApplicable(config) {
250
+ if (config.runtimeClassName === undefined || config.warmPoolName === undefined)
251
+ return;
252
+ throw new Error(`The kubernetes sandbox backend cannot apply runtimeClassName ${JSON.stringify(config.runtimeClassName)} to sandboxes claimed from warm pool ${JSON.stringify(config.warmPoolName)}: a pooled sandbox is already running under the RuntimeClass its SandboxTemplate named, and a claim cannot change it. Set runtimeClassName on that SandboxTemplate's podTemplate, or drop warmPoolName to have this backend create each Sandbox itself.`);
253
+ }
254
+ /**
255
+ * Build a {@link SandboxBackend} against a cluster running the agent-sandbox
256
+ * controller. Construction is synchronous and contacts nothing: readiness
257
+ * bounds and the config refusals are validated here so a misconfiguration
258
+ * surfaces during host wiring rather than mid-run, and the first API call
259
+ * happens on the first `create()`.
260
+ */
261
+ export function buildKubernetesBackend(config) {
262
+ const readiness = resolveKubernetesReadiness(config);
263
+ assertRuntimeClassIsApplicable(config);
264
+ // A hostname allowlist with no FQDN-capable engine declared is a
265
+ // configuration error, not a runtime one — it can be decided from
266
+ // `config.egress.policy.kind` alone, with no API call, so it is refused
267
+ // here, synchronously, the same moment the two checks above are.
268
+ if (config.egress) {
269
+ assertEgressPolicyIsEnforceable(config.egress.policy, config.egress.engine ?? 'core', config.egress.ciliumNarrowing);
270
+ }
271
+ // And the same for an egress PROFILE that could never be written as a
272
+ // label — a value the API server would reject leaves either a claim
273
+ // nothing binds or a policy nobody can apply, and both are decidable
274
+ // from config alone.
275
+ assertEgressProfileIsUsable(config.egress);
276
+ // And the same for per-sandbox egress: an engine that cannot express a
277
+ // hostname, an unnamed admission fence or a selector key the API server
278
+ // would refuse are all decidable from config alone, and a host that
279
+ // learns any of them from its first `setNetworkPolicy` call learns it an
280
+ // hour into a run that cannot be redone.
281
+ assertPerSandboxEgressIsUsable(config.egress);
282
+ // Resolved here as well as at each session, so a configuration this
283
+ // backend will never honour is refused while `buildKubernetesBackend` is
284
+ // still on the stack rather than on someone's first `create()`.
285
+ resolveStreamHeartbeatMs(config.streamHeartbeatMs);
286
+ const client = createKubernetesClient(clientAccess(config), clientOptions(config));
287
+ // Verify-not-trust runs once, lazily, on the first `create()` — never here,
288
+ // because `buildKubernetesBackend` is documented to contact nothing. A
289
+ // failed attempt is not cached: a transient API error should not wedge
290
+ // every later create() behind the same stale rejection forever. The
291
+ // boundary object holds both halves and both memos; it lives as long as
292
+ // this backend does, which is what makes the named-object check
293
+ // once-per-backend rather than once-per-create.
294
+ const egressBoundary = buildEgressBoundary(client, config, config.sandboxTemplateName);
295
+ // Ingress is cached PER LABEL SET rather than once per backend, because
296
+ // unlike the egress NAMED-object check it is a question about one pod: a
297
+ // pooled sandbox's labels come off the pool's template and a pool-less
298
+ // one's off this config, and a single memo would answer for a pod it never
299
+ // examined. Same failure handling as the egress memo — a failed attempt is
300
+ // dropped, so a transient API error does not wedge every later create()
301
+ // behind it. The egress UNION check is keyed the same way, for the same
302
+ // reason, and additionally expires: see {@link EGRESS_UNION_CACHE_TTL_MS}.
303
+ const ingressVerifier = buildIngressVerifier(client, config, new Map());
304
+ // One fence per backend, so its memo is shared by every sandbox this
305
+ // backend hands out rather than re-proved per handle. `undefined` when
306
+ // per-sandbox egress is not configured, which is what makes
307
+ // `setNetworkPolicy` absent from the handle — presence follows
308
+ // CONFIGURATION and never a runtime probe, so a caller's capability
309
+ // detection cannot depend on when it asked.
310
+ const perSandbox = config.egress?.perSandbox;
311
+ const fence = perSandbox === undefined ? undefined : buildAdmissionFence(client, perSandbox);
312
+ return {
313
+ tier: 'microvm',
314
+ name: 'kubernetes',
315
+ async create(options) {
316
+ await egressBoundary?.verifyNamedObject(options.signal);
317
+ const acquisition = await acquireKubernetesSandbox(client, config, options, readiness, ingressVerifier, egressBoundary);
318
+ const egress = config.egress;
319
+ const setNetworkPolicy = fence !== undefined &&
320
+ egress?.perSandbox !== undefined &&
321
+ acquisition.owner !== undefined &&
322
+ acquisition.perSandboxLabelValue !== undefined
323
+ ? buildPerSandboxPolicySetter({
324
+ client,
325
+ fence,
326
+ namespace: config.namespace,
327
+ egress: { ...egress, perSandbox: egress.perSandbox },
328
+ owner: acquisition.owner,
329
+ selectorValue: acquisition.perSandboxLabelValue,
330
+ })
331
+ : undefined;
332
+ return await admitProbedSandbox(acquisition, config, options, resolveProbeTimeoutMs(readiness.timeoutMs), setNetworkPolicy);
333
+ },
334
+ };
335
+ }
336
+ /**
337
+ * How long a union pass is trusted for one label set.
338
+ *
339
+ * Five minutes rather than the backend's lifetime, which is what the
340
+ * named-object check alone used to get: an operator who applies a widening
341
+ * policy at 10:00 should not have it go unnoticed until the host restarts.
342
+ * It is a cache, not a watch — `k8s-client.ts`'s "no watch, no informers, no
343
+ * resourceVersion tracking" invariant is untouched, because the only thing
344
+ * kept across calls is "this exact label set passed at this time".
345
+ */
346
+ export const EGRESS_UNION_CACHE_TTL_MS = 5 * 60 * 1_000;
347
+ /**
348
+ * Build the egress boundary this config asks for, or nothing at all.
349
+ *
350
+ * `sandboxTemplateName` is the template the caller is actually building from
351
+ * — it decides both the default policy name and the pod label the policy's
352
+ * selector has to match, and a workspace may be built from a different
353
+ * template than the task path's.
354
+ *
355
+ * `now` is injected only so the TTL above can be tested without waiting five
356
+ * minutes; nothing else passes it.
357
+ */
358
+ export function buildEgressBoundary(client, config, sandboxTemplateName, now = Date.now) {
359
+ const egress = config.egress;
360
+ if (egress === undefined)
361
+ return undefined;
362
+ const engine = egress.engine ?? 'core';
363
+ // One resolution of the profile, shared by the policy NAME and the policy
364
+ // SELECTOR: under a profile the default name gains the profile segment
365
+ // (one template under two profiles is two policy objects) and the
366
+ // selector gains the label, and the two must not be able to disagree.
367
+ const profile = egressProfileLabel(egress);
368
+ const target = {
369
+ namespace: config.namespace,
370
+ name: egress.networkPolicyName ?? defaultEgressPolicyName(sandboxTemplateName, profile?.value),
371
+ sandboxTemplateName,
372
+ ...(profile !== undefined ? { profile } : {}),
373
+ };
374
+ // Translated ONCE per boundary, not once per check: a `resolver` policy's
375
+ // `resolve()` is the host's own closure and may cost a network call, and
376
+ // running the two checks against two independently resolved allowlists
377
+ // would compare each against a different translation.
378
+ let translation;
379
+ const translate = () => {
380
+ translation ??= translateEgressPolicy(egress.policy, engine, target, egress.ciliumNarrowing).catch((err) => {
381
+ translation = undefined;
382
+ throw err;
383
+ });
384
+ return translation;
385
+ };
386
+ let namedObject;
387
+ const passes = new Map();
388
+ // The per-sandbox selector label carries a once-ever value, so it is
389
+ // excluded from the memo key — see {@link policyCacheKey}. Resolved once
390
+ // here rather than per check, because the resolver validates as it
391
+ // resolves and a per-check throw would surface from a cache lookup.
392
+ const perSandboxKey = perSandboxEgressLabelKey(egress);
393
+ return {
394
+ async verifyNamedObject(signal) {
395
+ namedObject ??= (async () => {
396
+ await verifyEgressPolicyApplied(client, await translate(), signal);
397
+ })().catch((err) => {
398
+ namedObject = undefined;
399
+ throw err;
400
+ });
401
+ await namedObject;
402
+ },
403
+ async verifyUnion(podLabels, subject, signal) {
404
+ if (!egressUnionVerificationEnabled(egress))
405
+ return;
406
+ const translated = await translate();
407
+ const key = policyCacheKey(podLabels, perSandboxKey);
408
+ const cached = passes.get(key);
409
+ if (cached !== undefined && now() - cached.at < EGRESS_UNION_CACHE_TTL_MS) {
410
+ await cached.pending;
411
+ return;
412
+ }
413
+ const pending = verifyEgressPolicyUnion(client, translated, { namespace: config.namespace, podLabels, engine, subject }, signal).catch((err) => {
414
+ // A failed attempt is never cached — same rule the named-object
415
+ // memo has always had.
416
+ passes.delete(key);
417
+ throw err;
418
+ });
419
+ passes.set(key, { at: now(), pending });
420
+ await pending;
421
+ },
422
+ };
423
+ }
424
+ /**
425
+ * Canonical key for one label set — order-independent, so two spellings of
426
+ * the same pod share a memo.
427
+ *
428
+ * `excludeKey` drops the PER-SANDBOX egress label, whose value is unique per
429
+ * acquire. Both memos exist to amortise a namespace-wide policy enumeration
430
+ * across every sandbox a backend produces, and a key that carried a
431
+ * once-ever value would give every acquire a miss and leave an entry behind
432
+ * that nothing ever looks up again — a full enumeration per sandbox, and a
433
+ * Map that grows for the host's whole life.
434
+ *
435
+ * Dropping it is sound at the moment these checks run: the only policy that
436
+ * could select a pod BY that key and value is that sandbox's own, whose name
437
+ * is generated in the same call and which does not exist yet. Every other
438
+ * policy selecting the pod — the operator's baseline, a per-profile one,
439
+ * anything hand-written — selects on the labels that remain, so two pods
440
+ * differing only in this label are the same question. The label itself is
441
+ * still PRESENT in the label set each check is run against; only the memo's
442
+ * key ignores it.
443
+ */
444
+ function policyCacheKey(podLabels, excludeKey) {
445
+ return JSON.stringify(Object.entries(podLabels)
446
+ .filter(([k]) => k !== excludeKey)
447
+ .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)));
448
+ }
449
+ /**
450
+ * Build the ingress check this config asks for, or nothing at all.
451
+ *
452
+ * `cache` is the provider path's per-label-set memo; a workspace passes none,
453
+ * and re-checks on every call for the same reason its egress check does —
454
+ * creating a workspace is a rare, explicit act with nothing to amortise, and
455
+ * a policy deleted since the last call must be noticed.
456
+ */
457
+ export function buildIngressVerifier(client, config, cache) {
458
+ if (!ingressVerificationEnabled(config.ingress))
459
+ return undefined;
460
+ const engine = resolveIngressEngine(config.ingress, config.egress?.engine);
461
+ const agentPort = config.agentPort ?? DEFAULT_AGENT_PORT;
462
+ // Same exclusion, same reason, as the egress union memo above: a
463
+ // once-ever label value in the key would make this memo a per-acquire
464
+ // miss and an unbounded Map. See {@link policyCacheKey}.
465
+ const perSandboxKey = perSandboxEgressLabelKey(config.egress);
466
+ return async (podLabels, subject, signal) => {
467
+ const target = { namespace: config.namespace, podLabels, agentPort, engine, subject };
468
+ if (cache === undefined) {
469
+ await verifyIngressPolicyApplied(client, target, signal);
470
+ return;
471
+ }
472
+ const key = policyCacheKey(podLabels, perSandboxKey);
473
+ let pending = cache.get(key);
474
+ if (pending === undefined) {
475
+ pending = verifyIngressPolicyApplied(client, target, signal).catch((err) => {
476
+ cache.delete(key);
477
+ throw err;
478
+ });
479
+ cache.set(key, pending);
480
+ }
481
+ await pending;
482
+ };
483
+ }
484
+ /** Config → the client's own access shape. Shared with `workspace.ts`. */
485
+ export function clientAccess(config) {
486
+ const access = config.access;
487
+ if (access.inCluster === true)
488
+ return { inCluster: true };
489
+ return {
490
+ server: access.server,
491
+ namespace: config.namespace,
492
+ getToken: access.getToken,
493
+ ...(access.ca !== undefined ? { ca: access.ca } : {}),
494
+ };
495
+ }
496
+ /**
497
+ * An acquire that was refused, carrying WHY in a field rather than in prose.
498
+ *
499
+ * Before this class a burst past node capacity and an API outage were the
500
+ * same plain `Error`, and a host could only tell them apart by matching
501
+ * message text that any release is free to reword. `reason` is the diagnosis,
502
+ * `retryable` is the advice that follows from it, and `cause` is the original
503
+ * failure — unmodified, so a host that already catches
504
+ * {@link ReadinessPollTimeout}, {@link KubernetesApiTimeoutError} or
505
+ * `KubernetesCredentialError` finds it there.
506
+ *
507
+ * `retryable` is about THIS acquire being worth attempting again, not about
508
+ * anything having been retried. Transient API failures are already retried
509
+ * inside the readiness budget, so a `retryable: true` that reaches a caller
510
+ * means the whole budget was spent on them.
511
+ *
512
+ * Not every acquire failure becomes one of these, and that is deliberate: a
513
+ * refusal this class cannot honestly diagnose — a malformed template, a 400
514
+ * from an admission webhook, a controller that reported Ready and named no
515
+ * sandbox — travels out as itself rather than being filed under whichever of
516
+ * the seven reasons is least wrong. `KubernetesApiError` carries the status
517
+ * for those.
518
+ */
519
+ export class KubernetesAcquireError extends Error {
520
+ name = 'KubernetesAcquireError';
521
+ reason;
522
+ /** Whether attempting the same acquire again could plausibly succeed. */
523
+ retryable;
524
+ /**
525
+ * `status.conditions[Ready].reason`, verbatim, when the controller
526
+ * refused the claim — `WarmPoolNotFound`, `TemplateNotFound`,
527
+ * `InvalidMetadata`, `EnvVarsInjectionRejected`. Present only for
528
+ * `'claim-rejected'`.
529
+ */
530
+ controllerReason;
531
+ /** The controller's own message for the same condition. */
532
+ controllerMessage;
533
+ constructor(details) {
534
+ super(details.message, details.cause !== undefined ? { cause: details.cause } : undefined);
535
+ this.reason = details.reason;
536
+ this.retryable = details.retryable;
537
+ if (details.controllerReason !== undefined)
538
+ this.controllerReason = details.controllerReason;
539
+ if (details.controllerMessage !== undefined)
540
+ this.controllerMessage = details.controllerMessage;
541
+ }
542
+ }
543
+ /**
544
+ * The `status.conditions[Ready].reason` values that mean the controller has
545
+ * DECIDED, so waiting is pointless.
546
+ *
547
+ * ## Where these strings came from
548
+ *
549
+ * Not from the issue that asked for this, and not from upstream source: this
550
+ * repo vendors none of agent-sandbox's Go, so a literal copied out of a
551
+ * changelog is a literal nobody here can check. Each of the four was produced
552
+ * against the deployed controller (kind v1.37.0, agent-sandbox v1.0.2,
553
+ * 2026-09-17) by making the claim it describes and reading the condition
554
+ * back:
555
+ *
556
+ * | reason | how it was produced | the controller's message |
557
+ * |---|---|---|
558
+ * | `WarmPoolNotFound` | claim at a pool that does not exist | `SandboxWarmPool "…" not found` |
559
+ * | `TemplateNotFound` | claim at a pool whose template does not exist | `SandboxTemplate "…" not found` |
560
+ * | `InvalidMetadata` | claim with an `additionalPodMetadata` label outside the allowed domains | `invalid additionalPodMetadata: …` |
561
+ * | `EnvVarsInjectionRejected` | claim with `spec.env` against a template that forbids injection | `environment variable injection rejected: …` |
562
+ *
563
+ * The transient reasons seen on the SAME cluster, which must NOT be in this
564
+ * set, were `DependenciesNotReady` (pod exists, still Pending) and
565
+ * `DependenciesReady` (the Ready=True reason).
566
+ *
567
+ * ## Why an unknown reason is not terminal
568
+ *
569
+ * A wrong literal here fails in one of two ways, and only one of them is
570
+ * recoverable. Too few entries: a rejected claim waits out the readiness
571
+ * budget, which is exactly the behaviour every release before this one had.
572
+ * Too many: an acquire that would have succeeded is refused on a guess. So
573
+ * the set is a closed list of measured strings and everything else falls
574
+ * through to the deadline.
575
+ */
576
+ // Frozen because it is exported from the package root: an array handed to
577
+ // every consumer is one a cast can push onto, and an entry added there would
578
+ // change fail-fast for the whole process. The `readonly string[]` annotation
579
+ // is deliberate rather than `as const` — `includes` on a literal tuple only
580
+ // accepts the literals, and the whole point is to ask it about a reason no
581
+ // one here has seen.
582
+ export const TERMINAL_CLAIM_REASONS = Object.freeze([
583
+ 'WarmPoolNotFound',
584
+ 'TemplateNotFound',
585
+ 'InvalidMetadata',
586
+ 'EnvVarsInjectionRejected',
587
+ ]);
588
+ /**
589
+ * The `Ready` condition a claim reports, whatever its status — the one
590
+ * {@link isConditionTrue} deliberately cannot return, because it answers a
591
+ * boolean question and this one needs the reason.
592
+ */
593
+ function readyCondition(conditions) {
594
+ return conditions?.find((c) => c.type === READY_CONDITION);
595
+ }
596
+ /**
597
+ * A claim the controller has refused, or `undefined` for one it is still
598
+ * working on.
599
+ *
600
+ * `status: 'False'` alone is not a refusal — it is also what a claim looks
601
+ * like for the whole of a cold start — so the REASON decides, against
602
+ * {@link TERMINAL_CLAIM_REASONS}.
603
+ *
604
+ * ONE class comes out of here whatever the reason, and that is deliberate:
605
+ * `InvalidMetadata` is how the controller refuses a pod label whose domain is
606
+ * not on its allowlist — the profile's, today — and a host's `catch` must not
607
+ * have to be written differently depending on whether a profile happens to be
608
+ * configured. `metadata` only decides what rides along as the `cause`: a
609
+ * {@link KubernetesPodLabelsRejectedError} naming the map that was sent
610
+ * and the `config.egress.profileLabelKey` that moves it, which the
611
+ * controller's own message cannot know about.
612
+ */
613
+ export function classifyClaimRejection(claim, claimName, metadata) {
614
+ const condition = readyCondition(claim?.status?.conditions);
615
+ if (condition === undefined || condition.status !== 'False')
616
+ return undefined;
617
+ const reason = condition.reason;
618
+ if (reason === undefined || !TERMINAL_CLAIM_REASONS.includes(reason))
619
+ return undefined;
620
+ // Only for the reason the pod metadata can actually cause, and only when
621
+ // this backend sent any: a `WarmPoolNotFound` carrying a labels-and-
622
+ // allowlist explanation would send an operator after the wrong thing.
623
+ const cause = reason === 'InvalidMetadata' &&
624
+ metadata !== undefined &&
625
+ Object.keys(metadata.requestedPodLabels).length > 0
626
+ ? new KubernetesPodLabelsRejectedError(metadata.requestedPodLabels, claimName, metadata.namespace, reason, condition.message ?? '(the controller reported no message)', metadata.profile)
627
+ : undefined;
628
+ return new KubernetesAcquireError({
629
+ reason: 'claim-rejected',
630
+ retryable: false,
631
+ controllerReason: reason,
632
+ ...(condition.message !== undefined ? { controllerMessage: condition.message } : {}),
633
+ ...(cause !== undefined ? { cause } : {}),
634
+ message: `kubernetes: the agent-sandbox controller refused SandboxClaim ${claimName} with reason ${reason}${condition.message !== undefined ? `: ${condition.message}` : ''}. That is a decision, not a delay, so the readiness budget was not waited out.${cause !== undefined ? ` ${cause.message}` : ''}`,
635
+ });
636
+ }
637
+ /**
638
+ * How long to wait before repeating a failed readiness read, or `undefined`
639
+ * when the failure is not worth repeating.
640
+ *
641
+ * Retryable: a connect failure (the socket, not the answer), a request the
642
+ * `apiRequestTimeoutMs` bound gave up on, a 429 (the API server's own
643
+ * priority-and-fairness queue shedding load) and any 5xx. Not retryable: 401
644
+ * and 403, which are a decision; 404 and 410, which are an answer; 409, which
645
+ * a caller resolves by re-reading; and everything this backend threw itself.
646
+ *
647
+ * The wait is the poll's own cadence unless the server named one — then its
648
+ * `Retry-After`, because the server knows when its queue drains and this code
649
+ * does not. Nothing here consults a clock: the caller's deadline owns the
650
+ * sleep, so a long `Retry-After` spends the readiness budget rather than
651
+ * extending it.
652
+ */
653
+ /**
654
+ * The largest delay a timer can hold — `2^31 - 1` ms, Node's own ceiling.
655
+ * Above it `setTimeout` warns and fires immediately, which is the opposite of
656
+ * what a long `Retry-After` asked for.
657
+ */
658
+ const MAX_RETRY_DELAY_MS = 2_147_483_647;
659
+ export function retryDelayForApiFailure(err, pollIntervalMs) {
660
+ if (err instanceof KubernetesApiTimeoutError)
661
+ return pollIntervalMs;
662
+ if (!(err instanceof KubernetesApiError))
663
+ return undefined;
664
+ if (err.transport === 'connect')
665
+ return pollIntervalMs;
666
+ const status = err.status;
667
+ if (status === undefined)
668
+ return undefined;
669
+ // `Retry-After` may only ever SLOW the poll down, and only within what a
670
+ // timer can express. A header of `0`, or one naming a moment already past,
671
+ // would otherwise turn the retry into a hot loop against a server that is
672
+ // already shedding load — the caller's own cadence is the rate this loop
673
+ // runs at when nothing is wrong. And a header naming a moment years away
674
+ // overflows `setTimeout`, which then fires at once rather than never,
675
+ // producing the same hot loop from the opposite direction. The readiness
676
+ // deadline ends the wait either way; the clamp only stops the wait from
677
+ // silently becoming no wait at all.
678
+ if (status === 429 || status >= 500) {
679
+ const asked = err.retryAfterMs ?? pollIntervalMs;
680
+ return Math.min(Math.max(asked, pollIntervalMs), MAX_RETRY_DELAY_MS);
681
+ }
682
+ return undefined;
683
+ }
684
+ /**
685
+ * How long the one diagnostic pod read after a failed acquire may take.
686
+ *
687
+ * Same shape and the same argument as `runFailureCleanup`'s grace: the
688
+ * readiness clock has already expired, so this cannot share it, and a
689
+ * diagnosis that could hang would keep `create()` pending past the budget the
690
+ * caller chose — for a nicer error message. One second, and a diagnosis that
691
+ * does not arrive is simply not made.
692
+ */
693
+ const ACQUIRE_DIAGNOSIS_GRACE_MS = 1_000;
694
+ /**
695
+ * The image-pull `status.containerStatuses[].state.waiting.reason` values the
696
+ * kubelet reports. Measured on kind v1.37.0 (2026-09-17): a container whose
697
+ * image does not exist waits as `ErrImagePull` for the first attempts and
698
+ * settles into `ImagePullBackOff`. The other two are the kubelet's names for
699
+ * a pull that resolved and then failed, and for a registry that cannot be
700
+ * reached at all.
701
+ */
702
+ const IMAGE_PULL_WAITING_REASONS = [
703
+ 'ErrImagePull',
704
+ 'ImagePullBackOff',
705
+ 'ImageInspectError',
706
+ 'RegistryUnavailable',
707
+ ];
708
+ /**
709
+ * Ask the pod why it is not ready, once, after the budget has already gone.
710
+ *
711
+ * Nothing on the healthy path calls this and nothing waits on it: it runs
712
+ * exactly when an acquire has already failed, and its whole output is a
713
+ * better {@link KubernetesAcquireFailureReason} than `'not-ready'`. A read
714
+ * that fails, a pod that is not there and a pod with nothing to say all
715
+ * produce `undefined`, which leaves the reason where it was.
716
+ *
717
+ * It must run BEFORE the cleanup DELETE, because the pod goes away with the
718
+ * object it belongs to.
719
+ *
720
+ * It takes the CALLER's signal where `runFailureCleanup` deliberately does
721
+ * not: cleanup must finish or the cluster keeps the object, while a diagnosis
722
+ * is only a better sentence for an error a caller who aborted will never
723
+ * read.
724
+ */
725
+ async function diagnoseUnreadyPod(client, namespace, podName, callerSignal) {
726
+ let pod;
727
+ try {
728
+ const deadline = new OperationDeadline(ACQUIRE_DIAGNOSIS_GRACE_MS, 'kubernetes acquire diagnosis', callerSignal);
729
+ pod = await deadline.run((signal) => client.request('GET', podPath(namespace, podName), undefined, signal));
730
+ }
731
+ catch {
732
+ // The acquire failure is the primary one and keeps its reason. A
733
+ // diagnosis that cannot be made is not a second failure to report.
734
+ return undefined;
735
+ }
736
+ const scheduled = pod?.status?.conditions?.find((c) => c.type === 'PodScheduled');
737
+ if (scheduled?.status === 'False' && scheduled.reason === 'Unschedulable')
738
+ return 'capacity';
739
+ for (const container of pod?.status?.containerStatuses ?? []) {
740
+ const reason = container.state?.waiting?.reason;
741
+ if (reason !== undefined && IMAGE_PULL_WAITING_REASONS.includes(reason))
742
+ return 'image-pull';
743
+ }
744
+ return undefined;
745
+ }
746
+ /**
747
+ * The refusal a caller sees, given the failure that actually happened and
748
+ * whatever the pod had to say about it.
749
+ *
750
+ * Returns `undefined` for a failure none of the seven reasons describes —
751
+ * see {@link KubernetesAcquireError} for why that is a deliberate hole rather
752
+ * than a missing case. A {@link KubernetesAcquireError} that arrived from
753
+ * deeper in (the claim rejection) is returned unchanged: it is already the
754
+ * diagnosis.
755
+ */
756
+ export function classifyAcquireFailure(err, podDiagnosis) {
757
+ if (err instanceof KubernetesAcquireError)
758
+ return err;
759
+ if (err instanceof KubernetesApiTimeoutError) {
760
+ return new KubernetesAcquireError({
761
+ reason: 'api-timeout',
762
+ retryable: true,
763
+ message: `kubernetes: the acquire was refused because the API server did not answer in time — ${err.message}`,
764
+ cause: err,
765
+ });
766
+ }
767
+ if (err instanceof KubernetesCredentialError) {
768
+ return new KubernetesAcquireError({
769
+ reason: 'forbidden',
770
+ retryable: false,
771
+ message: `kubernetes: the acquire was refused because the API server rejected this host's credential — ${err.message}. Check the host ServiceAccount's Role against the RBAC section of the Kubernetes sandbox documentation.`,
772
+ cause: err,
773
+ });
774
+ }
775
+ if (err instanceof KubernetesApiError) {
776
+ // The same predicate the poll retries on, so "worth trying again" has
777
+ // one definition and a caller cannot be told a failure is retryable
778
+ // that the poll would have declined to retry. The interval is
779
+ // irrelevant here — only whether an answer comes back at all.
780
+ if (retryDelayForApiFailure(err, 1) === undefined)
781
+ return undefined;
782
+ return new KubernetesAcquireError({
783
+ reason: 'api-unreachable',
784
+ retryable: true,
785
+ message: `kubernetes: the acquire was refused because the API server could not serve it — ${err.message}`,
786
+ cause: err,
787
+ });
788
+ }
789
+ if (err instanceof ReadinessPollTimeout) {
790
+ // ORDER MATTERS, and this is the order: what the cluster SAID beats
791
+ // what the failures suggest. A pod diagnosis is a condition the API
792
+ // server published about this pod, read after the budget had already
793
+ // gone; `err.cause` is at best the failure the poll was still meeting
794
+ // at that moment. On a saturated cluster both are present at once — a
795
+ // pod nothing can schedule AND an API server shedding load — and
796
+ // reporting `api-unreachable` there would hide the very reason this
797
+ // function exists to produce, and would turn `image-pull`'s
798
+ // `retryable: false` into a `true` that has a host retrying forever
799
+ // against an image reference that will never resolve. A diagnosis also
800
+ // cannot be stale in the way a cause can: it only exists because the
801
+ // API server answered one more read, moments ago.
802
+ if (podDiagnosis === 'capacity') {
803
+ return new KubernetesAcquireError({
804
+ reason: 'capacity',
805
+ retryable: true,
806
+ message: `kubernetes: the acquire was refused because its pod could not be scheduled — the cluster reports PodScheduled=False/Unschedulable. ${err.message}`,
807
+ cause: err,
808
+ });
809
+ }
810
+ if (podDiagnosis === 'image-pull') {
811
+ return new KubernetesAcquireError({
812
+ reason: 'image-pull',
813
+ retryable: false,
814
+ message: `kubernetes: the acquire was refused because its pod's container image will not pull. ${err.message}`,
815
+ cause: err,
816
+ });
817
+ }
818
+ // Nothing measured, so the failures the poll kept meeting decide: a
819
+ // poll that spent its budget retrying API failures did not fail
820
+ // because the sandbox was slow; it failed because the control plane
821
+ // was. `pollForBinding` carries the failure it was STILL meeting onto
822
+ // the timeout, and the reason follows it rather than the timeout.
823
+ const underlying = classifyAcquireFailure(err.cause, undefined);
824
+ if (underlying !== undefined) {
825
+ return new KubernetesAcquireError({
826
+ reason: underlying.reason,
827
+ retryable: underlying.retryable,
828
+ message: underlying.message,
829
+ cause: err,
830
+ });
831
+ }
832
+ return new KubernetesAcquireError({
833
+ reason: 'not-ready',
834
+ retryable: true,
835
+ message: err.message,
836
+ cause: err,
837
+ });
838
+ }
839
+ if (err instanceof OperationDeadlineExpired) {
840
+ return new KubernetesAcquireError({
841
+ reason: 'not-ready',
842
+ retryable: true,
843
+ message: `kubernetes: the acquire ran out of readiness budget — ${err.message}`,
844
+ cause: err,
845
+ });
846
+ }
847
+ return undefined;
848
+ }
849
+ /**
850
+ * Claim or create, wait for Ready, read the bound identity back, resolve the
851
+ * address and learn the pod's uid — or leave nothing behind trying.
852
+ *
853
+ * Exported because the sandbox surface is built on top of this record rather
854
+ * than beside it: one acquire path, one cleanup path, whatever ends up
855
+ * wrapping them.
856
+ *
857
+ * ## What it refuses with
858
+ *
859
+ * Every refusal this function can diagnose arrives as a
860
+ * {@link KubernetesAcquireError} naming one of seven reasons, with the
861
+ * original failure as its `cause`. Three things stay outside that:
862
+ * configuration refused before anything is created
863
+ * ({@link assertEnforceable}, {@link assertRuntimeClassIsApplicable}), a
864
+ * caller's own abort, and a failure none of the seven reasons honestly
865
+ * describes — see {@link KubernetesAcquireError} for why the last one is a
866
+ * hole on purpose.
867
+ *
868
+ * ## What it retries, and what it will not
869
+ *
870
+ * A readiness GET that fails transiently — a connect failure, a request the
871
+ * API bound gave up on, a 429, a 5xx — is repeated INSIDE the readiness
872
+ * deadline, honouring `Retry-After`. One clock, so a retry spends the budget
873
+ * rather than extending it, and a `create()` cannot outlive the timeout its
874
+ * caller chose. The create POST is never retried: it is not idempotent, and a
875
+ * POST whose answer never arrived may already have committed — which is why
876
+ * cleanup deletes the client-owned name whatever happened.
877
+ */
878
+ export async function acquireKubernetesSandbox(client, config, options, readiness, verifyIngress = buildIngressVerifier(client, config), egressBoundary = buildEgressBoundary(client, config, config.sandboxTemplateName)) {
879
+ options.signal?.throwIfAborted();
880
+ assertEnforceable(options);
881
+ assertRuntimeClassIsApplicable(config);
882
+ const namespace = config.namespace;
883
+ const ttlSeconds = config.claimTtlSeconds ?? DEFAULT_CLAIM_TTL_SECONDS;
884
+ const shutdownTime = new Date(Date.now() + ttlSeconds * 1_000).toISOString();
885
+ // Client-owned name, as on ACI: it lets failure cleanup DELETE the object
886
+ // even when the create response never arrived. `generateSandboxId` returns
887
+ // a lowercase UUID, which is already a legal DNS-1123 name suffix.
888
+ const objectName = `namzu-task-${generateSandboxId()}`;
889
+ const ownedPath = config.warmPoolName !== undefined
890
+ ? claimPath(namespace, objectName)
891
+ : sandboxPath(namespace, objectName);
892
+ const release = async (signal) => {
893
+ try {
894
+ await client.request('DELETE', ownedPath, undefined, signal);
895
+ }
896
+ catch (err) {
897
+ // The object is gone, which is the state DELETE was asking for.
898
+ if (!(err instanceof KubernetesAlreadyGoneError))
899
+ throw err;
900
+ }
901
+ };
902
+ // Where the expiry lives differs by KIND, and only this function knows
903
+ // which kind it created: a claim keeps it under `spec.lifecycle`, a
904
+ // directly created Sandbox at the top of `spec` (v1beta1 as served has
905
+ // not moved it under `lifecycle` yet). Merge-patch semantics (RFC 7386,
906
+ // the only content type this client's PATCH sends) merge the nested
907
+ // object, so `shutdownPolicy: Delete` survives every renewal.
908
+ // The parameter is deliberately NOT named `shutdownTime`: the stamp above
909
+ // is the one the create body carries, and no renewal ever re-sends it.
910
+ const renew = async (nextShutdownTime, signal) => {
911
+ const patch = config.warmPoolName !== undefined
912
+ ? { spec: { lifecycle: { shutdownTime: nextShutdownTime } } }
913
+ : { spec: { shutdownTime: nextShutdownTime } };
914
+ await client.request('PATCH', ownedPath, patch, signal);
915
+ };
916
+ // One clock over the whole path — the create POST included, so a hung API
917
+ // server cannot leave `create()` pending past the caller's timeout.
918
+ const deadline = new OperationDeadline(readiness.timeoutMs, 'kubernetes readiness', options.signal);
919
+ // The pool-less path reads its pod template BEFORE anything is created, so
920
+ // a missing or malformed template fails with nothing to clean up — hence
921
+ // this sits outside the cleanup block below. It is read per create rather
922
+ // than cached: an operator editing the template expects the next sandbox to
923
+ // use it, and this path is not the sub-second one.
924
+ const createPath = config.warmPoolName !== undefined
925
+ ? claimCollectionPath(namespace)
926
+ : sandboxCollectionPath(namespace);
927
+ // Composed ONCE, here, and handed to whichever body builder runs below:
928
+ // the claim's `additionalPodMetadata.labels` and a direct Sandbox's pod
929
+ // template metadata are the same map, and the translated policy's selector
930
+ // is built from the same resolution. See `composeAdditionalPodLabels`.
931
+ // Empty — the only case before a profile is configured — means every body
932
+ // below is byte for byte what it was.
933
+ // The per-sandbox selector label rides in the SAME map, through the same
934
+ // composer, as a second key rather than a second construction — the whole
935
+ // reason `composeAdditionalPodLabels` takes an `extra`. Its value is the
936
+ // name of the object this acquire is about to create, which is unique per
937
+ // acquire and known BEFORE the POST: that is what lets the label travel as
938
+ // claim-time pod metadata (warm-safe, no cold start) instead of as a patch
939
+ // to a running pod this backend has no verb for.
940
+ const perSandboxLabelKey = perSandboxEgressLabelKey(config.egress);
941
+ const podLabels = composeAdditionalPodLabels(config.egress, perSandboxLabelKey !== undefined ? { [perSandboxLabelKey]: objectName } : undefined);
942
+ const profile = egressProfileLabel(config.egress);
943
+ // Read off the create reply, and off the readiness polls if that reply
944
+ // carried no object — it is the uid a per-sandbox policy's
945
+ // `ownerReferences` names and the suffix of its name, so the cluster can
946
+ // garbage-collect the policy with the object this backend owns.
947
+ let ownerUid;
948
+ const buildCreateBody = async () => {
949
+ if (config.warmPoolName !== undefined) {
950
+ return buildClaimBody({
951
+ namespace,
952
+ name: objectName,
953
+ warmPoolName: config.warmPoolName,
954
+ shutdownTime,
955
+ ...(config.claimLabels !== undefined ? { labels: config.claimLabels } : {}),
956
+ podLabels,
957
+ });
958
+ }
959
+ const template = await deadline.run((signal) => readSandboxTemplate(client, namespace, config.sandboxTemplateName, signal));
960
+ // A direct Sandbox's pod labels are decided HERE, by the body below,
961
+ // so both network boundaries are checked against the real labels before
962
+ // anything is created — a refusal leaves no Sandbox and no PVC behind
963
+ // rather than one of each to clean up. This is EARLIER than the plan
964
+ // for the egress union check asked for (it said "after binding", for
965
+ // the bound pod's labels); a pool-less Sandbox's labels are knowable
966
+ // before the POST, and refusing with nothing created is strictly
967
+ // better than refusing with an object to clean up.
968
+ const directPodLabels = sandboxPodLabels(template, config.sandboxTemplateName, podLabels);
969
+ const directSubject = `to create Sandbox ${objectName} in namespace ${namespace}`;
970
+ if (verifyIngress !== undefined) {
971
+ await deadline.run((signal) => verifyIngress(directPodLabels, directSubject, signal));
972
+ }
973
+ if (egressBoundary !== undefined) {
974
+ await deadline.run((signal) => egressBoundary.verifyUnion(directPodLabels, directSubject, signal));
975
+ }
976
+ return buildSandboxBody({
977
+ namespace,
978
+ name: objectName,
979
+ template,
980
+ sandboxTemplateName: config.sandboxTemplateName,
981
+ shutdownTime,
982
+ podLabels,
983
+ ...(config.runtimeClassName !== undefined
984
+ ? { runtimeClassName: config.runtimeClassName }
985
+ : {}),
986
+ });
987
+ };
988
+ let createBody;
989
+ try {
990
+ createBody = await buildCreateBody();
991
+ }
992
+ catch (err) {
993
+ // Nothing exists yet, so there is nothing to clean up and no pod to
994
+ // ask — but a 403 on the template read is still a `forbidden` acquire,
995
+ // and a caller should not have to tell that apart by where it
996
+ // happened.
997
+ throw classifyAcquireFailure(err, undefined) ?? err;
998
+ }
999
+ // The pod the diagnosis below asks, when there is one. A directly created
1000
+ // Sandbox is backed by a pod of its own name; a CLAIM's pod is not knowable
1001
+ // until the controller has named a sandbox in `status.sandbox`, which it
1002
+ // does before Ready on a cold start and never on a rejected claim.
1003
+ let diagnosablePodName = config.warmPoolName !== undefined ? undefined : objectName;
1004
+ // One definition of "worth trying again", shared by the poll that retries
1005
+ // and the classification that reports — see {@link retryDelayForApiFailure}.
1006
+ const pollBehaviour = {
1007
+ retryDelayFor: (err) => retryDelayForApiFailure(err, readiness.pollIntervalMs),
1008
+ };
1009
+ try {
1010
+ // Inside the cleanup block: a POST that fails client-side may still have
1011
+ // committed, so the only safe assumption is that the object exists.
1012
+ const created = await deadline.run((signal) => client.request('POST', createPath, createBody, signal));
1013
+ ownerUid ??= created?.metadata?.uid;
1014
+ const binding = config.warmPoolName !== undefined
1015
+ ? await pollForBinding(async (signal) => {
1016
+ const claim = await client.request('GET', claimPath(namespace, objectName), undefined, signal);
1017
+ diagnosablePodName = claim?.status?.sandbox?.name ?? diagnosablePodName;
1018
+ ownerUid ??= claim?.metadata?.uid;
1019
+ // The pod labels this backend asked the controller for
1020
+ // travel into the read, not because the fail-fast needs
1021
+ // them — `InvalidMetadata` is already one of
1022
+ // `TERMINAL_CLAIM_REASONS`, and that is the ONE
1023
+ // taxonomy a refused claim is reported under — but so
1024
+ // the refusal can carry what was actually sent and how
1025
+ // to change it. See {@link ClaimPodMetadataContext}.
1026
+ return bindingFromClaim(claim, objectName, {
1027
+ namespace,
1028
+ requestedPodLabels: podLabels,
1029
+ ...(profile !== undefined ? { profile } : {}),
1030
+ });
1031
+ }, deadline, readiness, `claim ${objectName}`, pollBehaviour)
1032
+ : await pollForBinding(async (signal) => {
1033
+ const sandbox = await client.request('GET', sandboxPath(namespace, objectName), undefined, signal);
1034
+ ownerUid ??= sandbox?.metadata?.uid;
1035
+ return bindingFromSandbox(sandbox);
1036
+ }, deadline, readiness, `sandbox ${objectName}`, pollBehaviour);
1037
+ const agentPort = config.agentPort ?? DEFAULT_AGENT_PORT;
1038
+ const mode = config.agentAddress ?? 'service';
1039
+ // One read, two facts: the bind token and — under `'pod-ip'` — the
1040
+ // address, off the same pod. See {@link readAddressedPod}, which under
1041
+ // a configured profile also waits for the label the controller
1042
+ // patches onto the bound pod before anything is admitted.
1043
+ const pod = await readAddressedPod(client, namespace, binding, deadline, readiness, mode, podLabels);
1044
+ // The one refusal this capability must not skip: an unlabelled pod
1045
+ // handed back runs under whatever policy DOES select it while the host
1046
+ // believes it is on a narrower profile. This throws INSIDE the try
1047
+ // block, so the cleanup below releases the claim (or deletes the
1048
+ // Sandbox) exactly as a failed privilege probe does — and the probe
1049
+ // itself, which runs in `admitProbedSandbox` after this function
1050
+ // returns, is therefore never reached with the label unobserved.
1051
+ assertRequestedPodLabelsObserved(pod, podLabels, `Sandbox ${binding.name} in namespace ${namespace}`);
1052
+ // A CLAIMED sandbox's pod was built from the pool's own template, so
1053
+ // its labels are not knowable until the controller has bound one.
1054
+ // Checking here rather than not at all is the trade: a refusal
1055
+ // releases the claim through the cleanup below, which is the same
1056
+ // path a failed privilege probe takes.
1057
+ if (config.warmPoolName !== undefined) {
1058
+ const boundSubject = `the pod bound to Sandbox ${binding.name} in namespace ${namespace}`;
1059
+ if (verifyIngress !== undefined) {
1060
+ await deadline.run((signal) => verifyIngress(pod.labels ?? {}, boundSubject, signal));
1061
+ }
1062
+ // The egress union check asks the same question of the same labels
1063
+ // — which policies select THIS pod — so it runs at the same two
1064
+ // points, and a refusal here releases the claim through the cleanup
1065
+ // below, exactly as a failed privilege probe does.
1066
+ if (egressBoundary !== undefined) {
1067
+ await deadline.run((signal) => egressBoundary.verifyUnion(pod.labels ?? {}, boundSubject, signal));
1068
+ }
1069
+ }
1070
+ // Only when the capability is configured, and only after the pod has
1071
+ // been confirmed to carry the selector label: a policy written for a
1072
+ // pod that never got the label would select nothing while the caller
1073
+ // was told its egress had been narrowed.
1074
+ const owner = perSandboxLabelKey === undefined
1075
+ ? undefined
1076
+ : {
1077
+ kind: (config.warmPoolName !== undefined ? 'SandboxClaim' : 'Sandbox'),
1078
+ name: objectName,
1079
+ uid: assertOwnerUid(ownerUid, objectName, namespace),
1080
+ };
1081
+ return {
1082
+ binding,
1083
+ agent: resolveAgentAddress(binding, agentPort, pod.uid, {
1084
+ mode,
1085
+ ...(pod.podIP !== undefined ? { podIP: pod.podIP } : {}),
1086
+ }),
1087
+ // Only the literal-address mode gets one — see the field.
1088
+ ...(mode === 'pod-ip'
1089
+ ? { refreshAgent: buildAgentAddressRefresh(client, namespace, binding, agentPort, mode) }
1090
+ : {}),
1091
+ ownedPath,
1092
+ ...(owner !== undefined ? { owner, perSandboxLabelValue: objectName } : {}),
1093
+ ttlSeconds,
1094
+ release,
1095
+ renew,
1096
+ };
1097
+ }
1098
+ catch (err) {
1099
+ // Asked BEFORE cleanup, because the pod goes away with the object, and
1100
+ // only for a timeout — every other failure already knows what it was.
1101
+ const podDiagnosis = err instanceof ReadinessPollTimeout && diagnosablePodName !== undefined
1102
+ ? await diagnoseUnreadyPod(client, namespace, diagnosablePodName, options.signal)
1103
+ : undefined;
1104
+ // One cleanup for every way out of the block above, on its own short
1105
+ // budget: the readiness clock has already expired in the common case,
1106
+ // so spending it again would either skip cleanup or leave `create()`
1107
+ // pending without a bound. An object that is already gone is success.
1108
+ await runFailureCleanup(async (signal) => {
1109
+ await release(signal);
1110
+ });
1111
+ throw classifyAcquireFailure(err, podDiagnosis) ?? err;
1112
+ }
1113
+ }
1114
+ function buildClaimBody(options) {
1115
+ const { labels, podLabels } = options;
1116
+ return {
1117
+ apiVersion: `${SANDBOX_EXTENSIONS_API_GROUP}/${SANDBOX_API_VERSION}`,
1118
+ kind: 'SandboxClaim',
1119
+ metadata: {
1120
+ name: options.name,
1121
+ namespace: options.namespace,
1122
+ ...(labels !== undefined && Object.keys(labels).length > 0 ? { labels } : {}),
1123
+ },
1124
+ spec: {
1125
+ warmPoolRef: { name: options.warmPoolName },
1126
+ ...(podLabels !== undefined && Object.keys(podLabels).length > 0
1127
+ ? { additionalPodMetadata: { labels: podLabels } }
1128
+ : {}),
1129
+ lifecycle: { shutdownTime: options.shutdownTime, shutdownPolicy: 'Delete' },
1130
+ },
1131
+ };
1132
+ }
1133
+ /**
1134
+ * Refuse a bound pod that does not carry EVERY label this backend asked the
1135
+ * controller to put on it — the egress profile today, and whatever else
1136
+ * `composeAdditionalPodLabels` later contributes to the same map.
1137
+ *
1138
+ * The same set {@link readAddressedPod} waits for, deliberately: a wait that
1139
+ * covered more than the refusal would burn the readiness budget on a label
1140
+ * nothing then checked, and a refusal that covered more than the wait would
1141
+ * refuse a pod that had simply not been patched yet. An empty map (the
1142
+ * unprofiled path) checks nothing and returns.
1143
+ *
1144
+ * Called after {@link readAddressedPod} has already waited on the readiness
1145
+ * deadline, so reaching a missing label here means it never arrived, not that
1146
+ * it had not arrived yet.
1147
+ */
1148
+ function assertRequestedPodLabelsObserved(pod, requested, subject) {
1149
+ for (const [key, value] of Object.entries(requested)) {
1150
+ if (pod.labels?.[key] === value)
1151
+ continue;
1152
+ throw new KubernetesPodLabelNotObservedError({ key, value }, subject, pod.labels ?? {});
1153
+ }
1154
+ }
1155
+ /**
1156
+ * The uid of the object this acquire created, or a refusal naming what was
1157
+ * missing.
1158
+ *
1159
+ * Only reached when `config.egress.perSandbox` is configured, and only after
1160
+ * a create reply and every readiness poll have been read without one — the
1161
+ * API server assigns `metadata.uid` on admission and returns the object it
1162
+ * created, so an object with no uid is an API server that answered something
1163
+ * other than what it was asked for. Refusing is right: without the uid there
1164
+ * is no owner reference, and a per-sandbox policy with no owner is one the
1165
+ * cluster never collects.
1166
+ */
1167
+ function assertOwnerUid(uid, objectName, namespace) {
1168
+ if (uid !== undefined && uid !== '')
1169
+ return uid;
1170
+ throw new KubernetesOwnerUidMissingError(objectName, namespace);
1171
+ }
1172
+ /**
1173
+ * Recover a crashed host's claims: LIST every `SandboxClaim` carrying
1174
+ * `labelSelector`, `DELETE` each, and report what was removed.
1175
+ *
1176
+ * This deletes CLAIMS only. The controller's own garbage collection —
1177
+ * ownerReferences from claim to the Sandbox it bound, and from Sandbox to
1178
+ * Pod and Service — takes the rest down behind it; nothing here reads or
1179
+ * touches a Sandbox or a Pod directly. A claim already gone (raced by the
1180
+ * controller's own TTL reaper, or a second release call) counts as removed
1181
+ * rather than a failure, the same convention every other DELETE in this
1182
+ * backend follows.
1183
+ *
1184
+ * `labelSelector` is REQUIRED and refused, synchronously, before a single
1185
+ * request goes out, if it is absent or empty: a release that could fall back
1186
+ * to matching every claim (or every claim of the template) would delete a
1187
+ * live fleet's work the first time a caller passed one by mistake. There is
1188
+ * no default selector for exactly this reason.
1189
+ */
1190
+ export async function releaseKubernetesTaskSandboxes(config, options) {
1191
+ if (options.labelSelector === '') {
1192
+ throw new Error('kubernetes: releaseKubernetesTaskSandboxes requires a non-empty labelSelector — a release with no selector would delete every SandboxClaim in the namespace, including ones a live host still owns. Pass the selector that names only the claims you mean to recover.');
1193
+ }
1194
+ options.signal?.throwIfAborted();
1195
+ const namespace = config.namespace;
1196
+ const client = createKubernetesClient(clientAccess(config), clientOptions(config));
1197
+ const list = await client.request('GET', claimListPath(namespace, options.labelSelector), undefined, options.signal);
1198
+ const names = (list?.items ?? [])
1199
+ .map((claim) => claim.metadata?.name)
1200
+ .filter((name) => typeof name === 'string' && name !== '');
1201
+ // Concurrent, not one at a time: a crash-recovery release can carry a
1202
+ // whole host's worth of claims, and nothing here needs the ordering a
1203
+ // sequential loop would impose — each DELETE is independent and
1204
+ // idempotent (an already-gone claim is tolerated below). A non-tolerated
1205
+ // failure still rejects the whole call, exactly as a sequential loop
1206
+ // would have on its first such failure.
1207
+ await Promise.all(names.map(async (name) => {
1208
+ try {
1209
+ await client.request('DELETE', claimPath(namespace, name), undefined, options.signal);
1210
+ }
1211
+ catch (err) {
1212
+ if (!(err instanceof KubernetesAlreadyGoneError))
1213
+ throw err;
1214
+ }
1215
+ }));
1216
+ return { deleted: names.length, names };
1217
+ }
1218
+ /**
1219
+ * Read task-pool headroom before admitting more work: three GETs, no writes.
1220
+ *
1221
+ * `config.warmPoolName` is required — this reads the exact object a claim's
1222
+ * `warmPoolRef` names, so a pool-less backend (every create is a direct
1223
+ * Sandbox) has no pool to report on. `activeClaims` is every claim in the
1224
+ * namespace whose `spec.warmPoolRef.name` matches this pool, counted rather
1225
+ * than trusted from a label, because a claim's `warmPoolRef` is the one
1226
+ * field the API itself guarantees. `pendingPods` is every Pod in the
1227
+ * namespace still in phase `Pending` — a coarse but honest signal of
1228
+ * in-flight scale-up the ready-replica count alone does not carry, on
1229
+ * either the warm or the pool-less path.
1230
+ */
1231
+ export async function readKubernetesTaskCapacity(config, options) {
1232
+ options?.signal?.throwIfAborted();
1233
+ if (config.warmPoolName === undefined) {
1234
+ throw new Error('kubernetes: readKubernetesTaskCapacity requires config.warmPoolName — there is no SandboxWarmPool to report on for a backend that creates every sandbox directly.');
1235
+ }
1236
+ const namespace = config.namespace;
1237
+ const warmPoolName = config.warmPoolName;
1238
+ const client = createKubernetesClient(clientAccess(config), clientOptions(config));
1239
+ const [pool, claims, pods] = await Promise.all([
1240
+ client.request('GET', warmPoolPath(namespace, warmPoolName), undefined, options?.signal),
1241
+ client.request('GET', claimCollectionPath(namespace), undefined, options?.signal),
1242
+ client.request('GET', podCollectionPath(namespace), undefined, options?.signal),
1243
+ ]);
1244
+ const activeClaims = (claims?.items ?? []).filter((claim) => claim.spec?.warmPoolRef?.name === warmPoolName).length;
1245
+ const pendingPods = (pods?.items ?? []).filter((pod) => pod.status?.phase === 'Pending').length;
1246
+ return {
1247
+ warmPool: {
1248
+ ready: pool?.status?.readyReplicas ?? 0,
1249
+ desired: pool?.spec?.replicas ?? 0,
1250
+ },
1251
+ activeClaims,
1252
+ pendingPods,
1253
+ };
1254
+ }
1255
+ /**
1256
+ * The pool-less body. `Sandbox.spec` has no `templateRef` — only a
1257
+ * SandboxWarmPool consumes a SandboxTemplate — so the template's podTemplate
1258
+ * is copied in here by the client.
1259
+ *
1260
+ * `service: true` is forced rather than inherited: a Sandbox without a Service
1261
+ * has no `status.serviceFQDN`, and then the only address left is a pod IP that
1262
+ * changes on every resume.
1263
+ *
1264
+ * `volumeClaimTemplates` is copied VERBATIM when the template declares any.
1265
+ * Dropping it would be silent: the Sandbox would come up healthy with no disk,
1266
+ * the container's `volumeDevices`/`volumeMounts` entry would fail to resolve
1267
+ * (or, worse, resolve to an empty emptyDir on some paths), and the only
1268
+ * symptom of a workspace that lost its disk would be that yesterday's files
1269
+ * are gone. The controller wires the mount by the entry's own NAME,
1270
+ * StatefulSet style, so the copy needs no matching `volumes:` entry and this
1271
+ * function adds none.
1272
+ *
1273
+ * The podTemplate's metadata gains {@link sandboxTemplateLabel}: this Sandbox
1274
+ * is created DIRECTLY, never adopted out of a pool, so it never gets
1275
+ * agent-sandbox's own controller-owned
1276
+ * `agents.x-k8s.io/sandbox-template-ref-hash` label (that is written only on
1277
+ * bind). Without a label of its own a direct Sandbox's pod would carry
1278
+ * nothing `egress-policy.ts`'s translated `NetworkPolicy` could select it
1279
+ * by. Existing labels on the copied template are preserved — this ADDS to
1280
+ * them rather than replacing the object outright — but this backend's own
1281
+ * key always wins if the template happened to set it too, since this is the
1282
+ * label the translated policy is built to match.
1283
+ */
1284
+ export function buildSandboxBody(options) {
1285
+ return {
1286
+ apiVersion: `${SANDBOX_API_GROUP}/${SANDBOX_API_VERSION}`,
1287
+ kind: 'Sandbox',
1288
+ metadata: {
1289
+ name: options.name,
1290
+ namespace: options.namespace,
1291
+ ...(options.annotations !== undefined ? { annotations: options.annotations } : {}),
1292
+ },
1293
+ spec: {
1294
+ operatingMode: 'Running',
1295
+ service: true,
1296
+ ...(options.shutdownTime !== undefined
1297
+ ? { shutdownTime: options.shutdownTime, shutdownPolicy: 'Delete' }
1298
+ : {}),
1299
+ ...(options.template.volumeClaimTemplates !== undefined
1300
+ ? { volumeClaimTemplates: options.template.volumeClaimTemplates }
1301
+ : {}),
1302
+ podTemplate: sandboxPodTemplate(options.template, options.sandboxTemplateName, options.runtimeClassName, options.podLabels),
1303
+ },
1304
+ };
1305
+ }
1306
+ /**
1307
+ * The `spec.podTemplate` a directly created Sandbox carries: the template's,
1308
+ * with this backend's overlays — the template label and whatever
1309
+ * {@link composeAdditionalPodLabels} produced ({@link sandboxPodLabels}), and
1310
+ * the configured `runtimeClassName`.
1311
+ *
1312
+ * Its own function because it is now built twice: once into the create POST
1313
+ * by {@link buildSandboxBody}, and once into the JSON Patch that refreshes a
1314
+ * standing workspace's pod template (`workspace.ts`). Two expressions of the
1315
+ * same overlay would drift, and the one that drifted would report a workspace
1316
+ * as off-template forever — the hash under
1317
+ * `sandbox.namzu.ai/pod-template-hash` is taken over exactly this object, so
1318
+ * a second spelling is a second revision.
1319
+ *
1320
+ * `podLabels` is on this signature rather than only on the create body for
1321
+ * exactly that reason. A refresh rewrites `/spec/podTemplate` WHOLE, so a
1322
+ * refresh built without them would PATCH the egress profile off a pod
1323
+ * template that carries it — the replacement pod would come up selected by no
1324
+ * per-profile policy, on a path where nothing re-checks the label, and the
1325
+ * revision stamped beside it would be taken over a template the POST never
1326
+ * writes, so `templateCurrent` would report drift forever.
1327
+ */
1328
+ export function sandboxPodTemplate(template, sandboxTemplateName, runtimeClassName, podLabels) {
1329
+ const podTemplate = template.podTemplate;
1330
+ const spec = runtimeClassName !== undefined
1331
+ ? { ...podTemplate.spec, runtimeClassName }
1332
+ : { ...podTemplate.spec };
1333
+ return {
1334
+ ...podTemplate,
1335
+ metadata: {
1336
+ ...podTemplate.metadata,
1337
+ labels: sandboxPodLabels(template, sandboxTemplateName, podLabels),
1338
+ },
1339
+ spec,
1340
+ };
1341
+ }
1342
+ /**
1343
+ * The labels a directly created Sandbox's pod will carry: whatever the
1344
+ * template declares, plus this backend's own template label, which always
1345
+ * wins because it is the label a policy selector is built to match.
1346
+ *
1347
+ * Its own function because the ingress check has to reason about EXACTLY the
1348
+ * labels {@link buildSandboxBody} stamps, before the POST that stamps them.
1349
+ * Two expressions of the same rule would be one rename away from a check that
1350
+ * verifies a pod nobody creates.
1351
+ *
1352
+ * `extra` is `composeAdditionalPodLabels`'s map — the egress profile today.
1353
+ * It is applied LAST, and so wins over both, for the same reason the template
1354
+ * label wins over the copied template's own: it is a label the translated
1355
+ * policy's selector is built to match, and a pod that matched the selector
1356
+ * only sometimes would be a boundary that applied only sometimes.
1357
+ */
1358
+ export function sandboxPodLabels(template, sandboxTemplateName, extra) {
1359
+ return {
1360
+ ...template.podTemplate.metadata?.labels,
1361
+ ...sandboxTemplateLabel(sandboxTemplateName),
1362
+ ...extra,
1363
+ };
1364
+ }
1365
+ export async function readSandboxTemplate(client, namespace, sandboxTemplateName, signal) {
1366
+ const template = await client.request('GET', sandboxTemplatePath(namespace, sandboxTemplateName), undefined, signal);
1367
+ const podTemplate = template?.spec?.podTemplate;
1368
+ if (!podTemplate || typeof podTemplate.spec !== 'object' || podTemplate.spec === null) {
1369
+ throw new Error(`kubernetes: SandboxTemplate ${sandboxTemplateName} in namespace ${namespace} carries no spec.podTemplate.spec, so there is nothing to create a pool-less Sandbox from. Sandbox.spec has no templateRef — the podTemplate has to be copied in.`);
1370
+ }
1371
+ const volumeClaimTemplates = template?.spec?.volumeClaimTemplates;
1372
+ return {
1373
+ podTemplate,
1374
+ ...(volumeClaimTemplates !== undefined ? { volumeClaimTemplates } : {}),
1375
+ };
1376
+ }
1377
+ /**
1378
+ * Where the agent answers.
1379
+ *
1380
+ * Its own function, and under the default mode the Service FQDN wins over a
1381
+ * pod IP, because that address outlives the pod: a suspended-then-resumed
1382
+ * workspace comes back as a new pod with a new IP behind the same name, and
1383
+ * the transport re-resolves the name on every dial. A literal IP baked into a
1384
+ * long-lived handle is the bug that would produce — and it is exactly the bug
1385
+ * `'pod-ip'` accepts, deliberately, in exchange for an address a host outside
1386
+ * the cluster can resolve at all. That mode pays for it by re-reading the IP
1387
+ * on every resume and once after a failed connect.
1388
+ *
1389
+ * `'pod-ip'` takes the address from `pod`, the record the bind token was just
1390
+ * read out of, and NEVER falls back to `binding.podIPs`. The Sandbox's status
1391
+ * is a second source that can name a different pod — the one a resume is
1392
+ * replacing — and an address from one pod with a token from another is the
1393
+ * mismatch that arrives as a flat `unauthorized`.
1394
+ */
1395
+ export function resolveAgentAddress(binding, agentPort, token, options = {}) {
1396
+ if ((options.mode ?? 'service') === 'pod-ip') {
1397
+ const podIP = options.podIP;
1398
+ if (podIP === undefined || podIP === '') {
1399
+ throw new Error(`kubernetes: sandbox ${binding.name} is configured with agentAddress: 'pod-ip', but the live pod its bind token was read from reported no status.podIP (nor a status.podIPs entry), so there is no address to dial. Every path that binds a pod POLLS for that address until the readiness deadline before this is reached — see readAddressedPod — so a pod that still reports none was never given one: a CNI that did not attach it, or a pod that never got past scheduling. Nothing is taken from the Sandbox's own status here on purpose — that IP may belong to a different pod than the token does.`);
1400
+ }
1401
+ return { kind: 'tcp', host: podIP, port: agentPort, token };
1402
+ }
1403
+ const host = binding.serviceFQDN ?? binding.podIPs?.[0];
1404
+ if (host === undefined || host === '') {
1405
+ throw new Error(`kubernetes: sandbox ${binding.name} reported Ready with neither a serviceFQDN nor a pod IP, so its agent has no address to dial. Set 'service: true' on the SandboxTemplate the pool is built from.`);
1406
+ }
1407
+ return { kind: 'tcp', host, port: agentPort, token };
1408
+ }
1409
+ /**
1410
+ * Ready-or-not-yet, read off a `SandboxClaim`'s own status — plus the one
1411
+ * "not yet" that is really a "no".
1412
+ *
1413
+ * The rejection check runs BEFORE the readiness check rather than after,
1414
+ * because a refused claim is `Ready=False` forever and the readiness check
1415
+ * cannot tell that from a cold start in progress. See
1416
+ * {@link classifyClaimRejection}.
1417
+ */
1418
+ function bindingFromClaim(claim, claimName, metadata) {
1419
+ const rejection = classifyClaimRejection(claim, claimName, metadata);
1420
+ if (rejection !== undefined)
1421
+ throw rejection;
1422
+ if (!isConditionTrue(claim?.status?.conditions, READY_CONDITION))
1423
+ return undefined;
1424
+ const bound = claim?.status?.sandbox;
1425
+ // Ready with no bound name is the controller contradicting itself; polling
1426
+ // on would just burn the deadline waiting for a field that is finished.
1427
+ if (!bound?.name) {
1428
+ throw new Error(`kubernetes: SandboxClaim ${claimName} reported Ready but named no sandbox in status.sandbox.name, so there is nothing to address.`);
1429
+ }
1430
+ return {
1431
+ name: bound.name,
1432
+ ...(bound.podIPs !== undefined ? { podIPs: bound.podIPs } : {}),
1433
+ ...(bound.serviceFQDN !== undefined ? { serviceFQDN: bound.serviceFQDN } : {}),
1434
+ };
1435
+ }
1436
+ /** Ready-or-not-yet, read off a `Sandbox`'s own status. Shared with `workspace.ts`. */
1437
+ export function bindingFromSandbox(sandbox) {
1438
+ if (!isConditionTrue(sandbox?.status?.conditions, READY_CONDITION))
1439
+ return undefined;
1440
+ const name = sandbox?.metadata?.name;
1441
+ if (!name) {
1442
+ throw new Error('kubernetes: Sandbox reported Ready with no metadata.name');
1443
+ }
1444
+ const status = sandbox?.status;
1445
+ return {
1446
+ name,
1447
+ ...(status?.podIPs !== undefined ? { podIPs: status.podIPs } : {}),
1448
+ ...(status?.serviceFQDN !== undefined ? { serviceFQDN: status.serviceFQDN } : {}),
1449
+ ...(status?.selector !== undefined ? { podSelector: status.selector } : {}),
1450
+ };
1451
+ }
1452
+ /**
1453
+ * The readiness poll ran out of budget — and nothing else. Every OTHER
1454
+ * failure {@link pollForBinding} meets is rethrown as itself, so this class
1455
+ * is an exact answer to "was it the clock?", which a caller that has to
1456
+ * choose between two timeout messages needs and cannot get from the clock.
1457
+ *
1458
+ * Reading `remainingMs()` after the fact is NOT that answer: the expiry timer
1459
+ * and `performance.now()` are different clocks, and a timer that fires a
1460
+ * fraction of a millisecond early leaves a positive remainder behind an
1461
+ * expiry that has already happened.
1462
+ */
1463
+ export class ReadinessPollTimeout extends Error {
1464
+ name = 'ReadinessPollTimeout';
1465
+ }
1466
+ /**
1467
+ * Poll until `read` reports a binding. `read` returns `undefined` for "not
1468
+ * yet" and throws for a failure worth surfacing; the deadline owns every wait,
1469
+ * including the sleep between attempts, so an expired clock cannot be extended
1470
+ * by one more round trip. Shaped after ACI's `pollForRunningIp`.
1471
+ *
1472
+ * The only failure this raises on its own account is
1473
+ * {@link ReadinessPollTimeout}; anything `read` throws travels out unchanged,
1474
+ * unless `behaviour.retryDelayFor` claims it — in which case it is repeated
1475
+ * inside the same budget and, if the budget then runs out, carried onto the
1476
+ * timeout as its `cause`, so a poll that kept failing still says what it kept
1477
+ * seeing.
1478
+ */
1479
+ export async function pollForBinding(read, deadline, readiness, label, behaviour = {}) {
1480
+ /**
1481
+ * The last failure that was retried rather than raised, and only while it
1482
+ * is still the truth: a read that succeeds clears it, so the timeout
1483
+ * carries a failure the poll was STILL meeting when the budget ran out
1484
+ * rather than a blip it recovered from twenty polls earlier. The
1485
+ * difference is not cosmetic — the acquire's reason is read off this
1486
+ * cause, so a stale one would report an API outage for a poll whose API
1487
+ * was answering fine.
1488
+ */
1489
+ let lastRetried;
1490
+ while (deadline.remainingMs() > 0) {
1491
+ /** Overrides the poll cadence for one round when the server named one. */
1492
+ let nextDelayMs = readiness.pollIntervalMs;
1493
+ try {
1494
+ const binding = await deadline.run(read);
1495
+ lastRetried = undefined;
1496
+ if (binding)
1497
+ return binding;
1498
+ }
1499
+ catch (err) {
1500
+ if (err instanceof OperationDeadlineExpired)
1501
+ break;
1502
+ const retryDelayMs = behaviour.retryDelayFor?.(err);
1503
+ if (retryDelayMs === undefined)
1504
+ throw err;
1505
+ lastRetried = err;
1506
+ nextDelayMs = retryDelayMs;
1507
+ }
1508
+ try {
1509
+ await deadline.delay(nextDelayMs);
1510
+ }
1511
+ catch (err) {
1512
+ if (err instanceof OperationDeadlineExpired)
1513
+ break;
1514
+ throw err;
1515
+ }
1516
+ }
1517
+ throw new ReadinessPollTimeout(`kubernetes: ${label} never became Ready (${readiness.timeoutMs}ms)${lastRetried !== undefined
1518
+ ? `; the last API failure retried inside that budget was: ${lastRetried instanceof Error ? lastRetried.message : String(lastRetried)}`
1519
+ : ''}`, lastRetried !== undefined ? { cause: lastRetried } : undefined);
1520
+ }
1521
+ /**
1522
+ * Find the pod a sandbox is currently backed by, and read both facts off it.
1523
+ *
1524
+ * `uid` is the per-instance agent bind token; `podIP` is where that agent
1525
+ * listens, and only `agentAddress: 'pod-ip'` reads it.
1526
+ *
1527
+ * The pod is named after its Sandbox in agent-sandbox v1.0.2 — verified
1528
+ * against a running cluster — but that is an observation, not a documented
1529
+ * guarantee, and `Sandbox.status` exposes no pod name to fall back on. So the
1530
+ * fast path is one GET by that name, and the only cost of the name convention
1531
+ * changing upstream is a second round trip through `status.selector`, which is
1532
+ * exactly what the controller publishes the selector for.
1533
+ */
1534
+ export async function readBoundPod(client, namespace, binding, signal) {
1535
+ try {
1536
+ const pod = await client.request('GET', podPath(namespace, binding.name), undefined, signal);
1537
+ const uid = pod?.metadata?.uid;
1538
+ // `isPodLive` matters on the RESUME path in `workspace.ts`: a resumed
1539
+ // pod keeps its name, so for as long as the outgoing one is
1540
+ // terminating this GET can answer with the pod that is leaving and a
1541
+ // uid the new agent will refuse. On the acquire path nothing is
1542
+ // terminating and the filter never fires.
1543
+ if (uid && isPodLive(pod))
1544
+ return boundPod(uid, pod);
1545
+ }
1546
+ catch (err) {
1547
+ if (!(err instanceof KubernetesAlreadyGoneError))
1548
+ throw err;
1549
+ }
1550
+ const selector = binding.podSelector ?? (await readSandboxSelector(client, namespace, binding, signal));
1551
+ if (selector !== undefined && selector !== '') {
1552
+ const list = await client.request('GET', podListPath(namespace, selector), undefined, signal);
1553
+ for (const pod of list?.items ?? []) {
1554
+ const uid = pod.metadata?.uid;
1555
+ if (uid && isPodLive(pod))
1556
+ return boundPod(uid, pod);
1557
+ }
1558
+ }
1559
+ throw new Error(`kubernetes: could not read a pod uid for sandbox ${binding.name} in namespace ${namespace} — no live pod of that name, and its status.selector matched no live pod either (a pod carrying a deletionTimestamp, or in phase Succeeded/Failed, is never bound to). The pod uid is the agent's bind token, so the sandbox is refused rather than returned unauthenticated.`);
1560
+ }
1561
+ function boundPod(uid, pod) {
1562
+ const podIP = readPodIP(pod);
1563
+ const labels = pod.metadata?.labels;
1564
+ return {
1565
+ uid,
1566
+ ...(podIP !== undefined ? { podIP } : {}),
1567
+ ...(labels !== undefined ? { labels } : {}),
1568
+ };
1569
+ }
1570
+ /**
1571
+ * {@link readBoundPod}, plus — under `'pod-ip'` only — the wait for an
1572
+ * address to go with the token.
1573
+ *
1574
+ * Under the default mode this is the single read it has always been: one
1575
+ * `GET`, in the same place in the same order, because a Service FQDN is
1576
+ * published with the Sandbox and needs nothing from the pod but its uid.
1577
+ *
1578
+ * `'pod-ip'` has to wait, because a LIVE pod is not yet an ADDRESSED pod. A
1579
+ * pod is created `Pending` and carries no `status.podIP` until the CNI has
1580
+ * finished attaching it, and {@link isPodLive} accepts `Pending` on purpose —
1581
+ * the resume path in `workspace.ts` binds its replacement pod long before that
1582
+ * pod is Ready, because `Ready` stays True across the transition and the uid
1583
+ * is the only transition signal there is. Refusing an address-less pod outright
1584
+ * would therefore fail on the NORMAL path, in milliseconds, with the whole
1585
+ * readiness budget unspent. So "live, no address yet" is polled on the same
1586
+ * deadline as everything else on this path, and
1587
+ * {@link resolveAgentAddress}'s own refusal is left as the post-deadline
1588
+ * backstop for a pod that never gets an address at all.
1589
+ *
1590
+ * A failed READ stays fatal, exactly as it was: this is the acquire path,
1591
+ * where nothing is being replaced and a pod that cannot be read is not a pod
1592
+ * that is about to appear.
1593
+ *
1594
+ * `requiredLabels` is the second thing worth waiting for, and it is waited
1595
+ * for in the SAME loop rather than in a second one: an egress profile is a
1596
+ * label the CONTROLLER patches onto the pod it binds, so a pod read the
1597
+ * instant it was bound can be live, addressed and not yet labelled. Two
1598
+ * loops would be two deadlines and two answers to "is this pod ready to be
1599
+ * admitted". Like the address, an expired clock hands the pod back as it is
1600
+ * — the caller decides whether a missing label is fatal, and on the acquire
1601
+ * path it is: see `assertRequestedPodLabelsObserved`.
1602
+ */
1603
+ export async function readAddressedPod(client, namespace, binding, deadline, readiness, mode, requiredLabels) {
1604
+ const required = Object.entries(requiredLabels ?? {});
1605
+ const wanting = (pod) => (mode === 'pod-ip' && pod.podIP === undefined) ||
1606
+ required.some(([key, value]) => pod.labels?.[key] !== value);
1607
+ let pod = await deadline.run((signal) => readBoundPod(client, namespace, binding, signal));
1608
+ while (wanting(pod) && deadline.remainingMs() > 0) {
1609
+ try {
1610
+ await deadline.delay(readiness.pollIntervalMs);
1611
+ pod = await deadline.run((signal) => readBoundPod(client, namespace, binding, signal));
1612
+ }
1613
+ catch (err) {
1614
+ // An expired clock hands the incomplete pod back rather than
1615
+ // replacing it with a bare "deadline expired": the caller's
1616
+ // `resolveAgentAddress` (or `assertRequestedPodLabelsObserved`) then
1617
+ // reports WHICH fact never arrived.
1618
+ if (err instanceof OperationDeadlineExpired)
1619
+ break;
1620
+ throw err;
1621
+ }
1622
+ }
1623
+ return pod;
1624
+ }
1625
+ /**
1626
+ * The re-read a `'pod-ip'` handle follows a replaced pod with: one live-pod
1627
+ * read, then the same address resolution acquire did.
1628
+ *
1629
+ * Built here rather than inside the transport because finding the pod is a
1630
+ * CONTROL-plane act — the by-name GET, the selector fallback, the
1631
+ * liveness filter — and the transport owns none of that. It is handed over as
1632
+ * a closure so the transport can call it without learning what a Sandbox is.
1633
+ */
1634
+ export function buildAgentAddressRefresh(client, namespace, binding, agentPort, mode) {
1635
+ return async (signal) => {
1636
+ const pod = await readBoundPod(client, namespace, binding, signal);
1637
+ return resolveAgentAddress(binding, agentPort, pod.uid, {
1638
+ mode,
1639
+ ...(pod.podIP !== undefined ? { podIP: pod.podIP } : {}),
1640
+ });
1641
+ };
1642
+ }
1643
+ async function readSandboxSelector(client, namespace, binding, signal) {
1644
+ try {
1645
+ const sandbox = await client.request('GET', sandboxPath(namespace, binding.name), undefined, signal);
1646
+ return sandbox?.status?.selector;
1647
+ }
1648
+ catch (err) {
1649
+ if (err instanceof KubernetesAlreadyGoneError)
1650
+ return undefined;
1651
+ throw err;
1652
+ }
1653
+ }
1654
+ /**
1655
+ * Build the Sandbox, prove it is deprivileged, and only then hand it back.
1656
+ *
1657
+ * The probe runs BEFORE `create()` resolves, so a caller never holds a
1658
+ * reference to an under-hardened sandbox — a probe that refuses destroys the
1659
+ * instance on a bounded cleanup budget and rethrows, exactly as a readiness
1660
+ * failure does. There is no configuration that skips it: the whole value of
1661
+ * checking the deprivileging on every acquire rather than once by hand is
1662
+ * that it cannot be forgotten, and an off switch is a way to forget it.
1663
+ *
1664
+ * The probe goes through the Sandbox's own `exec`, not the raw transport, so
1665
+ * it traverses the same reserve/admit/stream/confirm path every later call
1666
+ * will. A sandbox that cannot answer the probe is one a caller could not use
1667
+ * either.
1668
+ *
1669
+ * ## And it runs on a clock
1670
+ *
1671
+ * This is the first thing on the acquire path that talks to the GUEST, and
1672
+ * the readiness deadline that bounded everything before it has already
1673
+ * expired. Left unbounded the probe would inherit the execution controller's
1674
+ * generic defaults instead — a five-minute observation, then a cancel-confirm
1675
+ * and a drain — so a pod whose agent has wedged (out of memory, an event loop
1676
+ * the workload blocked) would keep `create()` pending for minutes past the
1677
+ * caller's `readyTimeoutMs` with nothing reported. It gets its own deadline,
1678
+ * whose expiry takes the same cleanup-and-reject path every other refusal
1679
+ * does, in words that name the hang rather than blame a missing `cat`.
1680
+ */
1681
+ async function admitProbedSandbox(acquisition, config, options, probeTimeoutMs,
1682
+ /**
1683
+ * Present exactly when `config.egress.perSandbox` is configured — which
1684
+ * is what makes `setNetworkPolicy` present on the handle. See
1685
+ * `per-sandbox-policy.ts`.
1686
+ */
1687
+ setNetworkPolicy) {
1688
+ const sandbox = buildKubernetesSandbox({
1689
+ name: acquisition.binding.name,
1690
+ rootDir: options.workingDirectory,
1691
+ transport: new KubernetesAgentTransport(acquisition.agent, {
1692
+ // The backend opts in to the stream heartbeat; the transport
1693
+ // option it sets stays undefined for every other tier.
1694
+ heartbeatMs: resolveStreamHeartbeatMs(config.streamHeartbeatMs),
1695
+ ...(acquisition.refreshAgent !== undefined
1696
+ ? { refreshHandle: acquisition.refreshAgent }
1697
+ : {}),
1698
+ }),
1699
+ release: acquisition.release,
1700
+ renew: acquisition.renew,
1701
+ ttlSeconds: acquisition.ttlSeconds,
1702
+ ...(config.onLeaseRenewalError !== undefined
1703
+ ? { onRenewalError: config.onLeaseRenewalError }
1704
+ : {}),
1705
+ ...(setNetworkPolicy !== undefined ? { setNetworkPolicy } : {}),
1706
+ });
1707
+ try {
1708
+ await probeSandboxPrivileges(sandbox, acquisition.binding.name, probeTimeoutMs, options.signal);
1709
+ }
1710
+ catch (err) {
1711
+ await runFailureCleanup(async (signal) => {
1712
+ await sandbox.destroy({ signal });
1713
+ });
1714
+ throw err;
1715
+ }
1716
+ return sandbox;
1717
+ }
1718
+ /**
1719
+ * Run the probe against a built Sandbox, on its own clock, and throw if it
1720
+ * refuses. Cleanup is the CALLER's, and the two callers want opposite things:
1721
+ * a task acquire destroys the instance, while `workspace.ts` suspends it,
1722
+ * because deleting a workspace deletes its disk and a probe refusal is not a
1723
+ * reason to lose a caller's files.
1724
+ */
1725
+ export async function probeSandboxPrivileges(sandbox, sandboxName, probeTimeoutMs, signal) {
1726
+ // Labelled with the sandbox, so an expiry read off a log line says which
1727
+ // acquire stopped answering — and so the catch below can tell THIS
1728
+ // deadline from any other that might surface through the same exec.
1729
+ const probeLabel = `kubernetes privilege probe ${sandboxName}`;
1730
+ try {
1731
+ signal?.throwIfAborted();
1732
+ // The deadline's signal covers both ways this should stop early: it
1733
+ // aborts on expiry, and it aborts with the caller's own reason when
1734
+ // `signal` does. Handing it to `exec` is what releases the guest-side
1735
+ // execution rather than merely abandoning the wait.
1736
+ await new OperationDeadline(probeTimeoutMs, probeLabel, signal).run(async (execSignal) => await runPrivilegeProbe(async (command, args) => await sandbox.exec(command, args, { signal: execSignal }), sandboxName));
1737
+ // An abort that lands WHILE the probe is in flight must not leave a
1738
+ // live sandbox behind: the probe itself may well have finished, and
1739
+ // the caller who cancelled is about to stop holding the reference
1740
+ // that could destroy it. Same cleanup, one branch later.
1741
+ signal?.throwIfAborted();
1742
+ }
1743
+ catch (err) {
1744
+ // A caller who cancelled mid-probe gets THEIR reason, not the probe's
1745
+ // account of a command that was cancelled out from under it.
1746
+ signal?.throwIfAborted();
1747
+ // A probe that ran out of time is a probe that could not run, and is
1748
+ // refused in those words: `OperationDeadlineExpired` on its own would
1749
+ // leave a reader guessing which half of the acquire went quiet.
1750
+ if (err instanceof OperationDeadlineExpired && err.label === probeLabel) {
1751
+ throw privilegeProbeTimedOut(sandboxName, probeTimeoutMs, err);
1752
+ }
1753
+ throw err;
1754
+ }
1755
+ }
1756
+ //# sourceMappingURL=index.js.map