hammerkit 1.27.0-next.3 → 1.27.0-next.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/build.schema.json +188 -0
  2. package/dist/cache/backends/local-cache-backend.js +6 -0
  3. package/dist/cache/backends/registry-cache-backend.js +2 -2
  4. package/dist/cache/explain.js +17 -0
  5. package/dist/cli.js +2 -0
  6. package/dist/docker/item-network.js +13 -5
  7. package/dist/executer/deadline.js +17 -3
  8. package/dist/executer/deferred-restore.js +36 -0
  9. package/dist/executer/docker-service.js +30 -8
  10. package/dist/executer/docker-task.js +21 -1
  11. package/dist/executer/event-cache.js +20 -2
  12. package/dist/executer/execute-work-service.js +2 -0
  13. package/dist/executer/execute-work-task.js +10 -3
  14. package/dist/executer/execution-steps.js +25 -4
  15. package/dist/executer/reset-work-tree.js +4 -0
  16. package/dist/executer/scheduler/enqueue-next.js +57 -6
  17. package/dist/executer/scheduler/state-key.js +42 -5
  18. package/dist/executer/watch-loop.js +4 -1
  19. package/dist/harness/connect.js +20 -6
  20. package/dist/harness/mcp-command.js +22 -6
  21. package/dist/harness/mcp.js +63 -114
  22. package/dist/harness/opencode.js +158 -50
  23. package/dist/harness/pi.js +29 -0
  24. package/dist/harness/sandbox-client.js +168 -17
  25. package/dist/harness/service-tools.js +141 -0
  26. package/dist/harness/task-tools.js +221 -0
  27. package/dist/kubernetes/apply.js +27 -16
  28. package/dist/kubernetes/await-running-state.js +59 -13
  29. package/dist/kubernetes/claim-layout.js +71 -0
  30. package/dist/kubernetes/cluster-state.js +245 -0
  31. package/dist/kubernetes/ensure-http-route.js +1 -1
  32. package/dist/kubernetes/ensure-kubernetes-deployment-exists.js +35 -12
  33. package/dist/kubernetes/ensure-kubernetes-persistent-volume-claim-exists.js +1 -1
  34. package/dist/kubernetes/ensure-kubernetes-service-exists.js +6 -9
  35. package/dist/kubernetes/ensure-persistent-data.js +30 -20
  36. package/dist/kubernetes/follow-pod-logs.js +182 -0
  37. package/dist/kubernetes/get-service-ip.js +2 -2
  38. package/dist/kubernetes/job-placement.js +36 -12
  39. package/dist/kubernetes/job-secrets.js +33 -19
  40. package/dist/kubernetes/kept-claims.js +22 -10
  41. package/dist/kubernetes/network-policy.js +85 -27
  42. package/dist/kubernetes/remove-persistent-data.js +3 -1
  43. package/dist/kubernetes/resources.js +25 -3
  44. package/dist/kubernetes/restore-kubernetes-data.js +9 -3
  45. package/dist/kubernetes/sandbox-runner.js +628 -0
  46. package/dist/kubernetes/secrets.js +7 -11
  47. package/dist/kubernetes/volumes.js +6 -3
  48. package/dist/main.js +2 -0
  49. package/dist/network/egress-proxy.js +45 -6
  50. package/dist/optimizer/calculate-checksum.js +3 -2
  51. package/dist/optimizer/check-key.js +94 -0
  52. package/dist/optimizer/check-keys.js +64 -0
  53. package/dist/optimizer/get-work-cache-stats.js +17 -3
  54. package/dist/optimizer/work-task-cache-description.js +51 -1
  55. package/dist/planner/utils/append-work-dependencies.js +10 -5
  56. package/dist/planner/utils/append-work-service.js +19 -1
  57. package/dist/planner/utils/append-work-task.js +18 -3
  58. package/dist/planner/utils/get-work-runtime.js +6 -1
  59. package/dist/planner/utils/normalize-path.js +19 -1
  60. package/dist/planner/utils/parse-work-mount.js +7 -1
  61. package/dist/planner/utils/parse-work-port.js +4 -1
  62. package/dist/planner/utils/task-arguments.js +3 -0
  63. package/dist/planner/validate.js +22 -0
  64. package/dist/planner/work-network.js +3 -1
  65. package/dist/planner/work-runtime-aggregate.js +32 -0
  66. package/dist/planner/work-runtime-docker.js +2 -1
  67. package/dist/planner/work-runtime-kubernetes.js +124 -48
  68. package/dist/planner/work-runtime-local.js +2 -1
  69. package/dist/planner/work-task.js +6 -1
  70. package/dist/program.js +31 -10
  71. package/dist/remote/device-login.js +7 -2
  72. package/dist/remote/host-config.js +69 -3
  73. package/dist/remote/remote-commands.js +30 -7
  74. package/dist/remote/remote-work-commands.js +426 -45
  75. package/dist/remote/server-client.js +15 -2
  76. package/dist/remote/token-source.js +64 -26
  77. package/dist/run-program.js +9 -0
  78. package/dist/schema/build-file-base-task-schema.js +8 -1
  79. package/dist/schema/build-file-container-service-schema.js +12 -0
  80. package/dist/schema/build-file-container-task-schema.js +16 -1
  81. package/dist/schema/build-file-include-schema.js +18 -4
  82. package/dist/schema/build-file-kubernetes-service-schema.js +2 -0
  83. package/dist/schema/build-file-task-schema.js +5 -1
  84. package/dist/schema/capabilities-schema.js +56 -0
  85. package/dist/schema/reference-parser.js +120 -11
  86. package/dist/schema/schema-parser.js +92 -41
  87. package/dist/schema/work-scope-parser.js +33 -10
  88. package/dist/secrets/command-secret-provider.js +3 -1
  89. package/dist/secrets/github-app-token-command.js +3 -1
  90. package/dist/secrets/secret-catalog.js +43 -19
  91. package/dist/server/accounts-api.js +3 -1
  92. package/dist/server/approvals-api.js +1 -1
  93. package/dist/server/checks-api.js +83 -0
  94. package/dist/server/checks.js +116 -0
  95. package/dist/server/definitions-api.js +66 -3
  96. package/dist/server/endpoints.js +739 -0
  97. package/dist/server/executor.js +57 -5
  98. package/dist/server/kubernetes-backend.js +367 -65
  99. package/dist/server/kubernetes-sandbox-storage.js +48 -3
  100. package/dist/server/oauth.js +12 -1
  101. package/dist/server/plan.js +188 -103
  102. package/dist/server/policy.js +32 -2
  103. package/dist/server/repo-access.js +5 -0
  104. package/dist/server/run-plan.js +50 -19
  105. package/dist/server/run-request.js +58 -3
  106. package/dist/server/run-worker.js +245 -12
  107. package/dist/server/runs.js +410 -27
  108. package/dist/server/sandbox-pod.js +28 -0
  109. package/dist/server/sandbox-storage.js +23 -0
  110. package/dist/server/sandboxes-api.js +10 -4
  111. package/dist/server/sandboxes.js +8 -0
  112. package/dist/server/server-command.js +39 -20
  113. package/dist/server/server-config.js +255 -14
  114. package/dist/server/server.js +269 -15
  115. package/dist/server/service-runs.js +82 -0
  116. package/dist/utils/abort-on-signals.js +46 -6
  117. package/dist/utils/tar.js +36 -0
  118. package/dist/utils/terminal-prompt.js +53 -0
  119. package/package.json +1 -1
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.awaitRunObjectsGone = exports.removeRunObjects = exports.CallerNamespaces = exports.getVolumeScope = exports.KubernetesExecutor = exports.getRunRoleBinding = exports.getNamespacePolicies = exports.getClusterAccess = exports.isCallerNamespace = exports.getCallerNamespace = void 0;
3
+ exports.awaitRunObjectsGone = exports.removeRunObjects = exports.CallerNamespaces = exports.getVolumeScope = exports.getPrivilegedNamespaceProblem = exports.getCapabilitiesProblem = exports.getPodSecurityLabels = exports.KubernetesExecutor = exports.getRunRoleBinding = exports.removeServiceRunNamespace = exports.getServerIngressPolicy = exports.getNamespacePolicies = exports.getClusterAccess = exports.isCallerNamespace = exports.SERVICE_RUN_LABEL = exports.getServiceRunNamespace = exports.getCallerNamespace = void 0;
4
4
  const crypto_1 = require("crypto");
5
5
  const fs_1 = require("fs");
6
6
  const os_1 = require("os");
@@ -22,6 +22,8 @@ const POLICY_NAME = 'hammerkit';
22
22
  const MANAGED_LABEL = 'hammerkit.dev/managed';
23
23
  // how long the end of a run waits for its pods and claims to be gone
24
24
  const RELEASE_TIMEOUT = 120000;
25
+ // how long the setup of a caller's namespace is taken as it is, before it is applied again
26
+ const NAMESPACE_SETUP_REUSE_MS = 5 * 60000;
25
27
  // where a pod of the cluster finds the token and certificate of its ServiceAccount
26
28
  const IN_CLUSTER_NAMESPACE = '/var/run/secrets/kubernetes.io/serviceaccount/namespace';
27
29
  /**
@@ -32,6 +34,19 @@ function getCallerNamespace(prefix, owner) {
32
34
  return `${prefix}${(0, crypto_1.createHash)('sha256').update(`${owner.iss}\0${owner.sub}`).digest('hex').slice(0, 16)}`;
33
35
  }
34
36
  exports.getCallerNamespace = getCallerNamespace;
37
+ /**
38
+ * The namespace of a service run: one of its own, as the services stay up for hours while
39
+ * the caller's other runs go on in theirs (one at a time, each removing what the one
40
+ * before left).
41
+ */
42
+ function getServiceRunNamespace(prefix, run, wakes = 0) {
43
+ // a woken run starts in a namespace of its own, the one it slept in may still be removed
44
+ const name = wakes > 0 ? `up\0${run}\0${wakes}` : `up\0${run}`;
45
+ return `${prefix}${(0, crypto_1.createHash)('sha256').update(name).digest('hex').slice(0, 16)}`;
46
+ }
47
+ exports.getServiceRunNamespace = getServiceRunNamespace;
48
+ // the label of the namespace of a service run: the run label value of its run
49
+ exports.SERVICE_RUN_LABEL = 'hammerkit.dev/service-run';
35
50
  // Whether a namespace is one getCallerNamespace makes with this prefix: a server named `hk`
36
51
  // passes over the namespaces of a server named `hk-test` on the same cluster.
37
52
  function isCallerNamespace(prefix, name) {
@@ -143,6 +158,64 @@ function getNamespacePolicies(namespace, deniedEgress, allowedEgress = []) {
143
158
  ];
144
159
  }
145
160
  exports.getNamespacePolicies = getNamespacePolicies;
161
+ /**
162
+ * What lets the server into the namespace of a service run, to the ports of its endpoints:
163
+ * policies add up, so this one opens the pods to the server's namespace besides what the
164
+ * others let in, limited ones too.
165
+ */
166
+ // The server reaches the ports of the endpoints, from its own namespace, and no other port:
167
+ // what else the run starts (a database) stays for the pods of the run.
168
+ function getServerIngressPolicy(namespace, server, ports) {
169
+ return {
170
+ apiVersion: 'networking.k8s.io/v1',
171
+ kind: 'NetworkPolicy',
172
+ metadata: {
173
+ name: 'hammerkit-server-ingress',
174
+ namespace,
175
+ annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
176
+ labels: { [MANAGED_LABEL]: 'true' },
177
+ },
178
+ spec: {
179
+ podSelector: {},
180
+ policyTypes: ['Ingress'],
181
+ ingress: ports.length > 0
182
+ ? [
183
+ {
184
+ _from: [{ namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': server } } }],
185
+ ports: [...new Set(ports)].map((port) => ({ protocol: 'TCP', port })),
186
+ },
187
+ ]
188
+ : [],
189
+ },
190
+ };
191
+ }
192
+ exports.getServerIngressPolicy = getServerIngressPolicy;
193
+ // The namespace of a service run goes with all it holds; waits up to `timeout` for it.
194
+ async function removeServiceRunNamespace(instance, namespace, timeout) {
195
+ try {
196
+ await instance.coreApi.deleteNamespace({ name: namespace, propagationPolicy: 'Background' });
197
+ }
198
+ catch (e) {
199
+ if ((0, apply_1.statusCodeOf)(e) === 404) {
200
+ return;
201
+ }
202
+ throw e;
203
+ }
204
+ const deadline = Date.now() + timeout;
205
+ while (Date.now() < deadline) {
206
+ try {
207
+ await instance.coreApi.readNamespace({ name: namespace });
208
+ }
209
+ catch (e) {
210
+ if ((0, apply_1.statusCodeOf)(e) === 404) {
211
+ return;
212
+ }
213
+ throw e;
214
+ }
215
+ await (0, sleep_1.sleep)(1000);
216
+ }
217
+ }
218
+ exports.removeServiceRunNamespace = removeServiceRunNamespace;
146
219
  /**
147
220
  * The server's rights in a caller's namespace: the operator's ClusterRole, bound there
148
221
  * only, so the server needs no rights on Secrets or pods anywhere else.
@@ -167,22 +240,31 @@ exports.getRunRoleBinding = getRunRoleBinding;
167
240
  * process of the server as on a host (see HostExecutor); what changes is where its
168
241
  * container tasks and services go: into the caller's namespace, on the nodes of the
169
242
  * operator's pools, with the secrets of providers read in the Job itself. The runs
170
- * of one caller execute one after the other, since the objects of a task are named
171
- * by its cache id in the caller's namespace; different callers run side by side.
172
- * When a run ends, everything made for it is removed, but for the claims of its tasks
173
- * the server keeps (`backend.keepVolumes`), which go once they expired.
243
+ * of a caller execute side by side in their namespace, up to `runs.limits.running`:
244
+ * the objects of a run are named and found by its run label (getItemKey), so two runs
245
+ * of the same task have their own. What two runs share are the claims the server keeps
246
+ * (`backend.keepVolumes`) of a scope (getVolumeScope): the runs of one scope go one at
247
+ * a time (exclusive). When a run ends, everything made for it is removed, but for the
248
+ * kept claims, which go once they expired.
174
249
  */
175
250
  class KubernetesExecutor {
176
251
  constructor(config, processEnvs, options) {
177
252
  this.config = config;
178
- // the registry keeps a caller's next run in its queue, not running and timed here
179
- this.runningPerCaller = 1;
180
- // the run that executes, or the last one waiting, per caller namespace
253
+ // per caller namespace, what changes objects of the whole namespace: its setup, the
254
+ // removal of what a stopped server left, the sweeps of kept claims. The last one waiting.
181
255
  this.queues = new Map();
256
+ // per caller namespace, the runs that execute in it: by run label, their volume scope
257
+ this.active = new Map();
182
258
  // per caller namespace, when its kept claims are next looked at
183
259
  this.sweeps = new Map();
184
260
  // whether the kept claims a server left when it stopped were looked for
185
261
  this.swept = false;
262
+ // the namespaces where no run of this process left anything: what a run makes goes at its
263
+ // end, unless its release failed
264
+ this.tidy = new Set();
265
+ // the namespaces of the service runs that execute, and whether those of earlier ones went
266
+ this.serviceRuns = new Set();
267
+ this.serviceRunsSwept = false;
186
268
  this.access = getClusterAccess(config.backend, processEnvs);
187
269
  this.instance = (0, kubernetes_instance_1.createKubernetesInstances)({
188
270
  type: 'kubernetes',
@@ -203,17 +285,56 @@ class KubernetesExecutor {
203
285
  this.log = options.log;
204
286
  this.host = new executor_1.HostExecutor({ ...options, placement: this.placement() });
205
287
  }
206
- execute(run, context) {
207
- const namespace = getCallerNamespace(this.config.backend.namespacePrefix, run.owner);
208
- return this.enqueue(namespace, () => {
209
- if (context.signal.aborted) {
210
- throw new Error('the run was cancelled');
288
+ async execute(run, context) {
289
+ if (run.up) {
290
+ if (run.source.type === 'sandbox') {
291
+ throw new Error('on a cluster a service run starts from a branch, push the sandbox and start it from there');
211
292
  }
293
+ // in a namespace of its own, apart from the caller's runs and their claims
212
294
  return this.host.execute(run, context);
213
- });
295
+ }
296
+ const namespace = getCallerNamespace(this.config.backend.namespacePrefix, run.owner);
297
+ const label = (0, job_placement_1.getRunLabelValue)(run.id);
298
+ let runs = this.active.get(namespace);
299
+ if (!runs) {
300
+ runs = new Map();
301
+ this.active.set(namespace, runs);
302
+ }
303
+ runs.set(label, getVolumeScope(run));
304
+ try {
305
+ await this.host.execute(run, context);
306
+ }
307
+ finally {
308
+ runs.delete(label);
309
+ if (runs.size === 0 && this.active.get(namespace) === runs) {
310
+ this.active.delete(namespace);
311
+ }
312
+ }
214
313
  }
215
- // What happens in a caller's namespace happens one at a time: their runs, and the
216
- // removal of the claims their runs kept
314
+ namespaceOf(run) {
315
+ const prefix = this.config.backend.namespacePrefix;
316
+ return run.up ? getServiceRunNamespace(prefix, run.id, run.wakes) : getCallerNamespace(prefix, run.owner);
317
+ }
318
+ // The runs of a scope share its kept claims, which one of them may change at a time:
319
+ // what a task finds in one is what the last run left there. Without kept claims, runs
320
+ // share no object and nothing is held.
321
+ exclusive(run) {
322
+ if (this.keep === 0 || run.up) {
323
+ return undefined;
324
+ }
325
+ return `${getCallerNamespace(this.config.backend.namespacePrefix, run.owner)}/${getVolumeScope(run)}`;
326
+ }
327
+ // The scopes of the runs that execute in the namespace, but `except`'s: their kept claims
328
+ // are theirs to mount and stay. A server that keeps none mounts none.
329
+ scopesInUse(namespace, except) {
330
+ if (this.keep === 0) {
331
+ return new Set();
332
+ }
333
+ return new Set([...(this.active.get(namespace) ?? new Map()).entries()]
334
+ .filter(([label]) => label !== except)
335
+ .map(([, scope]) => scope));
336
+ }
337
+ // What changes objects of a whole namespace happens one at a time, not what its runs do
217
338
  enqueue(namespace, fn) {
218
339
  const previous = this.queues.get(namespace) ?? Promise.resolve();
219
340
  const execution = previous.then(fn);
@@ -227,54 +348,98 @@ class KubernetesExecutor {
227
348
  return execution;
228
349
  }
229
350
  placement() {
230
- const backend = this.config.backend;
231
351
  return {
232
352
  prepare: async (run, context, work) => {
233
- const namespace = getCallerNamespace(backend.namespacePrefix, run.owner);
234
- await this.namespaces.prepare(namespace);
235
- // what an earlier run left when the server stopped before its end; the runs
236
- // of a caller execute one at a time, so no other run is in the namespace
237
- await removeRunObjects(this.instance, namespace, `${job_placement_1.RUN_LABEL},${job_placement_1.RUN_LABEL}!=${(0, job_placement_1.getRunLabelValue)(run.id)}`, RELEASE_TIMEOUT);
238
- // the claims earlier runs kept that expired; the others this run may mount
239
- const { next } = await (0, kept_claims_1.sweepKeptClaims)(this.instance, namespace, {
240
- now: new Date(),
241
- keep: this.keep > 0,
242
- timeout: RELEASE_TIMEOUT,
243
- });
244
- if (next) {
245
- this.sweepLater(namespace, Math.max(0, next.getTime() - Date.now()));
353
+ const namespace = this.namespaceOf(run);
354
+ if (run.up) {
355
+ if (context.signal.aborted) {
356
+ throw new Error('the run was cancelled');
357
+ }
358
+ this.serviceRuns.add(namespace);
359
+ this.sweepServiceRuns();
360
+ // reached by the server at the Services of its endpoints
361
+ await this.namespaces.prepare(namespace, {
362
+ serviceRun: (0, job_placement_1.getRunLabelValue)(run.id),
363
+ server: this.namespace,
364
+ ports: (run.up.endpoints ?? []).map((endpoint) => endpoint.port),
365
+ quota: this.config.runs.up.quota,
366
+ });
367
+ context.emit({ type: 'log', task: 'hammerkit', stream: 'stdout', text: `runs in the namespace ${namespace}` });
368
+ return { kubernetes: this.environmentOf(run, namespace), env: this.access.env };
246
369
  }
370
+ const label = (0, job_placement_1.getRunLabelValue)(run.id);
371
+ let capabilitiesProblem;
372
+ await this.enqueue(namespace, async () => {
373
+ if (context.signal.aborted) {
374
+ throw new Error('the run was cancelled');
375
+ }
376
+ capabilitiesProblem = await this.namespaces.prepare(namespace);
377
+ // what an earlier run left when the server stopped before its end, or when its
378
+ // release failed: the objects of a run that does not execute. One that starts
379
+ // after this waits for it here before it makes any.
380
+ if (!this.tidy.has(namespace)) {
381
+ const running = [...new Set([label, ...(this.active.get(namespace)?.keys() ?? [])])];
382
+ await removeRunObjects(this.instance, namespace, `${job_placement_1.RUN_LABEL},${job_placement_1.RUN_LABEL} notin (${running.join(',')})`, RELEASE_TIMEOUT);
383
+ this.tidy.add(namespace);
384
+ }
385
+ // the claims earlier runs kept that expired; the others this run may mount
386
+ const { next } = await (0, kept_claims_1.sweepKeptClaims)(this.instance, namespace, {
387
+ now: new Date(),
388
+ keep: this.keep > 0,
389
+ timeout: RELEASE_TIMEOUT,
390
+ inUse: this.scopesInUse(namespace, label),
391
+ });
392
+ if (next) {
393
+ this.sweepLater(namespace, Math.max(0, next.getTime() - Date.now()));
394
+ }
395
+ });
247
396
  this.sweepLeftClaims();
248
397
  context.emit({ type: 'log', task: 'hammerkit', stream: 'stdout', text: `runs in the namespace ${namespace}` });
249
- const kubernetes = {
250
- type: 'kubernetes',
251
- namespace,
252
- kubeConfig: this.access.kubeConfig,
253
- context: this.access.context,
254
- httpRoutes: [],
255
- storageClass: backend.storageClass,
256
- volumeSize: backend.volumeSize,
257
- maxVolumeSize: (0, server_config_1.getMaxVolumeSize)(backend),
258
- networkPolicyEnforced: backend.networkPolicy === 'enforced',
259
- server: {
260
- run: run.id,
261
- namespace: this.namespace,
262
- pools: backend.pools,
263
- providers: backend.providers,
264
- accounts: Object.fromEntries(Object.entries(this.config.accounts).flatMap(([name, account]) => account.kubernetes ? [[name, account.kubernetes]] : [])),
265
- ...(work?.claim ? { workspace: getWorkspace(work.root, work.claim, namespace) } : {}),
266
- ...(this.keep > 0 ? { volumeScope: getVolumeScope(run) } : {}),
267
- },
398
+ return {
399
+ kubernetes: this.environmentOf(run, namespace, work, capabilitiesProblem),
400
+ env: this.access.env,
268
401
  };
269
- return { kubernetes, env: this.access.env };
270
402
  },
271
403
  release: async (run, context) => {
272
- const namespace = getCallerNamespace(backend.namespacePrefix, run.owner);
404
+ const namespace = this.namespaceOf(run);
405
+ if (run.up) {
406
+ try {
407
+ await removeServiceRunNamespace(this.instance, namespace, RELEASE_TIMEOUT);
408
+ }
409
+ catch (e) {
410
+ // at least nothing of it runs on: its objects are the server's to remove (runRole)
411
+ const forbidden = (0, apply_1.statusCodeOf)(e) === 403;
412
+ const text = forbidden
413
+ ? `the server may not delete the namespace ${namespace}, its ClusterRole needs delete on namespaces (see the backend docs); removing what runs in it`
414
+ : `the namespace ${namespace} could not be removed: ${(0, log_1.getErrorMessage)(e)}`;
415
+ this.log?.(text);
416
+ context.emit({ type: 'log', task: 'hammerkit', stream: 'stderr', text });
417
+ try {
418
+ await removeRunObjects(this.instance, namespace, job_placement_1.RUN_LABEL, RELEASE_TIMEOUT);
419
+ }
420
+ catch (inner) {
421
+ context.emit({
422
+ type: 'log',
423
+ task: 'hammerkit',
424
+ stream: 'stderr',
425
+ text: `what the run made in ${namespace} could not be removed: ${(0, log_1.getErrorMessage)(inner)}`,
426
+ });
427
+ }
428
+ // the next look removes the namespace
429
+ this.serviceRunsSwept = false;
430
+ }
431
+ finally {
432
+ this.serviceRuns.delete(namespace);
433
+ }
434
+ return;
435
+ }
273
436
  try {
274
437
  const selector = `${job_placement_1.RUN_LABEL}=${(0, job_placement_1.getRunLabelValue)(run.id)}`;
275
438
  await removeRunObjects(this.instance, namespace, selector, RELEASE_TIMEOUT);
276
439
  }
277
440
  catch (e) {
441
+ // the next run removes what is left
442
+ this.tidy.delete(namespace);
278
443
  context.emit({
279
444
  type: 'log',
280
445
  task: 'hammerkit',
@@ -300,8 +465,67 @@ class KubernetesExecutor {
300
465
  },
301
466
  };
302
467
  }
303
- // The kept claims of a namespace that expired go once no run of the caller is in it: at
304
- // the soonest expiry the server knows of
468
+ // The cluster as the process of a run sees it: the namespace its tasks and services go in.
469
+ environmentOf(run, namespace, work, capabilitiesProblem) {
470
+ const backend = this.config.backend;
471
+ return {
472
+ type: 'kubernetes',
473
+ namespace,
474
+ kubeConfig: this.access.kubeConfig,
475
+ context: this.access.context,
476
+ httpRoutes: [],
477
+ storageClass: backend.storageClass,
478
+ volumeSize: backend.volumeSize,
479
+ maxVolumeSize: (0, server_config_1.getMaxVolumeSize)(backend),
480
+ networkPolicyEnforced: backend.networkPolicy === 'enforced',
481
+ server: {
482
+ run: run.id,
483
+ namespace: this.namespace,
484
+ pools: backend.pools,
485
+ providers: backend.providers,
486
+ accounts: Object.fromEntries(Object.entries(this.config.accounts).flatMap(([name, account]) => account.kubernetes ? [[name, account.kubernetes]] : [])),
487
+ ...(work?.claim ? { workspace: getWorkspace(work.root, work.claim, namespace, backend.sandboxes.runner) } : {}),
488
+ // the claims of a service run go with its namespace
489
+ ...(this.keep > 0 && !run.up ? { volumeScope: getVolumeScope(run) } : {}),
490
+ ...(backend.capabilities
491
+ ? {
492
+ capabilities: {
493
+ runtimeClass: backend.capabilities.runtimeClass,
494
+ ...(capabilitiesProblem ? { unavailable: capabilitiesProblem } : {}),
495
+ },
496
+ }
497
+ : {}),
498
+ },
499
+ };
500
+ }
501
+ // Once, on the first service run: the namespaces of service runs the server left when it
502
+ // stopped, whose services went with it
503
+ // Once when the server starts, and on the next service run after a namespace could not be
504
+ // removed: the namespaces of service runs that none of the server's runs is in, whose
505
+ // services went with the process of a server that stopped
506
+ sweepServiceRuns() {
507
+ if (this.serviceRunsSwept) {
508
+ return;
509
+ }
510
+ this.serviceRunsSwept = true;
511
+ Promise.resolve()
512
+ .then(() => this.instance.coreApi.listNamespace({ labelSelector: `${MANAGED_LABEL}=true,${exports.SERVICE_RUN_LABEL}` }))
513
+ .then(async (namespaces) => {
514
+ for (const namespace of namespaces.items) {
515
+ const name = namespace.metadata?.name;
516
+ if (name &&
517
+ isCallerNamespace(this.config.backend.namespacePrefix, name) &&
518
+ !this.serviceRuns.has(name) &&
519
+ !namespace.metadata?.deletionTimestamp) {
520
+ this.log?.(`removing the namespace ${name} of a service run from before the server started`);
521
+ await removeServiceRunNamespace(this.instance, name, 0);
522
+ }
523
+ }
524
+ })
525
+ .catch((e) => this.log?.(`the namespaces of service runs could not be removed: ${(0, log_1.getErrorMessage)(e)}`));
526
+ }
527
+ // The kept claims of a namespace that expired go at the soonest expiry the server knows
528
+ // of, but those of a scope a run of the caller executes in
305
529
  sweepLater(namespace, delay) {
306
530
  const at = Date.now() + delay + 1000;
307
531
  const pending = this.sweeps.get(namespace);
@@ -321,6 +545,7 @@ class KubernetesExecutor {
321
545
  now: new Date(),
322
546
  keep: this.keep > 0,
323
547
  timeout: RELEASE_TIMEOUT,
548
+ inUse: this.scopesInUse(namespace),
324
549
  });
325
550
  if (removed.length > 0) {
326
551
  this.log?.(`removed ${removed.length} kept claims in ${namespace}`);
@@ -337,7 +562,7 @@ class KubernetesExecutor {
337
562
  }
338
563
  this.swept = true;
339
564
  Promise.resolve()
340
- .then(() => this.instance.coreApi.listNamespace({ labelSelector: `${MANAGED_LABEL}=true` }))
565
+ .then(() => this.instance.coreApi.listNamespace({ labelSelector: `${MANAGED_LABEL}=true,!${exports.SERVICE_RUN_LABEL}` }))
341
566
  .then((namespaces) => {
342
567
  for (const namespace of namespaces.items) {
343
568
  const name = namespace.metadata?.name;
@@ -352,11 +577,55 @@ class KubernetesExecutor {
352
577
  }
353
578
  }
354
579
  exports.KubernetesExecutor = KubernetesExecutor;
580
+ const ENFORCE_LABEL = 'pod-security.kubernetes.io/enforce';
581
+ const AUDIT_LABEL = 'pod-security.kubernetes.io/audit';
582
+ /**
583
+ * The Pod Security level a caller's namespace is made with: `baseline`, no privileged pods,
584
+ * host paths or host network, whatever a build asks. `baseline` also refuses the capabilities
585
+ * of `backend.capabilities`, sandboxed or not: with them the namespace enforces `privileged`
586
+ * and audits `baseline`, and the operator's admission policy (see the docs on capabilities)
587
+ * refuses what `baseline` would to a pod outside the sandbox's RuntimeClass.
588
+ */
589
+ function getPodSecurityLabels(backend) {
590
+ return backend.capabilities
591
+ ? { [ENFORCE_LABEL]: 'privileged', [AUDIT_LABEL]: 'baseline' }
592
+ : { [ENFORCE_LABEL]: 'baseline' };
593
+ }
594
+ exports.getPodSecurityLabels = getPodSecurityLabels;
595
+ /**
596
+ * The server never changes the level of a namespace it made (that would take `patch` on every
597
+ * namespace of the cluster): one made before `backend.capabilities` is relabelled by the
598
+ * operator. Until then its tasks that add capabilities fail with this, the others run.
599
+ */
600
+ function getCapabilitiesProblem(namespace, labels, backend) {
601
+ const enforce = labels?.[ENFORCE_LABEL];
602
+ if (!backend.capabilities || enforce === 'privileged') {
603
+ return undefined;
604
+ }
605
+ return (`the namespace ${namespace} was made before backend.capabilities and enforces ` +
606
+ `${enforce ? `Pod Security ${enforce}` : 'no Pod Security level of its own'}, not privileged: kubectl label namespace ${namespace} ` +
607
+ `${ENFORCE_LABEL}=privileged ${AUDIT_LABEL}=baseline --overwrite`);
608
+ }
609
+ exports.getCapabilitiesProblem = getCapabilitiesProblem;
610
+ /**
611
+ * A namespace made while the server had `backend.capabilities` admits privileged pods but for
612
+ * the operator's admission policy. Without capabilities the server runs nothing in it until the
613
+ * operator sets it back.
614
+ */
615
+ function getPrivilegedNamespaceProblem(namespace, labels, backend) {
616
+ if (backend.capabilities || labels?.[ENFORCE_LABEL] !== 'privileged') {
617
+ return undefined;
618
+ }
619
+ return (`the namespace ${namespace} enforces Pod Security privileged, made while the server had ` +
620
+ `backend.capabilities, and the server runs nothing in it without them: kubectl label namespace ` +
621
+ `${namespace} ${ENFORCE_LABEL}=baseline ${AUDIT_LABEL}- --overwrite`);
622
+ }
623
+ exports.getPrivilegedNamespaceProblem = getPrivilegedNamespaceProblem;
355
624
  // What may mount a claim a run kept: the runs of the same repository, as they share the
356
625
  // cache, and on a sandbox the runs of that sandbox only, whose claim is on the same node or
357
626
  // zone as the kept ones. The owner is the namespace's.
358
627
  function getVolumeScope(run) {
359
- if (run.source.type === 'sandbox') {
628
+ if (run.source.type === 'sandbox' && !run.source.check) {
360
629
  return `sandbox-${(0, crypto_1.createHash)('sha256').update(run.source.id).digest('hex').slice(0, 24)}`;
361
630
  }
362
631
  return (0, executor_1.cacheNamespace)(run);
@@ -370,6 +639,8 @@ class CallerNamespaces {
370
639
  constructor(instance, backend) {
371
640
  this.instance = instance;
372
641
  this.backend = backend;
642
+ // the namespaces this process set up, by their uid, and when
643
+ this.applied = new Map();
373
644
  }
374
645
  name(owner) {
375
646
  return getCallerNamespace(this.backend.namespacePrefix, owner);
@@ -377,9 +648,15 @@ class CallerNamespaces {
377
648
  // The caller's namespace: made by the server and kept apart from the others. The
378
649
  // server binds its rights there, isolates the network of the pods, applies the
379
650
  // operator's quota and gives containers without resources default requests, so they
380
- // are admitted under a quota on compute. All of it is replaced on every run, so the
381
- // namespace follows the config.
382
- async prepare(namespace) {
651
+ // are admitted under a quota on compute. All of it is replaced when the server sets up
652
+ // a namespace it did not set up in the last `NAMESPACE_SETUP_REUSE_MS` (it reads the
653
+ // config once, at its start), so the namespace follows the config; a namespace made anew
654
+ // since (deleted and made again) is set up again at once. Returns what keeps the tasks that
655
+ // add capabilities from running in it (getCapabilitiesProblem), if anything.
656
+ async prepare(namespace,
657
+ // a service run's own namespace: its label, the namespace of the server that reaches its
658
+ // endpoints, and its quota
659
+ serviceRun) {
383
660
  const backend = this.backend;
384
661
  let existing;
385
662
  try {
@@ -393,17 +670,33 @@ class CallerNamespaces {
393
670
  if (existing && existing.metadata?.labels?.[MANAGED_LABEL] !== 'true') {
394
671
  throw new Error(`the namespace ${namespace} was not made by hammerkit, the server does not run tasks in it`);
395
672
  }
673
+ const privileged = getPrivilegedNamespaceProblem(namespace, existing?.metadata?.labels, backend);
674
+ if (privileged) {
675
+ throw new Error(privileged);
676
+ }
677
+ // one made by another run of this server at the same time has the labels of its config
678
+ const problem = () => getCapabilitiesProblem(namespace, existing?.metadata?.labels ?? getPodSecurityLabels(backend), backend);
679
+ const started = Date.now();
680
+ const applied = this.applied.get(namespace);
681
+ if (existing &&
682
+ !existing.metadata?.deletionTimestamp &&
683
+ applied &&
684
+ applied.uid === existing.metadata?.uid &&
685
+ started - applied.at < NAMESPACE_SETUP_REUSE_MS) {
686
+ return problem();
687
+ }
688
+ this.applied.delete(namespace);
396
689
  if (!existing) {
397
690
  try {
398
- await this.instance.coreApi.createNamespace({
691
+ existing = await this.instance.coreApi.createNamespace({
399
692
  body: {
400
693
  metadata: {
401
694
  name: namespace,
402
695
  labels: {
403
696
  [MANAGED_LABEL]: 'true',
404
697
  'hammerkit.dev/caller': namespace,
405
- // no privileged pods, host paths or host network, whatever a build asks
406
- 'pod-security.kubernetes.io/enforce': 'baseline',
698
+ ...(serviceRun ? { [exports.SERVICE_RUN_LABEL]: serviceRun.serviceRun } : {}),
699
+ ...getPodSecurityLabels(backend),
407
700
  },
408
701
  },
409
702
  },
@@ -419,7 +712,10 @@ class CallerNamespaces {
419
712
  const [first, ...policies] = getNamespacePolicies(namespace, backend.deniedEgress, backend.allowedEgress);
420
713
  // the API server may refuse what the binding allows for a moment after it was made
421
714
  await untilAllowed(() => (0, apply_1.replace)(this.instance, first));
422
- for (const policy of policies) {
715
+ for (const policy of [
716
+ ...policies,
717
+ ...(serviceRun ? [getServerIngressPolicy(namespace, serviceRun.server, serviceRun.ports)] : []),
718
+ ]) {
423
719
  await (0, apply_1.replace)(this.instance, policy);
424
720
  }
425
721
  const metadata = {
@@ -432,7 +728,7 @@ class CallerNamespaces {
432
728
  apiVersion: 'v1',
433
729
  kind: 'ResourceQuota',
434
730
  metadata,
435
- spec: { hard: backend.quota },
731
+ spec: { hard: serviceRun?.quota ?? backend.quota },
436
732
  };
437
733
  const defaults = backend.defaultResources;
438
734
  const limits = {
@@ -450,7 +746,7 @@ class CallerNamespaces {
450
746
  },
451
747
  };
452
748
  for (const [object, wanted] of [
453
- [quota, !!backend.quota],
749
+ [quota, !!(serviceRun?.quota ?? backend.quota)],
454
750
  [limits, !!defaults],
455
751
  ]) {
456
752
  if (wanted) {
@@ -465,6 +761,11 @@ class CallerNamespaces {
465
761
  });
466
762
  }
467
763
  }
764
+ const uid = existing?.metadata?.uid;
765
+ if (uid) {
766
+ this.applied.set(namespace, { uid, at: started });
767
+ }
768
+ return problem();
468
769
  }
469
770
  whoAmI() {
470
771
  if (!this.subject) {
@@ -487,7 +788,7 @@ class CallerNamespaces {
487
788
  exports.CallerNamespaces = CallerNamespaces;
488
789
  // The claim of a sandbox as the Jobs of its run mount it: the run's copy of the files is
489
790
  // `root`, a source below it is the same path below `subPath` in the claim.
490
- function getWorkspace(root, claim, namespace) {
791
+ function getWorkspace(root, claim, namespace, runner) {
491
792
  if (claim.namespace !== namespace) {
492
793
  throw new Error(`the sandbox is in ${claim.namespace}, the run in ${namespace}`);
493
794
  }
@@ -497,6 +798,7 @@ function getWorkspace(root, claim, namespace) {
497
798
  subPath: claim.subPath,
498
799
  label: claim.label,
499
800
  ...(claim.pod ? { pod: claim.pod } : {}),
801
+ ...(runner ? { runner: true } : {}),
500
802
  };
501
803
  }
502
804
  // Everything a run made in the caller's namespace, by the label selector of the run: