hammerkit 1.27.0-next.3 → 1.27.0-next.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/build.schema.json +188 -0
  2. package/dist/cache/backends/local-cache-backend.js +6 -0
  3. package/dist/cache/backends/registry-cache-backend.js +2 -2
  4. package/dist/cache/explain.js +17 -0
  5. package/dist/cli.js +2 -0
  6. package/dist/docker/item-network.js +13 -5
  7. package/dist/executer/deadline.js +17 -3
  8. package/dist/executer/deferred-restore.js +36 -0
  9. package/dist/executer/docker-service.js +30 -8
  10. package/dist/executer/docker-task.js +21 -1
  11. package/dist/executer/event-cache.js +20 -2
  12. package/dist/executer/execute-work-service.js +2 -0
  13. package/dist/executer/execute-work-task.js +10 -3
  14. package/dist/executer/execution-steps.js +25 -4
  15. package/dist/executer/reset-work-tree.js +4 -0
  16. package/dist/executer/scheduler/enqueue-next.js +57 -6
  17. package/dist/executer/scheduler/state-key.js +42 -5
  18. package/dist/executer/watch-loop.js +4 -1
  19. package/dist/harness/connect.js +20 -6
  20. package/dist/harness/mcp-command.js +22 -6
  21. package/dist/harness/mcp.js +63 -114
  22. package/dist/harness/opencode.js +158 -50
  23. package/dist/harness/pi.js +29 -0
  24. package/dist/harness/sandbox-client.js +168 -17
  25. package/dist/harness/service-tools.js +141 -0
  26. package/dist/harness/task-tools.js +221 -0
  27. package/dist/kubernetes/apply.js +27 -16
  28. package/dist/kubernetes/await-running-state.js +59 -13
  29. package/dist/kubernetes/claim-layout.js +71 -0
  30. package/dist/kubernetes/cluster-state.js +245 -0
  31. package/dist/kubernetes/ensure-http-route.js +1 -1
  32. package/dist/kubernetes/ensure-kubernetes-deployment-exists.js +35 -12
  33. package/dist/kubernetes/ensure-kubernetes-persistent-volume-claim-exists.js +1 -1
  34. package/dist/kubernetes/ensure-kubernetes-service-exists.js +6 -9
  35. package/dist/kubernetes/ensure-persistent-data.js +30 -20
  36. package/dist/kubernetes/follow-pod-logs.js +182 -0
  37. package/dist/kubernetes/get-service-ip.js +2 -2
  38. package/dist/kubernetes/job-placement.js +36 -12
  39. package/dist/kubernetes/job-secrets.js +33 -19
  40. package/dist/kubernetes/kept-claims.js +22 -10
  41. package/dist/kubernetes/network-policy.js +85 -27
  42. package/dist/kubernetes/remove-persistent-data.js +3 -1
  43. package/dist/kubernetes/resources.js +25 -3
  44. package/dist/kubernetes/restore-kubernetes-data.js +9 -3
  45. package/dist/kubernetes/sandbox-runner.js +628 -0
  46. package/dist/kubernetes/secrets.js +7 -11
  47. package/dist/kubernetes/volumes.js +6 -3
  48. package/dist/main.js +2 -0
  49. package/dist/network/egress-proxy.js +45 -6
  50. package/dist/optimizer/calculate-checksum.js +3 -2
  51. package/dist/optimizer/check-key.js +94 -0
  52. package/dist/optimizer/check-keys.js +64 -0
  53. package/dist/optimizer/get-work-cache-stats.js +17 -3
  54. package/dist/optimizer/work-task-cache-description.js +51 -1
  55. package/dist/planner/utils/append-work-dependencies.js +10 -5
  56. package/dist/planner/utils/append-work-service.js +19 -1
  57. package/dist/planner/utils/append-work-task.js +18 -3
  58. package/dist/planner/utils/get-work-runtime.js +6 -1
  59. package/dist/planner/utils/normalize-path.js +19 -1
  60. package/dist/planner/utils/parse-work-mount.js +7 -1
  61. package/dist/planner/utils/parse-work-port.js +4 -1
  62. package/dist/planner/utils/task-arguments.js +3 -0
  63. package/dist/planner/validate.js +22 -0
  64. package/dist/planner/work-network.js +3 -1
  65. package/dist/planner/work-runtime-aggregate.js +32 -0
  66. package/dist/planner/work-runtime-docker.js +2 -1
  67. package/dist/planner/work-runtime-kubernetes.js +124 -48
  68. package/dist/planner/work-runtime-local.js +2 -1
  69. package/dist/planner/work-task.js +6 -1
  70. package/dist/program.js +31 -10
  71. package/dist/remote/device-login.js +7 -2
  72. package/dist/remote/host-config.js +69 -3
  73. package/dist/remote/remote-commands.js +30 -7
  74. package/dist/remote/remote-work-commands.js +426 -45
  75. package/dist/remote/server-client.js +15 -2
  76. package/dist/remote/token-source.js +64 -26
  77. package/dist/run-program.js +9 -0
  78. package/dist/schema/build-file-base-task-schema.js +8 -1
  79. package/dist/schema/build-file-container-service-schema.js +12 -0
  80. package/dist/schema/build-file-container-task-schema.js +16 -1
  81. package/dist/schema/build-file-include-schema.js +18 -4
  82. package/dist/schema/build-file-kubernetes-service-schema.js +2 -0
  83. package/dist/schema/build-file-task-schema.js +5 -1
  84. package/dist/schema/capabilities-schema.js +56 -0
  85. package/dist/schema/reference-parser.js +120 -11
  86. package/dist/schema/schema-parser.js +92 -41
  87. package/dist/schema/work-scope-parser.js +33 -10
  88. package/dist/secrets/command-secret-provider.js +3 -1
  89. package/dist/secrets/github-app-token-command.js +3 -1
  90. package/dist/secrets/secret-catalog.js +43 -19
  91. package/dist/server/accounts-api.js +3 -1
  92. package/dist/server/approvals-api.js +1 -1
  93. package/dist/server/checks-api.js +83 -0
  94. package/dist/server/checks.js +116 -0
  95. package/dist/server/definitions-api.js +66 -3
  96. package/dist/server/endpoints.js +739 -0
  97. package/dist/server/executor.js +57 -5
  98. package/dist/server/kubernetes-backend.js +367 -65
  99. package/dist/server/kubernetes-sandbox-storage.js +48 -3
  100. package/dist/server/oauth.js +12 -1
  101. package/dist/server/plan.js +188 -103
  102. package/dist/server/policy.js +32 -2
  103. package/dist/server/repo-access.js +5 -0
  104. package/dist/server/run-plan.js +50 -19
  105. package/dist/server/run-request.js +58 -3
  106. package/dist/server/run-worker.js +245 -12
  107. package/dist/server/runs.js +410 -27
  108. package/dist/server/sandbox-pod.js +28 -0
  109. package/dist/server/sandbox-storage.js +23 -0
  110. package/dist/server/sandboxes-api.js +10 -4
  111. package/dist/server/sandboxes.js +8 -0
  112. package/dist/server/server-command.js +39 -20
  113. package/dist/server/server-config.js +255 -14
  114. package/dist/server/server.js +269 -15
  115. package/dist/server/service-runs.js +82 -0
  116. package/dist/utils/abort-on-signals.js +46 -6
  117. package/dist/utils/tar.js +36 -0
  118. package/dist/utils/terminal-prompt.js +53 -0
  119. package/package.json +1 -1
package/build.schema.json CHANGED
@@ -289,10 +289,16 @@
289
289
  ],
290
290
  "description": "network the task or service reaches besides its needs: open (default), none or { allow: [hosts or host:port] }"
291
291
  },
292
+ "args": {
293
+ "type": "boolean"
294
+ },
292
295
  "runner": {
293
296
  "type": "string",
294
297
  "minLength": 1
295
298
  },
299
+ "publish": {
300
+ "type": "boolean"
301
+ },
296
302
  "image": {
297
303
  "type": "string"
298
304
  },
@@ -305,6 +311,56 @@
305
311
  "volumeSize": {
306
312
  "type": "string",
307
313
  "description": "size of its claim on Kubernetes, e.g. 10Gi"
314
+ },
315
+ "capAdd": {
316
+ "type": "array",
317
+ "items": {
318
+ "type": "string",
319
+ "enum": [
320
+ "AUDIT_CONTROL",
321
+ "AUDIT_READ",
322
+ "AUDIT_WRITE",
323
+ "BLOCK_SUSPEND",
324
+ "BPF",
325
+ "CHECKPOINT_RESTORE",
326
+ "CHOWN",
327
+ "DAC_OVERRIDE",
328
+ "DAC_READ_SEARCH",
329
+ "FOWNER",
330
+ "FSETID",
331
+ "IPC_LOCK",
332
+ "IPC_OWNER",
333
+ "KILL",
334
+ "LEASE",
335
+ "LINUX_IMMUTABLE",
336
+ "MAC_ADMIN",
337
+ "MAC_OVERRIDE",
338
+ "MKNOD",
339
+ "NET_ADMIN",
340
+ "NET_BIND_SERVICE",
341
+ "NET_BROADCAST",
342
+ "NET_RAW",
343
+ "PERFMON",
344
+ "SETFCAP",
345
+ "SETGID",
346
+ "SETPCAP",
347
+ "SETUID",
348
+ "SYS_ADMIN",
349
+ "SYS_BOOT",
350
+ "SYS_CHROOT",
351
+ "SYS_MODULE",
352
+ "SYS_NICE",
353
+ "SYS_PACCT",
354
+ "SYS_PTRACE",
355
+ "SYS_RAWIO",
356
+ "SYS_RESOURCE",
357
+ "SYS_TIME",
358
+ "SYS_TTY_CONFIG",
359
+ "SYSLOG",
360
+ "WAKE_ALARM"
361
+ ]
362
+ },
363
+ "description": "Linux capabilities added to the container, e.g. [SYS_ADMIN]"
308
364
  }
309
365
  },
310
366
  "required": [
@@ -370,9 +426,15 @@
370
426
  "network": {
371
427
  "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/network"
372
428
  },
429
+ "args": {
430
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/args"
431
+ },
373
432
  "runner": {
374
433
  "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/runner"
375
434
  },
435
+ "publish": {
436
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/publish"
437
+ },
376
438
  "platform": {
377
439
  "type": "object",
378
440
  "properties": {
@@ -397,6 +459,91 @@
397
459
  },
398
460
  "additionalProperties": false,
399
461
  "description": "local task"
462
+ },
463
+ {
464
+ "type": "object",
465
+ "properties": {
466
+ "deps": {
467
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/deps"
468
+ },
469
+ "needs": {
470
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/needs"
471
+ },
472
+ "description": {
473
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/description"
474
+ },
475
+ "extend": {
476
+ "type": "string"
477
+ },
478
+ "reset": {
479
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/reset"
480
+ },
481
+ "envs": {
482
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/envs"
483
+ },
484
+ "labels": {
485
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/labels"
486
+ },
487
+ "cache": {
488
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/cache"
489
+ },
490
+ "cmds": {
491
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/cmds"
492
+ },
493
+ "src": {
494
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/src"
495
+ },
496
+ "generates": {
497
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/generates"
498
+ },
499
+ "shell": {
500
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/shell"
501
+ },
502
+ "continuous": {
503
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/continuous"
504
+ },
505
+ "timeout": {
506
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/timeout"
507
+ },
508
+ "secrets": {
509
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/secrets"
510
+ },
511
+ "account": {
512
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/account"
513
+ },
514
+ "resources": {
515
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/resources"
516
+ },
517
+ "network": {
518
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/network"
519
+ },
520
+ "args": {
521
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/args"
522
+ },
523
+ "runner": {
524
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/runner"
525
+ },
526
+ "publish": {
527
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/publish"
528
+ },
529
+ "image": {
530
+ "type": "string"
531
+ },
532
+ "mounts": {
533
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/mounts"
534
+ },
535
+ "volumeSize": {
536
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/volumeSize"
537
+ },
538
+ "capAdd": {
539
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/capAdd"
540
+ }
541
+ },
542
+ "required": [
543
+ "extend"
544
+ ],
545
+ "additionalProperties": false,
546
+ "description": "container task taking its image from the task it extends"
400
547
  }
401
548
  ]
402
549
  }
@@ -504,6 +651,12 @@
504
651
  "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/resources",
505
652
  "description": "cpu and memory requests and limits"
506
653
  },
654
+ "restarts": {
655
+ "type": "integer",
656
+ "minimum": 0,
657
+ "maximum": 10,
658
+ "description": "crashes restarted before it fails on Kubernetes, 0 to 10, 2 by default"
659
+ },
507
660
  "volumeSize": {
508
661
  "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/volumeSize",
509
662
  "description": "size of its claim on Kubernetes, e.g. 10Gi"
@@ -511,6 +664,12 @@
511
664
  "network": {
512
665
  "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/network",
513
666
  "description": "network the task or service reaches besides its needs: open (default), none or { allow: [hosts or host:port] }"
667
+ },
668
+ "args": {
669
+ "type": "boolean"
670
+ },
671
+ "publish": {
672
+ "type": "boolean"
514
673
  }
515
674
  },
516
675
  "required": [
@@ -567,6 +726,9 @@
567
726
  "number"
568
727
  ]
569
728
  }
729
+ },
730
+ "publish": {
731
+ "type": "boolean"
570
732
  }
571
733
  },
572
734
  "required": [
@@ -586,6 +748,28 @@
586
748
  {
587
749
  "type": "string"
588
750
  },
751
+ {
752
+ "type": "object",
753
+ "properties": {
754
+ "path": {
755
+ "type": "string",
756
+ "minLength": 1,
757
+ "description": "Build file or directory, relative to the including file"
758
+ },
759
+ "with": {
760
+ "$ref": "#/definitions/hammerkit/properties/envs",
761
+ "description": "Environment values for the current build task\nhttps://hammerkit.dev/docs/build-file/environment-variables"
762
+ },
763
+ "publish": {
764
+ "type": "boolean",
765
+ "description": "false: a server leaves out what this file declares"
766
+ }
767
+ },
768
+ "required": [
769
+ "path"
770
+ ],
771
+ "additionalProperties": false
772
+ },
589
773
  {
590
774
  "type": "object",
591
775
  "properties": {
@@ -606,6 +790,10 @@
606
790
  "with": {
607
791
  "$ref": "#/definitions/hammerkit/properties/envs",
608
792
  "description": "Environment values for the current build task\nhttps://hammerkit.dev/docs/build-file/environment-variables"
793
+ },
794
+ "publish": {
795
+ "type": "boolean",
796
+ "description": "false: a server leaves out what this file declares"
609
797
  }
610
798
  },
611
799
  "required": [
@@ -80,6 +80,12 @@ function createLocalCacheBackend(spec) {
80
80
  return false;
81
81
  }
82
82
  },
83
+ async touch(taskId, stateKey, environment) {
84
+ const remoteDir = (0, path_1.join)(root, taskId, stateKey);
85
+ if (await environment.file.exists((0, path_1.join)(remoteDir, 'stats.json'))) {
86
+ await (0, promises_1.writeFile)((0, path_1.join)(remoteDir, LAST_ACCESS_FILE), `${Date.now()}`);
87
+ }
88
+ },
83
89
  async push(taskId, stateKey, from, environment) {
84
90
  const remoteDir = (0, path_1.join)(root, taskId, stateKey);
85
91
  await environment.file.createDirectory(remoteDir);
@@ -5,6 +5,7 @@ const crypto_1 = require("crypto");
5
5
  const os_1 = require("os");
6
6
  const path_1 = require("path");
7
7
  const tar_1 = require("tar");
8
+ const tar_2 = require("../../utils/tar");
8
9
  const registry_reference_1 = require("./registry/registry-reference");
9
10
  const registry_client_1 = require("./registry/registry-client");
10
11
  const MAX_TAG_LENGTH = 128;
@@ -74,8 +75,7 @@ function createRegistryCacheBackend(spec) {
74
75
  const layerFile = (0, path_1.join)(dir, 'layer.tar');
75
76
  const files = (await environment.file.listFiles(from)).sort();
76
77
  // portable + mtime-free so identical entries produce identical digests
77
- // `noMtime` is supported by tar 6 but missing from its type definitions
78
- await (0, tar_1.create)({ file: layerFile, cwd: from, portable: true, noMtime: true }, files);
78
+ await (0, tar_2.packArchive)({ file: layerFile, cwd: from, portable: true, noMtime: true }, files);
79
79
  const layer = await (0, registry_client_1.digestFile)(layerFile);
80
80
  const config = Buffer.from(JSON.stringify({
81
81
  architecture: 'amd64',
@@ -1,6 +1,7 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.explainWorkTree = exports.explainTask = exports.describeCause = void 0;
4
+ const work_task_1 = require("../planner/work-task");
4
5
  const state_key_1 = require("../executer/scheduler/state-key");
5
6
  const work_task_cache_description_1 = require("../optimizer/work-task-cache-description");
6
7
  const get_cache_directory_1 = require("../optimizer/get-cache-directory");
@@ -157,6 +158,22 @@ async function explainTask(item, defaultCacheMethod, environment, memo = new Map
157
158
  visiting.delete(item.id());
158
159
  return explanation;
159
160
  }
161
+ // Mirrors executeWorkTask: an aggregate is never looked up itself, and is reported
162
+ // cached when all its dependencies were. What changed is in its dependencies.
163
+ if ((0, work_task_1.isAggregateTask)(item.data)) {
164
+ const causes = [];
165
+ for (const dep of item.deps) {
166
+ const depExplanation = await explainTask(dep, defaultCacheMethod, environment, memo, visiting);
167
+ if (depExplanation.status !== 'hit') {
168
+ causes.push({ kind: 'dependency-changed', identifier: dep.name });
169
+ }
170
+ }
171
+ const status = causes.length === 0 ? 'hit' : 'miss';
172
+ const explanation = { taskId: item.id(), taskName: item.name, status, causes };
173
+ memo.set(item.id(), explanation);
174
+ visiting.delete(item.id());
175
+ return explanation;
176
+ }
160
177
  const runtimeStateKey = await safeCurrentStateKey(item, environment);
161
178
  const hasEntry = await safeHas(item, stateKey, environment);
162
179
  const predictedHit = runtimeStateKey === stateKey || hasEntry;
package/dist/cli.js CHANGED
@@ -79,7 +79,9 @@ class Cli {
79
79
  explain: options?.explain ?? false,
80
80
  cacheReadOnly: options?.cacheReadOnly ?? false,
81
81
  timeout: options?.timeout ?? null,
82
+ maxTimeout: options?.maxTimeout ?? null,
82
83
  skipDeps: options?.skipDeps ?? true,
84
+ restore: options?.restore ?? true,
83
85
  wait: options?.wait ?? 'ready',
84
86
  resourceCheck: options?.resourceCheck ?? true,
85
87
  network: options?.network ?? null,
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.removeOrphanedNetworks = exports.removeItemNetworks = exports.getNetworkHostConfig = exports.removeItemNetwork = exports.prepareItemNetwork = exports.checkNetworkLimitSupport = exports.getProxyNetworkName = exports.getRunNetworkName = exports.getItemNetworkName = void 0;
3
+ exports.removeOrphanedNetworks = exports.removeItemNetworks = exports.getNetworkHostConfig = exports.getPortBinding = exports.removeItemNetwork = exports.prepareItemNetwork = exports.checkNetworkLimitSupport = exports.getProxyNetworkName = exports.getRunNetworkName = exports.getItemNetworkName = void 0;
4
4
  const readline_1 = require("readline");
5
5
  const stream_1 = require("stream");
6
6
  const work_service_1 = require("../planner/work-service");
@@ -258,7 +258,7 @@ options) {
258
258
  await removeItemNetwork(docker, name);
259
259
  throw e;
260
260
  }
261
- const publish = (options.publish ?? []).filter((port) => !!port.hostPort);
261
+ const publish = (options.publish ?? []).filter((port) => port.hostPort !== null);
262
262
  return {
263
263
  options: {
264
264
  mode: name,
@@ -272,8 +272,9 @@ options) {
272
272
  await docker.getNetwork(getProxyNetworkName(name)).connect({ Container: container.id });
273
273
  }
274
274
  if (publish.length > 0) {
275
- await startPortRelay(docker, item, environment, name, container, publish, options);
275
+ return await startPortRelay(docker, item, environment, name, container, publish, options);
276
276
  }
277
+ return undefined;
277
278
  }
278
279
  catch (e) {
279
280
  await removeItemNetwork(docker, name);
@@ -374,12 +375,13 @@ async function startPortRelay(docker, item, environment, networkName, service, p
374
375
  ExposedPorts: Object.fromEntries(ports.map((port) => [`${port.containerPort}/tcp`, {}])),
375
376
  HostConfig: {
376
377
  ...HELPER_HOST_CONFIG,
377
- PortBindings: Object.fromEntries(ports.map((port) => [`${port.containerPort}/tcp`, [{ HostPort: `${port.hostPort}` }]])),
378
+ PortBindings: Object.fromEntries(ports.map((port) => [`${port.containerPort}/tcp`, [getPortBinding(port)]])),
378
379
  },
379
380
  });
380
381
  await docker.getNetwork(networkName).connect({ Container: container.id });
381
382
  await container.start();
382
- item.status.write('debug', `ports ${ports.map((port) => port.hostPort).join(', ')} published by a relay`);
383
+ item.status.write('debug', `ports ${ports.map((port) => port.hostPort || 'any').join(', ')} published by a relay`);
384
+ return container;
383
385
  }
384
386
  // The network, its proxy network, its proxy and relay, and every endpoint on
385
387
  // them: what an earlier run left behind, or what the item used.
@@ -429,6 +431,12 @@ async function removeNetwork(docker, name) {
429
431
  }
430
432
  }
431
433
  }
434
+ // A port of the host for a port of a container: a free one for 0, on one address of it
435
+ // when the port names one.
436
+ function getPortBinding(port) {
437
+ return { HostPort: port.hostPort ? `${port.hostPort}` : '', ...(port.hostIp ? { HostIp: port.hostIp } : {}) };
438
+ }
439
+ exports.getPortBinding = getPortBinding;
432
440
  function getNetworkHostConfig(network) {
433
441
  return {
434
442
  ...(network.mode ? { NetworkMode: network.mode } : {}),
@@ -1,14 +1,28 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.withDeadline = void 0;
3
+ exports.withDeadline = exports.getTaskTimeout = void 0;
4
4
  const abort_event_1 = require("../utils/abort-event");
5
+ // How long a task may run: its own `timeout`, else the run's default (`--timeout`), and
6
+ // never longer than the run's cap (`--max-timeout`), which also covers tasks with a
7
+ // longer timeout of their own. A caller can shorten a task this way, never extend it.
8
+ function getTaskTimeout(own, run) {
9
+ const timeout = own ?? run.timeout;
10
+ if (run.maxTimeout === null) {
11
+ return timeout;
12
+ }
13
+ return timeout === null ? run.maxTimeout : Math.min(timeout, run.maxTimeout);
14
+ }
15
+ exports.getTaskTimeout = getTaskTimeout;
16
+ // the longest delay of setTimeout: Node fires a longer one after 1 ms
17
+ const MAX_TIMER_MS = 2 ** 31 - 1;
5
18
  // An abort signal that follows `parent` and additionally fires after `timeout`
6
- // ms, so a task deadline reuses the runtimes' existing abort/cleanup path.
19
+ // ms, so a task deadline reuses the runtimes' existing abort/cleanup path. A
20
+ // timeout beyond what a timer can wait (about 24.8 days) sets no deadline.
7
21
  function withDeadline(parent, timeout) {
8
22
  const controller = new AbortController();
9
23
  let expired = false;
10
24
  const parentListener = (0, abort_event_1.listenOnAbort)(parent, () => controller.abort());
11
- const timer = timeout === null
25
+ const timer = timeout === null || timeout > MAX_TIMER_MS
12
26
  ? null
13
27
  : setTimeout(() => {
14
28
  expired = true;
@@ -0,0 +1,36 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.restoreDeferred = exports.clearDeferredRestore = exports.deferRestore = void 0;
4
+ const abort_1 = require("./abort");
5
+ // Keyed by the item's state, which every run replaces (resetWorkTree): a hit
6
+ // deferred in one run of a Cli is never restored in a later one, over outputs
7
+ // that run built.
8
+ const deferred = new WeakMap();
9
+ function deferRestore(item, run) {
10
+ deferred.set(item.state, { run, promise: null });
11
+ }
12
+ exports.deferRestore = deferRestore;
13
+ // Every cache check starts from scratch: a key that changed since a hit was
14
+ // deferred must not restore the old entry.
15
+ function clearDeferredRestore(item) {
16
+ deferred.delete(item.state);
17
+ }
18
+ exports.clearDeferredRestore = clearDeferredRestore;
19
+ // Restore every given item that was a hit without its outputs, in parallel,
20
+ // each at most once however many items wait on it.
21
+ async function restoreDeferred(work, items, abort) {
22
+ await Promise.all(items.map((item) => {
23
+ const entry = deferred.get(item.state);
24
+ if (!entry) {
25
+ return Promise.resolve();
26
+ }
27
+ if (!entry.promise) {
28
+ work.status.write('debug', `restore ${item.name} from cache, ${work.name} needs its outputs`);
29
+ entry.promise = entry.run();
30
+ }
31
+ return entry.promise;
32
+ }));
33
+ (0, abort_1.checkForAbort)(abort);
34
+ }
35
+ exports.restoreDeferred = restoreDeferred;
36
+ //# sourceMappingURL=deferred-restore.js.map
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.dockerService = exports.buildServiceCreateOptions = void 0;
3
+ exports.dockerService = exports.buildServiceCreateOptions = exports.readPublishedPorts = void 0;
4
4
  const work_service_1 = require("../planner/work-service");
5
5
  const run_labels_1 = require("../docker/run-labels");
6
6
  const service_hostname_1 = require("../planner/utils/service-hostname");
@@ -22,6 +22,28 @@ const work_healthcheck_1 = require("../planner/work-healthcheck");
22
22
  const docker_resources_1 = require("./docker-resources");
23
23
  const egress_proxy_1 = require("../network/egress-proxy");
24
24
  const HEALTHCHECK_INTERVAL_MS = 1000;
25
+ // Where docker published the ports of a container (or of its relay), for those that asked
26
+ // for a free one.
27
+ async function readPublishedPorts(container, ports) {
28
+ const wanted = ports.filter((port) => port.hostPort === 0);
29
+ if (wanted.length === 0) {
30
+ return [];
31
+ }
32
+ const bound = (await container.inspect()).NetworkSettings.Ports ?? {};
33
+ return wanted.flatMap((port) => {
34
+ const binding = bound[`${port.containerPort}/tcp`]?.find((candidate) => candidate.HostPort);
35
+ return binding
36
+ ? [
37
+ {
38
+ containerPort: port.containerPort,
39
+ host: port.hostIp ?? (binding.HostIp && binding.HostIp !== '0.0.0.0' ? binding.HostIp : '127.0.0.1'),
40
+ port: Number(binding.HostPort),
41
+ },
42
+ ]
43
+ : [];
44
+ });
45
+ }
46
+ exports.readPublishedPorts = readPublishedPorts;
25
47
  function buildServiceCreateOptions(item, options, network, secrets, resources = {}) {
26
48
  const envs = (0, replace_env_variables_1.getEnvironmentVariables)(item.data.envs);
27
49
  const command = (0, work_service_1.getServiceCommand)(item.data);
@@ -58,12 +80,9 @@ function buildServiceCreateOptions(item, options, network, secrets, resources =
58
80
  // collide with another run on the same host. A limited service's are
59
81
  // published by its relay.
60
82
  PortBindings: options.publishPorts && !network.relayPorts
61
- ? item.data.ports
62
- .filter((p) => !!p.hostPort)
63
- .reduce((map, port) => {
64
- map[`${port.containerPort}/tcp`] = [{ HostPort: `${port.hostPort}` }];
65
- return map;
66
- }, {})
83
+ ? Object.fromEntries(item.data.ports
84
+ .filter((p) => p.hostPort !== null)
85
+ .map((port) => [`${port.containerPort}/tcp`, [(0, item_network_1.getPortBinding)(port)]]))
67
86
  : {},
68
87
  ...resources,
69
88
  },
@@ -90,16 +109,18 @@ async function dockerService(docker, item, options, environment) {
90
109
  const resources = await (0, docker_resources_1.resolveDockerResources)(docker, item.data.resources, item.status);
91
110
  await (0, execution_steps_1.refuseHostLinks)(item, environment);
92
111
  container = await docker.createContainer(buildServiceCreateOptions(item, options, network.options, secrets, resources));
93
- await network.connect(container);
112
+ const relay = await network.connect(container);
94
113
  const stream = await container.attach({ stream: true, stdout: true, stderr: true });
95
114
  (0, stream_1.logStream)(item.status, stream);
96
115
  await container.start();
116
+ const published = options.publishPorts ? await readPublishedPorts(relay ?? container, item.data.ports) : [];
97
117
  if (!item.data.healthcheck || !options.waitForReady) {
98
118
  options.state.set({
99
119
  type: 'running',
100
120
  dns: { containerId: container.id },
101
121
  stateKey: options.stateKey,
102
122
  remote: null,
123
+ ...(published.length > 0 ? { published } : {}),
103
124
  });
104
125
  }
105
126
  else {
@@ -125,6 +146,7 @@ async function dockerService(docker, item, options, environment) {
125
146
  dns: { containerId: container.id },
126
147
  stateKey: options.stateKey,
127
148
  remote: null,
149
+ ...(published.length > 0 ? { published } : {}),
128
150
  });
129
151
  }
130
152
  }
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.dockerTask = exports.buildCreateOptions = void 0;
3
+ exports.dockerTask = exports.getCapabilityHostConfig = exports.buildCreateOptions = void 0;
4
4
  const run_labels_1 = require("../docker/run-labels");
5
5
  const execute_docker_1 = require("./execute-docker");
6
6
  const abort_1 = require("./abort");
@@ -47,10 +47,30 @@ function buildCreateOptions(item, stateKey, network, environment, secrets, resou
47
47
  ...(0, item_network_1.getNetworkHostConfig)(network),
48
48
  AutoRemove: true,
49
49
  ...resources,
50
+ ...getCapabilityHostConfig(item, environment),
50
51
  },
51
52
  };
52
53
  }
53
54
  exports.buildCreateOptions = buildCreateOptions;
55
+ /**
56
+ * The capabilities a task adds (`capAdd`). On a server they are only granted in the
57
+ * sandboxed runtime of `backend.capabilities`, never by the kernel of the docker host.
58
+ */
59
+ function getCapabilityHostConfig(item, environment) {
60
+ const capAdd = item.data.capAdd;
61
+ if (capAdd.length === 0) {
62
+ return {};
63
+ }
64
+ if (!environment.hosted) {
65
+ return { CapAdd: capAdd };
66
+ }
67
+ const runtime = environment.hosted.capabilityRuntime;
68
+ if (!runtime) {
69
+ throw new Error(`${item.name}: adds the capabilities ${capAdd.join(', ')}, which this server grants in no sandboxed runtime`);
70
+ }
71
+ return { CapAdd: capAdd, Runtime: runtime };
72
+ }
73
+ exports.getCapabilityHostConfig = getCapabilityHostConfig;
54
74
  async function dockerTask(docker, item, environment, options) {
55
75
  item.status.write('info', `execute ${item.name} in container`);
56
76
  let network = null;
@@ -8,6 +8,8 @@ const execute_docker_1 = require("./execute-docker");
8
8
  const plan_work_tasks_1 = require("../planner/utils/plan-work-tasks");
9
9
  const get_cache_directory_1 = require("../optimizer/get-cache-directory");
10
10
  const path_1 = require("path");
11
+ const crypto_1 = require("crypto");
12
+ const claim_layout_1 = require("../kubernetes/claim-layout");
11
13
  const move_files_1 = require("../file/move-files");
12
14
  const work_instance_id_1 = require("../planner/work-instance-id");
13
15
  async function restoreCache(environment, path, workTree) {
@@ -43,8 +45,24 @@ function* getArchivePaths(task, path) {
43
45
  }
44
46
  }
45
47
  else {
46
- for (const generate of task.generates.filter((g) => !g.inherited)) {
47
- const generateId = (0, path_1.relative)(task.cwd, generate.path).split(path_1.sep).join('-');
48
+ // `a-b` and `a/b` flatten alike: of those, the least deep (then the first in sorted
49
+ // order) keeps the name, the others' names carry a hash of their relative path, so a
50
+ // store and a restore in another directory or order find the same archive
51
+ const generates = task.generates
52
+ .filter((g) => !g.inherited)
53
+ .map((generate) => ({ generate, rel: (0, path_1.relative)(task.cwd, generate.path).split(path_1.sep).join(path_1.posix.sep) }));
54
+ const holders = new Map();
55
+ for (const { rel } of [...generates].sort((a, b) => (0, claim_layout_1.byDepth)(a.rel, b.rel))) {
56
+ const id = rel.split(path_1.posix.sep).join('-');
57
+ if (!holders.has(id)) {
58
+ holders.set(id, rel);
59
+ }
60
+ }
61
+ for (const { generate, rel } of generates) {
62
+ let generateId = rel.split(path_1.posix.sep).join('-');
63
+ if (holders.get(generateId) !== rel) {
64
+ generateId += '-' + (0, crypto_1.createHash)('sha256').update(rel).digest('hex').slice(0, 12);
65
+ }
48
66
  yield {
49
67
  filename: (0, path_1.join)(path, generateId + '-generates.tgz'),
50
68
  path: generate.path,
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.executeWorkService = exports.stopService = void 0;
4
4
  const work_item_1 = require("../planner/work-item");
5
5
  const await_completed_dependencies_1 = require("./await-completed-dependencies");
6
+ const deferred_restore_1 = require("./deferred-restore");
6
7
  const abort_1 = require("./abort");
7
8
  const log_1 = require("../log");
8
9
  const watch_loop_1 = require("./watch-loop");
@@ -35,6 +36,7 @@ async function executeWorkService(work, environment, options) {
35
36
  });
36
37
  await (0, watch_loop_1.watchLoop)(work, environment, options, async (cacheState, abort, stop) => {
37
38
  await (0, await_completed_dependencies_1.awaitCompletedDependencies)(work, work.deps, abort);
39
+ await (0, deferred_restore_1.restoreDeferred)(work, (0, await_completed_dependencies_1.transitiveDeps)(work.deps), abort);
38
40
  work.state.set({
39
41
  type: 'ready',
40
42
  stateKey: cacheState.stateKey,