hammerkit 1.27.0-next.3 → 1.27.0-next.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build.schema.json +188 -0
- package/dist/cache/backends/local-cache-backend.js +6 -0
- package/dist/cache/backends/registry-cache-backend.js +2 -2
- package/dist/cache/explain.js +17 -0
- package/dist/cli.js +2 -0
- package/dist/docker/item-network.js +13 -5
- package/dist/executer/deadline.js +17 -3
- package/dist/executer/deferred-restore.js +36 -0
- package/dist/executer/docker-service.js +30 -8
- package/dist/executer/docker-task.js +21 -1
- package/dist/executer/event-cache.js +20 -2
- package/dist/executer/execute-work-service.js +2 -0
- package/dist/executer/execute-work-task.js +10 -3
- package/dist/executer/execution-steps.js +25 -4
- package/dist/executer/reset-work-tree.js +4 -0
- package/dist/executer/scheduler/enqueue-next.js +57 -6
- package/dist/executer/scheduler/state-key.js +42 -5
- package/dist/executer/watch-loop.js +4 -1
- package/dist/harness/connect.js +20 -6
- package/dist/harness/mcp-command.js +22 -6
- package/dist/harness/mcp.js +63 -114
- package/dist/harness/opencode.js +158 -50
- package/dist/harness/pi.js +29 -0
- package/dist/harness/sandbox-client.js +168 -17
- package/dist/harness/service-tools.js +141 -0
- package/dist/harness/task-tools.js +221 -0
- package/dist/kubernetes/apply.js +27 -16
- package/dist/kubernetes/await-running-state.js +59 -13
- package/dist/kubernetes/claim-layout.js +71 -0
- package/dist/kubernetes/cluster-state.js +245 -0
- package/dist/kubernetes/ensure-http-route.js +1 -1
- package/dist/kubernetes/ensure-kubernetes-deployment-exists.js +35 -12
- package/dist/kubernetes/ensure-kubernetes-persistent-volume-claim-exists.js +1 -1
- package/dist/kubernetes/ensure-kubernetes-service-exists.js +6 -9
- package/dist/kubernetes/ensure-persistent-data.js +30 -20
- package/dist/kubernetes/follow-pod-logs.js +182 -0
- package/dist/kubernetes/get-service-ip.js +2 -2
- package/dist/kubernetes/job-placement.js +36 -12
- package/dist/kubernetes/job-secrets.js +33 -19
- package/dist/kubernetes/kept-claims.js +22 -10
- package/dist/kubernetes/network-policy.js +85 -27
- package/dist/kubernetes/remove-persistent-data.js +3 -1
- package/dist/kubernetes/resources.js +25 -3
- package/dist/kubernetes/restore-kubernetes-data.js +9 -3
- package/dist/kubernetes/sandbox-runner.js +628 -0
- package/dist/kubernetes/secrets.js +7 -11
- package/dist/kubernetes/volumes.js +6 -3
- package/dist/main.js +2 -0
- package/dist/network/egress-proxy.js +45 -6
- package/dist/optimizer/calculate-checksum.js +3 -2
- package/dist/optimizer/check-key.js +94 -0
- package/dist/optimizer/check-keys.js +64 -0
- package/dist/optimizer/get-work-cache-stats.js +17 -3
- package/dist/optimizer/work-task-cache-description.js +51 -1
- package/dist/planner/utils/append-work-dependencies.js +10 -5
- package/dist/planner/utils/append-work-service.js +19 -1
- package/dist/planner/utils/append-work-task.js +18 -3
- package/dist/planner/utils/get-work-runtime.js +6 -1
- package/dist/planner/utils/normalize-path.js +19 -1
- package/dist/planner/utils/parse-work-mount.js +7 -1
- package/dist/planner/utils/parse-work-port.js +4 -1
- package/dist/planner/utils/task-arguments.js +3 -0
- package/dist/planner/validate.js +22 -0
- package/dist/planner/work-network.js +3 -1
- package/dist/planner/work-runtime-aggregate.js +32 -0
- package/dist/planner/work-runtime-docker.js +2 -1
- package/dist/planner/work-runtime-kubernetes.js +124 -48
- package/dist/planner/work-runtime-local.js +2 -1
- package/dist/planner/work-task.js +6 -1
- package/dist/program.js +31 -10
- package/dist/remote/device-login.js +7 -2
- package/dist/remote/host-config.js +69 -3
- package/dist/remote/remote-commands.js +30 -7
- package/dist/remote/remote-work-commands.js +426 -45
- package/dist/remote/server-client.js +15 -2
- package/dist/remote/token-source.js +64 -26
- package/dist/run-program.js +9 -0
- package/dist/schema/build-file-base-task-schema.js +8 -1
- package/dist/schema/build-file-container-service-schema.js +12 -0
- package/dist/schema/build-file-container-task-schema.js +16 -1
- package/dist/schema/build-file-include-schema.js +18 -4
- package/dist/schema/build-file-kubernetes-service-schema.js +2 -0
- package/dist/schema/build-file-task-schema.js +5 -1
- package/dist/schema/capabilities-schema.js +56 -0
- package/dist/schema/reference-parser.js +120 -11
- package/dist/schema/schema-parser.js +92 -41
- package/dist/schema/work-scope-parser.js +33 -10
- package/dist/secrets/command-secret-provider.js +3 -1
- package/dist/secrets/github-app-token-command.js +3 -1
- package/dist/secrets/secret-catalog.js +43 -19
- package/dist/server/accounts-api.js +3 -1
- package/dist/server/approvals-api.js +1 -1
- package/dist/server/checks-api.js +83 -0
- package/dist/server/checks.js +116 -0
- package/dist/server/definitions-api.js +66 -3
- package/dist/server/endpoints.js +739 -0
- package/dist/server/executor.js +57 -5
- package/dist/server/kubernetes-backend.js +367 -65
- package/dist/server/kubernetes-sandbox-storage.js +48 -3
- package/dist/server/oauth.js +12 -1
- package/dist/server/plan.js +188 -103
- package/dist/server/policy.js +32 -2
- package/dist/server/repo-access.js +5 -0
- package/dist/server/run-plan.js +50 -19
- package/dist/server/run-request.js +58 -3
- package/dist/server/run-worker.js +245 -12
- package/dist/server/runs.js +410 -27
- package/dist/server/sandbox-pod.js +28 -0
- package/dist/server/sandbox-storage.js +23 -0
- package/dist/server/sandboxes-api.js +10 -4
- package/dist/server/sandboxes.js +8 -0
- package/dist/server/server-command.js +39 -20
- package/dist/server/server-config.js +255 -14
- package/dist/server/server.js +269 -15
- package/dist/server/service-runs.js +82 -0
- package/dist/utils/abort-on-signals.js +46 -6
- package/dist/utils/tar.js +36 -0
- package/dist/utils/terminal-prompt.js +53 -0
- package/package.json +1 -1
package/build.schema.json
CHANGED
|
@@ -289,10 +289,16 @@
|
|
|
289
289
|
],
|
|
290
290
|
"description": "network the task or service reaches besides its needs: open (default), none or { allow: [hosts or host:port] }"
|
|
291
291
|
},
|
|
292
|
+
"args": {
|
|
293
|
+
"type": "boolean"
|
|
294
|
+
},
|
|
292
295
|
"runner": {
|
|
293
296
|
"type": "string",
|
|
294
297
|
"minLength": 1
|
|
295
298
|
},
|
|
299
|
+
"publish": {
|
|
300
|
+
"type": "boolean"
|
|
301
|
+
},
|
|
296
302
|
"image": {
|
|
297
303
|
"type": "string"
|
|
298
304
|
},
|
|
@@ -305,6 +311,56 @@
|
|
|
305
311
|
"volumeSize": {
|
|
306
312
|
"type": "string",
|
|
307
313
|
"description": "size of its claim on Kubernetes, e.g. 10Gi"
|
|
314
|
+
},
|
|
315
|
+
"capAdd": {
|
|
316
|
+
"type": "array",
|
|
317
|
+
"items": {
|
|
318
|
+
"type": "string",
|
|
319
|
+
"enum": [
|
|
320
|
+
"AUDIT_CONTROL",
|
|
321
|
+
"AUDIT_READ",
|
|
322
|
+
"AUDIT_WRITE",
|
|
323
|
+
"BLOCK_SUSPEND",
|
|
324
|
+
"BPF",
|
|
325
|
+
"CHECKPOINT_RESTORE",
|
|
326
|
+
"CHOWN",
|
|
327
|
+
"DAC_OVERRIDE",
|
|
328
|
+
"DAC_READ_SEARCH",
|
|
329
|
+
"FOWNER",
|
|
330
|
+
"FSETID",
|
|
331
|
+
"IPC_LOCK",
|
|
332
|
+
"IPC_OWNER",
|
|
333
|
+
"KILL",
|
|
334
|
+
"LEASE",
|
|
335
|
+
"LINUX_IMMUTABLE",
|
|
336
|
+
"MAC_ADMIN",
|
|
337
|
+
"MAC_OVERRIDE",
|
|
338
|
+
"MKNOD",
|
|
339
|
+
"NET_ADMIN",
|
|
340
|
+
"NET_BIND_SERVICE",
|
|
341
|
+
"NET_BROADCAST",
|
|
342
|
+
"NET_RAW",
|
|
343
|
+
"PERFMON",
|
|
344
|
+
"SETFCAP",
|
|
345
|
+
"SETGID",
|
|
346
|
+
"SETPCAP",
|
|
347
|
+
"SETUID",
|
|
348
|
+
"SYS_ADMIN",
|
|
349
|
+
"SYS_BOOT",
|
|
350
|
+
"SYS_CHROOT",
|
|
351
|
+
"SYS_MODULE",
|
|
352
|
+
"SYS_NICE",
|
|
353
|
+
"SYS_PACCT",
|
|
354
|
+
"SYS_PTRACE",
|
|
355
|
+
"SYS_RAWIO",
|
|
356
|
+
"SYS_RESOURCE",
|
|
357
|
+
"SYS_TIME",
|
|
358
|
+
"SYS_TTY_CONFIG",
|
|
359
|
+
"SYSLOG",
|
|
360
|
+
"WAKE_ALARM"
|
|
361
|
+
]
|
|
362
|
+
},
|
|
363
|
+
"description": "Linux capabilities added to the container, e.g. [SYS_ADMIN]"
|
|
308
364
|
}
|
|
309
365
|
},
|
|
310
366
|
"required": [
|
|
@@ -370,9 +426,15 @@
|
|
|
370
426
|
"network": {
|
|
371
427
|
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/network"
|
|
372
428
|
},
|
|
429
|
+
"args": {
|
|
430
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/args"
|
|
431
|
+
},
|
|
373
432
|
"runner": {
|
|
374
433
|
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/runner"
|
|
375
434
|
},
|
|
435
|
+
"publish": {
|
|
436
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/publish"
|
|
437
|
+
},
|
|
376
438
|
"platform": {
|
|
377
439
|
"type": "object",
|
|
378
440
|
"properties": {
|
|
@@ -397,6 +459,91 @@
|
|
|
397
459
|
},
|
|
398
460
|
"additionalProperties": false,
|
|
399
461
|
"description": "local task"
|
|
462
|
+
},
|
|
463
|
+
{
|
|
464
|
+
"type": "object",
|
|
465
|
+
"properties": {
|
|
466
|
+
"deps": {
|
|
467
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/deps"
|
|
468
|
+
},
|
|
469
|
+
"needs": {
|
|
470
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/needs"
|
|
471
|
+
},
|
|
472
|
+
"description": {
|
|
473
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/description"
|
|
474
|
+
},
|
|
475
|
+
"extend": {
|
|
476
|
+
"type": "string"
|
|
477
|
+
},
|
|
478
|
+
"reset": {
|
|
479
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/reset"
|
|
480
|
+
},
|
|
481
|
+
"envs": {
|
|
482
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/envs"
|
|
483
|
+
},
|
|
484
|
+
"labels": {
|
|
485
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/labels"
|
|
486
|
+
},
|
|
487
|
+
"cache": {
|
|
488
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/cache"
|
|
489
|
+
},
|
|
490
|
+
"cmds": {
|
|
491
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/cmds"
|
|
492
|
+
},
|
|
493
|
+
"src": {
|
|
494
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/src"
|
|
495
|
+
},
|
|
496
|
+
"generates": {
|
|
497
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/generates"
|
|
498
|
+
},
|
|
499
|
+
"shell": {
|
|
500
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/shell"
|
|
501
|
+
},
|
|
502
|
+
"continuous": {
|
|
503
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/continuous"
|
|
504
|
+
},
|
|
505
|
+
"timeout": {
|
|
506
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/timeout"
|
|
507
|
+
},
|
|
508
|
+
"secrets": {
|
|
509
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/secrets"
|
|
510
|
+
},
|
|
511
|
+
"account": {
|
|
512
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/account"
|
|
513
|
+
},
|
|
514
|
+
"resources": {
|
|
515
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/resources"
|
|
516
|
+
},
|
|
517
|
+
"network": {
|
|
518
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/network"
|
|
519
|
+
},
|
|
520
|
+
"args": {
|
|
521
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/args"
|
|
522
|
+
},
|
|
523
|
+
"runner": {
|
|
524
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/runner"
|
|
525
|
+
},
|
|
526
|
+
"publish": {
|
|
527
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/publish"
|
|
528
|
+
},
|
|
529
|
+
"image": {
|
|
530
|
+
"type": "string"
|
|
531
|
+
},
|
|
532
|
+
"mounts": {
|
|
533
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/mounts"
|
|
534
|
+
},
|
|
535
|
+
"volumeSize": {
|
|
536
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/volumeSize"
|
|
537
|
+
},
|
|
538
|
+
"capAdd": {
|
|
539
|
+
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/capAdd"
|
|
540
|
+
}
|
|
541
|
+
},
|
|
542
|
+
"required": [
|
|
543
|
+
"extend"
|
|
544
|
+
],
|
|
545
|
+
"additionalProperties": false,
|
|
546
|
+
"description": "container task taking its image from the task it extends"
|
|
400
547
|
}
|
|
401
548
|
]
|
|
402
549
|
}
|
|
@@ -504,6 +651,12 @@
|
|
|
504
651
|
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/resources",
|
|
505
652
|
"description": "cpu and memory requests and limits"
|
|
506
653
|
},
|
|
654
|
+
"restarts": {
|
|
655
|
+
"type": "integer",
|
|
656
|
+
"minimum": 0,
|
|
657
|
+
"maximum": 10,
|
|
658
|
+
"description": "crashes restarted before it fails on Kubernetes, 0 to 10, 2 by default"
|
|
659
|
+
},
|
|
507
660
|
"volumeSize": {
|
|
508
661
|
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/volumeSize",
|
|
509
662
|
"description": "size of its claim on Kubernetes, e.g. 10Gi"
|
|
@@ -511,6 +664,12 @@
|
|
|
511
664
|
"network": {
|
|
512
665
|
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/network",
|
|
513
666
|
"description": "network the task or service reaches besides its needs: open (default), none or { allow: [hosts or host:port] }"
|
|
667
|
+
},
|
|
668
|
+
"args": {
|
|
669
|
+
"type": "boolean"
|
|
670
|
+
},
|
|
671
|
+
"publish": {
|
|
672
|
+
"type": "boolean"
|
|
514
673
|
}
|
|
515
674
|
},
|
|
516
675
|
"required": [
|
|
@@ -567,6 +726,9 @@
|
|
|
567
726
|
"number"
|
|
568
727
|
]
|
|
569
728
|
}
|
|
729
|
+
},
|
|
730
|
+
"publish": {
|
|
731
|
+
"type": "boolean"
|
|
570
732
|
}
|
|
571
733
|
},
|
|
572
734
|
"required": [
|
|
@@ -586,6 +748,28 @@
|
|
|
586
748
|
{
|
|
587
749
|
"type": "string"
|
|
588
750
|
},
|
|
751
|
+
{
|
|
752
|
+
"type": "object",
|
|
753
|
+
"properties": {
|
|
754
|
+
"path": {
|
|
755
|
+
"type": "string",
|
|
756
|
+
"minLength": 1,
|
|
757
|
+
"description": "Build file or directory, relative to the including file"
|
|
758
|
+
},
|
|
759
|
+
"with": {
|
|
760
|
+
"$ref": "#/definitions/hammerkit/properties/envs",
|
|
761
|
+
"description": "Environment values for the current build task\nhttps://hammerkit.dev/docs/build-file/environment-variables"
|
|
762
|
+
},
|
|
763
|
+
"publish": {
|
|
764
|
+
"type": "boolean",
|
|
765
|
+
"description": "false: a server leaves out what this file declares"
|
|
766
|
+
}
|
|
767
|
+
},
|
|
768
|
+
"required": [
|
|
769
|
+
"path"
|
|
770
|
+
],
|
|
771
|
+
"additionalProperties": false
|
|
772
|
+
},
|
|
589
773
|
{
|
|
590
774
|
"type": "object",
|
|
591
775
|
"properties": {
|
|
@@ -606,6 +790,10 @@
|
|
|
606
790
|
"with": {
|
|
607
791
|
"$ref": "#/definitions/hammerkit/properties/envs",
|
|
608
792
|
"description": "Environment values for the current build task\nhttps://hammerkit.dev/docs/build-file/environment-variables"
|
|
793
|
+
},
|
|
794
|
+
"publish": {
|
|
795
|
+
"type": "boolean",
|
|
796
|
+
"description": "false: a server leaves out what this file declares"
|
|
609
797
|
}
|
|
610
798
|
},
|
|
611
799
|
"required": [
|
|
@@ -80,6 +80,12 @@ function createLocalCacheBackend(spec) {
|
|
|
80
80
|
return false;
|
|
81
81
|
}
|
|
82
82
|
},
|
|
83
|
+
async touch(taskId, stateKey, environment) {
|
|
84
|
+
const remoteDir = (0, path_1.join)(root, taskId, stateKey);
|
|
85
|
+
if (await environment.file.exists((0, path_1.join)(remoteDir, 'stats.json'))) {
|
|
86
|
+
await (0, promises_1.writeFile)((0, path_1.join)(remoteDir, LAST_ACCESS_FILE), `${Date.now()}`);
|
|
87
|
+
}
|
|
88
|
+
},
|
|
83
89
|
async push(taskId, stateKey, from, environment) {
|
|
84
90
|
const remoteDir = (0, path_1.join)(root, taskId, stateKey);
|
|
85
91
|
await environment.file.createDirectory(remoteDir);
|
|
@@ -5,6 +5,7 @@ const crypto_1 = require("crypto");
|
|
|
5
5
|
const os_1 = require("os");
|
|
6
6
|
const path_1 = require("path");
|
|
7
7
|
const tar_1 = require("tar");
|
|
8
|
+
const tar_2 = require("../../utils/tar");
|
|
8
9
|
const registry_reference_1 = require("./registry/registry-reference");
|
|
9
10
|
const registry_client_1 = require("./registry/registry-client");
|
|
10
11
|
const MAX_TAG_LENGTH = 128;
|
|
@@ -74,8 +75,7 @@ function createRegistryCacheBackend(spec) {
|
|
|
74
75
|
const layerFile = (0, path_1.join)(dir, 'layer.tar');
|
|
75
76
|
const files = (await environment.file.listFiles(from)).sort();
|
|
76
77
|
// portable + mtime-free so identical entries produce identical digests
|
|
77
|
-
|
|
78
|
-
await (0, tar_1.create)({ file: layerFile, cwd: from, portable: true, noMtime: true }, files);
|
|
78
|
+
await (0, tar_2.packArchive)({ file: layerFile, cwd: from, portable: true, noMtime: true }, files);
|
|
79
79
|
const layer = await (0, registry_client_1.digestFile)(layerFile);
|
|
80
80
|
const config = Buffer.from(JSON.stringify({
|
|
81
81
|
architecture: 'amd64',
|
package/dist/cache/explain.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.explainWorkTree = exports.explainTask = exports.describeCause = void 0;
|
|
4
|
+
const work_task_1 = require("../planner/work-task");
|
|
4
5
|
const state_key_1 = require("../executer/scheduler/state-key");
|
|
5
6
|
const work_task_cache_description_1 = require("../optimizer/work-task-cache-description");
|
|
6
7
|
const get_cache_directory_1 = require("../optimizer/get-cache-directory");
|
|
@@ -157,6 +158,22 @@ async function explainTask(item, defaultCacheMethod, environment, memo = new Map
|
|
|
157
158
|
visiting.delete(item.id());
|
|
158
159
|
return explanation;
|
|
159
160
|
}
|
|
161
|
+
// Mirrors executeWorkTask: an aggregate is never looked up itself, and is reported
|
|
162
|
+
// cached when all its dependencies were. What changed is in its dependencies.
|
|
163
|
+
if ((0, work_task_1.isAggregateTask)(item.data)) {
|
|
164
|
+
const causes = [];
|
|
165
|
+
for (const dep of item.deps) {
|
|
166
|
+
const depExplanation = await explainTask(dep, defaultCacheMethod, environment, memo, visiting);
|
|
167
|
+
if (depExplanation.status !== 'hit') {
|
|
168
|
+
causes.push({ kind: 'dependency-changed', identifier: dep.name });
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
const status = causes.length === 0 ? 'hit' : 'miss';
|
|
172
|
+
const explanation = { taskId: item.id(), taskName: item.name, status, causes };
|
|
173
|
+
memo.set(item.id(), explanation);
|
|
174
|
+
visiting.delete(item.id());
|
|
175
|
+
return explanation;
|
|
176
|
+
}
|
|
160
177
|
const runtimeStateKey = await safeCurrentStateKey(item, environment);
|
|
161
178
|
const hasEntry = await safeHas(item, stateKey, environment);
|
|
162
179
|
const predictedHit = runtimeStateKey === stateKey || hasEntry;
|
package/dist/cli.js
CHANGED
|
@@ -79,7 +79,9 @@ class Cli {
|
|
|
79
79
|
explain: options?.explain ?? false,
|
|
80
80
|
cacheReadOnly: options?.cacheReadOnly ?? false,
|
|
81
81
|
timeout: options?.timeout ?? null,
|
|
82
|
+
maxTimeout: options?.maxTimeout ?? null,
|
|
82
83
|
skipDeps: options?.skipDeps ?? true,
|
|
84
|
+
restore: options?.restore ?? true,
|
|
83
85
|
wait: options?.wait ?? 'ready',
|
|
84
86
|
resourceCheck: options?.resourceCheck ?? true,
|
|
85
87
|
network: options?.network ?? null,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.removeOrphanedNetworks = exports.removeItemNetworks = exports.getNetworkHostConfig = exports.removeItemNetwork = exports.prepareItemNetwork = exports.checkNetworkLimitSupport = exports.getProxyNetworkName = exports.getRunNetworkName = exports.getItemNetworkName = void 0;
|
|
3
|
+
exports.removeOrphanedNetworks = exports.removeItemNetworks = exports.getNetworkHostConfig = exports.getPortBinding = exports.removeItemNetwork = exports.prepareItemNetwork = exports.checkNetworkLimitSupport = exports.getProxyNetworkName = exports.getRunNetworkName = exports.getItemNetworkName = void 0;
|
|
4
4
|
const readline_1 = require("readline");
|
|
5
5
|
const stream_1 = require("stream");
|
|
6
6
|
const work_service_1 = require("../planner/work-service");
|
|
@@ -258,7 +258,7 @@ options) {
|
|
|
258
258
|
await removeItemNetwork(docker, name);
|
|
259
259
|
throw e;
|
|
260
260
|
}
|
|
261
|
-
const publish = (options.publish ?? []).filter((port) =>
|
|
261
|
+
const publish = (options.publish ?? []).filter((port) => port.hostPort !== null);
|
|
262
262
|
return {
|
|
263
263
|
options: {
|
|
264
264
|
mode: name,
|
|
@@ -272,8 +272,9 @@ options) {
|
|
|
272
272
|
await docker.getNetwork(getProxyNetworkName(name)).connect({ Container: container.id });
|
|
273
273
|
}
|
|
274
274
|
if (publish.length > 0) {
|
|
275
|
-
await startPortRelay(docker, item, environment, name, container, publish, options);
|
|
275
|
+
return await startPortRelay(docker, item, environment, name, container, publish, options);
|
|
276
276
|
}
|
|
277
|
+
return undefined;
|
|
277
278
|
}
|
|
278
279
|
catch (e) {
|
|
279
280
|
await removeItemNetwork(docker, name);
|
|
@@ -374,12 +375,13 @@ async function startPortRelay(docker, item, environment, networkName, service, p
|
|
|
374
375
|
ExposedPorts: Object.fromEntries(ports.map((port) => [`${port.containerPort}/tcp`, {}])),
|
|
375
376
|
HostConfig: {
|
|
376
377
|
...HELPER_HOST_CONFIG,
|
|
377
|
-
PortBindings: Object.fromEntries(ports.map((port) => [`${port.containerPort}/tcp`, [
|
|
378
|
+
PortBindings: Object.fromEntries(ports.map((port) => [`${port.containerPort}/tcp`, [getPortBinding(port)]])),
|
|
378
379
|
},
|
|
379
380
|
});
|
|
380
381
|
await docker.getNetwork(networkName).connect({ Container: container.id });
|
|
381
382
|
await container.start();
|
|
382
|
-
item.status.write('debug', `ports ${ports.map((port) => port.hostPort).join(', ')} published by a relay`);
|
|
383
|
+
item.status.write('debug', `ports ${ports.map((port) => port.hostPort || 'any').join(', ')} published by a relay`);
|
|
384
|
+
return container;
|
|
383
385
|
}
|
|
384
386
|
// The network, its proxy network, its proxy and relay, and every endpoint on
|
|
385
387
|
// them: what an earlier run left behind, or what the item used.
|
|
@@ -429,6 +431,12 @@ async function removeNetwork(docker, name) {
|
|
|
429
431
|
}
|
|
430
432
|
}
|
|
431
433
|
}
|
|
434
|
+
// A port of the host for a port of a container: a free one for 0, on one address of it
|
|
435
|
+
// when the port names one.
|
|
436
|
+
function getPortBinding(port) {
|
|
437
|
+
return { HostPort: port.hostPort ? `${port.hostPort}` : '', ...(port.hostIp ? { HostIp: port.hostIp } : {}) };
|
|
438
|
+
}
|
|
439
|
+
exports.getPortBinding = getPortBinding;
|
|
432
440
|
function getNetworkHostConfig(network) {
|
|
433
441
|
return {
|
|
434
442
|
...(network.mode ? { NetworkMode: network.mode } : {}),
|
|
@@ -1,14 +1,28 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.withDeadline = void 0;
|
|
3
|
+
exports.withDeadline = exports.getTaskTimeout = void 0;
|
|
4
4
|
const abort_event_1 = require("../utils/abort-event");
|
|
5
|
+
// How long a task may run: its own `timeout`, else the run's default (`--timeout`), and
|
|
6
|
+
// never longer than the run's cap (`--max-timeout`), which also covers tasks with a
|
|
7
|
+
// longer timeout of their own. A caller can shorten a task this way, never extend it.
|
|
8
|
+
function getTaskTimeout(own, run) {
|
|
9
|
+
const timeout = own ?? run.timeout;
|
|
10
|
+
if (run.maxTimeout === null) {
|
|
11
|
+
return timeout;
|
|
12
|
+
}
|
|
13
|
+
return timeout === null ? run.maxTimeout : Math.min(timeout, run.maxTimeout);
|
|
14
|
+
}
|
|
15
|
+
exports.getTaskTimeout = getTaskTimeout;
|
|
16
|
+
// the longest delay of setTimeout: Node fires a longer one after 1 ms
|
|
17
|
+
const MAX_TIMER_MS = 2 ** 31 - 1;
|
|
5
18
|
// An abort signal that follows `parent` and additionally fires after `timeout`
|
|
6
|
-
// ms, so a task deadline reuses the runtimes' existing abort/cleanup path.
|
|
19
|
+
// ms, so a task deadline reuses the runtimes' existing abort/cleanup path. A
|
|
20
|
+
// timeout beyond what a timer can wait (about 24.8 days) sets no deadline.
|
|
7
21
|
function withDeadline(parent, timeout) {
|
|
8
22
|
const controller = new AbortController();
|
|
9
23
|
let expired = false;
|
|
10
24
|
const parentListener = (0, abort_event_1.listenOnAbort)(parent, () => controller.abort());
|
|
11
|
-
const timer = timeout === null
|
|
25
|
+
const timer = timeout === null || timeout > MAX_TIMER_MS
|
|
12
26
|
? null
|
|
13
27
|
: setTimeout(() => {
|
|
14
28
|
expired = true;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.restoreDeferred = exports.clearDeferredRestore = exports.deferRestore = void 0;
|
|
4
|
+
const abort_1 = require("./abort");
|
|
5
|
+
// Keyed by the item's state, which every run replaces (resetWorkTree): a hit
|
|
6
|
+
// deferred in one run of a Cli is never restored in a later one, over outputs
|
|
7
|
+
// that run built.
|
|
8
|
+
const deferred = new WeakMap();
|
|
9
|
+
function deferRestore(item, run) {
|
|
10
|
+
deferred.set(item.state, { run, promise: null });
|
|
11
|
+
}
|
|
12
|
+
exports.deferRestore = deferRestore;
|
|
13
|
+
// Every cache check starts from scratch: a key that changed since a hit was
|
|
14
|
+
// deferred must not restore the old entry.
|
|
15
|
+
function clearDeferredRestore(item) {
|
|
16
|
+
deferred.delete(item.state);
|
|
17
|
+
}
|
|
18
|
+
exports.clearDeferredRestore = clearDeferredRestore;
|
|
19
|
+
// Restore every given item that was a hit without its outputs, in parallel,
|
|
20
|
+
// each at most once however many items wait on it.
|
|
21
|
+
async function restoreDeferred(work, items, abort) {
|
|
22
|
+
await Promise.all(items.map((item) => {
|
|
23
|
+
const entry = deferred.get(item.state);
|
|
24
|
+
if (!entry) {
|
|
25
|
+
return Promise.resolve();
|
|
26
|
+
}
|
|
27
|
+
if (!entry.promise) {
|
|
28
|
+
work.status.write('debug', `restore ${item.name} from cache, ${work.name} needs its outputs`);
|
|
29
|
+
entry.promise = entry.run();
|
|
30
|
+
}
|
|
31
|
+
return entry.promise;
|
|
32
|
+
}));
|
|
33
|
+
(0, abort_1.checkForAbort)(abort);
|
|
34
|
+
}
|
|
35
|
+
exports.restoreDeferred = restoreDeferred;
|
|
36
|
+
//# sourceMappingURL=deferred-restore.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.dockerService = exports.buildServiceCreateOptions = void 0;
|
|
3
|
+
exports.dockerService = exports.buildServiceCreateOptions = exports.readPublishedPorts = void 0;
|
|
4
4
|
const work_service_1 = require("../planner/work-service");
|
|
5
5
|
const run_labels_1 = require("../docker/run-labels");
|
|
6
6
|
const service_hostname_1 = require("../planner/utils/service-hostname");
|
|
@@ -22,6 +22,28 @@ const work_healthcheck_1 = require("../planner/work-healthcheck");
|
|
|
22
22
|
const docker_resources_1 = require("./docker-resources");
|
|
23
23
|
const egress_proxy_1 = require("../network/egress-proxy");
|
|
24
24
|
const HEALTHCHECK_INTERVAL_MS = 1000;
|
|
25
|
+
// Where docker published the ports of a container (or of its relay), for those that asked
|
|
26
|
+
// for a free one.
|
|
27
|
+
async function readPublishedPorts(container, ports) {
|
|
28
|
+
const wanted = ports.filter((port) => port.hostPort === 0);
|
|
29
|
+
if (wanted.length === 0) {
|
|
30
|
+
return [];
|
|
31
|
+
}
|
|
32
|
+
const bound = (await container.inspect()).NetworkSettings.Ports ?? {};
|
|
33
|
+
return wanted.flatMap((port) => {
|
|
34
|
+
const binding = bound[`${port.containerPort}/tcp`]?.find((candidate) => candidate.HostPort);
|
|
35
|
+
return binding
|
|
36
|
+
? [
|
|
37
|
+
{
|
|
38
|
+
containerPort: port.containerPort,
|
|
39
|
+
host: port.hostIp ?? (binding.HostIp && binding.HostIp !== '0.0.0.0' ? binding.HostIp : '127.0.0.1'),
|
|
40
|
+
port: Number(binding.HostPort),
|
|
41
|
+
},
|
|
42
|
+
]
|
|
43
|
+
: [];
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
exports.readPublishedPorts = readPublishedPorts;
|
|
25
47
|
function buildServiceCreateOptions(item, options, network, secrets, resources = {}) {
|
|
26
48
|
const envs = (0, replace_env_variables_1.getEnvironmentVariables)(item.data.envs);
|
|
27
49
|
const command = (0, work_service_1.getServiceCommand)(item.data);
|
|
@@ -58,12 +80,9 @@ function buildServiceCreateOptions(item, options, network, secrets, resources =
|
|
|
58
80
|
// collide with another run on the same host. A limited service's are
|
|
59
81
|
// published by its relay.
|
|
60
82
|
PortBindings: options.publishPorts && !network.relayPorts
|
|
61
|
-
? item.data.ports
|
|
62
|
-
.filter((p) =>
|
|
63
|
-
.
|
|
64
|
-
map[`${port.containerPort}/tcp`] = [{ HostPort: `${port.hostPort}` }];
|
|
65
|
-
return map;
|
|
66
|
-
}, {})
|
|
83
|
+
? Object.fromEntries(item.data.ports
|
|
84
|
+
.filter((p) => p.hostPort !== null)
|
|
85
|
+
.map((port) => [`${port.containerPort}/tcp`, [(0, item_network_1.getPortBinding)(port)]]))
|
|
67
86
|
: {},
|
|
68
87
|
...resources,
|
|
69
88
|
},
|
|
@@ -90,16 +109,18 @@ async function dockerService(docker, item, options, environment) {
|
|
|
90
109
|
const resources = await (0, docker_resources_1.resolveDockerResources)(docker, item.data.resources, item.status);
|
|
91
110
|
await (0, execution_steps_1.refuseHostLinks)(item, environment);
|
|
92
111
|
container = await docker.createContainer(buildServiceCreateOptions(item, options, network.options, secrets, resources));
|
|
93
|
-
await network.connect(container);
|
|
112
|
+
const relay = await network.connect(container);
|
|
94
113
|
const stream = await container.attach({ stream: true, stdout: true, stderr: true });
|
|
95
114
|
(0, stream_1.logStream)(item.status, stream);
|
|
96
115
|
await container.start();
|
|
116
|
+
const published = options.publishPorts ? await readPublishedPorts(relay ?? container, item.data.ports) : [];
|
|
97
117
|
if (!item.data.healthcheck || !options.waitForReady) {
|
|
98
118
|
options.state.set({
|
|
99
119
|
type: 'running',
|
|
100
120
|
dns: { containerId: container.id },
|
|
101
121
|
stateKey: options.stateKey,
|
|
102
122
|
remote: null,
|
|
123
|
+
...(published.length > 0 ? { published } : {}),
|
|
103
124
|
});
|
|
104
125
|
}
|
|
105
126
|
else {
|
|
@@ -125,6 +146,7 @@ async function dockerService(docker, item, options, environment) {
|
|
|
125
146
|
dns: { containerId: container.id },
|
|
126
147
|
stateKey: options.stateKey,
|
|
127
148
|
remote: null,
|
|
149
|
+
...(published.length > 0 ? { published } : {}),
|
|
128
150
|
});
|
|
129
151
|
}
|
|
130
152
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.dockerTask = exports.buildCreateOptions = void 0;
|
|
3
|
+
exports.dockerTask = exports.getCapabilityHostConfig = exports.buildCreateOptions = void 0;
|
|
4
4
|
const run_labels_1 = require("../docker/run-labels");
|
|
5
5
|
const execute_docker_1 = require("./execute-docker");
|
|
6
6
|
const abort_1 = require("./abort");
|
|
@@ -47,10 +47,30 @@ function buildCreateOptions(item, stateKey, network, environment, secrets, resou
|
|
|
47
47
|
...(0, item_network_1.getNetworkHostConfig)(network),
|
|
48
48
|
AutoRemove: true,
|
|
49
49
|
...resources,
|
|
50
|
+
...getCapabilityHostConfig(item, environment),
|
|
50
51
|
},
|
|
51
52
|
};
|
|
52
53
|
}
|
|
53
54
|
exports.buildCreateOptions = buildCreateOptions;
|
|
55
|
+
/**
|
|
56
|
+
* The capabilities a task adds (`capAdd`). On a server they are only granted in the
|
|
57
|
+
* sandboxed runtime of `backend.capabilities`, never by the kernel of the docker host.
|
|
58
|
+
*/
|
|
59
|
+
function getCapabilityHostConfig(item, environment) {
|
|
60
|
+
const capAdd = item.data.capAdd;
|
|
61
|
+
if (capAdd.length === 0) {
|
|
62
|
+
return {};
|
|
63
|
+
}
|
|
64
|
+
if (!environment.hosted) {
|
|
65
|
+
return { CapAdd: capAdd };
|
|
66
|
+
}
|
|
67
|
+
const runtime = environment.hosted.capabilityRuntime;
|
|
68
|
+
if (!runtime) {
|
|
69
|
+
throw new Error(`${item.name}: adds the capabilities ${capAdd.join(', ')}, which this server grants in no sandboxed runtime`);
|
|
70
|
+
}
|
|
71
|
+
return { CapAdd: capAdd, Runtime: runtime };
|
|
72
|
+
}
|
|
73
|
+
exports.getCapabilityHostConfig = getCapabilityHostConfig;
|
|
54
74
|
async function dockerTask(docker, item, environment, options) {
|
|
55
75
|
item.status.write('info', `execute ${item.name} in container`);
|
|
56
76
|
let network = null;
|
|
@@ -8,6 +8,8 @@ const execute_docker_1 = require("./execute-docker");
|
|
|
8
8
|
const plan_work_tasks_1 = require("../planner/utils/plan-work-tasks");
|
|
9
9
|
const get_cache_directory_1 = require("../optimizer/get-cache-directory");
|
|
10
10
|
const path_1 = require("path");
|
|
11
|
+
const crypto_1 = require("crypto");
|
|
12
|
+
const claim_layout_1 = require("../kubernetes/claim-layout");
|
|
11
13
|
const move_files_1 = require("../file/move-files");
|
|
12
14
|
const work_instance_id_1 = require("../planner/work-instance-id");
|
|
13
15
|
async function restoreCache(environment, path, workTree) {
|
|
@@ -43,8 +45,24 @@ function* getArchivePaths(task, path) {
|
|
|
43
45
|
}
|
|
44
46
|
}
|
|
45
47
|
else {
|
|
46
|
-
|
|
47
|
-
|
|
48
|
+
// `a-b` and `a/b` flatten alike: of those, the least deep (then the first in sorted
|
|
49
|
+
// order) keeps the name, the others' names carry a hash of their relative path, so a
|
|
50
|
+
// store and a restore in another directory or order find the same archive
|
|
51
|
+
const generates = task.generates
|
|
52
|
+
.filter((g) => !g.inherited)
|
|
53
|
+
.map((generate) => ({ generate, rel: (0, path_1.relative)(task.cwd, generate.path).split(path_1.sep).join(path_1.posix.sep) }));
|
|
54
|
+
const holders = new Map();
|
|
55
|
+
for (const { rel } of [...generates].sort((a, b) => (0, claim_layout_1.byDepth)(a.rel, b.rel))) {
|
|
56
|
+
const id = rel.split(path_1.posix.sep).join('-');
|
|
57
|
+
if (!holders.has(id)) {
|
|
58
|
+
holders.set(id, rel);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
for (const { generate, rel } of generates) {
|
|
62
|
+
let generateId = rel.split(path_1.posix.sep).join('-');
|
|
63
|
+
if (holders.get(generateId) !== rel) {
|
|
64
|
+
generateId += '-' + (0, crypto_1.createHash)('sha256').update(rel).digest('hex').slice(0, 12);
|
|
65
|
+
}
|
|
48
66
|
yield {
|
|
49
67
|
filename: (0, path_1.join)(path, generateId + '-generates.tgz'),
|
|
50
68
|
path: generate.path,
|
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.executeWorkService = exports.stopService = void 0;
|
|
4
4
|
const work_item_1 = require("../planner/work-item");
|
|
5
5
|
const await_completed_dependencies_1 = require("./await-completed-dependencies");
|
|
6
|
+
const deferred_restore_1 = require("./deferred-restore");
|
|
6
7
|
const abort_1 = require("./abort");
|
|
7
8
|
const log_1 = require("../log");
|
|
8
9
|
const watch_loop_1 = require("./watch-loop");
|
|
@@ -35,6 +36,7 @@ async function executeWorkService(work, environment, options) {
|
|
|
35
36
|
});
|
|
36
37
|
await (0, watch_loop_1.watchLoop)(work, environment, options, async (cacheState, abort, stop) => {
|
|
37
38
|
await (0, await_completed_dependencies_1.awaitCompletedDependencies)(work, work.deps, abort);
|
|
39
|
+
await (0, deferred_restore_1.restoreDeferred)(work, (0, await_completed_dependencies_1.transitiveDeps)(work.deps), abort);
|
|
38
40
|
work.state.set({
|
|
39
41
|
type: 'ready',
|
|
40
42
|
stateKey: cacheState.stateKey,
|