hammerkit 1.7.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/build.schema.json +19 -13
- package/dist/docker/package.js +13 -3
- package/dist/docker/remove-orphaned-containers.js +53 -0
- package/dist/docker/run-labels.js +14 -0
- package/dist/docker/using-container.js +25 -26
- package/dist/executer/check-readiness.js +4 -4
- package/dist/executer/container-task-state.js +34 -0
- package/dist/executer/docker-service.js +23 -6
- package/dist/executer/docker-task.js +26 -4
- package/dist/executer/event-cache.js +8 -0
- package/dist/executer/execute-command.js +36 -6
- package/dist/executer/execute-docker.js +18 -2
- package/dist/executer/execute-work-service.js +4 -0
- package/dist/executer/execute-work-task.js +17 -2
- package/dist/executer/execution-steps.js +3 -8
- package/dist/executer/get-container-binds.js +13 -1
- package/dist/executer/main-process-user.js +52 -0
- package/dist/index.js +15 -3
- package/dist/kubernetes/apply.js +6 -7
- package/dist/kubernetes/await-running-state.js +48 -44
- package/dist/kubernetes/ensure-http-route.js +7 -10
- package/dist/kubernetes/ensure-kubernetes-deployment-exists.js +27 -6
- package/dist/kubernetes/ensure-namespace.js +9 -10
- package/dist/kubernetes/ensure-persistent-data.js +6 -5
- package/dist/kubernetes/get-service-ip.js +2 -2
- package/dist/kubernetes/kubernetes-instance.js +0 -4
- package/dist/kubernetes/remove-persistent-data.js +16 -6
- package/dist/kubernetes/resolve-pod-name.js +7 -7
- package/dist/planner/unreferenced-build-file-envs.js +35 -0
- package/dist/planner/utils/append-work-environment.js +1 -1
- package/dist/planner/utils/append-work-service.js +26 -2
- package/dist/planner/utils/get-outputs-to-reset.js +20 -0
- package/dist/planner/utils/parse-work-generate.js +2 -2
- package/dist/planner/validate.js +10 -0
- package/dist/planner/work-healthcheck.js +8 -0
- package/dist/planner/work-instance-id.js +3 -3
- package/dist/planner/work-runtime-docker.js +19 -19
- package/dist/planner/work-runtime-kubernetes.js +66 -48
- package/dist/planner/work-runtime-local.js +22 -5
- package/dist/planner/work-service-id.js +3 -0
- package/dist/planner/work-service.js +27 -1
- package/dist/program.js +6 -6
- package/dist/schema/build-file-container-service-schema.js +8 -0
- package/dist/schema/build-file-environment-schema-http-route.js +15 -0
- package/dist/schema/build-file-environment-schema.js +8 -2
- package/dist/schema/build-file-schema.js +1 -1
- package/dist/schema/build-file-service-container-healthcheck.js +4 -0
- package/dist/schema/envs-schema.js +1 -1
- package/dist/utils/abort-on-signals.js +17 -0
- package/package.json +9 -9
- package/dist/kubernetes/ensure-ingress.js +0 -54
- package/dist/schema/build-file-environment-schema-ingress.js +0 -14
|
@@ -7,9 +7,9 @@ function formatSelector(matchLabels) {
|
|
|
7
7
|
.join(',');
|
|
8
8
|
}
|
|
9
9
|
async function findReadyPod(coreApi, namespace, labelSelector) {
|
|
10
|
-
const pods = await coreApi.listNamespacedPod(namespace,
|
|
11
|
-
const ready = pods.
|
|
12
|
-
const pod = ready ?? pods.
|
|
10
|
+
const pods = await coreApi.listNamespacedPod({ namespace, labelSelector });
|
|
11
|
+
const ready = pods.items.find((p) => p.status?.conditions?.some((c) => c.type === 'Ready' && c.status === 'True'));
|
|
12
|
+
const pod = ready ?? pods.items[0];
|
|
13
13
|
if (!pod?.metadata?.name) {
|
|
14
14
|
throw new Error(`no pod found for selector ${labelSelector} in namespace ${namespace}`);
|
|
15
15
|
}
|
|
@@ -21,16 +21,16 @@ async function resolvePodName(coreApi, appsApi, namespace, selector) {
|
|
|
21
21
|
return selector.name;
|
|
22
22
|
}
|
|
23
23
|
if (type === 'service') {
|
|
24
|
-
const svc = await coreApi.readNamespacedService(selector.name, namespace);
|
|
25
|
-
const matchLabels = svc.
|
|
24
|
+
const svc = await coreApi.readNamespacedService({ name: selector.name, namespace });
|
|
25
|
+
const matchLabels = svc.spec?.selector;
|
|
26
26
|
if (!matchLabels || Object.keys(matchLabels).length === 0) {
|
|
27
27
|
throw new Error(`service ${selector.name} has no selector`);
|
|
28
28
|
}
|
|
29
29
|
return findReadyPod(coreApi, namespace, formatSelector(matchLabels));
|
|
30
30
|
}
|
|
31
31
|
if (type === 'deployment') {
|
|
32
|
-
const deploy = await appsApi.readNamespacedDeployment(selector.name, namespace);
|
|
33
|
-
const matchLabels = deploy.
|
|
32
|
+
const deploy = await appsApi.readNamespacedDeployment({ name: selector.name, namespace });
|
|
33
|
+
const matchLabels = deploy.spec?.selector?.matchLabels;
|
|
34
34
|
if (!matchLabels) {
|
|
35
35
|
throw new Error(`deployment ${selector.name} has no matchLabels`);
|
|
36
36
|
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getUnreferencedBuildFileEnvs = void 0;
|
|
4
|
+
// Build-file `envs` are passed to every task of the file and are part of every
|
|
5
|
+
// task's cache key, so a value only some tasks use (an image tag, a test token)
|
|
6
|
+
// reruns all of them when it changes. Returns the build-file envs the task's own
|
|
7
|
+
// definition never mentions as `$NAME` / `${NAME}`. The task may still read
|
|
8
|
+
// them implicitly at runtime (NODE_ENV), so this is only worth a warning.
|
|
9
|
+
function getUnreferencedBuildFileEnvs(task) {
|
|
10
|
+
const buildFileEnvs = task.scope.schema.envs ?? {};
|
|
11
|
+
const taskSchema = getTaskSchema(task);
|
|
12
|
+
// an aggregate (deps only) runs nothing, so a rerun costs nothing
|
|
13
|
+
if (!taskSchema || 'extend' in taskSchema || (taskSchema.cmds ?? []).length === 0) {
|
|
14
|
+
return [];
|
|
15
|
+
}
|
|
16
|
+
const taskEnvs = taskSchema.envs ?? {};
|
|
17
|
+
const definition = JSON.stringify(taskSchema);
|
|
18
|
+
return Object.keys(buildFileEnvs)
|
|
19
|
+
.filter((name) => !(name in taskEnvs))
|
|
20
|
+
.filter((name) => !isReferenced(definition, name));
|
|
21
|
+
}
|
|
22
|
+
exports.getUnreferencedBuildFileEnvs = getUnreferencedBuildFileEnvs;
|
|
23
|
+
function getTaskSchema(task) {
|
|
24
|
+
const prefix = task.scope.namePrefix.length > 0 ? `${task.scope.namePrefix}:` : '';
|
|
25
|
+
if (!task.name.startsWith(prefix)) {
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
const tasks = task.scope.schema.tasks ?? {};
|
|
29
|
+
return tasks[task.name.substring(prefix.length)] ?? null;
|
|
30
|
+
}
|
|
31
|
+
function isReferenced(definition, name) {
|
|
32
|
+
const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
33
|
+
return new RegExp(`\\$(\\{${escaped}\\}|${escaped}(?![A-Za-z0-9_]))`, 'i').test(definition);
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=unreferenced-build-file-envs.js.map
|
|
@@ -13,7 +13,7 @@ function appendWorkEnvironment(referenced) {
|
|
|
13
13
|
type: 'kubernetes',
|
|
14
14
|
context: referenced.schema.kubernetes.context,
|
|
15
15
|
kubeConfig: referenced.schema.kubernetes.kubeconfig,
|
|
16
|
-
|
|
16
|
+
httpRoutes: referenced.schema.kubernetes.httpRoutes || [],
|
|
17
17
|
namespace: referenced.schema.kubernetes.namespace ?? 'default',
|
|
18
18
|
};
|
|
19
19
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.appendWorkService = void 0;
|
|
3
|
+
exports.getHealthcheckTimeout = exports.DEFAULT_HEALTHCHECK_TIMEOUT = exports.appendWorkService = void 0;
|
|
4
4
|
const work_service_id_1 = require("../work-service-id");
|
|
5
5
|
const parse_work_ports_1 = require("./parse-work-ports");
|
|
6
6
|
const template_value_1 = require("./template-value");
|
|
@@ -17,6 +17,8 @@ const state_1 = require("../../executer/state");
|
|
|
17
17
|
const lazy_resolver_1 = require("../../executer/lazy-resolver");
|
|
18
18
|
const get_work_runtime_1 = require("./get-work-runtime");
|
|
19
19
|
const resolve_cache_1 = require("../../cache/resolve-cache");
|
|
20
|
+
const units_1 = require("../../utils/units");
|
|
21
|
+
const log_1 = require("../../log");
|
|
20
22
|
function appendWorkService(workTree, service, environment, context) {
|
|
21
23
|
const workService = parseService(workTree, service, environment, context);
|
|
22
24
|
if (!workTree.services[workService.name]) {
|
|
@@ -91,13 +93,16 @@ function parseService(workTree, service, environment, context) {
|
|
|
91
93
|
continuous: service.schema.continuous ?? false,
|
|
92
94
|
healthcheck: service.schema.healthcheck
|
|
93
95
|
? {
|
|
94
|
-
cmd: (0, parse_work_command_1.parseWorkCommand)(service.cwd, service.schema.healthcheck, envs),
|
|
96
|
+
cmd: (0, parse_work_command_1.parseWorkCommand)(service.cwd, service.schema.healthcheck.cmd, envs),
|
|
97
|
+
timeout: getHealthcheckTimeout(service.schema.healthcheck.timeout, environment),
|
|
95
98
|
}
|
|
96
99
|
: null,
|
|
97
100
|
envs: (0, replace_env_variables_1.buildEnvironmentVariables)(service.envs, environment, context),
|
|
98
101
|
image: (0, template_value_1.templateValue)(service.schema.image, envs),
|
|
99
102
|
cwd: service.cwd,
|
|
100
103
|
cmd: service.schema.cmd ? (0, parse_work_command_1.parseWorkCommand)(service.cwd, service.schema.cmd, envs) : null,
|
|
104
|
+
workdir: service.schema.workdir ? (0, template_value_1.templateValue)(service.schema.workdir, envs) : null,
|
|
105
|
+
shell: service.schema.shell ? (0, template_value_1.templateValue)(service.schema.shell, envs) : null,
|
|
101
106
|
volumes: (0, parse_work_volume_1.parseWorkVolumes)(service.cwd, service.schema.volumes, envs),
|
|
102
107
|
mounts: (0, parse_work_mounts_1.parseWorkMounts)(service.cwd, service.schema, envs),
|
|
103
108
|
src: (0, parse_work_source_1.parseWorkSource)(service.cwd, service.schema.src, envs),
|
|
@@ -105,4 +110,23 @@ function parseService(workTree, service, environment, context) {
|
|
|
105
110
|
};
|
|
106
111
|
}
|
|
107
112
|
}
|
|
113
|
+
exports.DEFAULT_HEALTHCHECK_TIMEOUT = '20s';
|
|
114
|
+
// A service's own `healthcheck.timeout`, else HAMMERKIT_HEALTHCHECK_TIMEOUT,
|
|
115
|
+
// else 20s: a check that never passes fails the run instead of hanging it.
|
|
116
|
+
function getHealthcheckTimeout(timeout, environment) {
|
|
117
|
+
if (timeout) {
|
|
118
|
+
return (0, units_1.parseDuration)(timeout);
|
|
119
|
+
}
|
|
120
|
+
const fromEnv = environment.processEnvs['HAMMERKIT_HEALTHCHECK_TIMEOUT'];
|
|
121
|
+
if (fromEnv) {
|
|
122
|
+
try {
|
|
123
|
+
return (0, units_1.parseDuration)(fromEnv);
|
|
124
|
+
}
|
|
125
|
+
catch (e) {
|
|
126
|
+
throw new Error(`HAMMERKIT_HEALTHCHECK_TIMEOUT: ${(0, log_1.getErrorMessage)(e)}`);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return (0, units_1.parseDuration)(exports.DEFAULT_HEALTHCHECK_TIMEOUT);
|
|
130
|
+
}
|
|
131
|
+
exports.getHealthcheckTimeout = getHealthcheckTimeout;
|
|
108
132
|
//# sourceMappingURL=append-work-service.js.map
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getOutputsToReset = void 0;
|
|
4
|
+
const path_1 = require("path");
|
|
5
|
+
function contains(parent, path) {
|
|
6
|
+
const rel = (0, path_1.relative)(parent, path);
|
|
7
|
+
return rel === '' || (!rel.startsWith('..') && !(0, path_1.isAbsolute)(rel));
|
|
8
|
+
}
|
|
9
|
+
// The outputs a run empties before the task's commands: its own outputs with
|
|
10
|
+
// resetOnChange (the default), except one that is a source or contains one
|
|
11
|
+
// (`package-lock.json` declared both ways) or contains the task's directory.
|
|
12
|
+
// Emptying those would delete what the task reads.
|
|
13
|
+
function getOutputsToReset(task) {
|
|
14
|
+
return task.generates.filter((generate) => generate.resetOnChange &&
|
|
15
|
+
!generate.inherited &&
|
|
16
|
+
!contains(generate.path, task.cwd) &&
|
|
17
|
+
!task.src.some((src) => contains(generate.path, src.absolutePath)));
|
|
18
|
+
}
|
|
19
|
+
exports.getOutputsToReset = getOutputsToReset;
|
|
20
|
+
//# sourceMappingURL=get-outputs-to-reset.js.map
|
|
@@ -24,12 +24,12 @@ function parseWorkGenerate(cwd, schema, envs) {
|
|
|
24
24
|
exports.parseWorkGenerate = parseWorkGenerate;
|
|
25
25
|
function mapGenerate(generate) {
|
|
26
26
|
if (typeof generate === 'string') {
|
|
27
|
-
return { path: generate, resetOnChange:
|
|
27
|
+
return { path: generate, resetOnChange: true, export: false };
|
|
28
28
|
}
|
|
29
29
|
else {
|
|
30
30
|
return {
|
|
31
31
|
path: generate.path,
|
|
32
|
-
resetOnChange: generate.resetOnChange ??
|
|
32
|
+
resetOnChange: generate.resetOnChange ?? true,
|
|
33
33
|
export: generate.export ?? false,
|
|
34
34
|
};
|
|
35
35
|
}
|
package/dist/planner/validate.js
CHANGED
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.checkIfContextExists = exports.hasDependencyCycle = exports.hasNeedCycle = exports.hasMixedCycle = exports.validate = void 0;
|
|
4
4
|
const read_build_file_1 = require("../parser/read-build-file");
|
|
5
5
|
const plan_work_tasks_1 = require("./utils/plan-work-tasks");
|
|
6
|
+
const unreferenced_build_file_envs_1 = require("./unreferenced-build-file-envs");
|
|
6
7
|
async function* validate(workTree, context) {
|
|
7
8
|
const cycleItems = [];
|
|
8
9
|
for (const item of (0, plan_work_tasks_1.iterateWorkServices)(workTree)) {
|
|
@@ -49,6 +50,15 @@ async function* validate(workTree, context) {
|
|
|
49
50
|
if ((!task.cmds || task.cmds.length === 0) && (!item.deps || item.deps.length === 0)) {
|
|
50
51
|
yield { type: 'warn', message: `task is empty`, item: task };
|
|
51
52
|
}
|
|
53
|
+
const unreferencedEnvs = (0, unreferenced_build_file_envs_1.getUnreferencedBuildFileEnvs)(task);
|
|
54
|
+
if (unreferencedEnvs.length > 0) {
|
|
55
|
+
yield {
|
|
56
|
+
type: 'warn',
|
|
57
|
+
message: `build-file envs ${unreferencedEnvs.join(', ')} are part of this task's cache key but not referenced ` +
|
|
58
|
+
`by it: changing them reruns it. If it doesn't read them at runtime, define them on the tasks that do`,
|
|
59
|
+
item: task,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
52
62
|
for (const src of task.src) {
|
|
53
63
|
if (!(await context.file.exists(src.absolutePath))) {
|
|
54
64
|
yield {
|
|
@@ -1,3 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getHealthcheckTimeoutMessage = void 0;
|
|
4
|
+
const units_1 = require("../utils/units");
|
|
5
|
+
function getHealthcheckTimeoutMessage(serviceName, healthcheck) {
|
|
6
|
+
return (`service ${serviceName} did not pass its healthcheck "${healthcheck.cmd.cmd}" within ` +
|
|
7
|
+
`${(0, units_1.formatDuration)(healthcheck.timeout)} of starting (see its log above; raise healthcheck.timeout ` +
|
|
8
|
+
`or HAMMERKIT_HEALTHCHECK_TIMEOUT if it is just slow)`);
|
|
9
|
+
}
|
|
10
|
+
exports.getHealthcheckTimeoutMessage = getHealthcheckTimeoutMessage;
|
|
3
11
|
//# sourceMappingURL=work-healthcheck.js.map
|
|
@@ -4,9 +4,9 @@ exports.getWorkInstanceId = void 0;
|
|
|
4
4
|
const crypto_1 = require("crypto");
|
|
5
5
|
// `id()` is the portable cache identity: identical for the same task in every
|
|
6
6
|
// checkout, so a shared cache hits across machines. Machine-local runtime state
|
|
7
|
-
// (docker container labels, the cache staging directory)
|
|
8
|
-
// scoped to one checkout — two worktrees of the same repo on
|
|
9
|
-
// otherwise see each other's
|
|
7
|
+
// (docker container labels, task state files, the cache staging directory)
|
|
8
|
+
// must instead stay scoped to one checkout — two worktrees of the same repo on
|
|
9
|
+
// one host would otherwise see each other's state records and staging files. The instance
|
|
10
10
|
// id folds the absolute project root into the portable id for that purpose.
|
|
11
11
|
function getWorkInstanceId(item) {
|
|
12
12
|
return (0, crypto_1.createHash)('sha1').update(`${item.id()}\0${item.data.projectRoot}`).digest('hex');
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.dockerServiceRuntime = exports.dockerTaskRuntime = void 0;
|
|
4
|
+
const run_labels_1 = require("../docker/run-labels");
|
|
4
5
|
const execute_docker_1 = require("../executer/execute-docker");
|
|
5
6
|
const remove_container_1 = require("../docker/remove-container");
|
|
6
7
|
const docker_task_1 = require("../executer/docker-task");
|
|
@@ -12,6 +13,8 @@ const path_1 = require("path");
|
|
|
12
13
|
const tar_1 = require("tar");
|
|
13
14
|
const plan_work_volume_1 = require("./utils/plan-work-volume");
|
|
14
15
|
const work_instance_id_1 = require("./work-instance-id");
|
|
16
|
+
const stream_1 = require("stream");
|
|
17
|
+
const container_task_state_1 = require("../executer/container-task-state");
|
|
15
18
|
function dockerTaskRuntime(task, workEnvironment) {
|
|
16
19
|
const docker = (0, execute_docker_1.getContainerCli)(workEnvironment);
|
|
17
20
|
return {
|
|
@@ -21,7 +24,8 @@ function dockerTaskRuntime(task, workEnvironment) {
|
|
|
21
24
|
label: [`hammerkit-id=${(0, work_instance_id_1.getWorkInstanceId)(task)}`],
|
|
22
25
|
},
|
|
23
26
|
});
|
|
24
|
-
|
|
27
|
+
// a paused container is a state record of hammerkit before 1.9, not a run
|
|
28
|
+
const currentTask = currentTasks.find((c) => c.State === 'running');
|
|
25
29
|
if (!currentTask) {
|
|
26
30
|
return;
|
|
27
31
|
}
|
|
@@ -52,8 +56,9 @@ function dockerTaskRuntime(task, workEnvironment) {
|
|
|
52
56
|
async execute(environment, options) {
|
|
53
57
|
await (0, docker_task_1.dockerTask)(docker, task, environment, options);
|
|
54
58
|
},
|
|
55
|
-
async remove() {
|
|
59
|
+
async remove(environment) {
|
|
56
60
|
await this.stop();
|
|
61
|
+
await (0, container_task_state_1.removeContainerTaskState)(environment, task);
|
|
57
62
|
for (const generate of task.data.generates) {
|
|
58
63
|
if (generate.inherited) {
|
|
59
64
|
continue;
|
|
@@ -69,17 +74,8 @@ function dockerTaskRuntime(task, workEnvironment) {
|
|
|
69
74
|
}
|
|
70
75
|
},
|
|
71
76
|
async currentStateKey(environment) {
|
|
72
|
-
const
|
|
73
|
-
|
|
74
|
-
filters: {
|
|
75
|
-
label: [`hammerkit-id=${(0, work_instance_id_1.getWorkInstanceId)(task)}`],
|
|
76
|
-
},
|
|
77
|
-
});
|
|
78
|
-
const container = containers[0];
|
|
79
|
-
if (!container) {
|
|
80
|
-
return null;
|
|
81
|
-
}
|
|
82
|
-
if (!container.Labels['hammerkit-state']) {
|
|
77
|
+
const stateKey = await (0, container_task_state_1.readContainerTaskState)(environment, task);
|
|
78
|
+
if (!stateKey) {
|
|
83
79
|
return null;
|
|
84
80
|
}
|
|
85
81
|
// outputs removed since the run (a deleted export, a pruned volume)
|
|
@@ -92,7 +88,7 @@ function dockerTaskRuntime(task, workEnvironment) {
|
|
|
92
88
|
return null;
|
|
93
89
|
}
|
|
94
90
|
}
|
|
95
|
-
return
|
|
91
|
+
return stateKey;
|
|
96
92
|
},
|
|
97
93
|
};
|
|
98
94
|
}
|
|
@@ -113,7 +109,7 @@ async function restoreContainer(docker, environment, item, path) {
|
|
|
113
109
|
Labels: {
|
|
114
110
|
app: 'hammerkit',
|
|
115
111
|
'hammerkit-id': (0, work_instance_id_1.getWorkInstanceId)(item),
|
|
116
|
-
|
|
112
|
+
...(0, run_labels_1.getRunLabels)(),
|
|
117
113
|
'hammerkit-type': 'task',
|
|
118
114
|
},
|
|
119
115
|
HostConfig: {
|
|
@@ -129,9 +125,12 @@ async function restoreContainer(docker, environment, item, path) {
|
|
|
129
125
|
.filter((v) => !v.inherited && !v.isFile)
|
|
130
126
|
.map((v) => `${v.volumeName}:${(0, execute_docker_1.convertToPosixPath)(v.path)}`),
|
|
131
127
|
},
|
|
132
|
-
},
|
|
128
|
+
}, async (container) => {
|
|
133
129
|
for (const generate of (0, event_cache_1.getArchivePaths)(item.data, path)) {
|
|
134
130
|
if (await environment.file.exists(generate.filename)) {
|
|
131
|
+
// a cache hit means exactly the stored outputs: whatever a failed or
|
|
132
|
+
// older run left in the volume goes first
|
|
133
|
+
await (0, execute_docker_1.clearContainerDirectory)(item.status, environment, container, generate.path);
|
|
135
134
|
await container.putArchive(environment.file.readStream(generate.filename), {
|
|
136
135
|
path: (0, path_1.dirname)(generate.path),
|
|
137
136
|
});
|
|
@@ -142,6 +141,7 @@ async function restoreContainer(docker, environment, item, path) {
|
|
|
142
141
|
const onHost = new Set(item.data.generates.filter((g) => !g.inherited && isOnHost(g)).map((g) => g.path));
|
|
143
142
|
for (const generate of (0, event_cache_1.getArchivePaths)(item.data, path)) {
|
|
144
143
|
if (onHost.has(generate.path) && (await environment.file.exists(generate.filename))) {
|
|
144
|
+
await environment.file.remove(generate.path);
|
|
145
145
|
await (0, tar_1.extract)({ file: generate.filename, cwd: (0, path_1.dirname)(generate.path) });
|
|
146
146
|
}
|
|
147
147
|
}
|
|
@@ -231,7 +231,7 @@ async function archiveContainer(docker, environment, item, path) {
|
|
|
231
231
|
Labels: {
|
|
232
232
|
app: 'hammerkit',
|
|
233
233
|
'hammerkit-id': (0, work_instance_id_1.getWorkInstanceId)(item),
|
|
234
|
-
|
|
234
|
+
...(0, run_labels_1.getRunLabels)(),
|
|
235
235
|
'hammerkit-type': 'task',
|
|
236
236
|
},
|
|
237
237
|
HostConfig: {
|
|
@@ -244,14 +244,14 @@ async function archiveContainer(docker, environment, item, path) {
|
|
|
244
244
|
.filter((v) => !v.inherited && !v.isFile)
|
|
245
245
|
.map((v) => `${v.volumeName}:${(0, execute_docker_1.convertToPosixPath)(v.path)}`),
|
|
246
246
|
},
|
|
247
|
-
},
|
|
247
|
+
}, async (container) => {
|
|
248
248
|
const hostFiles = new Set(item.data.type === 'container-task'
|
|
249
249
|
? item.data.generates.filter((g) => !g.inherited && g.isFile).map((g) => g.path)
|
|
250
250
|
: []);
|
|
251
251
|
for (const generatedArchive of (0, event_cache_1.getArchivePaths)(item.data, path)) {
|
|
252
252
|
// a file output is a bind mount of a host file, so archive it from the host
|
|
253
253
|
const readable = hostFiles.has(generatedArchive.path)
|
|
254
|
-
? (0, tar_1.create)({ cwd: (0, path_1.dirname)(generatedArchive.path) }, [(0, path_1.basename)(generatedArchive.path)])
|
|
254
|
+
? stream_1.Readable.from((0, tar_1.create)({ cwd: (0, path_1.dirname)(generatedArchive.path) }, [(0, path_1.basename)(generatedArchive.path)]))
|
|
255
255
|
: await container.getArchive({
|
|
256
256
|
path: generatedArchive.path,
|
|
257
257
|
});
|
|
@@ -16,7 +16,6 @@ const ensure_persistent_data_1 = require("../kubernetes/ensure-persistent-data")
|
|
|
16
16
|
const await_running_state_1 = require("../kubernetes/await-running-state");
|
|
17
17
|
const volumes_1 = require("../kubernetes/volumes");
|
|
18
18
|
const resources_1 = require("../kubernetes/resources");
|
|
19
|
-
const ensure_ingress_1 = require("../kubernetes/ensure-ingress");
|
|
20
19
|
const ensure_http_route_1 = require("../kubernetes/ensure-http-route");
|
|
21
20
|
const find_process_1 = __importDefault(require("find-process"));
|
|
22
21
|
const log_1 = require("../log");
|
|
@@ -24,13 +23,12 @@ const version_1 = require("../version");
|
|
|
24
23
|
const restore_kubernetes_data_1 = require("../kubernetes/restore-kubernetes-data");
|
|
25
24
|
const store_kubernetes_data_1 = require("../kubernetes/store-kubernetes-data");
|
|
26
25
|
const remove_persistent_data_1 = require("../kubernetes/remove-persistent-data");
|
|
27
|
-
function statusCodeOf(e) {
|
|
28
|
-
const err = e;
|
|
29
|
-
return err?.statusCode ?? err?.response?.statusCode ?? err?.body?.code;
|
|
30
|
-
}
|
|
31
26
|
async function listJobNames(instance, kubernetes, task) {
|
|
32
|
-
const jobs = await instance.batchApi.listNamespacedJob(
|
|
33
|
-
|
|
27
|
+
const jobs = await instance.batchApi.listNamespacedJob({
|
|
28
|
+
namespace: kubernetes.namespace,
|
|
29
|
+
labelSelector: `hammerkit.dev/id=${task.id()}`,
|
|
30
|
+
});
|
|
31
|
+
return jobs.items.flatMap((job) => (job.metadata?.name ? [job.metadata.name] : []));
|
|
34
32
|
}
|
|
35
33
|
function kubernetesTaskRuntime(task, kubernetes) {
|
|
36
34
|
const instance = (0, kubernetes_instance_1.createKubernetesInstances)(kubernetes);
|
|
@@ -128,22 +126,34 @@ function kubernetesTaskRuntime(task, kubernetes) {
|
|
|
128
126
|
}
|
|
129
127
|
},
|
|
130
128
|
async stop() {
|
|
131
|
-
const jobs = await instance.batchApi.listNamespacedJob(
|
|
132
|
-
|
|
129
|
+
const jobs = await instance.batchApi.listNamespacedJob({
|
|
130
|
+
namespace: kubernetes.namespace,
|
|
131
|
+
labelSelector: `hammerkit.dev/id=${task.id()}`,
|
|
132
|
+
});
|
|
133
|
+
for (const pod of jobs.items) {
|
|
133
134
|
if (pod.metadata?.name) {
|
|
134
|
-
await instance.batchApi.deleteNamespacedJob(pod.metadata.name, kubernetes.namespace);
|
|
135
|
+
await instance.batchApi.deleteNamespacedJob({ name: pod.metadata.name, namespace: kubernetes.namespace });
|
|
135
136
|
}
|
|
136
137
|
}
|
|
137
|
-
const deployments = await instance.appsApi.listNamespacedDeployment(
|
|
138
|
-
|
|
138
|
+
const deployments = await instance.appsApi.listNamespacedDeployment({
|
|
139
|
+
namespace: kubernetes.namespace,
|
|
140
|
+
labelSelector: `hammerkit.dev/id=${task.id()}`,
|
|
141
|
+
});
|
|
142
|
+
for (const deploy of deployments.items) {
|
|
139
143
|
if (deploy.metadata?.name) {
|
|
140
|
-
await instance.appsApi.deleteNamespacedDeployment(
|
|
144
|
+
await instance.appsApi.deleteNamespacedDeployment({
|
|
145
|
+
name: deploy.metadata.name,
|
|
146
|
+
namespace: kubernetes.namespace,
|
|
147
|
+
});
|
|
141
148
|
}
|
|
142
149
|
}
|
|
143
|
-
const pods = await instance.coreApi.listNamespacedPod(
|
|
144
|
-
|
|
150
|
+
const pods = await instance.coreApi.listNamespacedPod({
|
|
151
|
+
namespace: kubernetes.namespace,
|
|
152
|
+
labelSelector: `hammerkit.dev/id=${task.id()}`,
|
|
153
|
+
});
|
|
154
|
+
for (const pod of pods.items) {
|
|
145
155
|
if (pod.metadata?.name) {
|
|
146
|
-
await instance.coreApi.deleteNamespacedPod(pod.metadata.name, kubernetes.namespace);
|
|
156
|
+
await instance.coreApi.deleteNamespacedPod({ name: pod.metadata.name, namespace: kubernetes.namespace });
|
|
147
157
|
}
|
|
148
158
|
}
|
|
149
159
|
},
|
|
@@ -152,8 +162,11 @@ function kubernetesTaskRuntime(task, kubernetes) {
|
|
|
152
162
|
await (0, remove_persistent_data_1.removePersistentData)(instance, kubernetes, task);
|
|
153
163
|
},
|
|
154
164
|
async currentStateKey() {
|
|
155
|
-
const jobs = await instance.batchApi.listNamespacedJob(
|
|
156
|
-
|
|
165
|
+
const jobs = await instance.batchApi.listNamespacedJob({
|
|
166
|
+
namespace: kubernetes.namespace,
|
|
167
|
+
labelSelector: `hammerkit.dev/id=${task.id()}`,
|
|
168
|
+
});
|
|
169
|
+
const completedStates = jobs.items
|
|
157
170
|
.filter((j) => j.status?.succeeded && j.metadata?.labels?.['hammerkit.dev/state'])
|
|
158
171
|
.map((j) => j.metadata.labels['hammerkit.dev/state']);
|
|
159
172
|
const stateKey = completedStates[completedStates.length - 1] ?? null;
|
|
@@ -163,11 +176,14 @@ function kubernetesTaskRuntime(task, kubernetes) {
|
|
|
163
176
|
// outputs live in the task's claim; without it the finished job left
|
|
164
177
|
// nothing to reuse
|
|
165
178
|
try {
|
|
166
|
-
const claim = await instance.coreApi.readNamespacedPersistentVolumeClaim(
|
|
167
|
-
|
|
179
|
+
const claim = await instance.coreApi.readNamespacedPersistentVolumeClaim({
|
|
180
|
+
name: (0, volumes_1.getVolumeName)(task),
|
|
181
|
+
namespace: kubernetes.namespace,
|
|
182
|
+
});
|
|
183
|
+
return claim.metadata?.deletionTimestamp ? null : stateKey;
|
|
168
184
|
}
|
|
169
185
|
catch (e) {
|
|
170
|
-
if (statusCodeOf(e) === 404) {
|
|
186
|
+
if ((0, apply_1.statusCodeOf)(e) === 404) {
|
|
171
187
|
return null;
|
|
172
188
|
}
|
|
173
189
|
throw e;
|
|
@@ -189,10 +205,16 @@ function kubernetesServiceRuntime(service, kubernetes) {
|
|
|
189
205
|
await (0, store_kubernetes_data_1.storeKubernetesData)(service, kubernetes, instance, environment, path);
|
|
190
206
|
},
|
|
191
207
|
async stop() {
|
|
192
|
-
const deployments = await instance.appsApi.listNamespacedDeployment(
|
|
193
|
-
|
|
208
|
+
const deployments = await instance.appsApi.listNamespacedDeployment({
|
|
209
|
+
namespace: kubernetes.namespace,
|
|
210
|
+
labelSelector: `hammerkit.dev/id=${service.id()}`,
|
|
211
|
+
});
|
|
212
|
+
for (const deployment of deployments.items) {
|
|
194
213
|
if (deployment.metadata?.name) {
|
|
195
|
-
await instance.appsApi.deleteNamespacedDeployment(
|
|
214
|
+
await instance.appsApi.deleteNamespacedDeployment({
|
|
215
|
+
name: deployment.metadata.name,
|
|
216
|
+
namespace: kubernetes.namespace,
|
|
217
|
+
});
|
|
196
218
|
}
|
|
197
219
|
}
|
|
198
220
|
},
|
|
@@ -209,29 +231,16 @@ function kubernetesServiceRuntime(service, kubernetes) {
|
|
|
209
231
|
stateKey: options.stateKey,
|
|
210
232
|
remote: null,
|
|
211
233
|
});
|
|
212
|
-
const
|
|
213
|
-
|
|
214
|
-
if (ingress.kind === 'httproute') {
|
|
215
|
-
await (0, ensure_http_route_1.ensureHttpRoute)(instance, kubernetes, ingress, service);
|
|
216
|
-
}
|
|
217
|
-
else {
|
|
218
|
-
await (0, ensure_ingress_1.ensureIngress)(instance, kubernetes, ingress, service);
|
|
219
|
-
}
|
|
234
|
+
for (const route of kubernetes.httpRoutes.filter((r) => r.service === service.name)) {
|
|
235
|
+
await (0, ensure_http_route_1.ensureHttpRoute)(instance, kubernetes, route, service);
|
|
220
236
|
}
|
|
221
237
|
},
|
|
222
238
|
async remove() {
|
|
223
|
-
const ingresses = await instance.networkingApi.listNamespacedIngress(kubernetes.namespace, undefined, undefined, undefined, undefined, `hammerkit.dev/id=${service.id()}`);
|
|
224
|
-
for (const ingress of ingresses.body.items) {
|
|
225
|
-
if (ingress.metadata?.name) {
|
|
226
|
-
await instance.networkingApi.deleteNamespacedIngress(ingress.metadata.name, kubernetes.namespace);
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
239
|
// HTTPRoutes are a CRD, so they are deleted by name through the generic
|
|
230
240
|
// objectApi rather than listed (a list would error on clusters without the
|
|
231
|
-
// Gateway API CRDs even when none are declared). Only
|
|
232
|
-
//
|
|
233
|
-
const
|
|
234
|
-
for (const route of httpRoutes) {
|
|
241
|
+
// Gateway API CRDs even when none are declared). Only the routes this
|
|
242
|
+
// service declares are removed; a missing route (404) is fine.
|
|
243
|
+
for (const route of kubernetes.httpRoutes.filter((r) => r.service === service.name)) {
|
|
235
244
|
try {
|
|
236
245
|
await instance.objectApi.delete({
|
|
237
246
|
apiVersion: ensure_http_route_1.HTTP_ROUTE_API_VERSION,
|
|
@@ -240,23 +249,32 @@ function kubernetesServiceRuntime(service, kubernetes) {
|
|
|
240
249
|
});
|
|
241
250
|
}
|
|
242
251
|
catch (e) {
|
|
243
|
-
if (statusCodeOf(e) !== 404) {
|
|
252
|
+
if ((0, apply_1.statusCodeOf)(e) !== 404) {
|
|
244
253
|
throw e;
|
|
245
254
|
}
|
|
246
255
|
}
|
|
247
256
|
}
|
|
248
|
-
const services = await instance.coreApi.listNamespacedService(
|
|
249
|
-
|
|
257
|
+
const services = await instance.coreApi.listNamespacedService({
|
|
258
|
+
namespace: kubernetes.namespace,
|
|
259
|
+
labelSelector: `hammerkit.dev/id=${service.id()}`,
|
|
260
|
+
});
|
|
261
|
+
for (const service of services.items) {
|
|
250
262
|
if (service.metadata?.name) {
|
|
251
|
-
await instance.coreApi.deleteNamespacedService(
|
|
263
|
+
await instance.coreApi.deleteNamespacedService({
|
|
264
|
+
name: service.metadata.name,
|
|
265
|
+
namespace: kubernetes.namespace,
|
|
266
|
+
});
|
|
252
267
|
}
|
|
253
268
|
}
|
|
254
269
|
await this.stop();
|
|
255
270
|
await (0, remove_persistent_data_1.removePersistentData)(instance, kubernetes, service);
|
|
256
271
|
},
|
|
257
272
|
async currentStateKey() {
|
|
258
|
-
const deployments = await instance.appsApi.listNamespacedDeployment(
|
|
259
|
-
|
|
273
|
+
const deployments = await instance.appsApi.listNamespacedDeployment({
|
|
274
|
+
namespace: kubernetes.namespace,
|
|
275
|
+
labelSelector: `hammerkit.dev/id=${service.id()}`,
|
|
276
|
+
});
|
|
277
|
+
const deployment = deployments.items[0];
|
|
260
278
|
return deployment?.metadata?.labels?.['hammerkit.dev/state'] ?? null;
|
|
261
279
|
},
|
|
262
280
|
};
|
|
@@ -8,7 +8,9 @@ const tar_1 = require("tar");
|
|
|
8
8
|
const path_1 = require("path");
|
|
9
9
|
const local_task_1 = require("../executer/local-task");
|
|
10
10
|
const event_cache_1 = require("../executer/event-cache");
|
|
11
|
+
const get_outputs_to_reset_1 = require("./utils/get-outputs-to-reset");
|
|
11
12
|
const find_process_1 = __importDefault(require("find-process"));
|
|
13
|
+
const stream_1 = require("stream");
|
|
12
14
|
function getStateFilename(task) {
|
|
13
15
|
return (0, path_1.join)(task.data.cwd, '.hammerkit', `${task.id()}`);
|
|
14
16
|
}
|
|
@@ -56,8 +58,21 @@ function getLocalWorkRuntime(task) {
|
|
|
56
58
|
const pidFile = getPidFilename(task);
|
|
57
59
|
await environment.file.writeFile(pidFile, `${process.pid}`);
|
|
58
60
|
try {
|
|
61
|
+
// The record says "the outputs belong to this state key". A run that
|
|
62
|
+
// fails overwrites outputs, so the previous record goes first, and only
|
|
63
|
+
// a run that succeeded writes a new one.
|
|
64
|
+
const stateFile = getStateFilename(task);
|
|
65
|
+
if (await environment.file.exists(stateFile)) {
|
|
66
|
+
await environment.file.remove(stateFile);
|
|
67
|
+
}
|
|
68
|
+
for (const generate of (0, get_outputs_to_reset_1.getOutputsToReset)(task.data)) {
|
|
69
|
+
await environment.file.remove(generate.path);
|
|
70
|
+
}
|
|
59
71
|
await (0, local_task_1.localTask)(task, environment, options);
|
|
60
|
-
|
|
72
|
+
const failed = ['crash', 'error', 'canceled'].includes(options.state.current.type);
|
|
73
|
+
if (!failed) {
|
|
74
|
+
await environment.file.writeFile(stateFile, options.stateKey);
|
|
75
|
+
}
|
|
61
76
|
}
|
|
62
77
|
finally {
|
|
63
78
|
if (await environment.file.exists(pidFile)) {
|
|
@@ -96,15 +111,17 @@ function getLocalWorkRuntime(task) {
|
|
|
96
111
|
exports.getLocalWorkRuntime = getLocalWorkRuntime;
|
|
97
112
|
async function archiveLocal(environment, task, path) {
|
|
98
113
|
for (const generatedArchive of (0, event_cache_1.getArchivePaths)(task.data, path)) {
|
|
99
|
-
await environment.file.writeStream(generatedArchive.filename,
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
}, [(0, path_1.relative)(task.data.cwd, generatedArchive.path)]));
|
|
114
|
+
await environment.file.writeStream(generatedArchive.filename,
|
|
115
|
+
// tar's stream is a Minipass stream; Readable.from adapts it to a Node one
|
|
116
|
+
stream_1.Readable.from((0, tar_1.create)({ cwd: task.data.cwd, gzip: true }, [(0, path_1.relative)(task.data.cwd, generatedArchive.path)])));
|
|
103
117
|
}
|
|
104
118
|
}
|
|
105
119
|
async function restoreLocal(environment, task, path) {
|
|
106
120
|
for (const generate of (0, event_cache_1.getArchivePaths)(task, path)) {
|
|
107
121
|
if (await environment.file.exists(generate.filename)) {
|
|
122
|
+
// a cache hit means exactly the stored outputs: whatever a failed or
|
|
123
|
+
// older run left in the folder goes first
|
|
124
|
+
await environment.file.remove(generate.path);
|
|
108
125
|
await (0, tar_1.extract)({
|
|
109
126
|
file: generate.filename,
|
|
110
127
|
cwd: task.cwd,
|
|
@@ -23,6 +23,9 @@ function getWorkServiceId(service) {
|
|
|
23
23
|
.sort(),
|
|
24
24
|
src: service.src.map((s) => portable(s.absolutePath)).sort(),
|
|
25
25
|
mounts: service.mounts.map((m) => m.mount).sort(),
|
|
26
|
+
// only when declared, so existing services keep their ids
|
|
27
|
+
...(service.workdir ? { workdir: service.workdir } : {}),
|
|
28
|
+
...(service.shell ? { shell: service.shell } : {}),
|
|
26
29
|
}
|
|
27
30
|
: {
|
|
28
31
|
context: service.context,
|