hammerkit 1.27.0-next.27 → 1.27.0-next.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build.schema.json +6 -0
- package/dist/kubernetes/await-running-state.js +21 -9
- package/dist/kubernetes/cluster-state.js +29 -8
- package/dist/kubernetes/ensure-kubernetes-deployment-exists.js +14 -0
- package/dist/main.js +2 -0
- package/dist/planner/utils/append-work-service.js +1 -0
- package/dist/remote/remote-work-commands.js +128 -44
- package/dist/schema/build-file-container-service-schema.js +8 -0
- package/dist/utils/terminal-prompt.js +53 -0
- package/package.json +1 -1
package/build.schema.json
CHANGED
|
@@ -651,6 +651,12 @@
|
|
|
651
651
|
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/resources",
|
|
652
652
|
"description": "cpu and memory requests and limits"
|
|
653
653
|
},
|
|
654
|
+
"restarts": {
|
|
655
|
+
"type": "integer",
|
|
656
|
+
"minimum": 0,
|
|
657
|
+
"maximum": 10,
|
|
658
|
+
"description": "crashes restarted before it fails on Kubernetes, 0 to 10, 2 by default"
|
|
659
|
+
},
|
|
654
660
|
"volumeSize": {
|
|
655
661
|
"$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/volumeSize",
|
|
656
662
|
"description": "size of its claim on Kubernetes, e.g. 10Gi"
|
|
@@ -81,15 +81,18 @@ exports.deleteJob = deleteJob;
|
|
|
81
81
|
// container the runtime could not create) may last before it fails the service
|
|
82
82
|
exports.PASSING_FAILURE_MS = 60000;
|
|
83
83
|
// Waits for an available replica: a ready one, for a service without a healthcheck one that
|
|
84
|
-
// stayed ready for a while. The deadline only starts once
|
|
85
|
-
// image pull doesn't count against it (as on
|
|
86
|
-
// the container started). A pod that won't get there without a change (an image
|
|
87
|
-
// that isn't there, a container that crashed
|
|
84
|
+
// stayed ready for a while. The deadline only starts once the container runs, and again
|
|
85
|
+
// at each start of it, so an image pull or a back-off doesn't count against it (as on
|
|
86
|
+
// docker, where it starts after the container started). A pod that won't get there without a change (an image
|
|
87
|
+
// that isn't there, a container that crashed once more than its
|
|
88
|
+
// service's `restarts`, 2 by default) fails it at once, with
|
|
88
89
|
// the reason the cluster gives; one that may still pass fails once it lasted a minute.
|
|
89
90
|
// Only the pods of this Deployment count: not the ones of one deleted before it was
|
|
90
91
|
// created again, which may still be there, terminating or not yet.
|
|
91
92
|
async function awaitDeployRunningState(instance, env, name, readiness, pollInterval = 1000, now = Date.now) {
|
|
92
93
|
let runningSince = null;
|
|
94
|
+
// how often the container was restarted when last looked at
|
|
95
|
+
let restarts = 0;
|
|
93
96
|
let said;
|
|
94
97
|
// since when a pod has had a failure that may still pass, by pod
|
|
95
98
|
const failingSince = new Map();
|
|
@@ -108,7 +111,7 @@ async function awaitDeployRunningState(instance, env, name, readiness, pollInter
|
|
|
108
111
|
!(created && pod.metadata?.creationTimestamp && pod.metadata.creationTimestamp < created));
|
|
109
112
|
for (const [index, pod] of pods.entries()) {
|
|
110
113
|
const key = pod.metadata?.name ?? `${index}`;
|
|
111
|
-
const failure = (0, cluster_state_1.getPodFailure)(pod, readiness.container);
|
|
114
|
+
const failure = (0, cluster_state_1.getPodFailure)(pod, readiness.container, readiness.restarts);
|
|
112
115
|
if (!failure) {
|
|
113
116
|
failingSince.delete(key);
|
|
114
117
|
continue;
|
|
@@ -125,14 +128,23 @@ async function awaitDeployRunningState(instance, env, name, readiness, pollInter
|
|
|
125
128
|
}
|
|
126
129
|
said = waiting;
|
|
127
130
|
if (readiness.timeout !== null) {
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
131
|
+
// the deadline bounds each start of the container, as on Docker: a crash is counted by
|
|
132
|
+
// `restarts`, and the back-off before the next start doesn't count against it
|
|
133
|
+
const statuses = pods.flatMap((pod) => (pod.status?.containerStatuses ?? []).filter((status) => !readiness.container || status.name === readiness.container));
|
|
134
|
+
const running = statuses.length > 0
|
|
135
|
+
? statuses.some((status) => status.state?.running)
|
|
136
|
+
: pods.some((pod) => pod.status?.phase === 'Running');
|
|
137
|
+
const restarted = Math.max(0, ...statuses.map((status) => status.restartCount ?? 0));
|
|
138
|
+
if (!running) {
|
|
139
|
+
runningSince = null;
|
|
140
|
+
}
|
|
141
|
+
else if (runningSince === null || restarted !== restarts) {
|
|
142
|
+
runningSince = now();
|
|
132
143
|
}
|
|
133
144
|
else if (now() - runningSince >= readiness.timeout) {
|
|
134
145
|
throw new Error(readiness.timeoutMessage);
|
|
135
146
|
}
|
|
147
|
+
restarts = restarted;
|
|
136
148
|
}
|
|
137
149
|
await (0, sleep_1.sleep)(pollInterval);
|
|
138
150
|
}
|
|
@@ -12,8 +12,8 @@ const MAX_MESSAGE = 500;
|
|
|
12
12
|
const MAX_EVENTS = 20;
|
|
13
13
|
// events the result of a task says better: a Job of a failed command
|
|
14
14
|
const SAID_ELSEWHERE = ['BackoffLimitExceeded'];
|
|
15
|
-
// A container restarted this often
|
|
16
|
-
// a race with a dependency or a slow first start
|
|
15
|
+
// A container restarted this often (ended three times) is taken as one that keeps crashing,
|
|
16
|
+
// unless its service sets `restarts`: once may be a race with a dependency or a slow first start
|
|
17
17
|
exports.CRASHES_TO_FAIL = 2;
|
|
18
18
|
// what a pull error says when the image is not there to pull, or not for this cluster:
|
|
19
19
|
// waiting does not change it. Others (a registry that limits, a token not there yet) may pass.
|
|
@@ -21,13 +21,14 @@ const IMAGE_GONE = /not found|manifest unknown|unauthorized|denied|forbidden|inv
|
|
|
21
21
|
/**
|
|
22
22
|
* Why a pod of a service will not become ready without a change, as the cluster says it:
|
|
23
23
|
* an image that can not be pulled, a container that keeps crashing or can not be created,
|
|
24
|
-
* an init container that failed. Null while it may still get there.
|
|
24
|
+
* an init container that failed. Null while it may still get there. A container fails once
|
|
25
|
+
* it was restarted `restarts` times and crashed again.
|
|
25
26
|
*
|
|
26
27
|
* Only what the cluster wrote is quoted: the termination message of a container is its
|
|
27
28
|
* own output (and may hold a value it read), except for the network gate's, which says why
|
|
28
29
|
* the policy did not take hold.
|
|
29
30
|
*/
|
|
30
|
-
function getPodFailure(pod, container) {
|
|
31
|
+
function getPodFailure(pod, container, restarts = exports.CRASHES_TO_FAIL) {
|
|
31
32
|
for (const status of pod.status?.initContainerStatuses ?? []) {
|
|
32
33
|
const terminated = status.state?.terminated ?? status.lastState?.terminated;
|
|
33
34
|
if (!terminated || terminated.exitCode === 0) {
|
|
@@ -44,7 +45,7 @@ function getPodFailure(pod, container) {
|
|
|
44
45
|
}
|
|
45
46
|
}
|
|
46
47
|
for (const status of mainStatuses(pod, container)) {
|
|
47
|
-
const failure = getContainerFailure(status);
|
|
48
|
+
const failure = getContainerFailure(status, restartsEnded(pod, status), restarts);
|
|
48
49
|
if (failure) {
|
|
49
50
|
return failure;
|
|
50
51
|
}
|
|
@@ -52,7 +53,7 @@ function getPodFailure(pod, container) {
|
|
|
52
53
|
return null;
|
|
53
54
|
}
|
|
54
55
|
exports.getPodFailure = getPodFailure;
|
|
55
|
-
function getContainerFailure(status) {
|
|
56
|
+
function getContainerFailure(status, restartsEnded, restarts) {
|
|
56
57
|
const waiting = status.state?.waiting;
|
|
57
58
|
const reason = waiting?.reason;
|
|
58
59
|
const said = waiting?.message ? `: ${cut(waiting.message)}` : '';
|
|
@@ -76,15 +77,21 @@ function getContainerFailure(status) {
|
|
|
76
77
|
case 'RunContainerError':
|
|
77
78
|
return { text: `its container could not be started (${reason})${said}`, final: false };
|
|
78
79
|
case 'CrashLoopBackOff':
|
|
79
|
-
return status.restartCount >=
|
|
80
|
+
return status.restartCount >= restarts
|
|
80
81
|
? { text: `it keeps crashing: ${describeLastExit(status)}`, final: true }
|
|
81
82
|
: null;
|
|
82
83
|
}
|
|
84
|
+
// a kubelet may leave an instance that ended as it is until its back-off is over, and say
|
|
85
|
+
// CrashLoopBackOff only when it looks at the pod again, a minute later: the same crash
|
|
86
|
+
if (restartsEnded && status.state?.terminated && status.restartCount >= restarts) {
|
|
87
|
+
return { text: `it keeps crashing: ${describeLastExit(status)}`, final: true };
|
|
88
|
+
}
|
|
83
89
|
return null;
|
|
84
90
|
}
|
|
85
91
|
// How the last instance of a container ended, as the cluster says it.
|
|
86
92
|
function describeLastExit(status) {
|
|
87
|
-
|
|
93
|
+
// the instance that ended now, else the one before the one that waits or runs
|
|
94
|
+
const last = status.state?.terminated ?? status.lastState?.terminated;
|
|
88
95
|
const restarts = `, restarted ${status.restartCount} time${status.restartCount === 1 ? '' : 's'}`;
|
|
89
96
|
if (!last) {
|
|
90
97
|
return `it ended${restarts}`;
|
|
@@ -120,6 +127,10 @@ function getPodWaiting(pod, container) {
|
|
|
120
127
|
case 'CrashLoopBackOff':
|
|
121
128
|
return `crashed and starts again: ${describeLastExit(status)}`;
|
|
122
129
|
}
|
|
130
|
+
const ended = status.state?.terminated;
|
|
131
|
+
if (ended && restartsEnded(pod, status)) {
|
|
132
|
+
return `${ended.exitCode === 0 ? 'ended' : 'crashed'} and starts again: ${describeLastExit(status)}`;
|
|
133
|
+
}
|
|
123
134
|
if (status.restartCount > 0 && status.state?.running && !status.ready) {
|
|
124
135
|
return `restarted: ${describeLastExit(status)}`;
|
|
125
136
|
}
|
|
@@ -127,6 +138,16 @@ function getPodWaiting(pod, container) {
|
|
|
127
138
|
return undefined;
|
|
128
139
|
}
|
|
129
140
|
exports.getPodWaiting = getPodWaiting;
|
|
141
|
+
// Whether the kubelet starts a container that ended again: not one of a pod that finished (a
|
|
142
|
+
// task's Job, an evicted pod, which its ReplicaSet replaces) or that never restarts.
|
|
143
|
+
function restartsEnded(pod, status) {
|
|
144
|
+
const phase = pod.status?.phase;
|
|
145
|
+
if (phase === 'Succeeded' || phase === 'Failed') {
|
|
146
|
+
return false;
|
|
147
|
+
}
|
|
148
|
+
const policy = pod.spec?.restartPolicy ?? 'Always';
|
|
149
|
+
return policy === 'Always' || (policy === 'OnFailure' && status.state?.terminated?.exitCode !== 0);
|
|
150
|
+
}
|
|
130
151
|
function mainStatuses(pod, container) {
|
|
131
152
|
return (pod.status?.containerStatuses ?? []).filter((status) => !container || status.name === container);
|
|
132
153
|
}
|
|
@@ -17,6 +17,8 @@ const network_policy_1 = require("./network-policy");
|
|
|
17
17
|
const egress_proxy_1 = require("../network/egress-proxy");
|
|
18
18
|
// what the service was created from, to tell a changed one
|
|
19
19
|
exports.DEFINITION_LABEL = 'hammerkit.dev/definition';
|
|
20
|
+
// how often a starting container is checked: as the liveness probe
|
|
21
|
+
const STARTUP_PERIOD_SECONDS = 5;
|
|
20
22
|
async function ensureKubernetesDeploymentExists(instance, env, service, persistence, secrets, stateKey,
|
|
21
23
|
// the needed services by name, and the proxy of a limited network
|
|
22
24
|
network) {
|
|
@@ -34,6 +36,16 @@ network) {
|
|
|
34
36
|
failureThreshold: 3,
|
|
35
37
|
}
|
|
36
38
|
: undefined;
|
|
39
|
+
// the healthcheck's `timeout` for each start: until it passed once, the liveness probe
|
|
40
|
+
// doesn't kill a container that starts slowly, and the wait fails it with the timeout
|
|
41
|
+
const startupProbe = probe && service.data.healthcheck
|
|
42
|
+
? {
|
|
43
|
+
...probe,
|
|
44
|
+
// a check more than the timeout holds, and one more for the first check's offset: the
|
|
45
|
+
// wait fails it with the timeout before the kubelet restarts it
|
|
46
|
+
failureThreshold: Math.ceil(service.data.healthcheck.timeout / (STARTUP_PERIOD_SECONDS * 1000)) + 2,
|
|
47
|
+
}
|
|
48
|
+
: undefined;
|
|
37
49
|
const server = env.server;
|
|
38
50
|
const placement = server ? (0, job_placement_1.getPodPlacement)(server, undefined) : null;
|
|
39
51
|
const workspace = (0, volumes_1.getWorkspacePodSpec)(env, persistence);
|
|
@@ -103,6 +115,7 @@ network) {
|
|
|
103
115
|
})),
|
|
104
116
|
readinessProbe: probe,
|
|
105
117
|
livenessProbe: probe,
|
|
118
|
+
startupProbe,
|
|
106
119
|
...(0, container_resources_1.getKubernetesResources)(service.data.resources),
|
|
107
120
|
volumeMounts: [...persistence.mounts.map((m) => m.mount), ...secrets.mounts],
|
|
108
121
|
...(0, job_placement_1.getContainerSecurity)(env),
|
|
@@ -134,6 +147,7 @@ network) {
|
|
|
134
147
|
container: getServiceContainerName(service),
|
|
135
148
|
subject: `service ${service.name}`,
|
|
136
149
|
onWaiting: (text) => service.status.write('warn', `${service.name} ${text}`),
|
|
150
|
+
...(service.data.restarts !== null ? { restarts: service.data.restarts } : {}),
|
|
137
151
|
});
|
|
138
152
|
}
|
|
139
153
|
exports.ensureKubernetesDeploymentExists = ensureKubernetesDeploymentExists;
|
package/dist/main.js
CHANGED
|
@@ -8,6 +8,7 @@ const redact_1 = require("./utils/redact");
|
|
|
8
8
|
const empty_writable_1 = require("./utils/empty-writable");
|
|
9
9
|
const run_program_1 = require("./run-program");
|
|
10
10
|
const abort_on_signals_1 = require("./utils/abort-on-signals");
|
|
11
|
+
const terminal_prompt_1 = require("./utils/terminal-prompt");
|
|
11
12
|
const abortCtrl = new AbortController();
|
|
12
13
|
(0, abort_on_signals_1.abortOnSignals)(process, abortCtrl);
|
|
13
14
|
let settled = false;
|
|
@@ -32,6 +33,7 @@ const secrets = (0, redact_1.createSecretRegistry)();
|
|
|
32
33
|
stdout: process.stdout,
|
|
33
34
|
stderr: process.stderr,
|
|
34
35
|
stdoutColumns: process.stdout.columns,
|
|
36
|
+
prompt: (0, terminal_prompt_1.getTerminalPrompt)(process.env, abortCtrl.signal),
|
|
35
37
|
}, process.argv, false).then(() => {
|
|
36
38
|
settled = true;
|
|
37
39
|
}, () => {
|
|
@@ -118,6 +118,7 @@ function parseService(workTree, service, environment, context) {
|
|
|
118
118
|
secrets: (0, work_secret_1.parseWorkSecrets)(service.cwd, context.projectRoot, service.schema.secrets, envs, environment, context.secrets, service.schema.account),
|
|
119
119
|
resources: (0, work_resources_1.parseWorkResources)(service.schema.resources),
|
|
120
120
|
volumeSize: service.schema.volumeSize ?? null,
|
|
121
|
+
restarts: service.schema.restarts ?? null,
|
|
121
122
|
network: (0, work_network_1.parseWorkNetwork)(service.schema.network),
|
|
122
123
|
networkCeiling: null,
|
|
123
124
|
args: service.schema.args ?? null,
|
|
@@ -17,6 +17,7 @@ const units_1 = require("../utils/units");
|
|
|
17
17
|
const service_tools_1 = require("../harness/service-tools");
|
|
18
18
|
const path_1 = require("path");
|
|
19
19
|
const check_keys_1 = require("../optimizer/check-keys");
|
|
20
|
+
const terminal_prompt_1 = require("../utils/terminal-prompt");
|
|
20
21
|
function describeOutcome(planned) {
|
|
21
22
|
switch (planned.outcome) {
|
|
22
23
|
case 'published':
|
|
@@ -174,6 +175,125 @@ function registerWorkCommands(remote, context) {
|
|
|
174
175
|
const token = await (0, token_source_1.getTokenSource)(environment, chosen.name, chosen.server).token();
|
|
175
176
|
return { ...chosen, token };
|
|
176
177
|
};
|
|
178
|
+
// publishes the bundle with where it comes from and says what it is now: published,
|
|
179
|
+
// already published, or only proposed
|
|
180
|
+
const publishBundle = async (serverName, server, token, bundle, annotations) => {
|
|
181
|
+
const answer = await (0, server_client_1.callServer)(server, token, 'POST', '/v1/definitions', {
|
|
182
|
+
...bundle,
|
|
183
|
+
annotations,
|
|
184
|
+
});
|
|
185
|
+
const version = answer.body;
|
|
186
|
+
const files = Object.keys(bundle.files).length;
|
|
187
|
+
// a proposal that already waits answers 200 like a published version, with its state
|
|
188
|
+
const proposed = answer.status === 202 || version.state !== undefined;
|
|
189
|
+
const verb = answer.status === 202
|
|
190
|
+
? 'proposed'
|
|
191
|
+
: proposed
|
|
192
|
+
? 'already proposed'
|
|
193
|
+
: answer.status === 201
|
|
194
|
+
? 'published'
|
|
195
|
+
: 'already published';
|
|
196
|
+
environment.stdout.write(`${verb} to ${serverName}: ${(0, server_client_1.printable)(version.hash)} (${files} ${files === 1 ? 'file' : 'files'})\n`);
|
|
197
|
+
const names = [...(version.tasks ?? []), ...(version.services ?? [])];
|
|
198
|
+
environment.stdout.write(`tasks and services: ${names.map(server_client_1.printable).join(', ') || 'none'}\n`);
|
|
199
|
+
for (const line of annotationLines(version.annotations, 'from')) {
|
|
200
|
+
environment.stdout.write(`${line.trimStart()}\n`);
|
|
201
|
+
}
|
|
202
|
+
if (proposed) {
|
|
203
|
+
environment.stdout.write(`it can not run before somebody who may publish approves it, until ${(0, server_client_1.printable)(version.expiresAt)}\n`);
|
|
204
|
+
}
|
|
205
|
+
return { version, proposed };
|
|
206
|
+
};
|
|
207
|
+
// what publishing the bundle would do, undefined from a server that can't say (before 1.27)
|
|
208
|
+
const planBundle = async (server, token, bundle) => {
|
|
209
|
+
try {
|
|
210
|
+
return (await (0, server_client_1.callServer)(server, token, 'POST', '/v1/definitions/plan', bundle)).body;
|
|
211
|
+
}
|
|
212
|
+
catch (e) {
|
|
213
|
+
if (e instanceof server_client_1.ServerError && e.status === 404) {
|
|
214
|
+
return undefined;
|
|
215
|
+
}
|
|
216
|
+
throw e;
|
|
217
|
+
}
|
|
218
|
+
};
|
|
219
|
+
// The build file a run or up names, and what `--publish` may publish: the hash of the
|
|
220
|
+
// build file here, unless `--build` or HAMMERKIT_BUILD names a version.
|
|
221
|
+
const runDefinition = async (options) => {
|
|
222
|
+
const named = options.build ?? environment.processEnvs.HAMMERKIT_BUILD;
|
|
223
|
+
if (named !== undefined) {
|
|
224
|
+
if (options.publish) {
|
|
225
|
+
throw new Error(`--publish publishes the build file here, it does not go with ${options.build !== undefined ? '--build' : 'HAMMERKIT_BUILD'}`);
|
|
226
|
+
}
|
|
227
|
+
if (!bundle_1.HASH_PATTERN.test(named)) {
|
|
228
|
+
throw new Error(`${(0, server_client_1.printable)(named)} is not the hash of a published version (sha256:...)`);
|
|
229
|
+
}
|
|
230
|
+
return { definition: named };
|
|
231
|
+
}
|
|
232
|
+
const local = await (0, collect_bundle_1.collectBundle)(buildFile, environment);
|
|
233
|
+
return { definition: (0, bundle_1.hashBundle)(local), local };
|
|
234
|
+
};
|
|
235
|
+
// Submits a run of a published version. When the server does not have the build file
|
|
236
|
+
// here, it is published first: with --publish at once, else when the person at the
|
|
237
|
+
// terminal says so. Without a terminal and without --publish the run fails and says how
|
|
238
|
+
// to publish it, so CI never waits on a question.
|
|
239
|
+
const submitPublishing = async (target, request, local, publish) => {
|
|
240
|
+
const { serverName, server, connection } = target;
|
|
241
|
+
try {
|
|
242
|
+
return await (0, sandbox_client_1.submitRun)(connection, request);
|
|
243
|
+
}
|
|
244
|
+
catch (e) {
|
|
245
|
+
const status = e.status;
|
|
246
|
+
if (!local || status !== 400 || !/no published build file/.test(String(e))) {
|
|
247
|
+
throw e;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
const definition = request.definition;
|
|
251
|
+
const notPublished = `the build file here is not published on ${serverName} (${definition})`;
|
|
252
|
+
const token = await connection.token();
|
|
253
|
+
if (!publish) {
|
|
254
|
+
if (!environment.prompt) {
|
|
255
|
+
throw new Error(`${notPublished}, publish it first with: hammerkit remote publish ${serverName}, or pass --publish`);
|
|
256
|
+
}
|
|
257
|
+
// a server before 1.27 can't say what publishing would do: it is asked all the same,
|
|
258
|
+
// and refuses then what the caller may not do
|
|
259
|
+
const planned = await planBundle(server, token, local);
|
|
260
|
+
if (planned && planned.outcome === null) {
|
|
261
|
+
throw new Error(`${notPublished}, and you may only run published build files (builds: run): ` +
|
|
262
|
+
`somebody who may publish publishes it with: hammerkit remote publish ${serverName}`);
|
|
263
|
+
}
|
|
264
|
+
if (planned?.outcome !== 'already published') {
|
|
265
|
+
const files = Object.keys(local.files).length;
|
|
266
|
+
environment.stdout.write(`${notPublished}, ${files} ${files === 1 ? 'file' : 'files'}\n`);
|
|
267
|
+
if (planned) {
|
|
268
|
+
const names = [...(planned.tasks ?? []), ...(planned.services ?? [])];
|
|
269
|
+
environment.stdout.write(`tasks and services: ${names.map(server_client_1.printable).join(', ') || 'none'}\n`);
|
|
270
|
+
}
|
|
271
|
+
const proposing = planned?.outcome === 'proposed';
|
|
272
|
+
if (proposing) {
|
|
273
|
+
environment.stdout.write(`${describeOutcome(planned)}\n`);
|
|
274
|
+
}
|
|
275
|
+
const answer = await environment.prompt(proposing ? `propose it to ${serverName} now? [y/N] ` : `publish it to ${serverName} now and go on? [y/N] `);
|
|
276
|
+
if (environment.abortCtrl.signal.aborted) {
|
|
277
|
+
throw new Error(`stopped, nothing was published`);
|
|
278
|
+
}
|
|
279
|
+
if (!(0, terminal_prompt_1.isYes)(answer)) {
|
|
280
|
+
throw new Error(`${notPublished}, nothing was published`);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
const annotations = await (0, annotations_1.collectAnnotations)(buildFile, Object.keys(local.files));
|
|
285
|
+
const { version, proposed } = await publishBundle(serverName, server, token, local, annotations).catch((e) => {
|
|
286
|
+
throw e instanceof server_client_1.ServerError && e.status === 403 ? new Error(`${notPublished}, ${e.message}`) : e;
|
|
287
|
+
});
|
|
288
|
+
if (version.hash !== definition) {
|
|
289
|
+
throw new Error(`${serverName} published ${(0, server_client_1.printable)(version.hash)}, not ${definition} as computed here, nothing was started`);
|
|
290
|
+
}
|
|
291
|
+
if (proposed) {
|
|
292
|
+
throw new Error(`${definition} is only proposed on ${serverName}, nothing was started: ` +
|
|
293
|
+
'run this again once somebody who may publish approves it');
|
|
294
|
+
}
|
|
295
|
+
return (0, sandbox_client_1.submitRun)(connection, request);
|
|
296
|
+
};
|
|
177
297
|
remote
|
|
178
298
|
.command('publish [server]')
|
|
179
299
|
.description('publish the build file and the files it includes to a server, runs refer to it by its hash')
|
|
@@ -198,23 +318,7 @@ function registerWorkCommands(remote, context) {
|
|
|
198
318
|
}
|
|
199
319
|
const annotations = (0, annotations_1.applyAnnotations)(await (0, annotations_1.collectAnnotations)(buildFile, Object.keys(bundle.files)), options.annotation ?? []);
|
|
200
320
|
const { name: serverName, server, token } = await connect(name);
|
|
201
|
-
|
|
202
|
-
...bundle,
|
|
203
|
-
annotations,
|
|
204
|
-
});
|
|
205
|
-
const version = answer.body;
|
|
206
|
-
const files = Object.keys(bundle.files).length;
|
|
207
|
-
const proposed = answer.status === 202;
|
|
208
|
-
const verb = proposed ? 'proposed' : answer.status === 201 ? 'published' : 'already published';
|
|
209
|
-
environment.stdout.write(`${verb} to ${serverName}: ${(0, server_client_1.printable)(version.hash)} (${files} ${files === 1 ? 'file' : 'files'})\n`);
|
|
210
|
-
const names = [...(version.tasks ?? []), ...(version.services ?? [])];
|
|
211
|
-
environment.stdout.write(`tasks and services: ${names.map(server_client_1.printable).join(', ') || 'none'}\n`);
|
|
212
|
-
for (const line of annotationLines(version.annotations, 'from')) {
|
|
213
|
-
environment.stdout.write(`${line.trimStart()}\n`);
|
|
214
|
-
}
|
|
215
|
-
if (proposed) {
|
|
216
|
-
environment.stdout.write(`it can not run before somebody who may publish approves it, until ${(0, server_client_1.printable)(version.expiresAt)}\n`);
|
|
217
|
-
}
|
|
321
|
+
await publishBundle(serverName, server, token, bundle, annotations);
|
|
218
322
|
}));
|
|
219
323
|
remote
|
|
220
324
|
.command('run <tasks...>')
|
|
@@ -229,12 +333,9 @@ function registerWorkCommands(remote, context) {
|
|
|
229
333
|
.addOption(new commander_1.Option('--timeout <duration>', 'as --timeout of run: for tasks without their own, such as 30m'))
|
|
230
334
|
.addOption(new commander_1.Option('--max-timeout <duration>', 'as --max-timeout of run: for every task, also one with its own timeout'))
|
|
231
335
|
.addOption(new commander_1.Option('--detach', 'submit the run and exit, without following it: for a check that CI verifies later'))
|
|
336
|
+
.addOption(new commander_1.Option('--publish', 'publish the build file here first when the server does not have it, without asking; for CI and scripts'))
|
|
232
337
|
.action(guarded('Remote run was not successful', async (tasks, options) => {
|
|
233
|
-
const
|
|
234
|
-
const definition = named ?? (0, bundle_1.hashBundle)(await (0, collect_bundle_1.collectBundle)(buildFile, environment));
|
|
235
|
-
if (!bundle_1.HASH_PATTERN.test(definition)) {
|
|
236
|
-
throw new Error(`${(0, server_client_1.printable)(definition)} is not the hash of a published version (sha256:...)`);
|
|
237
|
-
}
|
|
338
|
+
const { definition, local } = await runDefinition(options);
|
|
238
339
|
const source = await refSource(options);
|
|
239
340
|
const ref = source.ref;
|
|
240
341
|
const { name: serverName, server } = await getServer(options.server);
|
|
@@ -248,21 +349,7 @@ function registerWorkCommands(remote, context) {
|
|
|
248
349
|
if (passthrough.length > 0) {
|
|
249
350
|
runOptions.args = passthrough;
|
|
250
351
|
}
|
|
251
|
-
|
|
252
|
-
try {
|
|
253
|
-
submitted = await (0, sandbox_client_1.submitRun)(connection, {
|
|
254
|
-
definition,
|
|
255
|
-
tasks,
|
|
256
|
-
source,
|
|
257
|
-
options: runOptions,
|
|
258
|
-
});
|
|
259
|
-
}
|
|
260
|
-
catch (e) {
|
|
261
|
-
if (e.status === 400 && !named && /no published build file/.test(String(e))) {
|
|
262
|
-
throw new Error(`the build file here is not published on ${serverName} (${definition}), publish it first`);
|
|
263
|
-
}
|
|
264
|
-
throw e;
|
|
265
|
-
}
|
|
352
|
+
const submitted = await submitPublishing({ serverName, server, connection }, { definition, tasks, source, options: runOptions }, local, !!options.publish);
|
|
266
353
|
const at = submitted.source.type === 'ref' ? ` at ${(0, server_client_1.printable)(submitted.source.commit)}` : '';
|
|
267
354
|
environment.stdout.write(`${(0, server_client_1.printable)(submitted.id)} on ${serverName}: ${tasks.map(server_client_1.printable).join(', ')} from ${(0, server_client_1.printable)(ref)}${at}\n`);
|
|
268
355
|
if (options.detach) {
|
|
@@ -314,17 +401,14 @@ function registerWorkCommands(remote, context) {
|
|
|
314
401
|
.addOption(new commander_1.Option('--ref <ref>', 'the branch or tag to start; the branch checked out here when left out'))
|
|
315
402
|
.addOption(new commander_1.Option('--commit <sha>', 'the commit the ref has to be at; the commit checked out here when --ref is left out'))
|
|
316
403
|
.addOption(new commander_1.Option('--env <name>', 'the environment of the build file, as --env of run'))
|
|
404
|
+
.addOption(new commander_1.Option('--publish', 'publish the build file here first when the server does not have it, without asking; for CI and scripts'))
|
|
317
405
|
.action(guarded('Remote up was not successful', async (services, options) => {
|
|
318
|
-
const
|
|
319
|
-
const definition = named ?? (0, bundle_1.hashBundle)(await (0, collect_bundle_1.collectBundle)(buildFile, environment));
|
|
320
|
-
if (!bundle_1.HASH_PATTERN.test(definition)) {
|
|
321
|
-
throw new Error(`${(0, server_client_1.printable)(definition)} is not the hash of a published version (sha256:...)`);
|
|
322
|
-
}
|
|
406
|
+
const { definition, local } = await runDefinition(options);
|
|
323
407
|
const endpoints = (options.endpoint ?? []).map(service_tools_1.parseEndpoint);
|
|
324
408
|
const source = await refSource(options);
|
|
325
409
|
const { name: serverName, server } = await getServer(options.server);
|
|
326
410
|
const connection = { url: server.url, ...(0, token_source_1.getTokenSource)(environment, serverName, server) };
|
|
327
|
-
const submitted = await (
|
|
411
|
+
const submitted = await submitPublishing({ serverName, server, connection }, {
|
|
328
412
|
definition,
|
|
329
413
|
services,
|
|
330
414
|
endpoints,
|
|
@@ -333,7 +417,7 @@ function registerWorkCommands(remote, context) {
|
|
|
333
417
|
...(options.name ? { name: options.name } : {}),
|
|
334
418
|
source,
|
|
335
419
|
options: options.env ? { env: options.env } : {},
|
|
336
|
-
});
|
|
420
|
+
}, local, !!options.publish);
|
|
337
421
|
environment.stdout.write(`${(0, server_client_1.printable)(submitted.id)} on ${serverName}: ${services.map(server_client_1.printable).join(', ')} from ${(0, server_client_1.printable)(source.ref)}${submitted.source.type === 'ref' ? ` at ${(0, server_client_1.printable)(submitted.source.commit)}` : ''}\n`);
|
|
338
422
|
const followed = await (0, sandbox_client_1.followRun)(connection, submitted, {
|
|
339
423
|
signal: environment.abortCtrl.signal,
|
|
@@ -41,6 +41,14 @@ exports.buildFileContainerServiceSchema = (0, zod_1.object)({
|
|
|
41
41
|
// the service account its provider secrets are read as, unless a secret names one
|
|
42
42
|
account: (0, zod_1.string)().optional(),
|
|
43
43
|
resources: resources_schema_1.resourcesSchema.optional(),
|
|
44
|
+
// on Kubernetes, how often its container may crash and be restarted before the
|
|
45
|
+
// service fails (2 by default): it fails when it crashes once more
|
|
46
|
+
restarts: (0, zod_1.number)()
|
|
47
|
+
.int()
|
|
48
|
+
.min(0)
|
|
49
|
+
.max(10)
|
|
50
|
+
.describe('crashes restarted before it fails on Kubernetes, 0 to 10, 2 by default')
|
|
51
|
+
.optional(),
|
|
44
52
|
// on Kubernetes, the size of the claim that holds its sources, mounts and volumes
|
|
45
53
|
volumeSize: size_schema_1.sizeSchema.describe('size of its claim on Kubernetes, e.g. 10Gi').optional(),
|
|
46
54
|
// what the service reaches besides its needs; tasks needing it always reach it
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.isYes = exports.getTerminalPrompt = exports.terminalPrompt = void 0;
|
|
4
|
+
const readline_1 = require("readline");
|
|
5
|
+
/**
|
|
6
|
+
* Asks the person at the terminal one question and answers the line they typed, or
|
|
7
|
+
* undefined when the input ended or the signal stopped the command first (Ctrl-C).
|
|
8
|
+
* The terminal stays in its own line mode, so Ctrl-C is a signal as anywhere else.
|
|
9
|
+
*/
|
|
10
|
+
function terminalPrompt(input, output, signal) {
|
|
11
|
+
return (question) => new Promise((resolve) => {
|
|
12
|
+
if (signal.aborted) {
|
|
13
|
+
resolve(undefined);
|
|
14
|
+
return;
|
|
15
|
+
}
|
|
16
|
+
const lines = (0, readline_1.createInterface)({ input, output, terminal: false });
|
|
17
|
+
let answer;
|
|
18
|
+
const stop = () => lines.close();
|
|
19
|
+
signal.addEventListener('abort', stop, { once: true });
|
|
20
|
+
lines.once('close', () => {
|
|
21
|
+
signal.removeEventListener('abort', stop);
|
|
22
|
+
if (answer === undefined) {
|
|
23
|
+
// what was asked ends on a line of its own
|
|
24
|
+
output.write('\n');
|
|
25
|
+
}
|
|
26
|
+
resolve(answer);
|
|
27
|
+
});
|
|
28
|
+
lines.question(question, (line) => {
|
|
29
|
+
answer = line;
|
|
30
|
+
lines.close();
|
|
31
|
+
});
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
exports.terminalPrompt = terminalPrompt;
|
|
35
|
+
/**
|
|
36
|
+
* The prompt of the CLI: only when somebody sits at a terminal to answer it, never in
|
|
37
|
+
* CI, where a question would wait until the job times out.
|
|
38
|
+
*/
|
|
39
|
+
function getTerminalPrompt(processEnvs, signal, stdio = process) {
|
|
40
|
+
if (!stdio.stdin.isTTY || !stdio.stdout.isTTY || isSet(processEnvs.CI)) {
|
|
41
|
+
return undefined;
|
|
42
|
+
}
|
|
43
|
+
return terminalPrompt(stdio.stdin, stdio.stdout, signal);
|
|
44
|
+
}
|
|
45
|
+
exports.getTerminalPrompt = getTerminalPrompt;
|
|
46
|
+
function isSet(value) {
|
|
47
|
+
return value !== undefined && value !== '' && value !== '0' && value.toLowerCase() !== 'false';
|
|
48
|
+
}
|
|
49
|
+
function isYes(answer) {
|
|
50
|
+
return /^\s*y(es)?\s*$/i.test(answer ?? '');
|
|
51
|
+
}
|
|
52
|
+
exports.isYes = isYes;
|
|
53
|
+
//# sourceMappingURL=terminal-prompt.js.map
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "hammerkit",
|
|
3
3
|
"description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
|
|
4
|
-
"version": "1.27.0-next.
|
|
4
|
+
"version": "1.27.0-next.29",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"build",
|
|
7
7
|
"build-tool",
|