hammerkit 1.27.0-next.3 → 1.27.0-next.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build.schema.json +188 -0
- package/dist/cache/backends/local-cache-backend.js +6 -0
- package/dist/cache/backends/registry-cache-backend.js +2 -2
- package/dist/cache/explain.js +17 -0
- package/dist/cli.js +2 -0
- package/dist/docker/item-network.js +13 -5
- package/dist/executer/deadline.js +17 -3
- package/dist/executer/deferred-restore.js +36 -0
- package/dist/executer/docker-service.js +30 -8
- package/dist/executer/docker-task.js +21 -1
- package/dist/executer/event-cache.js +20 -2
- package/dist/executer/execute-work-service.js +2 -0
- package/dist/executer/execute-work-task.js +10 -3
- package/dist/executer/execution-steps.js +25 -4
- package/dist/executer/reset-work-tree.js +4 -0
- package/dist/executer/scheduler/enqueue-next.js +57 -6
- package/dist/executer/scheduler/state-key.js +42 -5
- package/dist/executer/watch-loop.js +4 -1
- package/dist/harness/connect.js +20 -6
- package/dist/harness/mcp-command.js +22 -6
- package/dist/harness/mcp.js +63 -114
- package/dist/harness/opencode.js +158 -50
- package/dist/harness/pi.js +29 -0
- package/dist/harness/sandbox-client.js +168 -17
- package/dist/harness/service-tools.js +141 -0
- package/dist/harness/task-tools.js +221 -0
- package/dist/kubernetes/apply.js +27 -16
- package/dist/kubernetes/await-running-state.js +59 -13
- package/dist/kubernetes/claim-layout.js +71 -0
- package/dist/kubernetes/cluster-state.js +245 -0
- package/dist/kubernetes/ensure-http-route.js +1 -1
- package/dist/kubernetes/ensure-kubernetes-deployment-exists.js +35 -12
- package/dist/kubernetes/ensure-kubernetes-persistent-volume-claim-exists.js +1 -1
- package/dist/kubernetes/ensure-kubernetes-service-exists.js +6 -9
- package/dist/kubernetes/ensure-persistent-data.js +30 -20
- package/dist/kubernetes/follow-pod-logs.js +182 -0
- package/dist/kubernetes/get-service-ip.js +2 -2
- package/dist/kubernetes/job-placement.js +36 -12
- package/dist/kubernetes/job-secrets.js +33 -19
- package/dist/kubernetes/kept-claims.js +22 -10
- package/dist/kubernetes/network-policy.js +85 -27
- package/dist/kubernetes/remove-persistent-data.js +3 -1
- package/dist/kubernetes/resources.js +25 -3
- package/dist/kubernetes/restore-kubernetes-data.js +9 -3
- package/dist/kubernetes/sandbox-runner.js +628 -0
- package/dist/kubernetes/secrets.js +7 -11
- package/dist/kubernetes/volumes.js +6 -3
- package/dist/main.js +2 -0
- package/dist/network/egress-proxy.js +45 -6
- package/dist/optimizer/calculate-checksum.js +3 -2
- package/dist/optimizer/check-key.js +94 -0
- package/dist/optimizer/check-keys.js +64 -0
- package/dist/optimizer/get-work-cache-stats.js +17 -3
- package/dist/optimizer/work-task-cache-description.js +51 -1
- package/dist/planner/utils/append-work-dependencies.js +10 -5
- package/dist/planner/utils/append-work-service.js +19 -1
- package/dist/planner/utils/append-work-task.js +18 -3
- package/dist/planner/utils/get-work-runtime.js +6 -1
- package/dist/planner/utils/normalize-path.js +19 -1
- package/dist/planner/utils/parse-work-mount.js +7 -1
- package/dist/planner/utils/parse-work-port.js +4 -1
- package/dist/planner/utils/task-arguments.js +3 -0
- package/dist/planner/validate.js +22 -0
- package/dist/planner/work-network.js +3 -1
- package/dist/planner/work-runtime-aggregate.js +32 -0
- package/dist/planner/work-runtime-docker.js +2 -1
- package/dist/planner/work-runtime-kubernetes.js +124 -48
- package/dist/planner/work-runtime-local.js +2 -1
- package/dist/planner/work-task.js +6 -1
- package/dist/program.js +31 -10
- package/dist/remote/device-login.js +7 -2
- package/dist/remote/host-config.js +69 -3
- package/dist/remote/remote-commands.js +30 -7
- package/dist/remote/remote-work-commands.js +426 -45
- package/dist/remote/server-client.js +15 -2
- package/dist/remote/token-source.js +64 -26
- package/dist/run-program.js +9 -0
- package/dist/schema/build-file-base-task-schema.js +8 -1
- package/dist/schema/build-file-container-service-schema.js +12 -0
- package/dist/schema/build-file-container-task-schema.js +16 -1
- package/dist/schema/build-file-include-schema.js +18 -4
- package/dist/schema/build-file-kubernetes-service-schema.js +2 -0
- package/dist/schema/build-file-task-schema.js +5 -1
- package/dist/schema/capabilities-schema.js +56 -0
- package/dist/schema/reference-parser.js +120 -11
- package/dist/schema/schema-parser.js +92 -41
- package/dist/schema/work-scope-parser.js +33 -10
- package/dist/secrets/command-secret-provider.js +3 -1
- package/dist/secrets/github-app-token-command.js +3 -1
- package/dist/secrets/secret-catalog.js +43 -19
- package/dist/server/accounts-api.js +3 -1
- package/dist/server/approvals-api.js +1 -1
- package/dist/server/checks-api.js +83 -0
- package/dist/server/checks.js +116 -0
- package/dist/server/definitions-api.js +66 -3
- package/dist/server/endpoints.js +739 -0
- package/dist/server/executor.js +57 -5
- package/dist/server/kubernetes-backend.js +367 -65
- package/dist/server/kubernetes-sandbox-storage.js +48 -3
- package/dist/server/oauth.js +12 -1
- package/dist/server/plan.js +188 -103
- package/dist/server/policy.js +32 -2
- package/dist/server/repo-access.js +5 -0
- package/dist/server/run-plan.js +50 -19
- package/dist/server/run-request.js +58 -3
- package/dist/server/run-worker.js +245 -12
- package/dist/server/runs.js +410 -27
- package/dist/server/sandbox-pod.js +28 -0
- package/dist/server/sandbox-storage.js +23 -0
- package/dist/server/sandboxes-api.js +10 -4
- package/dist/server/sandboxes.js +8 -0
- package/dist/server/server-command.js +39 -20
- package/dist/server/server-config.js +255 -14
- package/dist/server/server.js +269 -15
- package/dist/server/service-runs.js +82 -0
- package/dist/utils/abort-on-signals.js +46 -6
- package/dist/utils/tar.js +36 -0
- package/dist/utils/terminal-prompt.js +53 -0
- package/package.json +1 -1
package/dist/server/runs.js
CHANGED
|
@@ -30,6 +30,8 @@ exports.SANDBOX_ID_PATTERN = /^sbx_[A-Za-z0-9_-]{16}$/;
|
|
|
30
30
|
function onSandbox(run, sandbox) {
|
|
31
31
|
return run.source.type === 'sandbox' && run.source.id === sandbox;
|
|
32
32
|
}
|
|
33
|
+
// the tasks of a run whose check keys are kept, a build file has fewer
|
|
34
|
+
const MAX_KEYED_TASKS = 1000;
|
|
33
35
|
// how much of the end of a task's output its result carries
|
|
34
36
|
const TAIL_LINES = 20;
|
|
35
37
|
const TAIL_LINE_LENGTH = 1000;
|
|
@@ -100,6 +102,7 @@ class RunRegistry {
|
|
|
100
102
|
bytes: 0,
|
|
101
103
|
tails: new Map(Object.entries(run.results ?? {}).map(([task, result]) => [task, result.lastLog])),
|
|
102
104
|
saving: Promise.resolve(),
|
|
105
|
+
keys: {},
|
|
103
106
|
};
|
|
104
107
|
this.entries.set(run.id, entry);
|
|
105
108
|
if (run.state === 'awaiting-approval') {
|
|
@@ -112,9 +115,27 @@ class RunRegistry {
|
|
|
112
115
|
this.finish(entry, 'rejected', 'nobody approved it in time');
|
|
113
116
|
}
|
|
114
117
|
}
|
|
118
|
+
else if (this.maySleep(entry) && ['queued', 'running', 'up', 'asleep'].includes(run.state)) {
|
|
119
|
+
// the services went with the process that held them, the next request wakes them
|
|
120
|
+
if (run.state === 'queued' || run.state === 'running' || run.state === 'up') {
|
|
121
|
+
this.sleep(entry);
|
|
122
|
+
}
|
|
123
|
+
else {
|
|
124
|
+
this.expireLater(entry);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
else if (run.state === 'asleep') {
|
|
128
|
+
// its time ran out while the server was down: it ends now, and gives its place back
|
|
129
|
+
run.stoppedBy ?? (run.stoppedBy = 'ttl');
|
|
130
|
+
this.finish(entry, 'succeeded');
|
|
131
|
+
}
|
|
115
132
|
else if (run.state === 'queued' || run.state === 'running') {
|
|
116
133
|
this.finish(entry, 'failed', `the server stopped while the run was ${run.state}, submit it again`);
|
|
117
134
|
}
|
|
135
|
+
else if (run.state === 'up') {
|
|
136
|
+
// the services went with the process that held them
|
|
137
|
+
this.finish(entry, 'failed', 'the server stopped while the services were up, start them again');
|
|
138
|
+
}
|
|
118
139
|
}
|
|
119
140
|
this.evict();
|
|
120
141
|
}
|
|
@@ -129,10 +150,18 @@ class RunRegistry {
|
|
|
129
150
|
throw new RunError('too many runs are waiting for an approval');
|
|
130
151
|
}
|
|
131
152
|
this.checkLimits(owner, waiting);
|
|
153
|
+
if (spec.up) {
|
|
154
|
+
this.checkUpLimits(owner, spec.up.name);
|
|
155
|
+
this.checkHosts(owner, spec.up);
|
|
156
|
+
}
|
|
132
157
|
const run = {
|
|
133
158
|
...spec,
|
|
134
159
|
id: `run_${(0, crypto_1.randomBytes)(12).toString('base64url')}`,
|
|
135
|
-
owner: {
|
|
160
|
+
owner: {
|
|
161
|
+
iss: owner.iss,
|
|
162
|
+
sub: owner.sub,
|
|
163
|
+
...(spec.up?.sleepTtl ? { groups: [...(owner.groups ?? [])], claims: { ...(owner.claims ?? {}) } } : {}),
|
|
164
|
+
},
|
|
136
165
|
state: waiting ? 'awaiting-approval' : 'queued',
|
|
137
166
|
createdAt: this.timestamp(),
|
|
138
167
|
};
|
|
@@ -145,6 +174,7 @@ class RunRegistry {
|
|
|
145
174
|
bytes: 0,
|
|
146
175
|
tails: new Map(),
|
|
147
176
|
saving: Promise.resolve(),
|
|
177
|
+
keys: {},
|
|
148
178
|
};
|
|
149
179
|
this.entries.set(run.id, entry);
|
|
150
180
|
if (waiting) {
|
|
@@ -179,30 +209,79 @@ class RunRegistry {
|
|
|
179
209
|
.map((entry) => this.copy(entry.run))
|
|
180
210
|
.reverse();
|
|
181
211
|
}
|
|
182
|
-
// A queued run is cancelled at once; a running one when its executor stops.
|
|
212
|
+
// A queued run is cancelled at once; a running one when its executor stops. A service run
|
|
213
|
+
// that is up stops its services, and says who stopped it.
|
|
183
214
|
cancel(owner, id, canUse) {
|
|
184
215
|
const entry = this.find(owner, id, canUse);
|
|
185
216
|
if (!entry) {
|
|
186
217
|
return undefined;
|
|
187
218
|
}
|
|
188
|
-
this.stop(entry);
|
|
219
|
+
this.stop(entry, undefined, { iss: owner.iss, sub: owner.sub });
|
|
189
220
|
return this.copy(entry.run);
|
|
190
221
|
}
|
|
222
|
+
/**
|
|
223
|
+
* Moves the end of a service run that is up (or asleep) to `ttl` (ms) from now, at most
|
|
224
|
+
* the server's maxTtl, and never sooner than it was. Its owner may, and whoever `mayOpen`
|
|
225
|
+
* says may open its endpoints.
|
|
226
|
+
*/
|
|
227
|
+
extend(caller, id, ttl, mayOpen) {
|
|
228
|
+
const entry = this.entries.get(id);
|
|
229
|
+
if (!entry || !(this.owns(caller, entry) || (entry.run.up && mayOpen?.(this.copy(entry.run))))) {
|
|
230
|
+
return { status: 'not-found' };
|
|
231
|
+
}
|
|
232
|
+
// up, asleep, or woken and starting again
|
|
233
|
+
if (!entry.run.upAt || isFinal(entry.run.state)) {
|
|
234
|
+
return { status: 'not-up', state: entry.run.state };
|
|
235
|
+
}
|
|
236
|
+
// never sooner than it was: who wants it gone stops it
|
|
237
|
+
const ms = Math.min(ttl, this.options.up?.maxTtl ?? ttl);
|
|
238
|
+
const expiresAt = Math.max(Date.parse(entry.run.expiresAt ?? '') || 0, this.now() + ms);
|
|
239
|
+
entry.run.expiresAt = new Date(expiresAt).toISOString();
|
|
240
|
+
this.record(entry, { type: 'extended', by: { iss: caller.iss, sub: caller.sub }, expiresAt: entry.run.expiresAt });
|
|
241
|
+
this.expireLater(entry);
|
|
242
|
+
return { status: 'ok', run: this.copy(entry.run) };
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* The service run an endpoint host belongs to: the newest that is up, so one that replaces
|
|
246
|
+
* a run of the same name takes its hosts over as soon as it is up; else the newest that
|
|
247
|
+
* had it, for a page that says how it stands. With the address of the endpoint when up.
|
|
248
|
+
*/
|
|
249
|
+
endpoint(host) {
|
|
250
|
+
let found;
|
|
251
|
+
for (const entry of this.entries.values()) {
|
|
252
|
+
const endpoint = entry.run.up?.endpoints.find((candidate) => candidate.host === host);
|
|
253
|
+
if (endpoint && (entry.run.state === 'up' || found?.entry.run.state !== 'up')) {
|
|
254
|
+
found = { entry, endpoint };
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
if (!found) {
|
|
258
|
+
return undefined;
|
|
259
|
+
}
|
|
260
|
+
const { entry, endpoint } = found;
|
|
261
|
+
const target = entry.run.state === 'up'
|
|
262
|
+
? entry.targets?.find((candidate) => candidate.service === endpoint.service && candidate.port === endpoint.port)
|
|
263
|
+
: undefined;
|
|
264
|
+
return { run: this.copy(entry.run), endpoint: { ...endpoint }, ...(target ? { target } : {}) };
|
|
265
|
+
}
|
|
191
266
|
// The run that holds a sandbox: the oldest on it that is not over. It holds it from
|
|
192
|
-
// when it is submitted, so the files are not moved under what an approver saw.
|
|
267
|
+
// when it is submitted, so the files are not moved under what an approver saw. A service
|
|
268
|
+
// run that is up works on a copy of the files, and holds it no longer.
|
|
193
269
|
holder(sandbox) {
|
|
194
|
-
return [...this.entries.values()].find((entry) => onSandbox(entry.run, sandbox) &&
|
|
195
|
-
.
|
|
270
|
+
return [...this.entries.values()].find((entry) => onSandbox(entry.run, sandbox) &&
|
|
271
|
+
!isFinal(entry.run.state) &&
|
|
272
|
+
entry.run.state !== 'up' &&
|
|
273
|
+
entry.run.state !== 'asleep')?.run.id;
|
|
196
274
|
}
|
|
197
275
|
// Cancels every run on a sandbox and resolves when none of them runs anymore.
|
|
198
276
|
async stopSandbox(sandbox, reason) {
|
|
199
277
|
const on = () => [...this.entries.values()].filter((entry) => onSandbox(entry.run, sandbox));
|
|
200
278
|
on().forEach((entry) => this.stop(entry, reason));
|
|
201
|
-
while (on().some((entry) => entry.run.state === 'running')) {
|
|
279
|
+
while (on().some((entry) => entry.run.state === 'running' || entry.run.state === 'up')) {
|
|
202
280
|
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
203
281
|
}
|
|
204
282
|
}
|
|
205
|
-
stop(entry, reason) {
|
|
283
|
+
stop(entry, reason, by) {
|
|
284
|
+
var _a, _b;
|
|
206
285
|
if (entry.run.state === 'queued') {
|
|
207
286
|
this.queue.splice(this.queue.indexOf(entry), 1);
|
|
208
287
|
this.finish(entry, 'cancelled', reason);
|
|
@@ -213,6 +292,15 @@ class RunRegistry {
|
|
|
213
292
|
else if (entry.run.state === 'running') {
|
|
214
293
|
entry.abort.abort();
|
|
215
294
|
}
|
|
295
|
+
else if (entry.run.state === 'up') {
|
|
296
|
+
(_a = entry.run).stoppedBy ?? (_a.stoppedBy = by ?? 'replaced');
|
|
297
|
+
entry.sleeping = false;
|
|
298
|
+
entry.abort.abort();
|
|
299
|
+
}
|
|
300
|
+
else if (entry.run.state === 'asleep') {
|
|
301
|
+
(_b = entry.run).stoppedBy ?? (_b.stoppedBy = by ?? 'replaced');
|
|
302
|
+
this.finish(entry, 'succeeded');
|
|
303
|
+
}
|
|
216
304
|
}
|
|
217
305
|
// The runs that wait for an approval, of anybody, oldest first. Who may see
|
|
218
306
|
// them is for the caller to decide.
|
|
@@ -289,18 +377,20 @@ class RunRegistry {
|
|
|
289
377
|
return { events, finished, stop: () => entry.listeners.delete(listener) };
|
|
290
378
|
}
|
|
291
379
|
/**
|
|
292
|
-
* The run once it is over, or as it is after `ms` or when
|
|
293
|
-
* that can not hold a stream of events open (a tool call of
|
|
380
|
+
* The run once it is over (a service run: once it is up), or as it is after `ms` or when
|
|
381
|
+
* `signal` aborts, for clients that can not hold a stream of events open (a tool call of
|
|
382
|
+
* an agent).
|
|
294
383
|
*/
|
|
295
384
|
async wait(owner, id, ms, canUse, signal) {
|
|
296
385
|
const entry = this.find(owner, id, canUse);
|
|
297
386
|
if (!entry) {
|
|
298
387
|
return undefined;
|
|
299
388
|
}
|
|
300
|
-
if (!isFinal(entry.run.state) && ms > 0 && !this.closed && !signal?.aborted) {
|
|
389
|
+
if (!isFinal(entry.run.state) && entry.run.state !== 'up' && ms > 0 && !this.closed && !signal?.aborted) {
|
|
301
390
|
await new Promise((resolve) => {
|
|
302
391
|
const listener = (event) => {
|
|
303
|
-
|
|
392
|
+
// a service run is there once it is up
|
|
393
|
+
if (event.type === 'state' && (isFinal(event.state) || event.state === 'up')) {
|
|
304
394
|
done();
|
|
305
395
|
}
|
|
306
396
|
};
|
|
@@ -335,10 +425,19 @@ class RunRegistry {
|
|
|
335
425
|
async close() {
|
|
336
426
|
this.closed = true;
|
|
337
427
|
for (const entry of this.queue.splice(0)) {
|
|
338
|
-
this.
|
|
428
|
+
if (this.maySleep(entry) && this.options.store) {
|
|
429
|
+
// a woken service run sleeps again, the next start keeps it
|
|
430
|
+
this.sleep(entry);
|
|
431
|
+
}
|
|
432
|
+
else {
|
|
433
|
+
this.finish(entry, 'cancelled');
|
|
434
|
+
}
|
|
339
435
|
}
|
|
340
436
|
for (const entry of this.entries.values()) {
|
|
341
|
-
if (entry.run.state === '
|
|
437
|
+
if (entry.run.state === 'asleep') {
|
|
438
|
+
this.clearTimer(entry);
|
|
439
|
+
}
|
|
440
|
+
else if (entry.run.state === 'awaiting-approval') {
|
|
342
441
|
if (this.options.store) {
|
|
343
442
|
// the record keeps it waiting, the next start takes it back
|
|
344
443
|
this.clearTimer(entry);
|
|
@@ -353,11 +452,15 @@ class RunRegistry {
|
|
|
353
452
|
done();
|
|
354
453
|
}
|
|
355
454
|
for (const entry of this.entries.values()) {
|
|
356
|
-
if (entry.run.state === 'running') {
|
|
455
|
+
if (entry.run.state === 'running' || entry.run.state === 'up') {
|
|
456
|
+
// a service run that may sleep is asleep in its record, the next request after the
|
|
457
|
+
// start wakes it
|
|
458
|
+
entry.sleeping = this.maySleep(entry) && !!this.options.store;
|
|
357
459
|
entry.abort.abort();
|
|
358
460
|
}
|
|
359
461
|
}
|
|
360
|
-
while (this.running > 0
|
|
462
|
+
while (this.running > 0 ||
|
|
463
|
+
[...this.entries.values()].some((entry) => entry.run.state === 'running' || entry.run.state === 'up')) {
|
|
361
464
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
362
465
|
}
|
|
363
466
|
// the runs that were stopped are written down before the server is gone
|
|
@@ -388,6 +491,8 @@ class RunRegistry {
|
|
|
388
491
|
copy(run) {
|
|
389
492
|
return {
|
|
390
493
|
...run,
|
|
494
|
+
...(run.up ? { up: JSON.parse(JSON.stringify(run.up)) } : {}),
|
|
495
|
+
...(run.stoppedBy && typeof run.stoppedBy === 'object' ? { stoppedBy: { ...run.stoppedBy } } : {}),
|
|
391
496
|
tasks: [...run.tasks],
|
|
392
497
|
source: { ...run.source },
|
|
393
498
|
options: JSON.parse(JSON.stringify(run.options)),
|
|
@@ -415,6 +520,76 @@ class RunRegistry {
|
|
|
415
520
|
throw new QuotaError(`you have ${limits.queued} runs in the queue, wait for one to start or cancel one`);
|
|
416
521
|
}
|
|
417
522
|
}
|
|
523
|
+
// Service runs that are starting or up, of the caller and of everybody, against what may
|
|
524
|
+
// be. One that is asleep does not count: it asks for a place when it wakes (visit). The runs
|
|
525
|
+
// of a caller with one name take one place: the newest replaces the others once it is up,
|
|
526
|
+
// and an older one that gets up after it goes (markUp), so a new run of a name the caller
|
|
527
|
+
// has does not need another. Several of one name that are still starting share it too:
|
|
528
|
+
// what they take while they start is held by runs.limits.running and maxConcurrent.
|
|
529
|
+
checkUpLimits(owner, name) {
|
|
530
|
+
// what a caller keeps asleep is held by a limit of its own: a run that sleeps may keep the
|
|
531
|
+
// volumes of its services. A run of a name the caller has a run of (asleep, up or
|
|
532
|
+
// starting) replaces it, it is not one more.
|
|
533
|
+
const asleep = this.options.up?.asleep;
|
|
534
|
+
if (asleep !== undefined) {
|
|
535
|
+
const open = [...this.entries.values()].filter((entry) => entry.run.up && !isFinal(entry.run.state) && this.owns(owner, entry));
|
|
536
|
+
const sleeping = new Set(open.filter((entry) => entry.run.state === 'asleep').map((entry) => entry.run.up?.name));
|
|
537
|
+
if (!open.some((entry) => entry.run.up?.name === name) && sleeping.size >= asleep) {
|
|
538
|
+
throw new QuotaError(`you have ${asleep} service runs asleep, stop one or wait for its end`);
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
const full = this.upLimitReached(owner, name);
|
|
542
|
+
if (full === 'caller') {
|
|
543
|
+
throw new QuotaError(`you have ${this.options.up?.perCaller} service runs starting or up, stop one or wait for its end`);
|
|
544
|
+
}
|
|
545
|
+
if (full === 'server') {
|
|
546
|
+
throw new RunError(`the server has ${this.options.up?.total} service runs starting or up, try again later`);
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
// Whose limit a service run of the caller with this name would go over, if any
|
|
550
|
+
upLimitReached(owner, name) {
|
|
551
|
+
const limits = this.options.up;
|
|
552
|
+
if (!limits) {
|
|
553
|
+
return undefined;
|
|
554
|
+
}
|
|
555
|
+
const places = new Set();
|
|
556
|
+
const mine = new Set();
|
|
557
|
+
for (const entry of this.entries.values()) {
|
|
558
|
+
if (!entry.run.up || isFinal(entry.run.state) || entry.run.state === 'asleep') {
|
|
559
|
+
continue;
|
|
560
|
+
}
|
|
561
|
+
places.add(JSON.stringify([entry.run.owner.iss, entry.run.owner.sub, entry.run.up.name]));
|
|
562
|
+
if (this.owns(owner, entry)) {
|
|
563
|
+
mine.add(entry.run.up.name);
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
if (mine.has(name)) {
|
|
567
|
+
return undefined;
|
|
568
|
+
}
|
|
569
|
+
if (mine.size >= limits.perCaller) {
|
|
570
|
+
return 'caller';
|
|
571
|
+
}
|
|
572
|
+
if (places.size >= limits.total) {
|
|
573
|
+
return 'server';
|
|
574
|
+
}
|
|
575
|
+
return undefined;
|
|
576
|
+
}
|
|
577
|
+
// A host is one run's, or a run's that replaces it (same owner and name) once it is up:
|
|
578
|
+
// two labels that come out the same are refused, they would reach each other's services.
|
|
579
|
+
checkHosts(owner, up) {
|
|
580
|
+
const hosts = new Set(up.endpoints.map((endpoint) => endpoint.host));
|
|
581
|
+
for (const entry of this.entries.values()) {
|
|
582
|
+
if (isFinal(entry.run.state) || !entry.run.up) {
|
|
583
|
+
continue;
|
|
584
|
+
}
|
|
585
|
+
const taken = entry.run.up.endpoints.find((endpoint) => hosts.has(endpoint.host));
|
|
586
|
+
if (taken && !(this.owns(owner, entry) && entry.run.up.name === up.name)) {
|
|
587
|
+
const error = new RunError(`the address ${taken.host} is another service run's, pick another name`);
|
|
588
|
+
error.status = 409;
|
|
589
|
+
throw error;
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
}
|
|
418
593
|
runningOf(owner) {
|
|
419
594
|
return [...this.entries.values()].filter((entry) => entry.run.state === 'running' && this.owns(owner, entry)).length;
|
|
420
595
|
}
|
|
@@ -424,20 +599,30 @@ class RunRegistry {
|
|
|
424
599
|
return (source.type === 'sandbox' &&
|
|
425
600
|
[...this.entries.values()].some((entry) => entry.run.state === 'running' && onSandbox(entry.run, source.id)));
|
|
426
601
|
}
|
|
602
|
+
// Whether a run that executes holds what the run would (RunExecutor.exclusive)
|
|
603
|
+
held(run) {
|
|
604
|
+
const exclusive = this.options.executor.exclusive?.bind(this.options.executor);
|
|
605
|
+
const key = exclusive?.(run);
|
|
606
|
+
return (key !== undefined &&
|
|
607
|
+
[...this.entries.values()].some((entry) => entry.run.state === 'running' && exclusive?.(entry.run) === key));
|
|
608
|
+
}
|
|
427
609
|
// The first run in the queue whose caller has a place left: one caller with many runs
|
|
428
610
|
// does not keep the others from the machine. A run on a sandbox another run works on
|
|
429
|
-
// waits for it.
|
|
611
|
+
// waits for it, as does one that needs what an executing run holds.
|
|
430
612
|
dispatch() {
|
|
431
613
|
while (this.running < this.maxConcurrent) {
|
|
432
|
-
const perCaller =
|
|
433
|
-
const index = this.queue.findIndex((entry) => this.runningOf(entry.run.owner) < perCaller && !this.sandboxBusy(entry.run));
|
|
614
|
+
const perCaller = this.options.limits?.running ?? Infinity;
|
|
615
|
+
const index = this.queue.findIndex((entry) => this.runningOf(entry.run.owner) < perCaller && !this.sandboxBusy(entry.run) && !this.held(entry.run));
|
|
434
616
|
if (index < 0) {
|
|
435
617
|
return;
|
|
436
618
|
}
|
|
437
619
|
const [entry] = this.queue.splice(index, 1);
|
|
438
620
|
this.running++;
|
|
439
621
|
this.execute(entry).finally(() => {
|
|
440
|
-
|
|
622
|
+
// a service run that is up gave its place back already
|
|
623
|
+
if (!entry.released) {
|
|
624
|
+
this.running--;
|
|
625
|
+
}
|
|
441
626
|
this.dispatch();
|
|
442
627
|
});
|
|
443
628
|
}
|
|
@@ -446,27 +631,209 @@ class RunRegistry {
|
|
|
446
631
|
entry.run.state = 'running';
|
|
447
632
|
entry.run.startedAt = this.timestamp();
|
|
448
633
|
this.record(entry, { type: 'state', state: 'running' });
|
|
634
|
+
// a service run gets this long to be up, then its ttl counts
|
|
449
635
|
const limit = this.options.maxDuration;
|
|
450
|
-
|
|
636
|
+
entry.durationTimer = limit
|
|
451
637
|
? setTimeout(() => {
|
|
452
638
|
entry.timedOut = true;
|
|
453
639
|
entry.abort.abort();
|
|
454
640
|
}, limit)
|
|
455
641
|
: undefined;
|
|
456
|
-
|
|
642
|
+
entry.durationTimer?.unref();
|
|
457
643
|
try {
|
|
458
644
|
await this.options.executor.execute(this.copy(entry.run), {
|
|
459
645
|
emit: (event) => this.record(entry, event),
|
|
460
646
|
signal: entry.abort.signal,
|
|
647
|
+
...(entry.run.up ? { up: (targets) => this.markUp(entry, targets) } : {}),
|
|
461
648
|
});
|
|
462
|
-
|
|
649
|
+
if (entry.sleeping) {
|
|
650
|
+
this.sleep(entry);
|
|
651
|
+
}
|
|
652
|
+
else if (entry.run.upAt) {
|
|
653
|
+
// services stay until they are stopped: ones that stopped on their own failed
|
|
654
|
+
this.endServices(entry, 'the services stopped on their own');
|
|
655
|
+
}
|
|
656
|
+
else {
|
|
657
|
+
this.finish(entry, entry.timedOut ? 'failed' : entry.abort.signal.aborted ? 'cancelled' : 'succeeded', entry.timedOut ? this.tooLong() : undefined);
|
|
658
|
+
}
|
|
463
659
|
}
|
|
464
660
|
catch (e) {
|
|
465
|
-
const
|
|
466
|
-
|
|
661
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
662
|
+
if (entry.sleeping) {
|
|
663
|
+
this.sleep(entry);
|
|
664
|
+
}
|
|
665
|
+
else if (entry.run.upAt) {
|
|
666
|
+
this.endServices(entry, message);
|
|
667
|
+
}
|
|
668
|
+
else {
|
|
669
|
+
const cancelled = entry.abort.signal.aborted && !entry.timedOut;
|
|
670
|
+
this.finish(entry, cancelled ? 'cancelled' : 'failed', entry.timedOut ? this.tooLong() : cancelled ? undefined : message);
|
|
671
|
+
}
|
|
467
672
|
}
|
|
468
673
|
finally {
|
|
469
|
-
clearTimeout(
|
|
674
|
+
clearTimeout(entry.durationTimer);
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
// The services of a service run are ready: its ttl counts from now, its place of
|
|
678
|
+
// maxConcurrent goes to the next run, and an older run of its owner with the same name
|
|
679
|
+
// stops, so its hosts are this run's.
|
|
680
|
+
markUp(entry, targets) {
|
|
681
|
+
const spec = entry.run.up;
|
|
682
|
+
if (!spec || entry.run.state !== 'running' || entry.abort.signal.aborted) {
|
|
683
|
+
return;
|
|
684
|
+
}
|
|
685
|
+
clearTimeout(entry.durationTimer);
|
|
686
|
+
entry.durationTimer = undefined;
|
|
687
|
+
entry.targets = targets.map((target) => ({ ...target, address: { ...target.address } }));
|
|
688
|
+
entry.run.state = 'up';
|
|
689
|
+
// one reading of the clock, so the ttl is exactly expiresAt - upAt
|
|
690
|
+
const now = this.now();
|
|
691
|
+
// the ttl counts from the first time, not from a wake
|
|
692
|
+
if (!entry.run.upAt) {
|
|
693
|
+
entry.run.upAt = new Date(now).toISOString();
|
|
694
|
+
entry.run.expiresAt = new Date(now + spec.ttl).toISOString();
|
|
695
|
+
}
|
|
696
|
+
if (spec.sleepTtl) {
|
|
697
|
+
entry.run.sleepsAt = new Date(now + spec.sleepTtl).toISOString();
|
|
698
|
+
}
|
|
699
|
+
this.record(entry, { type: 'state', state: 'up' });
|
|
700
|
+
this.expireLater(entry);
|
|
701
|
+
entry.released = true;
|
|
702
|
+
this.running--;
|
|
703
|
+
const entries = [...this.entries.values()];
|
|
704
|
+
for (const other of entries.slice(0, entries.indexOf(entry))) {
|
|
705
|
+
if (other.run.up?.name === spec.name &&
|
|
706
|
+
this.owns(entry.run.owner, other) &&
|
|
707
|
+
!isFinal(other.run.state) &&
|
|
708
|
+
other.run.state !== 'awaiting-approval') {
|
|
709
|
+
other.run.stoppedBy = 'replaced';
|
|
710
|
+
this.stop(other, `replaced by ${entry.run.id}`);
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
// A newer run of the name that is up or asleep already (this one waited for an approval
|
|
714
|
+
// while it got up) is what the owner started last: this one goes, so the runs of a name,
|
|
715
|
+
// which take one place of runs.up.limits, never hold more than one set of services.
|
|
716
|
+
const newer = entries
|
|
717
|
+
.slice(entries.indexOf(entry) + 1)
|
|
718
|
+
.find((other) => other.run.up?.name === spec.name &&
|
|
719
|
+
this.owns(entry.run.owner, other) &&
|
|
720
|
+
(other.run.state === 'up' || other.run.state === 'asleep'));
|
|
721
|
+
if (newer) {
|
|
722
|
+
entry.run.stoppedBy = 'replaced';
|
|
723
|
+
this.stop(entry, `replaced by ${newer.run.id}`);
|
|
724
|
+
}
|
|
725
|
+
this.dispatch();
|
|
726
|
+
}
|
|
727
|
+
// Stops a service run at its expiry, and puts one that is up to sleep at its sleepsAt. A
|
|
728
|
+
// timer can't wait longer than about 24 days, so a longer one looks again then.
|
|
729
|
+
expireLater(entry) {
|
|
730
|
+
this.clearTimer(entry);
|
|
731
|
+
const expiresAt = Date.parse(entry.run.expiresAt ?? '');
|
|
732
|
+
const sleepsAt = entry.run.state === 'up' ? Date.parse(entry.run.sleepsAt ?? '') : NaN;
|
|
733
|
+
const next = Number.isFinite(sleepsAt) ? Math.min(expiresAt, sleepsAt) : expiresAt;
|
|
734
|
+
entry.timer = setTimeout(() => {
|
|
735
|
+
var _a;
|
|
736
|
+
if (isFinal(entry.run.state)) {
|
|
737
|
+
return;
|
|
738
|
+
}
|
|
739
|
+
if (Date.parse(entry.run.expiresAt ?? '') <= this.now()) {
|
|
740
|
+
(_a = entry.run).stoppedBy ?? (_a.stoppedBy = 'ttl');
|
|
741
|
+
entry.sleeping = false;
|
|
742
|
+
if (entry.run.state === 'asleep') {
|
|
743
|
+
this.finish(entry, 'succeeded');
|
|
744
|
+
}
|
|
745
|
+
else {
|
|
746
|
+
this.stop(entry);
|
|
747
|
+
}
|
|
748
|
+
return;
|
|
749
|
+
}
|
|
750
|
+
if (entry.run.state === 'up' && Date.parse(entry.run.sleepsAt ?? '') <= this.now()) {
|
|
751
|
+
entry.sleeping = true;
|
|
752
|
+
entry.abort.abort();
|
|
753
|
+
return;
|
|
754
|
+
}
|
|
755
|
+
this.expireLater(entry);
|
|
756
|
+
}, Math.max(0, Math.min(next - this.now(), 2 ** 31 - 1)));
|
|
757
|
+
entry.timer.unref();
|
|
758
|
+
}
|
|
759
|
+
// Whether a service run sleeps instead of ending: it has a sleepTtl, its ttl has not
|
|
760
|
+
// ended, and it starts from a commit (the files of a sandbox change under it).
|
|
761
|
+
maySleep(entry) {
|
|
762
|
+
return (!!entry.run.up?.sleepTtl &&
|
|
763
|
+
!!entry.run.upAt &&
|
|
764
|
+
entry.run.source.type === 'ref' &&
|
|
765
|
+
Date.parse(entry.run.expiresAt ?? '') > this.now());
|
|
766
|
+
}
|
|
767
|
+
// The services stopped, the run waits for a request to wake it, or for its end.
|
|
768
|
+
sleep(entry) {
|
|
769
|
+
entry.sleeping = false;
|
|
770
|
+
entry.targets = undefined;
|
|
771
|
+
entry.run.state = 'asleep';
|
|
772
|
+
delete entry.run.sleepsAt;
|
|
773
|
+
this.record(entry, { type: 'state', state: 'asleep' });
|
|
774
|
+
this.expireLater(entry);
|
|
775
|
+
}
|
|
776
|
+
/**
|
|
777
|
+
* A request to an endpoint of a service run: one that is up stays up for its sleepTtl from
|
|
778
|
+
* now, one that is asleep is started again, if `mayWake` finds its owner may still run it
|
|
779
|
+
* (else it ends) and there is a place of runs.up.limits for it (else it sleeps on, and the
|
|
780
|
+
* next request asks again). Answers why it was not woken, or undefined.
|
|
781
|
+
*/
|
|
782
|
+
visit(id, mayWake) {
|
|
783
|
+
const entry = this.entries.get(id);
|
|
784
|
+
if (!entry?.run.up) {
|
|
785
|
+
return undefined;
|
|
786
|
+
}
|
|
787
|
+
const sleepTtl = entry.run.up.sleepTtl;
|
|
788
|
+
if (entry.run.state === 'up' && sleepTtl) {
|
|
789
|
+
// only kept in memory: a server that stops puts the run to sleep anyway
|
|
790
|
+
entry.run.sleepsAt = new Date(this.now() + sleepTtl).toISOString();
|
|
791
|
+
return undefined;
|
|
792
|
+
}
|
|
793
|
+
if (entry.run.state !== 'asleep' || this.closed) {
|
|
794
|
+
return undefined;
|
|
795
|
+
}
|
|
796
|
+
if (!this.maySleep(entry)) {
|
|
797
|
+
this.expireLater(entry);
|
|
798
|
+
return 'its time ran out';
|
|
799
|
+
}
|
|
800
|
+
const refused = mayWake?.(this.copy(entry.run));
|
|
801
|
+
if (refused) {
|
|
802
|
+
this.finish(entry, 'failed', refused);
|
|
803
|
+
return refused;
|
|
804
|
+
}
|
|
805
|
+
const full = this.upLimitReached(entry.run.owner, entry.run.up.name);
|
|
806
|
+
if (full === 'caller') {
|
|
807
|
+
return 'its owner has as many service runs starting or up as they may, it wakes once one of them stops or sleeps';
|
|
808
|
+
}
|
|
809
|
+
if (full === 'server') {
|
|
810
|
+
return 'the server has as many service runs starting or up as it may, it wakes once one of them stops or sleeps';
|
|
811
|
+
}
|
|
812
|
+
this.clearTimer(entry);
|
|
813
|
+
entry.abort = new AbortController();
|
|
814
|
+
entry.released = false;
|
|
815
|
+
entry.timedOut = false;
|
|
816
|
+
entry.run.wakes = (entry.run.wakes ?? 0) + 1;
|
|
817
|
+
entry.run.state = 'queued';
|
|
818
|
+
this.record(entry, { type: 'state', state: 'queued' });
|
|
819
|
+
this.queue.push(entry);
|
|
820
|
+
this.dispatch();
|
|
821
|
+
return undefined;
|
|
822
|
+
}
|
|
823
|
+
// The end of a run that was up: stopped by somebody, its ttl or a newer run it succeeded,
|
|
824
|
+
// stopped by the server going away it failed, as its record says after a restart.
|
|
825
|
+
endServices(entry, failure) {
|
|
826
|
+
if (entry.timedOut) {
|
|
827
|
+
this.finish(entry, 'failed', this.tooLong());
|
|
828
|
+
}
|
|
829
|
+
else if (!entry.abort.signal.aborted) {
|
|
830
|
+
this.finish(entry, 'failed', failure);
|
|
831
|
+
}
|
|
832
|
+
else if (entry.run.stoppedBy) {
|
|
833
|
+
this.finish(entry, 'succeeded');
|
|
834
|
+
}
|
|
835
|
+
else {
|
|
836
|
+
this.finish(entry, 'failed', 'the server stopped while the services were up, start them again');
|
|
470
837
|
}
|
|
471
838
|
}
|
|
472
839
|
tooLong() {
|
|
@@ -474,6 +841,8 @@ class RunRegistry {
|
|
|
474
841
|
}
|
|
475
842
|
finish(entry, state, error) {
|
|
476
843
|
this.clearTimer(entry);
|
|
844
|
+
clearTimeout(entry.durationTimer);
|
|
845
|
+
entry.targets = undefined;
|
|
477
846
|
entry.run.state = state;
|
|
478
847
|
entry.run.finishedAt = this.timestamp();
|
|
479
848
|
if (error) {
|
|
@@ -488,9 +857,23 @@ class RunRegistry {
|
|
|
488
857
|
}
|
|
489
858
|
this.record(entry, { type: 'state', state, ...(error ? { error } : {}) }, true);
|
|
490
859
|
entry.listeners.clear();
|
|
860
|
+
try {
|
|
861
|
+
this.options.onFinished?.(this.copy(entry.run), { ...entry.keys });
|
|
862
|
+
}
|
|
863
|
+
catch (e) {
|
|
864
|
+
this.options.log?.(`run ${entry.run.id} ended, but what it passed was not kept: ${e instanceof Error ? e.message : e}`);
|
|
865
|
+
}
|
|
491
866
|
this.evict();
|
|
492
867
|
}
|
|
493
868
|
record(entry, recorded, force = false) {
|
|
869
|
+
if (recorded.type === 'key') {
|
|
870
|
+
// a task name is the executor's; the first key it reports for a task is the one
|
|
871
|
+
// computed before anything of the run had run
|
|
872
|
+
if (!(recorded.task in entry.keys) && Object.keys(entry.keys).length < MAX_KEYED_TASKS) {
|
|
873
|
+
entry.keys[recorded.task] = recorded.key;
|
|
874
|
+
}
|
|
875
|
+
return;
|
|
876
|
+
}
|
|
494
877
|
const event = recorded.type === 'task' ? bounded(recorded) : recorded;
|
|
495
878
|
if (event.type === 'log') {
|
|
496
879
|
this.keepTail(entry, event.task, event.text);
|
|
@@ -572,7 +955,7 @@ class RunRegistry {
|
|
|
572
955
|
}
|
|
573
956
|
// what a restart must not lose: the runs that wait, and how a run ended. A running
|
|
574
957
|
// run is not kept up to date, it fails with the server.
|
|
575
|
-
if (event.type === 'state' || event.type === 'approval') {
|
|
958
|
+
if (event.type === 'state' || event.type === 'approval' || event.type === 'extended') {
|
|
576
959
|
this.persist(entry);
|
|
577
960
|
}
|
|
578
961
|
}
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.sandboxPodMain = exports.decodePodError = exports.encodeEditOps = exports.INCOMING_REF = exports.INCOMING = void 0;
|
|
4
|
+
const crypto_1 = require("crypto");
|
|
4
5
|
const promises_1 = require("fs/promises");
|
|
5
6
|
const path_1 = require("path");
|
|
6
7
|
const url_1 = require("url");
|
|
7
8
|
const git_checkout_1 = require("./git-checkout");
|
|
8
9
|
const sandbox_files_1 = require("./sandbox-files");
|
|
9
10
|
const sandbox_storage_1 = require("./sandbox-storage");
|
|
11
|
+
const tar_1 = require("../utils/tar");
|
|
10
12
|
// On a cluster a sandbox is a claim, and what the server does with its files runs in a
|
|
11
13
|
// short pod that mounts it (see KubernetesSandboxStorage): `hammerkit server-sandbox
|
|
12
14
|
// <directory>` reads one request on its standard input, does it with the code of a host
|
|
@@ -91,6 +93,32 @@ async function answer(storage, directory, request, out) {
|
|
|
91
93
|
}
|
|
92
94
|
return send({ files: plan.files, bytes: plan.bytes });
|
|
93
95
|
}
|
|
96
|
+
case 'archive': {
|
|
97
|
+
// written beside the sandbox on the claim, sent, and removed
|
|
98
|
+
const into = (0, path_1.join)(directory, `${request.id}-archive-${(0, crypto_1.randomBytes)(6).toString('hex')}`);
|
|
99
|
+
try {
|
|
100
|
+
await storage.exportTree(key, request.tree, into, AbortSignal.timeout(10 * 60000));
|
|
101
|
+
await write(out, JSON.stringify({ ok: true, result: null, stream: true }) + '\n');
|
|
102
|
+
const pack = (0, tar_1.packArchive)({ cwd: into, portable: true }, ['.']);
|
|
103
|
+
try {
|
|
104
|
+
for await (const chunk of pack) {
|
|
105
|
+
await write(out, chunk);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
catch (e) {
|
|
109
|
+
throw new AnswerStarted(e instanceof Error ? e.message : String(e));
|
|
110
|
+
}
|
|
111
|
+
finally {
|
|
112
|
+
// a reader that went away (over its size limit) leaves the pack reading files
|
|
113
|
+
// that are removed next: what fails then is no answer of anybody's
|
|
114
|
+
pack.on('error', () => undefined);
|
|
115
|
+
}
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
finally {
|
|
119
|
+
await (0, promises_1.rm)(into, { recursive: true, force: true });
|
|
120
|
+
}
|
|
121
|
+
}
|
|
94
122
|
case 'raw': {
|
|
95
123
|
const raw = await files().raw(request.path);
|
|
96
124
|
const header = {
|