hammerkit 1.27.0-next.3 → 1.27.0-next.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/build.schema.json +188 -0
  2. package/dist/cache/backends/local-cache-backend.js +6 -0
  3. package/dist/cache/backends/registry-cache-backend.js +2 -2
  4. package/dist/cache/explain.js +17 -0
  5. package/dist/cli.js +2 -0
  6. package/dist/docker/item-network.js +13 -5
  7. package/dist/executer/deadline.js +17 -3
  8. package/dist/executer/deferred-restore.js +36 -0
  9. package/dist/executer/docker-service.js +30 -8
  10. package/dist/executer/docker-task.js +21 -1
  11. package/dist/executer/event-cache.js +20 -2
  12. package/dist/executer/execute-work-service.js +2 -0
  13. package/dist/executer/execute-work-task.js +10 -3
  14. package/dist/executer/execution-steps.js +25 -4
  15. package/dist/executer/reset-work-tree.js +4 -0
  16. package/dist/executer/scheduler/enqueue-next.js +57 -6
  17. package/dist/executer/scheduler/state-key.js +42 -5
  18. package/dist/executer/watch-loop.js +4 -1
  19. package/dist/harness/connect.js +20 -6
  20. package/dist/harness/mcp-command.js +22 -6
  21. package/dist/harness/mcp.js +63 -114
  22. package/dist/harness/opencode.js +158 -50
  23. package/dist/harness/pi.js +29 -0
  24. package/dist/harness/sandbox-client.js +168 -17
  25. package/dist/harness/service-tools.js +141 -0
  26. package/dist/harness/task-tools.js +221 -0
  27. package/dist/kubernetes/apply.js +27 -16
  28. package/dist/kubernetes/await-running-state.js +59 -13
  29. package/dist/kubernetes/claim-layout.js +71 -0
  30. package/dist/kubernetes/cluster-state.js +245 -0
  31. package/dist/kubernetes/ensure-http-route.js +1 -1
  32. package/dist/kubernetes/ensure-kubernetes-deployment-exists.js +35 -12
  33. package/dist/kubernetes/ensure-kubernetes-persistent-volume-claim-exists.js +1 -1
  34. package/dist/kubernetes/ensure-kubernetes-service-exists.js +6 -9
  35. package/dist/kubernetes/ensure-persistent-data.js +30 -20
  36. package/dist/kubernetes/follow-pod-logs.js +182 -0
  37. package/dist/kubernetes/get-service-ip.js +2 -2
  38. package/dist/kubernetes/job-placement.js +36 -12
  39. package/dist/kubernetes/job-secrets.js +33 -19
  40. package/dist/kubernetes/kept-claims.js +22 -10
  41. package/dist/kubernetes/network-policy.js +85 -27
  42. package/dist/kubernetes/remove-persistent-data.js +3 -1
  43. package/dist/kubernetes/resources.js +25 -3
  44. package/dist/kubernetes/restore-kubernetes-data.js +9 -3
  45. package/dist/kubernetes/sandbox-runner.js +628 -0
  46. package/dist/kubernetes/secrets.js +7 -11
  47. package/dist/kubernetes/volumes.js +6 -3
  48. package/dist/main.js +2 -0
  49. package/dist/network/egress-proxy.js +45 -6
  50. package/dist/optimizer/calculate-checksum.js +3 -2
  51. package/dist/optimizer/check-key.js +94 -0
  52. package/dist/optimizer/check-keys.js +64 -0
  53. package/dist/optimizer/get-work-cache-stats.js +17 -3
  54. package/dist/optimizer/work-task-cache-description.js +51 -1
  55. package/dist/planner/utils/append-work-dependencies.js +10 -5
  56. package/dist/planner/utils/append-work-service.js +19 -1
  57. package/dist/planner/utils/append-work-task.js +18 -3
  58. package/dist/planner/utils/get-work-runtime.js +6 -1
  59. package/dist/planner/utils/normalize-path.js +19 -1
  60. package/dist/planner/utils/parse-work-mount.js +7 -1
  61. package/dist/planner/utils/parse-work-port.js +4 -1
  62. package/dist/planner/utils/task-arguments.js +3 -0
  63. package/dist/planner/validate.js +22 -0
  64. package/dist/planner/work-network.js +3 -1
  65. package/dist/planner/work-runtime-aggregate.js +32 -0
  66. package/dist/planner/work-runtime-docker.js +2 -1
  67. package/dist/planner/work-runtime-kubernetes.js +124 -48
  68. package/dist/planner/work-runtime-local.js +2 -1
  69. package/dist/planner/work-task.js +6 -1
  70. package/dist/program.js +31 -10
  71. package/dist/remote/device-login.js +7 -2
  72. package/dist/remote/host-config.js +69 -3
  73. package/dist/remote/remote-commands.js +30 -7
  74. package/dist/remote/remote-work-commands.js +426 -45
  75. package/dist/remote/server-client.js +15 -2
  76. package/dist/remote/token-source.js +64 -26
  77. package/dist/run-program.js +9 -0
  78. package/dist/schema/build-file-base-task-schema.js +8 -1
  79. package/dist/schema/build-file-container-service-schema.js +12 -0
  80. package/dist/schema/build-file-container-task-schema.js +16 -1
  81. package/dist/schema/build-file-include-schema.js +18 -4
  82. package/dist/schema/build-file-kubernetes-service-schema.js +2 -0
  83. package/dist/schema/build-file-task-schema.js +5 -1
  84. package/dist/schema/capabilities-schema.js +56 -0
  85. package/dist/schema/reference-parser.js +120 -11
  86. package/dist/schema/schema-parser.js +92 -41
  87. package/dist/schema/work-scope-parser.js +33 -10
  88. package/dist/secrets/command-secret-provider.js +3 -1
  89. package/dist/secrets/github-app-token-command.js +3 -1
  90. package/dist/secrets/secret-catalog.js +43 -19
  91. package/dist/server/accounts-api.js +3 -1
  92. package/dist/server/approvals-api.js +1 -1
  93. package/dist/server/checks-api.js +83 -0
  94. package/dist/server/checks.js +116 -0
  95. package/dist/server/definitions-api.js +66 -3
  96. package/dist/server/endpoints.js +739 -0
  97. package/dist/server/executor.js +57 -5
  98. package/dist/server/kubernetes-backend.js +367 -65
  99. package/dist/server/kubernetes-sandbox-storage.js +48 -3
  100. package/dist/server/oauth.js +12 -1
  101. package/dist/server/plan.js +188 -103
  102. package/dist/server/policy.js +32 -2
  103. package/dist/server/repo-access.js +5 -0
  104. package/dist/server/run-plan.js +50 -19
  105. package/dist/server/run-request.js +58 -3
  106. package/dist/server/run-worker.js +245 -12
  107. package/dist/server/runs.js +410 -27
  108. package/dist/server/sandbox-pod.js +28 -0
  109. package/dist/server/sandbox-storage.js +23 -0
  110. package/dist/server/sandboxes-api.js +10 -4
  111. package/dist/server/sandboxes.js +8 -0
  112. package/dist/server/server-command.js +39 -20
  113. package/dist/server/server-config.js +255 -14
  114. package/dist/server/server.js +269 -15
  115. package/dist/server/service-runs.js +82 -0
  116. package/dist/utils/abort-on-signals.js +46 -6
  117. package/dist/utils/tar.js +36 -0
  118. package/dist/utils/terminal-prompt.js +53 -0
  119. package/package.json +1 -1
@@ -30,6 +30,8 @@ exports.SANDBOX_ID_PATTERN = /^sbx_[A-Za-z0-9_-]{16}$/;
30
30
  function onSandbox(run, sandbox) {
31
31
  return run.source.type === 'sandbox' && run.source.id === sandbox;
32
32
  }
33
+ // the tasks of a run whose check keys are kept, a build file has fewer
34
+ const MAX_KEYED_TASKS = 1000;
33
35
  // how much of the end of a task's output its result carries
34
36
  const TAIL_LINES = 20;
35
37
  const TAIL_LINE_LENGTH = 1000;
@@ -100,6 +102,7 @@ class RunRegistry {
100
102
  bytes: 0,
101
103
  tails: new Map(Object.entries(run.results ?? {}).map(([task, result]) => [task, result.lastLog])),
102
104
  saving: Promise.resolve(),
105
+ keys: {},
103
106
  };
104
107
  this.entries.set(run.id, entry);
105
108
  if (run.state === 'awaiting-approval') {
@@ -112,9 +115,27 @@ class RunRegistry {
112
115
  this.finish(entry, 'rejected', 'nobody approved it in time');
113
116
  }
114
117
  }
118
+ else if (this.maySleep(entry) && ['queued', 'running', 'up', 'asleep'].includes(run.state)) {
119
+ // the services went with the process that held them, the next request wakes them
120
+ if (run.state === 'queued' || run.state === 'running' || run.state === 'up') {
121
+ this.sleep(entry);
122
+ }
123
+ else {
124
+ this.expireLater(entry);
125
+ }
126
+ }
127
+ else if (run.state === 'asleep') {
128
+ // its time ran out while the server was down: it ends now, and gives its place back
129
+ run.stoppedBy ?? (run.stoppedBy = 'ttl');
130
+ this.finish(entry, 'succeeded');
131
+ }
115
132
  else if (run.state === 'queued' || run.state === 'running') {
116
133
  this.finish(entry, 'failed', `the server stopped while the run was ${run.state}, submit it again`);
117
134
  }
135
+ else if (run.state === 'up') {
136
+ // the services went with the process that held them
137
+ this.finish(entry, 'failed', 'the server stopped while the services were up, start them again');
138
+ }
118
139
  }
119
140
  this.evict();
120
141
  }
@@ -129,10 +150,18 @@ class RunRegistry {
129
150
  throw new RunError('too many runs are waiting for an approval');
130
151
  }
131
152
  this.checkLimits(owner, waiting);
153
+ if (spec.up) {
154
+ this.checkUpLimits(owner, spec.up.name);
155
+ this.checkHosts(owner, spec.up);
156
+ }
132
157
  const run = {
133
158
  ...spec,
134
159
  id: `run_${(0, crypto_1.randomBytes)(12).toString('base64url')}`,
135
- owner: { iss: owner.iss, sub: owner.sub },
160
+ owner: {
161
+ iss: owner.iss,
162
+ sub: owner.sub,
163
+ ...(spec.up?.sleepTtl ? { groups: [...(owner.groups ?? [])], claims: { ...(owner.claims ?? {}) } } : {}),
164
+ },
136
165
  state: waiting ? 'awaiting-approval' : 'queued',
137
166
  createdAt: this.timestamp(),
138
167
  };
@@ -145,6 +174,7 @@ class RunRegistry {
145
174
  bytes: 0,
146
175
  tails: new Map(),
147
176
  saving: Promise.resolve(),
177
+ keys: {},
148
178
  };
149
179
  this.entries.set(run.id, entry);
150
180
  if (waiting) {
@@ -179,30 +209,79 @@ class RunRegistry {
179
209
  .map((entry) => this.copy(entry.run))
180
210
  .reverse();
181
211
  }
182
- // A queued run is cancelled at once; a running one when its executor stops.
212
+ // A queued run is cancelled at once; a running one when its executor stops. A service run
213
+ // that is up stops its services, and says who stopped it.
183
214
  cancel(owner, id, canUse) {
184
215
  const entry = this.find(owner, id, canUse);
185
216
  if (!entry) {
186
217
  return undefined;
187
218
  }
188
- this.stop(entry);
219
+ this.stop(entry, undefined, { iss: owner.iss, sub: owner.sub });
189
220
  return this.copy(entry.run);
190
221
  }
222
+ /**
223
+ * Moves the end of a service run that is up (or asleep) to `ttl` (ms) from now, at most
224
+ * the server's maxTtl, and never sooner than it was. Its owner may, and whoever `mayOpen`
225
+ * says may open its endpoints.
226
+ */
227
+ extend(caller, id, ttl, mayOpen) {
228
+ const entry = this.entries.get(id);
229
+ if (!entry || !(this.owns(caller, entry) || (entry.run.up && mayOpen?.(this.copy(entry.run))))) {
230
+ return { status: 'not-found' };
231
+ }
232
+ // up, asleep, or woken and starting again
233
+ if (!entry.run.upAt || isFinal(entry.run.state)) {
234
+ return { status: 'not-up', state: entry.run.state };
235
+ }
236
+ // never sooner than it was: who wants it gone stops it
237
+ const ms = Math.min(ttl, this.options.up?.maxTtl ?? ttl);
238
+ const expiresAt = Math.max(Date.parse(entry.run.expiresAt ?? '') || 0, this.now() + ms);
239
+ entry.run.expiresAt = new Date(expiresAt).toISOString();
240
+ this.record(entry, { type: 'extended', by: { iss: caller.iss, sub: caller.sub }, expiresAt: entry.run.expiresAt });
241
+ this.expireLater(entry);
242
+ return { status: 'ok', run: this.copy(entry.run) };
243
+ }
244
+ /**
245
+ * The service run an endpoint host belongs to: the newest that is up, so one that replaces
246
+ * a run of the same name takes its hosts over as soon as it is up; else the newest that
247
+ * had it, for a page that says how it stands. With the address of the endpoint when up.
248
+ */
249
+ endpoint(host) {
250
+ let found;
251
+ for (const entry of this.entries.values()) {
252
+ const endpoint = entry.run.up?.endpoints.find((candidate) => candidate.host === host);
253
+ if (endpoint && (entry.run.state === 'up' || found?.entry.run.state !== 'up')) {
254
+ found = { entry, endpoint };
255
+ }
256
+ }
257
+ if (!found) {
258
+ return undefined;
259
+ }
260
+ const { entry, endpoint } = found;
261
+ const target = entry.run.state === 'up'
262
+ ? entry.targets?.find((candidate) => candidate.service === endpoint.service && candidate.port === endpoint.port)
263
+ : undefined;
264
+ return { run: this.copy(entry.run), endpoint: { ...endpoint }, ...(target ? { target } : {}) };
265
+ }
191
266
  // The run that holds a sandbox: the oldest on it that is not over. It holds it from
192
- // when it is submitted, so the files are not moved under what an approver saw.
267
+ // when it is submitted, so the files are not moved under what an approver saw. A service
268
+ // run that is up works on a copy of the files, and holds it no longer.
193
269
  holder(sandbox) {
194
- return [...this.entries.values()].find((entry) => onSandbox(entry.run, sandbox) && !isFinal(entry.run.state))?.run
195
- .id;
270
+ return [...this.entries.values()].find((entry) => onSandbox(entry.run, sandbox) &&
271
+ !isFinal(entry.run.state) &&
272
+ entry.run.state !== 'up' &&
273
+ entry.run.state !== 'asleep')?.run.id;
196
274
  }
197
275
  // Cancels every run on a sandbox and resolves when none of them runs anymore.
198
276
  async stopSandbox(sandbox, reason) {
199
277
  const on = () => [...this.entries.values()].filter((entry) => onSandbox(entry.run, sandbox));
200
278
  on().forEach((entry) => this.stop(entry, reason));
201
- while (on().some((entry) => entry.run.state === 'running')) {
279
+ while (on().some((entry) => entry.run.state === 'running' || entry.run.state === 'up')) {
202
280
  await new Promise((resolve) => setTimeout(resolve, 20));
203
281
  }
204
282
  }
205
- stop(entry, reason) {
283
+ stop(entry, reason, by) {
284
+ var _a, _b;
206
285
  if (entry.run.state === 'queued') {
207
286
  this.queue.splice(this.queue.indexOf(entry), 1);
208
287
  this.finish(entry, 'cancelled', reason);
@@ -213,6 +292,15 @@ class RunRegistry {
213
292
  else if (entry.run.state === 'running') {
214
293
  entry.abort.abort();
215
294
  }
295
+ else if (entry.run.state === 'up') {
296
+ (_a = entry.run).stoppedBy ?? (_a.stoppedBy = by ?? 'replaced');
297
+ entry.sleeping = false;
298
+ entry.abort.abort();
299
+ }
300
+ else if (entry.run.state === 'asleep') {
301
+ (_b = entry.run).stoppedBy ?? (_b.stoppedBy = by ?? 'replaced');
302
+ this.finish(entry, 'succeeded');
303
+ }
216
304
  }
217
305
  // The runs that wait for an approval, of anybody, oldest first. Who may see
218
306
  // them is for the caller to decide.
@@ -289,18 +377,20 @@ class RunRegistry {
289
377
  return { events, finished, stop: () => entry.listeners.delete(listener) };
290
378
  }
291
379
  /**
292
- * The run once it is over, or as it is after `ms` or when `signal` aborts, for clients
293
- * that can not hold a stream of events open (a tool call of an agent).
380
+ * The run once it is over (a service run: once it is up), or as it is after `ms` or when
381
+ * `signal` aborts, for clients that can not hold a stream of events open (a tool call of
382
+ * an agent).
294
383
  */
295
384
  async wait(owner, id, ms, canUse, signal) {
296
385
  const entry = this.find(owner, id, canUse);
297
386
  if (!entry) {
298
387
  return undefined;
299
388
  }
300
- if (!isFinal(entry.run.state) && ms > 0 && !this.closed && !signal?.aborted) {
389
+ if (!isFinal(entry.run.state) && entry.run.state !== 'up' && ms > 0 && !this.closed && !signal?.aborted) {
301
390
  await new Promise((resolve) => {
302
391
  const listener = (event) => {
303
- if (event.type === 'state' && isFinal(event.state)) {
392
+ // a service run is there once it is up
393
+ if (event.type === 'state' && (isFinal(event.state) || event.state === 'up')) {
304
394
  done();
305
395
  }
306
396
  };
@@ -335,10 +425,19 @@ class RunRegistry {
335
425
  async close() {
336
426
  this.closed = true;
337
427
  for (const entry of this.queue.splice(0)) {
338
- this.finish(entry, 'cancelled');
428
+ if (this.maySleep(entry) && this.options.store) {
429
+ // a woken service run sleeps again, the next start keeps it
430
+ this.sleep(entry);
431
+ }
432
+ else {
433
+ this.finish(entry, 'cancelled');
434
+ }
339
435
  }
340
436
  for (const entry of this.entries.values()) {
341
- if (entry.run.state === 'awaiting-approval') {
437
+ if (entry.run.state === 'asleep') {
438
+ this.clearTimer(entry);
439
+ }
440
+ else if (entry.run.state === 'awaiting-approval') {
342
441
  if (this.options.store) {
343
442
  // the record keeps it waiting, the next start takes it back
344
443
  this.clearTimer(entry);
@@ -353,11 +452,15 @@ class RunRegistry {
353
452
  done();
354
453
  }
355
454
  for (const entry of this.entries.values()) {
356
- if (entry.run.state === 'running') {
455
+ if (entry.run.state === 'running' || entry.run.state === 'up') {
456
+ // a service run that may sleep is asleep in its record, the next request after the
457
+ // start wakes it
458
+ entry.sleeping = this.maySleep(entry) && !!this.options.store;
357
459
  entry.abort.abort();
358
460
  }
359
461
  }
360
- while (this.running > 0) {
462
+ while (this.running > 0 ||
463
+ [...this.entries.values()].some((entry) => entry.run.state === 'running' || entry.run.state === 'up')) {
361
464
  await new Promise((resolve) => setTimeout(resolve, 10));
362
465
  }
363
466
  // the runs that were stopped are written down before the server is gone
@@ -388,6 +491,8 @@ class RunRegistry {
388
491
  copy(run) {
389
492
  return {
390
493
  ...run,
494
+ ...(run.up ? { up: JSON.parse(JSON.stringify(run.up)) } : {}),
495
+ ...(run.stoppedBy && typeof run.stoppedBy === 'object' ? { stoppedBy: { ...run.stoppedBy } } : {}),
391
496
  tasks: [...run.tasks],
392
497
  source: { ...run.source },
393
498
  options: JSON.parse(JSON.stringify(run.options)),
@@ -415,6 +520,76 @@ class RunRegistry {
415
520
  throw new QuotaError(`you have ${limits.queued} runs in the queue, wait for one to start or cancel one`);
416
521
  }
417
522
  }
523
+ // Service runs that are starting or up, of the caller and of everybody, against what may
524
+ // be. One that is asleep does not count: it asks for a place when it wakes (visit). The runs
525
+ // of a caller with one name take one place: the newest replaces the others once it is up,
526
+ // and an older one that gets up after it goes (markUp), so a new run of a name the caller
527
+ // has does not need another. Several of one name that are still starting share it too:
528
+ // what they take while they start is held by runs.limits.running and maxConcurrent.
529
+ checkUpLimits(owner, name) {
530
+ // what a caller keeps asleep is held by a limit of its own: a run that sleeps may keep the
531
+ // volumes of its services. A run of a name the caller has a run of (asleep, up or
532
+ // starting) replaces it, it is not one more.
533
+ const asleep = this.options.up?.asleep;
534
+ if (asleep !== undefined) {
535
+ const open = [...this.entries.values()].filter((entry) => entry.run.up && !isFinal(entry.run.state) && this.owns(owner, entry));
536
+ const sleeping = new Set(open.filter((entry) => entry.run.state === 'asleep').map((entry) => entry.run.up?.name));
537
+ if (!open.some((entry) => entry.run.up?.name === name) && sleeping.size >= asleep) {
538
+ throw new QuotaError(`you have ${asleep} service runs asleep, stop one or wait for its end`);
539
+ }
540
+ }
541
+ const full = this.upLimitReached(owner, name);
542
+ if (full === 'caller') {
543
+ throw new QuotaError(`you have ${this.options.up?.perCaller} service runs starting or up, stop one or wait for its end`);
544
+ }
545
+ if (full === 'server') {
546
+ throw new RunError(`the server has ${this.options.up?.total} service runs starting or up, try again later`);
547
+ }
548
+ }
549
+ // Whose limit a service run of the caller with this name would go over, if any
550
+ upLimitReached(owner, name) {
551
+ const limits = this.options.up;
552
+ if (!limits) {
553
+ return undefined;
554
+ }
555
+ const places = new Set();
556
+ const mine = new Set();
557
+ for (const entry of this.entries.values()) {
558
+ if (!entry.run.up || isFinal(entry.run.state) || entry.run.state === 'asleep') {
559
+ continue;
560
+ }
561
+ places.add(JSON.stringify([entry.run.owner.iss, entry.run.owner.sub, entry.run.up.name]));
562
+ if (this.owns(owner, entry)) {
563
+ mine.add(entry.run.up.name);
564
+ }
565
+ }
566
+ if (mine.has(name)) {
567
+ return undefined;
568
+ }
569
+ if (mine.size >= limits.perCaller) {
570
+ return 'caller';
571
+ }
572
+ if (places.size >= limits.total) {
573
+ return 'server';
574
+ }
575
+ return undefined;
576
+ }
577
+ // A host is one run's, or a run's that replaces it (same owner and name) once it is up:
578
+ // two labels that come out the same are refused, they would reach each other's services.
579
+ checkHosts(owner, up) {
580
+ const hosts = new Set(up.endpoints.map((endpoint) => endpoint.host));
581
+ for (const entry of this.entries.values()) {
582
+ if (isFinal(entry.run.state) || !entry.run.up) {
583
+ continue;
584
+ }
585
+ const taken = entry.run.up.endpoints.find((endpoint) => hosts.has(endpoint.host));
586
+ if (taken && !(this.owns(owner, entry) && entry.run.up.name === up.name)) {
587
+ const error = new RunError(`the address ${taken.host} is another service run's, pick another name`);
588
+ error.status = 409;
589
+ throw error;
590
+ }
591
+ }
592
+ }
418
593
  runningOf(owner) {
419
594
  return [...this.entries.values()].filter((entry) => entry.run.state === 'running' && this.owns(owner, entry)).length;
420
595
  }
@@ -424,20 +599,30 @@ class RunRegistry {
424
599
  return (source.type === 'sandbox' &&
425
600
  [...this.entries.values()].some((entry) => entry.run.state === 'running' && onSandbox(entry.run, source.id)));
426
601
  }
602
+ // Whether a run that executes holds what the run would (RunExecutor.exclusive)
603
+ held(run) {
604
+ const exclusive = this.options.executor.exclusive?.bind(this.options.executor);
605
+ const key = exclusive?.(run);
606
+ return (key !== undefined &&
607
+ [...this.entries.values()].some((entry) => entry.run.state === 'running' && exclusive?.(entry.run) === key));
608
+ }
427
609
  // The first run in the queue whose caller has a place left: one caller with many runs
428
610
  // does not keep the others from the machine. A run on a sandbox another run works on
429
- // waits for it.
611
+ // waits for it, as does one that needs what an executing run holds.
430
612
  dispatch() {
431
613
  while (this.running < this.maxConcurrent) {
432
- const perCaller = Math.min(this.options.limits?.running ?? Infinity, this.options.executor.runningPerCaller ?? Infinity);
433
- const index = this.queue.findIndex((entry) => this.runningOf(entry.run.owner) < perCaller && !this.sandboxBusy(entry.run));
614
+ const perCaller = this.options.limits?.running ?? Infinity;
615
+ const index = this.queue.findIndex((entry) => this.runningOf(entry.run.owner) < perCaller && !this.sandboxBusy(entry.run) && !this.held(entry.run));
434
616
  if (index < 0) {
435
617
  return;
436
618
  }
437
619
  const [entry] = this.queue.splice(index, 1);
438
620
  this.running++;
439
621
  this.execute(entry).finally(() => {
440
- this.running--;
622
+ // a service run that is up gave its place back already
623
+ if (!entry.released) {
624
+ this.running--;
625
+ }
441
626
  this.dispatch();
442
627
  });
443
628
  }
@@ -446,27 +631,209 @@ class RunRegistry {
446
631
  entry.run.state = 'running';
447
632
  entry.run.startedAt = this.timestamp();
448
633
  this.record(entry, { type: 'state', state: 'running' });
634
+ // a service run gets this long to be up, then its ttl counts
449
635
  const limit = this.options.maxDuration;
450
- const timer = limit
636
+ entry.durationTimer = limit
451
637
  ? setTimeout(() => {
452
638
  entry.timedOut = true;
453
639
  entry.abort.abort();
454
640
  }, limit)
455
641
  : undefined;
456
- timer?.unref();
642
+ entry.durationTimer?.unref();
457
643
  try {
458
644
  await this.options.executor.execute(this.copy(entry.run), {
459
645
  emit: (event) => this.record(entry, event),
460
646
  signal: entry.abort.signal,
647
+ ...(entry.run.up ? { up: (targets) => this.markUp(entry, targets) } : {}),
461
648
  });
462
- this.finish(entry, entry.timedOut ? 'failed' : entry.abort.signal.aborted ? 'cancelled' : 'succeeded', entry.timedOut ? this.tooLong() : undefined);
649
+ if (entry.sleeping) {
650
+ this.sleep(entry);
651
+ }
652
+ else if (entry.run.upAt) {
653
+ // services stay until they are stopped: ones that stopped on their own failed
654
+ this.endServices(entry, 'the services stopped on their own');
655
+ }
656
+ else {
657
+ this.finish(entry, entry.timedOut ? 'failed' : entry.abort.signal.aborted ? 'cancelled' : 'succeeded', entry.timedOut ? this.tooLong() : undefined);
658
+ }
463
659
  }
464
660
  catch (e) {
465
- const cancelled = entry.abort.signal.aborted && !entry.timedOut;
466
- this.finish(entry, cancelled ? 'cancelled' : 'failed', entry.timedOut ? this.tooLong() : cancelled ? undefined : e instanceof Error ? e.message : String(e));
661
+ const message = e instanceof Error ? e.message : String(e);
662
+ if (entry.sleeping) {
663
+ this.sleep(entry);
664
+ }
665
+ else if (entry.run.upAt) {
666
+ this.endServices(entry, message);
667
+ }
668
+ else {
669
+ const cancelled = entry.abort.signal.aborted && !entry.timedOut;
670
+ this.finish(entry, cancelled ? 'cancelled' : 'failed', entry.timedOut ? this.tooLong() : cancelled ? undefined : message);
671
+ }
467
672
  }
468
673
  finally {
469
- clearTimeout(timer);
674
+ clearTimeout(entry.durationTimer);
675
+ }
676
+ }
677
+ // The services of a service run are ready: its ttl counts from now, its place of
678
+ // maxConcurrent goes to the next run, and an older run of its owner with the same name
679
+ // stops, so its hosts are this run's.
680
+ markUp(entry, targets) {
681
+ const spec = entry.run.up;
682
+ if (!spec || entry.run.state !== 'running' || entry.abort.signal.aborted) {
683
+ return;
684
+ }
685
+ clearTimeout(entry.durationTimer);
686
+ entry.durationTimer = undefined;
687
+ entry.targets = targets.map((target) => ({ ...target, address: { ...target.address } }));
688
+ entry.run.state = 'up';
689
+ // one reading of the clock, so the ttl is exactly expiresAt - upAt
690
+ const now = this.now();
691
+ // the ttl counts from the first time, not from a wake
692
+ if (!entry.run.upAt) {
693
+ entry.run.upAt = new Date(now).toISOString();
694
+ entry.run.expiresAt = new Date(now + spec.ttl).toISOString();
695
+ }
696
+ if (spec.sleepTtl) {
697
+ entry.run.sleepsAt = new Date(now + spec.sleepTtl).toISOString();
698
+ }
699
+ this.record(entry, { type: 'state', state: 'up' });
700
+ this.expireLater(entry);
701
+ entry.released = true;
702
+ this.running--;
703
+ const entries = [...this.entries.values()];
704
+ for (const other of entries.slice(0, entries.indexOf(entry))) {
705
+ if (other.run.up?.name === spec.name &&
706
+ this.owns(entry.run.owner, other) &&
707
+ !isFinal(other.run.state) &&
708
+ other.run.state !== 'awaiting-approval') {
709
+ other.run.stoppedBy = 'replaced';
710
+ this.stop(other, `replaced by ${entry.run.id}`);
711
+ }
712
+ }
713
+ // A newer run of the name that is up or asleep already (this one waited for an approval
714
+ // while it got up) is what the owner started last: this one goes, so the runs of a name,
715
+ // which take one place of runs.up.limits, never hold more than one set of services.
716
+ const newer = entries
717
+ .slice(entries.indexOf(entry) + 1)
718
+ .find((other) => other.run.up?.name === spec.name &&
719
+ this.owns(entry.run.owner, other) &&
720
+ (other.run.state === 'up' || other.run.state === 'asleep'));
721
+ if (newer) {
722
+ entry.run.stoppedBy = 'replaced';
723
+ this.stop(entry, `replaced by ${newer.run.id}`);
724
+ }
725
+ this.dispatch();
726
+ }
727
+ // Stops a service run at its expiry, and puts one that is up to sleep at its sleepsAt. A
728
+ // timer can't wait longer than about 24 days, so a longer one looks again then.
729
+ expireLater(entry) {
730
+ this.clearTimer(entry);
731
+ const expiresAt = Date.parse(entry.run.expiresAt ?? '');
732
+ const sleepsAt = entry.run.state === 'up' ? Date.parse(entry.run.sleepsAt ?? '') : NaN;
733
+ const next = Number.isFinite(sleepsAt) ? Math.min(expiresAt, sleepsAt) : expiresAt;
734
+ entry.timer = setTimeout(() => {
735
+ var _a;
736
+ if (isFinal(entry.run.state)) {
737
+ return;
738
+ }
739
+ if (Date.parse(entry.run.expiresAt ?? '') <= this.now()) {
740
+ (_a = entry.run).stoppedBy ?? (_a.stoppedBy = 'ttl');
741
+ entry.sleeping = false;
742
+ if (entry.run.state === 'asleep') {
743
+ this.finish(entry, 'succeeded');
744
+ }
745
+ else {
746
+ this.stop(entry);
747
+ }
748
+ return;
749
+ }
750
+ if (entry.run.state === 'up' && Date.parse(entry.run.sleepsAt ?? '') <= this.now()) {
751
+ entry.sleeping = true;
752
+ entry.abort.abort();
753
+ return;
754
+ }
755
+ this.expireLater(entry);
756
+ }, Math.max(0, Math.min(next - this.now(), 2 ** 31 - 1)));
757
+ entry.timer.unref();
758
+ }
759
+ // Whether a service run sleeps instead of ending: it has a sleepTtl, its ttl has not
760
+ // ended, and it starts from a commit (the files of a sandbox change under it).
761
+ maySleep(entry) {
762
+ return (!!entry.run.up?.sleepTtl &&
763
+ !!entry.run.upAt &&
764
+ entry.run.source.type === 'ref' &&
765
+ Date.parse(entry.run.expiresAt ?? '') > this.now());
766
+ }
767
+ // The services stopped, the run waits for a request to wake it, or for its end.
768
+ sleep(entry) {
769
+ entry.sleeping = false;
770
+ entry.targets = undefined;
771
+ entry.run.state = 'asleep';
772
+ delete entry.run.sleepsAt;
773
+ this.record(entry, { type: 'state', state: 'asleep' });
774
+ this.expireLater(entry);
775
+ }
776
+ /**
777
+ * A request to an endpoint of a service run: one that is up stays up for its sleepTtl from
778
+ * now, one that is asleep is started again, if `mayWake` finds its owner may still run it
779
+ * (else it ends) and there is a place of runs.up.limits for it (else it sleeps on, and the
780
+ * next request asks again). Answers why it was not woken, or undefined.
781
+ */
782
+ visit(id, mayWake) {
783
+ const entry = this.entries.get(id);
784
+ if (!entry?.run.up) {
785
+ return undefined;
786
+ }
787
+ const sleepTtl = entry.run.up.sleepTtl;
788
+ if (entry.run.state === 'up' && sleepTtl) {
789
+ // only kept in memory: a server that stops puts the run to sleep anyway
790
+ entry.run.sleepsAt = new Date(this.now() + sleepTtl).toISOString();
791
+ return undefined;
792
+ }
793
+ if (entry.run.state !== 'asleep' || this.closed) {
794
+ return undefined;
795
+ }
796
+ if (!this.maySleep(entry)) {
797
+ this.expireLater(entry);
798
+ return 'its time ran out';
799
+ }
800
+ const refused = mayWake?.(this.copy(entry.run));
801
+ if (refused) {
802
+ this.finish(entry, 'failed', refused);
803
+ return refused;
804
+ }
805
+ const full = this.upLimitReached(entry.run.owner, entry.run.up.name);
806
+ if (full === 'caller') {
807
+ return 'its owner has as many service runs starting or up as they may, it wakes once one of them stops or sleeps';
808
+ }
809
+ if (full === 'server') {
810
+ return 'the server has as many service runs starting or up as it may, it wakes once one of them stops or sleeps';
811
+ }
812
+ this.clearTimer(entry);
813
+ entry.abort = new AbortController();
814
+ entry.released = false;
815
+ entry.timedOut = false;
816
+ entry.run.wakes = (entry.run.wakes ?? 0) + 1;
817
+ entry.run.state = 'queued';
818
+ this.record(entry, { type: 'state', state: 'queued' });
819
+ this.queue.push(entry);
820
+ this.dispatch();
821
+ return undefined;
822
+ }
823
+ // The end of a run that was up: stopped by somebody, its ttl or a newer run it succeeded,
824
+ // stopped by the server going away it failed, as its record says after a restart.
825
+ endServices(entry, failure) {
826
+ if (entry.timedOut) {
827
+ this.finish(entry, 'failed', this.tooLong());
828
+ }
829
+ else if (!entry.abort.signal.aborted) {
830
+ this.finish(entry, 'failed', failure);
831
+ }
832
+ else if (entry.run.stoppedBy) {
833
+ this.finish(entry, 'succeeded');
834
+ }
835
+ else {
836
+ this.finish(entry, 'failed', 'the server stopped while the services were up, start them again');
470
837
  }
471
838
  }
472
839
  tooLong() {
@@ -474,6 +841,8 @@ class RunRegistry {
474
841
  }
475
842
  finish(entry, state, error) {
476
843
  this.clearTimer(entry);
844
+ clearTimeout(entry.durationTimer);
845
+ entry.targets = undefined;
477
846
  entry.run.state = state;
478
847
  entry.run.finishedAt = this.timestamp();
479
848
  if (error) {
@@ -488,9 +857,23 @@ class RunRegistry {
488
857
  }
489
858
  this.record(entry, { type: 'state', state, ...(error ? { error } : {}) }, true);
490
859
  entry.listeners.clear();
860
+ try {
861
+ this.options.onFinished?.(this.copy(entry.run), { ...entry.keys });
862
+ }
863
+ catch (e) {
864
+ this.options.log?.(`run ${entry.run.id} ended, but what it passed was not kept: ${e instanceof Error ? e.message : e}`);
865
+ }
491
866
  this.evict();
492
867
  }
493
868
  record(entry, recorded, force = false) {
869
+ if (recorded.type === 'key') {
870
+ // a task name is the executor's; the first key it reports for a task is the one
871
+ // computed before anything of the run had run
872
+ if (!(recorded.task in entry.keys) && Object.keys(entry.keys).length < MAX_KEYED_TASKS) {
873
+ entry.keys[recorded.task] = recorded.key;
874
+ }
875
+ return;
876
+ }
494
877
  const event = recorded.type === 'task' ? bounded(recorded) : recorded;
495
878
  if (event.type === 'log') {
496
879
  this.keepTail(entry, event.task, event.text);
@@ -572,7 +955,7 @@ class RunRegistry {
572
955
  }
573
956
  // what a restart must not lose: the runs that wait, and how a run ended. A running
574
957
  // run is not kept up to date, it fails with the server.
575
- if (event.type === 'state' || event.type === 'approval') {
958
+ if (event.type === 'state' || event.type === 'approval' || event.type === 'extended') {
576
959
  this.persist(entry);
577
960
  }
578
961
  }
@@ -1,12 +1,14 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.sandboxPodMain = exports.decodePodError = exports.encodeEditOps = exports.INCOMING_REF = exports.INCOMING = void 0;
4
+ const crypto_1 = require("crypto");
4
5
  const promises_1 = require("fs/promises");
5
6
  const path_1 = require("path");
6
7
  const url_1 = require("url");
7
8
  const git_checkout_1 = require("./git-checkout");
8
9
  const sandbox_files_1 = require("./sandbox-files");
9
10
  const sandbox_storage_1 = require("./sandbox-storage");
11
+ const tar_1 = require("../utils/tar");
10
12
  // On a cluster a sandbox is a claim, and what the server does with its files runs in a
11
13
  // short pod that mounts it (see KubernetesSandboxStorage): `hammerkit server-sandbox
12
14
  // <directory>` reads one request on its standard input, does it with the code of a host
@@ -91,6 +93,32 @@ async function answer(storage, directory, request, out) {
91
93
  }
92
94
  return send({ files: plan.files, bytes: plan.bytes });
93
95
  }
96
+ case 'archive': {
97
+ // written beside the sandbox on the claim, sent, and removed
98
+ const into = (0, path_1.join)(directory, `${request.id}-archive-${(0, crypto_1.randomBytes)(6).toString('hex')}`);
99
+ try {
100
+ await storage.exportTree(key, request.tree, into, AbortSignal.timeout(10 * 60000));
101
+ await write(out, JSON.stringify({ ok: true, result: null, stream: true }) + '\n');
102
+ const pack = (0, tar_1.packArchive)({ cwd: into, portable: true }, ['.']);
103
+ try {
104
+ for await (const chunk of pack) {
105
+ await write(out, chunk);
106
+ }
107
+ }
108
+ catch (e) {
109
+ throw new AnswerStarted(e instanceof Error ? e.message : String(e));
110
+ }
111
+ finally {
112
+ // a reader that went away (over its size limit) leaves the pack reading files
113
+ // that are removed next: what fails then is no answer of anybody's
114
+ pack.on('error', () => undefined);
115
+ }
116
+ return;
117
+ }
118
+ finally {
119
+ await (0, promises_1.rm)(into, { recursive: true, force: true });
120
+ }
121
+ }
94
122
  case 'raw': {
95
123
  const raw = await files().raw(request.path);
96
124
  const header = {