hammerkit 1.26.0 → 1.27.0-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build.schema.json +53 -1
- package/dist/cache/backends/local-cache-backend.js +10 -2
- package/dist/cache/explain.js +13 -11
- package/dist/cli.js +3 -0
- package/dist/docker/item-network.js +471 -0
- package/dist/docker/pull.js +4 -2
- package/dist/docker/remove-orphaned-containers.js +13 -6
- package/dist/environment/replace-env-variables.js +8 -6
- package/dist/executer/await-completed-dependencies.js +2 -1
- package/dist/executer/check-readiness.js +2 -2
- package/dist/executer/docker-service.js +24 -13
- package/dist/executer/docker-task.js +18 -29
- package/dist/executer/event-cache.js +4 -0
- package/dist/executer/execute-command.js +18 -5
- package/dist/executer/execute-docker.js +4 -1
- package/dist/executer/execution-steps.js +33 -7
- package/dist/executer/get-docker-executor.js +14 -7
- package/dist/executer/scheduler/enqueue-next.js +17 -3
- package/dist/executer/watch-loop.js +2 -2
- package/dist/executer/watch-state-key.js +1 -1
- package/dist/file/within-root.js +42 -0
- package/dist/harness/connect.js +99 -0
- package/dist/harness/format.js +73 -0
- package/dist/harness/mcp-command.js +59 -0
- package/dist/harness/mcp.js +432 -0
- package/dist/harness/opencode.js +492 -0
- package/dist/harness/pi.js +217 -0
- package/dist/harness/sandbox-client.js +373 -0
- package/dist/index.js +33 -36
- package/dist/kubernetes/apply.js +31 -1
- package/dist/kubernetes/await-running-state.js +33 -1
- package/dist/kubernetes/ensure-kubernetes-deployment-exists.js +43 -21
- package/dist/kubernetes/ensure-kubernetes-persistent-volume-claim-exists.js +131 -4
- package/dist/kubernetes/ensure-kubernetes-service-exists.js +25 -4
- package/dist/kubernetes/ensure-persistent-data.js +48 -7
- package/dist/kubernetes/get-service-ip.js +15 -1
- package/dist/kubernetes/job-placement.js +99 -0
- package/dist/kubernetes/job-secrets.js +306 -0
- package/dist/kubernetes/kept-claims.js +191 -0
- package/dist/kubernetes/kubernetes-instance.js +10 -2
- package/dist/kubernetes/network-gate.js +71 -0
- package/dist/kubernetes/network-policy.js +526 -0
- package/dist/kubernetes/resources.js +13 -2
- package/dist/kubernetes/restore-kubernetes-data.js +19 -4
- package/dist/kubernetes/secrets.js +2 -0
- package/dist/kubernetes/store-kubernetes-data.js +1 -1
- package/dist/kubernetes/volumes.js +100 -11
- package/dist/main.js +41 -0
- package/dist/network/egress-proxy.js +368 -0
- package/dist/network/port-relay.js +35 -0
- package/dist/optimizer/get-work-cache-stats.js +16 -0
- package/dist/optimizer/work-task-cache-description.js +3 -2
- package/dist/planner/service-definition.js +5 -0
- package/dist/planner/utils/append-work-environment.js +2 -0
- package/dist/planner/utils/append-work-service.js +4 -0
- package/dist/planner/utils/append-work-task.js +17 -1
- package/dist/planner/utils/assert-item-paths.js +50 -0
- package/dist/planner/utils/hosted-volumes.js +35 -0
- package/dist/planner/work-network.js +193 -0
- package/dist/planner/work-runtime-docker.js +41 -0
- package/dist/planner/work-runtime-kubernetes.js +335 -102
- package/dist/program.js +21 -0
- package/dist/remote/annotations.js +103 -0
- package/dist/remote/bundle-text.js +24 -0
- package/dist/remote/bundle.js +9 -1
- package/dist/remote/remote-commands.js +4 -1
- package/dist/remote/remote-work-commands.js +182 -69
- package/dist/remote/server-client.js +18 -5
- package/dist/remote/source-manifest.js +15 -2
- package/dist/remote/token-source.js +67 -0
- package/dist/schema/build-file-base-task-schema.js +3 -0
- package/dist/schema/build-file-container-service-schema.js +6 -0
- package/dist/schema/build-file-container-task-schema.js +3 -0
- package/dist/schema/build-file-environment-schema.js +9 -0
- package/dist/schema/network-schema.js +78 -0
- package/dist/schema/reference-parser.js +1 -0
- package/dist/schema/schema-parser.js +16 -3
- package/dist/schema/work-scope-parser.js +5 -0
- package/dist/secrets/command-secret-provider.js +73 -10
- package/dist/secrets/github-app-token-command.js +23 -0
- package/dist/secrets/github-app-token.js +191 -0
- package/dist/server/accounts-api.js +9 -2
- package/dist/server/definitions-api.js +69 -47
- package/dist/server/definitions.js +76 -51
- package/dist/server/executor.js +156 -41
- package/dist/server/git-checkout.js +72 -8
- package/dist/server/kubernetes-backend.js +554 -0
- package/dist/server/kubernetes-sandbox-storage.js +996 -0
- package/dist/server/oauth.js +1 -0
- package/dist/server/plan.js +51 -1
- package/dist/server/policy.js +104 -15
- package/dist/server/proposals-api.js +24 -40
- package/dist/server/proposals.js +12 -25
- package/dist/server/repo-access.js +37 -6
- package/dist/server/run-logs.js +217 -0
- package/dist/server/run-plan.js +51 -2
- package/dist/server/run-request.js +23 -10
- package/dist/server/run-store.js +5 -0
- package/dist/server/run-worker-command.js +13 -1
- package/dist/server/run-worker.js +121 -24
- package/dist/server/runs.js +184 -10
- package/dist/server/sandbox-docker.js +62 -0
- package/dist/server/sandbox-files-api.js +269 -0
- package/dist/server/sandbox-files.js +1054 -0
- package/dist/server/sandbox-patch.js +481 -0
- package/dist/server/sandbox-pod-command.js +15 -0
- package/dist/server/sandbox-pod.js +146 -0
- package/dist/server/sandbox-storage.js +462 -0
- package/dist/server/sandboxes-api.js +210 -0
- package/dist/server/sandboxes.js +519 -0
- package/dist/server/server-config.js +412 -22
- package/dist/server/server.js +204 -39
- package/dist/utils/run-id.js +12 -0
- package/harness/opencode.ts +11 -0
- package/harness/pi.ts +7 -0
- package/package.json +8 -2
- package/dist/remote/text-diff.js +0 -89
package/dist/server/runs.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.unavailableExecutor = exports.RunRegistry = exports.RUN_ID_PATTERN = exports.QuotaError = exports.RunError = exports.isFinal = void 0;
|
|
3
|
+
exports.unavailableExecutor = exports.RunRegistry = exports.MAX_TASK_ERROR = exports.MAX_CAUSES = exports.SANDBOX_ID_PATTERN = exports.RUN_ID_PATTERN = exports.QuotaError = exports.RunError = exports.isFinal = void 0;
|
|
4
4
|
const crypto_1 = require("crypto");
|
|
5
|
+
const run_logs_1 = require("./run-logs");
|
|
5
6
|
const units_1 = require("../utils/units");
|
|
6
7
|
const FINAL_STATES = ['succeeded', 'failed', 'cancelled', 'rejected'];
|
|
7
8
|
function isFinal(state) {
|
|
@@ -25,6 +26,38 @@ class QuotaError extends RunError {
|
|
|
25
26
|
}
|
|
26
27
|
exports.QuotaError = QuotaError;
|
|
27
28
|
exports.RUN_ID_PATTERN = /^run_[A-Za-z0-9_-]{16}$/;
|
|
29
|
+
exports.SANDBOX_ID_PATTERN = /^sbx_[A-Za-z0-9_-]{16}$/;
|
|
30
|
+
function onSandbox(run, sandbox) {
|
|
31
|
+
return run.source.type === 'sandbox' && run.source.id === sandbox;
|
|
32
|
+
}
|
|
33
|
+
// how much of the end of a task's output its result carries
|
|
34
|
+
const TAIL_LINES = 20;
|
|
35
|
+
const TAIL_LINE_LENGTH = 1000;
|
|
36
|
+
// what a task event says of why a task ran or failed: a change to thousands of source
|
|
37
|
+
// files is a cause per file. The process of a run cuts them already; an executor that
|
|
38
|
+
// does not can still not make a result of any size.
|
|
39
|
+
exports.MAX_CAUSES = 20;
|
|
40
|
+
exports.MAX_TASK_ERROR = 2 * 1024;
|
|
41
|
+
function bounded(event) {
|
|
42
|
+
const causes = event.causes;
|
|
43
|
+
return {
|
|
44
|
+
...event,
|
|
45
|
+
...(causes
|
|
46
|
+
? {
|
|
47
|
+
causes: [
|
|
48
|
+
...causes.slice(0, exports.MAX_CAUSES).map((cause) => cause.slice(0, exports.MAX_TASK_ERROR)),
|
|
49
|
+
...(causes.length > exports.MAX_CAUSES ? [`… and ${causes.length - exports.MAX_CAUSES} more`] : []),
|
|
50
|
+
],
|
|
51
|
+
}
|
|
52
|
+
: {}),
|
|
53
|
+
...(event.error !== undefined ? { error: event.error.slice(0, exports.MAX_TASK_ERROR) } : {}),
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
// what a task event weighs against the log limits of a run
|
|
57
|
+
function taskEventSize(event) {
|
|
58
|
+
const causes = (event.causes ?? []).reduce((sum, cause) => sum + Buffer.byteLength(cause) + 4, 0);
|
|
59
|
+
return 64 + causes + Buffer.byteLength(event.error ?? '');
|
|
60
|
+
}
|
|
28
61
|
// Where runs are kept while the server is up. Everything is scoped to the
|
|
29
62
|
// caller that submitted the run: the same id asked for by someone else is
|
|
30
63
|
// "not found", so ids can't be used to find out which runs exist.
|
|
@@ -35,6 +68,8 @@ class RunRegistry {
|
|
|
35
68
|
this.queue = [];
|
|
36
69
|
this.running = 0;
|
|
37
70
|
this.closed = false;
|
|
71
|
+
// callers that wait for a run to finish, let go when the server stops
|
|
72
|
+
this.waiters = new Set();
|
|
38
73
|
this.maxConcurrent = options.maxConcurrent ?? 4;
|
|
39
74
|
this.keep = options.keep ?? 200;
|
|
40
75
|
this.maxEvents = options.maxEvents ?? 5000;
|
|
@@ -63,6 +98,7 @@ class RunRegistry {
|
|
|
63
98
|
abort: new AbortController(),
|
|
64
99
|
truncated: events.some((event) => event.type === 'truncated'),
|
|
65
100
|
bytes: 0,
|
|
101
|
+
tails: new Map(Object.entries(run.results ?? {}).map(([task, result]) => [task, result.lastLog])),
|
|
66
102
|
saving: Promise.resolve(),
|
|
67
103
|
};
|
|
68
104
|
this.entries.set(run.id, entry);
|
|
@@ -107,6 +143,7 @@ class RunRegistry {
|
|
|
107
143
|
abort: new AbortController(),
|
|
108
144
|
truncated: false,
|
|
109
145
|
bytes: 0,
|
|
146
|
+
tails: new Map(),
|
|
110
147
|
saving: Promise.resolve(),
|
|
111
148
|
};
|
|
112
149
|
this.entries.set(run.id, entry);
|
|
@@ -148,17 +185,34 @@ class RunRegistry {
|
|
|
148
185
|
if (!entry) {
|
|
149
186
|
return undefined;
|
|
150
187
|
}
|
|
188
|
+
this.stop(entry);
|
|
189
|
+
return this.copy(entry.run);
|
|
190
|
+
}
|
|
191
|
+
// The run that holds a sandbox: the oldest on it that is not over. It holds it from
|
|
192
|
+
// when it is submitted, so the files are not moved under what an approver saw.
|
|
193
|
+
holder(sandbox) {
|
|
194
|
+
return [...this.entries.values()].find((entry) => onSandbox(entry.run, sandbox) && !isFinal(entry.run.state))?.run
|
|
195
|
+
.id;
|
|
196
|
+
}
|
|
197
|
+
// Cancels every run on a sandbox and resolves when none of them runs anymore.
|
|
198
|
+
async stopSandbox(sandbox, reason) {
|
|
199
|
+
const on = () => [...this.entries.values()].filter((entry) => onSandbox(entry.run, sandbox));
|
|
200
|
+
on().forEach((entry) => this.stop(entry, reason));
|
|
201
|
+
while (on().some((entry) => entry.run.state === 'running')) {
|
|
202
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
stop(entry, reason) {
|
|
151
206
|
if (entry.run.state === 'queued') {
|
|
152
207
|
this.queue.splice(this.queue.indexOf(entry), 1);
|
|
153
|
-
this.finish(entry, 'cancelled');
|
|
208
|
+
this.finish(entry, 'cancelled', reason);
|
|
154
209
|
}
|
|
155
210
|
else if (entry.run.state === 'awaiting-approval') {
|
|
156
|
-
this.finish(entry, 'cancelled');
|
|
211
|
+
this.finish(entry, 'cancelled', reason);
|
|
157
212
|
}
|
|
158
213
|
else if (entry.run.state === 'running') {
|
|
159
214
|
entry.abort.abort();
|
|
160
215
|
}
|
|
161
|
-
return this.copy(entry.run);
|
|
162
216
|
}
|
|
163
217
|
// The runs that wait for an approval, of anybody, oldest first. Who may see
|
|
164
218
|
// them is for the caller to decide.
|
|
@@ -234,6 +288,49 @@ class RunRegistry {
|
|
|
234
288
|
}
|
|
235
289
|
return { events, finished, stop: () => entry.listeners.delete(listener) };
|
|
236
290
|
}
|
|
291
|
+
/**
|
|
292
|
+
* The run once it is over, or as it is after `ms` or when `signal` aborts, for clients
|
|
293
|
+
* that can not hold a stream of events open (a tool call of an agent).
|
|
294
|
+
*/
|
|
295
|
+
async wait(owner, id, ms, canUse, signal) {
|
|
296
|
+
const entry = this.find(owner, id, canUse);
|
|
297
|
+
if (!entry) {
|
|
298
|
+
return undefined;
|
|
299
|
+
}
|
|
300
|
+
if (!isFinal(entry.run.state) && ms > 0 && !this.closed && !signal?.aborted) {
|
|
301
|
+
await new Promise((resolve) => {
|
|
302
|
+
const listener = (event) => {
|
|
303
|
+
if (event.type === 'state' && isFinal(event.state)) {
|
|
304
|
+
done();
|
|
305
|
+
}
|
|
306
|
+
};
|
|
307
|
+
const done = () => {
|
|
308
|
+
clearTimeout(timer);
|
|
309
|
+
entry.listeners.delete(listener);
|
|
310
|
+
this.waiters.delete(done);
|
|
311
|
+
signal?.removeEventListener('abort', done);
|
|
312
|
+
resolve();
|
|
313
|
+
};
|
|
314
|
+
const timer = setTimeout(done, ms);
|
|
315
|
+
timer.unref();
|
|
316
|
+
entry.listeners.add(listener);
|
|
317
|
+
this.waiters.add(done);
|
|
318
|
+
signal?.addEventListener('abort', done, { once: true });
|
|
319
|
+
});
|
|
320
|
+
}
|
|
321
|
+
return this.copy(entry.run);
|
|
322
|
+
}
|
|
323
|
+
// The log of a run as far as it was kept, and whether it was cut off.
|
|
324
|
+
logs(owner, id, canUse) {
|
|
325
|
+
const entry = this.find(owner, id, canUse);
|
|
326
|
+
if (!entry) {
|
|
327
|
+
return undefined;
|
|
328
|
+
}
|
|
329
|
+
// only the events since the last read are looked at
|
|
330
|
+
entry.index ?? (entry.index = new run_logs_1.LogIndex());
|
|
331
|
+
entry.index.update(entry.events);
|
|
332
|
+
return { events: entry.events, index: entry.index, truncated: entry.truncated };
|
|
333
|
+
}
|
|
237
334
|
// Stops accepting runs and aborts the ones in flight.
|
|
238
335
|
async close() {
|
|
239
336
|
this.closed = true;
|
|
@@ -251,6 +348,10 @@ class RunRegistry {
|
|
|
251
348
|
}
|
|
252
349
|
}
|
|
253
350
|
}
|
|
351
|
+
// who waits for a run that waits for an approval is told how it stands now
|
|
352
|
+
for (const done of [...this.waiters]) {
|
|
353
|
+
done();
|
|
354
|
+
}
|
|
254
355
|
for (const entry of this.entries.values()) {
|
|
255
356
|
if (entry.run.state === 'running') {
|
|
256
357
|
entry.abort.abort();
|
|
@@ -288,12 +389,12 @@ class RunRegistry {
|
|
|
288
389
|
return {
|
|
289
390
|
...run,
|
|
290
391
|
tasks: [...run.tasks],
|
|
291
|
-
definition: { ...run.definition },
|
|
292
392
|
source: { ...run.source },
|
|
293
393
|
options: JSON.parse(JSON.stringify(run.options)),
|
|
294
394
|
owner: { ...run.owner },
|
|
295
395
|
accounts: [...run.accounts],
|
|
296
396
|
...(run.approvals ? { approvals: JSON.parse(JSON.stringify(run.approvals)) } : {}),
|
|
397
|
+
...(run.results ? { results: JSON.parse(JSON.stringify(run.results)) } : {}),
|
|
297
398
|
};
|
|
298
399
|
}
|
|
299
400
|
timestamp() {
|
|
@@ -317,11 +418,19 @@ class RunRegistry {
|
|
|
317
418
|
runningOf(owner) {
|
|
318
419
|
return [...this.entries.values()].filter((entry) => entry.run.state === 'running' && this.owns(owner, entry)).length;
|
|
319
420
|
}
|
|
421
|
+
// Whether a run on the same sandbox executes: a sandbox runs one at a time.
|
|
422
|
+
sandboxBusy(run) {
|
|
423
|
+
const source = run.source;
|
|
424
|
+
return (source.type === 'sandbox' &&
|
|
425
|
+
[...this.entries.values()].some((entry) => entry.run.state === 'running' && onSandbox(entry.run, source.id)));
|
|
426
|
+
}
|
|
320
427
|
// The first run in the queue whose caller has a place left: one caller with many runs
|
|
321
|
-
// does not keep the others from the machine.
|
|
428
|
+
// does not keep the others from the machine. A run on a sandbox another run works on
|
|
429
|
+
// waits for it.
|
|
322
430
|
dispatch() {
|
|
323
431
|
while (this.running < this.maxConcurrent) {
|
|
324
|
-
const
|
|
432
|
+
const perCaller = Math.min(this.options.limits?.running ?? Infinity, this.options.executor.runningPerCaller ?? Infinity);
|
|
433
|
+
const index = this.queue.findIndex((entry) => this.runningOf(entry.run.owner) < perCaller && !this.sandboxBusy(entry.run));
|
|
325
434
|
if (index < 0) {
|
|
326
435
|
return;
|
|
327
436
|
}
|
|
@@ -370,15 +479,29 @@ class RunRegistry {
|
|
|
370
479
|
if (error) {
|
|
371
480
|
entry.run.error = error;
|
|
372
481
|
}
|
|
482
|
+
// a task the executor did not see to its end did not finish
|
|
483
|
+
for (const result of Object.values(entry.run.results ?? {})) {
|
|
484
|
+
if (result.status === 'running') {
|
|
485
|
+
result.status = 'cancelled';
|
|
486
|
+
result.finishedAt = entry.run.finishedAt;
|
|
487
|
+
}
|
|
488
|
+
}
|
|
373
489
|
this.record(entry, { type: 'state', state, ...(error ? { error } : {}) }, true);
|
|
374
490
|
entry.listeners.clear();
|
|
375
491
|
this.evict();
|
|
376
492
|
}
|
|
377
|
-
record(entry,
|
|
493
|
+
record(entry, recorded, force = false) {
|
|
494
|
+
const event = recorded.type === 'task' ? bounded(recorded) : recorded;
|
|
495
|
+
if (event.type === 'log') {
|
|
496
|
+
this.keepTail(entry, event.task, event.text);
|
|
497
|
+
}
|
|
498
|
+
else if (event.type === 'task') {
|
|
499
|
+
this.keepResult(entry, event);
|
|
500
|
+
}
|
|
378
501
|
// an executor that keeps talking can't fill the memory of the server or its disk; the
|
|
379
|
-
// state changes, which are few, are always kept
|
|
502
|
+
// state changes, which are few, are always kept, and the results above are kept apart
|
|
380
503
|
if (!force && event.type !== 'state') {
|
|
381
|
-
const size = event.type === 'log' ? Buffer.byteLength(event.text) + 64 : 64;
|
|
504
|
+
const size = event.type === 'log' ? Buffer.byteLength(event.text) + 64 : event.type === 'task' ? taskEventSize(event) : 64;
|
|
382
505
|
if (entry.events.length >= this.maxEvents || entry.bytes + size > this.maxLogBytes) {
|
|
383
506
|
if (!entry.truncated) {
|
|
384
507
|
entry.truncated = true;
|
|
@@ -390,6 +513,57 @@ class RunRegistry {
|
|
|
390
513
|
}
|
|
391
514
|
this.push(entry, event);
|
|
392
515
|
}
|
|
516
|
+
keepTail(entry, task, text) {
|
|
517
|
+
let tail = entry.tails.get(task);
|
|
518
|
+
if (!tail) {
|
|
519
|
+
tail = [];
|
|
520
|
+
entry.tails.set(task, tail);
|
|
521
|
+
}
|
|
522
|
+
for (const line of text.replace(/\r?\n$/, '').split(/\r?\n/)) {
|
|
523
|
+
tail.push(line.length > TAIL_LINE_LENGTH ? `${line.slice(0, TAIL_LINE_LENGTH)}…` : line);
|
|
524
|
+
}
|
|
525
|
+
tail.splice(0, Math.max(0, tail.length - TAIL_LINES));
|
|
526
|
+
}
|
|
527
|
+
keepResult(entry, event) {
|
|
528
|
+
var _a;
|
|
529
|
+
const results = ((_a = entry.run).results ?? (_a.results = {}));
|
|
530
|
+
let tail = entry.tails.get(event.task);
|
|
531
|
+
if (!tail) {
|
|
532
|
+
tail = [];
|
|
533
|
+
entry.tails.set(event.task, tail);
|
|
534
|
+
}
|
|
535
|
+
const now = this.timestamp();
|
|
536
|
+
const previous = results[event.task];
|
|
537
|
+
// A task the run names in several scopes is checked again as a dependency of a later
|
|
538
|
+
// one: it is then cached or not needed, or stopped with the scope. What it did the
|
|
539
|
+
// first time is what the caller asked about.
|
|
540
|
+
if (previous &&
|
|
541
|
+
['succeeded', 'failed', 'cached'].includes(previous.status) &&
|
|
542
|
+
['cached', 'skipped', 'cancelled'].includes(event.state)) {
|
|
543
|
+
return;
|
|
544
|
+
}
|
|
545
|
+
const over = event.state !== 'running';
|
|
546
|
+
const result = {
|
|
547
|
+
status: event.state,
|
|
548
|
+
...(event.state === 'running'
|
|
549
|
+
? { startedAt: now }
|
|
550
|
+
: previous?.startedAt
|
|
551
|
+
? { startedAt: previous.startedAt }
|
|
552
|
+
: {}),
|
|
553
|
+
...(over ? { finishedAt: now } : {}),
|
|
554
|
+
...(event.durationMs !== undefined ? { durationMs: event.durationMs } : {}),
|
|
555
|
+
...(event.exitCode !== undefined ? { exitCode: event.exitCode } : {}),
|
|
556
|
+
...(event.error ? { error: event.error } : {}),
|
|
557
|
+
...(event.state === 'cached'
|
|
558
|
+
? { cache: { hit: true } }
|
|
559
|
+
: event.state === 'succeeded' || event.state === 'failed'
|
|
560
|
+
? { cache: { hit: false, ...(event.causes?.length ? { causes: [...event.causes] } : {}) } }
|
|
561
|
+
: {}),
|
|
562
|
+
// the same list the log adds to, so the result always has the newest lines
|
|
563
|
+
lastLog: tail,
|
|
564
|
+
};
|
|
565
|
+
results[event.task] = result;
|
|
566
|
+
}
|
|
393
567
|
push(entry, event) {
|
|
394
568
|
const stored = { ...event, seq: entry.events.length + 1, time: this.timestamp() };
|
|
395
569
|
entry.events.push(stored);
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DockerSandboxContainers = void 0;
|
|
4
|
+
const execute_docker_1 = require("../executer/execute-docker");
|
|
5
|
+
const get_docker_executor_1 = require("../executer/get-docker-executor");
|
|
6
|
+
const pull_1 = require("../docker/pull");
|
|
7
|
+
// what gives a sandbox back to the server's user: a shell with chown and chmod
|
|
8
|
+
const HELPER_IMAGE = 'alpine:3.21';
|
|
9
|
+
class DockerSandboxContainers {
|
|
10
|
+
constructor(host) {
|
|
11
|
+
this.docker = (0, execute_docker_1.createDockerClient)(host);
|
|
12
|
+
}
|
|
13
|
+
async reclaim(directory) {
|
|
14
|
+
const images = await this.docker.listImages({});
|
|
15
|
+
if (!(0, pull_1.isImagePresent)(images, HELPER_IMAGE)) {
|
|
16
|
+
const stream = await this.docker.pull(HELPER_IMAGE);
|
|
17
|
+
await new Promise((resolve, reject) => this.docker.modem.followProgress(stream, (err) => (err ? reject(err) : resolve())));
|
|
18
|
+
}
|
|
19
|
+
const user = `${process.getuid?.() ?? 0}:${process.getgid?.() ?? 0}`;
|
|
20
|
+
// a link in the sandbox points into this container, not to the server
|
|
21
|
+
const container = await this.docker.createContainer({
|
|
22
|
+
Image: HELPER_IMAGE,
|
|
23
|
+
Cmd: ['sh', '-c', `chown -R -h ${user} /sandbox && chmod -R u+rwX /sandbox`],
|
|
24
|
+
Labels: { app: 'hammerkit' },
|
|
25
|
+
NetworkDisabled: true,
|
|
26
|
+
HostConfig: { Binds: [`${directory}:/sandbox`], NetworkMode: 'none' },
|
|
27
|
+
});
|
|
28
|
+
try {
|
|
29
|
+
await container.start();
|
|
30
|
+
const result = await container.wait();
|
|
31
|
+
if (result.StatusCode !== 0) {
|
|
32
|
+
throw new Error(`giving the files back to the server failed with ${result.StatusCode}`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
finally {
|
|
36
|
+
await container.remove({ force: true }).catch(() => undefined);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
async volumes(directory) {
|
|
40
|
+
const { Volumes } = await this.docker.listVolumes({
|
|
41
|
+
filters: JSON.stringify({ label: [`${get_docker_executor_1.VOLUME_ROOT_LABEL}=${directory}`] }),
|
|
42
|
+
});
|
|
43
|
+
return (Volumes ?? []).map((volume) => volume.Name);
|
|
44
|
+
}
|
|
45
|
+
async volumeBytes(directory) {
|
|
46
|
+
const names = new Set(await this.volumes(directory));
|
|
47
|
+
if (names.size === 0) {
|
|
48
|
+
return 0;
|
|
49
|
+
}
|
|
50
|
+
const usage = await this.docker.df();
|
|
51
|
+
return (usage.Volumes ?? [])
|
|
52
|
+
.filter((volume) => names.has(volume.Name))
|
|
53
|
+
.reduce((bytes, volume) => bytes + Math.max(0, volume.UsageData?.Size ?? 0), 0);
|
|
54
|
+
}
|
|
55
|
+
async removeVolumes(directory) {
|
|
56
|
+
for (const name of await this.volumes(directory)) {
|
|
57
|
+
await this.docker.getVolume(name).remove({ force: true });
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
exports.DockerSandboxContainers = DockerSandboxContainers;
|
|
62
|
+
//# sourceMappingURL=sandbox-docker.js.map
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.handleSandboxFiles = exports.MAX_EDITS_BODY_BYTES = void 0;
|
|
4
|
+
const zod_1 = require("zod");
|
|
5
|
+
const git_checkout_1 = require("./git-checkout");
|
|
6
|
+
const http_1 = require("./http");
|
|
7
|
+
const sandbox_files_1 = require("./sandbox-files");
|
|
8
|
+
const sandboxes_1 = require("./sandboxes");
|
|
9
|
+
// a batch of edits carries the content it writes
|
|
10
|
+
exports.MAX_EDITS_BODY_BYTES = 16 * 1024 * 1024;
|
|
11
|
+
const hashSchema = zod_1.z.string().regex(/^[0-9a-fA-F]{64}$/, 'expected the sha256 of the file, 64 hex digits');
|
|
12
|
+
const opSchema = zod_1.z.discriminatedUnion('op', [
|
|
13
|
+
zod_1.z
|
|
14
|
+
.object({
|
|
15
|
+
op: zod_1.z.literal('replace'),
|
|
16
|
+
path: zod_1.z.string(),
|
|
17
|
+
old_string: zod_1.z.string(),
|
|
18
|
+
new_string: zod_1.z.string(),
|
|
19
|
+
replace_all: zod_1.z.boolean().optional(),
|
|
20
|
+
ifHash: hashSchema.optional(),
|
|
21
|
+
})
|
|
22
|
+
.strict(),
|
|
23
|
+
zod_1.z
|
|
24
|
+
.object({
|
|
25
|
+
op: zod_1.z.literal('write'),
|
|
26
|
+
path: zod_1.z.string(),
|
|
27
|
+
content: zod_1.z.string(),
|
|
28
|
+
encoding: zod_1.z.enum(['utf8', 'base64']).optional(),
|
|
29
|
+
ifHash: hashSchema.optional(),
|
|
30
|
+
})
|
|
31
|
+
.strict(),
|
|
32
|
+
zod_1.z.object({ op: zod_1.z.literal('delete'), path: zod_1.z.string(), ifHash: hashSchema.optional() }).strict(),
|
|
33
|
+
zod_1.z.object({ op: zod_1.z.literal('move'), path: zod_1.z.string(), to: zod_1.z.string(), ifHash: hashSchema.optional() }).strict(),
|
|
34
|
+
zod_1.z.object({ op: zod_1.z.literal('patch'), patch: zod_1.z.string().min(1) }).strict(),
|
|
35
|
+
]);
|
|
36
|
+
const editsSchema = zod_1.z.object({ dryRun: zod_1.z.boolean().optional(), ops: zod_1.z.array(opSchema).min(1).max(1000) }).strict();
|
|
37
|
+
// grep tools name the flags `-i`, `-A`: both spellings are taken
|
|
38
|
+
const searchSchema = zod_1.z.preprocess((body) => {
|
|
39
|
+
if (typeof body !== 'object' || body === null || Array.isArray(body)) {
|
|
40
|
+
return body;
|
|
41
|
+
}
|
|
42
|
+
const renamed = {};
|
|
43
|
+
for (const [key, value] of Object.entries(body)) {
|
|
44
|
+
renamed[/^-[iABC]$/.test(key) ? key.slice(1) : key] = value;
|
|
45
|
+
}
|
|
46
|
+
return renamed;
|
|
47
|
+
}, zod_1.z
|
|
48
|
+
.object({
|
|
49
|
+
pattern: zod_1.z.string().min(1).max(10000),
|
|
50
|
+
path: zod_1.z.string().optional(),
|
|
51
|
+
glob: zod_1.z
|
|
52
|
+
.union([zod_1.z.string().min(1).max(500), zod_1.z.array(zod_1.z.string().min(1).max(500)).max(20)])
|
|
53
|
+
.transform((glob) => (Array.isArray(glob) ? glob : [glob]))
|
|
54
|
+
.optional(),
|
|
55
|
+
type: zod_1.z.string().optional(),
|
|
56
|
+
i: zod_1.z.boolean().optional(),
|
|
57
|
+
fixed_strings: zod_1.z.boolean().optional(),
|
|
58
|
+
multiline: zod_1.z.boolean().optional(),
|
|
59
|
+
A: zod_1.z.number().int().min(0).max(100).optional(),
|
|
60
|
+
B: zod_1.z.number().int().min(0).max(100).optional(),
|
|
61
|
+
C: zod_1.z.number().int().min(0).max(100).optional(),
|
|
62
|
+
output_mode: zod_1.z.enum(['files_with_matches', 'content', 'count']).optional(),
|
|
63
|
+
head_limit: zod_1.z.number().int().min(1).max(5000).optional(),
|
|
64
|
+
cursor: zod_1.z.string().optional(),
|
|
65
|
+
})
|
|
66
|
+
.strict());
|
|
67
|
+
/**
|
|
68
|
+
* The files of a sandbox, for clients without tools of their own (typed) and for harnesses
|
|
69
|
+
* that only swap the backend of theirs (raw):
|
|
70
|
+
*
|
|
71
|
+
* GET /v1/sandboxes/<id>/tree?path=&depth=&glob=&limit=&cursor=
|
|
72
|
+
* GET /v1/sandboxes/<id>/files/<path>?offset=&limit= numbered lines
|
|
73
|
+
* GET /v1/sandboxes/<id>/files/<path>?raw bytes
|
|
74
|
+
* GET /v1/sandboxes/<id>/files/<path>?stat what it is
|
|
75
|
+
* HEAD /v1/sandboxes/<id>/files/<path> the same, as headers
|
|
76
|
+
* PUT /v1/sandboxes/<id>/files/<path> write bytes, If-Match: "<sha256>"
|
|
77
|
+
* POST /v1/sandboxes/<id>/search ripgrep
|
|
78
|
+
* POST /v1/sandboxes/<id>/edits { dryRun, ops }, all or nothing
|
|
79
|
+
* GET /v1/sandboxes/<id>/changes?format=summary|diff|patch
|
|
80
|
+
*
|
|
81
|
+
* Reading a file works at any time. Writes, and the reads that walk the files (tree, search,
|
|
82
|
+
* changes), are `409` while a run holds the sandbox.
|
|
83
|
+
*/
|
|
84
|
+
async function handleSandboxFiles(sandboxes, log, principal, sandbox, action, path, req, res, url) {
|
|
85
|
+
const who = `iss=${principal.iss} sub=${principal.sub}`;
|
|
86
|
+
const method = req.method;
|
|
87
|
+
try {
|
|
88
|
+
if (action === 'files') {
|
|
89
|
+
await handleFile(sandboxes, log, principal, sandbox, path, req, res, url);
|
|
90
|
+
}
|
|
91
|
+
else if (action === 'tree') {
|
|
92
|
+
if (method !== 'GET') {
|
|
93
|
+
return (0, http_1.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET' });
|
|
94
|
+
}
|
|
95
|
+
const query = {
|
|
96
|
+
path: url.searchParams.get('path') || undefined,
|
|
97
|
+
depth: intParam(url, 'depth'),
|
|
98
|
+
glob: url.searchParams.get('glob') || undefined,
|
|
99
|
+
limit: intParam(url, 'limit'),
|
|
100
|
+
cursor: url.searchParams.get('cursor') ?? undefined,
|
|
101
|
+
};
|
|
102
|
+
const tree = await sandboxes.read(principal, sandbox.id, (files) => files.tree(query));
|
|
103
|
+
(0, http_1.send)(res, 200, tree);
|
|
104
|
+
}
|
|
105
|
+
else if (action === 'search') {
|
|
106
|
+
if (method !== 'POST') {
|
|
107
|
+
return (0, http_1.send)(res, 405, { error: 'method not allowed' }, { allow: 'POST' });
|
|
108
|
+
}
|
|
109
|
+
const query = parse(searchSchema, await (0, http_1.readJson)(req));
|
|
110
|
+
(0, http_1.send)(res, 200, await sandboxes.read(principal, sandbox.id, (files) => files.search(query)));
|
|
111
|
+
}
|
|
112
|
+
else if (action === 'edits') {
|
|
113
|
+
if (method !== 'POST') {
|
|
114
|
+
return (0, http_1.send)(res, 405, { error: 'method not allowed' }, { allow: 'POST' });
|
|
115
|
+
}
|
|
116
|
+
const body = parse(editsSchema, await (0, http_1.readJson)(req, exports.MAX_EDITS_BODY_BYTES));
|
|
117
|
+
const files = await sandboxes.edit(principal, sandbox.id, body.ops, body.dryRun);
|
|
118
|
+
if (!body.dryRun) {
|
|
119
|
+
log(`sandbox edits ${sandbox.id} ${who} ops=${body.ops.length} files=${files.length}`);
|
|
120
|
+
}
|
|
121
|
+
(0, http_1.send)(res, 200, { dryRun: body.dryRun ?? false, files });
|
|
122
|
+
}
|
|
123
|
+
else {
|
|
124
|
+
if (method !== 'GET') {
|
|
125
|
+
return (0, http_1.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET' });
|
|
126
|
+
}
|
|
127
|
+
await handleChanges(sandboxes, principal, sandbox, res, url);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
catch (e) {
|
|
131
|
+
if (e instanceof sandbox_files_1.SandboxFileError) {
|
|
132
|
+
return (0, http_1.send)(res, e.status, { error: e.message, ...e.details });
|
|
133
|
+
}
|
|
134
|
+
if (e instanceof git_checkout_1.GitOutputTooLarge) {
|
|
135
|
+
return (0, http_1.send)(res, 413, { error: 'the changes are too large to answer, take them out in parts with a push' });
|
|
136
|
+
}
|
|
137
|
+
throw e;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
exports.handleSandboxFiles = handleSandboxFiles;
|
|
141
|
+
async function handleFile(sandboxes, log, principal, sandbox, path, req, res, url) {
|
|
142
|
+
const method = req.method;
|
|
143
|
+
const files = sandboxes.files(principal, sandbox.id);
|
|
144
|
+
if (method === 'HEAD') {
|
|
145
|
+
const stat = await files.stat(path);
|
|
146
|
+
if (!stat) {
|
|
147
|
+
res.writeHead(404, { 'cache-control': 'no-store' });
|
|
148
|
+
res.end();
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
res.writeHead(200, statHeaders(stat));
|
|
152
|
+
res.end();
|
|
153
|
+
}
|
|
154
|
+
else if (method === 'GET') {
|
|
155
|
+
if (url.searchParams.has('stat')) {
|
|
156
|
+
const stat = await files.stat(path);
|
|
157
|
+
return stat ? (0, http_1.send)(res, 200, stat) : (0, http_1.send)(res, 404, { error: `${path} does not exist` });
|
|
158
|
+
}
|
|
159
|
+
if (url.searchParams.has('raw')) {
|
|
160
|
+
const raw = await files.raw(path);
|
|
161
|
+
const headers = { ...statHeaders(raw.stat), 'content-type': 'application/octet-stream' };
|
|
162
|
+
if (raw.content) {
|
|
163
|
+
res.writeHead(200, { ...headers, etag: `"${raw.sha256}"` });
|
|
164
|
+
res.end(raw.content);
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
res.writeHead(200, headers);
|
|
168
|
+
const stream = raw.stream;
|
|
169
|
+
await new Promise((resolve) => {
|
|
170
|
+
stream.on('error', () => {
|
|
171
|
+
res.destroy();
|
|
172
|
+
resolve();
|
|
173
|
+
});
|
|
174
|
+
res.on('close', () => {
|
|
175
|
+
stream.destroy();
|
|
176
|
+
resolve();
|
|
177
|
+
});
|
|
178
|
+
stream.pipe(res);
|
|
179
|
+
});
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
(0, http_1.send)(res, 200, await files.read(path, { offset: intParam(url, 'offset'), limit: intParam(url, 'limit') }));
|
|
183
|
+
}
|
|
184
|
+
else if (method === 'PUT') {
|
|
185
|
+
const content = await readBytes(req, sandbox_files_1.MAX_FILE_BYTES);
|
|
186
|
+
const ifMatch = /^(?:W\/)?"?([0-9a-fA-F]{64})"?$/.exec(req.headers['if-match'] ?? '');
|
|
187
|
+
if (req.headers['if-match'] !== undefined && !ifMatch) {
|
|
188
|
+
return (0, http_1.send)(res, 400, { error: 'If-Match: expected the sha256 of the file in quotes' });
|
|
189
|
+
}
|
|
190
|
+
const [written] = await sandboxes.edit(principal, sandbox.id, [
|
|
191
|
+
{ op: 'write', path, content, ifHash: ifMatch?.[1] },
|
|
192
|
+
]);
|
|
193
|
+
log(`sandbox write ${sandbox.id} iss=${principal.iss} sub=${principal.sub} bytes=${content.length}`);
|
|
194
|
+
// writing what the file already holds changes nothing
|
|
195
|
+
(0, http_1.send)(res, written?.change === 'added' ? 201 : 200, written ?? { path, change: 'unchanged', size: content.length });
|
|
196
|
+
}
|
|
197
|
+
else {
|
|
198
|
+
(0, http_1.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET, HEAD, PUT' });
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
async function handleChanges(sandboxes, principal, sandbox, res, url) {
|
|
202
|
+
const format = url.searchParams.get('format') ?? 'summary';
|
|
203
|
+
if (format === 'summary') {
|
|
204
|
+
const files = await sandboxes.read(principal, sandbox.id, (files) => files.changes());
|
|
205
|
+
return (0, http_1.send)(res, 200, { commit: sandbox.commit, files });
|
|
206
|
+
}
|
|
207
|
+
if (format !== 'diff' && format !== 'patch') {
|
|
208
|
+
return (0, http_1.send)(res, 400, { error: 'format: expected summary, diff or patch' });
|
|
209
|
+
}
|
|
210
|
+
const author = {
|
|
211
|
+
name: textParam(url, 'name') ?? 'hammerkit',
|
|
212
|
+
email: textParam(url, 'email') ?? 'hammerkit@localhost',
|
|
213
|
+
message: textParam(url, 'message', 4000, true) ?? `Changes of sandbox ${sandbox.id}`,
|
|
214
|
+
};
|
|
215
|
+
const text = await sandboxes.read(principal, sandbox.id, (files) => files.diff(format, author));
|
|
216
|
+
res.writeHead(200, { 'content-type': 'text/x-diff; charset=utf-8', 'cache-control': 'no-store' });
|
|
217
|
+
res.end(text);
|
|
218
|
+
}
|
|
219
|
+
function statHeaders(stat) {
|
|
220
|
+
return {
|
|
221
|
+
'cache-control': 'no-store',
|
|
222
|
+
'content-length': String(stat.size),
|
|
223
|
+
'last-modified': new Date(stat.modifiedAt).toUTCString(),
|
|
224
|
+
'x-hammerkit-type': stat.type,
|
|
225
|
+
'x-hammerkit-exec': String(stat.exec),
|
|
226
|
+
...(stat.target !== undefined ? { 'x-hammerkit-target': encodeURIComponent(stat.target) } : {}),
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
function intParam(url, name) {
|
|
230
|
+
const value = url.searchParams.get(name);
|
|
231
|
+
if (value === null || value === '') {
|
|
232
|
+
return undefined;
|
|
233
|
+
}
|
|
234
|
+
if (!/^\d{1,9}$/.test(value)) {
|
|
235
|
+
throw new sandbox_files_1.SandboxFileError(400, `${name}: expected a whole number`);
|
|
236
|
+
}
|
|
237
|
+
return Number(value);
|
|
238
|
+
}
|
|
239
|
+
function textParam(url, name, max = 200, lines = false) {
|
|
240
|
+
const value = url.searchParams.get(name);
|
|
241
|
+
if (value === null || value === '') {
|
|
242
|
+
return undefined;
|
|
243
|
+
}
|
|
244
|
+
if (value.length > max || (lines ? /\0/ : /[\0\r\n<>]/).test(value)) {
|
|
245
|
+
throw new sandbox_files_1.SandboxFileError(400, `${name}: at most ${max} characters${lines ? '' : ', no line breaks or <>'}`);
|
|
246
|
+
}
|
|
247
|
+
return value;
|
|
248
|
+
}
|
|
249
|
+
async function readBytes(req, limit) {
|
|
250
|
+
const chunks = [];
|
|
251
|
+
let size = 0;
|
|
252
|
+
for await (const chunk of req) {
|
|
253
|
+
size += chunk.length;
|
|
254
|
+
if (size > limit) {
|
|
255
|
+
throw new http_1.HttpError(413, `the file is larger than ${limit} bytes`);
|
|
256
|
+
}
|
|
257
|
+
chunks.push(chunk);
|
|
258
|
+
}
|
|
259
|
+
return Buffer.concat(chunks);
|
|
260
|
+
}
|
|
261
|
+
function parse(schema, body) {
|
|
262
|
+
const parsed = schema.safeParse(body);
|
|
263
|
+
if (!parsed.success) {
|
|
264
|
+
const issue = parsed.error.issues[0];
|
|
265
|
+
throw new sandboxes_1.SandboxError(400, `${issue.path.join('.') || 'request'}: ${issue.message}`);
|
|
266
|
+
}
|
|
267
|
+
return parsed.data;
|
|
268
|
+
}
|
|
269
|
+
//# sourceMappingURL=sandbox-files-api.js.map
|