hammerkit 1.26.0 → 1.27.0-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build.schema.json +53 -1
- package/dist/cache/backends/local-cache-backend.js +10 -2
- package/dist/cache/explain.js +13 -11
- package/dist/cli.js +3 -0
- package/dist/docker/item-network.js +471 -0
- package/dist/docker/pull.js +4 -2
- package/dist/docker/remove-orphaned-containers.js +13 -6
- package/dist/environment/replace-env-variables.js +8 -6
- package/dist/executer/await-completed-dependencies.js +2 -1
- package/dist/executer/check-readiness.js +2 -2
- package/dist/executer/docker-service.js +24 -13
- package/dist/executer/docker-task.js +18 -29
- package/dist/executer/event-cache.js +4 -0
- package/dist/executer/execute-command.js +18 -5
- package/dist/executer/execute-docker.js +4 -1
- package/dist/executer/execution-steps.js +33 -7
- package/dist/executer/get-docker-executor.js +14 -7
- package/dist/executer/scheduler/enqueue-next.js +17 -3
- package/dist/executer/watch-loop.js +2 -2
- package/dist/executer/watch-state-key.js +1 -1
- package/dist/file/within-root.js +42 -0
- package/dist/harness/connect.js +99 -0
- package/dist/harness/format.js +73 -0
- package/dist/harness/mcp-command.js +59 -0
- package/dist/harness/mcp.js +432 -0
- package/dist/harness/opencode.js +492 -0
- package/dist/harness/pi.js +217 -0
- package/dist/harness/sandbox-client.js +373 -0
- package/dist/index.js +33 -36
- package/dist/kubernetes/apply.js +31 -1
- package/dist/kubernetes/await-running-state.js +33 -1
- package/dist/kubernetes/ensure-kubernetes-deployment-exists.js +43 -21
- package/dist/kubernetes/ensure-kubernetes-persistent-volume-claim-exists.js +131 -4
- package/dist/kubernetes/ensure-kubernetes-service-exists.js +25 -4
- package/dist/kubernetes/ensure-persistent-data.js +48 -7
- package/dist/kubernetes/get-service-ip.js +15 -1
- package/dist/kubernetes/job-placement.js +99 -0
- package/dist/kubernetes/job-secrets.js +306 -0
- package/dist/kubernetes/kept-claims.js +191 -0
- package/dist/kubernetes/kubernetes-instance.js +10 -2
- package/dist/kubernetes/network-gate.js +71 -0
- package/dist/kubernetes/network-policy.js +526 -0
- package/dist/kubernetes/resources.js +13 -2
- package/dist/kubernetes/restore-kubernetes-data.js +19 -4
- package/dist/kubernetes/secrets.js +2 -0
- package/dist/kubernetes/store-kubernetes-data.js +1 -1
- package/dist/kubernetes/volumes.js +100 -11
- package/dist/main.js +41 -0
- package/dist/network/egress-proxy.js +368 -0
- package/dist/network/port-relay.js +35 -0
- package/dist/optimizer/get-work-cache-stats.js +16 -0
- package/dist/optimizer/work-task-cache-description.js +3 -2
- package/dist/planner/service-definition.js +5 -0
- package/dist/planner/utils/append-work-environment.js +2 -0
- package/dist/planner/utils/append-work-service.js +4 -0
- package/dist/planner/utils/append-work-task.js +17 -1
- package/dist/planner/utils/assert-item-paths.js +50 -0
- package/dist/planner/utils/hosted-volumes.js +35 -0
- package/dist/planner/work-network.js +193 -0
- package/dist/planner/work-runtime-docker.js +41 -0
- package/dist/planner/work-runtime-kubernetes.js +335 -102
- package/dist/program.js +21 -0
- package/dist/remote/annotations.js +103 -0
- package/dist/remote/bundle-text.js +24 -0
- package/dist/remote/bundle.js +9 -1
- package/dist/remote/remote-commands.js +4 -1
- package/dist/remote/remote-work-commands.js +182 -69
- package/dist/remote/server-client.js +18 -5
- package/dist/remote/source-manifest.js +15 -2
- package/dist/remote/token-source.js +67 -0
- package/dist/schema/build-file-base-task-schema.js +3 -0
- package/dist/schema/build-file-container-service-schema.js +6 -0
- package/dist/schema/build-file-container-task-schema.js +3 -0
- package/dist/schema/build-file-environment-schema.js +9 -0
- package/dist/schema/network-schema.js +78 -0
- package/dist/schema/reference-parser.js +1 -0
- package/dist/schema/schema-parser.js +16 -3
- package/dist/schema/work-scope-parser.js +5 -0
- package/dist/secrets/command-secret-provider.js +73 -10
- package/dist/secrets/github-app-token-command.js +23 -0
- package/dist/secrets/github-app-token.js +191 -0
- package/dist/server/accounts-api.js +9 -2
- package/dist/server/definitions-api.js +69 -47
- package/dist/server/definitions.js +76 -51
- package/dist/server/executor.js +156 -41
- package/dist/server/git-checkout.js +72 -8
- package/dist/server/kubernetes-backend.js +554 -0
- package/dist/server/kubernetes-sandbox-storage.js +996 -0
- package/dist/server/oauth.js +1 -0
- package/dist/server/plan.js +51 -1
- package/dist/server/policy.js +104 -15
- package/dist/server/proposals-api.js +24 -40
- package/dist/server/proposals.js +12 -25
- package/dist/server/repo-access.js +37 -6
- package/dist/server/run-logs.js +217 -0
- package/dist/server/run-plan.js +51 -2
- package/dist/server/run-request.js +23 -10
- package/dist/server/run-store.js +5 -0
- package/dist/server/run-worker-command.js +13 -1
- package/dist/server/run-worker.js +121 -24
- package/dist/server/runs.js +184 -10
- package/dist/server/sandbox-docker.js +62 -0
- package/dist/server/sandbox-files-api.js +269 -0
- package/dist/server/sandbox-files.js +1054 -0
- package/dist/server/sandbox-patch.js +481 -0
- package/dist/server/sandbox-pod-command.js +15 -0
- package/dist/server/sandbox-pod.js +146 -0
- package/dist/server/sandbox-storage.js +462 -0
- package/dist/server/sandboxes-api.js +210 -0
- package/dist/server/sandboxes.js +519 -0
- package/dist/server/server-config.js +412 -22
- package/dist/server/server.js +204 -39
- package/dist/utils/run-id.js +12 -0
- package/harness/opencode.ts +11 -0
- package/harness/pi.ts +7 -0
- package/package.json +8 -2
- package/dist/remote/text-diff.js +0 -89
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.readLogs = exports.LogIndex = exports.parseLogQuery = exports.MAX_LOG_LINES = void 0;
|
|
4
|
+
// what one call returns at most, and by default
|
|
5
|
+
exports.MAX_LOG_LINES = 1000;
|
|
6
|
+
const DEFAULT_LOG_LINES = 200;
|
|
7
|
+
const MAX_CONTEXT = 20;
|
|
8
|
+
const MAX_PATTERN = 200;
|
|
9
|
+
const PARAMETERS = ['task', 'tail', 'from', 'limit', 'grep', 'i', 'context'];
|
|
10
|
+
function count(params, name, min, max) {
|
|
11
|
+
const value = params.get(name);
|
|
12
|
+
if (value === null) {
|
|
13
|
+
return undefined;
|
|
14
|
+
}
|
|
15
|
+
if (!/^\d{1,9}$/.test(value) || Number(value) < min || Number(value) > max) {
|
|
16
|
+
throw new Error(`${name} is a number from ${min} to ${max}`);
|
|
17
|
+
}
|
|
18
|
+
return Number(value);
|
|
19
|
+
}
|
|
20
|
+
// Reads the query of `GET /v1/runs/<id>/logs`; throws with what is wrong.
|
|
21
|
+
function parseLogQuery(params) {
|
|
22
|
+
for (const name of new Set(params.keys())) {
|
|
23
|
+
if (!PARAMETERS.includes(name)) {
|
|
24
|
+
throw new Error(`there is no parameter ${name}, the log takes ${PARAMETERS.join(', ')}`);
|
|
25
|
+
}
|
|
26
|
+
if (params.getAll(name).length > 1) {
|
|
27
|
+
throw new Error(`${name} is given more than once`);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
const tail = count(params, 'tail', 1, exports.MAX_LOG_LINES);
|
|
31
|
+
const from = count(params, 'from', 1, 999999999);
|
|
32
|
+
if (tail !== undefined && from !== undefined) {
|
|
33
|
+
throw new Error('tail and from do not go together, tail reads the end and from reads on from a line');
|
|
34
|
+
}
|
|
35
|
+
const grep = params.get('grep') ?? undefined;
|
|
36
|
+
if (grep !== undefined && (grep.length === 0 || grep.length > MAX_PATTERN || /[\r\n]/.test(grep))) {
|
|
37
|
+
throw new Error(`grep is a text of 1 to ${MAX_PATTERN} characters on one line`);
|
|
38
|
+
}
|
|
39
|
+
const i = params.get('i');
|
|
40
|
+
if (i !== null && !['true', 'false', '1', '0'].includes(i)) {
|
|
41
|
+
throw new Error('i is true or false');
|
|
42
|
+
}
|
|
43
|
+
const task = params.get('task') ?? undefined;
|
|
44
|
+
if (task === '') {
|
|
45
|
+
throw new Error('task is the name of a task, leave it out for the lines of every task');
|
|
46
|
+
}
|
|
47
|
+
return {
|
|
48
|
+
...(task !== undefined ? { task } : {}),
|
|
49
|
+
tail,
|
|
50
|
+
from,
|
|
51
|
+
// `tail=500` means 500 lines, unless a smaller limit says otherwise
|
|
52
|
+
limit: count(params, 'limit', 1, exports.MAX_LOG_LINES) ?? tail ?? DEFAULT_LOG_LINES,
|
|
53
|
+
grep,
|
|
54
|
+
ignoreCase: i === 'true' || i === '1',
|
|
55
|
+
context: count(params, 'context', 0, MAX_CONTEXT) ?? 0,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
exports.parseLogQuery = parseLogQuery;
|
|
59
|
+
/**
|
|
60
|
+
* Where the lines of a run's log are: built as the log grows, so a read costs the lines it
|
|
61
|
+
* returns and not the whole log. A line is a range of the text of one log event, the line
|
|
62
|
+
* break and a `\r` before it not included.
|
|
63
|
+
*/
|
|
64
|
+
class LogIndex {
|
|
65
|
+
constructor() {
|
|
66
|
+
// the events looked at so far
|
|
67
|
+
this.scanned = 0;
|
|
68
|
+
this.lineEvent = [];
|
|
69
|
+
this.lineStart = [];
|
|
70
|
+
this.lineEnd = [];
|
|
71
|
+
// per task, its lines by their index in the whole log
|
|
72
|
+
this.taskLines = new Map();
|
|
73
|
+
// per log event, its first line and how many it has
|
|
74
|
+
this.eventLines = new Map();
|
|
75
|
+
}
|
|
76
|
+
update(events) {
|
|
77
|
+
for (; this.scanned < events.length; this.scanned++) {
|
|
78
|
+
const event = events[this.scanned];
|
|
79
|
+
if (event.type !== 'log') {
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
let lines = this.taskLines.get(event.task);
|
|
83
|
+
if (!lines) {
|
|
84
|
+
lines = [];
|
|
85
|
+
this.taskLines.set(event.task, lines);
|
|
86
|
+
}
|
|
87
|
+
const text = event.text;
|
|
88
|
+
const end = text.endsWith('\r\n') ? text.length - 2 : text.endsWith('\n') ? text.length - 1 : text.length;
|
|
89
|
+
const first = this.lineStart.length;
|
|
90
|
+
let start = 0;
|
|
91
|
+
for (;;) {
|
|
92
|
+
const lineBreak = text.indexOf('\n', start);
|
|
93
|
+
const stop = lineBreak < 0 || lineBreak >= end ? end : lineBreak;
|
|
94
|
+
this.lineEvent.push(this.scanned);
|
|
95
|
+
this.lineStart.push(start);
|
|
96
|
+
this.lineEnd.push(stop > start && text[stop - 1] === '\r' ? stop - 1 : stop);
|
|
97
|
+
lines.push(this.lineStart.length - 1);
|
|
98
|
+
if (stop >= end) {
|
|
99
|
+
break;
|
|
100
|
+
}
|
|
101
|
+
start = stop + 1;
|
|
102
|
+
}
|
|
103
|
+
this.eventLines.set(this.scanned, [first, this.lineStart.length - first]);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
get size() {
|
|
107
|
+
return this.lineStart.length;
|
|
108
|
+
}
|
|
109
|
+
lines(task) {
|
|
110
|
+
return this.taskLines.get(task) ?? [];
|
|
111
|
+
}
|
|
112
|
+
text(events, line) {
|
|
113
|
+
const event = events[this.lineEvent[line]];
|
|
114
|
+
return { task: event.task, stream: event.stream, text: event.text.slice(this.lineStart[line], this.lineEnd[line]) };
|
|
115
|
+
}
|
|
116
|
+
// The lines that contain `needle`, found per event so a line is not copied to be searched.
|
|
117
|
+
matches(events, needle, ignoreCase) {
|
|
118
|
+
const hits = new Uint8Array(this.size);
|
|
119
|
+
// a literal pattern, so the search is linear; the `i` flag folds case without changing
|
|
120
|
+
// where in the text a match is
|
|
121
|
+
const pattern = new RegExp(needle.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), ignoreCase ? 'gi' : 'g');
|
|
122
|
+
for (const [index, [first, length]] of this.eventLines) {
|
|
123
|
+
const text = events[index].text;
|
|
124
|
+
pattern.lastIndex = 0;
|
|
125
|
+
let line = first;
|
|
126
|
+
let found;
|
|
127
|
+
while ((found = pattern.exec(text)) !== null) {
|
|
128
|
+
while (line < first + length - 1 && this.lineEnd[line] < found.index + needle.length) {
|
|
129
|
+
line++;
|
|
130
|
+
}
|
|
131
|
+
if (this.lineEnd[line] < found.index + needle.length) {
|
|
132
|
+
break;
|
|
133
|
+
}
|
|
134
|
+
hits[line] = 1;
|
|
135
|
+
// one hit per line is enough
|
|
136
|
+
pattern.lastIndex = this.lineEnd[line] + 1;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
return hits;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
exports.LogIndex = LogIndex;
|
|
143
|
+
// The lines of the log a query selects, see LogQuery.
|
|
144
|
+
function readLogs(events, index, query, truncated) {
|
|
145
|
+
const selected = query.task !== undefined ? index.lines(query.task) : undefined;
|
|
146
|
+
const total = selected ? selected.length : index.size;
|
|
147
|
+
const at = (position) => (selected ? selected[position] : position);
|
|
148
|
+
const hits = query.grep !== undefined ? index.matches(events, query.grep, query.ignoreCase) : undefined;
|
|
149
|
+
const toLine = (position) => ({
|
|
150
|
+
line: position + 1,
|
|
151
|
+
...index.text(events, at(position)),
|
|
152
|
+
...(hits ? { match: hits[at(position)] === 1 } : {}),
|
|
153
|
+
});
|
|
154
|
+
// the positions of the selection to answer, in order
|
|
155
|
+
const positions = [];
|
|
156
|
+
const want = query.tail !== undefined ? Math.min(query.tail, query.limit) : query.limit;
|
|
157
|
+
let next;
|
|
158
|
+
if (!hits) {
|
|
159
|
+
if (query.tail !== undefined) {
|
|
160
|
+
for (let position = Math.max(0, total - want); position < total; position++) {
|
|
161
|
+
positions.push(position);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
const start = (query.from ?? 1) - 1;
|
|
166
|
+
for (let position = start; position < Math.min(total, start + query.limit); position++) {
|
|
167
|
+
positions.push(position);
|
|
168
|
+
}
|
|
169
|
+
next = start + query.limit < total ? start + query.limit + 1 : undefined;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
else if (query.tail !== undefined) {
|
|
173
|
+
// from the end: the last matches with the lines around them
|
|
174
|
+
let lowest = total;
|
|
175
|
+
for (let position = total - 1; position >= 0 && positions.length < want; position--) {
|
|
176
|
+
if (hits[at(position)] !== 1) {
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
const high = Math.min(total - 1, position + query.context, lowest - 1);
|
|
180
|
+
for (let near = high; near >= Math.max(0, position - query.context) && positions.length < want; near--) {
|
|
181
|
+
positions.push(near);
|
|
182
|
+
lowest = near;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
positions.reverse();
|
|
186
|
+
}
|
|
187
|
+
else {
|
|
188
|
+
const start = (query.from ?? 1) - 1;
|
|
189
|
+
// a match just before `from` still shows the lines after it
|
|
190
|
+
let emitted = start - 1;
|
|
191
|
+
for (let position = Math.max(0, start - query.context); position < total; position++) {
|
|
192
|
+
if (hits[at(position)] !== 1) {
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
const high = Math.min(total - 1, position + query.context);
|
|
196
|
+
for (let near = Math.max(emitted + 1, position - query.context); near <= high; near++) {
|
|
197
|
+
if (positions.length === query.limit) {
|
|
198
|
+
next = near + 1;
|
|
199
|
+
break;
|
|
200
|
+
}
|
|
201
|
+
positions.push(near);
|
|
202
|
+
emitted = near;
|
|
203
|
+
}
|
|
204
|
+
if (next !== undefined) {
|
|
205
|
+
break;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
return {
|
|
210
|
+
lines: positions.map(toLine),
|
|
211
|
+
total,
|
|
212
|
+
...(next !== undefined ? { next } : {}),
|
|
213
|
+
truncated,
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
exports.readLogs = readLogs;
|
|
217
|
+
//# sourceMappingURL=run-logs.js.map
|
package/dist/server/run-plan.js
CHANGED
|
@@ -1,17 +1,22 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.planRun = exports.getRunScopes = void 0;
|
|
3
|
+
exports.refuseArguments = exports.planRun = exports.getRunScopes = void 0;
|
|
4
4
|
const parse_label_arguments_1 = require("../parser/parse-label-arguments");
|
|
5
5
|
const plan_1 = require("./plan");
|
|
6
6
|
function labels(values) {
|
|
7
7
|
return values && values.length > 0 ? (0, parse_label_arguments_1.parseLabelArguments)(values) : undefined;
|
|
8
8
|
}
|
|
9
9
|
// The tasks in the order they were named, one scope each; by labels when none is.
|
|
10
|
+
// The arguments, if any, go to the one task named (the request allows no more). That
|
|
11
|
+
// task runs uncached unless the request names a `cache` method, as `run --cache` does.
|
|
10
12
|
function getRunScopes(tasks, options) {
|
|
13
|
+
const args = options.args && options.args.length > 0
|
|
14
|
+
? { values: options.args, services: false, cache: options.cache !== undefined }
|
|
15
|
+
: undefined;
|
|
11
16
|
return tasks.length > 0
|
|
12
17
|
? tasks.map((task) => ({
|
|
13
18
|
name: task,
|
|
14
|
-
scope: { taskName: task, environmentName: options.env ?? null },
|
|
19
|
+
scope: { taskName: task, environmentName: options.env ?? null, ...(args ? { args } : {}) },
|
|
15
20
|
}))
|
|
16
21
|
: [
|
|
17
22
|
{
|
|
@@ -41,4 +46,48 @@ async function planRun(bundle, run, environment, policy) {
|
|
|
41
46
|
};
|
|
42
47
|
}
|
|
43
48
|
exports.planRun = planRun;
|
|
49
|
+
// Variables of a server that hold no credential (FR-008): a task may read these and
|
|
50
|
+
// still take arguments.
|
|
51
|
+
const PLAIN_SERVER_ENVS = ['PATH', 'HOME', 'CI'];
|
|
52
|
+
function readsOf(entry) {
|
|
53
|
+
return [
|
|
54
|
+
...entry.secrets.map((secret) => `the secret ${secret.target}`),
|
|
55
|
+
...(entry.serverEnvs ?? [])
|
|
56
|
+
.filter((env) => !PLAIN_SERVER_ENVS.includes(env.toUpperCase()))
|
|
57
|
+
.map((env) => `$${env} of the server`),
|
|
58
|
+
];
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Why a run may not pass its arguments, or null when it may. They go to a task, never a
|
|
62
|
+
* service, and never to a run in which an item reads a secret or takes a variable of the
|
|
63
|
+
* server into its `envs`: an argument can make the command print what the task holds,
|
|
64
|
+
* and what its dependencies write into their outputs or its services serve, past the
|
|
65
|
+
* masking of secrets. So a shell on a server runs with no account and no credential of
|
|
66
|
+
* the server.
|
|
67
|
+
*/
|
|
68
|
+
function refuseArguments(run, plan) {
|
|
69
|
+
if (!run.options.args || run.options.args.length === 0) {
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
const [name] = run.tasks;
|
|
73
|
+
const entry = plan[name];
|
|
74
|
+
// a service named as a task is planned as nothing
|
|
75
|
+
if (!entry || entry.kind !== 'task') {
|
|
76
|
+
return `${name} is not a task of this version`;
|
|
77
|
+
}
|
|
78
|
+
const reads = readsOf(entry);
|
|
79
|
+
if (reads.length > 0) {
|
|
80
|
+
return `${name} reads ${reads.join(', ')}, it takes no arguments on a server: an argument could make it print them`;
|
|
81
|
+
}
|
|
82
|
+
// what a dependency or a service of the run holds reaches the task through its outputs or
|
|
83
|
+
// its answers, where an argument can print it in a form no mask knows (`rev token.txt`)
|
|
84
|
+
for (const [other, item] of Object.entries(plan).sort(([a], [b]) => a.localeCompare(b))) {
|
|
85
|
+
const held = (other === name ? [] : readsOf(item)).join(', ');
|
|
86
|
+
if (held.length > 0) {
|
|
87
|
+
return `${name} takes no arguments on a server: ${other} in its run reads ${held}, and an argument could make ${name} print what ${other} hands on`;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return null;
|
|
91
|
+
}
|
|
92
|
+
exports.refuseArguments = refuseArguments;
|
|
44
93
|
//# sourceMappingURL=run-plan.js.map
|
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.runRequestSchema = exports.runOptionsSchema = void 0;
|
|
3
|
+
exports.runRequestSchema = exports.runOptionsSchema = exports.MAX_ARGS_BYTES = exports.MAX_ARGS = void 0;
|
|
4
4
|
const zod_1 = require("zod");
|
|
5
5
|
const units_1 = require("../utils/units");
|
|
6
6
|
const bundle_1 = require("../remote/bundle");
|
|
7
|
-
const server_config_1 = require("./server-config");
|
|
8
7
|
const repo_1 = require("./repo");
|
|
8
|
+
const runs_1 = require("./runs");
|
|
9
9
|
const name = zod_1.z.string().min(1).max(200);
|
|
10
|
+
// what one run may pass after `--`
|
|
11
|
+
exports.MAX_ARGS = 100;
|
|
12
|
+
exports.MAX_ARGS_BYTES = 16 * 1024;
|
|
10
13
|
// What `POST /v1/runs` takes. Who is asking and which accounts they may use is
|
|
11
14
|
// decided from the token, never from this body; the accounts a run uses are those
|
|
12
15
|
// its secrets are read as, which the server derives from the published build file.
|
|
@@ -37,18 +40,20 @@ exports.runOptionsSchema = zod_1.z
|
|
|
37
40
|
timeout: duration.optional(),
|
|
38
41
|
// as --no-skip-deps when false
|
|
39
42
|
skipDeps: zod_1.z.boolean().optional(),
|
|
43
|
+
// as the text after `--`: appended to the last command of the one task the run
|
|
44
|
+
// names, each quoted for its shell. A task that reads a secret or a variable of the
|
|
45
|
+
// server takes none (see refuseArguments).
|
|
46
|
+
args: zod_1.z
|
|
47
|
+
.array(zod_1.z.string().refine((arg) => !arg.includes('\0'), 'an argument can not contain a NUL character'))
|
|
48
|
+
.max(exports.MAX_ARGS)
|
|
49
|
+
.refine((args) => args.reduce((total, arg) => total + Buffer.byteLength(arg), 0) <= exports.MAX_ARGS_BYTES, `the arguments are longer than ${exports.MAX_ARGS_BYTES / 1024} KiB together`)
|
|
50
|
+
.optional(),
|
|
40
51
|
})
|
|
41
52
|
.strict();
|
|
42
53
|
exports.runRequestSchema = zod_1.z
|
|
43
54
|
.object({
|
|
44
|
-
// the published build file to run,
|
|
45
|
-
|
|
46
|
-
definition: zod_1.z
|
|
47
|
-
.object({
|
|
48
|
-
name: server_config_1.definitionName,
|
|
49
|
-
hash: zod_1.z.union([zod_1.z.literal('latest'), zod_1.z.string().regex(bundle_1.HASH_PATTERN)]).default('latest'),
|
|
50
|
-
})
|
|
51
|
-
.strict(),
|
|
55
|
+
// the published version of the build file to run, by its hash
|
|
56
|
+
definition: zod_1.z.string().regex(bundle_1.HASH_PATTERN, 'expected the hash of a published build file, sha256:...'),
|
|
52
57
|
// tasks by their name in the build file, as on the command line; labels in
|
|
53
58
|
// the options can select tasks instead
|
|
54
59
|
tasks: zod_1.z.array(name).max(100).default([]),
|
|
@@ -74,6 +79,10 @@ exports.runRequestSchema = zod_1.z
|
|
|
74
79
|
.strict(),
|
|
75
80
|
// the name of a tree announced with POST /v1/sources
|
|
76
81
|
zod_1.z.object({ type: zod_1.z.literal('upload'), snapshot: zod_1.z.string().regex(bundle_1.HASH_PATTERN) }).strict(),
|
|
82
|
+
// a sandbox of the caller's, see POST /v1/sandboxes
|
|
83
|
+
zod_1.z
|
|
84
|
+
.object({ type: zod_1.z.literal('sandbox'), id: zod_1.z.string().regex(runs_1.SANDBOX_ID_PATTERN, 'expected a sandbox id') })
|
|
85
|
+
.strict(),
|
|
77
86
|
]),
|
|
78
87
|
options: exports.runOptionsSchema.default({}),
|
|
79
88
|
})
|
|
@@ -81,5 +90,9 @@ exports.runRequestSchema = zod_1.z
|
|
|
81
90
|
.refine((request) => request.tasks.length > 0 || (request.options.filter ?? []).length > 0, {
|
|
82
91
|
message: 'name at least one task, or filter by labels',
|
|
83
92
|
path: ['tasks'],
|
|
93
|
+
})
|
|
94
|
+
.refine((request) => (request.options.args ?? []).length === 0 || request.tasks.length === 1, {
|
|
95
|
+
message: 'arguments go to one task, name exactly one in tasks',
|
|
96
|
+
path: ['options', 'args'],
|
|
84
97
|
});
|
|
85
98
|
//# sourceMappingURL=run-request.js.map
|
package/dist/server/run-store.js
CHANGED
|
@@ -47,6 +47,11 @@ class FileRunStore {
|
|
|
47
47
|
if (!runs_1.RUN_ID_PATTERN.test(record.run.id) || `${record.run.id}.json` !== name || !Array.isArray(record.events)) {
|
|
48
48
|
throw new Error('not a run record');
|
|
49
49
|
}
|
|
50
|
+
// an older server kept the name the build file was published under with its hash
|
|
51
|
+
const definition = record.run.definition;
|
|
52
|
+
if (definition && typeof definition === 'object' && 'hash' in definition) {
|
|
53
|
+
record.run.definition = String(definition.hash);
|
|
54
|
+
}
|
|
50
55
|
records.push(record);
|
|
51
56
|
}
|
|
52
57
|
catch {
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.registerRunWorkerCommand = void 0;
|
|
3
|
+
exports.registerSandboxPodCommand = exports.registerRunWorkerCommand = void 0;
|
|
4
4
|
const run_worker_1 = require("./run-worker");
|
|
5
|
+
const sandbox_pod_command_1 = require("./sandbox-pod-command");
|
|
5
6
|
// `hammerkit server-run`: what the server starts for each run. It reads the run on its
|
|
6
7
|
// standard input, executes it in the current directory and reports on its standard output.
|
|
7
8
|
// Not for people: the input comes from the server and says which accounts it may read.
|
|
@@ -14,4 +15,15 @@ function registerRunWorkerCommand(program, environment) {
|
|
|
14
15
|
});
|
|
15
16
|
}
|
|
16
17
|
exports.registerRunWorkerCommand = registerRunWorkerCommand;
|
|
18
|
+
// `hammerkit server-sandbox <directory>`, see sandbox-pod-command.ts. index.ts starts it
|
|
19
|
+
// without loading the rest of the CLI; the program has it too, as every command.
|
|
20
|
+
function registerSandboxPodCommand(program, environment) {
|
|
21
|
+
program
|
|
22
|
+
.command('server-sandbox <directory>', { hidden: true })
|
|
23
|
+
.description('read or change the files of a sandbox for a hammerkit server (internal)')
|
|
24
|
+
.action(async (directory) => {
|
|
25
|
+
await (0, sandbox_pod_command_1.runSandboxPodCommand)(directory, process.stdin, environment.stdout);
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
exports.registerSandboxPodCommand = registerSandboxPodCommand;
|
|
17
29
|
//# sourceMappingURL=run-worker-command.js.map
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.workerMain = exports.runTasks = exports.workerInputSchema = void 0;
|
|
3
|
+
exports.workerMain = exports.taskCauses = exports.runTasks = exports.getRunNetwork = exports.workerInputSchema = void 0;
|
|
4
|
+
const readline_1 = require("readline");
|
|
4
5
|
const zod_1 = require("zod");
|
|
5
6
|
const cli_1 = require("../cli");
|
|
6
7
|
const get_file_context_1 = require("../file/get-file-context");
|
|
@@ -10,11 +11,16 @@ const reference_parser_1 = require("../schema/reference-parser");
|
|
|
10
11
|
const work_scope_parser_1 = require("../schema/work-scope-parser");
|
|
11
12
|
const work_item_status_1 = require("../planner/work-item-status");
|
|
12
13
|
const plan_work_tasks_1 = require("../planner/utils/plan-work-tasks");
|
|
14
|
+
const assert_item_paths_1 = require("../planner/utils/assert-item-paths");
|
|
15
|
+
const hosted_volumes_1 = require("../planner/utils/hosted-volumes");
|
|
13
16
|
const redact_1 = require("../utils/redact");
|
|
14
17
|
const empty_writable_1 = require("../utils/empty-writable");
|
|
15
18
|
const units_1 = require("../utils/units");
|
|
16
19
|
const run_request_1 = require("./run-request");
|
|
20
|
+
const network_schema_1 = require("../schema/network-schema");
|
|
21
|
+
const work_network_1 = require("../planner/work-network");
|
|
17
22
|
const run_plan_1 = require("./run-plan");
|
|
23
|
+
const runs_1 = require("./runs");
|
|
18
24
|
// one log line of a task is cut here, the registry caps how many events a run keeps
|
|
19
25
|
const MAX_LOG_TEXT = 8 * 1024;
|
|
20
26
|
/**
|
|
@@ -27,14 +33,41 @@ exports.workerInputSchema = zod_1.z
|
|
|
27
33
|
.object({
|
|
28
34
|
// the build file to start from, in the checkout
|
|
29
35
|
entry: zod_1.z.string().min(1),
|
|
36
|
+
// set when the build files are read from a directory of the run and work on the
|
|
37
|
+
// files of a sandbox
|
|
38
|
+
directory: zod_1.z
|
|
39
|
+
.object({ read: zod_1.z.string().min(1), work: zod_1.z.string().min(1) })
|
|
40
|
+
.strict()
|
|
41
|
+
.optional(),
|
|
30
42
|
tasks: zod_1.z.array(zod_1.z.string()),
|
|
31
43
|
options: run_request_1.runOptionsSchema,
|
|
32
44
|
// the accounts the run was admitted for; a secret of any other is refused
|
|
33
45
|
accounts: zod_1.z.array(zod_1.z.string()),
|
|
34
46
|
// the operator's providers and accounts, as the server read them from its config
|
|
35
47
|
catalog: zod_1.z.custom((value) => typeof value === 'object' && value !== null),
|
|
48
|
+
// the cluster and namespace container tasks and services run in, on a Kubernetes backend
|
|
49
|
+
kubernetes: zod_1.z
|
|
50
|
+
.custom((value) => typeof value === 'object' && value !== null && value.type === 'kubernetes')
|
|
51
|
+
.optional(),
|
|
52
|
+
// the most any task or service reaches besides its needs, as `--network`: `runs.network`
|
|
53
|
+
network: network_schema_1.networkSchema.optional(),
|
|
54
|
+
// for a run on a sandbox, `sandboxes.network`: the run gets the lower of the two
|
|
55
|
+
sandboxNetwork: network_schema_1.networkSchema.optional(),
|
|
56
|
+
// the caller and source trust of the run, in front of the names of its services' docker
|
|
57
|
+
// volumes (see scopeServiceVolumes)
|
|
58
|
+
volumeScope: zod_1.z
|
|
59
|
+
.string()
|
|
60
|
+
.regex(/^[a-z]+-[0-9a-f]{24}$/)
|
|
61
|
+
.optional(),
|
|
36
62
|
})
|
|
37
63
|
.strict();
|
|
64
|
+
// The ceiling of the run, combined here and not in the server: the lower of two allow lists
|
|
65
|
+
// marks what one allows only through a pattern, which a build file's `network` can't say.
|
|
66
|
+
function getRunNetwork(input) {
|
|
67
|
+
const runs = input.network ? (0, work_network_1.parseWorkNetwork)(input.network) : null;
|
|
68
|
+
return input.sandboxNetwork ? (0, work_network_1.limitNetwork)((0, work_network_1.parseWorkNetwork)(input.sandboxNetwork), runs) : runs;
|
|
69
|
+
}
|
|
70
|
+
exports.getRunNetwork = getRunNetwork;
|
|
38
71
|
/**
|
|
39
72
|
* Executes the tasks of a run with the engine of `hammerkit run`. The caller says which
|
|
40
73
|
* process this is in: it reports what happens through `emit` and stops when `abortCtrl`
|
|
@@ -45,14 +78,16 @@ async function runTasks(input, cwd, processEnvs, abortCtrl,
|
|
|
45
78
|
cancelled, emit) {
|
|
46
79
|
const secrets = (0, redact_1.createSecretRegistry)();
|
|
47
80
|
const status = (0, work_item_status_1.statusConsole)((0, empty_writable_1.emptyWritable)(), secrets);
|
|
81
|
+
// the process starts in a directory of the server, the tasks work on the run's files
|
|
82
|
+
const work = input.directory?.work ?? cwd;
|
|
48
83
|
// what the engine would print to a terminal is not wanted, its status console
|
|
49
84
|
// is what is streamed: it is redacted with the secrets of this run
|
|
50
85
|
const environment = {
|
|
51
86
|
processEnvs,
|
|
52
|
-
hosted: { catalog: input.catalog, accounts: input.accounts },
|
|
87
|
+
hosted: { catalog: input.catalog, accounts: input.accounts, kubernetes: input.kubernetes },
|
|
53
88
|
abortCtrl,
|
|
54
|
-
cwd,
|
|
55
|
-
file: (0, get_file_context_1.getFileContext)(
|
|
89
|
+
cwd: work,
|
|
90
|
+
file: (0, get_file_context_1.getFileContext)(work),
|
|
56
91
|
console: (0, log_1.consoleContext)((0, empty_writable_1.emptyWritable)()),
|
|
57
92
|
status,
|
|
58
93
|
secrets,
|
|
@@ -86,20 +121,30 @@ cancelled, emit) {
|
|
|
86
121
|
cacheReadOnly: options.cacheReadOnly ?? false,
|
|
87
122
|
timeout: options.timeout ? (0, units_1.parseDuration)(options.timeout) : null,
|
|
88
123
|
skipDeps: options.skipDeps !== false,
|
|
124
|
+
network: getRunNetwork(input),
|
|
125
|
+
// why a task was not taken from the cache goes into its result
|
|
126
|
+
explain: true,
|
|
89
127
|
};
|
|
90
128
|
for (const { name, scope } of (0, run_plan_1.getRunScopes)(input.tasks, options)) {
|
|
91
|
-
const { ctx, scope: parsed } = await (0, schema_parser_1.createParseContext)(input.entry, environment
|
|
129
|
+
const { ctx, scope: parsed } = await (0, schema_parser_1.createParseContext)(input.entry, environment, {
|
|
130
|
+
directory: input.directory,
|
|
131
|
+
});
|
|
92
132
|
const workTree = (0, work_scope_parser_1.getWorkContext)(await (0, reference_parser_1.parseReferences)(ctx, parsed, environment), scope, environment);
|
|
133
|
+
await (0, assert_item_paths_1.assertWorkTreePathsWithinRoot)(workTree, work);
|
|
134
|
+
if (input.volumeScope && !input.kubernetes) {
|
|
135
|
+
(0, hosted_volumes_1.scopeServiceVolumes)(workTree, input.volumeScope);
|
|
136
|
+
}
|
|
93
137
|
const cli = (0, cli_1.getCli)(workTree, environment);
|
|
94
138
|
if (cli.tasks().length === 0) {
|
|
95
139
|
throw new Error(`no tasks found for ${name}`);
|
|
96
140
|
}
|
|
97
141
|
const execution = cli.exec({ ...execOptions, workers: cli.defaultWorkers() });
|
|
98
142
|
const seen = new Map();
|
|
99
|
-
const
|
|
143
|
+
const report = () => reportTasks(emit, execution.state.current, seen, (text) => secrets.redact(text));
|
|
144
|
+
const watch = execution.state.on('run-events', report);
|
|
100
145
|
try {
|
|
101
146
|
const result = await execution.start();
|
|
102
|
-
|
|
147
|
+
report();
|
|
103
148
|
if (!result.success) {
|
|
104
149
|
throw new Error(cancelled() ? 'the run was cancelled' : `${name} failed`);
|
|
105
150
|
}
|
|
@@ -114,32 +159,80 @@ cancelled, emit) {
|
|
|
114
159
|
}
|
|
115
160
|
}
|
|
116
161
|
exports.runTasks = runTasks;
|
|
117
|
-
//
|
|
118
|
-
function
|
|
162
|
+
// Why a task ran, as a task event carries it: the first causes, masked and cut short.
|
|
163
|
+
function taskCauses(causes, redact) {
|
|
164
|
+
return causes?.length
|
|
165
|
+
? {
|
|
166
|
+
causes: [
|
|
167
|
+
...causes.slice(0, runs_1.MAX_CAUSES).map((cause) => redact(cause).slice(0, runs_1.MAX_TASK_ERROR)),
|
|
168
|
+
...(causes.length > runs_1.MAX_CAUSES ? [`… and ${causes.length - runs_1.MAX_CAUSES} more`] : []),
|
|
169
|
+
],
|
|
170
|
+
}
|
|
171
|
+
: {};
|
|
172
|
+
}
|
|
173
|
+
exports.taskCauses = taskCauses;
|
|
174
|
+
// A task event when a task has moved on since the last look, with what its result needs
|
|
175
|
+
// once it is over.
|
|
176
|
+
function reportTasks(emit, workTree, seen,
|
|
177
|
+
// what leaves the process is masked like the log is: an error may quote a value
|
|
178
|
+
redact) {
|
|
119
179
|
for (const task of (0, plan_work_tasks_1.iterateWorkTasks)(workTree)) {
|
|
120
180
|
const state = task.state.current;
|
|
181
|
+
const before = seen.get(task.name);
|
|
182
|
+
const started = 'started' in state ? state.started.getTime() : before?.started;
|
|
183
|
+
const since = () => (started !== undefined ? { durationMs: Math.max(0, Date.now() - started) } : {});
|
|
121
184
|
let event = null;
|
|
122
185
|
if (state.type === 'running') {
|
|
123
|
-
event = 'running';
|
|
186
|
+
event = { type: 'task', task: task.name, state: 'running' };
|
|
124
187
|
}
|
|
125
188
|
else if (state.type === 'completed') {
|
|
126
|
-
|
|
189
|
+
const status = state.skipped ? 'skipped' : state.cached ? 'cached' : 'succeeded';
|
|
190
|
+
event = {
|
|
191
|
+
type: 'task',
|
|
192
|
+
task: task.name,
|
|
193
|
+
state: status,
|
|
194
|
+
...(status === 'skipped' ? {} : { durationMs: state.duration }),
|
|
195
|
+
...taskCauses(state.missCauses, redact),
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
else if (state.type === 'crash') {
|
|
199
|
+
event = { type: 'task', task: task.name, state: 'failed', exitCode: state.exitCode, ...since() };
|
|
200
|
+
}
|
|
201
|
+
else if (state.type === 'error') {
|
|
202
|
+
const error = redact(state.errorMessage).slice(0, runs_1.MAX_TASK_ERROR);
|
|
203
|
+
event = { type: 'task', task: task.name, state: 'failed', error, ...since() };
|
|
127
204
|
}
|
|
128
|
-
else if (state.type === '
|
|
129
|
-
event = '
|
|
205
|
+
else if (state.type === 'canceled') {
|
|
206
|
+
event = { type: 'task', task: task.name, state: 'cancelled' };
|
|
130
207
|
}
|
|
131
|
-
if (event &&
|
|
132
|
-
seen.set(task.name, event);
|
|
133
|
-
emit(
|
|
208
|
+
if (event && before?.status !== event.state) {
|
|
209
|
+
seen.set(task.name, { status: event.state, started });
|
|
210
|
+
emit(event);
|
|
211
|
+
}
|
|
212
|
+
else if (started !== undefined && before?.started === undefined) {
|
|
213
|
+
seen.set(task.name, { status: before?.status, started });
|
|
134
214
|
}
|
|
135
215
|
}
|
|
136
216
|
}
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
}
|
|
142
|
-
|
|
217
|
+
// The run is the first line of the standard input. The server keeps it open and writes
|
|
218
|
+
// `cancel` on it to stop the run: that works the same on every platform, where a signal
|
|
219
|
+
// on Windows ends the process without letting it stop its tasks.
|
|
220
|
+
function readInput(stdin, cancel) {
|
|
221
|
+
const lines = (0, readline_1.createInterface)({ input: stdin, crlfDelay: Infinity });
|
|
222
|
+
let first;
|
|
223
|
+
const input = new Promise((resolve) => {
|
|
224
|
+
lines.on('line', (line) => {
|
|
225
|
+
if (first === undefined) {
|
|
226
|
+
first = line;
|
|
227
|
+
resolve(line);
|
|
228
|
+
}
|
|
229
|
+
else if (line.trim() === 'cancel') {
|
|
230
|
+
cancel();
|
|
231
|
+
}
|
|
232
|
+
});
|
|
233
|
+
lines.on('close', () => resolve(first ?? ''));
|
|
234
|
+
});
|
|
235
|
+
return { input, close: () => lines.close() };
|
|
143
236
|
}
|
|
144
237
|
/**
|
|
145
238
|
* The process of one run: reads its input, executes the tasks in the current directory
|
|
@@ -148,7 +241,7 @@ async function readAll(stream) {
|
|
|
148
241
|
*/
|
|
149
242
|
async function workerMain(stdin, stdout, processEnvs, cwd, abortCtrl) {
|
|
150
243
|
const say = (message) => stdout.write(`${JSON.stringify(message)}\n`);
|
|
151
|
-
// the server stops a run with a signal
|
|
244
|
+
// the server stops a run with `cancel` on the standard input, anybody else with a signal
|
|
152
245
|
let cancelled = false;
|
|
153
246
|
const stop = () => {
|
|
154
247
|
cancelled = true;
|
|
@@ -156,11 +249,14 @@ async function workerMain(stdin, stdout, processEnvs, cwd, abortCtrl) {
|
|
|
156
249
|
};
|
|
157
250
|
const signals = ['SIGINT', 'SIGTERM', 'SIGHUP'];
|
|
158
251
|
signals.forEach((signal) => process.on(signal, stop));
|
|
252
|
+
const reader = readInput(stdin, stop);
|
|
159
253
|
let input;
|
|
160
254
|
try {
|
|
161
|
-
input = exports.workerInputSchema.parse(JSON.parse(await
|
|
255
|
+
input = exports.workerInputSchema.parse(JSON.parse(await reader.input));
|
|
162
256
|
}
|
|
163
257
|
catch (e) {
|
|
258
|
+
reader.close();
|
|
259
|
+
signals.forEach((signal) => process.off(signal, stop));
|
|
164
260
|
say({ kind: 'result', ok: false, error: `the run could not be read: ${e instanceof Error ? e.message : e}` });
|
|
165
261
|
return;
|
|
166
262
|
}
|
|
@@ -172,6 +268,7 @@ async function workerMain(stdin, stdout, processEnvs, cwd, abortCtrl) {
|
|
|
172
268
|
say({ kind: 'result', ok: false, error: e instanceof Error ? e.message : String(e) });
|
|
173
269
|
}
|
|
174
270
|
finally {
|
|
271
|
+
reader.close();
|
|
175
272
|
signals.forEach((signal) => process.off(signal, stop));
|
|
176
273
|
}
|
|
177
274
|
}
|