hammerkit 1.27.0-next.16 → 1.27.0-next.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/executer/execution-steps.js +5 -2
- package/dist/executer/scheduler/state-key.js +37 -5
- package/dist/harness/mcp.js +2 -1
- package/dist/harness/opencode.js +9 -2
- package/dist/harness/pi.js +15 -0
- package/dist/harness/sandbox-client.js +26 -2
- package/dist/harness/task-tools.js +12 -1
- package/dist/optimizer/calculate-checksum.js +3 -2
- package/dist/optimizer/check-key.js +94 -0
- package/dist/optimizer/check-keys.js +64 -0
- package/dist/optimizer/get-work-cache-stats.js +17 -3
- package/dist/optimizer/work-task-cache-description.js +49 -1
- package/dist/planner/utils/normalize-path.js +19 -1
- package/dist/planner/utils/parse-work-mount.js +7 -1
- package/dist/planner/validate.js +22 -0
- package/dist/remote/remote-work-commands.js +70 -0
- package/dist/remote/server-client.js +15 -2
- package/dist/run-program.js +9 -0
- package/dist/schema/reference-parser.js +8 -1
- package/dist/secrets/github-app-token-command.js +3 -1
- package/dist/server/accounts-api.js +1 -0
- package/dist/server/checks-api.js +83 -0
- package/dist/server/checks.js +116 -0
- package/dist/server/executor.js +41 -3
- package/dist/server/kubernetes-backend.js +1 -1
- package/dist/server/kubernetes-sandbox-storage.js +41 -1
- package/dist/server/plan.js +103 -78
- package/dist/server/policy.js +10 -2
- package/dist/server/run-request.js +12 -1
- package/dist/server/run-worker.js +16 -2
- package/dist/server/runs.js +18 -0
- package/dist/server/sandbox-pod.js +28 -0
- package/dist/server/sandbox-storage.js +23 -0
- package/dist/server/sandboxes.js +8 -0
- package/dist/server/server-config.js +14 -1
- package/dist/server/server.js +46 -3
- package/package.json +1 -1
package/dist/server/plan.js
CHANGED
|
@@ -13,6 +13,7 @@ const work_scope_parser_1 = require("../schema/work-scope-parser");
|
|
|
13
13
|
const work_task_1 = require("../planner/work-task");
|
|
14
14
|
const work_service_1 = require("../planner/work-service");
|
|
15
15
|
const hosted_volumes_1 = require("../planner/utils/hosted-volumes");
|
|
16
|
+
const normalize_path_1 = require("../planner/utils/normalize-path");
|
|
16
17
|
const log_1 = require("../log");
|
|
17
18
|
const parse_error_1 = require("../schema/parse-error");
|
|
18
19
|
const task_arguments_1 = require("../planner/utils/task-arguments");
|
|
@@ -165,101 +166,125 @@ function describeSecret(secret) {
|
|
|
165
166
|
// server has no account, so nothing would stand between the build file and the
|
|
166
167
|
// operator's own credentials.
|
|
167
168
|
function findPolicyProblems(items, policy, root) {
|
|
168
|
-
|
|
169
|
+
// the problems of each item, said once for the items of a directory that share one, such
|
|
170
|
+
// as the tasks of an included file that all mount the docker socket
|
|
171
|
+
const grouped = new Map();
|
|
169
172
|
for (const item of items) {
|
|
170
|
-
const
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
for (const source of 'src' in data ? data.src : []) {
|
|
178
|
-
if (!source.inherited && insideBundle(root, data.cwd, source.absolutePath).length === 0) {
|
|
179
|
-
problems.push(`${item.name}: src ${source.source} is outside of the bundle`);
|
|
173
|
+
const prefix = `${item.name}: `;
|
|
174
|
+
for (const problem of findItemPolicyProblems(item, policy, root)) {
|
|
175
|
+
const said = problem.startsWith(prefix) ? problem.slice(prefix.length) : problem;
|
|
176
|
+
const key = `${item.data.cwd}\0${said}`;
|
|
177
|
+
const group = grouped.get(key);
|
|
178
|
+
if (group) {
|
|
179
|
+
group.names.push(item.name);
|
|
180
180
|
}
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
if (!generate.inherited && insideBundle(root, data.cwd, generate.path).length === 0) {
|
|
184
|
-
problems.push(`${item.name}: generates ${generate.path}, which is outside of the bundle`);
|
|
181
|
+
else {
|
|
182
|
+
grouped.set(key, { names: [item.name], problem: said });
|
|
185
183
|
}
|
|
186
184
|
}
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
185
|
+
}
|
|
186
|
+
return [...grouped.values()].map(({ names: [name, ...others], problem }) => others.length > 0 ? `${name} (also ${others.join(', ')}): ${problem}` : `${name}: ${problem}`);
|
|
187
|
+
}
|
|
188
|
+
function findItemPolicyProblems(item, policy, root) {
|
|
189
|
+
const problems = [];
|
|
190
|
+
const runtime = item.data.type === 'local-task' ? 'local' : 'docker';
|
|
191
|
+
if (policy.runtimes && !policy.runtimes.includes(runtime)) {
|
|
192
|
+
problems.push(`${item.name}: runs on the ${runtime} runtime, which this server does not allow (runtimes: ${policy.runtimes.join(', ')})`);
|
|
193
|
+
}
|
|
194
|
+
const data = item.data;
|
|
195
|
+
// what a task reads and writes is the bundle's: `src: ../..` would have the server
|
|
196
|
+
// copy its own files (its credentials, other callers' caches) into the task
|
|
197
|
+
for (const source of 'src' in data ? data.src : []) {
|
|
198
|
+
if (!source.inherited && insideBundle(root, data.cwd, source.absolutePath).length === 0) {
|
|
199
|
+
problems.push(`${item.name}: src ${source.source} is outside of the bundle`);
|
|
196
200
|
}
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
reason: ` to what reads ${reads.join(', ')}`,
|
|
202
|
-
});
|
|
201
|
+
}
|
|
202
|
+
for (const generate of (0, work_task_1.isWorkTask)(data) ? data.generates : []) {
|
|
203
|
+
if (!generate.inherited && insideBundle(root, data.cwd, generate.path).length === 0) {
|
|
204
|
+
problems.push(`${item.name}: generates ${generate.path}, which is outside of the bundle`);
|
|
203
205
|
}
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
206
|
+
}
|
|
207
|
+
// what holds a credential gets at most `runs.secretNetwork`, so a tool it runs can't
|
|
208
|
+
// send the credential anywhere else
|
|
209
|
+
const reads = getCredentialReads({
|
|
210
|
+
secrets: secretsOf(item).map(describeSecret),
|
|
211
|
+
serverEnvs: 'envs' in data ? data.envs.processNames ?? [] : [],
|
|
212
|
+
});
|
|
213
|
+
const ceilings = [];
|
|
214
|
+
if (policy.network && (0, work_network_1.isLimitedNetwork)(policy.network)) {
|
|
215
|
+
ceilings.push({ ceiling: policy.network, setting: 'runs.network', reason: '' });
|
|
216
|
+
}
|
|
217
|
+
if (reads.length > 0 && policy.secretNetwork && (0, work_network_1.isLimitedNetwork)(policy.secretNetwork)) {
|
|
218
|
+
ceilings.push({
|
|
219
|
+
ceiling: policy.secretNetwork,
|
|
220
|
+
setting: 'runs.secretNetwork',
|
|
221
|
+
reason: ` to what reads ${reads.join(', ')}`,
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
for (const { ceiling, setting, reason } of ceilings) {
|
|
225
|
+
// on a cluster a local task is a pod of a pool's tool image, limited like any other
|
|
226
|
+
if (data.type === 'local-task' && policy.backend !== 'kubernetes') {
|
|
227
|
+
problems.push(`${item.name}: is a local task, which can't be limited to the network this server allows${reason} ` +
|
|
228
|
+
`(${setting}: ${(0, work_network_1.formatNetwork)(ceiling)})`);
|
|
214
229
|
}
|
|
215
|
-
if (
|
|
216
|
-
for (
|
|
217
|
-
|
|
218
|
-
if (insideBundle(root, data.cwd, mount.localPath).length === 0) {
|
|
219
|
-
problems.push(`${item.name}: mounts ${mount.localPath}, which is outside of the bundle`);
|
|
220
|
-
}
|
|
221
|
-
}
|
|
230
|
+
else if ('network' in data && (0, work_network_1.exceedsNetwork)(data.network, ceiling)) {
|
|
231
|
+
problems.push(`${item.name}: asks for network ${(0, work_network_1.formatNetwork)(data.network)}, more than this server allows${reason} ` +
|
|
232
|
+
`(${setting}: ${(0, work_network_1.formatNetwork)(ceiling)})`);
|
|
222
233
|
}
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
234
|
+
}
|
|
235
|
+
if ((0, work_task_1.isContainerWorkTask)(data) || (0, work_service_1.isContainerWorkService)(data)) {
|
|
236
|
+
for (const mount of data.mounts) {
|
|
237
|
+
// a path of the server is not the build file's to mount: the docker socket, /etc, a home directory
|
|
238
|
+
if (insideBundle(root, data.cwd, mount.localPath).length === 0) {
|
|
239
|
+
problems.push(`${item.name}: mounts ${describeMountSource(mount)}, which is outside of the bundle`);
|
|
227
240
|
}
|
|
228
241
|
}
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
// provider only through the egress proxy
|
|
244
|
-
problems.push(`${item.name}: the secret ${describeSecret(secret).target} is read from ${source.provider} in the ` +
|
|
245
|
-
`Job of the task, which network ${(0, work_network_1.formatNetwork)(item.data.network)} would cut off from it: ` +
|
|
246
|
-
`this server names no endpoints of ${source.provider} in backend.providers.${source.provider}.allow ` +
|
|
247
|
-
'for the egress proxy to let through, so the task keeps an open network');
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
if (source.type === 'provider' && policy.reserved?.includes(source.account)) {
|
|
251
|
-
problems.push(`${item.name}: the secret ${describeSecret(secret).target} is read as ${source.account}, which the server uses to read repositories`);
|
|
242
|
+
}
|
|
243
|
+
for (const volume of (0, work_service_1.isContainerWorkService)(data) ? data.volumes : []) {
|
|
244
|
+
// docker takes anything else for a path of the host, such as the docker socket
|
|
245
|
+
if (!(0, hosted_volumes_1.isVolumeName)(volume.name)) {
|
|
246
|
+
problems.push(`${item.name}: volume ${volume.name} is not a volume name, a path of the server can't be mounted`);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
for (const secret of secretsOf(item)) {
|
|
250
|
+
const source = secret.source;
|
|
251
|
+
if (policy.backend === 'kubernetes' && source.type === 'provider') {
|
|
252
|
+
// the server reads no secret of a provider on a cluster: the Job of a task does
|
|
253
|
+
if (!(0, work_task_1.isWorkTask)(item.data)) {
|
|
254
|
+
problems.push(`${item.name}: the secret ${describeSecret(secret).target} is read from ${source.provider}, which ` +
|
|
255
|
+
'this server does for tasks only, a service on its cluster gets no provider secret yet');
|
|
252
256
|
}
|
|
253
|
-
else if (
|
|
254
|
-
problems.push(`${item.name}:
|
|
257
|
+
else if (secret.cache) {
|
|
258
|
+
problems.push(`${item.name}: the secret ${describeSecret(secret).target} has cache: true, its digest would need ` +
|
|
259
|
+
'the value before the Job of the task reads it, which this server does not support yet');
|
|
255
260
|
}
|
|
256
|
-
else if (
|
|
257
|
-
|
|
261
|
+
else if ((0, work_network_1.isLimitedNetwork)(item.data.network) && !policy.proxiedProviders?.includes(source.provider)) {
|
|
262
|
+
// the init containers that read it share the pod's NetworkPolicy, and reach the
|
|
263
|
+
// provider only through the egress proxy
|
|
264
|
+
problems.push(`${item.name}: the secret ${describeSecret(secret).target} is read from ${source.provider} in the ` +
|
|
265
|
+
`Job of the task, which network ${(0, work_network_1.formatNetwork)(item.data.network)} would cut off from it: ` +
|
|
266
|
+
`this server names no endpoints of ${source.provider} in backend.providers.${source.provider}.allow ` +
|
|
267
|
+
'for the egress proxy to let through, so the task keeps an open network');
|
|
258
268
|
}
|
|
259
269
|
}
|
|
270
|
+
if (source.type === 'provider' && policy.reserved?.includes(source.account)) {
|
|
271
|
+
problems.push(`${item.name}: the secret ${describeSecret(secret).target} is read as ${source.account}, which the server uses to read repositories`);
|
|
272
|
+
}
|
|
273
|
+
else if (source.type === 'file') {
|
|
274
|
+
problems.push(`${item.name}: a secret can not read a file of the server (file:), use a secret provider`);
|
|
275
|
+
}
|
|
276
|
+
else if (source.type === 'env' && !policy.envs.includes(source.name)) {
|
|
277
|
+
problems.push(`${item.name}: the secret ${describeSecret(secret).target} reads env:${source.name}, which the server does not list in ${policy.envsFrom ?? 'runs.env.allow'}`);
|
|
278
|
+
}
|
|
260
279
|
}
|
|
261
280
|
return problems;
|
|
262
281
|
}
|
|
282
|
+
// A mount of the home directory says so: `$PWD` in a mount is the home directory, not the
|
|
283
|
+
// project, which reads like a path of the checkout
|
|
284
|
+
function describeMountSource(mount) {
|
|
285
|
+
const home = (0, normalize_path_1.getHomeReference)(mount.mount.split(':')[0]);
|
|
286
|
+
return home ? `${mount.localPath} (${home} is the home directory)` : mount.localPath;
|
|
287
|
+
}
|
|
263
288
|
function toPlan(root, item) {
|
|
264
289
|
const data = item.data;
|
|
265
290
|
const secrets = secretsOf(item).map(describeSecret);
|
package/dist/server/policy.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.decideRun = exports.sandboxRepos = exports.decideSandbox = exports.sandboxPushBranches = exports.sandboxBranchType = exports.endpointLevel = exports.getGrant = exports.usedAccounts = exports.mayYank = exports.canUse = exports.maySeeDefinitions = exports.mayPublish = exports.mayPropose = exports.mayRun = exports.matches = void 0;
|
|
3
|
+
exports.decideRun = exports.sandboxRepos = exports.decideSandbox = exports.sandboxPushBranches = exports.sandboxBranchType = exports.endpointLevel = exports.getGrant = exports.usedAccounts = exports.mayYank = exports.canUse = exports.maySeeDefinitions = exports.mayPublish = exports.mayPropose = exports.mayReadChecks = exports.mayRun = exports.matches = void 0;
|
|
4
4
|
const repo_1 = require("./repo");
|
|
5
5
|
const BUILDS_ORDER = ['run', 'propose', 'publish'];
|
|
6
6
|
function matches(entitlement, principal) {
|
|
@@ -22,6 +22,12 @@ function atLeast(grant, level) {
|
|
|
22
22
|
}
|
|
23
23
|
const mayRun = (grant) => atLeast(grant, 'run');
|
|
24
24
|
exports.mayRun = mayRun;
|
|
25
|
+
// Who may run from a repository's refs may see what passed for its files, and so may who
|
|
26
|
+
// is entitled to just that (a CI job that only verifies).
|
|
27
|
+
function mayReadChecks(grant, repo) {
|
|
28
|
+
return (grant.checks || grant.builds !== null) && (0, repo_1.matchesAnyRepo)(grant.repos, repo);
|
|
29
|
+
}
|
|
30
|
+
exports.mayReadChecks = mayReadChecks;
|
|
25
31
|
const mayPropose = (grant) => atLeast(grant, 'propose');
|
|
26
32
|
exports.mayPropose = mayPropose;
|
|
27
33
|
const mayPublish = (grant) => atLeast(grant, 'publish');
|
|
@@ -57,11 +63,13 @@ function getGrant(entitlements, principal) {
|
|
|
57
63
|
const sandboxes = [];
|
|
58
64
|
const endpoints = [];
|
|
59
65
|
let builds = null;
|
|
66
|
+
let checks = false;
|
|
60
67
|
entitlements.forEach((entitlement, index) => {
|
|
61
68
|
if (!matches(entitlement, principal)) {
|
|
62
69
|
return;
|
|
63
70
|
}
|
|
64
71
|
matched.push(index);
|
|
72
|
+
checks = checks || entitlement.checks === 'read';
|
|
65
73
|
for (const [account, level] of Object.entries(entitlement.accounts)) {
|
|
66
74
|
if (accounts[account] !== 'use') {
|
|
67
75
|
accounts[account] = level;
|
|
@@ -96,7 +104,7 @@ function getGrant(entitlements, principal) {
|
|
|
96
104
|
}
|
|
97
105
|
}
|
|
98
106
|
});
|
|
99
|
-
return { accounts, repos, builds, sandboxes, endpoints, matched };
|
|
107
|
+
return { accounts, repos, builds, sandboxes, endpoints, checks, matched };
|
|
100
108
|
}
|
|
101
109
|
exports.getGrant = getGrant;
|
|
102
110
|
// What the caller may do with the endpoints of service runs of a repository: the higher
|
|
@@ -94,7 +94,13 @@ exports.runRequestSchema = zod_1.z
|
|
|
94
94
|
zod_1.z.object({ type: zod_1.z.literal('upload'), snapshot: zod_1.z.string().regex(bundle_1.HASH_PATTERN) }).strict(),
|
|
95
95
|
// a sandbox of the caller's, see POST /v1/sandboxes
|
|
96
96
|
zod_1.z
|
|
97
|
-
.object({
|
|
97
|
+
.object({
|
|
98
|
+
type: zod_1.z.literal('sandbox'),
|
|
99
|
+
id: zod_1.z.string().regex(runs_1.SANDBOX_ID_PATTERN, 'expected a sandbox id'),
|
|
100
|
+
// a check: run on a copy of the files a push of the sandbox would commit, so what
|
|
101
|
+
// passes counts for that commit (remote verify)
|
|
102
|
+
check: zod_1.z.literal(true).optional(),
|
|
103
|
+
})
|
|
98
104
|
.strict(),
|
|
99
105
|
]),
|
|
100
106
|
options: exports.runOptionsSchema.default({}),
|
|
@@ -144,5 +150,10 @@ exports.runRequestSchema = zod_1.z
|
|
|
144
150
|
.refine((request) => (request.options.args ?? []).length === 0 || request.tasks.length === 1, {
|
|
145
151
|
message: 'arguments go to one task, name exactly one in tasks',
|
|
146
152
|
path: ['options', 'args'],
|
|
153
|
+
})
|
|
154
|
+
.refine((request) => request.source.type !== 'sandbox' || !request.source.check || (request.options.args ?? []).length === 0, { message: 'a check runs the task as the build file says, it takes no arguments', path: ['options', 'args'] })
|
|
155
|
+
.refine((request) => request.source.type !== 'sandbox' || !request.source.check || request.services.length === 0, {
|
|
156
|
+
message: 'a check runs tasks, a run of services keeps them up and passes nothing',
|
|
157
|
+
path: ['services'],
|
|
147
158
|
});
|
|
148
159
|
//# sourceMappingURL=run-request.js.map
|
|
@@ -22,6 +22,7 @@ const work_network_1 = require("../planner/work-network");
|
|
|
22
22
|
const run_plan_1 = require("./run-plan");
|
|
23
23
|
const runs_1 = require("./runs");
|
|
24
24
|
const resources_1 = require("../kubernetes/resources");
|
|
25
|
+
const check_keys_1 = require("../optimizer/check-keys");
|
|
25
26
|
// one log line of a task is cut here, the registry caps how many events a run keeps
|
|
26
27
|
const MAX_LOG_TEXT = 8 * 1024;
|
|
27
28
|
/**
|
|
@@ -69,6 +70,8 @@ exports.workerInputSchema = zod_1.z
|
|
|
69
70
|
})
|
|
70
71
|
.strict()
|
|
71
72
|
.optional(),
|
|
73
|
+
// the run's results count as checks: report each task's check key before it runs
|
|
74
|
+
check: zod_1.z.boolean().optional(),
|
|
72
75
|
})
|
|
73
76
|
.strict();
|
|
74
77
|
// The ceiling of the run, combined here and not in the server: the lower of two allow lists
|
|
@@ -183,7 +186,8 @@ up) {
|
|
|
183
186
|
const options = input.options;
|
|
184
187
|
const execOptions = {
|
|
185
188
|
logMode: 'live',
|
|
186
|
-
|
|
189
|
+
// a check takes a cached result only for the same files, never for the same dates
|
|
190
|
+
cacheDefault: input.check && options.cache === 'modify-date' ? 'checksum' : options.cache ?? 'checksum',
|
|
187
191
|
cacheReadOnly: options.cacheReadOnly ?? false,
|
|
188
192
|
timeout: options.timeout ? (0, units_1.parseDuration)(options.timeout) : null,
|
|
189
193
|
maxTimeout: options.maxTimeout ? (0, units_1.parseDuration)(options.maxTimeout) : null,
|
|
@@ -192,7 +196,17 @@ up) {
|
|
|
192
196
|
// why a task was not taken from the cache goes into its result
|
|
193
197
|
explain: true,
|
|
194
198
|
};
|
|
195
|
-
|
|
199
|
+
const scopes = (0, run_plan_1.getRunScopes)(input.tasks, options, input.up?.services);
|
|
200
|
+
if (input.check) {
|
|
201
|
+
// the files as they are before any task wrote to them, a second task's too: a first
|
|
202
|
+
// one that formats or generates into src would key it by files no commit holds
|
|
203
|
+
for (const { scope } of scopes) {
|
|
204
|
+
for (const [task, { key }] of Object.entries(await (0, check_keys_1.getCheckKeys)(input.entry, environment, scope))) {
|
|
205
|
+
emit({ type: 'key', task, key });
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
for (const { name, scope } of scopes) {
|
|
196
210
|
const { ctx, scope: parsed } = await (0, schema_parser_1.createParseContext)(input.entry, environment, {
|
|
197
211
|
directory: input.directory,
|
|
198
212
|
});
|
package/dist/server/runs.js
CHANGED
|
@@ -30,6 +30,8 @@ exports.SANDBOX_ID_PATTERN = /^sbx_[A-Za-z0-9_-]{16}$/;
|
|
|
30
30
|
function onSandbox(run, sandbox) {
|
|
31
31
|
return run.source.type === 'sandbox' && run.source.id === sandbox;
|
|
32
32
|
}
|
|
33
|
+
// the tasks of a run whose check keys are kept, a build file has fewer
|
|
34
|
+
const MAX_KEYED_TASKS = 1000;
|
|
33
35
|
// how much of the end of a task's output its result carries
|
|
34
36
|
const TAIL_LINES = 20;
|
|
35
37
|
const TAIL_LINE_LENGTH = 1000;
|
|
@@ -100,6 +102,7 @@ class RunRegistry {
|
|
|
100
102
|
bytes: 0,
|
|
101
103
|
tails: new Map(Object.entries(run.results ?? {}).map(([task, result]) => [task, result.lastLog])),
|
|
102
104
|
saving: Promise.resolve(),
|
|
105
|
+
keys: {},
|
|
103
106
|
};
|
|
104
107
|
this.entries.set(run.id, entry);
|
|
105
108
|
if (run.state === 'awaiting-approval') {
|
|
@@ -166,6 +169,7 @@ class RunRegistry {
|
|
|
166
169
|
bytes: 0,
|
|
167
170
|
tails: new Map(),
|
|
168
171
|
saving: Promise.resolve(),
|
|
172
|
+
keys: {},
|
|
169
173
|
};
|
|
170
174
|
this.entries.set(run.id, entry);
|
|
171
175
|
if (waiting) {
|
|
@@ -790,9 +794,23 @@ class RunRegistry {
|
|
|
790
794
|
}
|
|
791
795
|
this.record(entry, { type: 'state', state, ...(error ? { error } : {}) }, true);
|
|
792
796
|
entry.listeners.clear();
|
|
797
|
+
try {
|
|
798
|
+
this.options.onFinished?.(this.copy(entry.run), { ...entry.keys });
|
|
799
|
+
}
|
|
800
|
+
catch (e) {
|
|
801
|
+
this.options.log?.(`run ${entry.run.id} ended, but what it passed was not kept: ${e instanceof Error ? e.message : e}`);
|
|
802
|
+
}
|
|
793
803
|
this.evict();
|
|
794
804
|
}
|
|
795
805
|
record(entry, recorded, force = false) {
|
|
806
|
+
if (recorded.type === 'key') {
|
|
807
|
+
// a task name is the executor's; the first key it reports for a task is the one
|
|
808
|
+
// computed before anything of the run had run
|
|
809
|
+
if (!(recorded.task in entry.keys) && Object.keys(entry.keys).length < MAX_KEYED_TASKS) {
|
|
810
|
+
entry.keys[recorded.task] = recorded.key;
|
|
811
|
+
}
|
|
812
|
+
return;
|
|
813
|
+
}
|
|
796
814
|
const event = recorded.type === 'task' ? bounded(recorded) : recorded;
|
|
797
815
|
if (event.type === 'log') {
|
|
798
816
|
this.keepTail(entry, event.task, event.text);
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.sandboxPodMain = exports.decodePodError = exports.encodeEditOps = exports.INCOMING_REF = exports.INCOMING = void 0;
|
|
4
|
+
const crypto_1 = require("crypto");
|
|
4
5
|
const promises_1 = require("fs/promises");
|
|
5
6
|
const path_1 = require("path");
|
|
6
7
|
const url_1 = require("url");
|
|
7
8
|
const git_checkout_1 = require("./git-checkout");
|
|
8
9
|
const sandbox_files_1 = require("./sandbox-files");
|
|
9
10
|
const sandbox_storage_1 = require("./sandbox-storage");
|
|
11
|
+
const tar_1 = require("../utils/tar");
|
|
10
12
|
// On a cluster a sandbox is a claim, and what the server does with its files runs in a
|
|
11
13
|
// short pod that mounts it (see KubernetesSandboxStorage): `hammerkit server-sandbox
|
|
12
14
|
// <directory>` reads one request on its standard input, does it with the code of a host
|
|
@@ -91,6 +93,32 @@ async function answer(storage, directory, request, out) {
|
|
|
91
93
|
}
|
|
92
94
|
return send({ files: plan.files, bytes: plan.bytes });
|
|
93
95
|
}
|
|
96
|
+
case 'archive': {
|
|
97
|
+
// written beside the sandbox on the claim, sent, and removed
|
|
98
|
+
const into = (0, path_1.join)(directory, `${request.id}-archive-${(0, crypto_1.randomBytes)(6).toString('hex')}`);
|
|
99
|
+
try {
|
|
100
|
+
await storage.exportTree(key, request.tree, into, AbortSignal.timeout(10 * 60000));
|
|
101
|
+
await write(out, JSON.stringify({ ok: true, result: null, stream: true }) + '\n');
|
|
102
|
+
const pack = (0, tar_1.packArchive)({ cwd: into, portable: true }, ['.']);
|
|
103
|
+
try {
|
|
104
|
+
for await (const chunk of pack) {
|
|
105
|
+
await write(out, chunk);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
catch (e) {
|
|
109
|
+
throw new AnswerStarted(e instanceof Error ? e.message : String(e));
|
|
110
|
+
}
|
|
111
|
+
finally {
|
|
112
|
+
// a reader that went away (over its size limit) leaves the pack reading files
|
|
113
|
+
// that are removed next: what fails then is no answer of anybody's
|
|
114
|
+
pack.on('error', () => undefined);
|
|
115
|
+
}
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
finally {
|
|
119
|
+
await (0, promises_1.rm)(into, { recursive: true, force: true });
|
|
120
|
+
}
|
|
121
|
+
}
|
|
94
122
|
case 'raw': {
|
|
95
123
|
const raw = await files().raw(request.path);
|
|
96
124
|
const header = {
|
|
@@ -276,6 +276,29 @@ class HostSandboxStorage {
|
|
|
276
276
|
// files change, and what it wrote to the git data is pruned again. Files a task made
|
|
277
277
|
// unreadable are made readable first: git would leave them out and call the sandbox
|
|
278
278
|
// clean.
|
|
279
|
+
async exportTree({ id }, tree, into, signal) {
|
|
280
|
+
if (!/^[0-9a-f]{40}([0-9a-f]{24})?$/.test(tree)) {
|
|
281
|
+
throw new Error(`${JSON.stringify(tree.slice(0, 80))} is not a git tree`);
|
|
282
|
+
}
|
|
283
|
+
return this.withIndex(id, signal, async (run) => {
|
|
284
|
+
const now = (await run(['write-tree'], 'read')).trim();
|
|
285
|
+
if (now !== tree) {
|
|
286
|
+
throw new Error(`the files of the sandbox ${id} are ${now} now, not ${tree}`);
|
|
287
|
+
}
|
|
288
|
+
// an index of its own holds the tree alone, whatever the sandbox's index says of its
|
|
289
|
+
// entries; the files are written as git has them, through no filter of the server's
|
|
290
|
+
const index = (0, path_1.join)(this.root(id), `export-${(0, crypto_1.randomBytes)(6).toString('hex')}`);
|
|
291
|
+
await (0, promises_1.mkdir)(into, { recursive: true });
|
|
292
|
+
try {
|
|
293
|
+
const env = this.readEnv({ GIT_INDEX_FILE: index });
|
|
294
|
+
await this.git(id, ['read-tree', tree], env, signal, 'read');
|
|
295
|
+
await this.git(id, ['checkout-index', '--all', '--force', `--prefix=${(0, path_1.join)(into, '/')}`], env, signal, 'read');
|
|
296
|
+
}
|
|
297
|
+
finally {
|
|
298
|
+
await (0, promises_1.rm)(index, { force: true });
|
|
299
|
+
}
|
|
300
|
+
});
|
|
301
|
+
}
|
|
279
302
|
async state({ id }, signal = AbortSignal.timeout(5 * 60 * 1000)) {
|
|
280
303
|
return this.withIndex(id, signal, async (run) => {
|
|
281
304
|
const tree = (await run(['write-tree'], 'read')).trim();
|
package/dist/server/sandboxes.js
CHANGED
|
@@ -412,6 +412,14 @@ class SandboxRegistry {
|
|
|
412
412
|
async state(id, signal) {
|
|
413
413
|
return this.options.storage.state(keyOf(this.existing(id)), signal);
|
|
414
414
|
}
|
|
415
|
+
// The files of `tree`, which the sandbox's files must be now, written into `into`: what a
|
|
416
|
+
// check runs on.
|
|
417
|
+
async exportTree(id, tree, into, signal) {
|
|
418
|
+
const sandbox = this.existing(id);
|
|
419
|
+
sandbox.lastUsedAt = this.timestamp();
|
|
420
|
+
await this.save(sandbox);
|
|
421
|
+
await this.options.storage.exportTree(keyOf(sandbox), tree, into, signal);
|
|
422
|
+
}
|
|
415
423
|
// What a run works on, `into` a directory of the run when the storage copies the files;
|
|
416
424
|
// the run is the sandbox's use.
|
|
417
425
|
async attach(id, into) {
|
|
@@ -341,6 +341,9 @@ exports.entitlementSchema = zod_1.z
|
|
|
341
341
|
// the repositories (`host/path` patterns) the caller may run from a git ref
|
|
342
342
|
repos: zod_1.z.array(repoPatternSchema).default([]),
|
|
343
343
|
builds: exports.buildsLevelSchema.optional(),
|
|
344
|
+
// `read`: may look up which tasks runs passed for files of the `repos` (remote verify),
|
|
345
|
+
// and nothing more; a builds level includes it
|
|
346
|
+
checks: zod_1.z.literal('read').optional(),
|
|
344
347
|
// the repositories the caller may keep sandboxes of, checkouts on the server that runs
|
|
345
348
|
// work on: a list of its own, the `repos` above are for runs from a ref
|
|
346
349
|
sandboxes: zod_1.z.array(exports.sandboxGrantSchema).optional(),
|
|
@@ -352,11 +355,19 @@ exports.entitlementSchema = zod_1.z
|
|
|
352
355
|
checkWho(entitlement, ctx);
|
|
353
356
|
if (Object.keys(entitlement.accounts).length === 0 &&
|
|
354
357
|
entitlement.builds === undefined &&
|
|
358
|
+
entitlement.checks === undefined &&
|
|
355
359
|
!entitlement.sandboxes?.length &&
|
|
356
360
|
!entitlement.endpoints?.length) {
|
|
357
361
|
ctx.addIssue({
|
|
358
362
|
code: zod_1.z.ZodIssueCode.custom,
|
|
359
|
-
message: 'name accounts to use or request, a builds level, sandboxes or endpoints',
|
|
363
|
+
message: 'name accounts to use or request, a builds level, checks, sandboxes or endpoints',
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
if (entitlement.checks && entitlement.repos.length === 0) {
|
|
367
|
+
ctx.addIssue({
|
|
368
|
+
code: zod_1.z.ZodIssueCode.custom,
|
|
369
|
+
path: ['repos'],
|
|
370
|
+
message: 'checks are read for repositories: name them in repos',
|
|
360
371
|
});
|
|
361
372
|
}
|
|
362
373
|
});
|
|
@@ -509,6 +520,8 @@ exports.serverConfigSchema = zod_1.z
|
|
|
509
520
|
})
|
|
510
521
|
.strict()
|
|
511
522
|
.default({}),
|
|
523
|
+
// how long a passed check stays for `remote verify` to find (see checks.ts)
|
|
524
|
+
checksKeep: duration.default('30d'),
|
|
512
525
|
})
|
|
513
526
|
.strict()
|
|
514
527
|
.default({}),
|