hammerkit 1.26.0 → 1.27.0-next.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/build.schema.json +53 -1
  2. package/dist/cache/backends/local-cache-backend.js +10 -2
  3. package/dist/cache/explain.js +13 -11
  4. package/dist/cli.js +3 -0
  5. package/dist/docker/item-network.js +471 -0
  6. package/dist/docker/pull.js +4 -2
  7. package/dist/docker/remove-orphaned-containers.js +13 -6
  8. package/dist/environment/replace-env-variables.js +8 -6
  9. package/dist/executer/await-completed-dependencies.js +2 -1
  10. package/dist/executer/check-readiness.js +2 -2
  11. package/dist/executer/docker-service.js +24 -13
  12. package/dist/executer/docker-task.js +18 -29
  13. package/dist/executer/event-cache.js +4 -0
  14. package/dist/executer/execute-command.js +18 -5
  15. package/dist/executer/execute-docker.js +4 -1
  16. package/dist/executer/execution-steps.js +33 -7
  17. package/dist/executer/get-docker-executor.js +14 -7
  18. package/dist/executer/scheduler/enqueue-next.js +17 -3
  19. package/dist/executer/watch-loop.js +2 -2
  20. package/dist/executer/watch-state-key.js +1 -1
  21. package/dist/file/within-root.js +42 -0
  22. package/dist/harness/connect.js +99 -0
  23. package/dist/harness/format.js +73 -0
  24. package/dist/harness/mcp-command.js +59 -0
  25. package/dist/harness/mcp.js +432 -0
  26. package/dist/harness/opencode.js +492 -0
  27. package/dist/harness/pi.js +217 -0
  28. package/dist/harness/sandbox-client.js +373 -0
  29. package/dist/index.js +33 -36
  30. package/dist/kubernetes/apply.js +31 -1
  31. package/dist/kubernetes/await-running-state.js +33 -1
  32. package/dist/kubernetes/ensure-kubernetes-deployment-exists.js +43 -21
  33. package/dist/kubernetes/ensure-kubernetes-persistent-volume-claim-exists.js +131 -4
  34. package/dist/kubernetes/ensure-kubernetes-service-exists.js +25 -4
  35. package/dist/kubernetes/ensure-persistent-data.js +48 -7
  36. package/dist/kubernetes/get-service-ip.js +15 -1
  37. package/dist/kubernetes/job-placement.js +99 -0
  38. package/dist/kubernetes/job-secrets.js +306 -0
  39. package/dist/kubernetes/kept-claims.js +191 -0
  40. package/dist/kubernetes/kubernetes-instance.js +10 -2
  41. package/dist/kubernetes/network-gate.js +71 -0
  42. package/dist/kubernetes/network-policy.js +526 -0
  43. package/dist/kubernetes/resources.js +13 -2
  44. package/dist/kubernetes/restore-kubernetes-data.js +19 -4
  45. package/dist/kubernetes/secrets.js +2 -0
  46. package/dist/kubernetes/store-kubernetes-data.js +1 -1
  47. package/dist/kubernetes/volumes.js +100 -11
  48. package/dist/log.js +1 -1
  49. package/dist/main.js +41 -0
  50. package/dist/network/egress-proxy.js +368 -0
  51. package/dist/network/port-relay.js +35 -0
  52. package/dist/optimizer/get-work-cache-stats.js +16 -0
  53. package/dist/optimizer/work-task-cache-description.js +3 -2
  54. package/dist/planner/service-definition.js +5 -0
  55. package/dist/planner/utils/append-work-environment.js +2 -0
  56. package/dist/planner/utils/append-work-service.js +4 -0
  57. package/dist/planner/utils/append-work-task.js +17 -1
  58. package/dist/planner/utils/assert-item-paths.js +50 -0
  59. package/dist/planner/utils/hosted-volumes.js +35 -0
  60. package/dist/planner/work-network.js +193 -0
  61. package/dist/planner/work-runtime-docker.js +41 -0
  62. package/dist/planner/work-runtime-kubernetes.js +335 -102
  63. package/dist/program.js +23 -1
  64. package/dist/remote/annotations.js +103 -0
  65. package/dist/remote/bundle-text.js +24 -0
  66. package/dist/remote/bundle.js +9 -1
  67. package/dist/remote/remote-commands.js +4 -1
  68. package/dist/remote/remote-work-commands.js +182 -69
  69. package/dist/remote/server-client.js +18 -5
  70. package/dist/remote/source-manifest.js +15 -2
  71. package/dist/remote/token-source.js +67 -0
  72. package/dist/schema/build-file-base-task-schema.js +3 -0
  73. package/dist/schema/build-file-container-service-schema.js +6 -0
  74. package/dist/schema/build-file-container-task-schema.js +3 -0
  75. package/dist/schema/build-file-environment-schema.js +9 -0
  76. package/dist/schema/network-schema.js +78 -0
  77. package/dist/schema/reference-parser.js +1 -0
  78. package/dist/schema/schema-parser.js +16 -3
  79. package/dist/schema/work-scope-parser.js +5 -0
  80. package/dist/secrets/command-secret-provider.js +73 -10
  81. package/dist/secrets/github-app-token-command.js +26 -0
  82. package/dist/secrets/github-app-token.js +191 -0
  83. package/dist/server/accounts-api.js +9 -2
  84. package/dist/server/definitions-api.js +69 -47
  85. package/dist/server/definitions.js +76 -51
  86. package/dist/server/executor.js +156 -41
  87. package/dist/server/git-checkout.js +72 -8
  88. package/dist/server/kubernetes-backend.js +562 -0
  89. package/dist/server/kubernetes-sandbox-storage.js +996 -0
  90. package/dist/server/oauth.js +1 -0
  91. package/dist/server/plan.js +51 -1
  92. package/dist/server/policy.js +104 -15
  93. package/dist/server/proposals-api.js +24 -40
  94. package/dist/server/proposals.js +12 -25
  95. package/dist/server/repo-access.js +37 -6
  96. package/dist/server/run-logs.js +217 -0
  97. package/dist/server/run-plan.js +51 -2
  98. package/dist/server/run-request.js +23 -10
  99. package/dist/server/run-store.js +5 -0
  100. package/dist/server/run-worker-command.js +13 -1
  101. package/dist/server/run-worker.js +121 -24
  102. package/dist/server/runs.js +184 -10
  103. package/dist/server/sandbox-docker.js +62 -0
  104. package/dist/server/sandbox-files-api.js +269 -0
  105. package/dist/server/sandbox-files.js +1054 -0
  106. package/dist/server/sandbox-patch.js +481 -0
  107. package/dist/server/sandbox-pod-command.js +15 -0
  108. package/dist/server/sandbox-pod.js +146 -0
  109. package/dist/server/sandbox-storage.js +462 -0
  110. package/dist/server/sandboxes-api.js +210 -0
  111. package/dist/server/sandboxes.js +519 -0
  112. package/dist/server/server-config.js +412 -22
  113. package/dist/server/server.js +204 -39
  114. package/dist/utils/run-id.js +12 -0
  115. package/harness/opencode.ts +11 -0
  116. package/harness/pi.ts +7 -0
  117. package/package.json +8 -2
  118. package/dist/remote/text-diff.js +0 -89
@@ -0,0 +1,462 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.HostSandboxStorage = exports.SandboxPushFailed = exports.SandboxPushConflict = void 0;
4
+ const crypto_1 = require("crypto");
5
+ const promises_1 = require("fs/promises");
6
+ const path_1 = require("path");
7
+ const git_checkout_1 = require("./git-checkout");
8
+ const sandbox_files_1 = require("./sandbox-files");
9
+ // how many directories the walk reads at once, and how many entries of one it looks at at once
10
+ const WALK_DIRECTORIES = 16;
11
+ const WALK_ENTRIES = 256;
12
+ class SandboxPushConflict extends Error {
13
+ }
14
+ exports.SandboxPushConflict = SandboxPushConflict;
15
+ // A push that failed after its commit was written: the remote may have taken it anyway (a
16
+ // timeout after it answered), which the caller checks against the branch.
17
+ class SandboxPushFailed extends Error {
18
+ constructor(commit, cause) {
19
+ super(cause instanceof Error ? cause.message : String(cause));
20
+ this.commit = commit;
21
+ this.cause = cause;
22
+ }
23
+ }
24
+ exports.SandboxPushFailed = SandboxPushFailed;
25
+ // the commit a sandbox pushed last, in its own git data
26
+ const PUSHED_REF = 'refs/hammerkit/pushed';
27
+ /**
28
+ * Sandboxes as directories below `directory`, one per sandbox: `files` is the checkout the
29
+ * tasks work on, `git` its git data. The git data is kept outside of the files on purpose:
30
+ * tasks write to the files, and a `.git/config` or a hook they wrote would be read by the
31
+ * git of the server, which runs as the server's user. Every git command names its git
32
+ * directory and work tree, and runs without hooks or a file system monitor.
33
+ */
34
+ class HostSandboxStorage {
35
+ constructor(directory,
36
+ // the environment of git when it reads a sandbox: the variables runs see
37
+ env,
38
+ // what containers left of a sandbox, when runs on it use containers
39
+ containers) {
40
+ this.directory = directory;
41
+ this.env = env;
42
+ this.containers = containers;
43
+ // one git command or walk at a time per sandbox: the objects `state` writes are pruned
44
+ // again, which must not meet another command that still needs them
45
+ this.queues = new Map();
46
+ // the sandboxes walked since a run last wrote to them, with the links the walk found: only
47
+ // a run's tasks write to a sandbox, between its `attach` and the `usage` it ends with, so
48
+ // until the next attach a `state` needs no walk. The edits of the file API write readable
49
+ // files and never a link, so a walk still holds after them: a link it found that an edit
50
+ // replaced is no link any more, and is added as the file it is.
51
+ this.walked = new Map();
52
+ // the sandboxes a run attached and that were not measured since, and how often each was
53
+ // attached: a walk while a run may write, or that a run attached during, is not reused
54
+ this.attached = new Set();
55
+ this.attaches = new Map();
56
+ }
57
+ exclusive(id, fn) {
58
+ const previous = this.queues.get(id) ?? Promise.resolve();
59
+ const next = previous.then(fn, fn);
60
+ const settled = next.catch(() => undefined);
61
+ this.queues.set(id, settled);
62
+ settled.then(() => {
63
+ if (this.queues.get(id) === settled) {
64
+ this.queues.delete(id);
65
+ }
66
+ });
67
+ return next;
68
+ }
69
+ root(id) {
70
+ return (0, path_1.join)(this.directory, id);
71
+ }
72
+ workTree(id) {
73
+ return (0, path_1.join)(this.root(id), 'files');
74
+ }
75
+ async git(id, args, env, signal, action, options = {}) {
76
+ const root = this.root(id);
77
+ return (0, git_checkout_1.git)([
78
+ `--git-dir=${(0, path_1.join)(root, 'git')}`,
79
+ `--work-tree=${this.workTree(id)}`,
80
+ '-c',
81
+ `core.hooksPath=${(0, path_1.join)(root, 'no-hooks')}`,
82
+ '-c',
83
+ 'core.fsmonitor=false',
84
+ // files as committed, on every system: a checkout with the machine's line endings
85
+ // and a read without its config would see every file changed (Git for Windows)
86
+ '-c',
87
+ 'core.autocrlf=false',
88
+ '-c',
89
+ 'advice.detachedHead=false',
90
+ '-c',
91
+ 'advice.addEmbeddedRepo=false',
92
+ ...args,
93
+ ], env, signal, `sandbox ${id}`, action, options);
94
+ }
95
+ // The environment of git when it reads the sandbox, without an index of its own.
96
+ readEnv(extra = {}) {
97
+ const env = {};
98
+ for (const [key, value] of Object.entries(this.env)) {
99
+ if (value !== undefined && key !== 'GIT_INDEX_FILE') {
100
+ env[key] = value;
101
+ }
102
+ }
103
+ // reading the files needs no configuration of the server's: a filter or attributes the
104
+ // machine's git config names would run on files a task wrote
105
+ env.GIT_CONFIG_NOSYSTEM = '1';
106
+ // a file that is never there, which git reads as empty: Git for Windows refuses `\\.\nul`
107
+ env.GIT_CONFIG_GLOBAL = (0, path_1.join)(this.directory, 'no-git-config');
108
+ return { ...env, ...extra };
109
+ }
110
+ // `fn` with a copy of the sandbox's index that holds the files as they are (`addFiles`), so
111
+ // neither the index nor the files change, and what it wrote to the git data is pruned again.
112
+ // Files a task made unreadable are made readable first: git would leave them out and call
113
+ // them unchanged. Holds the sandbox. No walk when nothing wrote since the last one.
114
+ withIndex(id, signal, fn) {
115
+ return this.exclusive(id, async () => {
116
+ const links = this.walked.get(id) ?? (await this.walkOnce(id)).links;
117
+ try {
118
+ return await this.indexed(id, signal, links, fn);
119
+ }
120
+ finally {
121
+ await this.prune(id);
122
+ }
123
+ });
124
+ }
125
+ // a copy of the index that holds the files as they are (`addFiles`)
126
+ async indexed(id, signal, links, fn) {
127
+ const root = this.root(id);
128
+ const index = (0, path_1.join)(root, `index-${(0, crypto_1.randomBytes)(6).toString('hex')}`);
129
+ try {
130
+ const original = (0, path_1.join)(root, 'git', 'index');
131
+ await (0, promises_1.copyFile)(original, index).then(
132
+ // with the time of the index: git takes a file changed in the same second as it was
133
+ // written for changed only when the index is no newer than the file (racy git)
134
+ async () => {
135
+ const stats = await (0, promises_1.stat)(original);
136
+ await (0, promises_1.utimes)(index, stats.atime, stats.mtime);
137
+ }, () => undefined);
138
+ const env = this.readEnv({ GIT_INDEX_FILE: index });
139
+ await this.addFiles(id, env, signal, links);
140
+ return await fn((args, action, options = {}) => this.git(id, args, { ...env, ...options.env }, signal, action, { maxBuffer: options.maxBuffer }));
141
+ }
142
+ finally {
143
+ await (0, promises_1.rm)(index, { force: true });
144
+ }
145
+ }
146
+ // the sandbox's own index, HEAD and refs keep what they need
147
+ async prune(id) {
148
+ await this.git(id, ['prune', '--expire=now'], this.readEnv(), AbortSignal.timeout(60000), 'read').catch(() => undefined);
149
+ }
150
+ // `maxChangesBytes` is for tests, which make changes larger than it without writing 64 MiB
151
+ files({ id }, maxChangesBytes) {
152
+ return this.filesOf(id, (fn) => this.withIndex(id, AbortSignal.timeout(5 * 60 * 1000), fn), maxChangesBytes);
153
+ }
154
+ filesOf(id, withIndex, maxChangesBytes) {
155
+ const timeout = () => AbortSignal.timeout(5 * 60 * 1000);
156
+ return new sandbox_files_1.HostSandboxFiles(this.workTree(id), {
157
+ run: (args, action, options = {}) => this.git(id, args, this.readEnv(options.env), timeout(), action, { maxBuffer: options.maxBuffer }),
158
+ withIndex,
159
+ }, this.env, maxChangesBytes);
160
+ }
161
+ async create({ id }, fetch) {
162
+ const root = this.root(id);
163
+ // the sandboxes are their owners', nobody else on the machine reads them
164
+ await (0, promises_1.mkdir)(this.directory, { recursive: true, mode: 0o700 });
165
+ await (0, promises_1.mkdir)(this.workTree(id), { recursive: true });
166
+ try {
167
+ await (0, git_checkout_1.git)(['init', '--quiet', '--bare', (0, path_1.join)(root, 'git')], fetch.env, fetch.signal, `sandbox ${id}`, 'create');
168
+ return await this.checkout(id, fetch, true);
169
+ }
170
+ catch (e) {
171
+ await this.removeId(id);
172
+ throw e;
173
+ }
174
+ }
175
+ async fetch({ id }, fetch) {
176
+ return this.exclusive(id, () => this.checkout(id, fetch, false));
177
+ }
178
+ async checkout(id, fetch, force) {
179
+ if (!(0, git_checkout_1.isValidRef)(fetch.ref)) {
180
+ throw new Error(`${JSON.stringify(fetch.ref.slice(0, 80))} is not a branch`);
181
+ }
182
+ this.walked.delete(id);
183
+ // the branch by its full name: a tag of the same name is never taken
184
+ await this.git(id, ['fetch', '--quiet', '--depth', '1', '--', fetch.url, `refs/heads/${fetch.ref}`], fetch.env, fetch.signal, 'fetch');
185
+ await this.git(id, ['checkout', '--quiet', ...(force ? ['--force'] : []), 'FETCH_HEAD'], fetch.env, fetch.signal, 'check out');
186
+ return (await this.git(id, ['rev-parse', 'HEAD'], fetch.env, fetch.signal, 'read')).trim();
187
+ }
188
+ // One hold of the sandbox from the commit to the push: a prune in between would remove the
189
+ // commit. A ref keeps the commit until the sandbox is moved to it, also when the push
190
+ // failed late and the remote has it after all.
191
+ async push({ id }, push) {
192
+ return this.exclusive(id, async () => {
193
+ const links = this.walked.get(id) ?? (await this.walkOnce(id)).links;
194
+ try {
195
+ return await this.commitAndPush(id, push, links);
196
+ }
197
+ finally {
198
+ await this.prune(id);
199
+ }
200
+ });
201
+ }
202
+ // Commits the changes for a push from elsewhere (the server, for the pod of a sandbox on a
203
+ // cluster), and keeps the commit as the push does until the sandbox is moved to it.
204
+ async commit({ id }, author, signal) {
205
+ return this.exclusive(id, async () => {
206
+ const links = this.walked.get(id) ?? (await this.walkOnce(id)).links;
207
+ try {
208
+ return await this.keptCommit(id, author, signal, links);
209
+ }
210
+ finally {
211
+ await this.prune(id);
212
+ }
213
+ });
214
+ }
215
+ async keptCommit(id, author, signal, links) {
216
+ const commit = await this.filesOf(id, (fn) => this.indexed(id, signal, links, fn)).commit(author);
217
+ if (commit) {
218
+ await this.git(id, ['update-ref', PUSHED_REF, commit], this.readEnv(), signal, 'push');
219
+ }
220
+ return commit;
221
+ }
222
+ async commitAndPush(id, push, links) {
223
+ const commit = await this.keptCommit(id, push.author, push.signal, links);
224
+ if (!commit) {
225
+ return undefined;
226
+ }
227
+ const run = (args, action, env = this.readEnv()) => this.git(id, args, env, push.signal, action);
228
+ // a branch this sandbox pushed before only moves forward: the sandbox is at what it
229
+ // pushed last, or it was moved since and its changes are on another commit
230
+ if (push.expect) {
231
+ const ancestor = await run(['merge-base', '--is-ancestor', push.expect, commit], 'read').then(() => true, () => false);
232
+ if (!ancestor) {
233
+ throw new SandboxPushConflict(`${push.branch} is at ${push.expect}, which the sandbox is no longer on: push to a new branch`);
234
+ }
235
+ }
236
+ // the lease makes the remote refuse when the branch moved since it was read, and when
237
+ // somebody created it meanwhile
238
+ let printed;
239
+ try {
240
+ printed = await this.git(id, [
241
+ 'push',
242
+ // a ref the remote refuses is a line with the reason (a GitHub App without the
243
+ // workflows permission, a ruleset), where git's last line only says it failed
244
+ '--porcelain',
245
+ '--no-verify',
246
+ `--force-with-lease=refs/heads/${push.branch}:${push.expect ?? ''}`,
247
+ '--',
248
+ push.url,
249
+ `${commit}:refs/heads/${push.branch}`,
250
+ ],
251
+ // the write credential is in the environment of this one git process, which is
252
+ // that environment alone: nothing of the server's own git configuration
253
+ push.env, push.signal, 'push', { exitCodes: [1] });
254
+ }
255
+ catch (e) {
256
+ throw new SandboxPushFailed(commit, e);
257
+ }
258
+ // `<flag>\t<from>:<to>\t<summary>`; `!` is a refused ref
259
+ const line = printed
260
+ .split('\n')
261
+ .map((printedLine) => printedLine.split('\t'))
262
+ .find((fields) => fields.length >= 3 && fields[1].endsWith(`:refs/heads/${push.branch}`));
263
+ if (!line || line[0] === '!') {
264
+ const reason = line ? line.slice(2).join(' ') : printed.trim() || 'git printed nothing';
265
+ throw new SandboxPushFailed(commit, new Error(`the remote refused the push to ${push.branch}: ${reason}`));
266
+ }
267
+ return commit;
268
+ }
269
+ async moveTo({ id }, commit) {
270
+ if (!/^[0-9a-f]{40}([0-9a-f]{24})?$/.test(commit)) {
271
+ throw new Error(`${JSON.stringify(commit.slice(0, 80))} is not a commit`);
272
+ }
273
+ await this.exclusive(id, () => this.git(id, ['reset', '--quiet', commit], this.readEnv(), AbortSignal.timeout(5 * 60 * 1000), 'move'));
274
+ }
275
+ // The tree is written with an index of its own, so neither the sandbox's index nor its
276
+ // files change, and what it wrote to the git data is pruned again. Files a task made
277
+ // unreadable are made readable first: git would leave them out and call the sandbox
278
+ // clean.
279
+ async state({ id }, signal = AbortSignal.timeout(5 * 60 * 1000)) {
280
+ return this.withIndex(id, signal, async (run) => {
281
+ const tree = (await run(['write-tree'], 'read')).trim();
282
+ const [commit, base] = (await run(['rev-parse', 'HEAD', 'HEAD^{tree}'], 'read')).trim().split('\n');
283
+ return { commit, tree, clean: tree === base };
284
+ });
285
+ }
286
+ // Adds the files as they are to the index of `env`. A link is written as a link and never
287
+ // walked: Git for Windows takes a junction for a directory, and would read what a
288
+ // `mklink /J x C:\` of a task points to.
289
+ async addFiles(id, env, signal, links) {
290
+ const written = await this.unignoredLinks(id, links, env, signal);
291
+ const pathspecs = ['.', ...written.map((link) => `:(top,exclude,literal)${link.path}`)];
292
+ await this.git(id, ['add', '--all', '--pathspec-from-file=-', '--pathspec-file-nul'], env, signal, 'read', {
293
+ input: pathspecs.map((pathspec) => `${pathspec}\0`).join(''),
294
+ });
295
+ if (written.length > 0) {
296
+ const entries = [];
297
+ for (const link of written) {
298
+ const blob = await this.git(id, ['hash-object', '-w', '--stdin'], env, signal, 'read', { input: link.target });
299
+ entries.push(`120000 ${blob.trim()}\t${link.path}\0`);
300
+ }
301
+ await this.git(id, ['update-index', '-z', '--add', '--replace', '--index-info'], env, signal, 'read', {
302
+ input: entries.join(''),
303
+ });
304
+ }
305
+ }
306
+ // the links of the files that are still links and that git does not ignore, by their path
307
+ // in the files ('/'-separated) and their target
308
+ async unignoredLinks(id, paths, env, signal) {
309
+ const files = this.workTree(id);
310
+ const links = [];
311
+ for (const path of paths) {
312
+ if (!path.startsWith(files + path_1.sep) || !(await (0, promises_1.lstat)(path).catch(() => undefined))?.isSymbolicLink()) {
313
+ continue;
314
+ }
315
+ links.push({ path: (0, path_1.relative)(files, path).split(path_1.sep).join('/'), target: await (0, promises_1.readlink)(path, 'buffer') });
316
+ }
317
+ if (links.length === 0) {
318
+ return [];
319
+ }
320
+ const ignored = await this.git(id, ['check-ignore', '-z', '--stdin'], env, signal, 'read', {
321
+ input: links.map((link) => `${link.path}\0`).join(''),
322
+ exitCodes: [1],
323
+ });
324
+ const skip = new Set(ignored.split('\0'));
325
+ return links.filter((link) => !skip.has(link.path));
326
+ }
327
+ // the run works in the sandbox itself
328
+ async attach({ id }) {
329
+ const files = this.workTree(id);
330
+ const stats = await (0, promises_1.lstat)(files).catch(() => undefined);
331
+ if (!stats?.isDirectory()) {
332
+ throw new Error(`the sandbox ${id} has no files on this server`);
333
+ }
334
+ // the run's tasks write from now on
335
+ this.walked.delete(id);
336
+ this.attached.add(id);
337
+ this.attaches.set(id, (this.attaches.get(id) ?? 0) + 1);
338
+ return { directory: files };
339
+ }
340
+ // What a task wrote is counted whatever it did to the modes, and with the volumes of its
341
+ // outputs. Symbolic links are counted, not followed: a link a task made points anywhere.
342
+ async usage({ id }) {
343
+ return this.exclusive(id, async () => {
344
+ // after a run: its tasks are done
345
+ this.attached.delete(id);
346
+ const { bytes } = await this.walkOnce(id);
347
+ return bytes + ((await this.containers?.volumeBytes(this.workTree(id)).catch(() => 0)) ?? 0);
348
+ });
349
+ }
350
+ async remove({ id }) {
351
+ await this.removeId(id);
352
+ }
353
+ async removeId(id) {
354
+ await this.exclusive(id, async () => {
355
+ this.walked.delete(id);
356
+ this.attached.delete(id);
357
+ this.attaches.delete(id);
358
+ await this.reclaim(id).catch(() => undefined);
359
+ await (0, promises_1.rm)(this.root(id), { recursive: true, force: true, maxRetries: 3 });
360
+ // without docker there is nothing to remove now; volumes stay labelled with the files
361
+ await this.containers?.removeVolumes(this.workTree(id)).catch(() => undefined);
362
+ });
363
+ }
364
+ // Walks the sandbox, and keeps the walk for the next `state` when no run wrote meanwhile.
365
+ async walkOnce(id) {
366
+ const attaches = this.attaches.get(id);
367
+ const walk = await this.reclaim(id);
368
+ if (!this.attached.has(id) && this.attaches.get(id) === attaches) {
369
+ this.walked.set(id, walk.links);
370
+ }
371
+ return walk;
372
+ }
373
+ // Walks the sandbox, makes every directory and file the server's user can read and write
374
+ // (a task runs as that user and may have taken its own permissions away) and answers the
375
+ // bytes. What a container left as another user is given back through a container first.
376
+ async reclaim(id) {
377
+ try {
378
+ return await this.walk(this.root(id));
379
+ }
380
+ catch (e) {
381
+ const code = e.code;
382
+ if (!this.containers || (code !== 'EPERM' && code !== 'EACCES')) {
383
+ throw e;
384
+ }
385
+ await this.containers.reclaim(this.root(id));
386
+ return await this.walk(this.root(id));
387
+ }
388
+ }
389
+ // Directories a few at a time and the entries of each together: a `node_modules` holds
390
+ // tens of thousands of files, and one at a time took seconds for every run.
391
+ async walk(root) {
392
+ let bytes = 0;
393
+ const links = [];
394
+ let pending = [root];
395
+ while (pending.length > 0) {
396
+ const directories = pending.slice(0, WALK_DIRECTORIES);
397
+ pending = pending.slice(WALK_DIRECTORIES);
398
+ for (const found of await Promise.all(directories.map((directory) => this.walkDirectory(directory, root)))) {
399
+ bytes += found.bytes;
400
+ links.push(...found.links);
401
+ pending.push(...found.directories);
402
+ }
403
+ }
404
+ return { bytes, links };
405
+ }
406
+ async walkDirectory(current, root) {
407
+ await ensureMode(current, 0o700);
408
+ const entries = await (0, promises_1.readdir)(current, { withFileTypes: true }).catch((e) => {
409
+ if (e.code === 'ENOENT' && current !== root) {
410
+ return [];
411
+ }
412
+ throw e;
413
+ });
414
+ const directories = [];
415
+ const links = [];
416
+ let bytes = 0;
417
+ for (let start = 0; start < entries.length; start += WALK_ENTRIES) {
418
+ const sizes = await Promise.all(entries.slice(start, start + WALK_ENTRIES).map(async (entry) => {
419
+ const path = (0, path_1.join)(current, entry.name);
420
+ if (entry.isDirectory()) {
421
+ directories.push(path);
422
+ return 0;
423
+ }
424
+ if (entry.isSymbolicLink()) {
425
+ links.push(path);
426
+ }
427
+ // one lstat for the mode and the size; a link is counted, not followed
428
+ return (await ensureMode(path, entry.isFile() ? 0o600 : 0))?.size ?? 0;
429
+ }));
430
+ bytes += sizes.reduce((sum, size) => sum + size, 0);
431
+ }
432
+ return { bytes, directories, links };
433
+ }
434
+ async list() {
435
+ // only a server that never kept a sandbox has no directory; any other error is one
436
+ const entries = await (0, promises_1.readdir)(this.directory, { withFileTypes: true }).catch((e) => {
437
+ if (e.code === 'ENOENT') {
438
+ return [];
439
+ }
440
+ throw e;
441
+ });
442
+ return entries
443
+ .filter((entry) => entry.isDirectory())
444
+ .map((entry) => ({ id: entry.name, remove: () => this.removeId(entry.name) }));
445
+ }
446
+ }
447
+ exports.HostSandboxStorage = HostSandboxStorage;
448
+ // adds the bits of the owner a path lacks and answers what it found; a path that is gone
449
+ // since it was listed is fine
450
+ async function ensureMode(path, bits) {
451
+ const stats = await (0, promises_1.lstat)(path).catch((e) => {
452
+ if (e.code === 'ENOENT') {
453
+ return undefined;
454
+ }
455
+ throw e;
456
+ });
457
+ if (stats && (stats.mode & bits) !== bits) {
458
+ await (0, promises_1.chmod)(path, (stats.mode & 0o7777) | bits);
459
+ }
460
+ return stats;
461
+ }
462
+ //# sourceMappingURL=sandbox-storage.js.map
@@ -0,0 +1,210 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.author = exports.handleSandboxes = void 0;
4
+ const zod_1 = require("zod");
5
+ const http_1 = require("./http");
6
+ const policy_1 = require("./policy");
7
+ const repo_1 = require("./repo");
8
+ const runs_1 = require("./runs");
9
+ const sandbox_files_api_1 = require("./sandbox-files-api");
10
+ const sandboxes_1 = require("./sandboxes");
11
+ const refSchema = zod_1.z.string().min(1).max(200);
12
+ const createSchema = zod_1.z
13
+ .object({
14
+ // `host/path`, as a ref run names it
15
+ repo: zod_1.z.string().transform((value, ctx) => {
16
+ const repo = (0, repo_1.normalizeRepo)(value);
17
+ if (!repo) {
18
+ ctx.addIssue({ code: zod_1.z.ZodIssueCode.custom, message: 'expected a repository like github.com/corp/payments' });
19
+ return zod_1.z.NEVER;
20
+ }
21
+ return repo;
22
+ }),
23
+ // a branch: a sandbox never starts from a tag or a commit
24
+ ref: refSchema,
25
+ })
26
+ .strict();
27
+ const fetchSchema = zod_1.z.object({ ref: refSchema }).strict();
28
+ // trailers by which a person attests a commit, which a caller could otherwise put in another
29
+ // person's name
30
+ const ATTESTATION = /^\s*(signed-off-by|reviewed-by|acked-by|tested-by)\s*:/im;
31
+ const pushSchema = zod_1.z
32
+ .object({
33
+ branch: refSchema,
34
+ // the message of the commit, as `git commit -m` takes it; it can not sign off for, or
35
+ // claim the review of, anybody
36
+ message: zod_1.z
37
+ .string()
38
+ .trim()
39
+ .min(1)
40
+ .max(10000)
41
+ .refine((message) => !ATTESTATION.test(message), 'the message may not carry Signed-off-by, Reviewed-by, Acked-by or Tested-by: the server pushes as the caller'),
42
+ })
43
+ .strict();
44
+ /**
45
+ * The sandboxes of a caller: checkouts on the server that runs work on.
46
+ *
47
+ * POST /v1/sandboxes { repo, ref } fetch a branch into a new sandbox
48
+ * GET /v1/sandboxes the caller's sandboxes, their limits and usage
49
+ * GET /v1/sandboxes/<id> one sandbox
50
+ * DELETE /v1/sandboxes/<id> cancel its runs and remove it
51
+ * POST /v1/sandboxes/<id>/fetch { ref } move a sandbox without changes to a branch's commit
52
+ * POST /v1/sandboxes/<id>/push { branch, message } commit the changes and push them
53
+ * ... /v1/sandboxes/<id>/tree, files/<path>, search, edits, changes: its files, see
54
+ * `handleSandboxFiles`
55
+ *
56
+ * A sandbox is its owner's: anybody else gets `404`.
57
+ */
58
+ async function handleSandboxes(config, sandboxes, repos, log, principal, req, res, url) {
59
+ const match = /^\/v1\/sandboxes(?:\/([^/]+)(?:\/(fetch|push|tree|search|edits|changes)|\/(files)\/(.+))?)?$/.exec(url.pathname);
60
+ if (!match) {
61
+ return false;
62
+ }
63
+ const [, id, named, files, encodedPath] = match;
64
+ const action = named ?? files;
65
+ const method = req.method;
66
+ const who = `iss=${principal.iss} sub=${principal.sub}`;
67
+ try {
68
+ if (!id) {
69
+ if (method === 'GET') {
70
+ const limits = sandboxes.limits;
71
+ (0, http_1.send)(res, 200, {
72
+ sandboxes: sandboxes.list(principal).map((sandbox) => view(sandboxes, sandbox)),
73
+ limits: { count: limits.count, bytes: limits.bytes },
74
+ used: sandboxes.usage(principal),
75
+ });
76
+ }
77
+ else if (method === 'POST') {
78
+ const body = parse(createSchema, await (0, http_1.readJson)(req));
79
+ const refused = (0, policy_1.decideSandbox)((0, policy_1.getGrant)(config.entitlements, principal), body.repo, body.ref, (repo) => repos.readable(repo));
80
+ if (refused) {
81
+ log(`refuse sandbox ${who} repo=${body.repo}: ${refused.error}`);
82
+ (0, http_1.send)(res, refused.status, { error: refused.error });
83
+ return true;
84
+ }
85
+ const sandbox = await sandboxes.create(principal, body.repo, body.ref);
86
+ log(`sandbox create ${sandbox.id} ${who} repo=${sandbox.repo} commit=${sandbox.commit} bytes=${sandbox.bytes}`);
87
+ (0, http_1.send)(res, 201, view(sandboxes, sandbox));
88
+ }
89
+ else {
90
+ (0, http_1.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET, POST' });
91
+ }
92
+ return true;
93
+ }
94
+ const sandbox = runs_1.SANDBOX_ID_PATTERN.test(id) ? sandboxes.get(principal, id) : undefined;
95
+ if (!sandbox) {
96
+ (0, http_1.send)(res, 404, { error: 'not found' });
97
+ return true;
98
+ }
99
+ if (action && action !== 'fetch' && action !== 'push') {
100
+ // the caller may have lost the repository, or the sandbox's branch, since they created it
101
+ const refused = (0, policy_1.decideSandbox)((0, policy_1.getGrant)(config.entitlements, principal), sandbox.repo, sandbox.ref, (repo) => repos.readable(repo));
102
+ if (refused) {
103
+ log(`refuse sandbox ${action} ${id} ${who}: ${refused.error}`);
104
+ (0, http_1.send)(res, refused.status, { error: refused.error });
105
+ return true;
106
+ }
107
+ let path;
108
+ try {
109
+ path = encodedPath === undefined ? undefined : decodeURIComponent(encodedPath);
110
+ }
111
+ catch {
112
+ (0, http_1.send)(res, 400, { error: 'the path is not valid percent-encoding' });
113
+ return true;
114
+ }
115
+ await (0, sandbox_files_api_1.handleSandboxFiles)(sandboxes, log, principal, sandbox, action, path, req, res, url);
116
+ return true;
117
+ }
118
+ if (action === 'fetch' || action === 'push') {
119
+ if (method !== 'POST') {
120
+ (0, http_1.send)(res, 405, { error: 'method not allowed' }, { allow: 'POST' });
121
+ return true;
122
+ }
123
+ const body = action === 'fetch' ? parse(fetchSchema, await (0, http_1.readJson)(req)) : parse(pushSchema, await (0, http_1.readJson)(req));
124
+ // the caller may have lost the repository or the sandbox's branch since they created the
125
+ // sandbox, and a fetch needs the new branch
126
+ const grant = (0, policy_1.getGrant)(config.entitlements, principal);
127
+ const refused = (0, policy_1.decideSandbox)(grant, sandbox.repo, 'ref' in body ? body.ref : sandbox.ref, (repo) => repos.readable(repo));
128
+ if (refused) {
129
+ log(`refuse sandbox ${action} ${id} ${who}: ${refused.error}`);
130
+ (0, http_1.send)(res, refused.status, { error: refused.error });
131
+ return true;
132
+ }
133
+ if ('branch' in body) {
134
+ const pushed = await sandboxes.push(principal, id, body.branch, { ...author(principal), message: body.message }, (0, policy_1.sandboxPushBranches)(grant, sandbox.repo));
135
+ log(`sandbox push ${id} ${who} branch=${body.branch} commit=${pushed.commit}`);
136
+ (0, http_1.send)(res, 200, { branch: body.branch, commit: pushed.commit, sandbox: view(sandboxes, pushed) });
137
+ return true;
138
+ }
139
+ const fetched = await sandboxes.fetch(principal, id, body.ref);
140
+ log(`sandbox fetch ${id} ${who} ref=${body.ref} commit=${fetched.commit}`);
141
+ (0, http_1.send)(res, 200, view(sandboxes, fetched));
142
+ }
143
+ else if (method === 'GET') {
144
+ (0, http_1.send)(res, 200, view(sandboxes, sandbox));
145
+ }
146
+ else if (method === 'DELETE') {
147
+ // a caller who lost the entitlement can still delete what they keep
148
+ const removed = await sandboxes.remove(principal, id);
149
+ log(`sandbox delete ${id} ${who} found=${removed}`);
150
+ removed ? (0, http_1.send)(res, 200, { deleted: id }) : (0, http_1.send)(res, 404, { error: 'not found' });
151
+ }
152
+ else {
153
+ (0, http_1.send)(res, 405, { error: 'method not allowed' }, { allow: 'GET, DELETE' });
154
+ }
155
+ }
156
+ catch (e) {
157
+ if (e instanceof sandboxes_1.SandboxError) {
158
+ log(`refuse sandbox ${id ?? 'create'} ${who}: ${e.message}`);
159
+ (0, http_1.send)(res, e.status, { error: e.message });
160
+ return true;
161
+ }
162
+ throw e;
163
+ }
164
+ return true;
165
+ }
166
+ exports.handleSandboxes = handleSandboxes;
167
+ function parse(schema, body) {
168
+ const parsed = schema.safeParse(body);
169
+ if (!parsed.success) {
170
+ const issue = parsed.error.issues[0];
171
+ throw new sandboxes_1.SandboxError(400, `${issue.path.join('.') || 'request'}: ${issue.message}`);
172
+ }
173
+ return parsed.data;
174
+ }
175
+ /**
176
+ * Who the commit of a push is by: the caller, as their token names them (`name` and
177
+ * `email`, unless the issuer says the address is not verified), else by their subject at the
178
+ * host of their issuer. Never what the request says, so a commit can not claim to be
179
+ * somebody else's.
180
+ */
181
+ function author(principal) {
182
+ const clean = (value) => (value ?? '')
183
+ .replace(/[<>\0\r\n]/g, '')
184
+ .trim()
185
+ .slice(0, 200);
186
+ const name = clean(principal.claims.name) || clean(principal.claims.preferred_username) || clean(principal.sub);
187
+ // an address the issuer says it did not verify is whatever the caller typed in
188
+ const claimed = principal.emailVerified === false ? '' : clean(principal.claims.email);
189
+ if (/^[^\s@]+@[^\s@]+$/.test(claimed)) {
190
+ return { name, email: claimed };
191
+ }
192
+ let host = 'hammerkit';
193
+ try {
194
+ host = new URL(principal.iss).hostname || host;
195
+ }
196
+ catch {
197
+ // an issuer that is not a URL keeps the default
198
+ }
199
+ const local = principal.sub.replace(/[^A-Za-z0-9._+-]/g, '-').slice(0, 64) || 'sandbox';
200
+ return { name, email: `${local}@${host}` };
201
+ }
202
+ exports.author = author;
203
+ // What the owner sees: everything but the owner, and the run that holds it.
204
+ function view(sandboxes, sandbox) {
205
+ const { owner, ...visible } = sandbox;
206
+ void owner;
207
+ const run = sandboxes.holder(sandbox.id);
208
+ return { ...visible, busy: run !== undefined, run: run ?? null };
209
+ }
210
+ //# sourceMappingURL=sandboxes-api.js.map