hammerkit 1.27.0-next.1 → 1.27.0-next.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/build.schema.json +9 -0
  2. package/dist/cache/backends/registry-cache-backend.js +2 -2
  3. package/dist/executer/event-cache.js +20 -2
  4. package/dist/harness/sandbox-client.js +104 -66
  5. package/dist/kubernetes/apply.js +27 -16
  6. package/dist/kubernetes/claim-layout.js +71 -0
  7. package/dist/kubernetes/ensure-http-route.js +1 -1
  8. package/dist/kubernetes/ensure-kubernetes-deployment-exists.js +5 -9
  9. package/dist/kubernetes/ensure-kubernetes-persistent-volume-claim-exists.js +1 -1
  10. package/dist/kubernetes/ensure-kubernetes-service-exists.js +4 -9
  11. package/dist/kubernetes/ensure-persistent-data.js +30 -20
  12. package/dist/kubernetes/get-service-ip.js +2 -2
  13. package/dist/kubernetes/job-secrets.js +33 -19
  14. package/dist/kubernetes/kept-claims.js +22 -10
  15. package/dist/kubernetes/network-policy.js +83 -26
  16. package/dist/kubernetes/remove-persistent-data.js +3 -1
  17. package/dist/kubernetes/resources.js +25 -3
  18. package/dist/kubernetes/restore-kubernetes-data.js +9 -3
  19. package/dist/kubernetes/secrets.js +7 -11
  20. package/dist/kubernetes/volumes.js +6 -3
  21. package/dist/log.js +1 -1
  22. package/dist/network/egress-proxy.js +45 -6
  23. package/dist/optimizer/work-task-cache-description.js +2 -0
  24. package/dist/planner/utils/append-work-dependencies.js +5 -2
  25. package/dist/planner/utils/append-work-service.js +1 -0
  26. package/dist/planner/utils/append-work-task.js +4 -1
  27. package/dist/planner/utils/task-arguments.js +3 -0
  28. package/dist/planner/work-runtime-docker.js +2 -1
  29. package/dist/planner/work-runtime-kubernetes.js +74 -48
  30. package/dist/planner/work-runtime-local.js +2 -1
  31. package/dist/program.js +12 -5
  32. package/dist/remote/annotations.js +13 -1
  33. package/dist/remote/remote-commands.js +11 -2
  34. package/dist/remote/remote-work-commands.js +89 -1
  35. package/dist/schema/build-file-base-task-schema.js +5 -1
  36. package/dist/schema/build-file-container-service-schema.js +2 -0
  37. package/dist/schema/reference-parser.js +1 -0
  38. package/dist/secrets/github-app-token-command.js +4 -1
  39. package/dist/server/executor.js +1 -1
  40. package/dist/server/kubernetes-backend.js +80 -31
  41. package/dist/server/kubernetes-sandbox-storage.js +2 -1
  42. package/dist/server/oauth.js +12 -1
  43. package/dist/server/plan.js +46 -10
  44. package/dist/server/run-plan.js +12 -15
  45. package/dist/server/run-request.js +6 -0
  46. package/dist/server/runs.js +10 -3
  47. package/dist/server/sandboxes-api.js +10 -4
  48. package/dist/server/server-command.js +39 -20
  49. package/dist/server/server-config.js +35 -4
  50. package/dist/server/server.js +16 -2
  51. package/dist/utils/abort-on-signals.js +46 -6
  52. package/dist/utils/tar.js +25 -0
  53. package/package.json +1 -1
@@ -12,6 +12,8 @@ const bundle_1 = require("./bundle");
12
12
  const annotations_1 = require("./annotations");
13
13
  const server_client_1 = require("./server-client");
14
14
  const bundle_text_1 = require("./bundle-text");
15
+ const sandbox_client_1 = require("../harness/sandbox-client");
16
+ const path_1 = require("path");
15
17
  const RUN_ID = /^run_[A-Za-z0-9_-]{16}$/;
16
18
  // what the publisher said about where a version comes from, one line
17
19
  // What a publisher said about where a version comes from: the server takes it as said,
@@ -29,6 +31,11 @@ function yankLines(version) {
29
31
  ]
30
32
  : [];
31
33
  }
34
+ // A line a task printed, shown as it is but for control characters, so it can't rewrite
35
+ // the terminal; tabs stay.
36
+ function printableLine(text) {
37
+ return text.split('\t').map(server_client_1.printableWhole).join('\t');
38
+ }
32
39
  const collectOption = (value, previous = []) => [...previous, value];
33
40
  // What a person who approves is shown of a run: everything the decision rests on.
34
41
  function describeApproval(run) {
@@ -89,7 +96,7 @@ function describePlan(plan) {
89
96
  }
90
97
  exports.describePlan = describePlan;
91
98
  function registerWorkCommands(remote, context) {
92
- const { environment, buildFile, getServer, guarded } = context;
99
+ const { environment, buildFile, getServer, guarded, passthrough, fail } = context;
93
100
  const connect = async (name) => {
94
101
  const chosen = await getServer(name);
95
102
  // a CI job or an agent brings its own token, a person signs in
@@ -122,6 +129,87 @@ function registerWorkCommands(remote, context) {
122
129
  environment.stdout.write(`it can not run before somebody who may publish approves it, until ${(0, server_client_1.printable)(version.expiresAt)}\n`);
123
130
  }
124
131
  }));
132
+ remote
133
+ .command('run <tasks...>')
134
+ .description('run tasks of a published build file on a server from a git ref, print their output and exit as they did')
135
+ .addOption(new commander_1.Option('--server <name>', 'the server to run on; the default of remote use when left out'))
136
+ .addOption(new commander_1.Option('--build <hash>', 'the published version to run (sha256:...), or HAMMERKIT_BUILD; the hash of the build file here when left out'))
137
+ .addOption(new commander_1.Option('--repo <host/path>', 'the repository, such as github.com/corp/payments; origin of the checkout here when left out'))
138
+ .addOption(new commander_1.Option('--ref <ref>', 'the branch or tag to run; the branch checked out here when left out'))
139
+ .addOption(new commander_1.Option('--commit <sha>', 'the commit the ref has to be at, else the run is refused; the commit checked out here when --ref is left out'))
140
+ .addOption(new commander_1.Option('--env <name>', 'the environment of the build file, as --env of run'))
141
+ .addOption(new commander_1.Option('--cache <method>', 'as --cache of run').choices(['checksum', 'modify-date', 'none']))
142
+ .addOption(new commander_1.Option('--timeout <duration>', 'stop the run after this long, such as 30m'))
143
+ .action(guarded('Remote run was not successful', async (tasks, options) => {
144
+ const named = options.build ?? environment.processEnvs.HAMMERKIT_BUILD;
145
+ const definition = named ?? (0, bundle_1.hashBundle)(await (0, collect_bundle_1.collectBundle)(buildFile, environment));
146
+ if (!bundle_1.HASH_PATTERN.test(definition)) {
147
+ throw new Error(`${(0, server_client_1.printable)(definition)} is not the hash of a published version (sha256:...)`);
148
+ }
149
+ const checkout = options.repo && options.ref ? {} : await (0, annotations_1.readCheckout)((0, path_1.dirname)(buildFile));
150
+ const repo = options.repo ?? checkout.repo;
151
+ const ref = options.ref ?? checkout.branch;
152
+ // the commit checked out here is the code meant, when the ref is the branch it is on
153
+ const commit = options.commit ?? (options.ref ? undefined : checkout.commit);
154
+ if (!repo) {
155
+ throw new Error('name the repository with --repo, the checkout here has no origin it can be told by');
156
+ }
157
+ if (!ref) {
158
+ throw new Error('name the branch or tag with --ref, the checkout here is on no branch');
159
+ }
160
+ const { name: serverName, server } = await getServer(options.server);
161
+ const connection = { url: server.url, ...(0, token_source_1.getTokenSource)(environment, serverName, server) };
162
+ const runOptions = {};
163
+ for (const key of ['env', 'cache', 'timeout']) {
164
+ if (options[key] !== undefined) {
165
+ runOptions[key] = options[key];
166
+ }
167
+ }
168
+ if (passthrough.length > 0) {
169
+ runOptions.args = passthrough;
170
+ }
171
+ let submitted;
172
+ try {
173
+ submitted = await (0, sandbox_client_1.submitRun)(connection, {
174
+ definition,
175
+ tasks,
176
+ source: { type: 'ref', repo, ref, ...(commit ? { commit } : {}) },
177
+ options: runOptions,
178
+ });
179
+ }
180
+ catch (e) {
181
+ if (e.status === 400 && !named && /no published build file/.test(String(e))) {
182
+ throw new Error(`the build file here is not published on ${serverName} (${definition}), publish it first`);
183
+ }
184
+ throw e;
185
+ }
186
+ const at = submitted.source.type === 'ref' ? ` at ${(0, server_client_1.printable)(submitted.source.commit)}` : '';
187
+ environment.stdout.write(`${(0, server_client_1.printable)(submitted.id)} on ${serverName}: ${tasks.map(server_client_1.printable).join(', ')} from ${(0, server_client_1.printable)(ref)}${at}\n`);
188
+ const prefixed = tasks.length > 1;
189
+ const followed = await (0, sandbox_client_1.followRun)(connection, submitted, {
190
+ signal: environment.abortCtrl.signal,
191
+ onLine: (line) => {
192
+ environment.stdout.write(`${prefixed || line.task !== tasks[0] ? `[${(0, server_client_1.printable)(line.task)}] ` : ''}${printableLine(line.text)}\n`);
193
+ },
194
+ onWaiting: (run) => {
195
+ for (const approval of run.approvals ?? []) {
196
+ environment.stdout.write(`waiting until somebody who uses ${(0, server_client_1.printable)(approval.account)} approves it, before ${(0, server_client_1.printable)(approval.expiresAt)}: hammerkit remote approve ${(0, server_client_1.printable)(run.id)}\n`);
197
+ }
198
+ },
199
+ });
200
+ const run = followed.run;
201
+ if (run.state === 'succeeded') {
202
+ environment.stdout.write(`${(0, server_client_1.printable)(run.id)} succeeded\n`);
203
+ return;
204
+ }
205
+ // the exit code of the first task that failed, as `hammerkit run` would end
206
+ const failed = tasks
207
+ .map((task) => run.results?.[task])
208
+ .concat(Object.values(run.results ?? {}))
209
+ .find((result) => result?.status === 'failed');
210
+ const exitCode = failed?.exitCode && failed.exitCode > 0 ? failed.exitCode : 1;
211
+ fail(`${(0, server_client_1.printable)(run.id)} ${run.state}${run.error ? `: ${(0, server_client_1.printable)(run.error)}` : ''}`, exitCode);
212
+ }));
125
213
  remote
126
214
  .command('versions [server]')
127
215
  .description('list the published versions of build files on a server, newest first, with where they come from')
@@ -40,8 +40,12 @@ exports.buildFileBaseTaskSchema = (0, zod_1.object)({
40
40
  account: (0, zod_1.string)().optional(),
41
41
  // CPU and memory limits of the container; a local task is not limited
42
42
  resources: resources_schema_1.resourcesSchema.optional(),
43
- // what the container reaches besides its needs; a local task can't be limited
43
+ // what the container reaches besides its needs; a local task can be limited only where
44
+ // it runs in a container, on a server's cluster
44
45
  network: network_schema_1.networkSchema.optional(),
46
+ // whether its last command takes the arguments after `--`: false refuses them, true
47
+ // says it is meant to, which a server with `runs.args: declared` requires
48
+ args: (0, zod_1.boolean)().optional(),
45
49
  // Name of the server (see `hammerkit remote`) the task has to run on; defaults
46
50
  // to the build file's `runner`. Never a local fallback when it is unknown.
47
51
  runner: runner_schema_1.runnerSchema.optional(),
@@ -45,6 +45,8 @@ exports.buildFileContainerServiceSchema = (0, zod_1.object)({
45
45
  volumeSize: size_schema_1.sizeSchema.describe('size of its claim on Kubernetes, e.g. 10Gi').optional(),
46
46
  // what the service reaches besides its needs; tasks needing it always reach it
47
47
  network: network_schema_1.networkSchema.optional(),
48
+ // whether its `cmd` takes the arguments after `--` of `up`: false refuses them
49
+ args: (0, zod_1.boolean)().optional(),
48
50
  })
49
51
  .strict()
50
52
  .describe('container service');
@@ -129,6 +129,7 @@ function applySchemaExtension(reference, task) {
129
129
  task.schema.timeout = task.schema.timeout ?? extend.schema.timeout;
130
130
  task.schema.runner = task.schema.runner ?? extend.schema.runner;
131
131
  task.schema.network = task.schema.network ?? extend.schema.network;
132
+ task.schema.args = task.schema.args ?? extend.schema.args;
132
133
  if (!reset.has('cmds'))
133
134
  task.schema.cmds = extendArray(task.schema.cmds, extend.schema.cmds);
134
135
  if (!reset.has('src'))
@@ -6,6 +6,8 @@ const github_app_token_1 = require("./github-app-token");
6
6
  // `hammerkit github-app-token <owner/repo>`: a secret provider command for the token a
7
7
  // server reads and pushes a repository with, so the released binary mints it without a
8
8
  // script or a runtime of its own. It prints the token and nothing else, without a newline.
9
+ // A failure goes to stderr, which the server puts in its error: the CLI's console writes to
10
+ // stdout, and the server drops what a failed provider printed there.
9
11
  function registerGithubAppTokenCommand(program, environment) {
10
12
  program
11
13
  .command('github-app-token <repo>')
@@ -15,7 +17,8 @@ function registerGithubAppTokenCommand(program, environment) {
15
17
  environment.stdout.write(await (0, github_app_token_1.githubAppToken)(repo, environment.processEnvs));
16
18
  }
17
19
  catch (e) {
18
- program.error(`github-app-token: ${(0, log_1.getErrorMessage)(e)}`, { exitCode: 1 });
20
+ environment.stderr.write(`github-app-token: ${(0, log_1.getErrorMessage)(e)}\n`);
21
+ throw e;
19
22
  }
20
23
  });
21
24
  }
@@ -100,7 +100,7 @@ class HostExecutor {
100
100
  if (extra.length > 0) {
101
101
  throw new Error(`the run would use ${extra.join(', ')}, which it was not admitted for`);
102
102
  }
103
- const refused = (0, run_plan_1.refuseArguments)(run, planned.plan);
103
+ const refused = (0, run_plan_1.refuseArguments)(run, planned.plan, policy);
104
104
  if (refused) {
105
105
  throw new Error(refused);
106
106
  }
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.awaitRunObjectsGone = exports.removeRunObjects = exports.CallerNamespaces = exports.getVolumeScope = exports.KubernetesExecutor = exports.getRunRoleBinding = exports.getNamespacePolicies = exports.getClusterAccess = exports.getCallerNamespace = void 0;
3
+ exports.awaitRunObjectsGone = exports.removeRunObjects = exports.CallerNamespaces = exports.getVolumeScope = exports.KubernetesExecutor = exports.getRunRoleBinding = exports.getNamespacePolicies = exports.getClusterAccess = exports.isCallerNamespace = exports.getCallerNamespace = void 0;
4
4
  const crypto_1 = require("crypto");
5
5
  const fs_1 = require("fs");
6
6
  const os_1 = require("os");
@@ -32,6 +32,12 @@ function getCallerNamespace(prefix, owner) {
32
32
  return `${prefix}${(0, crypto_1.createHash)('sha256').update(`${owner.iss}\0${owner.sub}`).digest('hex').slice(0, 16)}`;
33
33
  }
34
34
  exports.getCallerNamespace = getCallerNamespace;
35
+ // Whether a namespace is one getCallerNamespace makes with this prefix: a server named `hk`
36
+ // passes over the namespaces of a server named `hk-test` on the same cluster.
37
+ function isCallerNamespace(prefix, name) {
38
+ return name.startsWith(prefix) && /^[0-9a-f]{16}$/.test(name.slice(prefix.length));
39
+ }
40
+ exports.isCallerNamespace = isCallerNamespace;
35
41
  // How the server and the processes of its runs reach the cluster: a kubeconfig and
36
42
  // context the operator names, else the server's default kubeconfig, else the cluster
37
43
  // the server runs in (its pod's ServiceAccount, found through the variables
@@ -161,18 +167,21 @@ exports.getRunRoleBinding = getRunRoleBinding;
161
167
  * process of the server as on a host (see HostExecutor); what changes is where its
162
168
  * container tasks and services go: into the caller's namespace, on the nodes of the
163
169
  * operator's pools, with the secrets of providers read in the Job itself. The runs
164
- * of one caller execute one after the other, since the objects of a task are named
165
- * by its cache id in the caller's namespace; different callers run side by side.
166
- * When a run ends, everything made for it is removed, but for the claims of its tasks
167
- * the server keeps (`backend.keepVolumes`), which go once they expired.
170
+ * of a caller execute side by side in their namespace, up to `runs.limits.running`:
171
+ * the objects of a run are named and found by its run label (getItemKey), so two runs
172
+ * of the same task have their own. What two runs share are the claims the server keeps
173
+ * (`backend.keepVolumes`) of a scope (getVolumeScope): the runs of one scope go one at
174
+ * a time (exclusive). When a run ends, everything made for it is removed, but for the
175
+ * kept claims, which go once they expired.
168
176
  */
169
177
  class KubernetesExecutor {
170
178
  constructor(config, processEnvs, options) {
171
179
  this.config = config;
172
- // the registry keeps a caller's next run in its queue, not running and timed here
173
- this.runningPerCaller = 1;
174
- // the run that executes, or the last one waiting, per caller namespace
180
+ // per caller namespace, what changes objects of the whole namespace: its setup, the
181
+ // removal of what a stopped server left, the sweeps of kept claims. The last one waiting.
175
182
  this.queues = new Map();
183
+ // per caller namespace, the runs that execute in it: by run label, their volume scope
184
+ this.active = new Map();
176
185
  // per caller namespace, when its kept claims are next looked at
177
186
  this.sweeps = new Map();
178
187
  // whether the kept claims a server left when it stopped were looked for
@@ -197,17 +206,45 @@ class KubernetesExecutor {
197
206
  this.log = options.log;
198
207
  this.host = new executor_1.HostExecutor({ ...options, placement: this.placement() });
199
208
  }
200
- execute(run, context) {
209
+ async execute(run, context) {
201
210
  const namespace = getCallerNamespace(this.config.backend.namespacePrefix, run.owner);
202
- return this.enqueue(namespace, () => {
203
- if (context.signal.aborted) {
204
- throw new Error('the run was cancelled');
211
+ const label = (0, job_placement_1.getRunLabelValue)(run.id);
212
+ let runs = this.active.get(namespace);
213
+ if (!runs) {
214
+ runs = new Map();
215
+ this.active.set(namespace, runs);
216
+ }
217
+ runs.set(label, getVolumeScope(run));
218
+ try {
219
+ await this.host.execute(run, context);
220
+ }
221
+ finally {
222
+ runs.delete(label);
223
+ if (runs.size === 0 && this.active.get(namespace) === runs) {
224
+ this.active.delete(namespace);
205
225
  }
206
- return this.host.execute(run, context);
207
- });
226
+ }
227
+ }
228
+ // The runs of a scope share its kept claims, which one of them may change at a time:
229
+ // what a task finds in one is what the last run left there. Without kept claims, runs
230
+ // share no object and nothing is held.
231
+ exclusive(run) {
232
+ if (this.keep === 0) {
233
+ return undefined;
234
+ }
235
+ return `${getCallerNamespace(this.config.backend.namespacePrefix, run.owner)}/${getVolumeScope(run)}`;
236
+ }
237
+ // The scopes of the runs that execute in the namespace, but `except`'s: their kept claims
238
+ // are theirs to mount and stay. A server that keeps none mounts none.
239
+ scopesInUse(namespace, except) {
240
+ if (this.keep === 0) {
241
+ return new Set();
242
+ }
243
+ return new Set([...(this.active.get(namespace) ?? new Map()).entries()]
244
+ .filter(([label]) => label !== except)
245
+ .map(([, scope]) => scope));
208
246
  }
209
- // What happens in a caller's namespace happens one at a time: their runs, and the
210
- // removal of the claims their runs kept
247
+ // What changes objects of a whole namespace happens one at a time, not what its runs do
211
248
  enqueue(namespace, fn) {
212
249
  const previous = this.queues.get(namespace) ?? Promise.resolve();
213
250
  const execution = previous.then(fn);
@@ -225,19 +262,28 @@ class KubernetesExecutor {
225
262
  return {
226
263
  prepare: async (run, context, work) => {
227
264
  const namespace = getCallerNamespace(backend.namespacePrefix, run.owner);
228
- await this.namespaces.prepare(namespace);
229
- // what an earlier run left when the server stopped before its end; the runs
230
- // of a caller execute one at a time, so no other run is in the namespace
231
- await removeRunObjects(this.instance, namespace, `${job_placement_1.RUN_LABEL},${job_placement_1.RUN_LABEL}!=${(0, job_placement_1.getRunLabelValue)(run.id)}`, RELEASE_TIMEOUT);
232
- // the claims earlier runs kept that expired; the others this run may mount
233
- const { next } = await (0, kept_claims_1.sweepKeptClaims)(this.instance, namespace, {
234
- now: new Date(),
235
- keep: this.keep > 0,
236
- timeout: RELEASE_TIMEOUT,
265
+ const label = (0, job_placement_1.getRunLabelValue)(run.id);
266
+ await this.enqueue(namespace, async () => {
267
+ if (context.signal.aborted) {
268
+ throw new Error('the run was cancelled');
269
+ }
270
+ await this.namespaces.prepare(namespace);
271
+ // what an earlier run left when the server stopped before its end: the objects of
272
+ // a run that does not execute. One that starts after this waits for it here
273
+ // before it makes any.
274
+ const running = [...new Set([label, ...(this.active.get(namespace)?.keys() ?? [])])];
275
+ await removeRunObjects(this.instance, namespace, `${job_placement_1.RUN_LABEL},${job_placement_1.RUN_LABEL} notin (${running.join(',')})`, RELEASE_TIMEOUT);
276
+ // the claims earlier runs kept that expired; the others this run may mount
277
+ const { next } = await (0, kept_claims_1.sweepKeptClaims)(this.instance, namespace, {
278
+ now: new Date(),
279
+ keep: this.keep > 0,
280
+ timeout: RELEASE_TIMEOUT,
281
+ inUse: this.scopesInUse(namespace, label),
282
+ });
283
+ if (next) {
284
+ this.sweepLater(namespace, Math.max(0, next.getTime() - Date.now()));
285
+ }
237
286
  });
238
- if (next) {
239
- this.sweepLater(namespace, Math.max(0, next.getTime() - Date.now()));
240
- }
241
287
  this.sweepLeftClaims();
242
288
  context.emit({ type: 'log', task: 'hammerkit', stream: 'stdout', text: `runs in the namespace ${namespace}` });
243
289
  const kubernetes = {
@@ -294,8 +340,8 @@ class KubernetesExecutor {
294
340
  },
295
341
  };
296
342
  }
297
- // The kept claims of a namespace that expired go once no run of the caller is in it: at
298
- // the soonest expiry the server knows of
343
+ // The kept claims of a namespace that expired go at the soonest expiry the server knows
344
+ // of, but those of a scope a run of the caller executes in
299
345
  sweepLater(namespace, delay) {
300
346
  const at = Date.now() + delay + 1000;
301
347
  const pending = this.sweeps.get(namespace);
@@ -315,6 +361,7 @@ class KubernetesExecutor {
315
361
  now: new Date(),
316
362
  keep: this.keep > 0,
317
363
  timeout: RELEASE_TIMEOUT,
364
+ inUse: this.scopesInUse(namespace),
318
365
  });
319
366
  if (removed.length > 0) {
320
367
  this.log?.(`removed ${removed.length} kept claims in ${namespace}`);
@@ -335,7 +382,9 @@ class KubernetesExecutor {
335
382
  .then((namespaces) => {
336
383
  for (const namespace of namespaces.items) {
337
384
  const name = namespace.metadata?.name;
338
- if (name?.startsWith(this.config.backend.namespacePrefix) && !namespace.metadata?.deletionTimestamp) {
385
+ if (name &&
386
+ isCallerNamespace(this.config.backend.namespacePrefix, name) &&
387
+ !namespace.metadata?.deletionTimestamp) {
339
388
  this.sweepLater(name, 0);
340
389
  }
341
390
  }
@@ -7,6 +7,7 @@ const path_1 = require("path");
7
7
  const stream_1 = require("stream");
8
8
  const promises_2 = require("stream/promises");
9
9
  const tar_1 = require("tar");
10
+ const tar_2 = require("../utils/tar");
10
11
  const apply_1 = require("../kubernetes/apply");
11
12
  const kubernetes_instance_1 = require("../kubernetes/kubernetes-instance");
12
13
  const network_policy_1 = require("../kubernetes/network-policy");
@@ -463,7 +464,7 @@ class KubernetesSandboxStorage {
463
464
  await (0, git_checkout_1.git)([`--git-dir=${repository}`, 'fetch', '--quiet', '--depth', '1', '--', fetch.url, fetch.ref], fetch.env, fetch.signal, what);
464
465
  await (0, git_checkout_1.git)([`--git-dir=${repository}`, 'update-ref', `refs/heads/${sandbox_pod_1.INCOMING_REF}`, 'FETCH_HEAD'], fetch.env, fetch.signal, what, op);
465
466
  const incoming = `${CLAIM_PATH}/${sandbox_pod_1.INCOMING}`;
466
- await this.withPod(key, (pod) => this.exec(pod, ['sh', '-c', 'rm -rf "$1" && mkdir -p "$1" && tar xf - -C "$1"', 'sh', incoming], stream_1.Readable.from((0, tar_1.create)({ cwd: repository, portable: true }, ['.'])), null, fetch.signal));
467
+ await this.withPod(key, (pod) => this.exec(pod, ['sh', '-c', 'rm -rf "$1" && mkdir -p "$1" && tar xf - -C "$1"', 'sh', incoming], stream_1.Readable.from((0, tar_2.packArchive)({ cwd: repository, portable: true }, ['.'])), null, fetch.signal));
467
468
  return (await this.request(key, { op, id: key.id }, fetch.signal)).result;
468
469
  }
469
470
  finally {
@@ -106,7 +106,7 @@ class TokenVerifier {
106
106
  ? [groups]
107
107
  : [],
108
108
  claims,
109
- ...(typeof payload.email_verified === 'boolean' ? { emailVerified: payload.email_verified } : {}),
109
+ ...emailVerified(payload.email_verified),
110
110
  };
111
111
  }
112
112
  async findKey(issuer, algorithm, kid) {
@@ -224,4 +224,15 @@ function isSecureUrl(value) {
224
224
  return false;
225
225
  }
226
226
  }
227
+ // Some issuers send `email_verified` as the string "true" or "false": either says it as
228
+ // much as the boolean does, anything else says nothing
229
+ function emailVerified(value) {
230
+ if (value === true || value === 'true') {
231
+ return { emailVerified: true };
232
+ }
233
+ if (value === false || value === 'false') {
234
+ return { emailVerified: false };
235
+ }
236
+ return {};
237
+ }
227
238
  //# sourceMappingURL=oauth.js.map
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.inspectBundle = exports.planBundle = exports.DefinitionError = void 0;
3
+ exports.inspectBundle = exports.planBundle = exports.getCredentialReads = exports.PLAIN_SERVER_ENVS = exports.DefinitionError = void 0;
4
4
  const promises_1 = require("fs/promises");
5
5
  const os_1 = require("os");
6
6
  const path_1 = require("path");
@@ -20,6 +20,20 @@ const work_network_1 = require("../planner/work-network");
20
20
  class DefinitionError extends Error {
21
21
  }
22
22
  exports.DefinitionError = DefinitionError;
23
+ // Variables of a server that hold no credential (FR-008): a task may read these and
24
+ // still take arguments, or reach any network.
25
+ exports.PLAIN_SERVER_ENVS = ['PATH', 'HOME', 'CI'];
26
+ // What an item holds that its command must not hand out: its secrets, and the variables
27
+ // of the server its `envs` read, but the plain ones.
28
+ function getCredentialReads(item) {
29
+ return [
30
+ ...item.secrets.map((secret) => `the secret ${secret.target}`),
31
+ ...(item.serverEnvs ?? [])
32
+ .filter((env) => !exports.PLAIN_SERVER_ENVS.includes(env.toUpperCase()))
33
+ .map((env) => `$${env} of the server`),
34
+ ];
35
+ }
36
+ exports.getCredentialReads = getCredentialReads;
23
37
  // 40 (sha-1) or 64 (sha-256) hex digits
24
38
  const COMMIT_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/i;
25
39
  // A hash alone can only name what is inside the bundle, so everything a build
@@ -170,13 +184,32 @@ function findPolicyProblems(items, policy, root) {
170
184
  problems.push(`${item.name}: generates ${generate.path}, which is outside of the bundle`);
171
185
  }
172
186
  }
173
- const ceiling = policy.network;
174
- if (ceiling && (0, work_network_1.isLimitedNetwork)(ceiling)) {
175
- if (data.type === 'local-task') {
176
- problems.push(`${item.name}: is a local task, which can't be limited to the network this server allows (runs.network: ${(0, work_network_1.formatNetwork)(ceiling)})`);
187
+ // what holds a credential gets at most `runs.secretNetwork`, so a tool it runs can't
188
+ // send the credential anywhere else
189
+ const reads = getCredentialReads({
190
+ secrets: secretsOf(item).map(describeSecret),
191
+ serverEnvs: 'envs' in data ? data.envs.processNames ?? [] : [],
192
+ });
193
+ const ceilings = [];
194
+ if (policy.network && (0, work_network_1.isLimitedNetwork)(policy.network)) {
195
+ ceilings.push({ ceiling: policy.network, setting: 'runs.network', reason: '' });
196
+ }
197
+ if (reads.length > 0 && policy.secretNetwork && (0, work_network_1.isLimitedNetwork)(policy.secretNetwork)) {
198
+ ceilings.push({
199
+ ceiling: policy.secretNetwork,
200
+ setting: 'runs.secretNetwork',
201
+ reason: ` to what reads ${reads.join(', ')}`,
202
+ });
203
+ }
204
+ for (const { ceiling, setting, reason } of ceilings) {
205
+ // on a cluster a local task is a pod of a pool's tool image, limited like any other
206
+ if (data.type === 'local-task' && policy.backend !== 'kubernetes') {
207
+ problems.push(`${item.name}: is a local task, which can't be limited to the network this server allows${reason} ` +
208
+ `(${setting}: ${(0, work_network_1.formatNetwork)(ceiling)})`);
177
209
  }
178
210
  else if ('network' in data && (0, work_network_1.exceedsNetwork)(data.network, ceiling)) {
179
- problems.push(`${item.name}: asks for network ${(0, work_network_1.formatNetwork)(data.network)}, more than this server allows (runs.network: ${(0, work_network_1.formatNetwork)(ceiling)})`);
211
+ problems.push(`${item.name}: asks for network ${(0, work_network_1.formatNetwork)(data.network)}, more than this server allows${reason} ` +
212
+ `(${setting}: ${(0, work_network_1.formatNetwork)(ceiling)})`);
180
213
  }
181
214
  }
182
215
  if ((0, work_task_1.isContainerWorkTask)(data) || (0, work_service_1.isContainerWorkService)(data)) {
@@ -205,11 +238,13 @@ function findPolicyProblems(items, policy, root) {
205
238
  problems.push(`${item.name}: the secret ${describeSecret(secret).target} has cache: true, its digest would need ` +
206
239
  'the value before the Job of the task reads it, which this server does not support yet');
207
240
  }
208
- else if (item.data.network.type !== 'open') {
209
- // the init containers that read it share the pod's NetworkPolicy
241
+ else if ((0, work_network_1.isLimitedNetwork)(item.data.network) && !policy.proxiedProviders?.includes(source.provider)) {
242
+ // the init containers that read it share the pod's NetworkPolicy, and reach the
243
+ // provider only through the egress proxy
210
244
  problems.push(`${item.name}: the secret ${describeSecret(secret).target} is read from ${source.provider} in the ` +
211
- `Job of the task, which network ${(0, work_network_1.formatNetwork)(item.data.network)} would cut off from it, ` +
212
- 'a task that reads secrets of a provider keeps an open network on this server');
245
+ `Job of the task, which network ${(0, work_network_1.formatNetwork)(item.data.network)} would cut off from it: ` +
246
+ `this server names no endpoints of ${source.provider} in backend.providers.${source.provider}.allow ` +
247
+ 'for the egress proxy to let through, so the task keeps an open network');
213
248
  }
214
249
  }
215
250
  if (source.type === 'provider' && policy.reserved?.includes(source.account)) {
@@ -254,6 +289,7 @@ function toPlan(root, item) {
254
289
  cmds: 'cmds' in data ? data.cmds.map((cmd) => cmd.cmd) : 'cmd' in data && data.cmd ? [data.cmd.cmd] : [],
255
290
  command: (0, task_arguments_1.getArgumentsTarget)(data)?.cmd ?? null,
256
291
  network: (0, work_network_1.toNetworkSchema)('network' in data ? data.network : { type: 'open' }),
292
+ ...('args' in data && data.args !== null ? { args: data.args } : {}),
257
293
  deps: item.deps.map((dep) => dep.name).sort(),
258
294
  needs: item.needs.map((need) => need.name).sort(),
259
295
  accounts,
@@ -46,26 +46,16 @@ async function planRun(bundle, run, environment, policy) {
46
46
  };
47
47
  }
48
48
  exports.planRun = planRun;
49
- // Variables of a server that hold no credential (FR-008): a task may read these and
50
- // still take arguments.
51
- const PLAIN_SERVER_ENVS = ['PATH', 'HOME', 'CI'];
52
- function readsOf(entry) {
53
- return [
54
- ...entry.secrets.map((secret) => `the secret ${secret.target}`),
55
- ...(entry.serverEnvs ?? [])
56
- .filter((env) => !PLAIN_SERVER_ENVS.includes(env.toUpperCase()))
57
- .map((env) => `$${env} of the server`),
58
- ];
59
- }
60
49
  /**
61
50
  * Why a run may not pass its arguments, or null when it may. They go to a task, never a
62
51
  * service, and never to a run in which an item reads a secret or takes a variable of the
63
52
  * server into its `envs`: an argument can make the command print what the task holds,
64
53
  * and what its dependencies write into their outputs or its services serve, past the
65
54
  * masking of secrets. So a shell on a server runs with no account and no credential of
66
- * the server.
55
+ * the server. Under `runs.args: declared` they go only to a task that says `args: true`,
56
+ * so a task nobody meant to take them never does.
67
57
  */
68
- function refuseArguments(run, plan) {
58
+ function refuseArguments(run, plan, policy = {}) {
69
59
  if (!run.options.args || run.options.args.length === 0) {
70
60
  return null;
71
61
  }
@@ -75,14 +65,21 @@ function refuseArguments(run, plan) {
75
65
  if (!entry || entry.kind !== 'task') {
76
66
  return `${name} is not a task of this version`;
77
67
  }
78
- const reads = readsOf(entry);
68
+ // planning the run refuses it before, this keeps the check with the others
69
+ if (entry.args === false) {
70
+ return `${name} takes no arguments (args: false)`;
71
+ }
72
+ if (policy.args === 'declared' && entry.args !== true) {
73
+ return `${name} takes no arguments on this server: it takes them only for tasks that say args: true (runs.args: declared)`;
74
+ }
75
+ const reads = (0, plan_1.getCredentialReads)(entry);
79
76
  if (reads.length > 0) {
80
77
  return `${name} reads ${reads.join(', ')}, it takes no arguments on a server: an argument could make it print them`;
81
78
  }
82
79
  // what a dependency or a service of the run holds reaches the task through its outputs or
83
80
  // its answers, where an argument can print it in a form no mask knows (`rev token.txt`)
84
81
  for (const [other, item] of Object.entries(plan).sort(([a], [b]) => a.localeCompare(b))) {
85
- const held = (other === name ? [] : readsOf(item)).join(', ');
82
+ const held = (other === name ? [] : (0, plan_1.getCredentialReads)(item)).join(', ');
86
83
  if (held.length > 0) {
87
84
  return `${name} takes no arguments on a server: ${other} in its run reads ${held}, and an argument could make ${name} print what ${other} hands on`;
88
85
  }
@@ -75,6 +75,12 @@ exports.runRequestSchema = zod_1.z
75
75
  return repo;
76
76
  }),
77
77
  ref: name,
78
+ // the commit the ref has to be at, such as the one a CI job checked out: a ref that
79
+ // moved since is refused, so the run executes the code that was asked for
80
+ commit: zod_1.z
81
+ .string()
82
+ .regex(/^([0-9a-f]{40}|[0-9a-f]{64})$/, 'expected a full commit hash, 40 or 64 hex digits')
83
+ .optional(),
78
84
  })
79
85
  .strict(),
80
86
  // the name of a tree announced with POST /v1/sources
@@ -424,13 +424,20 @@ class RunRegistry {
424
424
  return (source.type === 'sandbox' &&
425
425
  [...this.entries.values()].some((entry) => entry.run.state === 'running' && onSandbox(entry.run, source.id)));
426
426
  }
427
+ // Whether a run that executes holds what the run would (RunExecutor.exclusive)
428
+ held(run) {
429
+ const exclusive = this.options.executor.exclusive?.bind(this.options.executor);
430
+ const key = exclusive?.(run);
431
+ return (key !== undefined &&
432
+ [...this.entries.values()].some((entry) => entry.run.state === 'running' && exclusive?.(entry.run) === key));
433
+ }
427
434
  // The first run in the queue whose caller has a place left: one caller with many runs
428
435
  // does not keep the others from the machine. A run on a sandbox another run works on
429
- // waits for it.
436
+ // waits for it, as does one that needs what an executing run holds.
430
437
  dispatch() {
431
438
  while (this.running < this.maxConcurrent) {
432
- const perCaller = Math.min(this.options.limits?.running ?? Infinity, this.options.executor.runningPerCaller ?? Infinity);
433
- const index = this.queue.findIndex((entry) => this.runningOf(entry.run.owner) < perCaller && !this.sandboxBusy(entry.run));
439
+ const perCaller = this.options.limits?.running ?? Infinity;
440
+ const index = this.queue.findIndex((entry) => this.runningOf(entry.run.owner) < perCaller && !this.sandboxBusy(entry.run) && !this.held(entry.run));
434
441
  if (index < 0) {
435
442
  return;
436
443
  }
@@ -175,7 +175,8 @@ function parse(schema, body) {
175
175
  /**
176
176
  * Who the commit of a push is by: the caller, as their token names them (`name` and
177
177
  * `email`, unless the issuer says the address is not verified), else by their subject at the
178
- * host of their issuer. Never what the request says, so a commit can not claim to be
178
+ * host of their issuer. Without a name, the local part of the address names them, then the
179
+ * subject. Never what the request says, so a commit can not claim to be
179
180
  * somebody else's.
180
181
  */
181
182
  function author(principal) {
@@ -183,11 +184,16 @@ function author(principal) {
183
184
  .replace(/[<>\0\r\n]/g, '')
184
185
  .trim()
185
186
  .slice(0, 200);
186
- const name = clean(principal.claims.name) || clean(principal.claims.preferred_username) || clean(principal.sub);
187
187
  // an address the issuer says it did not verify is whatever the caller typed in
188
188
  const claimed = principal.emailVerified === false ? '' : clean(principal.claims.email);
189
- if (/^[^\s@]+@[^\s@]+$/.test(claimed)) {
190
- return { name, email: claimed };
189
+ const email = /^[^\s@]+@[^\s@]+$/.test(claimed) ? claimed : '';
190
+ // a Google service account's token has an address but no name: its local part reads better than the number in sub
191
+ const name = clean(principal.claims.name) ||
192
+ clean(principal.claims.preferred_username) ||
193
+ email.slice(0, email.indexOf('@')) ||
194
+ clean(principal.sub);
195
+ if (email) {
196
+ return { name, email };
191
197
  }
192
198
  let host = 'hammerkit';
193
199
  try {