hammerkit 1.27.0-next.3 → 1.27.0-next.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/build.schema.json +188 -0
  2. package/dist/cache/backends/local-cache-backend.js +6 -0
  3. package/dist/cache/backends/registry-cache-backend.js +2 -2
  4. package/dist/cache/explain.js +17 -0
  5. package/dist/cli.js +2 -0
  6. package/dist/docker/item-network.js +13 -5
  7. package/dist/executer/deadline.js +17 -3
  8. package/dist/executer/deferred-restore.js +36 -0
  9. package/dist/executer/docker-service.js +30 -8
  10. package/dist/executer/docker-task.js +21 -1
  11. package/dist/executer/event-cache.js +20 -2
  12. package/dist/executer/execute-work-service.js +2 -0
  13. package/dist/executer/execute-work-task.js +10 -3
  14. package/dist/executer/execution-steps.js +25 -4
  15. package/dist/executer/reset-work-tree.js +4 -0
  16. package/dist/executer/scheduler/enqueue-next.js +57 -6
  17. package/dist/executer/scheduler/state-key.js +42 -5
  18. package/dist/executer/watch-loop.js +4 -1
  19. package/dist/harness/connect.js +20 -6
  20. package/dist/harness/mcp-command.js +22 -6
  21. package/dist/harness/mcp.js +63 -114
  22. package/dist/harness/opencode.js +158 -50
  23. package/dist/harness/pi.js +29 -0
  24. package/dist/harness/sandbox-client.js +168 -17
  25. package/dist/harness/service-tools.js +141 -0
  26. package/dist/harness/task-tools.js +221 -0
  27. package/dist/kubernetes/apply.js +27 -16
  28. package/dist/kubernetes/await-running-state.js +59 -13
  29. package/dist/kubernetes/claim-layout.js +71 -0
  30. package/dist/kubernetes/cluster-state.js +245 -0
  31. package/dist/kubernetes/ensure-http-route.js +1 -1
  32. package/dist/kubernetes/ensure-kubernetes-deployment-exists.js +35 -12
  33. package/dist/kubernetes/ensure-kubernetes-persistent-volume-claim-exists.js +1 -1
  34. package/dist/kubernetes/ensure-kubernetes-service-exists.js +6 -9
  35. package/dist/kubernetes/ensure-persistent-data.js +30 -20
  36. package/dist/kubernetes/follow-pod-logs.js +182 -0
  37. package/dist/kubernetes/get-service-ip.js +2 -2
  38. package/dist/kubernetes/job-placement.js +36 -12
  39. package/dist/kubernetes/job-secrets.js +33 -19
  40. package/dist/kubernetes/kept-claims.js +22 -10
  41. package/dist/kubernetes/network-policy.js +85 -27
  42. package/dist/kubernetes/remove-persistent-data.js +3 -1
  43. package/dist/kubernetes/resources.js +25 -3
  44. package/dist/kubernetes/restore-kubernetes-data.js +9 -3
  45. package/dist/kubernetes/sandbox-runner.js +628 -0
  46. package/dist/kubernetes/secrets.js +7 -11
  47. package/dist/kubernetes/volumes.js +6 -3
  48. package/dist/main.js +2 -0
  49. package/dist/network/egress-proxy.js +45 -6
  50. package/dist/optimizer/calculate-checksum.js +3 -2
  51. package/dist/optimizer/check-key.js +94 -0
  52. package/dist/optimizer/check-keys.js +64 -0
  53. package/dist/optimizer/get-work-cache-stats.js +17 -3
  54. package/dist/optimizer/work-task-cache-description.js +51 -1
  55. package/dist/planner/utils/append-work-dependencies.js +10 -5
  56. package/dist/planner/utils/append-work-service.js +19 -1
  57. package/dist/planner/utils/append-work-task.js +18 -3
  58. package/dist/planner/utils/get-work-runtime.js +6 -1
  59. package/dist/planner/utils/normalize-path.js +19 -1
  60. package/dist/planner/utils/parse-work-mount.js +7 -1
  61. package/dist/planner/utils/parse-work-port.js +4 -1
  62. package/dist/planner/utils/task-arguments.js +3 -0
  63. package/dist/planner/validate.js +22 -0
  64. package/dist/planner/work-network.js +3 -1
  65. package/dist/planner/work-runtime-aggregate.js +32 -0
  66. package/dist/planner/work-runtime-docker.js +2 -1
  67. package/dist/planner/work-runtime-kubernetes.js +124 -48
  68. package/dist/planner/work-runtime-local.js +2 -1
  69. package/dist/planner/work-task.js +6 -1
  70. package/dist/program.js +31 -10
  71. package/dist/remote/device-login.js +7 -2
  72. package/dist/remote/host-config.js +69 -3
  73. package/dist/remote/remote-commands.js +30 -7
  74. package/dist/remote/remote-work-commands.js +426 -45
  75. package/dist/remote/server-client.js +15 -2
  76. package/dist/remote/token-source.js +64 -26
  77. package/dist/run-program.js +9 -0
  78. package/dist/schema/build-file-base-task-schema.js +8 -1
  79. package/dist/schema/build-file-container-service-schema.js +12 -0
  80. package/dist/schema/build-file-container-task-schema.js +16 -1
  81. package/dist/schema/build-file-include-schema.js +18 -4
  82. package/dist/schema/build-file-kubernetes-service-schema.js +2 -0
  83. package/dist/schema/build-file-task-schema.js +5 -1
  84. package/dist/schema/capabilities-schema.js +56 -0
  85. package/dist/schema/reference-parser.js +120 -11
  86. package/dist/schema/schema-parser.js +92 -41
  87. package/dist/schema/work-scope-parser.js +33 -10
  88. package/dist/secrets/command-secret-provider.js +3 -1
  89. package/dist/secrets/github-app-token-command.js +3 -1
  90. package/dist/secrets/secret-catalog.js +43 -19
  91. package/dist/server/accounts-api.js +3 -1
  92. package/dist/server/approvals-api.js +1 -1
  93. package/dist/server/checks-api.js +83 -0
  94. package/dist/server/checks.js +116 -0
  95. package/dist/server/definitions-api.js +66 -3
  96. package/dist/server/endpoints.js +739 -0
  97. package/dist/server/executor.js +57 -5
  98. package/dist/server/kubernetes-backend.js +367 -65
  99. package/dist/server/kubernetes-sandbox-storage.js +48 -3
  100. package/dist/server/oauth.js +12 -1
  101. package/dist/server/plan.js +188 -103
  102. package/dist/server/policy.js +32 -2
  103. package/dist/server/repo-access.js +5 -0
  104. package/dist/server/run-plan.js +50 -19
  105. package/dist/server/run-request.js +58 -3
  106. package/dist/server/run-worker.js +245 -12
  107. package/dist/server/runs.js +410 -27
  108. package/dist/server/sandbox-pod.js +28 -0
  109. package/dist/server/sandbox-storage.js +23 -0
  110. package/dist/server/sandboxes-api.js +10 -4
  111. package/dist/server/sandboxes.js +8 -0
  112. package/dist/server/server-command.js +39 -20
  113. package/dist/server/server-config.js +255 -14
  114. package/dist/server/server.js +269 -15
  115. package/dist/server/service-runs.js +82 -0
  116. package/dist/utils/abort-on-signals.js +46 -6
  117. package/dist/utils/tar.js +36 -0
  118. package/dist/utils/terminal-prompt.js +53 -0
  119. package/package.json +1 -1
@@ -7,9 +7,11 @@ const path_1 = require("path");
7
7
  const stream_1 = require("stream");
8
8
  const promises_2 = require("stream/promises");
9
9
  const tar_1 = require("tar");
10
+ const tar_2 = require("../utils/tar");
10
11
  const apply_1 = require("../kubernetes/apply");
11
12
  const kubernetes_instance_1 = require("../kubernetes/kubernetes-instance");
12
13
  const network_policy_1 = require("../kubernetes/network-policy");
14
+ const sandbox_runner_1 = require("../kubernetes/sandbox-runner");
13
15
  const volumes_1 = require("../kubernetes/volumes");
14
16
  const log_1 = require("../log");
15
17
  const sleep_1 = require("../utils/sleep");
@@ -259,6 +261,41 @@ class KubernetesSandboxStorage {
259
261
  await (0, promises_1.rm)(repository, { recursive: true, force: true });
260
262
  }
261
263
  }
264
+ // The pod writes the tree's files beside the sandbox on the claim and sends them as a tar,
265
+ // which is extracted here as the files of `attach` are: what a run plans with, up to the
266
+ // bytes a sandbox may take.
267
+ async exportTree(key, tree, into, signal) {
268
+ await (0, promises_1.mkdir)(into, { recursive: true });
269
+ let bytes = 0;
270
+ const extract = (0, tar_1.extract)({
271
+ cwd: into,
272
+ filter: (_path, entry) => 'type' in entry && ['File', 'OldFile', 'Directory', 'SymbolicLink'].includes(entry.type),
273
+ });
274
+ const counted = new stream_1.PassThrough();
275
+ counted.on('data', (chunk) => {
276
+ bytes += chunk.length;
277
+ if (bytes > this.limits.bytes) {
278
+ counted.destroy(new Error(`the files of the sandbox ${key.id} take more than ${(0, units_1.formatSize)(this.limits.bytes)}`));
279
+ }
280
+ });
281
+ const extracted = new Promise((resolve, reject) => {
282
+ extract.on('finish', resolve);
283
+ extract.on('error', reject);
284
+ counted.on('error', reject);
285
+ });
286
+ extracted.catch(() => undefined);
287
+ counted.pipe(extract);
288
+ const answer = await this.exclusive(key.id, () => this.call(key, { op: 'archive', id: key.id, tree }, signal, true));
289
+ if ('stream' in answer) {
290
+ // a pipeline, so the answer is ended with the copy: an error of the pod after the copy
291
+ // stopped (over the size limit) lands nowhere else
292
+ await (0, promises_2.pipeline)(answer.stream, counted);
293
+ }
294
+ else {
295
+ counted.end(answer.body);
296
+ }
297
+ await extracted;
298
+ }
262
299
  async moveTo(key, commit) {
263
300
  await this.request(key, { op: 'move', id: key.id, commit }, AbortSignal.timeout(FETCH_TIMEOUT_MS));
264
301
  }
@@ -283,6 +320,8 @@ class KubernetesSandboxStorage {
283
320
  // runs on a sandbox go one at a time: a proxy the pod of the files does not own was
284
321
  // left by a run before (the server stopped, or its release could not remove it)
285
322
  await (0, network_policy_1.removeSandboxProxies)(this.instance, namespace, { sandbox: names.label }, (deployment) => isOwnedBy(deployment, pod.uid)).catch(() => undefined);
323
+ // and a runner, which the pod did not own when it made way for a run
324
+ await (0, sandbox_runner_1.removeSandboxRunners)(this.instance, namespace, names.label, (runner) => isOwnedBy(runner, pod.uid)).catch(() => undefined);
286
325
  await (0, promises_1.mkdir)(into, { recursive: true });
287
326
  await this.copyOut(key, ['tar', 'cf', '-', '-C', `${CLAIM_PATH}/${key.id}/files`, '.'], into,
288
327
  // what the run plans with: files, directories and links, nothing outside of it
@@ -387,7 +426,8 @@ class KubernetesSandboxStorage {
387
426
  async removeIn(namespace, id) {
388
427
  const names = getSandboxNames(id);
389
428
  await this.stopPod({ id, owner: { iss: '', sub: '' } }, namespace);
390
- // the cluster removes the proxies the pod owned; those it did not own go here
429
+ // the cluster removes the proxies and the runner the pod owned; those it did not own go here
430
+ await (0, sandbox_runner_1.removeSandboxRunners)(this.instance, namespace, names.label);
391
431
  await (0, network_policy_1.removeSandboxProxies)(this.instance, namespace, { sandbox: names.label });
392
432
  for (const name of [names.claim, names.git]) {
393
433
  try {
@@ -463,7 +503,7 @@ class KubernetesSandboxStorage {
463
503
  await (0, git_checkout_1.git)([`--git-dir=${repository}`, 'fetch', '--quiet', '--depth', '1', '--', fetch.url, fetch.ref], fetch.env, fetch.signal, what);
464
504
  await (0, git_checkout_1.git)([`--git-dir=${repository}`, 'update-ref', `refs/heads/${sandbox_pod_1.INCOMING_REF}`, 'FETCH_HEAD'], fetch.env, fetch.signal, what, op);
465
505
  const incoming = `${CLAIM_PATH}/${sandbox_pod_1.INCOMING}`;
466
- await this.withPod(key, (pod) => this.exec(pod, ['sh', '-c', 'rm -rf "$1" && mkdir -p "$1" && tar xf - -C "$1"', 'sh', incoming], stream_1.Readable.from((0, tar_1.create)({ cwd: repository, portable: true }, ['.'])), null, fetch.signal));
506
+ await this.withPod(key, (pod) => this.exec(pod, ['sh', '-c', 'rm -rf "$1" && mkdir -p "$1" && tar xf - -C "$1"', 'sh', incoming], stream_1.Readable.from((0, tar_2.packArchive)({ cwd: repository, portable: true }, ['.'])), null, fetch.signal));
467
507
  return (await this.request(key, { op, id: key.id }, fetch.signal)).result;
468
508
  }
469
509
  finally {
@@ -501,7 +541,7 @@ class KubernetesSandboxStorage {
501
541
  const reader = new AnswerReader(body);
502
542
  // the process ends when the server stops waiting for it: closing the connection does
503
543
  // not end a command in a pod
504
- const seconds = Math.ceil((['create', 'fetch', 'move'].includes(request.op) ? FETCH_TIMEOUT_MS : REQUEST_TIMEOUT_MS) / 1000);
544
+ const seconds = Math.ceil((['create', 'fetch', 'move', 'archive'].includes(request.op) ? FETCH_TIMEOUT_MS : REQUEST_TIMEOUT_MS) / 1000);
505
545
  const done = this.withPod(key, (pod) => this.exec(pod, ['timeout', String(seconds), 'hammerkit', 'server-sandbox', CLAIM_PATH], stream_1.Readable.from([Buffer.from(JSON.stringify(encoded))]), reader, signal));
506
546
  done.then(() => reader.complete(), (e) => reader.fail(e));
507
547
  const answer = await reader.answer;
@@ -884,6 +924,11 @@ class AnswerReader extends stream_1.Writable {
884
924
  }
885
925
  _write(chunk, _encoding, callback) {
886
926
  if (this.stream) {
927
+ if (this.stream.destroyed) {
928
+ // its reader stopped (over a size limit): the rest of the answer is nobody's
929
+ callback();
930
+ return;
931
+ }
887
932
  this.stream.write(chunk, callback);
888
933
  return;
889
934
  }
@@ -106,7 +106,7 @@ class TokenVerifier {
106
106
  ? [groups]
107
107
  : [],
108
108
  claims,
109
- ...(typeof payload.email_verified === 'boolean' ? { emailVerified: payload.email_verified } : {}),
109
+ ...emailVerified(payload.email_verified),
110
110
  };
111
111
  }
112
112
  async findKey(issuer, algorithm, kid) {
@@ -224,4 +224,15 @@ function isSecureUrl(value) {
224
224
  return false;
225
225
  }
226
226
  }
227
+ // Some issuers send `email_verified` as the string "true" or "false": either says it as
228
+ // much as the boolean does, anything else says nothing
229
+ function emailVerified(value) {
230
+ if (value === true || value === 'true') {
231
+ return { emailVerified: true };
232
+ }
233
+ if (value === false || value === 'false') {
234
+ return { emailVerified: false };
235
+ }
236
+ return {};
237
+ }
227
238
  //# sourceMappingURL=oauth.js.map
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.inspectBundle = exports.planBundle = exports.DefinitionError = void 0;
3
+ exports.inspectBundle = exports.planBundle = exports.getCredentialReads = exports.PLAIN_SERVER_ENVS = exports.DefinitionError = void 0;
4
4
  const promises_1 = require("fs/promises");
5
5
  const os_1 = require("os");
6
6
  const path_1 = require("path");
@@ -13,6 +13,7 @@ const work_scope_parser_1 = require("../schema/work-scope-parser");
13
13
  const work_task_1 = require("../planner/work-task");
14
14
  const work_service_1 = require("../planner/work-service");
15
15
  const hosted_volumes_1 = require("../planner/utils/hosted-volumes");
16
+ const normalize_path_1 = require("../planner/utils/normalize-path");
16
17
  const log_1 = require("../log");
17
18
  const parse_error_1 = require("../schema/parse-error");
18
19
  const task_arguments_1 = require("../planner/utils/task-arguments");
@@ -20,55 +21,52 @@ const work_network_1 = require("../planner/work-network");
20
21
  class DefinitionError extends Error {
21
22
  }
22
23
  exports.DefinitionError = DefinitionError;
24
+ // Variables of a server that hold no credential (FR-008): a task may read these and
25
+ // still take arguments, or reach any network.
26
+ exports.PLAIN_SERVER_ENVS = ['PATH', 'HOME', 'CI'];
27
+ // What an item holds that its command must not hand out: its secrets, and the variables
28
+ // of the server its `envs` read, but the plain ones.
29
+ function getCredentialReads(item) {
30
+ return [
31
+ ...item.secrets.map((secret) => `the secret ${secret.target}`),
32
+ ...(item.serverEnvs ?? [])
33
+ .filter((env) => !exports.PLAIN_SERVER_ENVS.includes(env.toUpperCase()))
34
+ .map((env) => `$${env} of the server`),
35
+ ];
36
+ }
37
+ exports.getCredentialReads = getCredentialReads;
23
38
  // 40 (sha-1) or 64 (sha-256) hex digits
24
39
  const COMMIT_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/i;
25
40
  // A hash alone can only name what is inside the bundle, so everything a build
26
41
  // file pulls in from elsewhere has to be pinned: a git include needs a commit
27
42
  // and a remote that is not a path on the server.
28
- function findLooseIncludes(bundle, noRemote = false) {
43
+ function checkGitSource(path, where, source, noRemote) {
44
+ if (noRemote === 'proposal') {
45
+ throw new DefinitionError(`${path}: ${where}: a proposal may not include from git, the server would fetch it before anybody approved it`);
46
+ }
47
+ if (noRemote === 'check') {
48
+ throw new DefinitionError(`${path}: ${where}: checking a build file that includes from git needs builds: publish, the server would fetch it`);
49
+ }
29
50
  const problems = [];
30
- for (const [path, content] of Object.entries(bundle.files)) {
31
- if (!/\.ya?ml$/i.test(path)) {
32
- continue;
33
- }
34
- let document;
35
- try {
36
- document = (0, yaml_1.parse)(content);
37
- }
38
- catch {
39
- continue; // a file that isn't valid yaml is reported when the bundle is parsed
40
- }
41
- for (const block of ['includes', 'references']) {
42
- for (const [name, include] of Object.entries(document?.[block] ?? {})) {
43
- const source = include;
44
- if (!source || typeof source !== 'object' || source.git === undefined) {
45
- continue;
46
- }
47
- if (noRemote) {
48
- problems.push(`${path}: ${block}.${name}: a proposal may not include from git, the server would fetch it before anybody approved it`);
49
- continue;
50
- }
51
- if (typeof source.git !== 'string' || !/^(https:\/\/|ssh:\/\/|git@[\w.-]+:)/.test(source.git)) {
52
- problems.push(`${path}: ${block}.${name}: git has to be an https:// or ssh:// address`);
53
- }
54
- if (typeof source.ref !== 'string' || !COMMIT_PATTERN.test(source.ref)) {
55
- problems.push(`${path}: ${block}.${name}: pin ref to a commit, a branch or tag can move`);
56
- }
57
- }
58
- }
51
+ if (!/^(https:\/\/|ssh:\/\/|git@[\w.-]+:)/.test(source.git)) {
52
+ problems.push(`${path}: ${where}: git has to be an https:// or ssh:// address`);
53
+ }
54
+ if (typeof source.ref !== 'string' || !COMMIT_PATTERN.test(source.ref)) {
55
+ problems.push(`${path}: ${where}: pin ref to a commit, a branch or tag can move`);
56
+ }
57
+ if (problems.length > 0) {
58
+ throw new DefinitionError(problems.join('\n'));
59
59
  }
60
- return problems;
61
60
  }
62
61
  // What a build file may not name on a server: the places it would run or store things
63
62
  // are the operator's. The docker host, a Kubernetes cluster and a remote cache are
64
63
  // credentials of the machine, not of the build file, and a project file that could carry
65
64
  // them could send runs, or the cache, anywhere.
66
- function findHostProblems(bundle) {
65
+ // A file a server leaves out (`publish: false`) is not checked, and neither is a service
66
+ // with `publish: false`: what is left out is never run there.
67
+ function findHostProblems(files) {
67
68
  const problems = [];
68
- for (const [path, content] of Object.entries(bundle.files)) {
69
- if (!/\.ya?ml$/i.test(path)) {
70
- continue;
71
- }
69
+ for (const [path, content] of Object.entries(files)) {
72
70
  let document;
73
71
  try {
74
72
  document = (0, yaml_1.parse)(content);
@@ -90,6 +88,7 @@ function findHostProblems(bundle) {
90
88
  for (const [name, service] of Object.entries(document.services ?? {})) {
91
89
  if (service &&
92
90
  typeof service === 'object' &&
91
+ service.publish !== false &&
93
92
  ('selector' in service || 'kubeconfig' in service || 'context' in service)) {
94
93
  problems.push(`${path}: services.${name}: Kubernetes services are not supported on a server`);
95
94
  }
@@ -97,6 +96,17 @@ function findHostProblems(bundle) {
97
96
  }
98
97
  return problems;
99
98
  }
99
+ // The files of the bundle a server publishes, see getPublishedFiles
100
+ function publishedBundleFiles(bundle, directory, entry) {
101
+ const files = {};
102
+ for (const scope of (0, reference_parser_1.getPublishedFiles)(entry)) {
103
+ const path = (0, path_1.relative)(directory, scope.fileName).split(path_1.sep).join('/');
104
+ if (!scope.remote && path in bundle.files) {
105
+ files[path] = bundle.files[path];
106
+ }
107
+ }
108
+ return files;
109
+ }
100
110
  // What a build file may read while it is inspected: its own files. The rest of
101
111
  // the server is not its business (a missing file is "not there", not an error).
102
112
  function confined(files, root) {
@@ -151,80 +161,139 @@ function describeSecret(secret) {
151
161
  // server has no account, so nothing would stand between the build file and the
152
162
  // operator's own credentials.
153
163
  function findPolicyProblems(items, policy, root) {
154
- const problems = [];
164
+ // the problems of each item, said once for the items of a directory that share one, such
165
+ // as the tasks of an included file that all mount the docker socket
166
+ const grouped = new Map();
155
167
  for (const item of items) {
156
- const runtime = item.data.type === 'local-task' ? 'local' : 'docker';
157
- if (policy.runtimes && !policy.runtimes.includes(runtime)) {
158
- problems.push(`${item.name}: runs on the ${runtime} runtime, which this server does not allow (runtimes: ${policy.runtimes.join(', ')})`);
159
- }
160
- const data = item.data;
161
- // what a task reads and writes is the bundle's: `src: ../..` would have the server
162
- // copy its own files (its credentials, other callers' caches) into the task
163
- for (const source of 'src' in data ? data.src : []) {
164
- if (!source.inherited && insideBundle(root, data.cwd, source.absolutePath).length === 0) {
165
- problems.push(`${item.name}: src ${source.source} is outside of the bundle`);
168
+ const prefix = `${item.name}: `;
169
+ for (const problem of findItemPolicyProblems(item, policy, root)) {
170
+ const said = problem.startsWith(prefix) ? problem.slice(prefix.length) : problem;
171
+ const key = `${item.data.cwd}\0${said}`;
172
+ const group = grouped.get(key);
173
+ if (group) {
174
+ group.names.push(item.name);
166
175
  }
167
- }
168
- for (const generate of (0, work_task_1.isWorkTask)(data) ? data.generates : []) {
169
- if (!generate.inherited && insideBundle(root, data.cwd, generate.path).length === 0) {
170
- problems.push(`${item.name}: generates ${generate.path}, which is outside of the bundle`);
176
+ else {
177
+ grouped.set(key, { names: [item.name], problem: said });
171
178
  }
172
179
  }
173
- const ceiling = policy.network;
174
- if (ceiling && (0, work_network_1.isLimitedNetwork)(ceiling)) {
175
- if (data.type === 'local-task') {
176
- problems.push(`${item.name}: is a local task, which can't be limited to the network this server allows (runs.network: ${(0, work_network_1.formatNetwork)(ceiling)})`);
177
- }
178
- else if ('network' in data && (0, work_network_1.exceedsNetwork)(data.network, ceiling)) {
179
- problems.push(`${item.name}: asks for network ${(0, work_network_1.formatNetwork)(data.network)}, more than this server allows (runs.network: ${(0, work_network_1.formatNetwork)(ceiling)})`);
180
- }
180
+ }
181
+ return [...grouped.values()].map(({ names: [name, ...others], problem }) => others.length > 0 ? `${name} (also ${others.join(', ')}): ${problem}` : `${name}: ${problem}`);
182
+ }
183
+ function findItemPolicyProblems(item, policy, root) {
184
+ const problems = [];
185
+ const runtime = item.data.type === 'local-task' ? 'local' : 'docker';
186
+ // an aggregate runs nothing, on no runtime
187
+ if (policy.runtimes && !(0, work_task_1.isAggregateTask)(item.data) && !policy.runtimes.includes(runtime)) {
188
+ problems.push(`${item.name}: runs on the ${runtime} runtime, which this server does not allow (runtimes: ${policy.runtimes.join(', ')})`);
189
+ }
190
+ const data = item.data;
191
+ // a capability holds in the sandboxed runtime the operator named for it, so only those
192
+ // the operator allows are added
193
+ const refused = policy.capabilities
194
+ ? ('capAdd' in data ? data.capAdd : []).filter((capability) => !policy.capabilities?.includes(capability))
195
+ : [];
196
+ if (refused.length > 0) {
197
+ const from = policy.capabilitiesFrom ?? 'backend.capabilities.allow';
198
+ problems.push(`${item.name}: adds the ${refused.length > 1 ? 'capabilities' : 'capability'} ${refused.join(', ')}, ` +
199
+ (policy.capabilities && policy.capabilities.length > 0
200
+ ? `which this server does not allow (${from}: ${policy.capabilities.join(', ')})`
201
+ : `and this server allows none (${from})`));
202
+ }
203
+ // what a task reads and writes is the bundle's: `src: ../..` would have the server
204
+ // copy its own files (its credentials, other callers' caches) into the task
205
+ for (const source of 'src' in data ? data.src : []) {
206
+ if (!source.inherited && insideBundle(root, data.cwd, source.absolutePath).length === 0) {
207
+ problems.push(`${item.name}: src ${source.source} is outside of the bundle`);
181
208
  }
182
- if ((0, work_task_1.isContainerWorkTask)(data) || (0, work_service_1.isContainerWorkService)(data)) {
183
- for (const mount of data.mounts) {
184
- // a path of the server is not the build file's to mount: the docker socket, /etc, a home directory
185
- if (insideBundle(root, data.cwd, mount.localPath).length === 0) {
186
- problems.push(`${item.name}: mounts ${mount.localPath}, which is outside of the bundle`);
187
- }
188
- }
209
+ }
210
+ for (const generate of (0, work_task_1.isWorkTask)(data) ? data.generates : []) {
211
+ if (!generate.inherited && insideBundle(root, data.cwd, generate.path).length === 0) {
212
+ problems.push(`${item.name}: generates ${generate.path}, which is outside of the bundle`);
189
213
  }
190
- for (const volume of (0, work_service_1.isContainerWorkService)(data) ? data.volumes : []) {
191
- // docker takes anything else for a path of the host, such as the docker socket
192
- if (!(0, hosted_volumes_1.isVolumeName)(volume.name)) {
193
- problems.push(`${item.name}: volume ${volume.name} is not a volume name, a path of the server can't be mounted`);
194
- }
214
+ }
215
+ // what holds a credential gets at most `runs.secretNetwork`, so a tool it runs can't
216
+ // send the credential anywhere else
217
+ const reads = getCredentialReads({
218
+ secrets: secretsOf(item).map(describeSecret),
219
+ serverEnvs: 'envs' in data ? data.envs.processNames ?? [] : [],
220
+ });
221
+ const ceilings = [];
222
+ if (policy.network && (0, work_network_1.isLimitedNetwork)(policy.network)) {
223
+ ceilings.push({ ceiling: policy.network, setting: 'runs.network', reason: '' });
224
+ }
225
+ if (reads.length > 0 && policy.secretNetwork && (0, work_network_1.isLimitedNetwork)(policy.secretNetwork)) {
226
+ ceilings.push({
227
+ ceiling: policy.secretNetwork,
228
+ setting: 'runs.secretNetwork',
229
+ reason: ` to what reads ${reads.join(', ')}`,
230
+ });
231
+ }
232
+ // an aggregate reaches nothing: it runs no command
233
+ for (const { ceiling, setting, reason } of (0, work_task_1.isAggregateTask)(data) ? [] : ceilings) {
234
+ // on a cluster a local task is a pod of a pool's tool image, limited like any other
235
+ if (data.type === 'local-task' && policy.backend !== 'kubernetes') {
236
+ problems.push(`${item.name}: is a local task, which can't be limited to the network this server allows${reason} ` +
237
+ `(${setting}: ${(0, work_network_1.formatNetwork)(ceiling)})`);
195
238
  }
196
- for (const secret of secretsOf(item)) {
197
- const source = secret.source;
198
- if (policy.backend === 'kubernetes' && source.type === 'provider') {
199
- // the server reads no secret of a provider on a cluster: the Job of a task does
200
- if (!(0, work_task_1.isWorkTask)(item.data)) {
201
- problems.push(`${item.name}: the secret ${describeSecret(secret).target} is read from ${source.provider}, which ` +
202
- 'this server does for tasks only, a service on its cluster gets no provider secret yet');
203
- }
204
- else if (secret.cache) {
205
- problems.push(`${item.name}: the secret ${describeSecret(secret).target} has cache: true, its digest would need ` +
206
- 'the value before the Job of the task reads it, which this server does not support yet');
207
- }
208
- else if (item.data.network.type !== 'open') {
209
- // the init containers that read it share the pod's NetworkPolicy
210
- problems.push(`${item.name}: the secret ${describeSecret(secret).target} is read from ${source.provider} in the ` +
211
- `Job of the task, which network ${(0, work_network_1.formatNetwork)(item.data.network)} would cut off from it, ` +
212
- 'a task that reads secrets of a provider keeps an open network on this server');
213
- }
239
+ else if ('network' in data && (0, work_network_1.exceedsNetwork)(data.network, ceiling)) {
240
+ problems.push(`${item.name}: asks for network ${(0, work_network_1.formatNetwork)(data.network)}, more than this server allows${reason} ` +
241
+ `(${setting}: ${(0, work_network_1.formatNetwork)(ceiling)})`);
242
+ }
243
+ }
244
+ if ((0, work_task_1.isContainerWorkTask)(data) || (0, work_service_1.isContainerWorkService)(data)) {
245
+ for (const mount of data.mounts) {
246
+ // a path of the server is not the build file's to mount: the docker socket, /etc, a home directory
247
+ if (insideBundle(root, data.cwd, mount.localPath).length === 0) {
248
+ problems.push(`${item.name}: mounts ${describeMountSource(mount)}, which is outside of the bundle`);
214
249
  }
215
- if (source.type === 'provider' && policy.reserved?.includes(source.account)) {
216
- problems.push(`${item.name}: the secret ${describeSecret(secret).target} is read as ${source.account}, which the server uses to read repositories`);
250
+ }
251
+ }
252
+ for (const volume of (0, work_service_1.isContainerWorkService)(data) ? data.volumes : []) {
253
+ // docker takes anything else for a path of the host, such as the docker socket
254
+ if (!(0, hosted_volumes_1.isVolumeName)(volume.name)) {
255
+ problems.push(`${item.name}: volume ${volume.name} is not a volume name, a path of the server can't be mounted`);
256
+ }
257
+ }
258
+ for (const secret of secretsOf(item)) {
259
+ const source = secret.source;
260
+ if (policy.backend === 'kubernetes' && source.type === 'provider') {
261
+ // the server reads no secret of a provider on a cluster: the Job of a task does
262
+ if (!(0, work_task_1.isWorkTask)(item.data)) {
263
+ problems.push(`${item.name}: the secret ${describeSecret(secret).target} is read from ${source.provider}, which ` +
264
+ 'this server does for tasks only, a service on its cluster gets no provider secret yet');
217
265
  }
218
- else if (source.type === 'file') {
219
- problems.push(`${item.name}: a secret can not read a file of the server (file:), use a secret provider`);
266
+ else if (secret.cache) {
267
+ problems.push(`${item.name}: the secret ${describeSecret(secret).target} has cache: true, its digest would need ` +
268
+ 'the value before the Job of the task reads it, which this server does not support yet');
220
269
  }
221
- else if (source.type === 'env' && !policy.envs.includes(source.name)) {
222
- problems.push(`${item.name}: the secret ${describeSecret(secret).target} reads env:${source.name}, which the server does not list in ${policy.envsFrom ?? 'runs.env.allow'}`);
270
+ else if ((0, work_network_1.isLimitedNetwork)(item.data.network) && !policy.proxiedProviders?.includes(source.provider)) {
271
+ // the init containers that read it share the pod's NetworkPolicy, and reach the
272
+ // provider only through the egress proxy
273
+ problems.push(`${item.name}: the secret ${describeSecret(secret).target} is read from ${source.provider} in the ` +
274
+ `Job of the task, which network ${(0, work_network_1.formatNetwork)(item.data.network)} would cut off from it: ` +
275
+ `this server names no endpoints of ${source.provider} in backend.providers.${source.provider}.allow ` +
276
+ 'for the egress proxy to let through, so the task keeps an open network');
223
277
  }
224
278
  }
279
+ if (source.type === 'provider' && policy.reserved?.includes(source.account)) {
280
+ problems.push(`${item.name}: the secret ${describeSecret(secret).target} is read as ${source.account}, which the server uses to read repositories`);
281
+ }
282
+ else if (source.type === 'file') {
283
+ problems.push(`${item.name}: a secret can not read a file of the server (file:), use a secret provider`);
284
+ }
285
+ else if (source.type === 'env' && !policy.envs.includes(source.name)) {
286
+ problems.push(`${item.name}: the secret ${describeSecret(secret).target} reads env:${source.name}, which the server does not list in ${policy.envsFrom ?? 'runs.env.allow'}`);
287
+ }
225
288
  }
226
289
  return problems;
227
290
  }
291
+ // A mount of the home directory says so: `$PWD` in a mount is the home directory, not the
292
+ // project, which reads like a path of the checkout
293
+ function describeMountSource(mount) {
294
+ const home = (0, normalize_path_1.getHomeReference)(mount.mount.split(':')[0]);
295
+ return home ? `${mount.localPath} (${home} is the home directory)` : mount.localPath;
296
+ }
228
297
  function toPlan(root, item) {
229
298
  const data = item.data;
230
299
  const secrets = secretsOf(item).map(describeSecret);
@@ -237,7 +306,7 @@ function toPlan(root, item) {
237
306
  : [];
238
307
  const ports = 'ports' in data ? data.ports.map((port) => `${port.hostPort ?? ''}:${port.containerPort}`) : [];
239
308
  const flags = [];
240
- if (data.type === 'local-task') {
309
+ if (data.type === 'local-task' && !(0, work_task_1.isAggregateTask)(data)) {
241
310
  flags.push('local-task');
242
311
  }
243
312
  if ('ports' in data && data.ports.some((port) => port.hostPort !== null)) {
@@ -246,14 +315,20 @@ function toPlan(root, item) {
246
315
  if (data.scope.remote) {
247
316
  flags.push('remote-include');
248
317
  }
318
+ const capAdd = (0, work_task_1.isContainerWorkTask)(data) ? data.capAdd : [];
319
+ if (capAdd.length > 0) {
320
+ flags.push('capabilities');
321
+ }
249
322
  const isTask = data.type === 'local-task' || data.type === 'container-task';
250
323
  return {
251
324
  kind: isTask ? 'task' : 'service',
252
- runtime: data.type === 'local-task' ? 'local' : 'docker',
325
+ ...(data.description ? { description: data.description } : {}),
326
+ runtime: (0, work_task_1.isAggregateTask)(data) ? 'none' : data.type === 'local-task' ? 'local' : 'docker',
253
327
  ...((0, work_task_1.isContainerWorkTask)(data) || (0, work_service_1.isContainerWorkService)(data) ? { image: data.image } : {}),
254
328
  cmds: 'cmds' in data ? data.cmds.map((cmd) => cmd.cmd) : 'cmd' in data && data.cmd ? [data.cmd.cmd] : [],
255
329
  command: (0, task_arguments_1.getArgumentsTarget)(data)?.cmd ?? null,
256
330
  network: (0, work_network_1.toNetworkSchema)('network' in data ? data.network : { type: 'open' }),
331
+ ...('args' in data && data.args !== null ? { args: data.args } : {}),
257
332
  deps: item.deps.map((dep) => dep.name).sort(),
258
333
  needs: item.needs.map((need) => need.name).sort(),
259
334
  accounts,
@@ -262,6 +337,7 @@ function toPlan(root, item) {
262
337
  mounts,
263
338
  ports,
264
339
  generates: 'generates' in data ? data.generates.map((generate) => generate.path) : [],
340
+ ...(capAdd.length > 0 ? { capAdd: [...capAdd] } : {}),
265
341
  flags,
266
342
  };
267
343
  }
@@ -278,10 +354,6 @@ function describeFailure(e) {
278
354
  // when somebody wants to run it. The same function answers what a run will do,
279
355
  // with the scope of the run: which tasks, services and accounts it involves.
280
356
  async function planBundle(bundle, environment, options = {}) {
281
- const problems = [...findLooseIncludes(bundle, options.noRemote), ...(options.policy ? findHostProblems(bundle) : [])];
282
- if (problems.length > 0) {
283
- throw new DefinitionError(problems.join('\n'));
284
- }
285
357
  const directory = await (0, promises_1.mkdtemp)((0, path_1.join)((0, os_1.tmpdir)(), 'hammerkit-definition-'));
286
358
  try {
287
359
  for (const [path, content] of Object.entries(bundle.files)) {
@@ -294,7 +366,20 @@ async function planBundle(bundle, environment, options = {}) {
294
366
  file: confined((0, get_file_context_1.getFileContext)(directory), directory),
295
367
  };
296
368
  try {
297
- const { ctx, scope } = await (0, schema_parser_1.createParseContext)((0, path_1.join)(directory, bundle.entry), inspected);
369
+ const { ctx, scope } = await (0, schema_parser_1.createParseContext)((0, path_1.join)(directory, bundle.entry), inspected, {
370
+ // a hash alone can only name what is inside the bundle, see checkGitSource
371
+ checkGitSource: (source, from, block, name) => checkGitSource(
372
+ // a file of a git include is named by its repository
373
+ from.remote
374
+ ? `${from.remote.git} ${(0, path_1.relative)(from.remote.root, from.fileName).split(path_1.sep).join('/')}`
375
+ : (0, path_1.relative)(directory, from.fileName), `${block}.${name}`, source, options.noRemote),
376
+ });
377
+ if (options.policy) {
378
+ const problems = findHostProblems(publishedBundleFiles(bundle, directory, scope));
379
+ if (problems.length > 0) {
380
+ throw new DefinitionError(problems.join('\n'));
381
+ }
382
+ }
298
383
  const context = await (0, reference_parser_1.parseReferences)(ctx, scope, inspected);
299
384
  const workTree = (0, work_scope_parser_1.getWorkContext)(context, options.scope ?? { environmentName: null }, inspected);
300
385
  const items = itemsOf(workTree);
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.decideRun = exports.sandboxRepos = exports.decideSandbox = exports.sandboxPushBranches = exports.sandboxBranchType = exports.getGrant = exports.usedAccounts = exports.mayYank = exports.canUse = exports.maySeeDefinitions = exports.mayPublish = exports.mayPropose = exports.mayRun = exports.matches = void 0;
3
+ exports.decideRun = exports.sandboxRepos = exports.decideSandbox = exports.sandboxPushBranches = exports.sandboxBranchType = exports.endpointLevel = exports.getGrant = exports.usedAccounts = exports.mayYank = exports.canUse = exports.maySeeDefinitions = exports.mayPublish = exports.mayPropose = exports.mayReadChecks = exports.mayRun = exports.matches = void 0;
4
4
  const repo_1 = require("./repo");
5
5
  const BUILDS_ORDER = ['run', 'propose', 'publish'];
6
6
  function matches(entitlement, principal) {
@@ -22,6 +22,12 @@ function atLeast(grant, level) {
22
22
  }
23
23
  const mayRun = (grant) => atLeast(grant, 'run');
24
24
  exports.mayRun = mayRun;
25
+ // Who may run from a repository's refs may see what passed for its files, and so may who
26
+ // is entitled to just that (a CI job that only verifies).
27
+ function mayReadChecks(grant, repo) {
28
+ return (grant.checks || grant.builds !== null) && (0, repo_1.matchesAnyRepo)(grant.repos, repo);
29
+ }
30
+ exports.mayReadChecks = mayReadChecks;
25
31
  const mayPropose = (grant) => atLeast(grant, 'propose');
26
32
  exports.mayPropose = mayPropose;
27
33
  const mayPublish = (grant) => atLeast(grant, 'publish');
@@ -55,12 +61,15 @@ function getGrant(entitlements, principal) {
55
61
  const accounts = {};
56
62
  const repos = [];
57
63
  const sandboxes = [];
64
+ const endpoints = [];
58
65
  let builds = null;
66
+ let checks = false;
59
67
  entitlements.forEach((entitlement, index) => {
60
68
  if (!matches(entitlement, principal)) {
61
69
  return;
62
70
  }
63
71
  matched.push(index);
72
+ checks = checks || entitlement.checks === 'read';
64
73
  for (const [account, level] of Object.entries(entitlement.accounts)) {
65
74
  if (accounts[account] !== 'use') {
66
75
  accounts[account] = level;
@@ -85,10 +94,31 @@ function getGrant(entitlements, principal) {
85
94
  known.branches[pattern] = higherType(known.branches[pattern], type);
86
95
  }
87
96
  }
97
+ for (const endpoint of entitlement.endpoints ?? []) {
98
+ const known = endpoints.find((entry) => entry.repo === endpoint.repo);
99
+ if (!known) {
100
+ endpoints.push({ ...endpoint });
101
+ }
102
+ else if (endpoint.level === 'publish') {
103
+ known.level = 'publish';
104
+ }
105
+ }
88
106
  });
89
- return { accounts, repos, builds, sandboxes, matched };
107
+ return { accounts, repos, builds, sandboxes, endpoints, checks, matched };
90
108
  }
91
109
  exports.getGrant = getGrant;
110
+ // What the caller may do with the endpoints of service runs of a repository: the higher
111
+ // level of the entries whose `repo` covers it, null for nothing.
112
+ function endpointLevel(grant, repo) {
113
+ let found = null;
114
+ for (const endpoint of grant.endpoints) {
115
+ if ((0, repo_1.matchesRepo)(endpoint.repo, repo) && found !== 'publish') {
116
+ found = endpoint.level;
117
+ }
118
+ }
119
+ return found;
120
+ }
121
+ exports.endpointLevel = endpointLevel;
92
122
  const BRANCH_TYPES = ['pull', 'push'];
93
123
  function higherType(a, b) {
94
124
  return a !== undefined && BRANCH_TYPES.indexOf(a) > BRANCH_TYPES.indexOf(b) ? a : b;
@@ -105,6 +105,11 @@ class RepoAccess {
105
105
  env.GIT_CONFIG_GLOBAL = '/dev/null';
106
106
  return { env, values: credentials.values };
107
107
  }
108
+ // A secret only the server reads, as an account no caller may use: the client secret of
109
+ // the sign-in at endpoints, read like the credential of a source.
110
+ serverSecret(secret) {
111
+ return this.token(secret);
112
+ }
108
113
  async token(secret) {
109
114
  const key = `${secret.account}\0${secret.from}`;
110
115
  const known = this.tokens.get(key);