hammerkit 1.27.0-next.3 → 1.27.0-next.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/build.schema.json CHANGED
@@ -289,6 +289,9 @@
289
289
  ],
290
290
  "description": "network the task or service reaches besides its needs: open (default), none or { allow: [hosts or host:port] }"
291
291
  },
292
+ "args": {
293
+ "type": "boolean"
294
+ },
292
295
  "runner": {
293
296
  "type": "string",
294
297
  "minLength": 1
@@ -370,6 +373,9 @@
370
373
  "network": {
371
374
  "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/network"
372
375
  },
376
+ "args": {
377
+ "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/args"
378
+ },
373
379
  "runner": {
374
380
  "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/runner"
375
381
  },
@@ -511,6 +517,9 @@
511
517
  "network": {
512
518
  "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/network",
513
519
  "description": "network the task or service reaches besides its needs: open (default), none or { allow: [hosts or host:port] }"
520
+ },
521
+ "args": {
522
+ "type": "boolean"
514
523
  }
515
524
  },
516
525
  "required": [
@@ -1,8 +1,9 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.getJobSecretSpec = exports.removeJobCredentials = exports.assertJobAccounts = exports.ensureJobCredentials = exports.getCredentialsName = exports.assertSecretsReachable = exports.splitJobSecrets = exports.ACCOUNT_PATH = exports.SECRET_VALUES_PATH = void 0;
3
+ exports.getJobSecretSpec = exports.removeJobCredentials = exports.assertJobAccounts = exports.ensureJobCredentials = exports.getCredentialsName = exports.getSecretHosts = exports.splitJobSecrets = exports.ACCOUNT_PATH = exports.SECRET_VALUES_PATH = void 0;
4
4
  const work_secret_1 = require("../planner/work-secret");
5
5
  const command_secret_provider_1 = require("../secrets/command-secret-provider");
6
+ const task_arguments_1 = require("../planner/utils/task-arguments");
6
7
  const version_1 = require("../version");
7
8
  const apply_1 = require("./apply");
8
9
  const resources_1 = require("./resources");
@@ -42,19 +43,24 @@ function splitJobSecrets(secrets, server) {
42
43
  exports.splitJobSecrets = splitJobSecrets;
43
44
  /**
44
45
  * The init containers that read secrets share the pod, and so the NetworkPolicy, of
45
- * the task: under a limited network they could not reach the secret manager. Until
46
- * they run in a pod of their own, a task that reads a provider's secret in its Job
47
- * keeps an open network.
46
+ * the task: under a limited network they reach the secret manager only through the
47
+ * task's egress proxy, which lets them reach the endpoints the server names for the
48
+ * provider (`backend.providers.<name>.allow`). A task that reads a secret of a provider
49
+ * without them keeps an open network. Answers the endpoints the proxy lets through.
48
50
  */
49
- function assertSecretsReachable(task, secrets, network = (0, work_network_1.getEffectiveNetwork)(task.data)) {
50
- if (secrets.length === 0 || network.type === 'open') {
51
- return;
51
+ function getSecretHosts(task, secrets, server, network = (0, work_network_1.getEffectiveNetwork)(task.data)) {
52
+ if (!server || secrets.length === 0 || network.type === 'open') {
53
+ return [];
54
+ }
55
+ const unreachable = secrets.find((secret) => !server.providers[secret.source.provider]?.allow?.length);
56
+ if (unreachable) {
57
+ throw new Error(`task ${task.name}: secret ${(0, work_secret_1.getSecretName)(unreachable.secret)} is read from ${(0, work_secret_1.describeSecretSource)(unreachable.secret)} in the Job of the task, which network ${(0, work_network_1.formatNetwork)(network)} would cut off from it; ` +
58
+ `on this cluster a task that reads secrets of ${unreachable.source.provider} keeps an open network, ` +
59
+ `unless the server names its endpoints in backend.providers.${unreachable.source.provider}.allow`);
52
60
  }
53
- const secret = secrets[0];
54
- throw new Error(`task ${task.name}: secret ${(0, work_secret_1.getSecretName)(secret.secret)} is read from ${(0, work_secret_1.describeSecretSource)(secret.secret)} in the Job of the task, which network ${(0, work_network_1.formatNetwork)(network)} would cut off from it; ` +
55
- 'on this cluster a task that reads secrets of a provider keeps an open network');
61
+ return [...new Set(secrets.flatMap((secret) => server.providers[secret.source.provider].allow ?? []))].sort();
56
62
  }
57
- exports.assertSecretsReachable = assertSecretsReachable;
63
+ exports.getSecretHosts = getSecretHosts;
58
64
  // The accounts of the secrets, in the order they are first read
59
65
  function accountsOf(secrets) {
60
66
  return [...new Set(secrets.map((s) => s.source.account))];
@@ -186,7 +192,9 @@ function quote(value) {
186
192
  * from it, and the task's output is masked before it leaves the pod, since the
187
193
  * server, which masks everything else, never sees these values.
188
194
  */
189
- function getJobSecretSpec(item, secrets, server, credentials, environment) {
195
+ function getJobSecretSpec(item, secrets, server, credentials, environment,
196
+ // under a limited network, what sends their requests through the egress proxy
197
+ proxyEnv = []) {
190
198
  if (!server || secrets.length === 0) {
191
199
  return {
192
200
  initContainers: [],
@@ -229,8 +237,11 @@ function getJobSecretSpec(item, secrets, server, credentials, environment) {
229
237
  '-c',
230
238
  'umask 022 && "$@" > "$HAMMERKIT_VALUE.part" && mv "$HAMMERKIT_VALUE.part" "$HAMMERKIT_VALUE"',
231
239
  ],
232
- args: ['sh', command, ...args],
240
+ // Kubernetes would expand a `$(NAME)` of the container's env in them, the proxy's token
241
+ // among them: the reference is text, as on any other backend
242
+ args: ['sh', command, ...args].map(task_arguments_1.escapeKubernetesText),
233
243
  env: [
244
+ ...proxyEnv,
234
245
  ...Object.entries(env).map(([name, value]) => ({ name, value })),
235
246
  { name: 'HAMMERKIT_VALUE', value: `${exports.SECRET_VALUES_PATH}/secret-${index}` },
236
247
  ],
@@ -23,6 +23,9 @@ const NETWORK_OF_LABEL = 'hammerkit.dev/network-of';
23
23
  // on the proxy of a task on a sandbox: the sandbox, whose pod of files it stays with
24
24
  const PROXY_OF_SANDBOX_LABEL = 'hammerkit.dev/proxy-of-sandbox';
25
25
  const PROXY_READY_TIMEOUT_MS = 60000;
26
+ // the token of the endpoints of secret providers, in a Secret named like the proxy
27
+ const SECRET_TOKEN_KEY = 'token';
28
+ const SECRET_TOKEN_VARIABLE = 'HAMMERKIT_PROXY_TOKEN';
26
29
  // how long the pods of an item may take to end before their policy goes
27
30
  const POD_END_TIMEOUT_MS = 120000;
28
31
  const POD_END_POLL_MS = 500;
@@ -109,10 +112,10 @@ function getIngress(env, item) {
109
112
  return [{ _from: [{ podSelector: from }] }];
110
113
  }
111
114
  // The NetworkPolicy of a limited item: egress to the pods of the services it
112
- // needs and, under `allow`, to its proxy; nothing else, DNS included (needs
113
- // and the proxy are host aliases). Ingress only from what needs it. An open
114
- // item has none.
115
- function buildNetworkPolicy(env, item, network, scope, proxy = getRunProxy(item, scope)) {
115
+ // needs and, under `allow` or with secrets read through it, to its proxy;
116
+ // nothing else, DNS included (needs and the proxy are host aliases). Ingress
117
+ // only from what needs it. An open item has none.
118
+ function buildNetworkPolicy(env, item, network, scope, proxy = getRunProxy(item, scope), proxied = network.type === 'allow') {
116
119
  const needs = item.needs.filter((need) => (0, work_item_1.isContainerWorkServiceItem)(need.service));
117
120
  return {
118
121
  apiVersion: 'networking.k8s.io/v1',
@@ -131,7 +134,7 @@ function buildNetworkPolicy(env, item, network, scope, proxy = getRunProxy(item,
131
134
  ...(needs.length > 0
132
135
  ? [{ to: needs.map((need) => ({ podSelector: { matchLabels: { [ID_LABEL]: need.service.id() } } })) }]
133
136
  : []),
134
- ...(network.type === 'allow'
137
+ ...(proxied
135
138
  ? [
136
139
  {
137
140
  to: [{ podSelector: { matchLabels: { [PROXY_LABEL]: proxy.key } } }],
@@ -179,6 +182,9 @@ exports.buildProxyPolicy = buildProxyPolicy;
179
182
  // left goes.
180
183
  // A task follows the proxy's log while its jobs run. A service outlives the
181
184
  // run on a cluster, so its refused hosts are in the proxy's own log.
185
+ // `secretHosts` are the endpoints of the providers whose secrets the init
186
+ // containers of a task on a server read: a limited task gets a proxy for them
187
+ // (under `none` too), which passes them to those containers only.
182
188
  async function ensureKubernetesItemNetwork(instance, env, item, options) {
183
189
  const network = (0, work_network_1.getEffectiveNetwork)(item.data);
184
190
  if (network.type !== 'open' && !env.networkPolicyEnforced) {
@@ -191,22 +197,30 @@ async function ensureKubernetesItemNetwork(instance, env, item, options) {
191
197
  // without the rights to manage NetworkPolicies, hammerkit never made one
192
198
  await removeKubernetesItemNetwork(instance, env, item, { run: null }, { ignoreForbidden: true });
193
199
  }
194
- return { env: [], hostAliases, initContainers: [], close() { } };
200
+ return { env: [], secretEnv: [], hostAliases, initContainers: [], close() { } };
201
+ }
202
+ const secretHosts = options.secretHosts ?? [];
203
+ if (secretHosts.length > 0 && options.sandbox) {
204
+ // a run on a sandbox uses no account, so it reads no secret of a provider
205
+ throw new Error(`${item.name}: a task on a sandbox reads no secret of a provider`);
195
206
  }
196
207
  const proxy = network.type === 'allow' && options.sandbox
197
208
  ? getSandboxProxy(env, item, network, hostAliases, options.sandbox)
198
209
  : getRunProxy(item, options);
199
- await (0, apply_1.apply)(instance, buildNetworkPolicy(env, item, network, options, proxy));
210
+ const proxied = network.type === 'allow' || secretHosts.length > 0;
211
+ await (0, apply_1.apply)(instance, buildNetworkPolicy(env, item, network, options, proxy, proxied));
200
212
  const initContainers = [(0, network_gate_1.getNetworkGateContainer)()];
201
- if (network.type === 'none') {
213
+ if (!proxied) {
202
214
  if (options.sandbox) {
203
215
  await removeSandboxProxies(instance, env.namespace, { sandbox: options.sandbox.label, item });
204
216
  }
205
217
  else {
206
218
  await removeProxy(instance, env.namespace, proxy.name, [404]);
207
219
  }
208
- return { env: [], hostAliases, initContainers, close() { } };
220
+ return { env: [], secretEnv: [], hostAliases, initContainers, close() { } };
209
221
  }
222
+ // under `none`, a proxy for the secrets only: it allows the item no host
223
+ const proxyNetwork = network.type === 'allow' ? network : { type: 'allow', hosts: [] };
210
224
  let started;
211
225
  try {
212
226
  if (proxy.sandbox) {
@@ -214,7 +228,7 @@ async function ensureKubernetesItemNetwork(instance, env, item, options) {
214
228
  await removeSandboxProxies(instance, env.namespace, { sandbox: proxy.sandbox.label, item }, (deployment) => deployment.metadata?.labels?.[PROXY_LABEL] === proxy.key);
215
229
  proxy.owner = await getFilesPodOwner(instance, env, proxy.sandbox);
216
230
  }
217
- started = await ensureProxy(instance, env, item, network, hostAliases, options, proxy);
231
+ started = await ensureProxy(instance, env, item, proxyNetwork, hostAliases, options, proxy, secretHosts);
218
232
  }
219
233
  catch (e) {
220
234
  await removeKubernetesItemNetwork(instance, env, item, options).catch(() => undefined);
@@ -223,11 +237,17 @@ async function ensureKubernetesItemNetwork(instance, env, item, options) {
223
237
  const follow = options.followLog
224
238
  ? await followProxyLog(instance, env, item, proxy, { tailLines: started.reused ? 0 : undefined })
225
239
  : null;
240
+ const needs = item.needs.map((need) => need.name);
241
+ const toEnv = (variables) => Object.entries(variables).map(([name, value]) => ({ name, value }));
226
242
  return {
227
- env: Object.entries((0, egress_proxy_1.getProxyClientEnvironment)(item.needs.map((need) => need.name))).map(([name, value]) => ({
228
- name,
229
- value,
230
- })),
243
+ env: network.type === 'allow' ? toEnv((0, egress_proxy_1.getProxyClientEnvironment)(needs)) : [],
244
+ // the token from the proxy's Secret, which the item's own container never mounts
245
+ secretEnv: secretHosts.length > 0
246
+ ? [
247
+ { name: SECRET_TOKEN_VARIABLE, valueFrom: { secretKeyRef: { name: proxy.name, key: SECRET_TOKEN_KEY } } },
248
+ ...toEnv((0, egress_proxy_1.getProxyClientEnvironment)(needs, `$(${SECRET_TOKEN_VARIABLE})`)),
249
+ ]
250
+ : [],
231
251
  // the item reaches each TCP forward under its host's name, at the proxy
232
252
  hostAliases: [
233
253
  ...hostAliases,
@@ -318,6 +338,7 @@ async function removeAllKubernetesItemNetworks(instance, env, item) {
318
338
  if (name) {
319
339
  await ignoreStatus(instance.appsApi.deleteNamespacedDeployment({ name, namespace: env.namespace }), [404]);
320
340
  await ignoreStatus(instance.coreApi.deleteNamespacedService({ name, namespace: env.namespace }), [404]);
341
+ await ignoreStatus(instance.coreApi.deleteNamespacedSecret({ name, namespace: env.namespace }), [404]);
321
342
  }
322
343
  }
323
344
  }
@@ -349,7 +370,7 @@ function getProxyReadinessScript(ports) {
349
370
  `.on('error', () => process.exit(1))`);
350
371
  }
351
372
  exports.getProxyReadinessScript = getProxyReadinessScript;
352
- async function ensureProxy(instance, env, item, network, hostAliases, scope, proxy) {
373
+ async function ensureProxy(instance, env, item, network, hostAliases, scope, proxy, secretHosts = []) {
353
374
  const name = proxy.name;
354
375
  const labels = { [PROXY_LABEL]: proxy.key };
355
376
  const metadata = {
@@ -388,7 +409,17 @@ async function ensureProxy(instance, env, item, network, hostAliases, scope, pro
388
409
  name: 'proxy',
389
410
  image: egress_proxy_1.EGRESS_PROXY_IMAGE,
390
411
  command: (0, egress_proxy_1.getEgressProxyCommand)(),
391
- env: Object.entries((0, egress_proxy_1.getItemProxyEnvironment)(item)).map(([key, value]) => ({ name: key, value })),
412
+ env: [
413
+ ...Object.entries({ ...(0, egress_proxy_1.getItemProxyEnvironment)(item), ...(0, egress_proxy_1.getSecretProxyEnvironment)(secretHosts) }).map(([key, value]) => ({ name: key, value })),
414
+ ...(secretHosts.length > 0
415
+ ? [
416
+ {
417
+ name: 'HAMMERKIT_SECRET_TOKEN',
418
+ valueFrom: { secretKeyRef: { name, key: SECRET_TOKEN_KEY } },
419
+ },
420
+ ]
421
+ : []),
422
+ ],
392
423
  ports: (0, egress_proxy_1.getEgressProxyPorts)(network).map((port) => ({ containerPort: port })),
393
424
  // in the pod, since its ingress policy may keep the kubelet out;
394
425
  // ready once it listens on every port, its TCP forwards' too
@@ -422,6 +453,21 @@ async function ensureProxy(instance, env, item, network, hostAliases, scope, pro
422
453
  ? await instance.objectApi.read(deployment).then((current) => !current.metadata?.deletionTimestamp, (e) => ((0, apply_1.statusCodeOf)(e) === 404 ? false : Promise.reject(e)))
423
454
  : false;
424
455
  await (0, apply_1.apply)(instance, buildProxyPolicy(env, item, network, scope, proxy));
456
+ if (secretHosts.length > 0) {
457
+ // new with each run's proxy, and kept while it is there: its pod read it when it started.
458
+ // The init containers that read secrets get it, the task not.
459
+ const secret = {
460
+ apiVersion: 'v1',
461
+ kind: 'Secret',
462
+ metadata: { ...metadata, annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() } },
463
+ type: 'Opaque',
464
+ data: { [SECRET_TOKEN_KEY]: Buffer.from((0, crypto_1.randomBytes)(32).toString('hex')).toString('base64') },
465
+ };
466
+ const exists = await instance.objectApi.read(secret).then(() => true, (e) => ((0, apply_1.statusCodeOf)(e) === 404 ? false : Promise.reject(e)));
467
+ if (!exists) {
468
+ await instance.objectApi.create(secret);
469
+ }
470
+ }
425
471
  await (0, apply_1.apply)(instance, deployment);
426
472
  const created = await (0, apply_1.apply)(instance, service);
427
473
  await (0, await_running_state_1.awaitDeployRunningState)(instance, env, name, {
@@ -433,7 +479,7 @@ async function ensureProxy(instance, env, item, network, hostAliases, scope, pro
433
479
  if (!ip) {
434
480
  throw new Error(`the egress proxy of ${item.name} has no cluster ip`);
435
481
  }
436
- item.status.write('debug', `${reused ? 'egress proxy of the sandbox' : 'egress proxy'} allows ${network.hosts.join(', ') || 'no host'}`);
482
+ item.status.write('debug', `${reused ? 'egress proxy of the sandbox' : 'egress proxy'} allows ${network.hosts.join(', ') || 'no host'}${secretHosts.length > 0 ? `, and ${secretHosts.join(', ')} to the reads of secrets` : ''}`);
437
483
  return { ip, reused };
438
484
  }
439
485
  // Refused hosts show up in the item's log, as on Docker; of a proxy from an
@@ -468,9 +514,13 @@ async function followProxyLog(instance, env, item, proxy, options) {
468
514
  return null;
469
515
  }
470
516
  }
471
- async function removeProxy(instance, namespace, name, ignored) {
517
+ // The token of a proxy for secrets goes with it; a proxy of a sandbox has none.
518
+ async function removeProxy(instance, namespace, name, ignored, options = { token: true }) {
472
519
  await ignoreStatus(instance.appsApi.deleteNamespacedDeployment({ name, namespace }), ignored);
473
520
  await ignoreStatus(instance.coreApi.deleteNamespacedService({ name, namespace }), ignored);
521
+ if (options.token) {
522
+ await ignoreStatus(instance.coreApi.deleteNamespacedSecret({ name, namespace }), ignored);
523
+ }
474
524
  await ignoreStatus(instance.objectApi.delete({
475
525
  apiVersion: 'networking.k8s.io/v1',
476
526
  kind: 'NetworkPolicy',
@@ -492,7 +542,7 @@ async function removeSandboxProxies(instance, namespace, of, keep = () => false)
492
542
  for (const deployment of deployments.items) {
493
543
  const name = deployment.metadata?.name;
494
544
  if (name && !keep(deployment)) {
495
- await removeProxy(instance, namespace, name, [404]);
545
+ await removeProxy(instance, namespace, name, [404], { token: false });
496
546
  }
497
547
  }
498
548
  }
@@ -5,10 +5,12 @@
5
5
  // `CONNECT` tunnels to an allowed host pass, everything else gets a 403 and a
6
6
  // `refused` event on stdout. For each `host:port` entry it also listens on that
7
7
  // port and forwards raw TCP to that host, which the item reaches under the
8
- // host's name. The script is passed to `node -e`, so the proxy
8
+ // host's name. On a server's cluster it also lets the init containers that read
9
+ // the task's secrets reach the endpoints of their providers: a tunnel to port 443
10
+ // that carries the proxy's token, which those containers get and the task not. The script is passed to `node -e`, so the proxy
9
11
  // needs no image of its own, only a stock Node image.
10
12
  Object.defineProperty(exports, "__esModule", { value: true });
11
- exports.formatEgressProxyEvent = exports.formatRefusedHost = exports.parseEgressProxyEvent = exports.getProxylessEnvironment = exports.getProxyClientEnvironment = exports.getEgressProxyPorts = exports.getItemProxyEnvironment = exports.getEgressProxyEnvironment = exports.getEgressProxyCommand = exports.EGRESS_PROXY_SCRIPT = exports.EGRESS_PROXY_HOST = exports.EGRESS_PROXY_PORT = exports.EGRESS_PROXY_IMAGE = void 0;
13
+ exports.formatEgressProxyEvent = exports.formatRefusedHost = exports.parseEgressProxyEvent = exports.getProxylessEnvironment = exports.getProxyClientEnvironment = exports.getEgressProxyPorts = exports.getItemProxyEnvironment = exports.getSecretProxyEnvironment = exports.getEgressProxyEnvironment = exports.getEgressProxyCommand = exports.EGRESS_PROXY_SCRIPT = exports.EGRESS_PROXY_HOST = exports.EGRESS_PROXY_PORT = exports.EGRESS_PROXY_IMAGE = void 0;
12
14
  const work_network_1 = require("../planner/work-network");
13
15
  const network_schema_1 = require("../schema/network-schema");
14
16
  // Pinned, so the code behind the boundary is the same on every host. To bump
@@ -25,6 +27,7 @@ exports.EGRESS_PROXY_SCRIPT = `
25
27
  const http = require('http')
26
28
  const net = require('net')
27
29
  const dns = require('dns')
30
+ const crypto = require('crypto')
28
31
  const entries = JSON.parse(process.env.HAMMERKIT_ALLOW || '[]').map((h) => h.toLowerCase())
29
32
  // names and patterns for HTTP(S)
30
33
  const allow = entries.filter((e) => !e.includes(':'))
@@ -38,6 +41,9 @@ const publicOnly = JSON.parse(process.env.HAMMERKIT_PUBLIC_ONLY || '[]').map((h)
38
41
  // the names of the needed services, for tools that ignore NO_PROXY: exact
39
42
  // names only, never patterns
40
43
  const needs = JSON.parse(process.env.HAMMERKIT_NEEDS || '[]').map((h) => h.toLowerCase())
44
+ // the endpoints of secret providers, for a request with the token only
45
+ const secretAllow = JSON.parse(process.env.HAMMERKIT_SECRET_ALLOW || '[]').map((h) => h.toLowerCase())
46
+ const secretToken = process.env.HAMMERKIT_SECRET_TOKEN || ''
41
47
  const port = parseInt(process.env.HAMMERKIT_PROXY_PORT || '${exports.EGRESS_PROXY_PORT}', 10)
42
48
  // the name whose address the proxy listens on, every address when unset
43
49
  const bind = process.env.HAMMERKIT_PROXY_BIND
@@ -64,6 +70,31 @@ function allowedBy(host, port) {
64
70
  }
65
71
  return allow.some((rule) => rule.startsWith('*.') && name.endsWith(rule.substring(1))) ? 'wildcard' : null
66
72
  }
73
+ // Whether the request carries the token as the password of Proxy-Authorization (Basic).
74
+ function hasSecretToken(req) {
75
+ if (secretToken.length === 0) {
76
+ return false
77
+ }
78
+ const match = /^basic\\s+(\\S+)$/i.exec(String(req.headers['proxy-authorization'] || ''))
79
+ if (!match) {
80
+ return false
81
+ }
82
+ const credentials = Buffer.from(match[1], 'base64').toString('utf8')
83
+ const colon = credentials.indexOf(':')
84
+ if (colon < 0) {
85
+ return false
86
+ }
87
+ const given = Buffer.from(credentials.substring(colon + 1), 'utf8')
88
+ const expected = Buffer.from(secretToken, 'utf8')
89
+ return given.length === expected.length && crypto.timingSafeEqual(given, expected)
90
+ }
91
+ // the rule of a tunnel to an endpoint of a secret provider: like a wildcard, it reaches
92
+ // no private address
93
+ function allowedForSecrets(req, host, port) {
94
+ const name = host.toLowerCase().replace(/\\.$/, '')
95
+ const listed = secretAllow.some((rule) => (rule.startsWith('*.') ? name.endsWith(rule.substring(1)) : name === rule))
96
+ return listed && port === 443 && hasSecretToken(req) ? 'wildcard' : null
97
+ }
67
98
  function ipv4(address) {
68
99
  const mapped = /^::ffff:(\\d+\\.\\d+\\.\\d+\\.\\d+)$/i.exec(address)
69
100
  const value = mapped ? mapped[1] : address
@@ -197,7 +228,7 @@ const server = http.createServer((req, res) => {
197
228
  server.on('connect', (req, socket, head) => {
198
229
  const target = parseAuthority(req.url || '')
199
230
  socket.on('error', () => socket.destroy())
200
- const rule = target ? allowedBy(target.host, target.port) : null
231
+ const rule = target ? allowedBy(target.host, target.port) || allowedForSecrets(req, target.host, target.port) : null
201
232
  if (!rule) {
202
233
  refuse(target ? target.host : String(req.url))
203
234
  socket.end('HTTP/1.1 403 Forbidden\\r\\ncontent-type: text/plain\\r\\n\\r\\nhammerkit: network access is not allowed\\n')
@@ -290,7 +321,8 @@ exports.getEgressProxyCommand = getEgressProxyCommand;
290
321
  // `needs` are the names of the needed services, allowed as exactly those
291
322
  // names. `bind` is the name the proxy listens on alone, so that nothing on its
292
323
  // other networks reaches it. `publicOnly` are the entries of `hosts` that reach
293
- // no private address (see WorkNetwork).
324
+ // no private address (see WorkNetwork). The endpoints of secret providers and
325
+ // the token they need are set by the caller (getSecretProxyEnvironment).
294
326
  function getEgressProxyEnvironment(hosts, needs, bind, publicOnly = []) {
295
327
  return {
296
328
  HAMMERKIT_ALLOW: JSON.stringify(hosts),
@@ -301,6 +333,12 @@ function getEgressProxyEnvironment(hosts, needs, bind, publicOnly = []) {
301
333
  };
302
334
  }
303
335
  exports.getEgressProxyEnvironment = getEgressProxyEnvironment;
336
+ // The endpoints of secret providers the proxy lets a tunnel with its token reach; the
337
+ // token itself is set where the proxy runs, see HAMMERKIT_SECRET_TOKEN.
338
+ function getSecretProxyEnvironment(hosts) {
339
+ return hosts.length > 0 ? { HAMMERKIT_SECRET_ALLOW: JSON.stringify(hosts) } : {};
340
+ }
341
+ exports.getSecretProxyEnvironment = getSecretProxyEnvironment;
304
342
  // The proxy of a task or service: the hosts of its network in this run (under
305
343
  // the run's ceiling), and the names of the services it needs, on every runtime.
306
344
  function getItemProxyEnvironment(item, bind) {
@@ -315,8 +353,9 @@ function getEgressProxyPorts(network) {
315
353
  return [exports.EGRESS_PROXY_PORT, ...(0, work_network_1.getTcpForwards)(network).map((forward) => forward.port)];
316
354
  }
317
355
  exports.getEgressProxyPorts = getEgressProxyPorts;
318
- function getProxyClientEnvironment(directHosts) {
319
- const proxy = `http://${exports.EGRESS_PROXY_HOST}:${exports.EGRESS_PROXY_PORT}`;
356
+ // `token` is sent as the password, for the endpoints of secret providers.
357
+ function getProxyClientEnvironment(directHosts, token) {
358
+ const proxy = `http://${token !== undefined ? `hammerkit:${token}@` : ''}${exports.EGRESS_PROXY_HOST}:${exports.EGRESS_PROXY_PORT}`;
320
359
  const noProxy = ['localhost', '127.0.0.1', ...directHosts].join(',');
321
360
  return {
322
361
  HTTP_PROXY: proxy,
@@ -120,6 +120,7 @@ function parseService(workTree, service, environment, context) {
120
120
  volumeSize: service.schema.volumeSize ?? null,
121
121
  network: (0, work_network_1.parseWorkNetwork)(service.schema.network),
122
122
  networkCeiling: null,
123
+ args: service.schema.args ?? null,
123
124
  };
124
125
  }
125
126
  }
@@ -83,6 +83,7 @@ function parseTask(cwd, task, environment, context) {
83
83
  resources: (0, work_resources_1.parseWorkResources)(task.schema.resources),
84
84
  network: (0, work_network_1.parseWorkNetwork)(task.schema.network),
85
85
  networkCeiling: null,
86
+ args: task.schema.args ?? null,
86
87
  runner: task.schema.runner ?? task.scope.schema.runner ?? null,
87
88
  };
88
89
  if ((0, build_file_task_schema_1.isBuildFileContainerTaskSchema)(task.schema)) {
@@ -100,7 +101,9 @@ function parseTask(cwd, task, environment, context) {
100
101
  if (fileSecret) {
101
102
  throw new Error(`task ${task.relativeName}: a local task takes secrets as env only (${(0, work_secret_1.getSecretName)(fileSecret)})`);
102
103
  }
103
- if (baseWorkNode.network.type !== 'open') {
104
+ // on a server's cluster it runs in a pod of a pool's tool image (runInToolImage)
105
+ const contained = !!environment.hosted?.kubernetes || !!environment.hosted?.localTasksInContainers;
106
+ if (baseWorkNode.network.type !== 'open' && !contained) {
104
107
  throw new Error(`task ${task.relativeName}: network ${(0, work_network_1.formatNetwork)(baseWorkNode.network)} needs a container, ` +
105
108
  `hammerkit can't limit the network of a local task`);
106
109
  }
@@ -101,6 +101,9 @@ function applyArguments(selection, args, keepCache = false) {
101
101
  if (item.type === 'kubernetes-service') {
102
102
  throw new Error(`service ${item.name} is a kubernetes service and has no cmd for the arguments after --`);
103
103
  }
104
+ if (item.args === false) {
105
+ throw new Error(`${item.type === 'container-service' ? 'service' : 'task'} ${item.name} takes no arguments after -- (args: false)`);
106
+ }
104
107
  const target = getArgumentsTarget(item);
105
108
  if (!target) {
106
109
  throw new Error(item.type === 'container-service'
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.kubernetesForwardRuntime = exports.kubernetesServiceRuntime = exports.kubernetesTaskRuntime = exports.describeFailedJob = void 0;
6
+ exports.kubernetesForwardRuntime = exports.kubernetesServiceRuntime = exports.kubernetesTaskRuntime = exports.getTaskJobContainers = exports.describeFailedJob = void 0;
7
7
  const abort_1 = require("../executer/abort");
8
8
  const kubernetes_service_1 = require("../executer/kubernetes-service");
9
9
  const kubernetes_instance_1 = require("../kubernetes/kubernetes-instance");
@@ -125,6 +125,41 @@ async function describeFailedJob(instance, kubernetes, task, job, container, sec
125
125
  return undefined;
126
126
  }
127
127
  exports.describeFailedJob = describeFailedJob;
128
+ /**
129
+ * The containers of the Job of a command: the network gate first, nothing runs before the
130
+ * item's policy holds, then the init containers that read its secrets, then the command. The
131
+ * proxy's token (`network.secretEnv`) reaches the init containers through `secretSpec` only:
132
+ * the command's container gets the proxy variables of a limited network without it.
133
+ */
134
+ function getTaskJobContainers(options) {
135
+ const { task, cmd, envs, secrets, network, secretSpec, kubernetes } = options;
136
+ // the arguments after `--` keep their `$` as text, Kubernetes expands `$(NAME)`
137
+ const command = secretSpec.command(task.data.shell, (0, task_arguments_1.getKubernetesCommandText)(cmd));
138
+ const initContainers = [...network.initContainers, ...secretSpec.initContainers];
139
+ return {
140
+ ...(initContainers.length > 0 ? { initContainers } : {}),
141
+ containers: [
142
+ {
143
+ image: task.data.image,
144
+ // through the task's shell, as on Docker
145
+ command: command.command,
146
+ args: command.args,
147
+ workingDir: cmd.cwd,
148
+ env: [
149
+ ...Object.entries(envs).map(([key, value]) => ({ name: key, value })),
150
+ ...secrets.env,
151
+ ...network.env,
152
+ ...command.env,
153
+ ],
154
+ name: options.container,
155
+ volumeMounts: [...options.mounts, ...secrets.mounts, ...secretSpec.mounts],
156
+ ...(0, container_resources_1.getKubernetesResources)(task.data.resources),
157
+ ...(0, job_placement_1.getContainerSecurity)(kubernetes),
158
+ },
159
+ ],
160
+ };
161
+ }
162
+ exports.getTaskJobContainers = getTaskJobContainers;
128
163
  function kubernetesTaskRuntime(task, kubernetes) {
129
164
  const instance = (0, kubernetes_instance_1.createKubernetesInstances)(kubernetes);
130
165
  // on a server that keeps them, the task's claim stays after the run with the state of its
@@ -168,8 +203,9 @@ function kubernetesTaskRuntime(task, kubernetes) {
168
203
  const { delivered, inJob } = (0, job_secrets_1.splitJobSecrets)(task.data.secrets, server);
169
204
  const secrets = (0, secrets_1.getKubernetesSecretRefs)(task, delivered);
170
205
  const admitted = environment.hosted?.accounts;
206
+ let secretHosts;
171
207
  try {
172
- (0, job_secrets_1.assertSecretsReachable)(task, inJob);
208
+ secretHosts = (0, job_secrets_1.getSecretHosts)(task, inJob, server);
173
209
  if (server) {
174
210
  (0, job_secrets_1.assertJobAccounts)(inJob, server, admitted);
175
211
  }
@@ -210,7 +246,11 @@ function kubernetesTaskRuntime(task, kubernetes) {
210
246
  // NetworkPolicy and proxy of a limited network, both of this run
211
247
  let network;
212
248
  try {
213
- network = await (0, network_policy_1.ensureKubernetesItemNetwork)(instance, kubernetes, task, { followLog: true, ...scope });
249
+ network = await (0, network_policy_1.ensureKubernetesItemNetwork)(instance, kubernetes, task, {
250
+ followLog: true,
251
+ secretHosts,
252
+ ...scope,
253
+ });
214
254
  }
215
255
  catch (e) {
216
256
  await removeTaskSecrets(instance, kubernetes, task).catch(() => undefined);
@@ -226,15 +266,13 @@ function kubernetesTaskRuntime(task, kubernetes) {
226
266
  try {
227
267
  // made for each Job right before it, so a later command's token is as fresh
228
268
  const credentials = await (0, job_secrets_1.ensureJobCredentials)(instance, kubernetes, task, inJob, admitted);
229
- secretSpec = (0, job_secrets_1.getJobSecretSpec)(task, inJob, server, credentials, environment);
269
+ secretSpec = (0, job_secrets_1.getJobSecretSpec)(task, inJob, server, credentials, environment, network.secretEnv);
230
270
  }
231
271
  catch (e) {
232
272
  await removeTaskSecrets(instance, kubernetes, task).catch(() => undefined);
233
273
  options.state.set({ stateKey: options.stateKey, type: 'error', errorMessage: (0, log_1.getErrorMessage)(e) });
234
274
  return;
235
275
  }
236
- // the arguments after `--` keep their `$` as text, Kubernetes expands `$(NAME)`
237
- const command = secretSpec.command(task.data.shell, (0, task_arguments_1.getKubernetesCommandText)(cmd));
238
276
  const spec = {
239
277
  kind: 'Job',
240
278
  apiVersion: 'batch/v1',
@@ -255,33 +293,17 @@ function kubernetesTaskRuntime(task, kubernetes) {
255
293
  labels,
256
294
  },
257
295
  spec: {
258
- // the network gate first: nothing runs before the item's policy holds
259
- ...(network.initContainers.length + secretSpec.initContainers.length > 0
260
- ? { initContainers: [...network.initContainers, ...secretSpec.initContainers] }
261
- : {}),
262
- containers: [
263
- {
264
- image: task.data.image,
265
- // through the task's shell, as on Docker
266
- command: command.command,
267
- args: command.args,
268
- workingDir: cmd.cwd,
269
- env: [
270
- ...Object.entries(envs).map(([key, value]) => ({ name: key, value })),
271
- ...secrets.env,
272
- ...network.env,
273
- ...command.env,
274
- ],
275
- name: container,
276
- volumeMounts: [
277
- ...persistence.mounts.map((m) => m.mount),
278
- ...secrets.mounts,
279
- ...secretSpec.mounts,
280
- ],
281
- ...(0, container_resources_1.getKubernetesResources)(task.data.resources),
282
- ...(0, job_placement_1.getContainerSecurity)(kubernetes),
283
- },
284
- ],
296
+ ...getTaskJobContainers({
297
+ task,
298
+ cmd,
299
+ container,
300
+ envs,
301
+ secrets,
302
+ network,
303
+ secretSpec,
304
+ mounts: persistence.mounts.map((m) => m.mount),
305
+ kubernetes,
306
+ }),
285
307
  volumes: [
286
308
  ...persistence.volumes,
287
309
  ...persistence.workspace,
package/dist/program.js CHANGED
@@ -84,7 +84,8 @@ function printTaskDefinition(environment, definition) {
84
84
  }
85
85
  }
86
86
  // `ls --json`: what a caller (an agent picking a task) needs to know, with the
87
- // command that arguments after `--` are appended to (null: it takes none).
87
+ // command that arguments after `--` are appended to (null: it takes none) and
88
+ // `args` as the build file says it (false: it refuses them).
88
89
  function getListJson(tasks, services) {
89
90
  return {
90
91
  tasks: tasks.map(({ item }) => ({
@@ -96,6 +97,7 @@ function getListJson(tasks, services) {
96
97
  needs: item.needs.map((n) => n.name),
97
98
  labels: item.data.labels,
98
99
  command: (0, task_arguments_1.getArgumentsTarget)(item.data)?.cmd ?? null,
100
+ args: item.data.args,
99
101
  })),
100
102
  services: services.map(({ item }) => ({
101
103
  name: item.name,
@@ -105,6 +107,7 @@ function getListJson(tasks, services) {
105
107
  ports: item.data.ports.map((p) => ({ host: p.hostPort, container: p.containerPort })),
106
108
  labels: item.data.labels,
107
109
  command: (0, task_arguments_1.getArgumentsTarget)(item.data)?.cmd ?? null,
110
+ args: item.data.type === 'container-service' ? item.data.args : null,
108
111
  })),
109
112
  };
110
113
  }
@@ -40,8 +40,12 @@ exports.buildFileBaseTaskSchema = (0, zod_1.object)({
40
40
  account: (0, zod_1.string)().optional(),
41
41
  // CPU and memory limits of the container; a local task is not limited
42
42
  resources: resources_schema_1.resourcesSchema.optional(),
43
- // what the container reaches besides its needs; a local task can't be limited
43
+ // what the container reaches besides its needs; a local task can be limited only where
44
+ // it runs in a container, on a server's cluster
44
45
  network: network_schema_1.networkSchema.optional(),
46
+ // whether its last command takes the arguments after `--`: false refuses them, true
47
+ // says it is meant to, which a server with `runs.args: declared` requires
48
+ args: (0, zod_1.boolean)().optional(),
45
49
  // Name of the server (see `hammerkit remote`) the task has to run on; defaults
46
50
  // to the build file's `runner`. Never a local fallback when it is unknown.
47
51
  runner: runner_schema_1.runnerSchema.optional(),
@@ -45,6 +45,8 @@ exports.buildFileContainerServiceSchema = (0, zod_1.object)({
45
45
  volumeSize: size_schema_1.sizeSchema.describe('size of its claim on Kubernetes, e.g. 10Gi').optional(),
46
46
  // what the service reaches besides its needs; tasks needing it always reach it
47
47
  network: network_schema_1.networkSchema.optional(),
48
+ // whether its `cmd` takes the arguments after `--` of `up`: false refuses them
49
+ args: (0, zod_1.boolean)().optional(),
48
50
  })
49
51
  .strict()
50
52
  .describe('container service');
@@ -129,6 +129,7 @@ function applySchemaExtension(reference, task) {
129
129
  task.schema.timeout = task.schema.timeout ?? extend.schema.timeout;
130
130
  task.schema.runner = task.schema.runner ?? extend.schema.runner;
131
131
  task.schema.network = task.schema.network ?? extend.schema.network;
132
+ task.schema.args = task.schema.args ?? extend.schema.args;
132
133
  if (!reset.has('cmds'))
133
134
  task.schema.cmds = extendArray(task.schema.cmds, extend.schema.cmds);
134
135
  if (!reset.has('src'))
@@ -100,7 +100,7 @@ class HostExecutor {
100
100
  if (extra.length > 0) {
101
101
  throw new Error(`the run would use ${extra.join(', ')}, which it was not admitted for`);
102
102
  }
103
- const refused = (0, run_plan_1.refuseArguments)(run, planned.plan);
103
+ const refused = (0, run_plan_1.refuseArguments)(run, planned.plan, policy);
104
104
  if (refused) {
105
105
  throw new Error(refused);
106
106
  }
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.inspectBundle = exports.planBundle = exports.DefinitionError = void 0;
3
+ exports.inspectBundle = exports.planBundle = exports.getCredentialReads = exports.PLAIN_SERVER_ENVS = exports.DefinitionError = void 0;
4
4
  const promises_1 = require("fs/promises");
5
5
  const os_1 = require("os");
6
6
  const path_1 = require("path");
@@ -20,6 +20,20 @@ const work_network_1 = require("../planner/work-network");
20
20
  class DefinitionError extends Error {
21
21
  }
22
22
  exports.DefinitionError = DefinitionError;
23
+ // Variables of a server that hold no credential (FR-008): a task may read these and
24
+ // still take arguments, or reach any network.
25
+ exports.PLAIN_SERVER_ENVS = ['PATH', 'HOME', 'CI'];
26
+ // What an item holds that its command must not hand out: its secrets, and the variables
27
+ // of the server its `envs` read, but the plain ones.
28
+ function getCredentialReads(item) {
29
+ return [
30
+ ...item.secrets.map((secret) => `the secret ${secret.target}`),
31
+ ...(item.serverEnvs ?? [])
32
+ .filter((env) => !exports.PLAIN_SERVER_ENVS.includes(env.toUpperCase()))
33
+ .map((env) => `$${env} of the server`),
34
+ ];
35
+ }
36
+ exports.getCredentialReads = getCredentialReads;
23
37
  // 40 (sha-1) or 64 (sha-256) hex digits
24
38
  const COMMIT_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/i;
25
39
  // A hash alone can only name what is inside the bundle, so everything a build
@@ -170,13 +184,32 @@ function findPolicyProblems(items, policy, root) {
170
184
  problems.push(`${item.name}: generates ${generate.path}, which is outside of the bundle`);
171
185
  }
172
186
  }
173
- const ceiling = policy.network;
174
- if (ceiling && (0, work_network_1.isLimitedNetwork)(ceiling)) {
175
- if (data.type === 'local-task') {
176
- problems.push(`${item.name}: is a local task, which can't be limited to the network this server allows (runs.network: ${(0, work_network_1.formatNetwork)(ceiling)})`);
187
+ // what holds a credential gets at most `runs.secretNetwork`, so a tool it runs can't
188
+ // send the credential anywhere else
189
+ const reads = getCredentialReads({
190
+ secrets: secretsOf(item).map(describeSecret),
191
+ serverEnvs: 'envs' in data ? data.envs.processNames ?? [] : [],
192
+ });
193
+ const ceilings = [];
194
+ if (policy.network && (0, work_network_1.isLimitedNetwork)(policy.network)) {
195
+ ceilings.push({ ceiling: policy.network, setting: 'runs.network', reason: '' });
196
+ }
197
+ if (reads.length > 0 && policy.secretNetwork && (0, work_network_1.isLimitedNetwork)(policy.secretNetwork)) {
198
+ ceilings.push({
199
+ ceiling: policy.secretNetwork,
200
+ setting: 'runs.secretNetwork',
201
+ reason: ` to what reads ${reads.join(', ')}`,
202
+ });
203
+ }
204
+ for (const { ceiling, setting, reason } of ceilings) {
205
+ // on a cluster a local task is a pod of a pool's tool image, limited like any other
206
+ if (data.type === 'local-task' && policy.backend !== 'kubernetes') {
207
+ problems.push(`${item.name}: is a local task, which can't be limited to the network this server allows${reason} ` +
208
+ `(${setting}: ${(0, work_network_1.formatNetwork)(ceiling)})`);
177
209
  }
178
210
  else if ('network' in data && (0, work_network_1.exceedsNetwork)(data.network, ceiling)) {
179
- problems.push(`${item.name}: asks for network ${(0, work_network_1.formatNetwork)(data.network)}, more than this server allows (runs.network: ${(0, work_network_1.formatNetwork)(ceiling)})`);
211
+ problems.push(`${item.name}: asks for network ${(0, work_network_1.formatNetwork)(data.network)}, more than this server allows${reason} ` +
212
+ `(${setting}: ${(0, work_network_1.formatNetwork)(ceiling)})`);
180
213
  }
181
214
  }
182
215
  if ((0, work_task_1.isContainerWorkTask)(data) || (0, work_service_1.isContainerWorkService)(data)) {
@@ -205,11 +238,13 @@ function findPolicyProblems(items, policy, root) {
205
238
  problems.push(`${item.name}: the secret ${describeSecret(secret).target} has cache: true, its digest would need ` +
206
239
  'the value before the Job of the task reads it, which this server does not support yet');
207
240
  }
208
- else if (item.data.network.type !== 'open') {
209
- // the init containers that read it share the pod's NetworkPolicy
241
+ else if ((0, work_network_1.isLimitedNetwork)(item.data.network) && !policy.proxiedProviders?.includes(source.provider)) {
242
+ // the init containers that read it share the pod's NetworkPolicy, and reach the
243
+ // provider only through the egress proxy
210
244
  problems.push(`${item.name}: the secret ${describeSecret(secret).target} is read from ${source.provider} in the ` +
211
- `Job of the task, which network ${(0, work_network_1.formatNetwork)(item.data.network)} would cut off from it, ` +
212
- 'a task that reads secrets of a provider keeps an open network on this server');
245
+ `Job of the task, which network ${(0, work_network_1.formatNetwork)(item.data.network)} would cut off from it: ` +
246
+ `this server names no endpoints of ${source.provider} in backend.providers.${source.provider}.allow ` +
247
+ 'for the egress proxy to let through, so the task keeps an open network');
213
248
  }
214
249
  }
215
250
  if (source.type === 'provider' && policy.reserved?.includes(source.account)) {
@@ -254,6 +289,7 @@ function toPlan(root, item) {
254
289
  cmds: 'cmds' in data ? data.cmds.map((cmd) => cmd.cmd) : 'cmd' in data && data.cmd ? [data.cmd.cmd] : [],
255
290
  command: (0, task_arguments_1.getArgumentsTarget)(data)?.cmd ?? null,
256
291
  network: (0, work_network_1.toNetworkSchema)('network' in data ? data.network : { type: 'open' }),
292
+ ...('args' in data && data.args !== null ? { args: data.args } : {}),
257
293
  deps: item.deps.map((dep) => dep.name).sort(),
258
294
  needs: item.needs.map((need) => need.name).sort(),
259
295
  accounts,
@@ -46,26 +46,16 @@ async function planRun(bundle, run, environment, policy) {
46
46
  };
47
47
  }
48
48
  exports.planRun = planRun;
49
- // Variables of a server that hold no credential (FR-008): a task may read these and
50
- // still take arguments.
51
- const PLAIN_SERVER_ENVS = ['PATH', 'HOME', 'CI'];
52
- function readsOf(entry) {
53
- return [
54
- ...entry.secrets.map((secret) => `the secret ${secret.target}`),
55
- ...(entry.serverEnvs ?? [])
56
- .filter((env) => !PLAIN_SERVER_ENVS.includes(env.toUpperCase()))
57
- .map((env) => `$${env} of the server`),
58
- ];
59
- }
60
49
  /**
61
50
  * Why a run may not pass its arguments, or null when it may. They go to a task, never a
62
51
  * service, and never to a run in which an item reads a secret or takes a variable of the
63
52
  * server into its `envs`: an argument can make the command print what the task holds,
64
53
  * and what its dependencies write into their outputs or its services serve, past the
65
54
  * masking of secrets. So a shell on a server runs with no account and no credential of
66
- * the server.
55
+ * the server. Under `runs.args: declared` they go only to a task that says `args: true`,
56
+ * so a task nobody meant to take them never does.
67
57
  */
68
- function refuseArguments(run, plan) {
58
+ function refuseArguments(run, plan, policy = {}) {
69
59
  if (!run.options.args || run.options.args.length === 0) {
70
60
  return null;
71
61
  }
@@ -75,14 +65,21 @@ function refuseArguments(run, plan) {
75
65
  if (!entry || entry.kind !== 'task') {
76
66
  return `${name} is not a task of this version`;
77
67
  }
78
- const reads = readsOf(entry);
68
+ // planning the run refuses it before, this keeps the check with the others
69
+ if (entry.args === false) {
70
+ return `${name} takes no arguments (args: false)`;
71
+ }
72
+ if (policy.args === 'declared' && entry.args !== true) {
73
+ return `${name} takes no arguments on this server: it takes them only for tasks that say args: true (runs.args: declared)`;
74
+ }
75
+ const reads = (0, plan_1.getCredentialReads)(entry);
79
76
  if (reads.length > 0) {
80
77
  return `${name} reads ${reads.join(', ')}, it takes no arguments on a server: an argument could make it print them`;
81
78
  }
82
79
  // what a dependency or a service of the run holds reaches the task through its outputs or
83
80
  // its answers, where an argument can print it in a form no mask knows (`rev token.txt`)
84
81
  for (const [other, item] of Object.entries(plan).sort(([a], [b]) => a.localeCompare(b))) {
85
- const held = (other === name ? [] : readsOf(item)).join(', ');
82
+ const held = (other === name ? [] : (0, plan_1.getCredentialReads)(item)).join(', ');
86
83
  if (held.length > 0) {
87
84
  return `${name} takes no arguments on a server: ${other} in its run reads ${held}, and an argument could make ${name} print what ${other} hands on`;
88
85
  }
@@ -5,6 +5,7 @@ const commander_1 = require("commander");
5
5
  const log_1 = require("../log");
6
6
  const server_config_1 = require("./server-config");
7
7
  const server_1 = require("./server");
8
+ const abort_on_signals_1 = require("../utils/abort-on-signals");
8
9
  // Runs this machine as a hammerkit server until interrupted. Like `remote`, it
9
10
  // only needs its own config file, not a build file.
10
11
  function registerServerCommand(program, environment) {
@@ -13,31 +14,46 @@ function registerServerCommand(program, environment) {
13
14
  .description('serve this machine to remote runs (experimental)')
14
15
  .addOption(new commander_1.Option('-c, --config <file>', 'server config file').env('HAMMERKIT_SERVER_CONFIG'))
15
16
  .action(async (options) => {
17
+ // SIGHUP reads the entitlements again, so who may do what changes without a
18
+ // restart; the rest of the config needs one. The server takes it over from the
19
+ // abort of the CLI, which stops other commands on it, before it starts: a SIGHUP
20
+ // while it reads its config and starts waits for it. Windows sends SIGHUP when the
21
+ // console closes, which stops the server there too.
22
+ let server;
23
+ let reloadWhenStarted = false;
24
+ const reload = async (running) => {
25
+ try {
26
+ const result = running.reload(await (0, server_config_1.readServerConfig)(environment, options.config));
27
+ if (result.restart.length > 0) {
28
+ environment.stdout.write(`changed but not reloaded, restart for: ${result.restart.join(', ')}\n`);
29
+ }
30
+ }
31
+ catch (e) {
32
+ environment.stdout.write(`entitlements were not reloaded: ${(0, log_1.getErrorMessage)(e)}\n`);
33
+ }
34
+ };
35
+ const releaseReload = process.platform !== 'win32'
36
+ ? (0, abort_on_signals_1.takeOverSignal)(process, 'SIGHUP', () => {
37
+ if (server) {
38
+ void reload(server);
39
+ }
40
+ else {
41
+ reloadWhenStarted = true;
42
+ }
43
+ })
44
+ : () => undefined;
16
45
  try {
17
46
  if (!options.config) {
18
47
  throw new Error('pass the server config with --config or HAMMERKIT_SERVER_CONFIG');
19
48
  }
20
49
  const config = await (0, server_config_1.readServerConfig)(environment, options.config);
21
- const server = await (0, server_1.startServer)(config, environment, {
50
+ const running = await (0, server_1.startServer)(config, environment, {
22
51
  log: (line) => environment.stdout.write(`${line}\n`),
23
52
  });
24
- environment.stdout.write(`hammerkit server listening on ${server.url}\n`);
25
- // SIGHUP reads the entitlements again, so who may do what changes without
26
- // a restart; the rest of the config needs one
27
- const reload = async () => {
28
- try {
29
- const result = server.reload(await (0, server_config_1.readServerConfig)(environment, options.config));
30
- if (result.restart.length > 0) {
31
- environment.stdout.write(`changed but not reloaded, restart for: ${result.restart.join(', ')}\n`);
32
- }
33
- }
34
- catch (e) {
35
- environment.stdout.write(`entitlements were not reloaded: ${(0, log_1.getErrorMessage)(e)}\n`);
36
- }
37
- };
38
- const onReload = () => void reload();
39
- if (process.platform !== 'win32') {
40
- process.on('SIGHUP', onReload);
53
+ server = running;
54
+ environment.stdout.write(`hammerkit server listening on ${running.url}\n`);
55
+ if (reloadWhenStarted) {
56
+ void reload(running);
41
57
  }
42
58
  await new Promise((resolve) => {
43
59
  if (environment.abortCtrl.signal.aborted) {
@@ -45,8 +61,8 @@ function registerServerCommand(program, environment) {
45
61
  }
46
62
  environment.abortCtrl.signal.addEventListener('abort', () => resolve(), { once: true });
47
63
  });
48
- process.off('SIGHUP', onReload);
49
- await server.close();
64
+ releaseReload();
65
+ await running.close();
50
66
  environment.stdout.write('hammerkit server stopped\n');
51
67
  }
52
68
  catch (e) {
@@ -55,6 +71,9 @@ function registerServerCommand(program, environment) {
55
71
  }
56
72
  program.error(`Server was not successful: ${(0, log_1.getErrorMessage)(e)}`, { exitCode: 1 });
57
73
  }
74
+ finally {
75
+ releaseReload();
76
+ }
58
77
  });
59
78
  }
60
79
  exports.registerServerCommand = registerServerCommand;
@@ -238,8 +238,22 @@ const kubernetesBackendSchema = zod_1.z
238
238
  .min(1)
239
239
  .default([{ name: 'default', platform: { os: 'linux', arch: 'amd64' }, nodeSelector: {}, tolerations: [] }]),
240
240
  // per secret provider, the image its command runs in: secrets are read in the
241
- // Job of the task, not by the server
242
- providers: zod_1.z.record(zod_1.z.object({ image: zod_1.z.string().min(1) }).strict()).default({}),
241
+ // Job of the task, not by the server. `allow` are the hosts its command reaches, such
242
+ // as secretmanager.googleapis.com: the egress proxy of a task with a limited network
243
+ // lets the init containers that read its secrets reach them, and nothing else does
244
+ providers: zod_1.z
245
+ .record(zod_1.z
246
+ .object({
247
+ image: zod_1.z.string().min(1),
248
+ allow: zod_1.z
249
+ .array(zod_1.z
250
+ .string()
251
+ .refine((host) => (0, network_schema_1.isAllowEntry)(host) && !host.includes(':'), 'expected a host name such as secretmanager.googleapis.com or *.example.com'))
252
+ .min(1)
253
+ .optional(),
254
+ })
255
+ .strict())
256
+ .default({}),
243
257
  // the ResourceQuota of every caller's namespace, such as `requests.cpu: '8'`
244
258
  quota: zod_1.z.record(zod_1.z.union([zod_1.z.string().min(1), zod_1.z.number()]).transform(String)).optional(),
245
259
  // what a container without `resources` requests (a LimitRange), so it is
@@ -451,6 +465,13 @@ exports.serverConfigSchema = zod_1.z
451
465
  // the most any task or service of a run reaches besides its needs: none, open
452
466
  // or { allow: [hosts] }. A version that asks for more is refused at publish.
453
467
  network: network_schema_1.networkSchema.default('open'),
468
+ // the most a task or service that reads a secret, or a variable of the server
469
+ // other than PATH, HOME and CI, reaches, under `network`: so a tool it runs can't
470
+ // send the value anywhere. Checked like `network`.
471
+ secretNetwork: network_schema_1.networkSchema.default('open'),
472
+ // which tasks take arguments: `any` but those that say `args: false`, or
473
+ // `declared`, only those that say `args: true`
474
+ args: zod_1.z.enum(['any', 'declared']).default('any'),
454
475
  })
455
476
  .strict()
456
477
  .default({}),
@@ -586,7 +607,10 @@ exports.serverConfigSchema = zod_1.z
586
607
  message: `under runs.network, ${shared}`,
587
608
  });
588
609
  }
589
- if (config.sandboxes.runtimes?.includes('local') && config.sandboxes.network !== 'open') {
610
+ // on a cluster a local task runs in a pod of a pool's tool image, which can be limited
611
+ if (config.sandboxes.runtimes?.includes('local') &&
612
+ config.sandboxes.network !== 'open' &&
613
+ config.backend.type === 'host') {
590
614
  ctx.addIssue({
591
615
  code: zod_1.z.ZodIssueCode.custom,
592
616
  path: ['sandboxes', 'network'],
@@ -697,6 +721,13 @@ exports.serverConfigSchema = zod_1.z
697
721
  message: 'runs.network limits every task with a NetworkPolicy, set networkPolicy: enforced once the network plugin of the cluster enforces them',
698
722
  });
699
723
  }
724
+ if (config.runs.secretNetwork !== 'open' && backend.networkPolicy !== 'enforced') {
725
+ ctx.addIssue({
726
+ code: zod_1.z.ZodIssueCode.custom,
727
+ path: ['backend', 'networkPolicy'],
728
+ message: 'runs.secretNetwork limits the tasks that read secrets with a NetworkPolicy, set networkPolicy: enforced once the network plugin of the cluster enforces them',
729
+ });
730
+ }
700
731
  if (config.sandboxes.network !== 'open' &&
701
732
  backend.networkPolicy !== 'enforced' &&
702
733
  config.entitlements.some((entitlement) => entitlement.sandboxes?.length)) {
@@ -707,7 +738,7 @@ exports.serverConfigSchema = zod_1.z
707
738
  });
708
739
  }
709
740
  }
710
- if (config.runs.network !== 'open' && config.runtimes.includes('local')) {
741
+ if (config.runs.network !== 'open' && config.runtimes.includes('local') && backend.type === 'host') {
711
742
  ctx.addIssue({
712
743
  code: zod_1.z.ZodIssueCode.custom,
713
744
  path: ['runs', 'network'],
@@ -275,7 +275,7 @@ async function submitRun(config, registry, definitions, sources, sandboxes, plan
275
275
  (0, http_2.send)(res, 400, { error: `the run does not load: ${e instanceof Error ? e.message : String(e)}` });
276
276
  return;
277
277
  }
278
- const refused = (0, run_plan_1.refuseArguments)(request, planned.plan);
278
+ const refused = (0, run_plan_1.refuseArguments)(request, planned.plan, sandbox ? sandboxes.policy : planning.policy);
279
279
  if (refused) {
280
280
  log(`refuse run iss=${principal.iss} sub=${principal.sub}: ${refused}`);
281
281
  (0, http_2.send)(res, 400, { error: refused });
@@ -442,7 +442,8 @@ async function startServer(config, environment, options = {}) {
442
442
  environment: {
443
443
  ...environment,
444
444
  processEnvs: (0, run_environment_1.runEnvironment)(config, environment.processEnvs),
445
- hosted: { catalog: (0, server_config_1.getSecretCatalog)(config) },
445
+ // on a cluster a local task runs in a pod of a pool's tool image
446
+ hosted: { catalog: (0, server_config_1.getSecretCatalog)(config), localTasksInContainers: config.backend.type === 'kubernetes' },
446
447
  },
447
448
  policy: {
448
449
  envs: config.runs.env.allow,
@@ -450,6 +451,13 @@ async function startServer(config, environment, options = {}) {
450
451
  runtimes: config.runtimes,
451
452
  backend: config.backend.type,
452
453
  network: (0, work_network_1.parseWorkNetwork)(config.runs.network),
454
+ secretNetwork: (0, work_network_1.parseWorkNetwork)(config.runs.secretNetwork),
455
+ args: config.runs.args,
456
+ proxiedProviders: config.backend.type === 'kubernetes'
457
+ ? Object.entries(config.backend.providers)
458
+ .filter(([, provider]) => provider.allow)
459
+ .map(([name]) => name)
460
+ : [],
453
461
  },
454
462
  };
455
463
  const storage = config.storage.directory ?? (0, path_1.join)((0, get_cache_directory_1.getHammerkitDirectory)(), 'server');
@@ -1,17 +1,57 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.abortOnSignals = exports.ABORT_SIGNALS = void 0;
3
+ exports.takeOverSignal = exports.abortOnSignals = exports.ABORT_SIGNALS = void 0;
4
4
  // Ctrl-C (SIGINT), a CI job being cancelled or a supervisor stopping hammerkit
5
5
  // (SIGTERM) and a closed terminal (SIGHUP) all stop the run the same way: abort,
6
6
  // so services are removed and local tasks' process groups are killed. Without a
7
7
  // handler Node exits on SIGTERM/SIGHUP at once and leaves both behind.
8
8
  exports.ABORT_SIGNALS = ['SIGINT', 'SIGTERM', 'SIGHUP'];
9
- function abortOnSignals(source, abortCtrl) {
10
- for (const signal of exports.ABORT_SIGNALS) {
11
- source.on(signal, () => {
12
- abortCtrl.abort();
13
- });
9
+ // signals a running command gives a meaning of its own (the server reloads its
10
+ // entitlements on SIGHUP), per source and counted by how many hold them: they don't
11
+ // abort the run
12
+ const takenOver = new WeakMap();
13
+ function heldOn(source) {
14
+ let held = takenOver.get(source);
15
+ if (!held) {
16
+ held = new Map();
17
+ takenOver.set(source, held);
14
18
  }
19
+ return held;
20
+ }
21
+ /**
22
+ * Stops the run on the abort signals, all but those a command took over with
23
+ * `takeOverSignal`. Returns what removes the listeners again.
24
+ */
25
+ function abortOnSignals(source, abortCtrl) {
26
+ const listeners = exports.ABORT_SIGNALS.map((signal) => {
27
+ const listener = () => {
28
+ if (!takenOver.get(source)?.get(signal)) {
29
+ abortCtrl.abort();
30
+ }
31
+ };
32
+ source.on(signal, listener);
33
+ return { signal, listener };
34
+ });
35
+ return () => listeners.forEach(({ signal, listener }) => source.off(signal, listener));
15
36
  }
16
37
  exports.abortOnSignals = abortOnSignals;
38
+ /**
39
+ * Calls `handler` on `signal` instead of stopping the run, until the returned function
40
+ * gives the signal back.
41
+ */
42
+ function takeOverSignal(source, signal, handler) {
43
+ const held = heldOn(source);
44
+ held.set(signal, (held.get(signal) ?? 0) + 1);
45
+ source.on(signal, handler);
46
+ let released = false;
47
+ return () => {
48
+ if (released) {
49
+ return;
50
+ }
51
+ released = true;
52
+ source.off(signal, handler);
53
+ held.set(signal, (held.get(signal) ?? 1) - 1);
54
+ };
55
+ }
56
+ exports.takeOverSignal = takeOverSignal;
17
57
  //# sourceMappingURL=abort-on-signals.js.map
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "hammerkit",
3
3
  "description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
4
- "version": "1.27.0-next.3",
4
+ "version": "1.27.0-next.5",
5
5
  "keywords": [
6
6
  "build",
7
7
  "build-tool",