hammerkit 1.27.0-next.4 → 1.27.0-next.6

package/build.schema.json

@@ -289,6 +289,9 @@
                     ],
                     "description": "network the task or service reaches besides its needs: open (default), none or { allow: [hosts or host:port] }"
                   },
+                  "args": {
+                    "type": "boolean"
+                  },
                   "runner": {
                     "type": "string",
                     "minLength": 1
@@ -370,6 +373,9 @@
                   "network": {
                     "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/network"
                   },
+                  "args": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/args"
+                  },
                   "runner": {
                     "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/runner"
                   },
@@ -511,6 +517,9 @@
                   "network": {
                     "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/network",
                     "description": "network the task or service reaches besides its needs: open (default), none or { allow: [hosts or host:port] }"
+                  },
+                  "args": {
+                    "type": "boolean"
                   }
                 },
                 "required": [

package/dist/executer/event-cache.js

@@ -8,6 +8,8 @@ const execute_docker_1 = require("./execute-docker");
 const plan_work_tasks_1 = require("../planner/utils/plan-work-tasks");
 const get_cache_directory_1 = require("../optimizer/get-cache-directory");
 const path_1 = require("path");
+const crypto_1 = require("crypto");
+const claim_layout_1 = require("../kubernetes/claim-layout");
 const move_files_1 = require("../file/move-files");
 const work_instance_id_1 = require("../planner/work-instance-id");
 async function restoreCache(environment, path, workTree) {
@@ -43,8 +45,24 @@ function* getArchivePaths(task, path) {
         }
     }
     else {
-        for (const generate of task.generates.filter((g) => !g.inherited)) {
-            const generateId = (0, path_1.relative)(task.cwd, generate.path).split(path_1.sep).join('-');
+        // `a-b` and `a/b` flatten alike: of those, the least deep (then the first in sorted
+        // order) keeps the name, the others' names carry a hash of their relative path, so a
+        // store and a restore in another directory or order find the same archive
+        const generates = task.generates
+            .filter((g) => !g.inherited)
+            .map((generate) => ({ generate, rel: (0, path_1.relative)(task.cwd, generate.path).split(path_1.sep).join(path_1.posix.sep) }));
+        const holders = new Map();
+        for (const { rel } of [...generates].sort((a, b) => (0, claim_layout_1.byDepth)(a.rel, b.rel))) {
+            const id = rel.split(path_1.posix.sep).join('-');
+            if (!holders.has(id)) {
+                holders.set(id, rel);
+            }
+        }
+        for (const { generate, rel } of generates) {
+            let generateId = rel.split(path_1.posix.sep).join('-');
+            if (holders.get(generateId) !== rel) {
+                generateId += '-' + (0, crypto_1.createHash)('sha256').update(rel).digest('hex').slice(0, 12);
+            }
             yield {
                 filename: (0, path_1.join)(path, generateId + '-generates.tgz'),
                 path: generate.path,

package/dist/kubernetes/claim-layout.js

@@ -0,0 +1,71 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.hasOutputsFlatteningAlike = exports.hasPathsEndingAlike = exports.byDepth = exports.getClaimSubPath = void 0;
+const crypto_1 = require("crypto");
+const path_1 = require("path");
+const portable_path_1 = require("../planner/utils/portable-path");
+// The paths a work item keeps in its claim. Its consumers mount its outputs by the same
+// paths, so they find the same folders.
+function getClaimPaths(work) {
+    const data = work.data;
+    const paths = [];
+    if ('mounts' in data) {
+        paths.push(...data.mounts.map((mount) => mount.containerPath));
+    }
+    if ('src' in data) {
+        paths.push(...data.src.filter((src) => !src.inherited).map((src) => src.absolutePath));
+    }
+    if ('generates' in data) {
+        paths.push(...data.generates.filter((generate) => !generate.inherited).map((generate) => generate.path));
+    }
+    if ('volumes' in data) {
+        paths.push(...data.volumes.filter((volume) => !volume.inherited).map((volume) => volume.containerPath));
+    }
+    return paths;
+}
+// Where `path` is in the claim of `work`, relative to its root. A path is named after its
+// last segment, so claims without two such paths are laid out as they always were (a
+// service's data among them). Of the paths with the same last segment, the least deep one
+// (then the first in sorted order) keeps the name, and the others also carry a hash of
+// their path. Both go by the path
+// relative to the project, as the task's id does, so a run in another checkout directory
+// (each run of a server has its own) or with its `generates` in another order finds the
+// same folders in a kept claim.
+function getClaimSubPath(work, path) {
+    const name = (0, path_1.basename)(path) || 'root';
+    const alike = getClaimPaths(work).filter((other) => other !== path && (0, path_1.basename)(other) === (0, path_1.basename)(path));
+    if (alike.length === 0) {
+        return name;
+    }
+    const own = (0, portable_path_1.portablePath)(work.data.projectRoot, path);
+    const first = [own, ...alike.map((other) => (0, portable_path_1.portablePath)(work.data.projectRoot, other))].sort(byDepth)[0];
+    if (first === own) {
+        return name;
+    }
+    return `${name}-${(0, crypto_1.createHash)('sha256').update(own).digest('hex').slice(0, 12)}`;
+}
+exports.getClaimSubPath = getClaimSubPath;
+function byDepth(a, b) {
+    return a.split('/').length - b.split('/').length || (a < b ? -1 : a > b ? 1 : 0);
+}
+exports.byDepth = byDepth;
+// Whether two different paths of the claim of `work` end alike. Before getClaimSubPath
+// named them apart, they shared one folder, and outputs made then are mixed.
+function hasPathsEndingAlike(work) {
+    const paths = new Set(getClaimPaths(work));
+    return new Set([...paths].map((path) => (0, path_1.basename)(path))).size < paths.size;
+}
+exports.hasPathsEndingAlike = hasPathsEndingAlike;
+// Whether two outputs of `work` flatten to the same name of a cache archive (`a-b` and
+// `a/b`, see getArchivePaths). Before they were named apart, one archive held both.
+function hasOutputsFlatteningAlike(work) {
+    const data = work.data;
+    if (!('generates' in data) || !('cwd' in data)) {
+        return false;
+    }
+    const paths = new Set(data.generates.filter((generate) => !generate.inherited).map((generate) => generate.path));
+    const names = new Set([...paths].map((path) => (0, path_1.relative)(data.cwd, path).split(path_1.sep).join('-')));
+    return names.size < paths.size;
+}
+exports.hasOutputsFlatteningAlike = hasOutputsFlatteningAlike;
+//# sourceMappingURL=claim-layout.js.map

package/dist/kubernetes/ensure-persistent-data.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ensurePersistentData = exports.clearClaim = exports.getPodForPersistence = void 0;
+exports.ensurePersistentData = exports.packSource = exports.clearClaim = exports.getPodForPersistence = void 0;
 const apply_1 = require("./apply");
 const path_1 = require("path");
 const tar_1 = require("tar");
@@ -137,6 +137,25 @@ async function clearClaim(instance, env, work, pod) {
     });
 }
 exports.clearClaim = clearClaim;
+// The archive that uploads a source into the claim: its file or directory, at its place in
+// the claim (see getClaimSubPath) instead of under its own name
+function packSource(source) {
+    const fileItem = (0, path_1.basename)(source.localPath);
+    return stream_1.Readable.from((0, tar_1.create)({
+        cwd: (0, path_1.dirname)(source.localPath),
+        filter(path) {
+            return source.matcher(path, source.localPath);
+        },
+        onWriteEntry(entry) {
+            entry.path = source.subPath + entry.path.slice(fileItem.length);
+            // the second of two hard-linked files links to the first by its path in the archive
+            if (entry.type === 'Link' && entry.linkpath?.startsWith(fileItem)) {
+                entry.linkpath = source.subPath + entry.linkpath.slice(fileItem.length);
+            }
+        },
+    }, [fileItem]));
+}
+exports.packSource = packSource;
 async function ensurePersistentData(instance, env, environment, service, persistence, 
 // the kept claim of the work holds what an earlier run left, which goes first
 options = {}) {
@@ -163,13 +182,7 @@ options = {}) {
             if (!(await environment.file.exists(source.localPath))) {
                 continue;
             }
-            const fileItem = (0, path_1.basename)(source.localPath);
-            const data = stream_1.Readable.from((0, tar_1.create)({
-                cwd: (0, path_1.dirname)(source.localPath),
-                filter(path) {
-                    return source.matcher(path, source.localPath);
-                },
-            }, [fileItem]));
+            const data = packSource(source);
             service.status.console('stdout', `upload ${source.localPath}`);
             const targetPath = `/dev/hammerkit/${service.id()}`;
             const res = await instance.exec.exec(env.namespace, name, 'volume', ['tar', 'xf', '-', '-C', targetPath], process.stdout, process.stderr, data, false, (status) => {

package/dist/kubernetes/job-secrets.js

@@ -1,8 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getJobSecretSpec = exports.removeJobCredentials = exports.assertJobAccounts = exports.ensureJobCredentials = exports.getCredentialsName = exports.assertSecretsReachable = exports.splitJobSecrets = exports.ACCOUNT_PATH = exports.SECRET_VALUES_PATH = void 0;
+exports.getJobSecretSpec = exports.removeJobCredentials = exports.assertJobAccounts = exports.ensureJobCredentials = exports.getCredentialsName = exports.getSecretHosts = exports.splitJobSecrets = exports.ACCOUNT_PATH = exports.SECRET_VALUES_PATH = void 0;
 const work_secret_1 = require("../planner/work-secret");
 const command_secret_provider_1 = require("../secrets/command-secret-provider");
+const task_arguments_1 = require("../planner/utils/task-arguments");
 const version_1 = require("../version");
 const apply_1 = require("./apply");
 const resources_1 = require("./resources");
@@ -42,19 +43,24 @@ function splitJobSecrets(secrets, server) {
 exports.splitJobSecrets = splitJobSecrets;
 /**
  * The init containers that read secrets share the pod, and so the NetworkPolicy, of
- * the task: under a limited network they could not reach the secret manager. Until
- * they run in a pod of their own, a task that reads a provider's secret in its Job
- * keeps an open network.
+ * the task: under a limited network they reach the secret manager only through the
+ * task's egress proxy, which lets them reach the endpoints the server names for the
+ * provider (`backend.providers.<name>.allow`). A task that reads a secret of a provider
+ * without them keeps an open network. Answers the endpoints the proxy lets through.
  */
-function assertSecretsReachable(task, secrets, network = (0, work_network_1.getEffectiveNetwork)(task.data)) {
-    if (secrets.length === 0 || network.type === 'open') {
-        return;
+function getSecretHosts(task, secrets, server, network = (0, work_network_1.getEffectiveNetwork)(task.data)) {
+    if (!server || secrets.length === 0 || network.type === 'open') {
+        return [];
+    }
+    const unreachable = secrets.find((secret) => !server.providers[secret.source.provider]?.allow?.length);
+    if (unreachable) {
+        throw new Error(`task ${task.name}: secret ${(0, work_secret_1.getSecretName)(unreachable.secret)} is read from ${(0, work_secret_1.describeSecretSource)(unreachable.secret)} in the Job of the task, which network ${(0, work_network_1.formatNetwork)(network)} would cut off from it; ` +
+            `on this cluster a task that reads secrets of ${unreachable.source.provider} keeps an open network, ` +
+            `unless the server names its endpoints in backend.providers.${unreachable.source.provider}.allow`);
     }
-    const secret = secrets[0];
-    throw new Error(`task ${task.name}: secret ${(0, work_secret_1.getSecretName)(secret.secret)} is read from ${(0, work_secret_1.describeSecretSource)(secret.secret)} in the Job of the task, which network ${(0, work_network_1.formatNetwork)(network)} would cut off from it; ` +
-        'on this cluster a task that reads secrets of a provider keeps an open network');
+    return [...new Set(secrets.flatMap((secret) => server.providers[secret.source.provider].allow ?? []))].sort();
 }
-exports.assertSecretsReachable = assertSecretsReachable;
+exports.getSecretHosts = getSecretHosts;
 // The accounts of the secrets, in the order they are first read
 function accountsOf(secrets) {
     return [...new Set(secrets.map((s) => s.source.account))];
@@ -186,7 +192,9 @@ function quote(value) {
  * from it, and the task's output is masked before it leaves the pod, since the
  * server, which masks everything else, never sees these values.
  */
-function getJobSecretSpec(item, secrets, server, credentials, environment) {
+function getJobSecretSpec(item, secrets, server, credentials, environment, 
+// under a limited network, what sends their requests through the egress proxy
+proxyEnv = []) {
     if (!server || secrets.length === 0) {
         return {
             initContainers: [],
@@ -229,8 +237,11 @@ function getJobSecretSpec(item, secrets, server, credentials, environment) {
                 '-c',
                 'umask 022 && "$@" > "$HAMMERKIT_VALUE.part" && mv "$HAMMERKIT_VALUE.part" "$HAMMERKIT_VALUE"',
             ],
-            args: ['sh', command, ...args],
+            // Kubernetes would expand a `$(NAME)` of the container's env in them, the proxy's token
+            // among them: the reference is text, as on any other backend
+            args: ['sh', command, ...args].map(task_arguments_1.escapeKubernetesText),
             env: [
+                ...proxyEnv,
                 ...Object.entries(env).map(([name, value]) => ({ name, value })),
                 { name: 'HAMMERKIT_VALUE', value: `${exports.SECRET_VALUES_PATH}/secret-${index}` },
             ],

package/dist/kubernetes/network-policy.js

@@ -23,6 +23,9 @@ const NETWORK_OF_LABEL = 'hammerkit.dev/network-of';
 // on the proxy of a task on a sandbox: the sandbox, whose pod of files it stays with
 const PROXY_OF_SANDBOX_LABEL = 'hammerkit.dev/proxy-of-sandbox';
 const PROXY_READY_TIMEOUT_MS = 60000;
+// the token of the endpoints of secret providers, in a Secret named like the proxy
+const SECRET_TOKEN_KEY = 'token';
+const SECRET_TOKEN_VARIABLE = 'HAMMERKIT_PROXY_TOKEN';
 // how long the pods of an item may take to end before their policy goes
 const POD_END_TIMEOUT_MS = 120000;
 const POD_END_POLL_MS = 500;
@@ -109,10 +112,10 @@ function getIngress(env, item) {
     return [{ _from: [{ podSelector: from }] }];
 }
 // The NetworkPolicy of a limited item: egress to the pods of the services it
-// needs and, under `allow`, to its proxy; nothing else, DNS included (needs
-// and the proxy are host aliases). Ingress only from what needs it. An open
-// item has none.
-function buildNetworkPolicy(env, item, network, scope, proxy = getRunProxy(item, scope)) {
+// needs and, under `allow` or with secrets read through it, to its proxy;
+// nothing else, DNS included (needs and the proxy are host aliases). Ingress
+// only from what needs it. An open item has none.
+function buildNetworkPolicy(env, item, network, scope, proxy = getRunProxy(item, scope), proxied = network.type === 'allow') {
     const needs = item.needs.filter((need) => (0, work_item_1.isContainerWorkServiceItem)(need.service));
     return {
         apiVersion: 'networking.k8s.io/v1',
@@ -131,7 +134,7 @@ function buildNetworkPolicy(env, item, network, scope, proxy = getRunProxy(item,
                 ...(needs.length > 0
                     ? [{ to: needs.map((need) => ({ podSelector: { matchLabels: { [ID_LABEL]: need.service.id() } } })) }]
                     : []),
-                ...(network.type === 'allow'
+                ...(proxied
                     ? [
                         {
                             to: [{ podSelector: { matchLabels: { [PROXY_LABEL]: proxy.key } } }],
@@ -179,6 +182,9 @@ exports.buildProxyPolicy = buildProxyPolicy;
 // left goes.
 // A task follows the proxy's log while its jobs run. A service outlives the
 // run on a cluster, so its refused hosts are in the proxy's own log.
+// `secretHosts` are the endpoints of the providers whose secrets the init
+// containers of a task on a server read: a limited task gets a proxy for them
+// (under `none` too), which passes them to those containers only.
 async function ensureKubernetesItemNetwork(instance, env, item, options) {
     const network = (0, work_network_1.getEffectiveNetwork)(item.data);
     if (network.type !== 'open' && !env.networkPolicyEnforced) {
@@ -191,22 +197,30 @@ async function ensureKubernetesItemNetwork(instance, env, item, options) {
             // without the rights to manage NetworkPolicies, hammerkit never made one
             await removeKubernetesItemNetwork(instance, env, item, { run: null }, { ignoreForbidden: true });
         }
-        return { env: [], hostAliases, initContainers: [], close() { } };
+        return { env: [], secretEnv: [], hostAliases, initContainers: [], close() { } };
+    }
+    const secretHosts = options.secretHosts ?? [];
+    if (secretHosts.length > 0 && options.sandbox) {
+        // a run on a sandbox uses no account, so it reads no secret of a provider
+        throw new Error(`${item.name}: a task on a sandbox reads no secret of a provider`);
     }
     const proxy = network.type === 'allow' && options.sandbox
         ? getSandboxProxy(env, item, network, hostAliases, options.sandbox)
         : getRunProxy(item, options);
-    await (0, apply_1.apply)(instance, buildNetworkPolicy(env, item, network, options, proxy));
+    const proxied = network.type === 'allow' || secretHosts.length > 0;
+    await (0, apply_1.apply)(instance, buildNetworkPolicy(env, item, network, options, proxy, proxied));
     const initContainers = [(0, network_gate_1.getNetworkGateContainer)()];
-    if (network.type === 'none') {
+    if (!proxied) {
         if (options.sandbox) {
             await removeSandboxProxies(instance, env.namespace, { sandbox: options.sandbox.label, item });
         }
         else {
             await removeProxy(instance, env.namespace, proxy.name, [404]);
         }
-        return { env: [], hostAliases, initContainers, close() { } };
+        return { env: [], secretEnv: [], hostAliases, initContainers, close() { } };
     }
+    // under `none`, a proxy for the secrets only: it allows the item no host
+    const proxyNetwork = network.type === 'allow' ? network : { type: 'allow', hosts: [] };
     let started;
     try {
         if (proxy.sandbox) {
@@ -214,7 +228,7 @@ async function ensureKubernetesItemNetwork(instance, env, item, options) {
             await removeSandboxProxies(instance, env.namespace, { sandbox: proxy.sandbox.label, item }, (deployment) => deployment.metadata?.labels?.[PROXY_LABEL] === proxy.key);
             proxy.owner = await getFilesPodOwner(instance, env, proxy.sandbox);
         }
-        started = await ensureProxy(instance, env, item, network, hostAliases, options, proxy);
+        started = await ensureProxy(instance, env, item, proxyNetwork, hostAliases, options, proxy, secretHosts);
     }
     catch (e) {
         await removeKubernetesItemNetwork(instance, env, item, options).catch(() => undefined);
@@ -223,11 +237,17 @@ async function ensureKubernetesItemNetwork(instance, env, item, options) {
     const follow = options.followLog
         ? await followProxyLog(instance, env, item, proxy, { tailLines: started.reused ? 0 : undefined })
         : null;
+    const needs = item.needs.map((need) => need.name);
+    const toEnv = (variables) => Object.entries(variables).map(([name, value]) => ({ name, value }));
     return {
-        env: Object.entries((0, egress_proxy_1.getProxyClientEnvironment)(item.needs.map((need) => need.name))).map(([name, value]) => ({
-            name,
-            value,
-        })),
+        env: network.type === 'allow' ? toEnv((0, egress_proxy_1.getProxyClientEnvironment)(needs)) : [],
+        // the token from the proxy's Secret, which the item's own container never mounts
+        secretEnv: secretHosts.length > 0
+            ? [
+                { name: SECRET_TOKEN_VARIABLE, valueFrom: { secretKeyRef: { name: proxy.name, key: SECRET_TOKEN_KEY } } },
+                ...toEnv((0, egress_proxy_1.getProxyClientEnvironment)(needs, `$(${SECRET_TOKEN_VARIABLE})`)),
+            ]
+            : [],
         // the item reaches each TCP forward under its host's name, at the proxy
         hostAliases: [
             ...hostAliases,
@@ -318,6 +338,7 @@ async function removeAllKubernetesItemNetworks(instance, env, item) {
         if (name) {
             await ignoreStatus(instance.appsApi.deleteNamespacedDeployment({ name, namespace: env.namespace }), [404]);
             await ignoreStatus(instance.coreApi.deleteNamespacedService({ name, namespace: env.namespace }), [404]);
+            await ignoreStatus(instance.coreApi.deleteNamespacedSecret({ name, namespace: env.namespace }), [404]);
         }
     }
 }
@@ -349,7 +370,7 @@ function getProxyReadinessScript(ports) {
         `.on('error', () => process.exit(1))`);
 }
 exports.getProxyReadinessScript = getProxyReadinessScript;
-async function ensureProxy(instance, env, item, network, hostAliases, scope, proxy) {
+async function ensureProxy(instance, env, item, network, hostAliases, scope, proxy, secretHosts = []) {
     const name = proxy.name;
     const labels = { [PROXY_LABEL]: proxy.key };
     const metadata = {
@@ -388,7 +409,17 @@ async function ensureProxy(instance, env, item, network, hostAliases, scope, pro
                             name: 'proxy',
                             image: egress_proxy_1.EGRESS_PROXY_IMAGE,
                             command: (0, egress_proxy_1.getEgressProxyCommand)(),
-                            env: Object.entries((0, egress_proxy_1.getItemProxyEnvironment)(item)).map(([key, value]) => ({ name: key, value })),
+                            env: [
+                                ...Object.entries({ ...(0, egress_proxy_1.getItemProxyEnvironment)(item), ...(0, egress_proxy_1.getSecretProxyEnvironment)(secretHosts) }).map(([key, value]) => ({ name: key, value })),
+                                ...(secretHosts.length > 0
+                                    ? [
+                                        {
+                                            name: 'HAMMERKIT_SECRET_TOKEN',
+                                            valueFrom: { secretKeyRef: { name, key: SECRET_TOKEN_KEY } },
+                                        },
+                                    ]
+                                    : []),
+                            ],
                             ports: (0, egress_proxy_1.getEgressProxyPorts)(network).map((port) => ({ containerPort: port })),
                             // in the pod, since its ingress policy may keep the kubelet out;
                             // ready once it listens on every port, its TCP forwards' too
@@ -422,6 +453,21 @@ async function ensureProxy(instance, env, item, network, hostAliases, scope, pro
         ? await instance.objectApi.read(deployment).then((current) => !current.metadata?.deletionTimestamp, (e) => ((0, apply_1.statusCodeOf)(e) === 404 ? false : Promise.reject(e)))
         : false;
     await (0, apply_1.apply)(instance, buildProxyPolicy(env, item, network, scope, proxy));
+    if (secretHosts.length > 0) {
+        // new with each run's proxy, and kept while it is there: its pod read it when it started.
+        // The init containers that read secrets get it, the task not.
+        const secret = {
+            apiVersion: 'v1',
+            kind: 'Secret',
+            metadata: { ...metadata, annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() } },
+            type: 'Opaque',
+            data: { [SECRET_TOKEN_KEY]: Buffer.from((0, crypto_1.randomBytes)(32).toString('hex')).toString('base64') },
+        };
+        const exists = await instance.objectApi.read(secret).then(() => true, (e) => ((0, apply_1.statusCodeOf)(e) === 404 ? false : Promise.reject(e)));
+        if (!exists) {
+            await instance.objectApi.create(secret);
+        }
+    }
     await (0, apply_1.apply)(instance, deployment);
     const created = await (0, apply_1.apply)(instance, service);
     await (0, await_running_state_1.awaitDeployRunningState)(instance, env, name, {
@@ -433,7 +479,7 @@ async function ensureProxy(instance, env, item, network, hostAliases, scope, pro
     if (!ip) {
         throw new Error(`the egress proxy of ${item.name} has no cluster ip`);
     }
-    item.status.write('debug', `${reused ? 'egress proxy of the sandbox' : 'egress proxy'} allows ${network.hosts.join(', ') || 'no host'}`);
+    item.status.write('debug', `${reused ? 'egress proxy of the sandbox' : 'egress proxy'} allows ${network.hosts.join(', ') || 'no host'}${secretHosts.length > 0 ? `, and ${secretHosts.join(', ')} to the reads of secrets` : ''}`);
     return { ip, reused };
 }
 // Refused hosts show up in the item's log, as on Docker; of a proxy from an
@@ -468,9 +514,13 @@ async function followProxyLog(instance, env, item, proxy, options) {
         return null;
     }
 }
-async function removeProxy(instance, namespace, name, ignored) {
+// The token of a proxy for secrets goes with it; a proxy of a sandbox has none.
+async function removeProxy(instance, namespace, name, ignored, options = { token: true }) {
     await ignoreStatus(instance.appsApi.deleteNamespacedDeployment({ name, namespace }), ignored);
     await ignoreStatus(instance.coreApi.deleteNamespacedService({ name, namespace }), ignored);
+    if (options.token) {
+        await ignoreStatus(instance.coreApi.deleteNamespacedSecret({ name, namespace }), ignored);
+    }
     await ignoreStatus(instance.objectApi.delete({
         apiVersion: 'networking.k8s.io/v1',
         kind: 'NetworkPolicy',
@@ -492,7 +542,7 @@ async function removeSandboxProxies(instance, namespace, of, keep = () => false)
     for (const deployment of deployments.items) {
         const name = deployment.metadata?.name;
         if (name && !keep(deployment)) {
-            await removeProxy(instance, namespace, name, [404]);
+            await removeProxy(instance, namespace, name, [404], { token: false });
         }
     }
 }

package/dist/kubernetes/restore-kubernetes-data.js

@@ -1,11 +1,17 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.restoreKubernetesData = void 0;
+exports.restoreKubernetesData = exports.getRestoreTarget = void 0;
 const event_cache_1 = require("../executer/event-cache");
 const log_1 = require("../log");
-const path_1 = require("path");
 const volumes_1 = require("./volumes");
 const ensure_persistent_data_1 = require("./ensure-persistent-data");
+const claim_layout_1 = require("./claim-layout");
+// Where the upload pod (which mounts the work's claim at /dev/hammerkit/<id>) unpacks the
+// archive of an output: the output's place in the claim, which its mount reads
+function getRestoreTarget(work, path) {
+    return `/dev/hammerkit/${work.id()}/${(0, claim_layout_1.getClaimSubPath)(work, path)}`;
+}
+exports.getRestoreTarget = getRestoreTarget;
 async function restoreKubernetesData(work, kubernetes, instance, environment, path, 
 // the kept claim of the work holds what an earlier run left, which goes first
 options = {}) {
@@ -18,7 +24,7 @@ options = {}) {
             if (!(await environment.file.exists(generatedArchive.filename))) {
                 continue;
             }
-            const targetPath = `/dev/hammerkit/${work.id()}/${(0, path_1.basename)(generatedArchive.path)}`;
+            const targetPath = getRestoreTarget(work, generatedArchive.path);
             // a claim made for this run (a server removes the claims of a run when it
             // ends) does not have the directory of the output yet
             let failure = null;

package/dist/kubernetes/volumes.js

@@ -4,6 +4,7 @@ exports.getSandboxAffinity = exports.SANDBOX_LABEL = exports.getWorkspacePodSpec
 const path_1 = require("path");
 const fs_1 = require("fs");
 const kept_claims_1 = require("./kept-claims");
+const claim_layout_1 = require("./claim-layout");
 function getVolumeName(service, kubernetes) {
     return (0, kept_claims_1.getClaimName)(service, kubernetes);
 }
@@ -83,6 +84,7 @@ async function getKubernetesPersistence(work, kubernetes) {
 exports.getKubernetesPersistence = getKubernetesPersistence;
 function appendVolume(sourceWork, persistence, containerPath, localPath, isFile, readOnly, kubernetes) {
     const volumeName = getVolumeName(sourceWork, kubernetes);
+    const subPath = (0, claim_layout_1.getClaimSubPath)(sourceWork, containerPath);
     const existVolume = persistence.volumes.some((v) => v.name === volumeName);
     if (!existVolume) {
         persistence.volumes.push({
@@ -101,7 +103,7 @@ function appendVolume(sourceWork, persistence, containerPath, localPath, isFile,
             readOnly,
             mountPath: containerPath,
             name: volumeName,
-            subPath: (0, path_1.basename)(containerPath),
+            subPath,
         },
         uploadPath: `/dev/hammerkit/${sourceWork.id()}`,
         cwd: isFile ? (0, path_1.dirname)(containerPath) : containerPath,
@@ -111,6 +113,7 @@ function appendVolume(sourceWork, persistence, containerPath, localPath, isFile,
             localPath: localPath,
             matcher: () => true, // TODO
             containerPath,
+            subPath,
             stateKey: '', // TODO
         });
     }

package/dist/network/egress-proxy.js

@@ -5,10 +5,12 @@
 // `CONNECT` tunnels to an allowed host pass, everything else gets a 403 and a
 // `refused` event on stdout. For each `host:port` entry it also listens on that
 // port and forwards raw TCP to that host, which the item reaches under the
-// host's name. The script is passed to `node -e`, so the proxy
+// host's name. On a server's cluster it also lets the init containers that read
+// the task's secrets reach the endpoints of their providers: a tunnel to port 443
+// that carries the proxy's token, which those containers get and the task not. The script is passed to `node -e`, so the proxy
 // needs no image of its own, only a stock Node image.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.formatEgressProxyEvent = exports.formatRefusedHost = exports.parseEgressProxyEvent = exports.getProxylessEnvironment = exports.getProxyClientEnvironment = exports.getEgressProxyPorts = exports.getItemProxyEnvironment = exports.getEgressProxyEnvironment = exports.getEgressProxyCommand = exports.EGRESS_PROXY_SCRIPT = exports.EGRESS_PROXY_HOST = exports.EGRESS_PROXY_PORT = exports.EGRESS_PROXY_IMAGE = void 0;
+exports.formatEgressProxyEvent = exports.formatRefusedHost = exports.parseEgressProxyEvent = exports.getProxylessEnvironment = exports.getProxyClientEnvironment = exports.getEgressProxyPorts = exports.getItemProxyEnvironment = exports.getSecretProxyEnvironment = exports.getEgressProxyEnvironment = exports.getEgressProxyCommand = exports.EGRESS_PROXY_SCRIPT = exports.EGRESS_PROXY_HOST = exports.EGRESS_PROXY_PORT = exports.EGRESS_PROXY_IMAGE = void 0;
 const work_network_1 = require("../planner/work-network");
 const network_schema_1 = require("../schema/network-schema");
 // Pinned, so the code behind the boundary is the same on every host. To bump
@@ -25,6 +27,7 @@ exports.EGRESS_PROXY_SCRIPT = `
 const http = require('http')
 const net = require('net')
 const dns = require('dns')
+const crypto = require('crypto')
 const entries = JSON.parse(process.env.HAMMERKIT_ALLOW || '[]').map((h) => h.toLowerCase())
 // names and patterns for HTTP(S)
 const allow = entries.filter((e) => !e.includes(':'))
@@ -38,6 +41,9 @@ const publicOnly = JSON.parse(process.env.HAMMERKIT_PUBLIC_ONLY || '[]').map((h)
 // the names of the needed services, for tools that ignore NO_PROXY: exact
 // names only, never patterns
 const needs = JSON.parse(process.env.HAMMERKIT_NEEDS || '[]').map((h) => h.toLowerCase())
+// the endpoints of secret providers, for a request with the token only
+const secretAllow = JSON.parse(process.env.HAMMERKIT_SECRET_ALLOW || '[]').map((h) => h.toLowerCase())
+const secretToken = process.env.HAMMERKIT_SECRET_TOKEN || ''
 const port = parseInt(process.env.HAMMERKIT_PROXY_PORT || '${exports.EGRESS_PROXY_PORT}', 10)
 // the name whose address the proxy listens on, every address when unset
 const bind = process.env.HAMMERKIT_PROXY_BIND
@@ -64,6 +70,31 @@ function allowedBy(host, port) {
   }
   return allow.some((rule) => rule.startsWith('*.') && name.endsWith(rule.substring(1))) ? 'wildcard' : null
 }
+// Whether the request carries the token as the password of Proxy-Authorization (Basic).
+function hasSecretToken(req) {
+  if (secretToken.length === 0) {
+    return false
+  }
+  const match = /^basic\\s+(\\S+)$/i.exec(String(req.headers['proxy-authorization'] || ''))
+  if (!match) {
+    return false
+  }
+  const credentials = Buffer.from(match[1], 'base64').toString('utf8')
+  const colon = credentials.indexOf(':')
+  if (colon < 0) {
+    return false
+  }
+  const given = Buffer.from(credentials.substring(colon + 1), 'utf8')
+  const expected = Buffer.from(secretToken, 'utf8')
+  return given.length === expected.length && crypto.timingSafeEqual(given, expected)
+}
+// the rule of a tunnel to an endpoint of a secret provider: like a wildcard, it reaches
+// no private address
+function allowedForSecrets(req, host, port) {
+  const name = host.toLowerCase().replace(/\\.$/, '')
+  const listed = secretAllow.some((rule) => (rule.startsWith('*.') ? name.endsWith(rule.substring(1)) : name === rule))
+  return listed && port === 443 && hasSecretToken(req) ? 'wildcard' : null
+}
 function ipv4(address) {
   const mapped = /^::ffff:(\\d+\\.\\d+\\.\\d+\\.\\d+)$/i.exec(address)
   const value = mapped ? mapped[1] : address
@@ -197,7 +228,7 @@ const server = http.createServer((req, res) => {
 server.on('connect', (req, socket, head) => {
   const target = parseAuthority(req.url || '')
   socket.on('error', () => socket.destroy())
-  const rule = target ? allowedBy(target.host, target.port) : null
+  const rule = target ? allowedBy(target.host, target.port) || allowedForSecrets(req, target.host, target.port) : null
   if (!rule) {
     refuse(target ? target.host : String(req.url))
     socket.end('HTTP/1.1 403 Forbidden\\r\\ncontent-type: text/plain\\r\\n\\r\\nhammerkit: network access is not allowed\\n')
@@ -290,7 +321,8 @@ exports.getEgressProxyCommand = getEgressProxyCommand;
 // `needs` are the names of the needed services, allowed as exactly those
 // names. `bind` is the name the proxy listens on alone, so that nothing on its
 // other networks reaches it. `publicOnly` are the entries of `hosts` that reach
-// no private address (see WorkNetwork).
+// no private address (see WorkNetwork). The endpoints of secret providers and
+// the token they need are set by the caller (getSecretProxyEnvironment).
 function getEgressProxyEnvironment(hosts, needs, bind, publicOnly = []) {
     return {
         HAMMERKIT_ALLOW: JSON.stringify(hosts),
@@ -301,6 +333,12 @@ function getEgressProxyEnvironment(hosts, needs, bind, publicOnly = []) {
     };
 }
 exports.getEgressProxyEnvironment = getEgressProxyEnvironment;
+// The endpoints of secret providers the proxy lets a tunnel with its token reach; the
+// token itself is set where the proxy runs, see HAMMERKIT_SECRET_TOKEN.
+function getSecretProxyEnvironment(hosts) {
+    return hosts.length > 0 ? { HAMMERKIT_SECRET_ALLOW: JSON.stringify(hosts) } : {};
+}
+exports.getSecretProxyEnvironment = getSecretProxyEnvironment;
 // The proxy of a task or service: the hosts of its network in this run (under
 // the run's ceiling), and the names of the services it needs, on every runtime.
 function getItemProxyEnvironment(item, bind) {
@@ -315,8 +353,9 @@ function getEgressProxyPorts(network) {
     return [exports.EGRESS_PROXY_PORT, ...(0, work_network_1.getTcpForwards)(network).map((forward) => forward.port)];
 }
 exports.getEgressProxyPorts = getEgressProxyPorts;
-function getProxyClientEnvironment(directHosts) {
-    const proxy = `http://${exports.EGRESS_PROXY_HOST}:${exports.EGRESS_PROXY_PORT}`;
+// `token` is sent as the password, for the endpoints of secret providers.
+function getProxyClientEnvironment(directHosts, token) {
+    const proxy = `http://${token !== undefined ? `hammerkit:${token}@` : ''}${exports.EGRESS_PROXY_HOST}:${exports.EGRESS_PROXY_PORT}`;
     const noProxy = ['localhost', '127.0.0.1', ...directHosts].join(',');
     return {
         HTTP_PROXY: proxy,

package/dist/optimizer/work-task-cache-description.js

@@ -6,6 +6,7 @@ const crypto_1 = require("crypto");
 const replace_env_variables_1 = require("../environment/replace-env-variables");
 const portable_path_1 = require("../planner/utils/portable-path");
 const work_secret_1 = require("../planner/work-secret");
+const claim_layout_1 = require("../kubernetes/claim-layout");
 // Every path is made project-relative (portablePath) so the description, and the
 // task id hashed from it, is identical wherever the project is checked out.
 //
@@ -40,6 +41,7 @@ function getWorkTaskCacheDescription(item) {
         cwd: portable(task.cwd),
         deps: item.deps.length > 0 ? item.deps.map((dep) => dep.id()).sort() : undefined,
         secrets: cachedSecrets.length > 0 ? cachedSecrets : undefined,
+        claimLayout: (0, claim_layout_1.hasPathsEndingAlike)(item) || (0, claim_layout_1.hasOutputsFlatteningAlike)(item) ? 2 : undefined,
     };
 }
 exports.getWorkTaskCacheDescription = getWorkTaskCacheDescription;

package/dist/planner/utils/append-work-dependencies.js

@@ -19,7 +19,10 @@ function appendWorkDependencies(workTree, referenced, item, environment, context
         }
         if (((0, work_task_1.isContainerWorkTask)(depNode.data) || (0, work_service_1.isContainerWorkService)(depNode.data)) &&
             ((0, work_task_1.isContainerWorkTask)(item.data) || (0, work_service_1.isContainerWorkService)(item.data))) {
+            // the outputs of the dep's own deps too, which stay those of the task that made them
+            // (on Kubernetes, they are in its claim)
             for (const generate of depNode.data.generates) {
+                const inherited = generate.inherited ?? depNode;
                 if (generate.isFile) {
                     // TODO check if thats correct
                     continue;
@@ -31,7 +34,7 @@ function appendWorkDependencies(workTree, referenced, item, environment, context
                             export: false,
                             resetOnChange: false,
                             containerPath: generate.path,
-                            inherited: depNode,
+                            inherited,
                             readOnly: false,
                         });
                     }
@@ -44,7 +47,7 @@ function appendWorkDependencies(workTree, referenced, item, environment, context
                             exportAlways: false,
                             resetOnChange: false,
                             path: generate.path,
-                            inherited: depNode,
+                            inherited,
                             isFile: false,
                         });
                     }

package/dist/planner/utils/append-work-service.js

@@ -120,6 +120,7 @@ function parseService(workTree, service, environment, context) {
             volumeSize: service.schema.volumeSize ?? null,
             network: (0, work_network_1.parseWorkNetwork)(service.schema.network),
             networkCeiling: null,
+            args: service.schema.args ?? null,
         };
     }
 }

package/dist/planner/utils/append-work-task.js

@@ -83,6 +83,7 @@ function parseTask(cwd, task, environment, context) {
         resources: (0, work_resources_1.parseWorkResources)(task.schema.resources),
         network: (0, work_network_1.parseWorkNetwork)(task.schema.network),
         networkCeiling: null,
+        args: task.schema.args ?? null,
         runner: task.schema.runner ?? task.scope.schema.runner ?? null,
     };
     if ((0, build_file_task_schema_1.isBuildFileContainerTaskSchema)(task.schema)) {
@@ -100,7 +101,9 @@ function parseTask(cwd, task, environment, context) {
         if (fileSecret) {
             throw new Error(`task ${task.relativeName}: a local task takes secrets as env only (${(0, work_secret_1.getSecretName)(fileSecret)})`);
         }
-        if (baseWorkNode.network.type !== 'open') {
+        // on a server's cluster it runs in a pod of a pool's tool image (runInToolImage)
+        const contained = !!environment.hosted?.kubernetes || !!environment.hosted?.localTasksInContainers;
+        if (baseWorkNode.network.type !== 'open' && !contained) {
             throw new Error(`task ${task.relativeName}: network ${(0, work_network_1.formatNetwork)(baseWorkNode.network)} needs a container, ` +
                 `hammerkit can't limit the network of a local task`);
         }

package/dist/planner/utils/task-arguments.js

@@ -101,6 +101,9 @@ function applyArguments(selection, args, keepCache = false) {
     if (item.type === 'kubernetes-service') {
         throw new Error(`service ${item.name} is a kubernetes service and has no cmd for the arguments after --`);
     }
+    if (item.args === false) {
+        throw new Error(`${item.type === 'container-service' ? 'service' : 'task'} ${item.name} takes no arguments after -- (args: false)`);
+    }
     const target = getArgumentsTarget(item);
     if (!target) {
         throw new Error(item.type === 'container-service'

package/dist/planner/work-runtime-kubernetes.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.kubernetesForwardRuntime = exports.kubernetesServiceRuntime = exports.kubernetesTaskRuntime = exports.describeFailedJob = void 0;
+exports.kubernetesForwardRuntime = exports.kubernetesServiceRuntime = exports.kubernetesTaskRuntime = exports.getTaskJobContainers = exports.describeFailedJob = void 0;
 const abort_1 = require("../executer/abort");
 const kubernetes_service_1 = require("../executer/kubernetes-service");
 const kubernetes_instance_1 = require("../kubernetes/kubernetes-instance");
@@ -125,6 +125,41 @@ async function describeFailedJob(instance, kubernetes, task, job, container, sec
     return undefined;
 }
 exports.describeFailedJob = describeFailedJob;
+/**
+ * The containers of the Job of a command: the network gate first, nothing runs before the
+ * item's policy holds, then the init containers that read its secrets, then the command. The
+ * proxy's token (`network.secretEnv`) reaches the init containers through `secretSpec` only:
+ * the command's container gets the proxy variables of a limited network without it.
+ */
+function getTaskJobContainers(options) {
+    const { task, cmd, envs, secrets, network, secretSpec, kubernetes } = options;
+    // the arguments after `--` keep their `$` as text, Kubernetes expands `$(NAME)`
+    const command = secretSpec.command(task.data.shell, (0, task_arguments_1.getKubernetesCommandText)(cmd));
+    const initContainers = [...network.initContainers, ...secretSpec.initContainers];
+    return {
+        ...(initContainers.length > 0 ? { initContainers } : {}),
+        containers: [
+            {
+                image: task.data.image,
+                // through the task's shell, as on Docker
+                command: command.command,
+                args: command.args,
+                workingDir: cmd.cwd,
+                env: [
+                    ...Object.entries(envs).map(([key, value]) => ({ name: key, value })),
+                    ...secrets.env,
+                    ...network.env,
+                    ...command.env,
+                ],
+                name: options.container,
+                volumeMounts: [...options.mounts, ...secrets.mounts, ...secretSpec.mounts],
+                ...(0, container_resources_1.getKubernetesResources)(task.data.resources),
+                ...(0, job_placement_1.getContainerSecurity)(kubernetes),
+            },
+        ],
+    };
+}
+exports.getTaskJobContainers = getTaskJobContainers;
 function kubernetesTaskRuntime(task, kubernetes) {
     const instance = (0, kubernetes_instance_1.createKubernetesInstances)(kubernetes);
     // on a server that keeps them, the task's claim stays after the run with the state of its
@@ -168,8 +203,9 @@ function kubernetesTaskRuntime(task, kubernetes) {
             const { delivered, inJob } = (0, job_secrets_1.splitJobSecrets)(task.data.secrets, server);
             const secrets = (0, secrets_1.getKubernetesSecretRefs)(task, delivered);
             const admitted = environment.hosted?.accounts;
+            let secretHosts;
             try {
-                (0, job_secrets_1.assertSecretsReachable)(task, inJob);
+                secretHosts = (0, job_secrets_1.getSecretHosts)(task, inJob, server);
                 if (server) {
                     (0, job_secrets_1.assertJobAccounts)(inJob, server, admitted);
                 }
@@ -210,7 +246,11 @@ function kubernetesTaskRuntime(task, kubernetes) {
             // NetworkPolicy and proxy of a limited network, both of this run
             let network;
             try {
-                network = await (0, network_policy_1.ensureKubernetesItemNetwork)(instance, kubernetes, task, { followLog: true, ...scope });
+                network = await (0, network_policy_1.ensureKubernetesItemNetwork)(instance, kubernetes, task, {
+                    followLog: true,
+                    secretHosts,
+                    ...scope,
+                });
             }
             catch (e) {
                 await removeTaskSecrets(instance, kubernetes, task).catch(() => undefined);
@@ -226,15 +266,13 @@ function kubernetesTaskRuntime(task, kubernetes) {
                     try {
                         // made for each Job right before it, so a later command's token is as fresh
                         const credentials = await (0, job_secrets_1.ensureJobCredentials)(instance, kubernetes, task, inJob, admitted);
-                        secretSpec = (0, job_secrets_1.getJobSecretSpec)(task, inJob, server, credentials, environment);
+                        secretSpec = (0, job_secrets_1.getJobSecretSpec)(task, inJob, server, credentials, environment, network.secretEnv);
                     }
                     catch (e) {
                         await removeTaskSecrets(instance, kubernetes, task).catch(() => undefined);
                         options.state.set({ stateKey: options.stateKey, type: 'error', errorMessage: (0, log_1.getErrorMessage)(e) });
                         return;
                     }
-                    // the arguments after `--` keep their `$` as text, Kubernetes expands `$(NAME)`
-                    const command = secretSpec.command(task.data.shell, (0, task_arguments_1.getKubernetesCommandText)(cmd));
                     const spec = {
                         kind: 'Job',
                         apiVersion: 'batch/v1',
@@ -255,33 +293,17 @@ function kubernetesTaskRuntime(task, kubernetes) {
                                     labels,
                                 },
                                 spec: {
-                                    // the network gate first: nothing runs before the item's policy holds
-                                    ...(network.initContainers.length + secretSpec.initContainers.length > 0
-                                        ? { initContainers: [...network.initContainers, ...secretSpec.initContainers] }
-                                        : {}),
-                                    containers: [
-                                        {
-                                            image: task.data.image,
-                                            // through the task's shell, as on Docker
-                                            command: command.command,
-                                            args: command.args,
-                                            workingDir: cmd.cwd,
-                                            env: [
-                                                ...Object.entries(envs).map(([key, value]) => ({ name: key, value })),
-                                                ...secrets.env,
-                                                ...network.env,
-                                                ...command.env,
-                                            ],
-                                            name: container,
-                                            volumeMounts: [
-                                                ...persistence.mounts.map((m) => m.mount),
-                                                ...secrets.mounts,
-                                                ...secretSpec.mounts,
-                                            ],
-                                            ...(0, container_resources_1.getKubernetesResources)(task.data.resources),
-                                            ...(0, job_placement_1.getContainerSecurity)(kubernetes),
-                                        },
-                                    ],
+                                    ...getTaskJobContainers({
+                                        task,
+                                        cmd,
+                                        container,
+                                        envs,
+                                        secrets,
+                                        network,
+                                        secretSpec,
+                                        mounts: persistence.mounts.map((m) => m.mount),
+                                        kubernetes,
+                                    }),
                                     volumes: [
                                         ...persistence.volumes,
                                         ...persistence.workspace,

package/dist/program.js

@@ -84,7 +84,8 @@ function printTaskDefinition(environment, definition) {
     }
 }
 // `ls --json`: what a caller (an agent picking a task) needs to know, with the
-// command that arguments after `--` are appended to (null: it takes none).
+// command that arguments after `--` are appended to (null: it takes none) and
+// `args` as the build file says it (false: it refuses them).
 function getListJson(tasks, services) {
     return {
         tasks: tasks.map(({ item }) => ({
@@ -96,6 +97,7 @@ function getListJson(tasks, services) {
             needs: item.needs.map((n) => n.name),
             labels: item.data.labels,
             command: (0, task_arguments_1.getArgumentsTarget)(item.data)?.cmd ?? null,
+            args: item.data.args,
         })),
         services: services.map(({ item }) => ({
             name: item.name,
@@ -105,6 +107,7 @@ function getListJson(tasks, services) {
             ports: item.data.ports.map((p) => ({ host: p.hostPort, container: p.containerPort })),
             labels: item.data.labels,
             command: (0, task_arguments_1.getArgumentsTarget)(item.data)?.cmd ?? null,
+            args: item.data.type === 'container-service' ? item.data.args : null,
         })),
     };
 }

package/dist/schema/build-file-base-task-schema.js

@@ -40,8 +40,12 @@ exports.buildFileBaseTaskSchema = (0, zod_1.object)({
     account: (0, zod_1.string)().optional(),
     // CPU and memory limits of the container; a local task is not limited
     resources: resources_schema_1.resourcesSchema.optional(),
-    // what the container reaches besides its needs; a local task can't be limited
+    // what the container reaches besides its needs; a local task can be limited only where
+    // it runs in a container, on a server's cluster
     network: network_schema_1.networkSchema.optional(),
+    // whether its last command takes the arguments after `--`: false refuses them, true
+    // says it is meant to, which a server with `runs.args: declared` requires
+    args: (0, zod_1.boolean)().optional(),
     // Name of the server (see `hammerkit remote`) the task has to run on; defaults
     // to the build file's `runner`. Never a local fallback when it is unknown.
     runner: runner_schema_1.runnerSchema.optional(),

package/dist/schema/build-file-container-service-schema.js

@@ -45,6 +45,8 @@ exports.buildFileContainerServiceSchema = (0, zod_1.object)({
     volumeSize: size_schema_1.sizeSchema.describe('size of its claim on Kubernetes, e.g. 10Gi').optional(),
     // what the service reaches besides its needs; tasks needing it always reach it
     network: network_schema_1.networkSchema.optional(),
+    // whether its `cmd` takes the arguments after `--` of `up`: false refuses them
+    args: (0, zod_1.boolean)().optional(),
 })
     .strict()
     .describe('container service');

package/dist/schema/reference-parser.js

@@ -129,6 +129,7 @@ function applySchemaExtension(reference, task) {
     task.schema.timeout = task.schema.timeout ?? extend.schema.timeout;
     task.schema.runner = task.schema.runner ?? extend.schema.runner;
     task.schema.network = task.schema.network ?? extend.schema.network;
+    task.schema.args = task.schema.args ?? extend.schema.args;
     if (!reset.has('cmds'))
         task.schema.cmds = extendArray(task.schema.cmds, extend.schema.cmds);
     if (!reset.has('src'))

package/dist/server/executor.js

@@ -100,7 +100,7 @@ class HostExecutor {
         if (extra.length > 0) {
             throw new Error(`the run would use ${extra.join(', ')}, which it was not admitted for`);
         }
-        const refused = (0, run_plan_1.refuseArguments)(run, planned.plan);
+        const refused = (0, run_plan_1.refuseArguments)(run, planned.plan, policy);
         if (refused) {
             throw new Error(refused);
         }

package/dist/server/plan.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.inspectBundle = exports.planBundle = exports.DefinitionError = void 0;
+exports.inspectBundle = exports.planBundle = exports.getCredentialReads = exports.PLAIN_SERVER_ENVS = exports.DefinitionError = void 0;
 const promises_1 = require("fs/promises");
 const os_1 = require("os");
 const path_1 = require("path");
@@ -20,6 +20,20 @@ const work_network_1 = require("../planner/work-network");
 class DefinitionError extends Error {
 }
 exports.DefinitionError = DefinitionError;
+// Variables of a server that hold no credential (FR-008): a task may read these and
+// still take arguments, or reach any network.
+exports.PLAIN_SERVER_ENVS = ['PATH', 'HOME', 'CI'];
+// What an item holds that its command must not hand out: its secrets, and the variables
+// of the server its `envs` read, but the plain ones.
+function getCredentialReads(item) {
+    return [
+        ...item.secrets.map((secret) => `the secret ${secret.target}`),
+        ...(item.serverEnvs ?? [])
+            .filter((env) => !exports.PLAIN_SERVER_ENVS.includes(env.toUpperCase()))
+            .map((env) => `$${env} of the server`),
+    ];
+}
+exports.getCredentialReads = getCredentialReads;
 // 40 (sha-1) or 64 (sha-256) hex digits
 const COMMIT_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/i;
 // A hash alone can only name what is inside the bundle, so everything a build
@@ -170,13 +184,32 @@ function findPolicyProblems(items, policy, root) {
                 problems.push(`${item.name}: generates ${generate.path}, which is outside of the bundle`);
             }
         }
-        const ceiling = policy.network;
-        if (ceiling && (0, work_network_1.isLimitedNetwork)(ceiling)) {
-            if (data.type === 'local-task') {
-                problems.push(`${item.name}: is a local task, which can't be limited to the network this server allows (runs.network: ${(0, work_network_1.formatNetwork)(ceiling)})`);
+        // what holds a credential gets at most `runs.secretNetwork`, so a tool it runs can't
+        // send the credential anywhere else
+        const reads = getCredentialReads({
+            secrets: secretsOf(item).map(describeSecret),
+            serverEnvs: 'envs' in data ? data.envs.processNames ?? [] : [],
+        });
+        const ceilings = [];
+        if (policy.network && (0, work_network_1.isLimitedNetwork)(policy.network)) {
+            ceilings.push({ ceiling: policy.network, setting: 'runs.network', reason: '' });
+        }
+        if (reads.length > 0 && policy.secretNetwork && (0, work_network_1.isLimitedNetwork)(policy.secretNetwork)) {
+            ceilings.push({
+                ceiling: policy.secretNetwork,
+                setting: 'runs.secretNetwork',
+                reason: ` to what reads ${reads.join(', ')}`,
+            });
+        }
+        for (const { ceiling, setting, reason } of ceilings) {
+            // on a cluster a local task is a pod of a pool's tool image, limited like any other
+            if (data.type === 'local-task' && policy.backend !== 'kubernetes') {
+                problems.push(`${item.name}: is a local task, which can't be limited to the network this server allows${reason} ` +
+                    `(${setting}: ${(0, work_network_1.formatNetwork)(ceiling)})`);
             }
             else if ('network' in data && (0, work_network_1.exceedsNetwork)(data.network, ceiling)) {
-                problems.push(`${item.name}: asks for network ${(0, work_network_1.formatNetwork)(data.network)}, more than this server allows (runs.network: ${(0, work_network_1.formatNetwork)(ceiling)})`);
+                problems.push(`${item.name}: asks for network ${(0, work_network_1.formatNetwork)(data.network)}, more than this server allows${reason} ` +
+                    `(${setting}: ${(0, work_network_1.formatNetwork)(ceiling)})`);
             }
         }
         if ((0, work_task_1.isContainerWorkTask)(data) || (0, work_service_1.isContainerWorkService)(data)) {
@@ -205,11 +238,13 @@ function findPolicyProblems(items, policy, root) {
                     problems.push(`${item.name}: the secret ${describeSecret(secret).target} has cache: true, its digest would need ` +
                         'the value before the Job of the task reads it, which this server does not support yet');
                 }
-                else if (item.data.network.type !== 'open') {
-                    // the init containers that read it share the pod's NetworkPolicy
+                else if ((0, work_network_1.isLimitedNetwork)(item.data.network) && !policy.proxiedProviders?.includes(source.provider)) {
+                    // the init containers that read it share the pod's NetworkPolicy, and reach the
+                    // provider only through the egress proxy
                     problems.push(`${item.name}: the secret ${describeSecret(secret).target} is read from ${source.provider} in the ` +
-                        `Job of the task, which network ${(0, work_network_1.formatNetwork)(item.data.network)} would cut off from it, ` +
-                        'a task that reads secrets of a provider keeps an open network on this server');
+                        `Job of the task, which network ${(0, work_network_1.formatNetwork)(item.data.network)} would cut off from it: ` +
+                        `this server names no endpoints of ${source.provider} in backend.providers.${source.provider}.allow ` +
+                        'for the egress proxy to let through, so the task keeps an open network');
                 }
             }
             if (source.type === 'provider' && policy.reserved?.includes(source.account)) {
@@ -254,6 +289,7 @@ function toPlan(root, item) {
         cmds: 'cmds' in data ? data.cmds.map((cmd) => cmd.cmd) : 'cmd' in data && data.cmd ? [data.cmd.cmd] : [],
         command: (0, task_arguments_1.getArgumentsTarget)(data)?.cmd ?? null,
         network: (0, work_network_1.toNetworkSchema)('network' in data ? data.network : { type: 'open' }),
+        ...('args' in data && data.args !== null ? { args: data.args } : {}),
         deps: item.deps.map((dep) => dep.name).sort(),
         needs: item.needs.map((need) => need.name).sort(),
         accounts,

package/dist/server/run-plan.js

@@ -46,26 +46,16 @@ async function planRun(bundle, run, environment, policy) {
     };
 }
 exports.planRun = planRun;
-// Variables of a server that hold no credential (FR-008): a task may read these and
-// still take arguments.
-const PLAIN_SERVER_ENVS = ['PATH', 'HOME', 'CI'];
-function readsOf(entry) {
-    return [
-        ...entry.secrets.map((secret) => `the secret ${secret.target}`),
-        ...(entry.serverEnvs ?? [])
-            .filter((env) => !PLAIN_SERVER_ENVS.includes(env.toUpperCase()))
-            .map((env) => `$${env} of the server`),
-    ];
-}
 /**
  * Why a run may not pass its arguments, or null when it may. They go to a task, never a
  * service, and never to a run in which an item reads a secret or takes a variable of the
  * server into its `envs`: an argument can make the command print what the task holds,
  * and what its dependencies write into their outputs or its services serve, past the
  * masking of secrets. So a shell on a server runs with no account and no credential of
- * the server.
+ * the server. Under `runs.args: declared` they go only to a task that says `args: true`,
+ * so a task nobody meant to take them never does.
  */
-function refuseArguments(run, plan) {
+function refuseArguments(run, plan, policy = {}) {
     if (!run.options.args || run.options.args.length === 0) {
         return null;
     }
@@ -75,14 +65,21 @@ function refuseArguments(run, plan) {
     if (!entry || entry.kind !== 'task') {
         return `${name} is not a task of this version`;
     }
-    const reads = readsOf(entry);
+    // planning the run refuses it before, this keeps the check with the others
+    if (entry.args === false) {
+        return `${name} takes no arguments (args: false)`;
+    }
+    if (policy.args === 'declared' && entry.args !== true) {
+        return `${name} takes no arguments on this server: it takes them only for tasks that say args: true (runs.args: declared)`;
+    }
+    const reads = (0, plan_1.getCredentialReads)(entry);
     if (reads.length > 0) {
         return `${name} reads ${reads.join(', ')}, it takes no arguments on a server: an argument could make it print them`;
     }
     // what a dependency or a service of the run holds reaches the task through its outputs or
     // its answers, where an argument can print it in a form no mask knows (`rev token.txt`)
     for (const [other, item] of Object.entries(plan).sort(([a], [b]) => a.localeCompare(b))) {
-        const held = (other === name ? [] : readsOf(item)).join(', ');
+        const held = (other === name ? [] : (0, plan_1.getCredentialReads)(item)).join(', ');
         if (held.length > 0) {
             return `${name} takes no arguments on a server: ${other} in its run reads ${held}, and an argument could make ${name} print what ${other} hands on`;
         }

package/dist/server/server-config.js

@@ -238,8 +238,22 @@ const kubernetesBackendSchema = zod_1.z
         .min(1)
         .default([{ name: 'default', platform: { os: 'linux', arch: 'amd64' }, nodeSelector: {}, tolerations: [] }]),
     // per secret provider, the image its command runs in: secrets are read in the
-    // Job of the task, not by the server
-    providers: zod_1.z.record(zod_1.z.object({ image: zod_1.z.string().min(1) }).strict()).default({}),
+    // Job of the task, not by the server. `allow` are the hosts its command reaches, such
+    // as secretmanager.googleapis.com: the egress proxy of a task with a limited network
+    // lets the init containers that read its secrets reach them, and nothing else does
+    providers: zod_1.z
+        .record(zod_1.z
+        .object({
+        image: zod_1.z.string().min(1),
+        allow: zod_1.z
+            .array(zod_1.z
+            .string()
+            .refine((host) => (0, network_schema_1.isAllowEntry)(host) && !host.includes(':'), 'expected a host name such as secretmanager.googleapis.com or *.example.com'))
+            .min(1)
+            .optional(),
+    })
+        .strict())
+        .default({}),
     // the ResourceQuota of every caller's namespace, such as `requests.cpu: '8'`
     quota: zod_1.z.record(zod_1.z.union([zod_1.z.string().min(1), zod_1.z.number()]).transform(String)).optional(),
     // what a container without `resources` requests (a LimitRange), so it is
@@ -451,6 +465,13 @@ exports.serverConfigSchema = zod_1.z
         // the most any task or service of a run reaches besides its needs: none, open
         // or { allow: [hosts] }. A version that asks for more is refused at publish.
         network: network_schema_1.networkSchema.default('open'),
+        // the most a task or service that reads a secret, or a variable of the server
+        // other than PATH, HOME and CI, reaches, under `network`: so a tool it runs can't
+        // send the value anywhere. Checked like `network`.
+        secretNetwork: network_schema_1.networkSchema.default('open'),
+        // which tasks take arguments: `any` but those that say `args: false`, or
+        // `declared`, only those that say `args: true`
+        args: zod_1.z.enum(['any', 'declared']).default('any'),
     })
         .strict()
         .default({}),
@@ -586,7 +607,10 @@ exports.serverConfigSchema = zod_1.z
             message: `under runs.network, ${shared}`,
         });
     }
-    if (config.sandboxes.runtimes?.includes('local') && config.sandboxes.network !== 'open') {
+    // on a cluster a local task runs in a pod of a pool's tool image, which can be limited
+    if (config.sandboxes.runtimes?.includes('local') &&
+        config.sandboxes.network !== 'open' &&
+        config.backend.type === 'host') {
         ctx.addIssue({
             code: zod_1.z.ZodIssueCode.custom,
             path: ['sandboxes', 'network'],
@@ -697,6 +721,13 @@ exports.serverConfigSchema = zod_1.z
                 message: 'runs.network limits every task with a NetworkPolicy, set networkPolicy: enforced once the network plugin of the cluster enforces them',
             });
         }
+        if (config.runs.secretNetwork !== 'open' && backend.networkPolicy !== 'enforced') {
+            ctx.addIssue({
+                code: zod_1.z.ZodIssueCode.custom,
+                path: ['backend', 'networkPolicy'],
+                message: 'runs.secretNetwork limits the tasks that read secrets with a NetworkPolicy, set networkPolicy: enforced once the network plugin of the cluster enforces them',
+            });
+        }
         if (config.sandboxes.network !== 'open' &&
             backend.networkPolicy !== 'enforced' &&
             config.entitlements.some((entitlement) => entitlement.sandboxes?.length)) {
@@ -707,7 +738,7 @@ exports.serverConfigSchema = zod_1.z
             });
         }
     }
-    if (config.runs.network !== 'open' && config.runtimes.includes('local')) {
+    if (config.runs.network !== 'open' && config.runtimes.includes('local') && backend.type === 'host') {
         ctx.addIssue({
             code: zod_1.z.ZodIssueCode.custom,
             path: ['runs', 'network'],

package/dist/server/server.js

@@ -275,7 +275,7 @@ async function submitRun(config, registry, definitions, sources, sandboxes, plan
         (0, http_2.send)(res, 400, { error: `the run does not load: ${e instanceof Error ? e.message : String(e)}` });
         return;
     }
-    const refused = (0, run_plan_1.refuseArguments)(request, planned.plan);
+    const refused = (0, run_plan_1.refuseArguments)(request, planned.plan, sandbox ? sandboxes.policy : planning.policy);
     if (refused) {
         log(`refuse run iss=${principal.iss} sub=${principal.sub}: ${refused}`);
         (0, http_2.send)(res, 400, { error: refused });
@@ -442,7 +442,8 @@ async function startServer(config, environment, options = {}) {
         environment: {
             ...environment,
             processEnvs: (0, run_environment_1.runEnvironment)(config, environment.processEnvs),
-            hosted: { catalog: (0, server_config_1.getSecretCatalog)(config) },
+            // on a cluster a local task runs in a pod of a pool's tool image
+            hosted: { catalog: (0, server_config_1.getSecretCatalog)(config), localTasksInContainers: config.backend.type === 'kubernetes' },
         },
         policy: {
             envs: config.runs.env.allow,
@@ -450,6 +451,13 @@ async function startServer(config, environment, options = {}) {
             runtimes: config.runtimes,
             backend: config.backend.type,
             network: (0, work_network_1.parseWorkNetwork)(config.runs.network),
+            secretNetwork: (0, work_network_1.parseWorkNetwork)(config.runs.secretNetwork),
+            args: config.runs.args,
+            proxiedProviders: config.backend.type === 'kubernetes'
+                ? Object.entries(config.backend.providers)
+                    .filter(([, provider]) => provider.allow)
+                    .map(([name]) => name)
+                : [],
         },
     };
     const storage = config.storage.directory ?? (0, path_1.join)((0, get_cache_directory_1.getHammerkitDirectory)(), 'server');

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "hammerkit",
   "description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
-  "version": "1.27.0-next.4",
+  "version": "1.27.0-next.6",
   "keywords": [
     "build",
     "build-tool",