hammerkit 1.27.0-next.1 → 1.27.0-next.11

package/dist/kubernetes/job-secrets.js

@@ -1,8 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getJobSecretSpec = exports.removeJobCredentials = exports.assertJobAccounts = exports.ensureJobCredentials = exports.getCredentialsName = exports.assertSecretsReachable = exports.splitJobSecrets = exports.ACCOUNT_PATH = exports.SECRET_VALUES_PATH = void 0;
+exports.getJobSecretSpec = exports.removeJobCredentials = exports.assertJobAccounts = exports.ensureJobCredentials = exports.getCredentialsName = exports.getSecretHosts = exports.splitJobSecrets = exports.ACCOUNT_PATH = exports.SECRET_VALUES_PATH = void 0;
 const work_secret_1 = require("../planner/work-secret");
 const command_secret_provider_1 = require("../secrets/command-secret-provider");
+const task_arguments_1 = require("../planner/utils/task-arguments");
 const version_1 = require("../version");
 const apply_1 = require("./apply");
 const resources_1 = require("./resources");
@@ -42,25 +43,30 @@ function splitJobSecrets(secrets, server) {
 exports.splitJobSecrets = splitJobSecrets;
 /**
  * The init containers that read secrets share the pod, and so the NetworkPolicy, of
- * the task: under a limited network they could not reach the secret manager. Until
- * they run in a pod of their own, a task that reads a provider's secret in its Job
- * keeps an open network.
+ * the task: under a limited network they reach the secret manager only through the
+ * task's egress proxy, which lets them reach the endpoints the server names for the
+ * provider (`backend.providers.<name>.allow`). A task that reads a secret of a provider
+ * without them keeps an open network. Answers the endpoints the proxy lets through.
  */
-function assertSecretsReachable(task, secrets, network = (0, work_network_1.getEffectiveNetwork)(task.data)) {
-    if (secrets.length === 0 || network.type === 'open') {
-        return;
+function getSecretHosts(task, secrets, server, network = (0, work_network_1.getEffectiveNetwork)(task.data)) {
+    if (!server || secrets.length === 0 || network.type === 'open') {
+        return [];
+    }
+    const unreachable = secrets.find((secret) => !server.providers[secret.source.provider]?.allow?.length);
+    if (unreachable) {
+        throw new Error(`task ${task.name}: secret ${(0, work_secret_1.getSecretName)(unreachable.secret)} is read from ${(0, work_secret_1.describeSecretSource)(unreachable.secret)} in the Job of the task, which network ${(0, work_network_1.formatNetwork)(network)} would cut off from it; ` +
+            `on this cluster a task that reads secrets of ${unreachable.source.provider} keeps an open network, ` +
+            `unless the server names its endpoints in backend.providers.${unreachable.source.provider}.allow`);
     }
-    const secret = secrets[0];
-    throw new Error(`task ${task.name}: secret ${(0, work_secret_1.getSecretName)(secret.secret)} is read from ${(0, work_secret_1.describeSecretSource)(secret.secret)} in the Job of the task, which network ${(0, work_network_1.formatNetwork)(network)} would cut off from it; ` +
-        'on this cluster a task that reads secrets of a provider keeps an open network');
+    return [...new Set(secrets.flatMap((secret) => server.providers[secret.source.provider].allow ?? []))].sort();
 }
-exports.assertSecretsReachable = assertSecretsReachable;
+exports.getSecretHosts = getSecretHosts;
 // The accounts of the secrets, in the order they are first read
 function accountsOf(secrets) {
     return [...new Set(secrets.map((s) => s.source.account))];
 }
-function getCredentialsName(item) {
-    return (0, resources_1.getResourceName)(item, '-credentials');
+function getCredentialsName(item, kubernetes) {
+    return (0, resources_1.getResourceName)(item, kubernetes, '-credentials');
 }
 exports.getCredentialsName = getCredentialsName;
 // The keys of the credentials Secret: `<account>.token` and `<account>.file.<key>`
@@ -130,9 +136,9 @@ async function ensureJobCredentials(instance, kubernetes, item, secrets, admitte
         apiVersion: 'v1',
         metadata: {
             namespace: kubernetes.namespace,
-            name: getCredentialsName(item),
+            name: getCredentialsName(item, kubernetes),
             annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
-            labels: { 'hammerkit.dev/id': item.id(), ...(0, job_placement_1.getRunLabels)(kubernetes) },
+            labels: (0, resources_1.getItemLabels)(item, kubernetes),
         },
         type: 'Opaque',
         data,
@@ -163,7 +169,10 @@ async function removeJobCredentials(instance, kubernetes, item) {
         return;
     }
     try {
-        await instance.coreApi.deleteNamespacedSecret({ name: getCredentialsName(item), namespace: kubernetes.namespace });
+        await instance.coreApi.deleteNamespacedSecret({
+            name: getCredentialsName(item, kubernetes),
+            namespace: kubernetes.namespace,
+        });
     }
     catch (e) {
         if ((0, apply_1.statusCodeOf)(e) !== 404) {
@@ -186,7 +195,9 @@ function quote(value) {
  * from it, and the task's output is masked before it leaves the pod, since the
  * server, which masks everything else, never sees these values.
  */
-function getJobSecretSpec(item, secrets, server, credentials, environment) {
+function getJobSecretSpec(item, secrets, server, credentials, environment, 
+// under a limited network, what sends their requests through the egress proxy
+proxyEnv = []) {
     if (!server || secrets.length === 0) {
         return {
             initContainers: [],
@@ -205,7 +216,7 @@ function getJobSecretSpec(item, secrets, server, credentials, environment) {
         }
         const name = `hammerkit-account-${index}`;
         accountVolume.set(account, name);
-        volumes.push({ name, secret: { secretName: getCredentialsName(item), items, defaultMode: 0o444 } });
+        volumes.push({ name, secret: { secretName: getCredentialsName(item, { server }), items, defaultMode: 0o444 } });
     });
     const initContainers = secrets.map(({ index, secret, source }) => {
         const label = `secret ${(0, work_secret_1.getSecretName)(secret)} (${(0, work_secret_1.describeSecretSource)(secret)})`;
@@ -229,8 +240,11 @@ function getJobSecretSpec(item, secrets, server, credentials, environment) {
                 '-c',
                 'umask 022 && "$@" > "$HAMMERKIT_VALUE.part" && mv "$HAMMERKIT_VALUE.part" "$HAMMERKIT_VALUE"',
             ],
-            args: ['sh', command, ...args],
+            // Kubernetes would expand a `$(NAME)` of the container's env in them, the proxy's token
+            // among them: the reference is text, as on any other backend
+            args: ['sh', command, ...args].map(task_arguments_1.escapeKubernetesText),
             env: [
+                ...proxyEnv,
                 ...Object.entries(env).map(([name, value]) => ({ name, value })),
                 { name: 'HAMMERKIT_VALUE', value: `${exports.SECRET_VALUES_PATH}/secret-${index}` },
             ],

package/dist/kubernetes/kept-claims.js

@@ -5,6 +5,7 @@ const crypto_1 = require("crypto");
 const apply_1 = require("./apply");
 const sleep_1 = require("../utils/sleep");
 const job_placement_1 = require("./job-placement");
+const resources_1 = require("./resources");
 // On a server with `backend.keepVolumes`, the claim of a task stays after the run, so the
 // next run of the same source finds the outputs in it and mounts them without restoring
 // them from the cache. A kept claim has no run label (the end of the run removes what has
@@ -18,11 +19,15 @@ exports.STATE_ANNOTATION = 'hammerkit.dev/state';
 // when it goes, set by the end of the run that mounted it last
 exports.EXPIRES_ANNOTATION = 'hammerkit.dev/expires';
 // The claim of a work item: a task's on a server that keeps them carries the scope, so a
-// run of another source never mounts it. 63 characters at most, a pod's volume name.
+// run of another source never mounts it, and the runs of one scope go one at a time
+// (KubernetesExecutor.exclusive). Any other claim on a server is the run's own, as the
+// runs of a caller share their namespace. 63 characters at most, a pod's volume name.
 function getClaimName(work, kubernetes) {
-    const scope = kubernetes?.server?.volumeScope;
-    const name = 'hammerkit-' + work.id();
-    return scope && isTask(work) ? `${name}-${(0, crypto_1.createHash)('sha256').update(scope).digest('hex').slice(0, 12)}` : name;
+    const scope = kubernetes.server?.volumeScope;
+    if (scope && isTask(work)) {
+        return `hammerkit-${work.id()}-${(0, crypto_1.createHash)('sha256').update(scope).digest('hex').slice(0, 12)}`;
+    }
+    return 'hammerkit-' + (0, resources_1.getItemKey)(work, kubernetes);
 }
 exports.getClaimName = getClaimName;
 // whether the claim of the work item stays after the run
@@ -156,14 +161,17 @@ async function expireKeptClaims(instance, namespace, runLabel, expires) {
     }
 }
 exports.expireKeptClaims = expireKeptClaims;
-// While no run of the caller is in the namespace: the kept claims that expired, and those
-// a run left without an expiry because the server stopped before its end (all of them
-// when the server keeps none any more). The names of those removed, and when the next of
-// the others expires.
+// The kept claims that expired, and those a run left without an expiry because the server
+// stopped before its end (all of them when the server keeps none any more), but those of
+// the scopes `inUse`: a run of the scope executes and mounts them, or may in a moment.
+// The names of those removed, and when the next of the others expires.
 async function sweepKeptClaims(instance, namespace, options) {
     const expired = [];
     let next = null;
     for (const claim of await listKeptClaims(instance, namespace)) {
+        if (options.inUse?.has(claim.metadata?.labels?.[exports.VOLUME_SCOPE_LABEL] ?? '')) {
+            continue;
+        }
         const expires = Date.parse(claim.metadata?.annotations?.[exports.EXPIRES_ANNOTATION] ?? '');
         if (!options.keep || !(expires > options.now.getTime())) {
             expired.push(claim);
@@ -178,14 +186,18 @@ async function sweepKeptClaims(instance, namespace, options) {
 exports.sweepKeptClaims = sweepKeptClaims;
 // When a run needs room under the quota of its namespace: the kept claims it does not use
 // go, those of the other sources and those of its own a task of this run did not take (one
-// renamed or changed since). The names of those removed.
+// renamed or changed since). Of another scope, only one its last run released, which has an
+// expiry: one without is mounted by a run of the caller that executes now (or was left by
+// a server that stopped, which the next sweep removes). The names of those removed.
 async function evictKeptClaims(instance, kubernetes, timeout) {
     const server = kubernetes.server;
     if (!server?.volumeScope) {
         return [];
     }
     const unused = await listKeptClaims(instance, kubernetes.namespace, `${exports.VOLUME_SCOPE_LABEL},${exports.KEPT_BY_LABEL}!=${(0, job_placement_1.getRunLabelValue)(server.run)}`);
-    return removeClaims(instance, kubernetes.namespace, unused, timeout);
+    const released = unused.filter((claim) => claim.metadata?.labels?.[exports.VOLUME_SCOPE_LABEL] === server.volumeScope ||
+        !!claim.metadata?.annotations?.[exports.EXPIRES_ANNOTATION]);
+    return removeClaims(instance, kubernetes.namespace, released, timeout);
 }
 exports.evictKeptClaims = evictKeptClaims;
 //# sourceMappingURL=kept-claims.js.map

package/dist/kubernetes/network-policy.js

@@ -17,12 +17,15 @@ const sleep_1 = require("../utils/sleep");
 const network_gate_1 = require("./network-gate");
 const job_placement_1 = require("./job-placement");
 Object.defineProperty(exports, "RUN_LABEL", { enumerable: true, get: function () { return job_placement_1.RUN_LABEL; } });
-const ID_LABEL = 'hammerkit.dev/id';
+const resources_1 = require("./resources");
 const PROXY_LABEL = 'hammerkit.dev/proxy-for';
 const NETWORK_OF_LABEL = 'hammerkit.dev/network-of';
 // on the proxy of a task on a sandbox: the sandbox, whose pod of files it stays with
 const PROXY_OF_SANDBOX_LABEL = 'hammerkit.dev/proxy-of-sandbox';
 const PROXY_READY_TIMEOUT_MS = 60000;
+// the token of the endpoints of secret providers, in a Secret named like the proxy
+const SECRET_TOKEN_KEY = 'token';
+const SECRET_TOKEN_VARIABLE = 'HAMMERKIT_PROXY_TOKEN';
 // how long the pods of an item may take to end before their policy goes
 const POD_END_TIMEOUT_MS = 120000;
 const POD_END_POLL_MS = 500;
@@ -83,7 +86,7 @@ function getNetworkPolicyName(item, scope) {
 exports.getNetworkPolicyName = getNetworkPolicyName;
 // the pods of the item, of the run when the scope has one
 function getItemPodLabels(item, scope) {
-    return { [ID_LABEL]: item.id(), ...(scope.run ? { [job_placement_1.RUN_LABEL]: scope.run } : {}) };
+    return { [resources_1.ID_LABEL]: item.id(), ...(scope.run ? { [job_placement_1.RUN_LABEL]: scope.run } : {}) };
 }
 function toSelector(labels) {
     return Object.entries(labels)
@@ -104,15 +107,20 @@ function getIngress(env, item) {
     if (ids.length === 0) {
         return [];
     }
-    const from = { matchExpressions: [{ key: ID_LABEL, operator: 'In', values: ids }] };
+    // on a server, of this run: the runs of a caller share their namespace
+    const run = (0, job_placement_1.getRunLabels)(env);
+    const from = {
+        matchExpressions: [{ key: resources_1.ID_LABEL, operator: 'In', values: ids }],
+        ...(Object.keys(run).length > 0 ? { matchLabels: run } : {}),
+    };
     // the client's name for `from`, a reserved word
     return [{ _from: [{ podSelector: from }] }];
 }
 // The NetworkPolicy of a limited item: egress to the pods of the services it
-// needs and, under `allow`, to its proxy; nothing else, DNS included (needs
-// and the proxy are host aliases). Ingress only from what needs it. An open
-// item has none.
-function buildNetworkPolicy(env, item, network, scope, proxy = getRunProxy(item, scope)) {
+// needs and, under `allow` or with secrets read through it, to its proxy;
+// nothing else, DNS included (needs and the proxy are host aliases). Ingress
+// only from what needs it. An open item has none.
+function buildNetworkPolicy(env, item, network, scope, proxy = getRunProxy(item, scope), proxied = network.type === 'allow') {
     const needs = item.needs.filter((need) => (0, work_item_1.isContainerWorkServiceItem)(need.service));
     return {
         apiVersion: 'networking.k8s.io/v1',
@@ -129,9 +137,9 @@ function buildNetworkPolicy(env, item, network, scope, proxy = getRunProxy(item,
             ingress: getIngress(env, item),
             egress: [
                 ...(needs.length > 0
-                    ? [{ to: needs.map((need) => ({ podSelector: { matchLabels: { [ID_LABEL]: need.service.id() } } })) }]
+                    ? [{ to: needs.map((need) => ({ podSelector: { matchLabels: (0, resources_1.getItemLabels)(need.service, env) } })) }]
                     : []),
-                ...(network.type === 'allow'
+                ...(proxied
                     ? [
                         {
                             to: [{ podSelector: { matchLabels: { [PROXY_LABEL]: proxy.key } } }],
@@ -179,6 +187,9 @@ exports.buildProxyPolicy = buildProxyPolicy;
 // left goes.
 // A task follows the proxy's log while its jobs run. A service outlives the
 // run on a cluster, so its refused hosts are in the proxy's own log.
+// `secretHosts` are the endpoints of the providers whose secrets the init
+// containers of a task on a server read: a limited task gets a proxy for them
+// (under `none` too), which passes them to those containers only.
 async function ensureKubernetesItemNetwork(instance, env, item, options) {
     const network = (0, work_network_1.getEffectiveNetwork)(item.data);
     if (network.type !== 'open' && !env.networkPolicyEnforced) {
@@ -191,22 +202,30 @@ async function ensureKubernetesItemNetwork(instance, env, item, options) {
             // without the rights to manage NetworkPolicies, hammerkit never made one
             await removeKubernetesItemNetwork(instance, env, item, { run: null }, { ignoreForbidden: true });
         }
-        return { env: [], hostAliases, initContainers: [], close() { } };
+        return { env: [], secretEnv: [], hostAliases, initContainers: [], close() { } };
+    }
+    const secretHosts = options.secretHosts ?? [];
+    if (secretHosts.length > 0 && options.sandbox) {
+        // a run on a sandbox uses no account, so it reads no secret of a provider
+        throw new Error(`${item.name}: a task on a sandbox reads no secret of a provider`);
     }
     const proxy = network.type === 'allow' && options.sandbox
         ? getSandboxProxy(env, item, network, hostAliases, options.sandbox)
         : getRunProxy(item, options);
-    await (0, apply_1.apply)(instance, buildNetworkPolicy(env, item, network, options, proxy));
+    const proxied = network.type === 'allow' || secretHosts.length > 0;
+    await (0, apply_1.apply)(instance, buildNetworkPolicy(env, item, network, options, proxy, proxied));
     const initContainers = [(0, network_gate_1.getNetworkGateContainer)()];
-    if (network.type === 'none') {
+    if (!proxied) {
         if (options.sandbox) {
             await removeSandboxProxies(instance, env.namespace, { sandbox: options.sandbox.label, item });
         }
         else {
             await removeProxy(instance, env.namespace, proxy.name, [404]);
         }
-        return { env: [], hostAliases, initContainers, close() { } };
+        return { env: [], secretEnv: [], hostAliases, initContainers, close() { } };
     }
+    // under `none`, a proxy for the secrets only: it allows the item no host
+    const proxyNetwork = network.type === 'allow' ? network : { type: 'allow', hosts: [] };
     let started;
     try {
         if (proxy.sandbox) {
@@ -214,7 +233,7 @@ async function ensureKubernetesItemNetwork(instance, env, item, options) {
             await removeSandboxProxies(instance, env.namespace, { sandbox: proxy.sandbox.label, item }, (deployment) => deployment.metadata?.labels?.[PROXY_LABEL] === proxy.key);
             proxy.owner = await getFilesPodOwner(instance, env, proxy.sandbox);
         }
-        started = await ensureProxy(instance, env, item, network, hostAliases, options, proxy);
+        started = await ensureProxy(instance, env, item, proxyNetwork, hostAliases, options, proxy, secretHosts);
     }
     catch (e) {
         await removeKubernetesItemNetwork(instance, env, item, options).catch(() => undefined);
@@ -223,11 +242,17 @@ async function ensureKubernetesItemNetwork(instance, env, item, options) {
     const follow = options.followLog
         ? await followProxyLog(instance, env, item, proxy, { tailLines: started.reused ? 0 : undefined })
         : null;
+    const needs = item.needs.map((need) => need.name);
+    const toEnv = (variables) => Object.entries(variables).map(([name, value]) => ({ name, value }));
     return {
-        env: Object.entries((0, egress_proxy_1.getProxyClientEnvironment)(item.needs.map((need) => need.name))).map(([name, value]) => ({
-            name,
-            value,
-        })),
+        env: network.type === 'allow' ? toEnv((0, egress_proxy_1.getProxyClientEnvironment)(needs)) : [],
+        // the token from the proxy's Secret, which the item's own container never mounts
+        secretEnv: secretHosts.length > 0
+            ? [
+                { name: SECRET_TOKEN_VARIABLE, valueFrom: { secretKeyRef: { name: proxy.name, key: SECRET_TOKEN_KEY } } },
+                ...toEnv((0, egress_proxy_1.getProxyClientEnvironment)(needs, `$(${SECRET_TOKEN_VARIABLE})`)),
+            ]
+            : [],
         // the item reaches each TCP forward under its host's name, at the proxy
         hostAliases: [
             ...hostAliases,
@@ -286,12 +311,14 @@ async function releaseKubernetesItemNetwork(instance, env, item, scope) {
 }
 exports.releaseKubernetesItemNetwork = releaseKubernetesItemNetwork;
 // Every policy and proxy of the item, of any run, once none of its pods runs:
-// what `down` and `clean` remove.
+// what `down` and `clean` remove. On a server, those of the server's run only: the
+// runs of a caller share their namespace, and the pods of another run of the same item
+// would run unlimited without their policy.
 async function removeAllKubernetesItemNetworks(instance, env, item) {
-    if (!(await awaitPodsEnded(instance, env, { [ID_LABEL]: item.id() }))) {
+    if (!(await awaitPodsEnded(instance, env, (0, resources_1.getItemLabels)(item, env)))) {
         throw new Error(`pods of ${item.name} are still running, so its network policy stays`);
     }
-    const labelSelector = `${NETWORK_OF_LABEL}=${item.id()}`;
+    const labelSelector = toSelector({ [NETWORK_OF_LABEL]: item.id(), ...(0, job_placement_1.getRunLabels)(env) });
     let policies;
     try {
         const list = await instance.objectApi.list('networking.k8s.io/v1', 'NetworkPolicy', env.namespace, undefined, undefined, undefined, undefined, labelSelector);
@@ -318,6 +345,7 @@ async function removeAllKubernetesItemNetworks(instance, env, item) {
         if (name) {
             await ignoreStatus(instance.appsApi.deleteNamespacedDeployment({ name, namespace: env.namespace }), [404]);
             await ignoreStatus(instance.coreApi.deleteNamespacedService({ name, namespace: env.namespace }), [404]);
+            await ignoreStatus(instance.coreApi.deleteNamespacedSecret({ name, namespace: env.namespace }), [404]);
         }
     }
 }
@@ -349,7 +377,7 @@ function getProxyReadinessScript(ports) {
         `.on('error', () => process.exit(1))`);
 }
 exports.getProxyReadinessScript = getProxyReadinessScript;
-async function ensureProxy(instance, env, item, network, hostAliases, scope, proxy) {
+async function ensureProxy(instance, env, item, network, hostAliases, scope, proxy, secretHosts = []) {
     const name = proxy.name;
     const labels = { [PROXY_LABEL]: proxy.key };
     const metadata = {
@@ -388,7 +416,17 @@ async function ensureProxy(instance, env, item, network, hostAliases, scope, pro
                             name: 'proxy',
                             image: egress_proxy_1.EGRESS_PROXY_IMAGE,
                             command: (0, egress_proxy_1.getEgressProxyCommand)(),
-                            env: Object.entries((0, egress_proxy_1.getItemProxyEnvironment)(item)).map(([key, value]) => ({ name: key, value })),
+                            env: [
+                                ...Object.entries({ ...(0, egress_proxy_1.getItemProxyEnvironment)(item), ...(0, egress_proxy_1.getSecretProxyEnvironment)(secretHosts) }).map(([key, value]) => ({ name: key, value })),
+                                ...(secretHosts.length > 0
+                                    ? [
+                                        {
+                                            name: 'HAMMERKIT_SECRET_TOKEN',
+                                            valueFrom: { secretKeyRef: { name, key: SECRET_TOKEN_KEY } },
+                                        },
+                                    ]
+                                    : []),
+                            ],
                             ports: (0, egress_proxy_1.getEgressProxyPorts)(network).map((port) => ({ containerPort: port })),
                             // in the pod, since its ingress policy may keep the kubelet out;
                             // ready once it listens on every port, its TCP forwards' too
@@ -422,6 +460,21 @@ async function ensureProxy(instance, env, item, network, hostAliases, scope, pro
         ? await instance.objectApi.read(deployment).then((current) => !current.metadata?.deletionTimestamp, (e) => ((0, apply_1.statusCodeOf)(e) === 404 ? false : Promise.reject(e)))
         : false;
     await (0, apply_1.apply)(instance, buildProxyPolicy(env, item, network, scope, proxy));
+    if (secretHosts.length > 0) {
+        // new with each run's proxy, and kept while it is there: its pod read it when it started.
+        // The init containers that read secrets get it, the task not.
+        const secret = {
+            apiVersion: 'v1',
+            kind: 'Secret',
+            metadata: { ...metadata, annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() } },
+            type: 'Opaque',
+            data: { [SECRET_TOKEN_KEY]: Buffer.from((0, crypto_1.randomBytes)(32).toString('hex')).toString('base64') },
+        };
+        const exists = await instance.objectApi.read(secret).then(() => true, (e) => ((0, apply_1.statusCodeOf)(e) === 404 ? false : Promise.reject(e)));
+        if (!exists) {
+            await instance.objectApi.create(secret);
+        }
+    }
     await (0, apply_1.apply)(instance, deployment);
     const created = await (0, apply_1.apply)(instance, service);
     await (0, await_running_state_1.awaitDeployRunningState)(instance, env, name, {
@@ -433,7 +486,7 @@ async function ensureProxy(instance, env, item, network, hostAliases, scope, pro
     if (!ip) {
         throw new Error(`the egress proxy of ${item.name} has no cluster ip`);
     }
-    item.status.write('debug', `${reused ? 'egress proxy of the sandbox' : 'egress proxy'} allows ${network.hosts.join(', ') || 'no host'}`);
+    item.status.write('debug', `${reused ? 'egress proxy of the sandbox' : 'egress proxy'} allows ${network.hosts.join(', ') || 'no host'}${secretHosts.length > 0 ? `, and ${secretHosts.join(', ')} to the reads of secrets` : ''}`);
     return { ip, reused };
 }
 // Refused hosts show up in the item's log, as on Docker; of a proxy from an
@@ -468,9 +521,13 @@ async function followProxyLog(instance, env, item, proxy, options) {
         return null;
     }
 }
-async function removeProxy(instance, namespace, name, ignored) {
+// The token of a proxy for secrets goes with it; a proxy of a sandbox has none.
+async function removeProxy(instance, namespace, name, ignored, options = { token: true }) {
     await ignoreStatus(instance.appsApi.deleteNamespacedDeployment({ name, namespace }), ignored);
     await ignoreStatus(instance.coreApi.deleteNamespacedService({ name, namespace }), ignored);
+    if (options.token) {
+        await ignoreStatus(instance.coreApi.deleteNamespacedSecret({ name, namespace }), ignored);
+    }
     await ignoreStatus(instance.objectApi.delete({
         apiVersion: 'networking.k8s.io/v1',
         kind: 'NetworkPolicy',
@@ -492,7 +549,7 @@ async function removeSandboxProxies(instance, namespace, of, keep = () => false)
     for (const deployment of deployments.items) {
         const name = deployment.metadata?.name;
         if (name && !keep(deployment)) {
-            await removeProxy(instance, namespace, name, [404]);
+            await removeProxy(instance, namespace, name, [404], { token: false });
         }
     }
 }

package/dist/kubernetes/remove-persistent-data.js

@@ -1,8 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.removePersistentData = void 0;
+const resources_1 = require("./resources");
 async function removePersistentData(instance, kubernetes, task) {
-    const labelSelector = `hammerkit.dev/id=${task.id()}`;
+    // on a server, the run's: a claim it kept has no run label and goes once it expired
+    const labelSelector = (0, resources_1.getItemSelector)(task, kubernetes);
     const configmaps = await instance.coreApi.listNamespacedConfigMap({ namespace: kubernetes.namespace, labelSelector });
     for (const configMap of configmaps.items) {
         if (configMap.metadata?.name) {

package/dist/kubernetes/resources.js

@@ -1,12 +1,34 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.removeInvalidCharacters = exports.getResourceName = void 0;
+exports.removeInvalidCharacters = exports.getResourceName = exports.getItemSelector = exports.getItemLabels = exports.getItemKey = exports.ID_LABEL = void 0;
+const job_placement_1 = require("./job-placement");
 // An object name is a DNS label: lowercase letters, digits and `-`, at most 63
 // characters, starting and ending with a letter or digit. The id makes a name
 // unique; the item's name is there for people and is cut to fit.
 const MAX_NAME = 63;
-function getResourceName(item, suffix) {
-    const tail = `-${item.id()}${suffix ?? ''}`;
+exports.ID_LABEL = 'hammerkit.dev/id';
+// What tells the objects of an item apart in its namespace: its id, and on a server the
+// run's label too. The runs of a caller execute side by side in their namespace, and two
+// of them often run the same task (the same command on two sandboxes of a repository),
+// so each has objects of its own, which nothing of the other run finds or removes. The
+// first 8 digits of the run label tell runs apart, as they do for a run's proxies.
+function getItemKey(item, env) {
+    return env.server ? `${item.id().substring(0, 32)}-${(0, job_placement_1.getRunLabelValue)(env.server.run).substring(0, 8)}` : item.id();
+}
+exports.getItemKey = getItemKey;
+// The labels the objects of an item carry, by which they are found again
+function getItemLabels(item, env) {
+    return { [exports.ID_LABEL]: item.id(), ...(0, job_placement_1.getRunLabels)(env) };
+}
+exports.getItemLabels = getItemLabels;
+function getItemSelector(item, env) {
+    return Object.entries(getItemLabels(item, env))
+        .map(([key, value]) => `${key}=${value}`)
+        .join(',');
+}
+exports.getItemSelector = getItemSelector;
+function getResourceName(item, env, suffix) {
+    const tail = `-${getItemKey(item, env)}${suffix ?? ''}`;
     const name = removeInvalidCharacters(item.name)
         .slice(0, Math.max(0, MAX_NAME - tail.length))
         .replace(/-+$/, '');

package/dist/kubernetes/restore-kubernetes-data.js

@@ -1,11 +1,17 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.restoreKubernetesData = void 0;
+exports.restoreKubernetesData = exports.getRestoreTarget = void 0;
 const event_cache_1 = require("../executer/event-cache");
 const log_1 = require("../log");
-const path_1 = require("path");
 const volumes_1 = require("./volumes");
 const ensure_persistent_data_1 = require("./ensure-persistent-data");
+const claim_layout_1 = require("./claim-layout");
+// Where the upload pod (which mounts the work's claim at /dev/hammerkit/<id>) unpacks the
+// archive of an output: the output's place in the claim, which its mount reads
+function getRestoreTarget(work, path) {
+    return `/dev/hammerkit/${work.id()}/${(0, claim_layout_1.getClaimSubPath)(work, path)}`;
+}
+exports.getRestoreTarget = getRestoreTarget;
 async function restoreKubernetesData(work, kubernetes, instance, environment, path, 
 // the kept claim of the work holds what an earlier run left, which goes first
 options = {}) {
@@ -18,7 +24,7 @@ options = {}) {
             if (!(await environment.file.exists(generatedArchive.filename))) {
                 continue;
             }
-            const targetPath = `/dev/hammerkit/${work.id()}/${(0, path_1.basename)(generatedArchive.path)}`;
+            const targetPath = getRestoreTarget(work, generatedArchive.path);
             // a claim made for this run (a server removes the claims of a run when it
             // ends) does not have the directory of the output yet
             let failure = null;

package/dist/kubernetes/secrets.js

@@ -5,19 +5,18 @@ const work_secret_1 = require("../planner/work-secret");
 const apply_1 = require("./apply");
 const resources_1 = require("./resources");
 const version_1 = require("../version");
-const job_placement_1 = require("./job-placement");
 const SECRET_VOLUME_NAME = 'hammerkit-secrets';
-function getSecretResourceName(item) {
-    return (0, resources_1.getResourceName)(item, '-secrets');
+function getSecretResourceName(item, kubernetes) {
+    return (0, resources_1.getResourceName)(item, kubernetes, '-secrets');
 }
 exports.getSecretResourceName = getSecretResourceName;
 // The values go into a Secret of the item; the pod only references it, so no
 // value is part of a Job or Deployment spec.
-function getKubernetesSecretRefs(item, secrets) {
+function getKubernetesSecretRefs(item, kubernetes, secrets) {
     if (secrets.length === 0) {
         return { env: [], mounts: [], volumes: [] };
     }
-    const secretName = getSecretResourceName(item);
+    const secretName = getSecretResourceName(item, kubernetes);
     const refs = {
         env: [],
         mounts: [],
@@ -50,14 +49,11 @@ async function ensureKubernetesSecret(instance, kubernetes, item, secrets, envir
         apiVersion: 'v1',
         metadata: {
             namespace: kubernetes.namespace,
-            name: getSecretResourceName(item),
+            name: getSecretResourceName(item, kubernetes),
             annotations: {
                 'hammerkit.dev/version': (0, version_1.getVersion)(),
             },
-            labels: {
-                'hammerkit.dev/id': item.id(),
-                ...(0, job_placement_1.getRunLabels)(kubernetes),
-            },
+            labels: (0, resources_1.getItemLabels)(item, kubernetes),
         },
         type: 'Opaque',
         stringData,
@@ -68,7 +64,7 @@ exports.ensureKubernetesSecret = ensureKubernetesSecret;
 async function removeKubernetesSecret(instance, kubernetes, item) {
     try {
         await instance.coreApi.deleteNamespacedSecret({
-            name: getSecretResourceName(item),
+            name: getSecretResourceName(item, kubernetes),
             namespace: kubernetes.namespace,
         });
     }

package/dist/kubernetes/volumes.js

@@ -4,6 +4,7 @@ exports.getSandboxAffinity = exports.SANDBOX_LABEL = exports.getWorkspacePodSpec
 const path_1 = require("path");
 const fs_1 = require("fs");
 const kept_claims_1 = require("./kept-claims");
+const claim_layout_1 = require("./claim-layout");
 function getVolumeName(service, kubernetes) {
     return (0, kept_claims_1.getClaimName)(service, kubernetes);
 }
@@ -50,7 +51,7 @@ async function getKubernetesPersistence(work, kubernetes) {
         mounts: [],
         initCommands: [],
     };
-    const workspace = kubernetes?.server?.workspace;
+    const workspace = kubernetes.server?.workspace;
     for (const mount of work.data.mounts) {
         if (workspace &&
             appendWorkspaceMount(workspace, persistence, mount.containerPath, mount.localPath, mount.isFile, mount.readOnly)) {
@@ -83,6 +84,7 @@ async function getKubernetesPersistence(work, kubernetes) {
 exports.getKubernetesPersistence = getKubernetesPersistence;
 function appendVolume(sourceWork, persistence, containerPath, localPath, isFile, readOnly, kubernetes) {
     const volumeName = getVolumeName(sourceWork, kubernetes);
+    const subPath = (0, claim_layout_1.getClaimSubPath)(sourceWork, containerPath);
     const existVolume = persistence.volumes.some((v) => v.name === volumeName);
     if (!existVolume) {
         persistence.volumes.push({
@@ -92,7 +94,7 @@ function appendVolume(sourceWork, persistence, containerPath, localPath, isFile,
             },
         });
         persistence.sizes[volumeName] = 'volumeSize' in sourceWork.data ? sourceWork.data.volumeSize : null;
-        if (kubernetes && (0, kept_claims_1.isKeptClaim)(sourceWork, kubernetes)) {
+        if ((0, kept_claims_1.isKeptClaim)(sourceWork, kubernetes)) {
             persistence.kept.push(volumeName);
         }
     }
@@ -101,7 +103,7 @@ function appendVolume(sourceWork, persistence, containerPath, localPath, isFile,
             readOnly,
             mountPath: containerPath,
             name: volumeName,
-            subPath: (0, path_1.basename)(containerPath),
+            subPath,
         },
         uploadPath: `/dev/hammerkit/${sourceWork.id()}`,
         cwd: isFile ? (0, path_1.dirname)(containerPath) : containerPath,
@@ -111,6 +113,7 @@ function appendVolume(sourceWork, persistence, containerPath, localPath, isFile,
             localPath: localPath,
             matcher: () => true, // TODO
             containerPath,
+            subPath,
             stateKey: '', // TODO
         });
     }

package/dist/log.js

@@ -216,7 +216,7 @@ function writeWorkItemLogToConsole(env, log, maxNodeNameLength) {
 exports.writeWorkItemLogToConsole = writeWorkItemLogToConsole;
 function formatDate(date) {
     return colors_1.default.grey(`[${date.getFullYear()}-${(date.getMonth() + 1).toString().padStart(2, '0')}-${date
-        .getDay()
+        .getDate()
         .toString()
         .padStart(2, '0')} ${date.getHours().toString().padStart(2, '0')}:${date
         .getMinutes()