hammerkit 1.27.0-next.3 → 1.27.0-next.31

package/dist/kubernetes/ensure-persistent-data.js

@@ -1,9 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ensurePersistentData = exports.clearClaim = exports.getPodForPersistence = void 0;
+exports.ensurePersistentData = exports.packSource = exports.clearClaim = exports.getPodForPersistence = void 0;
 const apply_1 = require("./apply");
 const path_1 = require("path");
-const tar_1 = require("tar");
+const tar_1 = require("../utils/tar");
 const resources_1 = require("./resources");
 const await_running_state_1 = require("./await-running-state");
 const sleep_1 = require("../utils/sleep");
@@ -67,17 +67,14 @@ async function getPodForPersistence(instance, env, service, persistence, type, f
     for (const volume of persistence.volumes) {
         await (0, ensure_kubernetes_persistent_volume_claim_exists_1.ensureKubernetesPersistentVolumeClaimExists)(instance, env, volume, service, persistence.sizes[volume.name], persistence.kept.includes(volume.name));
     }
-    const name = (0, resources_1.getResourceName)(service, type === 'write' ? '-upload' : '-download');
+    const name = (0, resources_1.getResourceName)(service, env, type === 'write' ? '-upload' : '-download');
     const podSpec = {
         kind: 'Pod',
         apiVersion: 'v1',
         metadata: {
             namespace: env.namespace,
             name,
-            labels: {
-                'hammerkit.dev/id': service.id(),
-                ...(0, job_placement_1.getRunLabels)(env),
-            },
+            labels: (0, resources_1.getItemLabels)(service, env),
             annotations: {
                 'hammerkit.dev/version': (0, version_1.getVersion)(),
             },
@@ -104,18 +101,18 @@ async function getPodForPersistence(instance, env, service, persistence, type, f
             volumes: persistence.volumes,
         },
     };
-    service.status.console('stdout', `create pod ${name}`);
+    service.status.write('debug', `create pod ${name}`);
     await removeStalePod(instance, env.namespace, name);
     const pod = await (0, apply_1.apply)(instance, podSpec);
     if (pod.status?.phase !== 'Running') {
-        service.status.console('stdout', `await running state ${name}`);
+        service.status.write('debug', `await running state ${name}`);
         await (0, await_running_state_1.awaitRunningState)(instance, env, name, 'Running');
     }
     try {
         await fn(name);
     }
     finally {
-        service.status.console('stdout', `delete pod ${name}`);
+        service.status.write('debug', `delete pod ${name}`);
         await instance.coreApi.deleteNamespacedPod({ name, namespace: env.namespace });
     }
 }
@@ -124,7 +121,7 @@ exports.getPodForPersistence = getPodForPersistence;
 // the task runs on it or outputs are restored into it, so it holds what a new one would
 async function clearClaim(instance, env, work, pod) {
     const targetPath = `/dev/hammerkit/${work.id()}`;
-    work.status.console('stdout', `clear what an earlier run left in ${targetPath}`);
+    work.status.write('info', `clear what an earlier run left in ${targetPath}`);
     let failure = null;
     const res = await instance.exec.exec(env.namespace, pod, 'volume', ['find', targetPath, '-mindepth', '1', '-maxdepth', '1', '-exec', 'rm', '-rf', '{}', '+'], process.stdout, process.stderr, null, false, (status) => {
         if (status.status !== 'Success') {
@@ -137,6 +134,25 @@ async function clearClaim(instance, env, work, pod) {
     });
 }
 exports.clearClaim = clearClaim;
+// The archive that uploads a source into the claim: its file or directory, at its place in
+// the claim (see getClaimSubPath) instead of under its own name
+function packSource(source) {
+    const fileItem = (0, path_1.basename)(source.localPath);
+    return stream_1.Readable.from((0, tar_1.packArchive)({
+        cwd: (0, path_1.dirname)(source.localPath),
+        filter(path) {
+            return source.matcher(path, source.localPath);
+        },
+        onWriteEntry(entry) {
+            entry.path = source.subPath + entry.path.slice(fileItem.length);
+            // the second of two hard-linked files links to the first by its path in the archive
+            if (entry.type === 'Link' && entry.linkpath?.startsWith(fileItem)) {
+                entry.linkpath = source.subPath + entry.linkpath.slice(fileItem.length);
+            }
+        },
+    }, [fileItem]));
+}
+exports.packSource = packSource;
 async function ensurePersistentData(instance, env, environment, service, persistence, 
 // the kept claim of the work holds what an earlier run left, which goes first
 options = {}) {
@@ -153,7 +169,7 @@ options = {}) {
     if (persistence.sources.length === 0 && !options.reset) {
         return;
     }
-    service.status.console('stdout', 'start upload container');
+    service.status.write('debug', 'start upload container');
     await getPodForPersistence(instance, env, service, persistence, 'write', async (name) => {
         if (options.reset) {
             await clearClaim(instance, env, service, name);
@@ -163,14 +179,8 @@ options = {}) {
             if (!(await environment.file.exists(source.localPath))) {
                 continue;
             }
-            const fileItem = (0, path_1.basename)(source.localPath);
-            const data = stream_1.Readable.from((0, tar_1.create)({
-                cwd: (0, path_1.dirname)(source.localPath),
-                filter(path) {
-                    return source.matcher(path, source.localPath);
-                },
-            }, [fileItem]));
-            service.status.console('stdout', `upload ${source.localPath}`);
+            const data = packSource(source);
+            service.status.write('info', `upload ${source.localPath}`);
             const targetPath = `/dev/hammerkit/${service.id()}`;
             const res = await instance.exec.exec(env.namespace, name, 'volume', ['tar', 'xf', '-', '-C', targetPath], process.stdout, process.stderr, data, false, (status) => {
                 service.status.write('debug', `upload ${source.localPath} to ${name}: ${status}`);

package/dist/kubernetes/follow-pod-logs.js

@@ -0,0 +1,182 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.followPodLogs = void 0;
+const stream_1 = require("stream");
+const readline_1 = require("readline");
+const log_1 = require("../log");
+// what is read of an instance of a container that already ended
+const MAX_ENDED_LOG_BYTES = 1024 * 1024;
+/**
+ * Follows the log of a container in the pods of a selector, as `docker attach` does for a
+ * service on Docker: the instance that runs as it writes, and an instance that ended before
+ * it was seen (one that crashed and was restarted) once, from its start. Each instance is
+ * read once, by its container id.
+ */
+function followPodLogs(instance, namespace, labelSelector, container, onLine, options = {}) {
+    const seen = new Set();
+    const following = [];
+    const reading = new Set();
+    let stopped = false;
+    let failed = false;
+    let timer;
+    let polling = Promise.resolve();
+    let busy = false;
+    let again = false;
+    // the logs followed now, which end when their instance does
+    let streaming = 0;
+    const fail = (e) => {
+        if (!failed && !stopped) {
+            failed = true;
+            options.onError?.(`the log of ${container} could not be read: ${(0, log_1.getErrorMessage)(e)}`);
+        }
+    };
+    const track = (promise) => {
+        const tracked = promise.catch(fail).finally(() => reading.delete(tracked));
+        reading.add(tracked);
+    };
+    const readEnded = async (pod, previous) => {
+        const log = await instance.coreApi.readNamespacedPodLog({
+            name: pod,
+            namespace,
+            container,
+            previous,
+            limitBytes: MAX_ENDED_LOG_BYTES,
+        });
+        for (const line of String(log ?? '').split(/\r?\n/)) {
+            if (line.length > 0) {
+                onLine(line);
+            }
+        }
+    };
+    const follow = async (pod) => {
+        const output = new stream_1.PassThrough();
+        let ended = false;
+        // the instance ended or the request broke: a restarted one is looked for at once
+        const end = () => {
+            if (!ended) {
+                ended = true;
+                streaming--;
+                wake();
+            }
+        };
+        // stop() aborts the request, which errors the stream the client pipes in here
+        output.on('pipe', (source) => source.on('error', () => {
+            end();
+        }));
+        output.on('end', end);
+        (0, readline_1.createInterface)({ input: output }).on('line', (line) => onLine(line));
+        streaming++;
+        let controller;
+        try {
+            controller = await instance.log.log(namespace, pod, container, output, { follow: true });
+        }
+        catch (e) {
+            end();
+            throw e;
+        }
+        if (stopped) {
+            controller.abort();
+        }
+        else {
+            following.push(controller);
+        }
+    };
+    const look = (pod) => {
+        const name = pod.metadata?.name;
+        const status = pod.status?.containerStatuses?.find((s) => s.name === container);
+        // a pod that goes is of a Deployment before this one, or of a rollout that replaced it
+        if (!name || !status || pod.metadata?.deletionTimestamp) {
+            return;
+        }
+        // an instance that ended before this one, not seen while it ran: its lines go first
+        let before = Promise.resolve();
+        const last = status.lastState?.terminated?.containerID;
+        if (last && !seen.has(last)) {
+            seen.add(last);
+            before = readEnded(name, true).catch(fail);
+            track(before);
+        }
+        const current = status.containerID;
+        if (!current || seen.has(current)) {
+            return;
+        }
+        if (status.state?.running) {
+            seen.add(current);
+            track(before.then(() => follow(name)));
+        }
+        else if (status.state?.terminated) {
+            seen.add(current);
+            track(before.then(() => readEnded(name, false)));
+        }
+    };
+    const poll = async () => {
+        try {
+            const pods = await instance.coreApi.listNamespacedPod({ namespace, labelSelector });
+            if (!stopped) {
+                pods.items.forEach(look);
+            }
+        }
+        catch (e) {
+            fail(e);
+        }
+    };
+    const loop = () => {
+        if (stopped) {
+            return;
+        }
+        busy = true;
+        polling = poll().finally(() => {
+            busy = false;
+            if (stopped) {
+                return;
+            }
+            if (again) {
+                again = false;
+                loop();
+                return;
+            }
+            const interval = streaming > 0 ? options.followingInterval ?? 15000 : options.pollInterval ?? 1000;
+            timer = setTimeout(loop, interval);
+            timer.unref?.();
+        });
+    };
+    // looks now, not at the next interval
+    function wake() {
+        if (stopped) {
+            return;
+        }
+        if (busy) {
+            again = true;
+            return;
+        }
+        clearTimeout(timer);
+        loop();
+    }
+    loop();
+    return {
+        async settle() {
+            await polling;
+            await poll();
+            // what follows a running instance never ends by itself
+            const ended = [...reading];
+            let wait;
+            await Promise.race([
+                Promise.all(ended),
+                new Promise((resolve) => {
+                    wait = setTimeout(resolve, 5000);
+                    wait.unref?.();
+                }),
+            ]);
+            clearTimeout(wait);
+        },
+        stop() {
+            stopped = true;
+            clearTimeout(timer);
+            for (const controller of following.splice(0)) {
+                controller.abort();
+            }
+        },
+    };
+}
+exports.followPodLogs = followPodLogs;
+//# sourceMappingURL=follow-pod-logs.js.map

package/dist/kubernetes/get-service-ip.js

@@ -5,7 +5,7 @@ const resources_1 = require("./resources");
 // The address a task reaches the service on: its clusterIP, or for a headless
 // Service (a service without ports) the address of its ready pod.
 async function getServiceIp(instance, env, workService) {
-    const name = (0, resources_1.getResourceName)(workService);
+    const name = (0, resources_1.getResourceName)(workService, env);
     const svc = await instance.coreApi.readNamespacedService({ name, namespace: env.namespace });
     const clusterIP = svc.spec?.clusterIP ?? null;
     if (clusterIP !== 'None') {
@@ -13,7 +13,7 @@ async function getServiceIp(instance, env, workService) {
     }
     const pods = await instance.coreApi.listNamespacedPod({
         namespace: env.namespace,
-        labelSelector: `hammerkit.dev/id=${workService.id()}`,
+        labelSelector: (0, resources_1.getItemSelector)(workService, env),
     });
     const ready = pods.items.find((pod) => !pod.metadata?.deletionTimestamp &&
         pod.status?.podIP &&

package/dist/kubernetes/job-placement.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getContainerSecurity = exports.getRunLabels = exports.getRunLabelValue = exports.RUN_LABEL = exports.getPodPlacement = exports.placeOnPool = exports.getNodeArch = exports.runInToolImage = exports.getLocalTaskPool = void 0;
+exports.getPodRuntimeClass = exports.getContainerSecurity = exports.getRunLabels = exports.getRunLabelValue = exports.RUN_LABEL = exports.getPodPlacement = exports.placeOnPool = exports.getNodeArch = exports.runInToolImage = exports.getLocalTaskPool = void 0;
 const crypto_1 = require("crypto");
 /**
  * The pool a local task runs on: the first that offers its platform. A pool says
@@ -34,6 +34,7 @@ function runInToolImage(task, server, platform) {
         user: null,
         mounts: [],
         volumeSize: null,
+        capAdd: [],
         pool: pool.name,
         arch: getNodeArch(pool),
     };
@@ -82,18 +83,41 @@ exports.getRunLabels = getRunLabels;
 /**
  * What a container of a run may do on a server, where it shares nodes with the pods of
  * other callers: no privilege escalation (setuid binaries such as sudo), no raw sockets
- * (spoofing on a bridge network) and the runtime's seccomp profile.
+ * (spoofing on a bridge network) and the runtime's seccomp profile. The capabilities a
+ * task adds (`capAdd`) come on top, in the sandbox of getPodRuntimeClass on a server.
  */
-function getContainerSecurity(kubernetes) {
-    return kubernetes.server
-        ? {
-            securityContext: {
-                allowPrivilegeEscalation: false,
-                capabilities: { drop: ['NET_RAW'] },
-                seccompProfile: { type: 'RuntimeDefault' },
-            },
-        }
-        : {};
+function getContainerSecurity(kubernetes, capAdd = []) {
+    const add = capAdd.length > 0 ? { add: [...capAdd] } : {};
+    if (!kubernetes.server) {
+        return capAdd.length > 0 ? { securityContext: { capabilities: add } } : {};
+    }
+    const drop = capAdd.includes('NET_RAW') ? {} : { drop: ['NET_RAW'] };
+    return {
+        securityContext: {
+            allowPrivilegeEscalation: false,
+            capabilities: { ...drop, ...add },
+            seccompProfile: { type: 'RuntimeDefault' },
+        },
+    };
 }
 exports.getContainerSecurity = getContainerSecurity;
+/**
+ * The RuntimeClass of the pod of a task that adds capabilities on a server: the sandbox the
+ * operator named (`backend.capabilities.runtimeClass`), so the capabilities hold in its own
+ * kernel and never in the node's. Elsewhere the cluster is the build file's own.
+ */
+function getPodRuntimeClass(task, kubernetes) {
+    if (task.capAdd.length === 0 || !kubernetes.server) {
+        return {};
+    }
+    const sandbox = kubernetes.server.capabilities;
+    if (!sandbox) {
+        throw new Error(`${task.name}: adds the capabilities ${task.capAdd.join(', ')}, which this server grants in no sandboxed runtime`);
+    }
+    if (sandbox.unavailable) {
+        throw new Error(`${task.name}: adds the capabilities ${task.capAdd.join(', ')}, but ${sandbox.unavailable}`);
+    }
+    return { runtimeClassName: sandbox.runtimeClass };
+}
+exports.getPodRuntimeClass = getPodRuntimeClass;
 //# sourceMappingURL=job-placement.js.map

package/dist/kubernetes/job-secrets.js

@@ -1,8 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getJobSecretSpec = exports.removeJobCredentials = exports.assertJobAccounts = exports.ensureJobCredentials = exports.getCredentialsName = exports.assertSecretsReachable = exports.splitJobSecrets = exports.ACCOUNT_PATH = exports.SECRET_VALUES_PATH = void 0;
+exports.getJobSecretSpec = exports.removeJobCredentials = exports.assertJobAccounts = exports.ensureJobCredentials = exports.getCredentialsName = exports.getSecretHosts = exports.splitJobSecrets = exports.ACCOUNT_PATH = exports.SECRET_VALUES_PATH = void 0;
 const work_secret_1 = require("../planner/work-secret");
 const command_secret_provider_1 = require("../secrets/command-secret-provider");
+const task_arguments_1 = require("../planner/utils/task-arguments");
 const version_1 = require("../version");
 const apply_1 = require("./apply");
 const resources_1 = require("./resources");
@@ -42,25 +43,30 @@ function splitJobSecrets(secrets, server) {
 exports.splitJobSecrets = splitJobSecrets;
 /**
  * The init containers that read secrets share the pod, and so the NetworkPolicy, of
- * the task: under a limited network they could not reach the secret manager. Until
- * they run in a pod of their own, a task that reads a provider's secret in its Job
- * keeps an open network.
+ * the task: under a limited network they reach the secret manager only through the
+ * task's egress proxy, which lets them reach the endpoints the server names for the
+ * provider (`backend.providers.<name>.allow`). A task that reads a secret of a provider
+ * without them keeps an open network. Answers the endpoints the proxy lets through.
  */
-function assertSecretsReachable(task, secrets, network = (0, work_network_1.getEffectiveNetwork)(task.data)) {
-    if (secrets.length === 0 || network.type === 'open') {
-        return;
+function getSecretHosts(task, secrets, server, network = (0, work_network_1.getEffectiveNetwork)(task.data)) {
+    if (!server || secrets.length === 0 || network.type === 'open') {
+        return [];
+    }
+    const unreachable = secrets.find((secret) => !server.providers[secret.source.provider]?.allow?.length);
+    if (unreachable) {
+        throw new Error(`task ${task.name}: secret ${(0, work_secret_1.getSecretName)(unreachable.secret)} is read from ${(0, work_secret_1.describeSecretSource)(unreachable.secret)} in the Job of the task, which network ${(0, work_network_1.formatNetwork)(network)} would cut off from it; ` +
+            `on this cluster a task that reads secrets of ${unreachable.source.provider} keeps an open network, ` +
+            `unless the server names its endpoints in backend.providers.${unreachable.source.provider}.allow`);
     }
-    const secret = secrets[0];
-    throw new Error(`task ${task.name}: secret ${(0, work_secret_1.getSecretName)(secret.secret)} is read from ${(0, work_secret_1.describeSecretSource)(secret.secret)} in the Job of the task, which network ${(0, work_network_1.formatNetwork)(network)} would cut off from it; ` +
-        'on this cluster a task that reads secrets of a provider keeps an open network');
+    return [...new Set(secrets.flatMap((secret) => server.providers[secret.source.provider].allow ?? []))].sort();
 }
-exports.assertSecretsReachable = assertSecretsReachable;
+exports.getSecretHosts = getSecretHosts;
 // The accounts of the secrets, in the order they are first read
 function accountsOf(secrets) {
     return [...new Set(secrets.map((s) => s.source.account))];
 }
-function getCredentialsName(item) {
-    return (0, resources_1.getResourceName)(item, '-credentials');
+function getCredentialsName(item, kubernetes) {
+    return (0, resources_1.getResourceName)(item, kubernetes, '-credentials');
 }
 exports.getCredentialsName = getCredentialsName;
 // The keys of the credentials Secret: `<account>.token` and `<account>.file.<key>`
@@ -130,9 +136,9 @@ async function ensureJobCredentials(instance, kubernetes, item, secrets, admitte
         apiVersion: 'v1',
         metadata: {
             namespace: kubernetes.namespace,
-            name: getCredentialsName(item),
+            name: getCredentialsName(item, kubernetes),
             annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
-            labels: { 'hammerkit.dev/id': item.id(), ...(0, job_placement_1.getRunLabels)(kubernetes) },
+            labels: (0, resources_1.getItemLabels)(item, kubernetes),
         },
         type: 'Opaque',
         data,
@@ -163,7 +169,10 @@ async function removeJobCredentials(instance, kubernetes, item) {
         return;
     }
     try {
-        await instance.coreApi.deleteNamespacedSecret({ name: getCredentialsName(item), namespace: kubernetes.namespace });
+        await instance.coreApi.deleteNamespacedSecret({
+            name: getCredentialsName(item, kubernetes),
+            namespace: kubernetes.namespace,
+        });
     }
     catch (e) {
         if ((0, apply_1.statusCodeOf)(e) !== 404) {
@@ -186,7 +195,9 @@ function quote(value) {
  * from it, and the task's output is masked before it leaves the pod, since the
  * server, which masks everything else, never sees these values.
  */
-function getJobSecretSpec(item, secrets, server, credentials, environment) {
+function getJobSecretSpec(item, secrets, server, credentials, environment, 
+// under a limited network, what sends their requests through the egress proxy
+proxyEnv = []) {
     if (!server || secrets.length === 0) {
         return {
             initContainers: [],
@@ -205,7 +216,7 @@ function getJobSecretSpec(item, secrets, server, credentials, environment) {
         }
         const name = `hammerkit-account-${index}`;
         accountVolume.set(account, name);
-        volumes.push({ name, secret: { secretName: getCredentialsName(item), items, defaultMode: 0o444 } });
+        volumes.push({ name, secret: { secretName: getCredentialsName(item, { server }), items, defaultMode: 0o444 } });
     });
     const initContainers = secrets.map(({ index, secret, source }) => {
         const label = `secret ${(0, work_secret_1.getSecretName)(secret)} (${(0, work_secret_1.describeSecretSource)(secret)})`;
@@ -229,8 +240,11 @@ function getJobSecretSpec(item, secrets, server, credentials, environment) {
                 '-c',
                 'umask 022 && "$@" > "$HAMMERKIT_VALUE.part" && mv "$HAMMERKIT_VALUE.part" "$HAMMERKIT_VALUE"',
             ],
-            args: ['sh', command, ...args],
+            // Kubernetes would expand a `$(NAME)` of the container's env in them, the proxy's token
+            // among them: the reference is text, as on any other backend
+            args: ['sh', command, ...args].map(task_arguments_1.escapeKubernetesText),
             env: [
+                ...proxyEnv,
                 ...Object.entries(env).map(([name, value]) => ({ name, value })),
                 { name: 'HAMMERKIT_VALUE', value: `${exports.SECRET_VALUES_PATH}/secret-${index}` },
             ],

package/dist/kubernetes/kept-claims.js

@@ -5,6 +5,7 @@ const crypto_1 = require("crypto");
 const apply_1 = require("./apply");
 const sleep_1 = require("../utils/sleep");
 const job_placement_1 = require("./job-placement");
+const resources_1 = require("./resources");
 // On a server with `backend.keepVolumes`, the claim of a task stays after the run, so the
 // next run of the same source finds the outputs in it and mounts them without restoring
 // them from the cache. A kept claim has no run label (the end of the run removes what has
@@ -18,11 +19,15 @@ exports.STATE_ANNOTATION = 'hammerkit.dev/state';
 // when it goes, set by the end of the run that mounted it last
 exports.EXPIRES_ANNOTATION = 'hammerkit.dev/expires';
 // The claim of a work item: a task's on a server that keeps them carries the scope, so a
-// run of another source never mounts it. 63 characters at most, a pod's volume name.
+// run of another source never mounts it, and the runs of one scope go one at a time
+// (KubernetesExecutor.exclusive). Any other claim on a server is the run's own, as the
+// runs of a caller share their namespace. 63 characters at most, a pod's volume name.
 function getClaimName(work, kubernetes) {
-    const scope = kubernetes?.server?.volumeScope;
-    const name = 'hammerkit-' + work.id();
-    return scope && isTask(work) ? `${name}-${(0, crypto_1.createHash)('sha256').update(scope).digest('hex').slice(0, 12)}` : name;
+    const scope = kubernetes.server?.volumeScope;
+    if (scope && isTask(work)) {
+        return `hammerkit-${work.id()}-${(0, crypto_1.createHash)('sha256').update(scope).digest('hex').slice(0, 12)}`;
+    }
+    return 'hammerkit-' + (0, resources_1.getItemKey)(work, kubernetes);
 }
 exports.getClaimName = getClaimName;
 // whether the claim of the work item stays after the run
@@ -156,14 +161,17 @@ async function expireKeptClaims(instance, namespace, runLabel, expires) {
     }
 }
 exports.expireKeptClaims = expireKeptClaims;
-// While no run of the caller is in the namespace: the kept claims that expired, and those
-// a run left without an expiry because the server stopped before its end (all of them
-// when the server keeps none any more). The names of those removed, and when the next of
-// the others expires.
+// The kept claims that expired, and those a run left without an expiry because the server
+// stopped before its end (all of them when the server keeps none any more), but those of
+// the scopes `inUse`: a run of the scope executes and mounts them, or may in a moment.
+// The names of those removed, and when the next of the others expires.
 async function sweepKeptClaims(instance, namespace, options) {
     const expired = [];
     let next = null;
     for (const claim of await listKeptClaims(instance, namespace)) {
+        if (options.inUse?.has(claim.metadata?.labels?.[exports.VOLUME_SCOPE_LABEL] ?? '')) {
+            continue;
+        }
         const expires = Date.parse(claim.metadata?.annotations?.[exports.EXPIRES_ANNOTATION] ?? '');
         if (!options.keep || !(expires > options.now.getTime())) {
             expired.push(claim);
@@ -178,14 +186,18 @@ async function sweepKeptClaims(instance, namespace, options) {
 exports.sweepKeptClaims = sweepKeptClaims;
 // When a run needs room under the quota of its namespace: the kept claims it does not use
 // go, those of the other sources and those of its own a task of this run did not take (one
-// renamed or changed since). The names of those removed.
+// renamed or changed since). Of another scope, only one its last run released, which has an
+// expiry: one without is mounted by a run of the caller that executes now (or was left by
+// a server that stopped, which the next sweep removes). The names of those removed.
 async function evictKeptClaims(instance, kubernetes, timeout) {
     const server = kubernetes.server;
     if (!server?.volumeScope) {
         return [];
     }
     const unused = await listKeptClaims(instance, kubernetes.namespace, `${exports.VOLUME_SCOPE_LABEL},${exports.KEPT_BY_LABEL}!=${(0, job_placement_1.getRunLabelValue)(server.run)}`);
-    return removeClaims(instance, kubernetes.namespace, unused, timeout);
+    const released = unused.filter((claim) => claim.metadata?.labels?.[exports.VOLUME_SCOPE_LABEL] === server.volumeScope ||
+        !!claim.metadata?.annotations?.[exports.EXPIRES_ANNOTATION]);
+    return removeClaims(instance, kubernetes.namespace, released, timeout);
 }
 exports.evictKeptClaims = evictKeptClaims;
 //# sourceMappingURL=kept-claims.js.map

package/dist/kubernetes/kubernetes-instance.js

@@ -24,6 +24,7 @@ function createKubernetesInstances(kubernetes) {
         return {
             objectApi,
             log: new client_node_1.Log(kc),
+            metrics: new client_node_1.Metrics(kc),
             exec,
             appsApi,
             coreApi,