hammerkit 1.27.0-next.9 → 2.0.0-next.2

package/dist/kubernetes/ensure-persistent-data.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ensurePersistentData = exports.packSource = exports.clearClaim = exports.getPodForPersistence = void 0;
+exports.ensurePersistentData = exports.packSource = exports.restoreLiveFiles = exports.syncLiveFiles = exports.SYNC_FILES = exports.getStateDirsInUse = exports.REMOVE_OTHER_STATES = exports.execInPod = exports.clearClaim = exports.getPodForPersistence = void 0;
+const volumes_1 = require("./volumes");
 const apply_1 = require("./apply");
 const path_1 = require("path");
 const tar_1 = require("../utils/tar");
@@ -68,18 +69,24 @@ async function getPodForPersistence(instance, env, service, persistence, type, f
         await (0, ensure_kubernetes_persistent_volume_claim_exists_1.ensureKubernetesPersistentVolumeClaimExists)(instance, env, volume, service, persistence.sizes[volume.name], persistence.kept.includes(volume.name));
     }
     const name = (0, resources_1.getResourceName)(service, env, type === 'write' ? '-upload' : '-download');
+    // a service of a service run is up while its claim takes an upload (a move to another
+    // commit): the pod is not one its Service sends to
+    const serving = !!env.server?.serviceRun && service.data.type === 'container-service';
+    // and on a service run, it goes where the pods that mount the same claims run
+    const claims = env.server?.serviceRun ? (0, volumes_1.getClaimPodSpec)(persistence.volumes) : { labels: {} };
     const podSpec = {
         kind: 'Pod',
         apiVersion: 'v1',
         metadata: {
             namespace: env.namespace,
             name,
-            labels: (0, resources_1.getItemLabels)(service, env),
+            labels: { ...(serving ? (0, job_placement_1.getRunLabels)(env) : (0, resources_1.getItemLabels)(service, env)), ...claims.labels },
             annotations: {
                 'hammerkit.dev/version': (0, version_1.getVersion)(),
             },
         },
         spec: {
+            ...(claims.affinity ? { affinity: claims.affinity } : {}),
             // on a server the claims are bound where the pod of the task can go: the nodes
             // of its pool
             ...(env.server
@@ -122,18 +129,182 @@ exports.getPodForPersistence = getPodForPersistence;
 async function clearClaim(instance, env, work, pod) {
     const targetPath = `/dev/hammerkit/${work.id()}`;
     work.status.write('info', `clear what an earlier run left in ${targetPath}`);
+    const failure = await execInPod(instance, env, pod, [
+        'find',
+        targetPath,
+        '-mindepth',
+        '1',
+        '-maxdepth',
+        '1',
+        '-exec',
+        'rm',
+        '-rf',
+        '{}',
+        '+',
+    ]);
+    if (failure) {
+        throw new Error(`unable to clear ${targetPath} in ${pod}: ${failure}`);
+    }
+}
+exports.clearClaim = clearClaim;
+// Runs `command` in the upload pod `pod`: what failed, or null when it went well
+async function execInPod(instance, env, pod, command) {
     let failure = null;
-    const res = await instance.exec.exec(env.namespace, pod, 'volume', ['find', targetPath, '-mindepth', '1', '-maxdepth', '1', '-exec', 'rm', '-rf', '{}', '+'], process.stdout, process.stderr, null, false, (status) => {
+    const res = await instance.exec.exec(env.namespace, pod, 'volume', command, process.stdout, process.stderr, null, false, (status) => {
         if (status.status !== 'Success') {
             failure = status.message ?? status.reason ?? 'unknown';
         }
     });
     await new Promise((resolve, reject) => {
         res.on('error', reject);
-        res.on('close', () => failure ? reject(new Error(`unable to clear ${targetPath} in ${pod}: ${failure}`)) : resolve());
+        res.on('close', () => resolve());
     });
+    return failure;
+}
+exports.execInPod = execInPod;
+// marks the folder of a state whose upload went through
+const UPLOADED = '.uploaded';
+// Removes the folders of the states (see getStateFilesDir) in the claim at `$1` but those
+// named after it: the one the work runs at, and the ones pods of the run mount
+// (it says how it went itself: a loop ends with the status of its last test in some shells)
+exports.REMOVE_OTHER_STATES = `cd "$1" || exit 1
+shift
+for d in ${volumes_1.STATE_FILES_PREFIX}*; do
+  if [ ! -d "$d" ]; then continue; fi
+  keep=
+  for k in "$@"; do
+    if [ "$d" = "$k" ]; then keep=1; fi
+  done
+  if [ -z "$keep" ]; then rm -rf "$d" || exit 1; fi
+done
+exit 0`;
+/**
+ * The folders of the states (see getStateFilesDir) of the claim `claim` that the services of
+ * the run mount: the pods of its Deployments, which serve or start. A pod of a service mounts
+ * the outputs of a task only after the task completed, so what it mounts is whole.
+ */
+async function getStateDirsInUse(instance, namespace, claim) {
+    const pods = await instance.coreApi.listNamespacedPod({ namespace });
+    const dirs = new Set();
+    for (const pod of pods.items) {
+        const phase = pod.status?.phase;
+        if ((phase !== 'Running' && phase !== 'Pending') ||
+            !pod.metadata?.ownerReferences?.some((owner) => owner.kind === 'ReplicaSet')) {
+            continue;
+        }
+        const volumes = new Set((pod.spec?.volumes ?? [])
+            .filter((volume) => volume.persistentVolumeClaim?.claimName === claim)
+            .map((volume) => volume.name));
+        for (const container of [...(pod.spec?.initContainers ?? []), ...(pod.spec?.containers ?? [])]) {
+            for (const mount of container.volumeMounts ?? []) {
+                const dir = mount.subPath?.split('/')[0];
+                if (volumes.has(mount.name) && dir?.startsWith(volumes_1.STATE_FILES_PREFIX)) {
+                    dirs.add(dir);
+                }
+            }
+        }
+    }
+    return dirs;
+}
+exports.getStateDirsInUse = getStateDirsInUse;
+// Syncs the files in the claim from the folder at `$1` into the folder at `$2` (see
+// LIVE_FILES_DIR), which the pod of a service mounts while it runs: what is new or changed
+// is written, what is gone is removed, and nothing else is touched, so the service sees only
+// the changes. A file is written in place, and the paths the pod mounts (the others) are never
+// removed: the pod holds them by their inode, and would not see one made again.
+exports.SYNC_FILES = `src="$1"
+dst="$2"
+shift 2
+if [ ! -d "$src" ]; then echo "no files at $src" >&2; exit 1; fi
+mkdir -p "$dst" || exit 1
+cd "$src" || exit 1
+find . -mindepth 1 ! -path ./${UPLOADED} | while IFS= read -r f; do
+  t="$dst/$f"
+  if [ -L "$f" ]; then
+    l=$(readlink "$f")
+    if [ -L "$t" ] && [ "$(readlink "$t")" = "$l" ]; then continue; fi
+    rm -rf "$t" && ln -s "$l" "$t" || exit 1
+  elif [ -d "$f" ]; then
+    if [ -L "$t" ] || { [ -e "$t" ] && [ ! -d "$t" ]; }; then rm -rf "$t" || exit 1; fi
+    mkdir -p "$t" || exit 1
+  else
+    if [ -L "$t" ] || [ -d "$t" ]; then rm -rf "$t" || exit 1; fi
+    if [ -e "$t" ]; then
+      if cmp -s "$f" "$t"; then continue; fi
+      cat "$f" > "$t" && chmod "$(stat -c %a "$f")" "$t" || exit 1
+    else
+      cp -p "$f" "$t" || exit 1
+    fi
+  fi
+done || exit 1
+cd "$dst" || exit 1
+find . -mindepth 1 -depth | while IFS= read -r f; do
+  if [ -e "$src/$f" ] || [ -L "$src/$f" ]; then continue; fi
+  mounted=
+  for m in "$@"; do
+    if [ "$f" = "./$m" ]; then mounted=1; fi
+  done
+  if [ -z "$mounted" ]; then rm -rf "$f" || exit 1; fi
+done || exit 1
+exit 0`;
+/**
+ * Syncs the files of the state `from` (see getStateFilesDir) into the folder the pod of a
+ * service that watches its files mounts them from (LIVE_FILES_DIR).
+ */
+async function syncLiveFiles(instance, env, work, persistence, pod, from) {
+    const root = `/dev/hammerkit/${work.id()}`;
+    // the paths of the folder its pod mounts
+    const mounted = persistence.mounts.flatMap(({ mount }) => mount.subPath?.startsWith(`${volumes_1.LIVE_FILES_DIR}/`) ? [mount.subPath.slice(volumes_1.LIVE_FILES_DIR.length + 1)] : []);
+    const failure = await execInPod(instance, env, pod, [
+        'sh',
+        '-c',
+        exports.SYNC_FILES,
+        'sh',
+        `${root}/${from}`,
+        `${root}/${volumes_1.LIVE_FILES_DIR}`,
+        ...mounted,
+    ]);
+    if (failure) {
+        throw new Error(`unable to sync the files of ${work.name} into ${root}/${volumes_1.LIVE_FILES_DIR}: ${failure}`);
+    }
+}
+exports.syncLiveFiles = syncLiveFiles;
+// Puts the files of the state `from` back into the folder the pod of a service that watches
+// its files mounts them from: a move that did not get up leaves the service at the commit
+// it was at
+async function restoreLiveFiles(instance, env, work, persistence, from) {
+    await getPodForPersistence(instance, env, work, persistence, 'write', (pod) => syncLiveFiles(instance, env, work, persistence, pod, from));
+}
+exports.restoreLiveFiles = restoreLiveFiles;
+// Prepares the folder of a state in the claim of a work item of a service run before its
+// upload: the folders of other states go, but the ones pods of the run mount. A folder a pod
+// mounts is not touched: when it is the one of this state, its files are there already.
+// Returns whether the upload is needed.
+async function prepareStateFiles(instance, env, work, pod, dir, inUse) {
+    const targetPath = `/dev/hammerkit/${work.id()}`;
+    const removed = await execInPod(instance, env, pod, [
+        'sh',
+        '-c',
+        exports.REMOVE_OTHER_STATES,
+        'sh',
+        targetPath,
+        dir,
+        ...[...inUse].sort(),
+    ]);
+    if (removed) {
+        throw new Error(`unable to remove the files of other commits in ${targetPath}: ${removed}`);
+    }
+    if (inUse.has(dir) && !(await execInPod(instance, env, pod, ['test', '-e', `${targetPath}/${dir}/${UPLOADED}`]))) {
+        work.status.write('info', `the files of ${work.name} are there already`);
+        return false;
+    }
+    // what a failed upload or task left: tar does not remove files the commit does not have
+    const cleared = await execInPod(instance, env, pod, ['rm', '-rf', `${targetPath}/${dir}`]);
+    if (cleared) {
+        throw new Error(`unable to clear ${targetPath}/${dir}: ${cleared}`);
+    }
+    return true;
 }
-exports.clearClaim = clearClaim;
 // The archive that uploads a source into the claim: its file or directory, at its place in
 // the claim (see getClaimSubPath) instead of under its own name
 function packSource(source) {
@@ -166,14 +337,31 @@ options = {}) {
     for (const volume of persistence.volumes) {
         await (0, ensure_kubernetes_persistent_volume_claim_exists_1.ensureKubernetesPersistentVolumeClaimExists)(instance, env, volume, service, persistence.sizes[volume.name], persistence.kept.includes(volume.name));
     }
-    if (persistence.sources.length === 0 && !options.reset) {
+    // the folder of the state is in the work's own claim: one that has none (a service that
+    // only mounts what its deps made) has nothing to prepare
+    const claim = (0, volumes_1.getVolumeName)(service, env);
+    const files = persistence.volumes.some((volume) => volume.persistentVolumeClaim?.claimName === claim)
+        ? options.files
+        : undefined;
+    if (persistence.sources.length === 0 && !options.reset && !files) {
         return;
     }
+    // the folders of the states the pods of the run mount, before the upload pod mounts the claim
+    const inUse = files ? await getStateDirsInUse(instance, env.namespace, claim) : new Set();
+    for (const dir of files?.keep ?? []) {
+        inUse.add(dir);
+    }
     service.status.write('debug', 'start upload container');
     await getPodForPersistence(instance, env, service, persistence, 'write', async (name) => {
         if (options.reset) {
             await clearClaim(instance, env, service, name);
         }
+        if (files && !(await prepareStateFiles(instance, env, service, name, files.dir, inUse))) {
+            if (files.live && service.data.type === 'container-service') {
+                await syncLiveFiles(instance, env, service, persistence, name, files.dir);
+            }
+            return;
+        }
         for (const source of persistence.sources) {
             // skip sources that don't exist on disk — nothing to upload
             if (!(await environment.file.exists(source.localPath))) {
@@ -204,6 +392,23 @@ options = {}) {
                 }
             });
         }
+        if (files) {
+            const dir = `/dev/hammerkit/${service.id()}/${files.dir}`;
+            const marked = await execInPod(instance, env, name, [
+                'sh',
+                '-c',
+                'mkdir -p "$1" && touch "$1/$2"',
+                'sh',
+                dir,
+                UPLOADED,
+            ]);
+            if (marked) {
+                throw new Error(`unable to mark the upload of ${dir}: ${marked}`);
+            }
+            if (files.live && service.data.type === 'container-service') {
+                await syncLiveFiles(instance, env, service, persistence, name, files.dir);
+            }
+        }
     });
 }
 exports.ensurePersistentData = ensurePersistentData;

package/dist/kubernetes/follow-pod-logs.js

@@ -0,0 +1,182 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.followPodLogs = void 0;
+const stream_1 = require("stream");
+const readline_1 = require("readline");
+const log_1 = require("../log");
+// what is read of an instance of a container that already ended
+const MAX_ENDED_LOG_BYTES = 1024 * 1024;
+/**
+ * Follows the log of a container in the pods of a selector, as `docker attach` does for a
+ * service on Docker: the instance that runs as it writes, and an instance that ended before
+ * it was seen (one that crashed and was restarted) once, from its start. Each instance is
+ * read once, by its container id.
+ */
+function followPodLogs(instance, namespace, labelSelector, container, onLine, options = {}) {
+    const seen = new Set();
+    const following = [];
+    const reading = new Set();
+    let stopped = false;
+    let failed = false;
+    let timer;
+    let polling = Promise.resolve();
+    let busy = false;
+    let again = false;
+    // the logs followed now, which end when their instance does
+    let streaming = 0;
+    const fail = (e) => {
+        if (!failed && !stopped) {
+            failed = true;
+            options.onError?.(`the log of ${container} could not be read: ${(0, log_1.getErrorMessage)(e)}`);
+        }
+    };
+    const track = (promise) => {
+        const tracked = promise.catch(fail).finally(() => reading.delete(tracked));
+        reading.add(tracked);
+    };
+    const readEnded = async (pod, previous) => {
+        const log = await instance.coreApi.readNamespacedPodLog({
+            name: pod,
+            namespace,
+            container,
+            previous,
+            limitBytes: MAX_ENDED_LOG_BYTES,
+        });
+        for (const line of String(log ?? '').split(/\r?\n/)) {
+            if (line.length > 0) {
+                onLine(line);
+            }
+        }
+    };
+    const follow = async (pod) => {
+        const output = new stream_1.PassThrough();
+        let ended = false;
+        // the instance ended or the request broke: a restarted one is looked for at once
+        const end = () => {
+            if (!ended) {
+                ended = true;
+                streaming--;
+                wake();
+            }
+        };
+        // stop() aborts the request, which errors the stream the client pipes in here
+        output.on('pipe', (source) => source.on('error', () => {
+            end();
+        }));
+        output.on('end', end);
+        (0, readline_1.createInterface)({ input: output }).on('line', (line) => onLine(line));
+        streaming++;
+        let controller;
+        try {
+            controller = await instance.log.log(namespace, pod, container, output, { follow: true });
+        }
+        catch (e) {
+            end();
+            throw e;
+        }
+        if (stopped) {
+            controller.abort();
+        }
+        else {
+            following.push(controller);
+        }
+    };
+    const look = (pod) => {
+        const name = pod.metadata?.name;
+        const status = pod.status?.containerStatuses?.find((s) => s.name === container);
+        // a pod that goes is of a Deployment before this one, or of a rollout that replaced it
+        if (!name || !status || pod.metadata?.deletionTimestamp) {
+            return;
+        }
+        // an instance that ended before this one, not seen while it ran: its lines go first
+        let before = Promise.resolve();
+        const last = status.lastState?.terminated?.containerID;
+        if (last && !seen.has(last)) {
+            seen.add(last);
+            before = readEnded(name, true).catch(fail);
+            track(before);
+        }
+        const current = status.containerID;
+        if (!current || seen.has(current)) {
+            return;
+        }
+        if (status.state?.running) {
+            seen.add(current);
+            track(before.then(() => follow(name)));
+        }
+        else if (status.state?.terminated) {
+            seen.add(current);
+            track(before.then(() => readEnded(name, false)));
+        }
+    };
+    const poll = async () => {
+        try {
+            const pods = await instance.coreApi.listNamespacedPod({ namespace, labelSelector });
+            if (!stopped) {
+                pods.items.forEach(look);
+            }
+        }
+        catch (e) {
+            fail(e);
+        }
+    };
+    const loop = () => {
+        if (stopped) {
+            return;
+        }
+        busy = true;
+        polling = poll().finally(() => {
+            busy = false;
+            if (stopped) {
+                return;
+            }
+            if (again) {
+                again = false;
+                loop();
+                return;
+            }
+            const interval = streaming > 0 ? options.followingInterval ?? 15000 : options.pollInterval ?? 1000;
+            timer = setTimeout(loop, interval);
+            timer.unref?.();
+        });
+    };
+    // looks now, not at the next interval
+    function wake() {
+        if (stopped) {
+            return;
+        }
+        if (busy) {
+            again = true;
+            return;
+        }
+        clearTimeout(timer);
+        loop();
+    }
+    loop();
+    return {
+        async settle() {
+            await polling;
+            await poll();
+            // what follows a running instance never ends by itself
+            const ended = [...reading];
+            let wait;
+            await Promise.race([
+                Promise.all(ended),
+                new Promise((resolve) => {
+                    wait = setTimeout(resolve, 5000);
+                    wait.unref?.();
+                }),
+            ]);
+            clearTimeout(wait);
+        },
+        stop() {
+            stopped = true;
+            clearTimeout(timer);
+            for (const controller of following.splice(0)) {
+                controller.abort();
+            }
+        },
+    };
+}
+exports.followPodLogs = followPodLogs;
+//# sourceMappingURL=follow-pod-logs.js.map

package/dist/kubernetes/job-placement.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getContainerSecurity = exports.getRunLabels = exports.getRunLabelValue = exports.RUN_LABEL = exports.getPodPlacement = exports.placeOnPool = exports.getNodeArch = exports.runInToolImage = exports.getLocalTaskPool = void 0;
+exports.getPodRuntimeClass = exports.getContainerSecurity = exports.getRunLabels = exports.getRunLabelValue = exports.RUN_LABEL = exports.getPodPlacement = exports.placeOnPool = exports.getNodeArch = exports.runInToolImage = exports.getLocalTaskPool = void 0;
 const crypto_1 = require("crypto");
 /**
  * The pool a local task runs on: the first that offers its platform. A pool says
@@ -34,6 +34,7 @@ function runInToolImage(task, server, platform) {
         user: null,
         mounts: [],
         volumeSize: null,
+        capAdd: [],
         pool: pool.name,
         arch: getNodeArch(pool),
     };
@@ -82,18 +83,41 @@ exports.getRunLabels = getRunLabels;
 /**
  * What a container of a run may do on a server, where it shares nodes with the pods of
  * other callers: no privilege escalation (setuid binaries such as sudo), no raw sockets
- * (spoofing on a bridge network) and the runtime's seccomp profile.
+ * (spoofing on a bridge network) and the runtime's seccomp profile. The capabilities a
+ * task adds (`capAdd`) come on top, in the sandbox of getPodRuntimeClass on a server.
  */
-function getContainerSecurity(kubernetes) {
-    return kubernetes.server
-        ? {
-            securityContext: {
-                allowPrivilegeEscalation: false,
-                capabilities: { drop: ['NET_RAW'] },
-                seccompProfile: { type: 'RuntimeDefault' },
-            },
-        }
-        : {};
+function getContainerSecurity(kubernetes, capAdd = []) {
+    const add = capAdd.length > 0 ? { add: [...capAdd] } : {};
+    if (!kubernetes.server) {
+        return capAdd.length > 0 ? { securityContext: { capabilities: add } } : {};
+    }
+    const drop = capAdd.includes('NET_RAW') ? {} : { drop: ['NET_RAW'] };
+    return {
+        securityContext: {
+            allowPrivilegeEscalation: false,
+            capabilities: { ...drop, ...add },
+            seccompProfile: { type: 'RuntimeDefault' },
+        },
+    };
 }
 exports.getContainerSecurity = getContainerSecurity;
+/**
+ * The RuntimeClass of the pod of a task that adds capabilities on a server: the sandbox the
+ * operator named (`backend.capabilities.runtimeClass`), so the capabilities hold in its own
+ * kernel and never in the node's. Elsewhere the cluster is the build file's own.
+ */
+function getPodRuntimeClass(task, kubernetes) {
+    if (task.capAdd.length === 0 || !kubernetes.server) {
+        return {};
+    }
+    const sandbox = kubernetes.server.capabilities;
+    if (!sandbox) {
+        throw new Error(`${task.name}: adds the capabilities ${task.capAdd.join(', ')}, which this server grants in no sandboxed runtime`);
+    }
+    if (sandbox.unavailable) {
+        throw new Error(`${task.name}: adds the capabilities ${task.capAdd.join(', ')}, but ${sandbox.unavailable}`);
+    }
+    return { runtimeClassName: sandbox.runtimeClass };
+}
+exports.getPodRuntimeClass = getPodRuntimeClass;
 //# sourceMappingURL=job-placement.js.map

package/dist/kubernetes/kubernetes-instance.js

@@ -24,6 +24,7 @@ function createKubernetesInstances(kubernetes) {
         return {
             objectApi,
             log: new client_node_1.Log(kc),
+            metrics: new client_node_1.Metrics(kc),
             exec,
             appsApi,
             coreApi,

package/dist/kubernetes/network-policy.js

@@ -129,7 +129,8 @@ function buildNetworkPolicy(env, item, network, scope, proxy = getRunProxy(item,
             namespace: env.namespace,
             name: getNetworkPolicyName(item, scope),
             annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
-            labels: { [NETWORK_OF_LABEL]: item.id(), ...(0, job_placement_1.getRunLabels)(env) },
+            labels: { [NETWORK_OF_LABEL]: item.id(), ...(scope.owners ? {} : (0, job_placement_1.getRunLabels)(env)) },
+            ...(scope.owners ? { ownerReferences: scope.owners } : {}),
         },
         spec: {
             podSelector: { matchLabels: getItemPodLabels(item, scope) },

package/dist/kubernetes/remove-stale-items.js

@@ -0,0 +1,46 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getStaleItemSelector = exports.removeStaleItems = void 0;
+const job_placement_1 = require("./job-placement");
+const resources_1 = require("./resources");
+const apply_1 = require("./apply");
+/**
+ * What a service run made for items it has no more, once it moved to another commit: the
+ * Deployment, Service, Secret, policies and Jobs of a service or task whose id changed (a new
+ * image, other volumes). Their claims stay, with the namespace of the run.
+ */
+async function removeStaleItems(instance, env, ids) {
+    if (!env.server) {
+        return;
+    }
+    const namespace = env.namespace;
+    const labelSelector = getStaleItemSelector(env.server.run, ids);
+    const selector = { namespace, labelSelector };
+    await instance.appsApi.deleteCollectionNamespacedDeployment({ ...selector, propagationPolicy: 'Background' });
+    await instance.batchApi.deleteCollectionNamespacedJob({ ...selector, propagationPolicy: 'Background' });
+    await instance.coreApi.deleteCollectionNamespacedService(selector);
+    await instance.coreApi.deleteCollectionNamespacedSecret(selector);
+    const policies = await instance.objectApi.list('networking.k8s.io/v1', 'NetworkPolicy', namespace, undefined, undefined, undefined, undefined, labelSelector);
+    for (const policy of policies.items) {
+        const metadata = { name: policy.metadata?.name, namespace };
+        await instance.objectApi
+            .delete({ apiVersion: 'networking.k8s.io/v1', kind: 'NetworkPolicy', metadata })
+            .catch((e) => {
+            if ((0, apply_1.statusCodeOf)(e) !== 404) {
+                throw e;
+            }
+        });
+    }
+}
+exports.removeStaleItems = removeStaleItems;
+// the objects of the run's items but those of `ids`
+function getStaleItemSelector(run, ids) {
+    const current = [...new Set(ids)].sort();
+    return [
+        `${job_placement_1.RUN_LABEL}=${(0, job_placement_1.getRunLabelValue)(run)}`,
+        resources_1.ID_LABEL,
+        ...(current.length > 0 ? [`${resources_1.ID_LABEL} notin (${current.join(',')})`] : []),
+    ].join(',');
+}
+exports.getStaleItemSelector = getStaleItemSelector;
+//# sourceMappingURL=remove-stale-items.js.map

package/dist/kubernetes/restore-kubernetes-data.js

@@ -8,14 +8,18 @@ const ensure_persistent_data_1 = require("./ensure-persistent-data");
 const claim_layout_1 = require("./claim-layout");
 // Where the upload pod (which mounts the work's claim at /dev/hammerkit/<id>) unpacks the
 // archive of an output: the output's place in the claim, which its mount reads
-function getRestoreTarget(work, path) {
-    return `/dev/hammerkit/${work.id()}/${(0, claim_layout_1.getClaimSubPath)(work, path)}`;
+function getRestoreTarget(work, path, 
+// the folder of the state on a service run, see getStateFilesDir
+stateDir) {
+    return `/dev/hammerkit/${work.id()}/${stateDir ? `${stateDir}/` : ''}${(0, claim_layout_1.getClaimSubPath)(work, path)}`;
 }
 exports.getRestoreTarget = getRestoreTarget;
 async function restoreKubernetesData(work, kubernetes, instance, environment, path, 
-// the kept claim of the work holds what an earlier run left, which goes first
+// the kept claim of the work holds what an earlier run left, which goes first; on a
+// service run, the outputs go into the folder of `stateKey` (see getStateFilesDir)
 options = {}) {
-    const persistence = await (0, volumes_1.getKubernetesPersistence)(work, kubernetes);
+    const persistence = await (0, volumes_1.getKubernetesPersistence)(work, kubernetes, options.stateKey);
+    const stateDir = kubernetes.server?.serviceRun ? (0, volumes_1.getItemFilesDir)(work, options.stateKey) : undefined;
     await (0, ensure_persistent_data_1.getPodForPersistence)(instance, kubernetes, work, persistence, 'write', async (name) => {
         if (options.reset) {
             await (0, ensure_persistent_data_1.clearClaim)(instance, kubernetes, work, name);
@@ -24,7 +28,7 @@ options = {}) {
             if (!(await environment.file.exists(generatedArchive.filename))) {
                 continue;
             }
-            const targetPath = getRestoreTarget(work, generatedArchive.path);
+            const targetPath = getRestoreTarget(work, generatedArchive.path, stateDir);
             // a claim made for this run (a server removes the claims of a run when it
             // ends) does not have the directory of the output yet
             let failure = null;