hammerkit 1.27.0-next.9 → 2.0.0-next.1

package/dist/kubernetes/sandbox-runner.js

@@ -0,0 +1,628 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.removeSandboxRunners = exports.executeInRunner = exports.getExecExitCode = exports.buildRunnerPod = exports.getRunnerName = exports.getRunnerKey = exports.canUseRunner = exports.getWrapper = exports.RUNNER_CHANGED_EXIT = exports.getCleanUp = exports.getScratchDirectories = exports.getRunnerMounts = exports.RUNNER_SANDBOX_PATH = exports.IDLE_COMMAND = exports.RUNNER_TOOLS_IMAGE = exports.RUNNER_LABEL = void 0;
+const crypto_1 = require("crypto");
+const path_1 = require("path");
+const string_decoder_1 = require("string_decoder");
+const stream_1 = require("stream");
+const promises_1 = require("stream/promises");
+const work_network_1 = require("../planner/work-network");
+const replace_env_variables_1 = require("../environment/replace-env-variables");
+const log_1 = require("../log");
+const sleep_1 = require("../utils/sleep");
+const version_1 = require("../version");
+const apply_1 = require("./apply");
+const container_resources_1 = require("./container-resources");
+const job_placement_1 = require("./job-placement");
+const network_policy_1 = require("./network-policy");
+const resources_1 = require("./resources");
+const volumes_1 = require("./volumes");
+/**
+ * The runner of a sandbox: one pod that the commands of a run on the sandbox are exec'd
+ * into, one after the other, in place of a Job each. A Job's pod takes seconds to be
+ * scheduled and to start; an exec into a running pod a fraction of one.
+ *
+ * - Only a task shaped like an agent's shell gets it: a container task that mounts the
+ *   whole sandbox (`mounts: [.]`) and nothing else (no claims of its own or of its
+ *   dependencies, so no `generates`), with no `needs` and no secrets. Any other task runs
+ *   as a Job.
+ * - The pod is the task's container as its Job would have it: its image, variables,
+ *   resources, the sandbox's claim, the security context, the nodes of the pool and the
+ *   node of the pod of the sandbox's files. It is named after all of that (getRunnerKey),
+ *   so a change starts a new one and the one before goes. Its first process only waits.
+ * - Every command runs in a wrapper that first and last kills every other process of the
+ *   container and empties /tmp and /dev/shm (unless a mount of the task is in one): the
+ *   container is reused, what a command started is not. A command cancelled, out of time or whose connection broke is killed
+ *   the same way, or the pod is deleted. The first process and the wrapper run a static
+ *   busybox from a read-only mount, so a command can not replace what cleans up after it.
+ * - Under a limited network it has a NetworkPolicy of its own and, under `allow`, the
+ *   sandbox's egress proxy, both keyed like the pod. They carry no run, and go with the
+ *   pod of the sandbox's files, which owns them and the runner: the runner stays while
+ *   the sandbox has that pod, `backend.sandboxes.idle` after its last run or request.
+ */
+exports.RUNNER_LABEL = 'hammerkit.dev/runner';
+const RUNNER_CONTAINER = 'runner';
+const TMP_VOLUME = 'hammerkit-runner-tmp';
+const TOOLS_VOLUME = 'hammerkit-runner-tools';
+const TOOLS_PATH = '/.hammerkit';
+const BUSYBOX = `${TOOLS_PATH}/busybox`;
+// Pinned, static (it runs in any image), and copied into the runner by an init container.
+// To bump it, take the index digest of the tag (`docker buildx imagetools inspect
+// busybox:<version>-musl`).
+exports.RUNNER_TOOLS_IMAGE = 'busybox:1.37.0-musl@sha256:5cec3fc171c87218698e85a52af7087de727372aae264a787b8112901a5b0092';
+const RUNNER_START_MS = 5 * 60 * 1000;
+const RUNNER_POLL_MS = 250;
+const PULL_FAILURES = ['ErrImagePull', 'ImagePullBackOff', 'InvalidImageName'];
+const KILL_TIMEOUT_MS = 10000;
+// a command's output is kept to this, as the log of a Job
+const MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
+// Waits without a process of its own to signal: a command's `kill -KILL -1` ends the
+// sleep, not the loop (the first process of the container is spared), which reaps what
+// the commands left
+exports.IDLE_COMMAND = [BUSYBOX, 'sh', '-c', `trap "exit 0" TERM; while :; do ${BUSYBOX} sleep 3600 & wait; done`];
+const SCRATCH_DIRECTORIES = ['/tmp', '/dev/shm'];
+// Where the runner mounts the sandbox's files. A run has them in a directory of its own (the
+// server's directory of the run), which every command links here first: the pod and its key
+// stay the same from one run to the next.
+exports.RUNNER_SANDBOX_PATH = '/.hammerkit-sandbox';
+// The mounts of the task, from the directory of the run to the runner's
+function getRunnerMounts(workspace, persistence) {
+    return persistence.mounts.map(({ mount }) => ({ ...mount, mountPath: toRunnerPath(workspace, mount.mountPath) }));
+}
+exports.getRunnerMounts = getRunnerMounts;
+function toRunnerPath(workspace, path) {
+    const rel = (0, path_1.relative)(workspace.root, path);
+    return rel === '' ? exports.RUNNER_SANDBOX_PATH : path_1.posix.join(exports.RUNNER_SANDBOX_PATH, ...rel.split(path_1.sep));
+}
+// The scratch directories the runner empties: not one that holds a mount of the task, or is
+// in one, such as the sandbox's files when the server keeps its runs below /tmp
+function getScratchDirectories(mounts) {
+    const within = (path, directory) => path === directory || path.startsWith(`${directory}/`);
+    return SCRATCH_DIRECTORIES.filter((directory) => mounts.every((mount) => !within(mount.mountPath, directory) && !within(directory, mount.mountPath)));
+}
+exports.getScratchDirectories = getScratchDirectories;
+// Kills every process of the container but the first and the shell itself (`kill` is the
+// shell's own), again while one forked meanwhile, then empties the scratch directories
+function getCleanUp(directories) {
+    const patterns = directories.flatMap((directory) => [`${directory}/*`, `${directory}/.[!.]*`, `${directory}/..?*`]);
+    return [
+        'i=0',
+        'while kill -KILL -1 2>/dev/null && [ $i -lt 10 ]; do i=$((i+1)); done',
+        ...(patterns.length > 0 ? [`${BUSYBOX} rm -rf ${patterns.join(' ')} 2>/dev/null`] : []),
+    ].join('\n');
+}
+exports.getCleanUp = getCleanUp;
+// What the wrapper says on stderr, with RUNNER_CHANGED_EXIT, when a command before changed
+// the runner: the command did not run, the runner goes and the task fails
+exports.RUNNER_CHANGED_EXIT = 125;
+const RUNNER_CHANGED_MARK = `hammerkit-runner-changed-${(0, crypto_1.randomBytes)(8).toString('hex')}`;
+// Runs `"$@"` in the directory `$3`, alone in the container: what an earlier command left
+// is cleaned up before, what this one left after. The directory of the run `$2` is a link
+// to the sandbox's files first. Before the command runs, each of `mountPaths` must still be
+// the mount it was made as (the kernel's mountinfo, which no command writes): a command
+// before could have moved a folder a mount is in and put files of its own in its place.
+// When not, it says `$1` and exits RUNNER_CHANGED_EXIT. Else its exit code is the command's.
+function getWrapper(cleanUp, mountPaths) {
+    const sandbox = shellQuote(exports.RUNNER_SANDBOX_PATH);
+    return [
+        cleanUp,
+        `changed() { echo "$1" >&2; exit ${exports.RUNNER_CHANGED_EXIT}; }`,
+        `${BUSYBOX} mkdir -p "$(${BUSYBOX} dirname "$2")" && ${BUSYBOX} ln -sfn ${sandbox} "$2" || changed "$1"`,
+        ...mountPaths.map((path) => {
+            const quoted = shellQuote(path);
+            const field = shellQuote(`^[^ ]* [^ ]* [^ ]* [^ ]* ${toMountInfoPattern(path)} `);
+            return `[ "$(${BUSYBOX} realpath ${quoted})" = ${quoted} ] && ${BUSYBOX} grep -q ${field} /proc/self/mountinfo || changed "$1"`;
+        }),
+        'cd "$3" || exit 126',
+        `case "$(pwd -P)" in ${sandbox}|${sandbox}/*) ;; *) changed "$1" ;; esac`,
+        'shift 3',
+        '"$@"',
+        'code=$?',
+        cleanUp,
+        'exit $code',
+    ].join('\n');
+}
+exports.getWrapper = getWrapper;
+function shellQuote(value) {
+    return `'${value.replace(/'/g, `'\\''`)}'`;
+}
+// A mount point as /proc/self/mountinfo writes it (space, tab, newline and backslash as
+// octal escapes), as a basic regular expression
+function toMountInfoPattern(path) {
+    const escaped = path.replace(/[\\ \t\n]/g, (c) => `\\${c.charCodeAt(0).toString(8).padStart(3, '0')}`);
+    return escaped.replace(/[.[\]*^$\\]/g, (c) => `\\${c}`);
+}
+// the sandboxes whose runner a task of this process uses: one command at a time each, the
+// other tasks of the run run as Jobs meanwhile
+const inUse = new Set();
+// per sandbox, the runners whose container ended as it started (an image without `sh`, or
+// one that can not be pulled): their tasks run as Jobs while that pod of the files stays
+const failed = new Map();
+// Whether the task's commands may run in the sandbox's runner
+function canUseRunner(task, kubernetes, persistence) {
+    const workspace = kubernetes.server?.workspace;
+    return (!!workspace?.runner &&
+        !!workspace.pod &&
+        persistence.workspace.length > 0 &&
+        persistence.volumes.length === 0 &&
+        persistence.mounts.every((mount) => mount.workspace) &&
+        // the whole of the files at the root, so the runner has no writable directory of its own
+        // around the mounts that a command could leave files in for the next runs
+        persistence.mounts.some((mount) => (0, path_1.relative)(workspace.root, mount.mount.mountPath) === '') &&
+        task.needs.length === 0 &&
+        task.data.secrets.length === 0 &&
+        task.data.generates.length === 0 &&
+        // its pod would need the sandboxed runtime of a Job
+        task.data.capAdd.length === 0);
+}
+exports.canUseRunner = canUseRunner;
+// What the runner is made of: the same key for the same pod
+function getRunnerKey(task, kubernetes, persistence) {
+    const server = kubernetes.server;
+    const workspace = server.workspace;
+    return (0, crypto_1.createHash)('sha256')
+        .update(JSON.stringify({
+        sandbox: workspace.label,
+        pod: workspace.pod?.uid ?? null,
+        image: task.data.image,
+        envs: (0, replace_env_variables_1.getEnvironmentVariables)(task.data.envs),
+        resources: (0, container_resources_1.getKubernetesResources)(task.data.resources),
+        network: (0, work_network_1.getEffectiveNetwork)(task.data),
+        mounts: getRunnerMounts(workspace, persistence),
+        placement: (0, job_placement_1.getPodPlacement)(server, task.data.pool),
+        security: (0, job_placement_1.getContainerSecurity)(kubernetes),
+        version: (0, version_1.getVersion)(),
+    }))
+        .digest('hex')
+        .slice(0, 40);
+}
+exports.getRunnerKey = getRunnerKey;
+function getRunnerName(key) {
+    return `hammerkit-runner-${key.slice(0, 20)}`;
+}
+exports.getRunnerName = getRunnerName;
+function buildRunnerPod(options) {
+    const { task, kubernetes, persistence, key, network, owner } = options;
+    const server = kubernetes.server;
+    const workspace = server.workspace;
+    const placement = (0, job_placement_1.getPodPlacement)(server, task.data.pool);
+    const envs = (0, replace_env_variables_1.getEnvironmentVariables)(task.data.envs);
+    return {
+        apiVersion: 'v1',
+        kind: 'Pod',
+        metadata: {
+            namespace: kubernetes.namespace,
+            name: getRunnerName(key),
+            annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
+            labels: {
+                // what its policy and proxy select it by
+                [resources_1.ID_LABEL]: key,
+                [exports.RUNNER_LABEL]: key.slice(0, 20),
+                [volumes_1.SANDBOX_LABEL]: workspace.label,
+                ...(0, network_policy_1.getNetworkLabels)(kubernetes, task),
+            },
+            ownerReferences: [owner],
+        },
+        spec: {
+            automountServiceAccountToken: false,
+            enableServiceLinks: false,
+            restartPolicy: 'Never',
+            terminationGracePeriodSeconds: 0,
+            nodeSelector: placement.nodeSelector,
+            ...(placement.tolerations.length > 0 ? { tolerations: placement.tolerations } : {}),
+            affinity: (0, volumes_1.getSandboxAffinity)(workspace.label),
+            hostAliases: network.hostAliases,
+            initContainers: [
+                ...network.initContainers,
+                {
+                    name: 'hammerkit-runner-tools',
+                    image: exports.RUNNER_TOOLS_IMAGE,
+                    command: ['/bin/busybox', 'cp', '/bin/busybox', BUSYBOX],
+                    volumeMounts: [{ name: TOOLS_VOLUME, mountPath: TOOLS_PATH }],
+                    securityContext: {
+                        runAsNonRoot: true,
+                        runAsUser: 65534,
+                        allowPrivilegeEscalation: false,
+                        readOnlyRootFilesystem: true,
+                        capabilities: { drop: ['ALL'] },
+                    },
+                    resources: { limits: { memory: '16Mi' } },
+                },
+            ],
+            containers: [
+                {
+                    name: RUNNER_CONTAINER,
+                    image: task.data.image,
+                    command: exports.IDLE_COMMAND,
+                    workingDir: exports.RUNNER_SANDBOX_PATH,
+                    env: [...Object.entries(envs).map(([name, value]) => ({ name, value })), ...network.env],
+                    volumeMounts: [
+                        ...getRunnerMounts(workspace, persistence),
+                        { name: TMP_VOLUME, mountPath: '/tmp' },
+                        // not even root in the container writes it: it has no capability to remount
+                        { name: TOOLS_VOLUME, mountPath: TOOLS_PATH, readOnly: true },
+                    ],
+                    ...(0, container_resources_1.getKubernetesResources)(task.data.resources),
+                    ...(0, job_placement_1.getContainerSecurity)(kubernetes),
+                },
+            ],
+            volumes: [...persistence.workspace, { name: TMP_VOLUME, emptyDir: {} }, { name: TOOLS_VOLUME, emptyDir: {} }],
+        },
+    };
+}
+exports.buildRunnerPod = buildRunnerPod;
+// The exit code of an exec, from the status the API server ends it with
+function getExecExitCode(status) {
+    if (status?.status === 'Success') {
+        return 0;
+    }
+    const code = status?.details?.causes?.find((cause) => cause.reason === 'ExitCode')?.message;
+    return code !== undefined && /^\d+$/.test(code) ? Number(code) : undefined;
+}
+exports.getExecExitCode = getExecExitCode;
+/**
+ * Runs the task's commands in the sandbox's runner, started when there is none (or one of
+ * something else). `unavailable` when the task can not use it now (the runner is in use by
+ * another task of the run, the pod of the sandbox's files went, or the runner did not
+ * start): the task then runs as Jobs.
+ */
+async function executeInRunner(instance, task, kubernetes, persistence, abort) {
+    const workspace = kubernetes.server.workspace;
+    if (inUse.has(workspace.label)) {
+        return { type: 'unavailable' };
+    }
+    inUse.add(workspace.label);
+    try {
+        const filesPod = await getFilesPod(instance, kubernetes.namespace, workspace);
+        if (!filesPod) {
+            return { type: 'unavailable' };
+        }
+        const key = getRunnerKey(task, kubernetes, persistence);
+        const name = getRunnerName(key);
+        const failedOf = failed.get(workspace.label);
+        if (failedOf?.pod === filesPod.uid && failedOf.keys.has(key)) {
+            return { type: 'unavailable' };
+        }
+        // its policy and proxy select it by the key, and stay for the next runs
+        const item = { ...task, id: () => key };
+        const found = await getRunner(instance, kubernetes.namespace, name, filesPod.uid);
+        let runner = found?.ready ? found.pod : undefined;
+        const sandbox = { label: workspace.label, pod: workspace.pod };
+        const owners = (pod) => [filesPod, ...(pod ? [ownerOf(pod)] : [])];
+        const network = await (0, network_policy_1.ensureKubernetesItemNetwork)(instance, kubernetes, item, {
+            followLog: true,
+            run: null,
+            sandbox,
+            owners: owners(runner),
+        });
+        try {
+            if (!runner) {
+                let started;
+                if (found) {
+                    // one a task before started, which stopped while it did: it goes on starting
+                    started = await waitForRunner(instance, kubernetes.namespace, found.pod, abort);
+                }
+                else {
+                    // one of something else goes, with its limits
+                    await removeSandboxRunners(instance, kubernetes.namespace, workspace.label, (pod) => pod.metadata?.name === name);
+                    started = await startRunner(instance, buildRunnerPod({ task, kubernetes, persistence, key, network, owner: filesPod }), abort);
+                }
+                if (!('pod' in started)) {
+                    if (started.reason === 'ended') {
+                        if (failedOf?.pod !== filesPod.uid) {
+                            failed.set(workspace.label, { pod: filesPod.uid, keys: new Set() });
+                        }
+                        failed.get(workspace.label).keys.add(key);
+                    }
+                    task.status.write('debug', `the runner of the sandbox did not start (${started.message}), the task runs as a Job`);
+                    return { type: 'unavailable' };
+                }
+                runner = started.pod;
+                if (Object.keys((0, network_policy_1.getNetworkLabels)(kubernetes, task)).length > 0) {
+                    // the policy stays until the runner ended too
+                    await setPolicyOwners(instance, kubernetes.namespace, (0, network_policy_1.getNetworkPolicyName)(item, { run: null }), owners(runner));
+                }
+            }
+            else {
+                task.status.write('debug', `runs in the runner ${name} of the sandbox`);
+            }
+            const mounts = getRunnerMounts(workspace, persistence);
+            const cleanUp = getCleanUp(getScratchDirectories(mounts));
+            const wrapper = getWrapper(cleanUp, mounts.map((mount) => mount.mountPath));
+            for (const cmd of task.data.cmds) {
+                task.status.write('debug', `execute ${cmd.cmd} in the runner of the sandbox`);
+                const exitCode = await execInRunner(instance, kubernetes.namespace, name, workspace.root, cmd.cwd, [task.data.shell, '-c', cmd.cmd], { wrapper, cleanUp }, task, abort);
+                if (abort.aborted) {
+                    return { type: 'canceled' };
+                }
+                if (exitCode === 'changed') {
+                    // a fresh pod for the next task; this one fails rather than running again as a Job,
+                    // since a command can say so itself after it ran
+                    await removeSandboxRunners(instance, kubernetes.namespace, workspace.label, (pod) => pod.metadata?.name !== name);
+                    throw new Error('a command changed the runner of the sandbox, it was removed: run the task again');
+                }
+                if (exitCode !== 0) {
+                    return { type: 'crash', exitCode };
+                }
+            }
+            return { type: 'done' };
+        }
+        finally {
+            network.close();
+        }
+    }
+    finally {
+        inUse.delete(workspace.label);
+    }
+}
+exports.executeInRunner = executeInRunner;
+// The pod of the sandbox's files the run holds, as long as it is still that pod
+async function getFilesPod(instance, namespace, workspace) {
+    if (!workspace.pod) {
+        return undefined;
+    }
+    const pod = await instance.coreApi
+        .readNamespacedPod({ name: workspace.pod.name, namespace })
+        .catch((e) => ((0, apply_1.statusCodeOf)(e) === 404 ? undefined : Promise.reject(e)));
+    if (!pod || pod.metadata?.uid !== workspace.pod.uid || pod.metadata?.deletionTimestamp) {
+        return undefined;
+    }
+    return ownerOf(pod);
+}
+function ownerOf(pod) {
+    return { apiVersion: 'v1', kind: 'Pod', name: pod.metadata.name, uid: pod.metadata.uid };
+}
+// The runner of that name, when it belongs to the pod of the files and runs, or still
+// starts (`ready` false); one that ended or is going is deleted
+async function getRunner(instance, namespace, name, owner) {
+    const pod = await instance.coreApi
+        .readNamespacedPod({ name, namespace })
+        .catch((e) => ((0, apply_1.statusCodeOf)(e) === 404 ? undefined : Promise.reject(e)));
+    if (!pod) {
+        return undefined;
+    }
+    const ours = !pod.metadata?.deletionTimestamp &&
+        (pod.metadata?.ownerReferences ?? []).some((reference) => reference.uid === owner);
+    if (ours && isRunning(pod)) {
+        return { pod, ready: true };
+    }
+    if (ours && pod.status?.phase === 'Pending' && !getStartFailure(pod)) {
+        return { pod, ready: false };
+    }
+    await deletePod(instance, namespace, name);
+    return undefined;
+}
+function isRunning(pod) {
+    const status = pod.status?.containerStatuses?.find((s) => s.name === RUNNER_CONTAINER);
+    return pod.status?.phase === 'Running' && !!status?.state?.running;
+}
+// Why the runner will not run: its container ended, or an image can not be pulled, which
+// waits forever and is as good as ended
+function getStartFailure(pod) {
+    const status = pod.status?.containerStatuses?.find((s) => s.name === RUNNER_CONTAINER);
+    const unpulled = [...(pod.status?.initContainerStatuses ?? []), ...(pod.status?.containerStatuses ?? [])].find((s) => PULL_FAILURES.includes(s.state?.waiting?.reason ?? ''));
+    if (unpulled) {
+        return `${unpulled.state?.waiting?.reason} of ${unpulled.image}`;
+    }
+    if (pod.status?.phase === 'Failed' || pod.status?.phase === 'Succeeded' || status?.state?.terminated) {
+        return `its container ended with ${status?.state?.terminated?.exitCode ?? 'no code'}`;
+    }
+    return undefined;
+}
+// Waits until no pod of that name is left: a new one of the same name is refused before
+async function deletePod(instance, namespace, name) {
+    try {
+        await instance.coreApi.deleteNamespacedPod({ name, namespace, gracePeriodSeconds: 0 });
+    }
+    catch (e) {
+        if ((0, apply_1.statusCodeOf)(e) === 404) {
+            return;
+        }
+        throw e;
+    }
+    const deadline = Date.now() + RUNNER_START_MS;
+    while (Date.now() < deadline) {
+        const left = await instance.coreApi
+            .readNamespacedPod({ name, namespace })
+            .catch((e) => ((0, apply_1.statusCodeOf)(e) === 404 ? undefined : Promise.reject(e)));
+        if (!left) {
+            return;
+        }
+        await (0, sleep_1.sleep)(RUNNER_POLL_MS);
+    }
+    throw new Error(`the runner ${name} is still there`);
+}
+// Starts the runner and answers it once it runs, or why it does not: the cluster refused
+// it (a full quota, say, which a Job waits for), its container ended, or it took too long
+async function startRunner(instance, spec, abort) {
+    try {
+        await instance.objectApi.create(spec);
+    }
+    catch (e) {
+        return { reason: 'refused', message: (0, log_1.getErrorMessage)(e) };
+    }
+    return await waitForRunner(instance, spec.metadata.namespace, spec, abort);
+}
+// Waits until the runner runs. A task stopped meanwhile (cancelled, out of time) leaves it
+// starting, for the next command, which would time out the same way if each started anew;
+// one that ends, or does not run within RUNNER_START_MS of being made, goes.
+async function waitForRunner(instance, namespace, runner, abort) {
+    const name = runner.metadata.name;
+    const created = runner.metadata?.creationTimestamp ? new Date(runner.metadata.creationTimestamp).getTime() : NaN;
+    const deadline = (Number.isNaN(created) ? Date.now() : Math.min(created, Date.now())) + RUNNER_START_MS;
+    for (;;) {
+        const pod = await instance.coreApi.readNamespacedPod({ name, namespace });
+        if (isRunning(pod)) {
+            return { pod };
+        }
+        const failure = getStartFailure(pod);
+        if (failure || Date.now() > deadline) {
+            await deletePod(instance, namespace, name).catch(() => undefined);
+            return failure ? { reason: 'ended', message: failure } : { reason: 'stopped', message: 'it took too long' };
+        }
+        if (abort.aborted) {
+            return { reason: 'stopped', message: 'the task was stopped' };
+        }
+        await (0, sleep_1.sleep)(RUNNER_POLL_MS);
+    }
+}
+// Removes the runners of a sandbox but those `keep` takes, and once each is gone its policy
+// and proxy, which the pod of the sandbox's files would keep while it runs
+async function removeSandboxRunners(instance, namespace, sandbox, keep) {
+    if (!keep) {
+        // all of the sandbox's runners go: so does what was remembered of them
+        failed.delete(sandbox);
+    }
+    const pods = await instance.coreApi.listNamespacedPod({
+        namespace,
+        labelSelector: `${volumes_1.SANDBOX_LABEL}=${sandbox},${exports.RUNNER_LABEL}`,
+    });
+    for (const pod of pods.items) {
+        const name = pod.metadata?.name;
+        const key = pod.metadata?.labels?.[resources_1.ID_LABEL];
+        if (!name || keep?.(pod)) {
+            continue;
+        }
+        await deletePod(instance, namespace, name);
+        if (key) {
+            const item = { id: () => key };
+            await instance.objectApi
+                .delete({
+                apiVersion: 'networking.k8s.io/v1',
+                kind: 'NetworkPolicy',
+                metadata: { name: (0, network_policy_1.getNetworkPolicyName)(item, { run: null }), namespace },
+            })
+                .catch((e) => ((0, apply_1.statusCodeOf)(e) === 404 ? undefined : Promise.reject(e)));
+            await (0, network_policy_1.removeSandboxProxies)(instance, namespace, { sandbox, item });
+        }
+    }
+}
+exports.removeSandboxRunners = removeSandboxRunners;
+async function setPolicyOwners(instance, namespace, name, owners) {
+    await instance.objectApi.patch({
+        apiVersion: 'networking.k8s.io/v1',
+        kind: 'NetworkPolicy',
+        metadata: { name, namespace, ownerReferences: owners },
+    }, undefined, undefined, undefined, undefined, 'application/merge-patch+json');
+}
+// Runs a command in the runner and answers its exit code. Its output goes to the task's,
+// line by line as it comes. On abort (cancel, timeout) every process of the command is
+// killed, or the runner goes when that fails.
+async function execInRunner(instance, namespace, pod, 
+// the directory of the run's sandbox, and the command's in it
+root, cwd, command, scripts, task, abort) {
+    const { cleanUp } = scripts;
+    const output = { bytes: 0, cut: false };
+    let changed = false;
+    const lines = (stream) => {
+        let rest = '';
+        // a character split between two chunks stays whole
+        const decoder = new string_decoder_1.StringDecoder('utf8');
+        const writable = new stream_1.Writable({
+            write(chunk, _encoding, callback) {
+                if (output.bytes < MAX_OUTPUT_BYTES) {
+                    output.bytes += chunk.length;
+                    const parts = (rest + decoder.write(chunk)).split(/\r?\n/);
+                    rest = parts.pop() ?? '';
+                    for (const line of parts) {
+                        if (stream === 'stderr' && line === RUNNER_CHANGED_MARK) {
+                            changed = true;
+                            continue;
+                        }
+                        task.status.console(stream, line);
+                    }
+                }
+                else if (!output.cut) {
+                    output.cut = true;
+                    task.status.write('warn', `the output of ${task.name} was cut at ${MAX_OUTPUT_BYTES} bytes`);
+                }
+                callback();
+            },
+            final(callback) {
+                rest += decoder.end();
+                if (rest.length > 0) {
+                    task.status.console(stream, rest);
+                }
+                callback();
+            },
+        });
+        return writable;
+    };
+    const stdout = lines('stdout');
+    const stderr = lines('stderr');
+    let status;
+    const socket = await instance.exec.exec(namespace, pod, RUNNER_CONTAINER, [BUSYBOX, 'sh', '-c', scripts.wrapper, 'hammerkit', RUNNER_CHANGED_MARK, root, cwd, ...command], stdout, stderr, null, false, (received) => {
+        status = received;
+    });
+    const closed = new Promise((resolve, reject) => {
+        socket.on('close', () => resolve());
+        socket.on('error', (e) => reject(e));
+    });
+    const kill = async () => {
+        try {
+            await killInRunner(instance, namespace, pod, cleanUp);
+        }
+        catch (e) {
+            task.status.write('debug', `the command could not be stopped in the runner, it goes: ${(0, log_1.getErrorMessage)(e)}`);
+            await deletePod(instance, namespace, pod).catch(() => undefined);
+        }
+        // what the command printed after it was killed is not waited for long
+        await Promise.race([closed.catch(() => undefined), (0, sleep_1.sleep)(KILL_TIMEOUT_MS)]);
+        socket.close();
+    };
+    const aborted = () => {
+        void kill();
+    };
+    if (abort.aborted) {
+        aborted();
+    }
+    else {
+        abort.addEventListener('abort', aborted, { once: true });
+    }
+    let broken;
+    try {
+        await closed.catch((e) => {
+            broken = e;
+        });
+        if (!abort.aborted && getExecExitCode(status) === undefined) {
+            // the connection broke while the command may still run: it ends here as on a cancel,
+            // before the run lets go of the sandbox
+            await kill();
+        }
+    }
+    finally {
+        abort.removeEventListener('abort', aborted);
+        for (const stream of [stdout, stderr]) {
+            if (!stream.writableEnded) {
+                stream.end();
+            }
+        }
+        await Promise.all([(0, promises_1.finished)(stdout), (0, promises_1.finished)(stderr)]).catch(() => undefined);
+    }
+    if (abort.aborted) {
+        return 137;
+    }
+    const exitCode = getExecExitCode(status);
+    if (exitCode === undefined) {
+        throw new Error(`the command in the runner of the sandbox ended without an exit code: ${status?.message ?? (broken ? (0, log_1.getErrorMessage)(broken) : 'the connection closed')}`);
+    }
+    return exitCode === exports.RUNNER_CHANGED_EXIT && changed ? 'changed' : exitCode;
+}
+async function killInRunner(instance, namespace, pod, cleanUp) {
+    let status;
+    const socket = await instance.exec.exec(namespace, pod, RUNNER_CONTAINER, [BUSYBOX, 'sh', '-c', cleanUp], null, null, null, false, (received) => {
+        status = received;
+    });
+    await Promise.race([
+        new Promise((resolve, reject) => {
+            socket.on('close', () => resolve());
+            socket.on('error', reject);
+        }),
+        (0, sleep_1.sleep)(KILL_TIMEOUT_MS).then(() => Promise.reject(new Error('the kill did not answer'))),
+    ]);
+    // what it answers does not matter, that it answered does
+    if (getExecExitCode(status) === undefined) {
+        throw new Error(status?.message ?? 'the kill did not end');
+    }
+}
+//# sourceMappingURL=sandbox-runner.js.map