hammerkit 1.27.0-next.11 → 1.27.0-next.13

package/dist/cli.js

@@ -79,6 +79,7 @@ class Cli {
                         explain: options?.explain ?? false,
                         cacheReadOnly: options?.cacheReadOnly ?? false,
                         timeout: options?.timeout ?? null,
+                        maxTimeout: options?.maxTimeout ?? null,
                         skipDeps: options?.skipDeps ?? true,
                         wait: options?.wait ?? 'ready',
                         resourceCheck: options?.resourceCheck ?? true,

package/dist/executer/deadline.js

@@ -1,14 +1,28 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.withDeadline = void 0;
+exports.withDeadline = exports.getTaskTimeout = void 0;
 const abort_event_1 = require("../utils/abort-event");
+// How long a task may run: its own `timeout`, else the run's default (`--timeout`), and
+// never longer than the run's cap (`--max-timeout`), which also covers tasks with a
+// longer timeout of their own. A caller can shorten a task this way, never extend it.
+function getTaskTimeout(own, run) {
+    const timeout = own ?? run.timeout;
+    if (run.maxTimeout === null) {
+        return timeout;
+    }
+    return timeout === null ? run.maxTimeout : Math.min(timeout, run.maxTimeout);
+}
+exports.getTaskTimeout = getTaskTimeout;
+// the longest delay of setTimeout: Node fires a longer one after 1 ms
+const MAX_TIMER_MS = 2 ** 31 - 1;
 // An abort signal that follows `parent` and additionally fires after `timeout`
-// ms, so a task deadline reuses the runtimes' existing abort/cleanup path.
+// ms, so a task deadline reuses the runtimes' existing abort/cleanup path. A
+// timeout beyond what a timer can wait (about 24.8 days) sets no deadline.
 function withDeadline(parent, timeout) {
     const controller = new AbortController();
     let expired = false;
     const parentListener = (0, abort_event_1.listenOnAbort)(parent, () => controller.abort());
-    const timer = timeout === null
+    const timer = timeout === null || timeout > MAX_TIMER_MS
         ? null
         : setTimeout(() => {
             expired = true;

package/dist/executer/execute-work-task.js

@@ -94,7 +94,7 @@ dependents = null) {
                     stateKey: cacheState.stateKey,
                     started,
                 });
-                const timeout = work.data.timeout ?? options.timeout;
+                const timeout = (0, deadline_1.getTaskTimeout)(work.data.timeout, options);
                 const deadline = (0, deadline_1.withDeadline)(abort, timeout);
                 try {
                     await work.runtime.execute(environment, {

package/dist/harness/sandbox-client.js

@@ -115,7 +115,8 @@ class SandboxClient {
             runOptions.args = options.args;
         }
         if (options.timeoutSeconds) {
-            runOptions.timeout = `${Math.max(1, Math.ceil(options.timeoutSeconds))}s`;
+            // a cap: it also shortens a shell task that sets a longer timeout of its own
+            runOptions.maxTimeout = `${Math.max(1, Math.ceil(options.timeoutSeconds))}s`;
         }
         const submitted = await submitRun(this.connection, {
             definition: this.build.hash,

package/dist/kubernetes/network-policy.js

@@ -129,7 +129,8 @@ function buildNetworkPolicy(env, item, network, scope, proxy = getRunProxy(item,
             namespace: env.namespace,
             name: getNetworkPolicyName(item, scope),
             annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
-            labels: { [NETWORK_OF_LABEL]: item.id(), ...(0, job_placement_1.getRunLabels)(env) },
+            labels: { [NETWORK_OF_LABEL]: item.id(), ...(scope.owners ? {} : (0, job_placement_1.getRunLabels)(env)) },
+            ...(scope.owners ? { ownerReferences: scope.owners } : {}),
         },
         spec: {
             podSelector: { matchLabels: getItemPodLabels(item, scope) },

package/dist/kubernetes/sandbox-runner.js

@@ -0,0 +1,626 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.removeSandboxRunners = exports.executeInRunner = exports.getExecExitCode = exports.buildRunnerPod = exports.getRunnerName = exports.getRunnerKey = exports.canUseRunner = exports.getWrapper = exports.RUNNER_CHANGED_EXIT = exports.getCleanUp = exports.getScratchDirectories = exports.getRunnerMounts = exports.RUNNER_SANDBOX_PATH = exports.IDLE_COMMAND = exports.RUNNER_TOOLS_IMAGE = exports.RUNNER_LABEL = void 0;
+const crypto_1 = require("crypto");
+const path_1 = require("path");
+const string_decoder_1 = require("string_decoder");
+const stream_1 = require("stream");
+const promises_1 = require("stream/promises");
+const work_network_1 = require("../planner/work-network");
+const replace_env_variables_1 = require("../environment/replace-env-variables");
+const log_1 = require("../log");
+const sleep_1 = require("../utils/sleep");
+const version_1 = require("../version");
+const apply_1 = require("./apply");
+const container_resources_1 = require("./container-resources");
+const job_placement_1 = require("./job-placement");
+const network_policy_1 = require("./network-policy");
+const resources_1 = require("./resources");
+const volumes_1 = require("./volumes");
+/**
+ * The runner of a sandbox: one pod that the commands of a run on the sandbox are exec'd
+ * into, one after the other, in place of a Job each. A Job's pod takes seconds to be
+ * scheduled and to start; an exec into a running pod a fraction of one.
+ *
+ * - Only a task shaped like an agent's shell gets it: a container task that mounts the
+ *   whole sandbox (`mounts: [.]`) and nothing else (no claims of its own or of its
+ *   dependencies, so no `generates`), with no `needs` and no secrets. Any other task runs
+ *   as a Job.
+ * - The pod is the task's container as its Job would have it: its image, variables,
+ *   resources, the sandbox's claim, the security context, the nodes of the pool and the
+ *   node of the pod of the sandbox's files. It is named after all of that (getRunnerKey),
+ *   so a change starts a new one and the one before goes. Its first process only waits.
+ * - Every command runs in a wrapper that first and last kills every other process of the
+ *   container and empties /tmp and /dev/shm (unless a mount of the task is in one): the
+ *   container is reused, what a command started is not. A command cancelled, out of time or whose connection broke is killed
+ *   the same way, or the pod is deleted. The first process and the wrapper run a static
+ *   busybox from a read-only mount, so a command can not replace what cleans up after it.
+ * - Under a limited network it has a NetworkPolicy of its own and, under `allow`, the
+ *   sandbox's egress proxy, both keyed like the pod. They carry no run, and go with the
+ *   pod of the sandbox's files, which owns them and the runner: the runner stays while
+ *   the sandbox has that pod, `backend.sandboxes.idle` after its last run or request.
+ */
+exports.RUNNER_LABEL = 'hammerkit.dev/runner';
+const RUNNER_CONTAINER = 'runner';
+const TMP_VOLUME = 'hammerkit-runner-tmp';
+const TOOLS_VOLUME = 'hammerkit-runner-tools';
+const TOOLS_PATH = '/.hammerkit';
+const BUSYBOX = `${TOOLS_PATH}/busybox`;
+// Pinned, static (it runs in any image), and copied into the runner by an init container.
+// To bump it, take the index digest of the tag (`docker buildx imagetools inspect
+// busybox:<version>-musl`).
+exports.RUNNER_TOOLS_IMAGE = 'busybox:1.37.0-musl@sha256:5cec3fc171c87218698e85a52af7087de727372aae264a787b8112901a5b0092';
+const RUNNER_START_MS = 5 * 60 * 1000;
+const RUNNER_POLL_MS = 250;
+const PULL_FAILURES = ['ErrImagePull', 'ImagePullBackOff', 'InvalidImageName'];
+const KILL_TIMEOUT_MS = 10000;
+// a command's output is kept to this, as the log of a Job
+const MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
+// Waits without a process of its own to signal: a command's `kill -KILL -1` ends the
+// sleep, not the loop (the first process of the container is spared), which reaps what
+// the commands left
+exports.IDLE_COMMAND = [BUSYBOX, 'sh', '-c', `trap "exit 0" TERM; while :; do ${BUSYBOX} sleep 3600 & wait; done`];
+const SCRATCH_DIRECTORIES = ['/tmp', '/dev/shm'];
+// Where the runner mounts the sandbox's files. A run has them in a directory of its own (the
+// server's directory of the run), which every command links here first: the pod and its key
+// stay the same from one run to the next.
+exports.RUNNER_SANDBOX_PATH = '/.hammerkit-sandbox';
+// The mounts of the task, from the directory of the run to the runner's
+function getRunnerMounts(workspace, persistence) {
+    return persistence.mounts.map(({ mount }) => ({ ...mount, mountPath: toRunnerPath(workspace, mount.mountPath) }));
+}
+exports.getRunnerMounts = getRunnerMounts;
+function toRunnerPath(workspace, path) {
+    const rel = (0, path_1.relative)(workspace.root, path);
+    return rel === '' ? exports.RUNNER_SANDBOX_PATH : path_1.posix.join(exports.RUNNER_SANDBOX_PATH, ...rel.split(path_1.sep));
+}
+// The scratch directories the runner empties: not one that holds a mount of the task, or is
+// in one, such as the sandbox's files when the server keeps its runs below /tmp
+function getScratchDirectories(mounts) {
+    const within = (path, directory) => path === directory || path.startsWith(`${directory}/`);
+    return SCRATCH_DIRECTORIES.filter((directory) => mounts.every((mount) => !within(mount.mountPath, directory) && !within(directory, mount.mountPath)));
+}
+exports.getScratchDirectories = getScratchDirectories;
+// Kills every process of the container but the first and the shell itself (`kill` is the
+// shell's own), again while one forked meanwhile, then empties the scratch directories
+function getCleanUp(directories) {
+    const patterns = directories.flatMap((directory) => [`${directory}/*`, `${directory}/.[!.]*`, `${directory}/..?*`]);
+    return [
+        'i=0',
+        'while kill -KILL -1 2>/dev/null && [ $i -lt 10 ]; do i=$((i+1)); done',
+        ...(patterns.length > 0 ? [`${BUSYBOX} rm -rf ${patterns.join(' ')} 2>/dev/null`] : []),
+    ].join('\n');
+}
+exports.getCleanUp = getCleanUp;
+// What the wrapper says on stderr, with RUNNER_CHANGED_EXIT, when a command before changed
+// the runner: the command did not run, the runner goes and the task fails
+exports.RUNNER_CHANGED_EXIT = 125;
+const RUNNER_CHANGED_MARK = `hammerkit-runner-changed-${(0, crypto_1.randomBytes)(8).toString('hex')}`;
+// Runs `"$@"` in the directory `$3`, alone in the container: what an earlier command left
+// is cleaned up before, what this one left after. The directory of the run `$2` is a link
+// to the sandbox's files first. Before the command runs, each of `mountPaths` must still be
+// the mount it was made as (the kernel's mountinfo, which no command writes): a command
+// before could have moved a folder a mount is in and put files of its own in its place.
+// When not, it says `$1` and exits RUNNER_CHANGED_EXIT. Else its exit code is the command's.
+function getWrapper(cleanUp, mountPaths) {
+    const sandbox = shellQuote(exports.RUNNER_SANDBOX_PATH);
+    return [
+        cleanUp,
+        `changed() { echo "$1" >&2; exit ${exports.RUNNER_CHANGED_EXIT}; }`,
+        `${BUSYBOX} mkdir -p "$(${BUSYBOX} dirname "$2")" && ${BUSYBOX} ln -sfn ${sandbox} "$2" || changed "$1"`,
+        ...mountPaths.map((path) => {
+            const quoted = shellQuote(path);
+            const field = shellQuote(`^[^ ]* [^ ]* [^ ]* [^ ]* ${toMountInfoPattern(path)} `);
+            return `[ "$(${BUSYBOX} realpath ${quoted})" = ${quoted} ] && ${BUSYBOX} grep -q ${field} /proc/self/mountinfo || changed "$1"`;
+        }),
+        'cd "$3" || exit 126',
+        `case "$(pwd -P)" in ${sandbox}|${sandbox}/*) ;; *) changed "$1" ;; esac`,
+        'shift 3',
+        '"$@"',
+        'code=$?',
+        cleanUp,
+        'exit $code',
+    ].join('\n');
+}
+exports.getWrapper = getWrapper;
+function shellQuote(value) {
+    return `'${value.replace(/'/g, `'\\''`)}'`;
+}
+// A mount point as /proc/self/mountinfo writes it (space, tab, newline and backslash as
+// octal escapes), as a basic regular expression
+function toMountInfoPattern(path) {
+    const escaped = path.replace(/[\\ \t\n]/g, (c) => `\\${c.charCodeAt(0).toString(8).padStart(3, '0')}`);
+    return escaped.replace(/[.[\]*^$\\]/g, (c) => `\\${c}`);
+}
+// the sandboxes whose runner a task of this process uses: one command at a time each, the
+// other tasks of the run run as Jobs meanwhile
+const inUse = new Set();
+// per sandbox, the runners whose container ended as it started (an image without `sh`, or
+// one that can not be pulled): their tasks run as Jobs while that pod of the files stays
+const failed = new Map();
+// Whether the task's commands may run in the sandbox's runner
+function canUseRunner(task, kubernetes, persistence) {
+    const workspace = kubernetes.server?.workspace;
+    return (!!workspace?.runner &&
+        !!workspace.pod &&
+        persistence.workspace.length > 0 &&
+        persistence.volumes.length === 0 &&
+        persistence.mounts.every((mount) => mount.workspace) &&
+        // the whole of the files at the root, so the runner has no writable directory of its own
+        // around the mounts that a command could leave files in for the next runs
+        persistence.mounts.some((mount) => (0, path_1.relative)(workspace.root, mount.mount.mountPath) === '') &&
+        task.needs.length === 0 &&
+        task.data.secrets.length === 0 &&
+        task.data.generates.length === 0);
+}
+exports.canUseRunner = canUseRunner;
+// What the runner is made of: the same key for the same pod
+function getRunnerKey(task, kubernetes, persistence) {
+    const server = kubernetes.server;
+    const workspace = server.workspace;
+    return (0, crypto_1.createHash)('sha256')
+        .update(JSON.stringify({
+        sandbox: workspace.label,
+        pod: workspace.pod?.uid ?? null,
+        image: task.data.image,
+        envs: (0, replace_env_variables_1.getEnvironmentVariables)(task.data.envs),
+        resources: (0, container_resources_1.getKubernetesResources)(task.data.resources),
+        network: (0, work_network_1.getEffectiveNetwork)(task.data),
+        mounts: getRunnerMounts(workspace, persistence),
+        placement: (0, job_placement_1.getPodPlacement)(server, task.data.pool),
+        security: (0, job_placement_1.getContainerSecurity)(kubernetes),
+        version: (0, version_1.getVersion)(),
+    }))
+        .digest('hex')
+        .slice(0, 40);
+}
+exports.getRunnerKey = getRunnerKey;
+function getRunnerName(key) {
+    return `hammerkit-runner-${key.slice(0, 20)}`;
+}
+exports.getRunnerName = getRunnerName;
+function buildRunnerPod(options) {
+    const { task, kubernetes, persistence, key, network, owner } = options;
+    const server = kubernetes.server;
+    const workspace = server.workspace;
+    const placement = (0, job_placement_1.getPodPlacement)(server, task.data.pool);
+    const envs = (0, replace_env_variables_1.getEnvironmentVariables)(task.data.envs);
+    return {
+        apiVersion: 'v1',
+        kind: 'Pod',
+        metadata: {
+            namespace: kubernetes.namespace,
+            name: getRunnerName(key),
+            annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
+            labels: {
+                // what its policy and proxy select it by
+                [resources_1.ID_LABEL]: key,
+                [exports.RUNNER_LABEL]: key.slice(0, 20),
+                [volumes_1.SANDBOX_LABEL]: workspace.label,
+                ...(0, network_policy_1.getNetworkLabels)(kubernetes, task),
+            },
+            ownerReferences: [owner],
+        },
+        spec: {
+            automountServiceAccountToken: false,
+            enableServiceLinks: false,
+            restartPolicy: 'Never',
+            terminationGracePeriodSeconds: 0,
+            nodeSelector: placement.nodeSelector,
+            ...(placement.tolerations.length > 0 ? { tolerations: placement.tolerations } : {}),
+            affinity: (0, volumes_1.getSandboxAffinity)(workspace.label),
+            hostAliases: network.hostAliases,
+            initContainers: [
+                ...network.initContainers,
+                {
+                    name: 'hammerkit-runner-tools',
+                    image: exports.RUNNER_TOOLS_IMAGE,
+                    command: ['/bin/busybox', 'cp', '/bin/busybox', BUSYBOX],
+                    volumeMounts: [{ name: TOOLS_VOLUME, mountPath: TOOLS_PATH }],
+                    securityContext: {
+                        runAsNonRoot: true,
+                        runAsUser: 65534,
+                        allowPrivilegeEscalation: false,
+                        readOnlyRootFilesystem: true,
+                        capabilities: { drop: ['ALL'] },
+                    },
+                    resources: { limits: { memory: '16Mi' } },
+                },
+            ],
+            containers: [
+                {
+                    name: RUNNER_CONTAINER,
+                    image: task.data.image,
+                    command: exports.IDLE_COMMAND,
+                    workingDir: exports.RUNNER_SANDBOX_PATH,
+                    env: [...Object.entries(envs).map(([name, value]) => ({ name, value })), ...network.env],
+                    volumeMounts: [
+                        ...getRunnerMounts(workspace, persistence),
+                        { name: TMP_VOLUME, mountPath: '/tmp' },
+                        // not even root in the container writes it: it has no capability to remount
+                        { name: TOOLS_VOLUME, mountPath: TOOLS_PATH, readOnly: true },
+                    ],
+                    ...(0, container_resources_1.getKubernetesResources)(task.data.resources),
+                    ...(0, job_placement_1.getContainerSecurity)(kubernetes),
+                },
+            ],
+            volumes: [...persistence.workspace, { name: TMP_VOLUME, emptyDir: {} }, { name: TOOLS_VOLUME, emptyDir: {} }],
+        },
+    };
+}
+exports.buildRunnerPod = buildRunnerPod;
+// The exit code of an exec, from the status the API server ends it with
+function getExecExitCode(status) {
+    if (status?.status === 'Success') {
+        return 0;
+    }
+    const code = status?.details?.causes?.find((cause) => cause.reason === 'ExitCode')?.message;
+    return code !== undefined && /^\d+$/.test(code) ? Number(code) : undefined;
+}
+exports.getExecExitCode = getExecExitCode;
+/**
+ * Runs the task's commands in the sandbox's runner, started when there is none (or one of
+ * something else). `unavailable` when the task can not use it now (the runner is in use by
+ * another task of the run, the pod of the sandbox's files went, or the runner did not
+ * start): the task then runs as Jobs.
+ */
+async function executeInRunner(instance, task, kubernetes, persistence, abort) {
+    const workspace = kubernetes.server.workspace;
+    if (inUse.has(workspace.label)) {
+        return { type: 'unavailable' };
+    }
+    inUse.add(workspace.label);
+    try {
+        const filesPod = await getFilesPod(instance, kubernetes.namespace, workspace);
+        if (!filesPod) {
+            return { type: 'unavailable' };
+        }
+        const key = getRunnerKey(task, kubernetes, persistence);
+        const name = getRunnerName(key);
+        const failedOf = failed.get(workspace.label);
+        if (failedOf?.pod === filesPod.uid && failedOf.keys.has(key)) {
+            return { type: 'unavailable' };
+        }
+        // its policy and proxy select it by the key, and stay for the next runs
+        const item = { ...task, id: () => key };
+        const found = await getRunner(instance, kubernetes.namespace, name, filesPod.uid);
+        let runner = found?.ready ? found.pod : undefined;
+        const sandbox = { label: workspace.label, pod: workspace.pod };
+        const owners = (pod) => [filesPod, ...(pod ? [ownerOf(pod)] : [])];
+        const network = await (0, network_policy_1.ensureKubernetesItemNetwork)(instance, kubernetes, item, {
+            followLog: true,
+            run: null,
+            sandbox,
+            owners: owners(runner),
+        });
+        try {
+            if (!runner) {
+                let started;
+                if (found) {
+                    // one a task before started, which stopped while it did: it goes on starting
+                    started = await waitForRunner(instance, kubernetes.namespace, found.pod, abort);
+                }
+                else {
+                    // one of something else goes, with its limits
+                    await removeSandboxRunners(instance, kubernetes.namespace, workspace.label, (pod) => pod.metadata?.name === name);
+                    started = await startRunner(instance, buildRunnerPod({ task, kubernetes, persistence, key, network, owner: filesPod }), abort);
+                }
+                if (!('pod' in started)) {
+                    if (started.reason === 'ended') {
+                        if (failedOf?.pod !== filesPod.uid) {
+                            failed.set(workspace.label, { pod: filesPod.uid, keys: new Set() });
+                        }
+                        failed.get(workspace.label).keys.add(key);
+                    }
+                    task.status.write('debug', `the runner of the sandbox did not start (${started.message}), the task runs as a Job`);
+                    return { type: 'unavailable' };
+                }
+                runner = started.pod;
+                if (Object.keys((0, network_policy_1.getNetworkLabels)(kubernetes, task)).length > 0) {
+                    // the policy stays until the runner ended too
+                    await setPolicyOwners(instance, kubernetes.namespace, (0, network_policy_1.getNetworkPolicyName)(item, { run: null }), owners(runner));
+                }
+            }
+            else {
+                task.status.write('debug', `runs in the runner ${name} of the sandbox`);
+            }
+            const mounts = getRunnerMounts(workspace, persistence);
+            const cleanUp = getCleanUp(getScratchDirectories(mounts));
+            const wrapper = getWrapper(cleanUp, mounts.map((mount) => mount.mountPath));
+            for (const cmd of task.data.cmds) {
+                task.status.write('debug', `execute ${cmd.cmd} in the runner of the sandbox`);
+                const exitCode = await execInRunner(instance, kubernetes.namespace, name, workspace.root, cmd.cwd, [task.data.shell, '-c', cmd.cmd], { wrapper, cleanUp }, task, abort);
+                if (abort.aborted) {
+                    return { type: 'canceled' };
+                }
+                if (exitCode === 'changed') {
+                    // a fresh pod for the next task; this one fails rather than running again as a Job,
+                    // since a command can say so itself after it ran
+                    await removeSandboxRunners(instance, kubernetes.namespace, workspace.label, (pod) => pod.metadata?.name !== name);
+                    throw new Error('a command changed the runner of the sandbox, it was removed: run the task again');
+                }
+                if (exitCode !== 0) {
+                    return { type: 'crash', exitCode };
+                }
+            }
+            return { type: 'done' };
+        }
+        finally {
+            network.close();
+        }
+    }
+    finally {
+        inUse.delete(workspace.label);
+    }
+}
+exports.executeInRunner = executeInRunner;
+// The pod of the sandbox's files the run holds, as long as it is still that pod
+async function getFilesPod(instance, namespace, workspace) {
+    if (!workspace.pod) {
+        return undefined;
+    }
+    const pod = await instance.coreApi
+        .readNamespacedPod({ name: workspace.pod.name, namespace })
+        .catch((e) => ((0, apply_1.statusCodeOf)(e) === 404 ? undefined : Promise.reject(e)));
+    if (!pod || pod.metadata?.uid !== workspace.pod.uid || pod.metadata?.deletionTimestamp) {
+        return undefined;
+    }
+    return ownerOf(pod);
+}
+function ownerOf(pod) {
+    return { apiVersion: 'v1', kind: 'Pod', name: pod.metadata.name, uid: pod.metadata.uid };
+}
+// The runner of that name, when it belongs to the pod of the files and runs, or still
+// starts (`ready` false); one that ended or is going is deleted
+async function getRunner(instance, namespace, name, owner) {
+    const pod = await instance.coreApi
+        .readNamespacedPod({ name, namespace })
+        .catch((e) => ((0, apply_1.statusCodeOf)(e) === 404 ? undefined : Promise.reject(e)));
+    if (!pod) {
+        return undefined;
+    }
+    const ours = !pod.metadata?.deletionTimestamp &&
+        (pod.metadata?.ownerReferences ?? []).some((reference) => reference.uid === owner);
+    if (ours && isRunning(pod)) {
+        return { pod, ready: true };
+    }
+    if (ours && pod.status?.phase === 'Pending' && !getStartFailure(pod)) {
+        return { pod, ready: false };
+    }
+    await deletePod(instance, namespace, name);
+    return undefined;
+}
+function isRunning(pod) {
+    const status = pod.status?.containerStatuses?.find((s) => s.name === RUNNER_CONTAINER);
+    return pod.status?.phase === 'Running' && !!status?.state?.running;
+}
+// Why the runner will not run: its container ended, or an image can not be pulled, which
+// waits forever and is as good as ended
+function getStartFailure(pod) {
+    const status = pod.status?.containerStatuses?.find((s) => s.name === RUNNER_CONTAINER);
+    const unpulled = [...(pod.status?.initContainerStatuses ?? []), ...(pod.status?.containerStatuses ?? [])].find((s) => PULL_FAILURES.includes(s.state?.waiting?.reason ?? ''));
+    if (unpulled) {
+        return `${unpulled.state?.waiting?.reason} of ${unpulled.image}`;
+    }
+    if (pod.status?.phase === 'Failed' || pod.status?.phase === 'Succeeded' || status?.state?.terminated) {
+        return `its container ended with ${status?.state?.terminated?.exitCode ?? 'no code'}`;
+    }
+    return undefined;
+}
+// Waits until no pod of that name is left: a new one of the same name is refused before
+async function deletePod(instance, namespace, name) {
+    try {
+        await instance.coreApi.deleteNamespacedPod({ name, namespace, gracePeriodSeconds: 0 });
+    }
+    catch (e) {
+        if ((0, apply_1.statusCodeOf)(e) === 404) {
+            return;
+        }
+        throw e;
+    }
+    const deadline = Date.now() + RUNNER_START_MS;
+    while (Date.now() < deadline) {
+        const left = await instance.coreApi
+            .readNamespacedPod({ name, namespace })
+            .catch((e) => ((0, apply_1.statusCodeOf)(e) === 404 ? undefined : Promise.reject(e)));
+        if (!left) {
+            return;
+        }
+        await (0, sleep_1.sleep)(RUNNER_POLL_MS);
+    }
+    throw new Error(`the runner ${name} is still there`);
+}
+// Starts the runner and answers it once it runs, or why it does not: the cluster refused
+// it (a full quota, say, which a Job waits for), its container ended, or it took too long
+async function startRunner(instance, spec, abort) {
+    try {
+        await instance.objectApi.create(spec);
+    }
+    catch (e) {
+        return { reason: 'refused', message: (0, log_1.getErrorMessage)(e) };
+    }
+    return await waitForRunner(instance, spec.metadata.namespace, spec, abort);
+}
+// Waits until the runner runs. A task stopped meanwhile (cancelled, out of time) leaves it
+// starting, for the next command, which would time out the same way if each started anew;
+// one that ends, or does not run within RUNNER_START_MS of being made, goes.
+async function waitForRunner(instance, namespace, runner, abort) {
+    const name = runner.metadata.name;
+    const created = runner.metadata?.creationTimestamp ? new Date(runner.metadata.creationTimestamp).getTime() : NaN;
+    const deadline = (Number.isNaN(created) ? Date.now() : Math.min(created, Date.now())) + RUNNER_START_MS;
+    for (;;) {
+        const pod = await instance.coreApi.readNamespacedPod({ name, namespace });
+        if (isRunning(pod)) {
+            return { pod };
+        }
+        const failure = getStartFailure(pod);
+        if (failure || Date.now() > deadline) {
+            await deletePod(instance, namespace, name).catch(() => undefined);
+            return failure ? { reason: 'ended', message: failure } : { reason: 'stopped', message: 'it took too long' };
+        }
+        if (abort.aborted) {
+            return { reason: 'stopped', message: 'the task was stopped' };
+        }
+        await (0, sleep_1.sleep)(RUNNER_POLL_MS);
+    }
+}
+// Removes the runners of a sandbox but those `keep` takes, and once each is gone its policy
+// and proxy, which the pod of the sandbox's files would keep while it runs
+async function removeSandboxRunners(instance, namespace, sandbox, keep) {
+    if (!keep) {
+        // all of the sandbox's runners go: so does what was remembered of them
+        failed.delete(sandbox);
+    }
+    const pods = await instance.coreApi.listNamespacedPod({
+        namespace,
+        labelSelector: `${volumes_1.SANDBOX_LABEL}=${sandbox},${exports.RUNNER_LABEL}`,
+    });
+    for (const pod of pods.items) {
+        const name = pod.metadata?.name;
+        const key = pod.metadata?.labels?.[resources_1.ID_LABEL];
+        if (!name || keep?.(pod)) {
+            continue;
+        }
+        await deletePod(instance, namespace, name);
+        if (key) {
+            const item = { id: () => key };
+            await instance.objectApi
+                .delete({
+                apiVersion: 'networking.k8s.io/v1',
+                kind: 'NetworkPolicy',
+                metadata: { name: (0, network_policy_1.getNetworkPolicyName)(item, { run: null }), namespace },
+            })
+                .catch((e) => ((0, apply_1.statusCodeOf)(e) === 404 ? undefined : Promise.reject(e)));
+            await (0, network_policy_1.removeSandboxProxies)(instance, namespace, { sandbox, item });
+        }
+    }
+}
+exports.removeSandboxRunners = removeSandboxRunners;
+async function setPolicyOwners(instance, namespace, name, owners) {
+    await instance.objectApi.patch({
+        apiVersion: 'networking.k8s.io/v1',
+        kind: 'NetworkPolicy',
+        metadata: { name, namespace, ownerReferences: owners },
+    }, undefined, undefined, undefined, undefined, 'application/merge-patch+json');
+}
+// Runs a command in the runner and answers its exit code. Its output goes to the task's,
+// line by line as it comes. On abort (cancel, timeout) every process of the command is
+// killed, or the runner goes when that fails.
+async function execInRunner(instance, namespace, pod, 
+// the directory of the run's sandbox, and the command's in it
+root, cwd, command, scripts, task, abort) {
+    const { cleanUp } = scripts;
+    const output = { bytes: 0, cut: false };
+    let changed = false;
+    const lines = (stream) => {
+        let rest = '';
+        // a character split between two chunks stays whole
+        const decoder = new string_decoder_1.StringDecoder('utf8');
+        const writable = new stream_1.Writable({
+            write(chunk, _encoding, callback) {
+                if (output.bytes < MAX_OUTPUT_BYTES) {
+                    output.bytes += chunk.length;
+                    const parts = (rest + decoder.write(chunk)).split(/\r?\n/);
+                    rest = parts.pop() ?? '';
+                    for (const line of parts) {
+                        if (stream === 'stderr' && line === RUNNER_CHANGED_MARK) {
+                            changed = true;
+                            continue;
+                        }
+                        task.status.console(stream, line);
+                    }
+                }
+                else if (!output.cut) {
+                    output.cut = true;
+                    task.status.write('warn', `the output of ${task.name} was cut at ${MAX_OUTPUT_BYTES} bytes`);
+                }
+                callback();
+            },
+            final(callback) {
+                rest += decoder.end();
+                if (rest.length > 0) {
+                    task.status.console(stream, rest);
+                }
+                callback();
+            },
+        });
+        return writable;
+    };
+    const stdout = lines('stdout');
+    const stderr = lines('stderr');
+    let status;
+    const socket = await instance.exec.exec(namespace, pod, RUNNER_CONTAINER, [BUSYBOX, 'sh', '-c', scripts.wrapper, 'hammerkit', RUNNER_CHANGED_MARK, root, cwd, ...command], stdout, stderr, null, false, (received) => {
+        status = received;
+    });
+    const closed = new Promise((resolve, reject) => {
+        socket.on('close', () => resolve());
+        socket.on('error', (e) => reject(e));
+    });
+    const kill = async () => {
+        try {
+            await killInRunner(instance, namespace, pod, cleanUp);
+        }
+        catch (e) {
+            task.status.write('debug', `the command could not be stopped in the runner, it goes: ${(0, log_1.getErrorMessage)(e)}`);
+            await deletePod(instance, namespace, pod).catch(() => undefined);
+        }
+        // what the command printed after it was killed is not waited for long
+        await Promise.race([closed.catch(() => undefined), (0, sleep_1.sleep)(KILL_TIMEOUT_MS)]);
+        socket.close();
+    };
+    const aborted = () => {
+        void kill();
+    };
+    if (abort.aborted) {
+        aborted();
+    }
+    else {
+        abort.addEventListener('abort', aborted, { once: true });
+    }
+    let broken;
+    try {
+        await closed.catch((e) => {
+            broken = e;
+        });
+        if (!abort.aborted && getExecExitCode(status) === undefined) {
+            // the connection broke while the command may still run: it ends here as on a cancel,
+            // before the run lets go of the sandbox
+            await kill();
+        }
+    }
+    finally {
+        abort.removeEventListener('abort', aborted);
+        for (const stream of [stdout, stderr]) {
+            if (!stream.writableEnded) {
+                stream.end();
+            }
+        }
+        await Promise.all([(0, promises_1.finished)(stdout), (0, promises_1.finished)(stderr)]).catch(() => undefined);
+    }
+    if (abort.aborted) {
+        return 137;
+    }
+    const exitCode = getExecExitCode(status);
+    if (exitCode === undefined) {
+        throw new Error(`the command in the runner of the sandbox ended without an exit code: ${status?.message ?? (broken ? (0, log_1.getErrorMessage)(broken) : 'the connection closed')}`);
+    }
+    return exitCode === exports.RUNNER_CHANGED_EXIT && changed ? 'changed' : exitCode;
+}
+async function killInRunner(instance, namespace, pod, cleanUp) {
+    let status;
+    const socket = await instance.exec.exec(namespace, pod, RUNNER_CONTAINER, [BUSYBOX, 'sh', '-c', cleanUp], null, null, null, false, (received) => {
+        status = received;
+    });
+    await Promise.race([
+        new Promise((resolve, reject) => {
+            socket.on('close', () => resolve());
+            socket.on('error', reject);
+        }),
+        (0, sleep_1.sleep)(KILL_TIMEOUT_MS).then(() => Promise.reject(new Error('the kill did not answer'))),
+    ]);
+    // what it answers does not matter, that it answered does
+    if (getExecExitCode(status) === undefined) {
+        throw new Error(status?.message ?? 'the kill did not end');
+    }
+}
+//# sourceMappingURL=sandbox-runner.js.map

package/dist/planner/work-runtime-kubernetes.js

@@ -34,6 +34,7 @@ const job_placement_1 = require("../kubernetes/job-placement");
 const network_policy_1 = require("../kubernetes/network-policy");
 const run_id_1 = require("../utils/run-id");
 const work_network_1 = require("./work-network");
+const sandbox_runner_1 = require("../kubernetes/sandbox-runner");
 const work_secret_1 = require("./work-secret");
 async function listJobNames(instance, kubernetes, task) {
     const jobs = await instance.batchApi.listNamespacedJob({
@@ -188,6 +189,28 @@ function kubernetesTaskRuntime(task, kubernetes) {
             const envs = (0, replace_env_variables_1.getEnvironmentVariables)(task.data.envs);
             const persistence = await (0, volumes_1.getKubernetesPersistence)(task, kubernetes);
             const workspace = (0, volumes_1.getWorkspacePodSpec)(kubernetes, persistence, { name: task.name, pool: task.data.pool });
+            // a shell of a sandbox: its commands go into the sandbox's runner, when it has one now
+            if ((0, sandbox_runner_1.canUseRunner)(task, kubernetes, persistence)) {
+                try {
+                    const result = await (0, sandbox_runner_1.executeInRunner)(instance, task, kubernetes, persistence, options.abort);
+                    if (result.type === 'crash') {
+                        options.state.set({ stateKey: options.stateKey, type: 'crash', exitCode: result.exitCode });
+                        return;
+                    }
+                    // stopped while the runner started: no Job for it either
+                    if (result.type === 'canceled' || options.abort.aborted) {
+                        options.state.set({ stateKey: options.stateKey, type: 'canceled' });
+                        return;
+                    }
+                    if (result.type === 'done') {
+                        return;
+                    }
+                }
+                catch (e) {
+                    options.state.set({ stateKey: options.stateKey, type: 'error', errorMessage: (0, log_1.getErrorMessage)(e) });
+                    return;
+                }
+            }
             await (0, ensure_namespace_1.ensureNamespace)(instance, kubernetes.namespace);
             try {
                 // the task writes to it now: what it holds stops being the outputs of a state, and

package/dist/program.js

@@ -608,6 +608,7 @@ async function getProgram(environment, argv) {
             .choices(['checksum', 'modify-date', 'none']))
             .addOption(new commander_1.Option('--cache-read-only', `restore from cache backends but never push to them (or set ${read_only_1.CACHE_READ_ONLY_ENV}=1)`).default(false))
             .addOption(new commander_1.Option('--timeout <duration>', 'fail tasks without their own timeout after this long (e.g. 10m)').argParser(units_1.parseDuration))
+            .addOption(new commander_1.Option('--max-timeout <duration>', 'fail any task after this long, also one with a longer timeout of its own (e.g. 2m)').argParser(units_1.parseDuration))
             .addOption(new commander_1.Option('--network <level>', 'the most any task or service reaches besides its needs: none, open or allow:<host>,<host>').argParser(parseNetworkArgument))
             .action(async (options, command) => {
             try {
@@ -626,6 +627,7 @@ async function getProgram(environment, argv) {
                     wait: options.wait,
                     cacheReadOnly: (0, read_only_1.isCacheReadOnly)(options.cacheReadOnly, environment.processEnvs),
                     timeout: options.timeout ?? null,
+                    maxTimeout: options.maxTimeout ?? null,
                     network: options.network ?? null,
                 });
                 if (!result.success) {
@@ -688,6 +690,7 @@ async function getProgram(environment, argv) {
             .addOption(new commander_1.Option('--dry-run', 'print the execution plan with predicted cache hits/misses without running').default(false))
             .addOption(new commander_1.Option('--cache-read-only', `restore from cache backends but never push to them (or set ${read_only_1.CACHE_READ_ONLY_ENV}=1)`).default(false))
             .addOption(new commander_1.Option('--timeout <duration>', 'fail tasks without their own timeout after this long (e.g. 10m)').argParser(units_1.parseDuration))
+            .addOption(new commander_1.Option('--max-timeout <duration>', 'fail any task after this long, also one with a longer timeout of its own (e.g. 2m)').argParser(units_1.parseDuration))
             .addOption(new commander_1.Option('--network <level>', 'the most any task or service reaches besides its needs: none, open or allow:<host>,<host>').argParser(parseNetworkArgument))
             .addOption(new commander_1.Option('--no-skip-deps', 'run dependencies even when nothing needing them has to run'))
             .action(async (task, options, command) => {
@@ -722,6 +725,7 @@ async function getProgram(environment, argv) {
                     explain: options.explain,
                     cacheReadOnly: (0, read_only_1.isCacheReadOnly)(options.cacheReadOnly, environment.processEnvs),
                     timeout: options.timeout ?? null,
+                    maxTimeout: options.maxTimeout ?? null,
                     network: options.network ?? null,
                     skipDeps: options.skipDeps !== false,
                 });

package/dist/remote/remote-work-commands.js

@@ -139,7 +139,8 @@ function registerWorkCommands(remote, context) {
         .addOption(new commander_1.Option('--commit <sha>', 'the commit the ref has to be at, else the run is refused; the commit checked out here when --ref is left out'))
         .addOption(new commander_1.Option('--env <name>', 'the environment of the build file, as --env of run'))
         .addOption(new commander_1.Option('--cache <method>', 'as --cache of run').choices(['checksum', 'modify-date', 'none']))
-        .addOption(new commander_1.Option('--timeout <duration>', 'stop the run after this long, such as 30m'))
+        .addOption(new commander_1.Option('--timeout <duration>', 'as --timeout of run: for tasks without their own, such as 30m'))
+        .addOption(new commander_1.Option('--max-timeout <duration>', 'as --max-timeout of run: for every task, also one with its own timeout'))
         .action(guarded('Remote run was not successful', async (tasks, options) => {
         const named = options.build ?? environment.processEnvs.HAMMERKIT_BUILD;
         const definition = named ?? (0, bundle_1.hashBundle)(await (0, collect_bundle_1.collectBundle)(buildFile, environment));
@@ -160,7 +161,7 @@ function registerWorkCommands(remote, context) {
         const { name: serverName, server } = await getServer(options.server);
         const connection = { url: server.url, ...(0, token_source_1.getTokenSource)(environment, serverName, server) };
         const runOptions = {};
-        for (const key of ['env', 'cache', 'timeout']) {
+        for (const key of ['env', 'cache', 'timeout', 'maxTimeout']) {
             if (options[key] !== undefined) {
                 runOptions[key] = options[key];
             }

package/dist/server/kubernetes-backend.js

@@ -22,6 +22,8 @@ const POLICY_NAME = 'hammerkit';
 const MANAGED_LABEL = 'hammerkit.dev/managed';
 // how long the end of a run waits for its pods and claims to be gone
 const RELEASE_TIMEOUT = 120000;
+// how long the setup of a caller's namespace is taken as it is, before it is applied again
+const NAMESPACE_SETUP_REUSE_MS = 5 * 60000;
 // where a pod of the cluster finds the token and certificate of its ServiceAccount
 const IN_CLUSTER_NAMESPACE = '/var/run/secrets/kubernetes.io/serviceaccount/namespace';
 /**
@@ -186,6 +188,9 @@ class KubernetesExecutor {
         this.sweeps = new Map();
         // whether the kept claims a server left when it stopped were looked for
         this.swept = false;
+        // the namespaces where no run of this process left anything: what a run makes goes at its
+        // end, unless its release failed
+        this.tidy = new Set();
         this.access = getClusterAccess(config.backend, processEnvs);
         this.instance = (0, kubernetes_instance_1.createKubernetesInstances)({
             type: 'kubernetes',
@@ -268,11 +273,14 @@ class KubernetesExecutor {
                         throw new Error('the run was cancelled');
                     }
                     await this.namespaces.prepare(namespace);
-                    // what an earlier run left when the server stopped before its end: the objects of
-                    // a run that does not execute. One that starts after this waits for it here
-                    // before it makes any.
-                    const running = [...new Set([label, ...(this.active.get(namespace)?.keys() ?? [])])];
-                    await removeRunObjects(this.instance, namespace, `${job_placement_1.RUN_LABEL},${job_placement_1.RUN_LABEL} notin (${running.join(',')})`, RELEASE_TIMEOUT);
+                    // what an earlier run left when the server stopped before its end, or when its
+                    // release failed: the objects of a run that does not execute. One that starts
+                    // after this waits for it here before it makes any.
+                    if (!this.tidy.has(namespace)) {
+                        const running = [...new Set([label, ...(this.active.get(namespace)?.keys() ?? [])])];
+                        await removeRunObjects(this.instance, namespace, `${job_placement_1.RUN_LABEL},${job_placement_1.RUN_LABEL} notin (${running.join(',')})`, RELEASE_TIMEOUT);
+                        this.tidy.add(namespace);
+                    }
                     // the claims earlier runs kept that expired; the others this run may mount
                     const { next } = await (0, kept_claims_1.sweepKeptClaims)(this.instance, namespace, {
                         now: new Date(),
@@ -302,7 +310,9 @@ class KubernetesExecutor {
                         pools: backend.pools,
                         providers: backend.providers,
                         accounts: Object.fromEntries(Object.entries(this.config.accounts).flatMap(([name, account]) => account.kubernetes ? [[name, account.kubernetes]] : [])),
-                        ...(work?.claim ? { workspace: getWorkspace(work.root, work.claim, namespace) } : {}),
+                        ...(work?.claim
+                            ? { workspace: getWorkspace(work.root, work.claim, namespace, backend.sandboxes.runner) }
+                            : {}),
                         ...(this.keep > 0 ? { volumeScope: getVolumeScope(run) } : {}),
                     },
                 };
@@ -315,6 +325,8 @@ class KubernetesExecutor {
                     await removeRunObjects(this.instance, namespace, selector, RELEASE_TIMEOUT);
                 }
                 catch (e) {
+                    // the next run removes what is left
+                    this.tidy.delete(namespace);
                     context.emit({
                         type: 'log',
                         task: 'hammerkit',
@@ -411,6 +423,8 @@ class CallerNamespaces {
     constructor(instance, backend) {
         this.instance = instance;
         this.backend = backend;
+        // the namespaces this process set up, by their uid, and when
+        this.applied = new Map();
     }
     name(owner) {
         return getCallerNamespace(this.backend.namespacePrefix, owner);
@@ -418,8 +432,10 @@ class CallerNamespaces {
     // The caller's namespace: made by the server and kept apart from the others. The
     // server binds its rights there, isolates the network of the pods, applies the
     // operator's quota and gives containers without resources default requests, so they
-    // are admitted under a quota on compute. All of it is replaced on every run, so the
-    // namespace follows the config.
+    // are admitted under a quota on compute. All of it is replaced when the server sets up
+    // a namespace it did not set up in the last `NAMESPACE_SETUP_REUSE_MS` (it reads the
+    // config once, at its start), so the namespace follows the config; a namespace made anew
+    // since (deleted and made again) is set up again at once.
     async prepare(namespace) {
         const backend = this.backend;
         let existing;
@@ -434,9 +450,19 @@ class CallerNamespaces {
         if (existing && existing.metadata?.labels?.[MANAGED_LABEL] !== 'true') {
             throw new Error(`the namespace ${namespace} was not made by hammerkit, the server does not run tasks in it`);
         }
+        const started = Date.now();
+        const applied = this.applied.get(namespace);
+        if (existing &&
+            !existing.metadata?.deletionTimestamp &&
+            applied &&
+            applied.uid === existing.metadata?.uid &&
+            started - applied.at < NAMESPACE_SETUP_REUSE_MS) {
+            return;
+        }
+        this.applied.delete(namespace);
         if (!existing) {
             try {
-                await this.instance.coreApi.createNamespace({
+                existing = await this.instance.coreApi.createNamespace({
                     body: {
                         metadata: {
                             name: namespace,
@@ -506,6 +532,10 @@ class CallerNamespaces {
                 });
             }
         }
+        const uid = existing?.metadata?.uid;
+        if (uid) {
+            this.applied.set(namespace, { uid, at: started });
+        }
     }
     whoAmI() {
         if (!this.subject) {
@@ -528,7 +558,7 @@ class CallerNamespaces {
 exports.CallerNamespaces = CallerNamespaces;
 // The claim of a sandbox as the Jobs of its run mount it: the run's copy of the files is
 // `root`, a source below it is the same path below `subPath` in the claim.
-function getWorkspace(root, claim, namespace) {
+function getWorkspace(root, claim, namespace, runner) {
     if (claim.namespace !== namespace) {
         throw new Error(`the sandbox is in ${claim.namespace}, the run in ${namespace}`);
     }
@@ -538,6 +568,7 @@ function getWorkspace(root, claim, namespace) {
         subPath: claim.subPath,
         label: claim.label,
         ...(claim.pod ? { pod: claim.pod } : {}),
+        ...(runner ? { runner: true } : {}),
     };
 }
 // Everything a run made in the caller's namespace, by the label selector of the run:

package/dist/server/kubernetes-sandbox-storage.js

@@ -11,6 +11,7 @@ const tar_2 = require("../utils/tar");
 const apply_1 = require("../kubernetes/apply");
 const kubernetes_instance_1 = require("../kubernetes/kubernetes-instance");
 const network_policy_1 = require("../kubernetes/network-policy");
+const sandbox_runner_1 = require("../kubernetes/sandbox-runner");
 const volumes_1 = require("../kubernetes/volumes");
 const log_1 = require("../log");
 const sleep_1 = require("../utils/sleep");
@@ -284,6 +285,8 @@ class KubernetesSandboxStorage {
             // runs on a sandbox go one at a time: a proxy the pod of the files does not own was
             // left by a run before (the server stopped, or its release could not remove it)
             await (0, network_policy_1.removeSandboxProxies)(this.instance, namespace, { sandbox: names.label }, (deployment) => isOwnedBy(deployment, pod.uid)).catch(() => undefined);
+            // and a runner, which the pod did not own when it made way for a run
+            await (0, sandbox_runner_1.removeSandboxRunners)(this.instance, namespace, names.label, (runner) => isOwnedBy(runner, pod.uid)).catch(() => undefined);
             await (0, promises_1.mkdir)(into, { recursive: true });
             await this.copyOut(key, ['tar', 'cf', '-', '-C', `${CLAIM_PATH}/${key.id}/files`, '.'], into, 
             // what the run plans with: files, directories and links, nothing outside of it
@@ -388,7 +391,8 @@ class KubernetesSandboxStorage {
     async removeIn(namespace, id) {
         const names = getSandboxNames(id);
         await this.stopPod({ id, owner: { iss: '', sub: '' } }, namespace);
-        // the cluster removes the proxies the pod owned; those it did not own go here
+        // the cluster removes the proxies and the runner the pod owned; those it did not own go here
+        await (0, sandbox_runner_1.removeSandboxRunners)(this.instance, namespace, names.label);
         await (0, network_policy_1.removeSandboxProxies)(this.instance, namespace, { sandbox: names.label });
         for (const name of [names.claim, names.git]) {
             try {

package/dist/server/run-request.js

@@ -38,6 +38,8 @@ exports.runOptionsSchema = zod_1.z
     cacheReadOnly: zod_1.z.boolean().optional(),
     // as --timeout
     timeout: duration.optional(),
+    // as --max-timeout: caps every task, also one with a longer timeout of its own
+    maxTimeout: duration.optional(),
     // as --no-skip-deps when false
     skipDeps: zod_1.z.boolean().optional(),
     // as the text after `--`: appended to the last command of the one task the run

package/dist/server/run-worker.js

@@ -120,6 +120,7 @@ cancelled, emit) {
             cacheDefault: options.cache ?? 'checksum',
             cacheReadOnly: options.cacheReadOnly ?? false,
             timeout: options.timeout ? (0, units_1.parseDuration)(options.timeout) : null,
+            maxTimeout: options.maxTimeout ? (0, units_1.parseDuration)(options.maxTimeout) : null,
             skipDeps: options.skipDeps !== false,
             network: getRunNetwork(input),
             // why a task was not taken from the cache goes into its result

package/dist/server/server-config.js

@@ -188,8 +188,12 @@ const kubernetesSandboxesSchema = zod_1.z
     // when left out. With a class of their own, a ResourceQuota on that class in each
     // caller's namespace holds `sandboxes.limits`.
     storageClass: zod_1.z.string().min(1).optional(),
-    // how long the pod of a sandbox's files stays after the last request
+    // how long the pod of a sandbox's files stays after the last request; the runner of the
+    // sandbox goes with it
     idle: duration.default('1m'),
+    // exec the commands of a task shaped like a shell (it mounts the sandbox only, has no
+    // needs, outputs or secrets) into one pod per sandbox, in place of a Job each
+    runner: zod_1.z.boolean().default(true),
     // of the pod of a sandbox's files, which runs git and ripgrep over all of them
     resources: zod_1.z
         .object({ requests: quantitiesSchema.optional(), limits: quantitiesSchema.optional() })

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "hammerkit",
   "description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
-  "version": "1.27.0-next.11",
+  "version": "1.27.0-next.13",
   "keywords": [
     "build",
     "build-tool",