hammerkit 1.26.0 → 1.27.0-next.10

package/dist/server/kubernetes-sandbox-storage.js

@@ -0,0 +1,997 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getPushRefusal = exports.KubernetesSandboxStorage = exports.getSandboxNames = void 0;
+const crypto_1 = require("crypto");
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const stream_1 = require("stream");
+const promises_2 = require("stream/promises");
+const tar_1 = require("tar");
+const tar_2 = require("../utils/tar");
+const apply_1 = require("../kubernetes/apply");
+const kubernetes_instance_1 = require("../kubernetes/kubernetes-instance");
+const network_policy_1 = require("../kubernetes/network-policy");
+const volumes_1 = require("../kubernetes/volumes");
+const log_1 = require("../log");
+const sleep_1 = require("../utils/sleep");
+const units_1 = require("../utils/units");
+const version_1 = require("../version");
+const git_checkout_1 = require("./git-checkout");
+const kubernetes_backend_1 = require("./kubernetes-backend");
+const sandbox_files_1 = require("./sandbox-files");
+const sandbox_pod_1 = require("./sandbox-pod");
+const sandbox_storage_1 = require("./sandbox-storage");
+const sandboxes_1 = require("./sandboxes");
+// A sandbox has two claims. One holds its files (in `files/`), which the tasks of runs
+// mount; the other holds its git data, which only the pod of its files mounts, so nothing a
+// task does reaches the configuration or the hooks of git: a link a task makes in the files
+// leads at most to the root of the claim of the files.
+//
+// The pod mounts the claim of the git data where a host keeps its sandboxes, and the files
+// at `<id>/files` below it, so the code of a host (`hammerkit server-sandbox`) works on it
+// unchanged; what the server fetched for it goes to `incoming` there.
+const CLAIM_PATH = '/claim';
+const FILES_DIRECTORY = 'files';
+const MANAGED = { 'hammerkit.dev/managed': 'true' };
+// the claims of the sandboxes of one server (its prefix of caller namespaces and its own
+// namespace): another server on the cluster leaves them alone
+const SERVER_LABEL = 'hammerkit.dev/sandbox-server';
+// the pods of sandboxes' files: no network at all
+const FILES_LABEL = 'hammerkit.dev/sandbox-files';
+const FILES_POLICY = 'hammerkit-sandbox-files';
+const QUOTA_NAME = 'hammerkit-sandboxes';
+const ID_ANNOTATION = 'hammerkit.dev/sandbox-id';
+const ISS_ANNOTATION = 'hammerkit.dev/owner-iss';
+const SUB_ANNOTATION = 'hammerkit.dev/owner-sub';
+// a pod that is left behind (the server stopped) ends on its own
+const POD_DEADLINE_SECONDS = 6 * 60 * 60;
+// a pod this old is replaced when nothing uses it, well before its deadline ends it
+const POD_RENEW_MS = (POD_DEADLINE_SECONDS - 60 * 60) * 1000;
+// how often a run's pods are looked at, and how long one may find no room next to the pod
+// of the files before that pod makes way
+const UNSCHEDULABLE_POLL_MS = 3000;
+const UNSCHEDULABLE_MS = 10000;
+// how long a pod may take to start, its image pull included
+const POD_START_MS = 5 * 60 * 1000;
+const REQUEST_TIMEOUT_MS = 5 * 60 * 1000;
+const FETCH_TIMEOUT_MS = 10 * 60 * 1000;
+// an answer is read into memory up to this; a diff is up to 64 MiB, as JSON a bit more
+const MAX_ANSWER_BYTES = 160 * 1024 * 1024;
+const MAX_STDERR = 16 * 1024;
+// what is sent into a pod and not taken by the cluster yet, before the server waits
+const MAX_STDIN_BUFFERED = 4 * 1024 * 1024;
+// The names of a sandbox on the cluster: its id has letters of both cases and `_`, which
+// names and labels can not all take, so they carry a hash of it.
+function getSandboxNames(id) {
+    const label = (0, crypto_1.createHash)('sha256').update(id).digest('hex').slice(0, 20);
+    return {
+        claim: `hammerkit-sandbox-${label}`,
+        git: `hammerkit-sandbox-${label}-git`,
+        pod: `hammerkit-sandbox-${label}-files`,
+        label,
+    };
+}
+exports.getSandboxNames = getSandboxNames;
+/**
+ * Sandboxes on a cluster: PersistentVolumeClaims per sandbox in its caller's namespace (its
+ * files, and its git data), mounted only while a task of a run on it or a short pod for its
+ * files runs. An idle sandbox costs its storage only.
+ *
+ * - The server fetches the commit (it holds the read credential of `sources`, as for a
+ *   ref run) and hands the repository to the pod of the sandbox, which takes it into
+ *   the claim. Nothing of the credential is in the caller's namespace.
+ * - Reading, searching and editing the files, the state and the usage run in that pod,
+ *   with the code of a host (`hammerkit server-sandbox`, see sandbox-pod.ts), so the
+ *   rules on paths, links and the deny list are the same. The pod has no network and no
+ *   token, and stays while requests come, `backend.sandboxes.idle` after the last.
+ * - A run on the sandbox gets a copy of the files to plan with, and its tasks mount the
+ *   claim itself (WorkKubernetesWorkspace). The pod of its files stays while the run
+ *   works, and the pods of the run go to its node: a claim of one node at a time is then
+ *   never moved between nodes, which takes seconds each time.
+ * - A push is made by the server: the pod commits the changes, the server copies the
+ *   objects of the sandbox's git data, checks the commit and pushes it with the write
+ *   credential, which never enters the caller's namespace either.
+ * - With `backend.sandboxes.storageClass`, a ResourceQuota on that class in the
+ *   caller's namespace holds `sandboxes.limits` on the cluster as well.
+ */
+class KubernetesSandboxStorage {
+    constructor(config, limits, 
+    // the environment of git when the server fetches a repository: the variables runs see
+    env, 
+    // where the server fetches a commit before it goes into a claim
+    staging, instance) {
+        this.limits = limits;
+        this.env = env;
+        this.staging = staging;
+        this.pods = new Map();
+        // the namespaces whose policy and quota for sandboxes this process has applied
+        this.prepared = new Map();
+        // one request at a time per sandbox, as a host has it: the objects `state` writes are
+        // pruned again in the pod, which must not meet another request that still needs them
+        this.queues = new Map();
+        this.backend = config.backend;
+        const access = (0, kubernetes_backend_1.getClusterAccess)(config.backend, env);
+        this.instance =
+            instance ??
+                (0, kubernetes_instance_1.createKubernetesInstances)({
+                    type: 'kubernetes',
+                    namespace: 'default',
+                    kubeConfig: access.kubeConfig,
+                    context: access.context,
+                    httpRoutes: [],
+                    networkPolicyEnforced: true,
+                });
+        this.namespaces = new kubernetes_backend_1.CallerNamespaces(this.instance, config.backend);
+        this.server = (0, crypto_1.createHash)('sha256')
+            .update(`${config.backend.namespacePrefix}\n${config.backend.namespace ?? ''}`)
+            .digest('hex')
+            .slice(0, 20);
+    }
+    async create(key, fetch) {
+        const namespace = this.namespaces.name(key.owner);
+        await this.namespaces.prepare(namespace);
+        await this.prepare(namespace);
+        const names = getSandboxNames(key.id);
+        try {
+            await this.createClaim(key, namespace, names.git);
+            await this.createClaim(key, namespace, names.claim);
+            return await this.receive(key, fetch, 'create');
+        }
+        catch (e) {
+            await this.remove(key).catch(() => undefined);
+            throw e;
+        }
+    }
+    async createClaim(key, namespace, name) {
+        const sandboxes = this.backend.sandboxes;
+        const claim = {
+            apiVersion: 'v1',
+            kind: 'PersistentVolumeClaim',
+            metadata: {
+                name,
+                namespace,
+                labels: { ...MANAGED, [volumes_1.SANDBOX_LABEL]: getSandboxNames(key.id).label, [SERVER_LABEL]: this.server },
+                annotations: {
+                    'hammerkit.dev/version': (0, version_1.getVersion)(),
+                    [ID_ANNOTATION]: key.id,
+                    [ISS_ANNOTATION]: key.owner.iss,
+                    [SUB_ANNOTATION]: key.owner.sub,
+                },
+            },
+            spec: {
+                accessModes: ['ReadWriteOnce'],
+                storageClassName: sandboxes.storageClass ?? this.backend.storageClass,
+                resources: { requests: { storage: String((0, units_1.parseSize)(sandboxes.size)) } },
+            },
+        };
+        try {
+            await this.instance.objectApi.create(claim);
+        }
+        catch (e) {
+            // over the ResourceQuota of the caller's sandboxes
+            if ((0, apply_1.statusCodeOf)(e) === 403 && /exceeded quota/.test((0, log_1.getErrorMessage)(e))) {
+                throw new sandboxes_1.SandboxError(413, `your sandboxes are at your limit of ${this.limits.count} or ${(0, units_1.formatSize)(this.limits.bytes)}: delete one`);
+            }
+            throw e;
+        }
+    }
+    async fetch(key, fetch) {
+        return this.receive(key, fetch, 'fetch');
+    }
+    async state(key, signal) {
+        return (await this.request(key, { op: 'state', id: key.id }, signal)).result;
+    }
+    async usage(key) {
+        return (await this.request(key, { op: 'usage', id: key.id })).result;
+    }
+    /**
+     * The pod commits the changes; the server takes the objects of the sandbox's git data
+     * (not its configuration), checks that the commit is one on top of the sandbox's that
+     * leaves out the deny list, and pushes it. The pod of the files can not make the server
+     * push something else, and the write credential stays with the server.
+     */
+    async push(key, push) {
+        // the commit is copied out and pushed before a `state` may prune it again
+        return this.exclusive(key.id, () => this.commitAndPush(key, push));
+    }
+    async commitAndPush(key, push) {
+        const commit = (await this.requestNow(key, { op: 'commit', id: key.id, author: push.author }, push.signal)).result;
+        if (commit === null) {
+            return undefined;
+        }
+        if (!/^[0-9a-f]{40}([0-9a-f]{24})?$/.test(commit)) {
+            throw new Error(`the pod of the sandbox ${key.id} did not answer a commit`);
+        }
+        const repository = (0, path_1.join)(this.staging, `${key.id}-push-${(0, crypto_1.randomBytes)(6).toString('hex')}.git`);
+        const what = `sandbox ${key.id}`;
+        const env = defined(this.env);
+        const run = (args, action, runEnv = env, options = {}) => (0, git_checkout_1.git)([`--git-dir=${repository}`, '-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', ...args], runEnv, push.signal, what, action, options);
+        try {
+            await (0, promises_1.mkdir)(this.staging, { recursive: true, mode: 0o700 });
+            await (0, git_checkout_1.git)(['init', '--quiet', '--bare', repository], env, push.signal, what, 'push');
+            // objects and where the shallow history ends, nothing that configures git or points
+            // it at other repositories (objects/info)
+            await this.copyOut(key, ['tar', 'cf', '-', '-C', `${CLAIM_PATH}/${key.id}/git`, '.'], repository, (path, type) => (type === 'File' &&
+                /^(\.\/)?(shallow|objects\/[0-9a-f]{2}\/[0-9a-f]+|objects\/pack\/pack-[0-9a-f]+\.(pack|idx))$/.test(path)) ||
+                (type === 'Directory' && /^(\.\/)?(objects(\/([0-9a-f]{2}|pack))?\/?)?$/.test(path)), push.signal);
+            const [found, ...parents] = (await run(['rev-list', '--parents', '-n', '1', commit], 'read')).trim().split(' ');
+            if (found !== commit || parents.length !== 1) {
+                throw new Error(`${commit} of the sandbox ${key.id} is not one commit on top of the sandbox's`);
+            }
+            const changed = (await run(['diff-tree', '-r', '-z', '--no-commit-id', '--name-only', '--no-renames', parents[0], commit], 'read'))
+                .split('\0')
+                .filter((path) => path.length > 0);
+            const denied = changed.find((path) => (0, sandbox_files_1.deniedPath)(path) || (0, sandbox_files_1.isRunState)(path));
+            if (denied) {
+                throw new Error(`the commit of the sandbox ${key.id} has ${denied}, which never leaves the server`);
+            }
+            // a branch this sandbox pushed before only moves forward
+            if (push.expect) {
+                const ancestor = await run(['merge-base', '--is-ancestor', push.expect, commit], 'read').then(() => true, () => false);
+                if (!ancestor) {
+                    throw new sandbox_storage_1.SandboxPushConflict(`${push.branch} is at ${push.expect}, which the sandbox is no longer on: push to a new branch`);
+                }
+            }
+            let printed;
+            try {
+                printed = await run([
+                    'push',
+                    // a refused ref is a line with the remote's reason, as on a host
+                    '--porcelain',
+                    '--no-verify',
+                    `--force-with-lease=refs/heads/${push.branch}:${push.expect ?? ''}`,
+                    '--',
+                    push.url,
+                    `${commit}:refs/heads/${push.branch}`,
+                ], 'push', 
+                // the write credential is in the environment of this one git process
+                push.env, { exitCodes: [1] });
+            }
+            catch (e) {
+                throw new sandbox_storage_1.SandboxPushFailed(commit, e);
+            }
+            const refusal = getPushRefusal(printed, push.branch);
+            if (refusal) {
+                throw new sandbox_storage_1.SandboxPushFailed(commit, new Error(refusal));
+            }
+            return commit;
+        }
+        finally {
+            await (0, promises_1.rm)(repository, { recursive: true, force: true });
+        }
+    }
+    async moveTo(key, commit) {
+        await this.request(key, { op: 'move', id: key.id, commit }, AbortSignal.timeout(FETCH_TIMEOUT_MS));
+    }
+    files(key) {
+        return new KubernetesSandboxFiles((request, signal) => this.request(key, { ...request, id: key.id }, signal), (path) => this.raw(key, path));
+    }
+    /**
+     * Copies the files into `into` for the process of the run, which reads the sources of
+     * tasks to know what changed. The pod of the files stays until the run released it,
+     * so the pods of the run, which go next to it, find the claim on its node.
+     */
+    async attach(key, into) {
+        const names = getSandboxNames(key.id);
+        const namespace = this.namespaces.name(key.owner);
+        const pod = await this.usePod(key);
+        pod.runs++;
+        const leave = () => {
+            pod.runs--;
+            this.leavePod(key, pod);
+        };
+        try {
+            // runs on a sandbox go one at a time: a proxy the pod of the files does not own was
+            // left by a run before (the server stopped, or its release could not remove it)
+            await (0, network_policy_1.removeSandboxProxies)(this.instance, namespace, { sandbox: names.label }, (deployment) => isOwnedBy(deployment, pod.uid)).catch(() => undefined);
+            await (0, promises_1.mkdir)(into, { recursive: true });
+            await this.copyOut(key, ['tar', 'cf', '-', '-C', `${CLAIM_PATH}/${key.id}/files`, '.'], into, 
+            // what the run plans with: files, directories and links, nothing outside of it
+            (_path, type) => ['File', 'OldFile', 'ContiguousFile', 'Directory', 'SymbolicLink'].includes(type), AbortSignal.timeout(FETCH_TIMEOUT_MS));
+        }
+        catch (e) {
+            leave();
+            throw e;
+        }
+        const watching = new AbortController();
+        void this.makeWayForRun(key, pod, watching.signal);
+        return {
+            directory: into,
+            claim: {
+                name: names.claim,
+                namespace,
+                subPath: FILES_DIRECTORY,
+                label: names.label,
+                ...(pod.uid ? { pod: { name: pod.name, uid: pod.uid } } : {}),
+            },
+            release: () => {
+                if (!watching.signal.aborted) {
+                    watching.abort();
+                    leave();
+                    if (pod.movedAside) {
+                        // the egress proxies it owned stayed for the run; those of a pod after it stay
+                        (0, network_policy_1.removeSandboxProxies)(this.instance, namespace, { sandbox: names.label }, (deployment) => !isOwnedBy(deployment, pod.uid) && (deployment.metadata?.ownerReferences ?? []).length > 0).catch(() => undefined);
+                    }
+                }
+            },
+        };
+    }
+    /**
+     * The pods of a run go to the node of the pod of the files. When one finds no room there
+     * (a small node, which the cluster can not grow), the pod of the files goes, so the run's
+     * pod goes where there is room and the claim follows it, as it would without the pod.
+     */
+    async makeWayForRun(key, pod, signal) {
+        const selector = `${volumes_1.SANDBOX_LABEL}=${getSandboxNames(key.id).label},!${FILES_LABEL}`;
+        while (!signal.aborted && this.pods.get(key.id) === pod) {
+            await (0, sleep_1.sleep)(UNSCHEDULABLE_POLL_MS);
+            if (signal.aborted || this.pods.get(key.id) !== pod) {
+                return;
+            }
+            try {
+                const pods = await this.instance.coreApi.listNamespacedPod({
+                    namespace: pod.namespace,
+                    labelSelector: selector,
+                });
+                const stuck = pods.items.some((runPod) => runPod.status?.conditions?.some((condition) => condition.type === 'PodScheduled' &&
+                    condition.status === 'False' &&
+                    condition.reason === 'Unschedulable' &&
+                    Date.now() - new Date(condition.lastTransitionTime ?? Date.now()).getTime() >= UNSCHEDULABLE_MS));
+                if (stuck && !signal.aborted && this.pods.get(key.id) === pod) {
+                    // requests under way are answered first; the next one starts a pod next to the run's
+                    this.pods.delete(key.id);
+                    clearTimeout(pod.idle);
+                    for (let i = 0; pod.busy > pod.runs && i < REQUEST_TIMEOUT_MS / 100; i++) {
+                        await (0, sleep_1.sleep)(100);
+                    }
+                    // what it owns stays for the run (the egress proxies of its tasks) and goes
+                    // when the run released it
+                    pod.movedAside = true;
+                    await this.deletePod(pod.namespace, pod.name, 'Orphan');
+                    return;
+                }
+            }
+            catch {
+                // the next look may get an answer
+            }
+        }
+    }
+    // What a tar in the pod writes, extracted into `into`: the entries `keep` takes, up to
+    // the bytes a sandbox may take.
+    async copyOut(key, command, into, keep, signal) {
+        let bytes = 0;
+        const extract = (0, tar_1.extract)({
+            cwd: into,
+            filter: (path, entry) => 'type' in entry && keep(path, entry.type),
+        });
+        const counted = new stream_1.PassThrough();
+        counted.on('data', (chunk) => {
+            bytes += chunk.length;
+            if (bytes > this.limits.bytes) {
+                counted.destroy(new Error(`the sandbox ${key.id} takes more than ${(0, units_1.formatSize)(this.limits.bytes)}`));
+            }
+        });
+        const extracted = new Promise((resolve, reject) => {
+            extract.on('finish', resolve);
+            extract.on('error', reject);
+            counted.on('error', reject);
+        });
+        extracted.catch(() => undefined);
+        counted.pipe(extract);
+        await this.withPod(key, (pod) => this.exec(pod, command, null, counted, signal));
+        counted.end();
+        await extracted;
+    }
+    async remove(key) {
+        await this.removeIn(this.namespaces.name(key.owner), key.id);
+    }
+    async removeIn(namespace, id) {
+        const names = getSandboxNames(id);
+        await this.stopPod({ id, owner: { iss: '', sub: '' } }, namespace);
+        // the cluster removes the proxies the pod owned; those it did not own go here
+        await (0, network_policy_1.removeSandboxProxies)(this.instance, namespace, { sandbox: names.label });
+        for (const name of [names.claim, names.git]) {
+            try {
+                await this.instance.coreApi.deleteNamespacedPersistentVolumeClaim({ name, namespace });
+            }
+            catch (e) {
+                if ((0, apply_1.statusCodeOf)(e) !== 404) {
+                    throw e;
+                }
+            }
+        }
+    }
+    // The claims of sandboxes on the cluster. Those of this server, in the namespace their
+    // owner has, are removed when they have no record; the others are kept, and their records
+    // too (a server that changed its prefix still has them).
+    async list() {
+        // the server has rights in the namespaces it made only (runRole, bound there): it looks
+        // for claims in those, and passes over one it can not read, another server's
+        const namespaces = await this.instance.coreApi.listNamespace({ labelSelector: 'hammerkit.dev/managed=true' });
+        const claims = [];
+        for (const namespace of namespaces.items) {
+            const name = namespace.metadata?.name;
+            if (!name || namespace.metadata?.deletionTimestamp) {
+                continue;
+            }
+            try {
+                const listed = await this.instance.coreApi.listNamespacedPersistentVolumeClaim({
+                    namespace: name,
+                    labelSelector: `hammerkit.dev/managed=true,${volumes_1.SANDBOX_LABEL}`,
+                });
+                claims.push(...listed.items);
+            }
+            catch (e) {
+                if ((0, apply_1.statusCodeOf)(e) !== 403) {
+                    throw e;
+                }
+            }
+        }
+        const found = new Map();
+        for (const claim of claims) {
+            const annotations = claim.metadata?.annotations ?? {};
+            const id = annotations[ID_ANNOTATION];
+            const namespace = claim.metadata?.namespace;
+            if (!id || !namespace || claim.metadata?.deletionTimestamp) {
+                continue;
+            }
+            const names = getSandboxNames(id);
+            if (claim.metadata?.name !== names.claim && claim.metadata?.name !== names.git) {
+                continue;
+            }
+            const owner = { iss: annotations[ISS_ANNOTATION] ?? '', sub: annotations[SUB_ANNOTATION] ?? '' };
+            const ours = claim.metadata?.labels?.[SERVER_LABEL] === this.server && this.namespaces.name(owner) === namespace;
+            const created = claim.metadata?.creationTimestamp ? new Date(claim.metadata.creationTimestamp) : new Date();
+            const seen = found.get(id);
+            found.set(id, {
+                id,
+                ...((seen ? seen.remove : true) && ours ? { remove: () => this.removeIn(namespace, id) } : {}),
+                createdAt: seen?.createdAt && seen.createdAt > created ? seen.createdAt : created,
+            });
+        }
+        return [...found.values()];
+    }
+    // The commit, fetched by the server and taken into the claim by the pod.
+    async receive(key, fetch, op) {
+        if (!(0, git_checkout_1.isValidRef)(fetch.ref)) {
+            throw new Error(`${JSON.stringify(fetch.ref.slice(0, 80))} is not a branch, tag or commit`);
+        }
+        const repository = (0, path_1.join)(this.staging, `${key.id}-${(0, crypto_1.randomBytes)(6).toString('hex')}.git`);
+        try {
+            const what = `sandbox ${key.id}`;
+            await (0, promises_1.mkdir)(this.staging, { recursive: true, mode: 0o700 });
+            await (0, git_checkout_1.git)(['init', '--quiet', '--bare', repository], fetch.env, fetch.signal, what, op);
+            await (0, git_checkout_1.git)([`--git-dir=${repository}`, 'fetch', '--quiet', '--depth', '1', '--', fetch.url, fetch.ref], fetch.env, fetch.signal, what);
+            await (0, git_checkout_1.git)([`--git-dir=${repository}`, 'update-ref', `refs/heads/${sandbox_pod_1.INCOMING_REF}`, 'FETCH_HEAD'], fetch.env, fetch.signal, what, op);
+            const incoming = `${CLAIM_PATH}/${sandbox_pod_1.INCOMING}`;
+            await this.withPod(key, (pod) => this.exec(pod, ['sh', '-c', 'rm -rf "$1" && mkdir -p "$1" && tar xf - -C "$1"', 'sh', incoming], stream_1.Readable.from((0, tar_2.packArchive)({ cwd: repository, portable: true }, ['.'])), null, fetch.signal));
+            return (await this.request(key, { op, id: key.id }, fetch.signal)).result;
+        }
+        finally {
+            await (0, promises_1.rm)(repository, { recursive: true, force: true });
+        }
+    }
+    async raw(key, path) {
+        const answer = await this.call(key, { op: 'raw', id: key.id, path }, AbortSignal.timeout(REQUEST_TIMEOUT_MS), true);
+        const result = answer.result;
+        return 'stream' in answer ? { stat: result.stat, stream: answer.stream } : { ...result, content: answer.body };
+    }
+    request(key, request, signal = AbortSignal.timeout(REQUEST_TIMEOUT_MS)) {
+        return this.exclusive(key.id, () => this.requestNow(key, request, signal));
+    }
+    async requestNow(key, request, signal) {
+        const answer = await this.call(key, request, signal, false);
+        return { result: answer.result };
+    }
+    exclusive(id, fn) {
+        const previous = this.queues.get(id) ?? Promise.resolve();
+        const next = previous.then(fn, fn);
+        const settled = next.catch(() => undefined);
+        this.queues.set(id, settled);
+        void settled.then(() => {
+            if (this.queues.get(id) === settled) {
+                this.queues.delete(id);
+            }
+        });
+        return next;
+    }
+    // One request to `hammerkit server-sandbox` in the pod: the first line of its answer
+    // says how it went, a raw read sends the bytes after it.
+    async call(key, request, signal, body) {
+        const encoded = request.op === 'edit' ? { ...request, ops: (0, sandbox_pod_1.encodeEditOps)(request.ops) } : request;
+        const reader = new AnswerReader(body);
+        // the process ends when the server stops waiting for it: closing the connection does
+        // not end a command in a pod
+        const seconds = Math.ceil((['create', 'fetch', 'move'].includes(request.op) ? FETCH_TIMEOUT_MS : REQUEST_TIMEOUT_MS) / 1000);
+        const done = this.withPod(key, (pod) => this.exec(pod, ['timeout', String(seconds), 'hammerkit', 'server-sandbox', CLAIM_PATH], stream_1.Readable.from([Buffer.from(JSON.stringify(encoded))]), reader, signal));
+        done.then(() => reader.complete(), (e) => reader.fail(e));
+        const answer = await reader.answer;
+        if (!answer.header.ok) {
+            throw (0, sandbox_pod_1.decodePodError)(answer.header.error);
+        }
+        return 'stream' in answer
+            ? { result: answer.header.result, stream: answer.stream }
+            : { ...answer, result: answer.header.result };
+    }
+    /**
+     * Runs `fn` with the pod of the sandbox's files, made when there is none. It stays for
+     * the requests that follow and goes `idle` after the last one.
+     */
+    async withPod(key, fn) {
+        const pod = await this.usePod(key);
+        try {
+            return await fn(pod);
+        }
+        finally {
+            this.leavePod(key, pod);
+        }
+    }
+    // The running pod of the sandbox's files, which stays until `leavePod`.
+    async usePod(key) {
+        let pod = this.pods.get(key.id);
+        if (pod?.started !== undefined && pod.busy === 0 && Date.now() - pod.started > POD_RENEW_MS) {
+            // nothing uses it and its deadline nears: a new one, before a run needs it for long
+            await this.stopPod(key);
+            pod = this.pods.get(key.id);
+        }
+        if (!pod) {
+            const namespace = this.namespaces.name(key.owner);
+            const created = {
+                namespace,
+                name: getSandboxNames(key.id).pod,
+                busy: 0,
+                runs: 0,
+                ready: Promise.resolve(),
+            };
+            created.ready = this.startPod(key, namespace).then(({ started, uid }) => {
+                created.started = started;
+                created.uid = uid;
+            });
+            created.ready.catch(() => {
+                if (this.pods.get(key.id) === created) {
+                    this.pods.delete(key.id);
+                }
+            });
+            this.pods.set(key.id, created);
+            pod = created;
+        }
+        clearTimeout(pod.idle);
+        pod.busy++;
+        try {
+            await pod.ready;
+        }
+        catch (e) {
+            this.leavePod(key, pod);
+            throw e;
+        }
+        return pod;
+    }
+    leavePod(key, pod) {
+        pod.busy--;
+        if (pod.busy === 0 && this.pods.get(key.id) === pod) {
+            pod.idle = setTimeout(() => {
+                if (pod.busy === 0 && this.pods.get(key.id) === pod) {
+                    this.stopPod(key).catch(() => undefined);
+                }
+            }, (0, units_1.parseDuration)(this.backend.sandboxes.idle));
+            pod.idle.unref();
+        }
+    }
+    // Takes the pod of the sandbox's files down, once the requests under way are answered.
+    async stopPod(key, namespace = this.namespaces.name(key.owner)) {
+        const pod = this.pods.get(key.id);
+        if (pod) {
+            this.pods.delete(key.id);
+            clearTimeout(pod.idle);
+            for (let i = 0; pod.busy > 0 && i < REQUEST_TIMEOUT_MS / 100; i++) {
+                await (0, sleep_1.sleep)(100);
+            }
+        }
+        await this.deletePod(namespace, getSandboxNames(key.id).pod);
+    }
+    async deletePod(namespace, name, propagationPolicy) {
+        try {
+            await this.instance.coreApi.deleteNamespacedPod({
+                name,
+                namespace,
+                gracePeriodSeconds: 0,
+                ...(propagationPolicy ? { propagationPolicy } : {}),
+            });
+        }
+        catch (e) {
+            if ((0, apply_1.statusCodeOf)(e) === 404) {
+                return;
+            }
+            throw e;
+        }
+        for (let attempt = 0; attempt < 120; attempt++) {
+            try {
+                await this.instance.coreApi.readNamespacedPod({ name, namespace });
+            }
+            catch (e) {
+                if ((0, apply_1.statusCodeOf)(e) === 404) {
+                    return;
+                }
+                throw e;
+            }
+            await (0, sleep_1.sleep)(500);
+        }
+    }
+    // Starts the pod of the sandbox's files and answers when it started, and its uid.
+    async startPod(key, namespace) {
+        const names = getSandboxNames(key.id);
+        await this.prepare(namespace);
+        // one left by a server that stopped is used while it runs
+        const existing = await this.instance.coreApi
+            .readNamespacedPod({ name: names.pod, namespace })
+            .catch((e) => ((0, apply_1.statusCodeOf)(e) === 404 ? undefined : Promise.reject(e)));
+        if (existing?.status?.phase === 'Running' && !existing.metadata?.deletionTimestamp) {
+            return { started: startedAt(existing), uid: existing.metadata?.uid };
+        }
+        if (existing) {
+            await this.deletePod(namespace, names.pod);
+        }
+        // where the pods of runs go: the first pool, which container tasks run on, so that a
+        // claim a zonal class provisions for this pod is where the tasks can mount it
+        const pool = this.backend.pools[0];
+        const resources = this.backend.sandboxes.resources;
+        const pod = {
+            apiVersion: 'v1',
+            kind: 'Pod',
+            metadata: {
+                name: names.pod,
+                namespace,
+                // the NetworkPolicy of the caller's namespace for pods with an open network leaves
+                // it out: its own policy, no network at all, is all it gets
+                labels: { ...MANAGED, [volumes_1.SANDBOX_LABEL]: names.label, [FILES_LABEL]: 'true', [network_policy_1.NETWORK_LABEL]: 'limited' },
+                annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
+            },
+            spec: {
+                automountServiceAccountToken: false,
+                enableServiceLinks: false,
+                restartPolicy: 'Never',
+                activeDeadlineSeconds: POD_DEADLINE_SECONDS,
+                terminationGracePeriodSeconds: 0,
+                nodeSelector: {
+                    'kubernetes.io/os': pool.platform.os,
+                    'kubernetes.io/arch': pool.platform.arch,
+                    ...pool.nodeSelector,
+                },
+                ...(pool.tolerations.length > 0 ? { tolerations: pool.tolerations } : {}),
+                affinity: (0, volumes_1.getSandboxAffinity)(names.label),
+                securityContext: { seccompProfile: { type: 'RuntimeDefault' } },
+                containers: [
+                    {
+                        name: 'files',
+                        image: this.backend.sandboxes.image,
+                        command: ['sleep', String(POD_DEADLINE_SECONDS)],
+                        workingDir: CLAIM_PATH,
+                        env: [{ name: 'HOME', value: '/tmp' }],
+                        ...(resources.requests || resources.limits ? { resources } : {}),
+                        // root, as tasks may run as root and write files only root may change, but
+                        // with no more than the capabilities to own and change any file of the claims
+                        securityContext: {
+                            allowPrivilegeEscalation: false,
+                            readOnlyRootFilesystem: true,
+                            capabilities: { drop: ['ALL'], add: ['CHOWN', 'DAC_OVERRIDE', 'FOWNER'] },
+                        },
+                        volumeMounts: [
+                            { name: 'git', mountPath: CLAIM_PATH },
+                            { name: 'files', mountPath: `${CLAIM_PATH}/${key.id}/files`, subPath: FILES_DIRECTORY },
+                            { name: 'tmp', mountPath: '/tmp' },
+                        ],
+                    },
+                ],
+                volumes: [
+                    { name: 'git', persistentVolumeClaim: { claimName: names.git } },
+                    { name: 'files', persistentVolumeClaim: { claimName: names.claim } },
+                    { name: 'tmp', emptyDir: {} },
+                ],
+            },
+        };
+        await this.instance.objectApi.create(pod);
+        const deadline = Date.now() + POD_START_MS;
+        for (;;) {
+            const current = await this.instance.coreApi.readNamespacedPod({ name: names.pod, namespace });
+            const phase = current.status?.phase;
+            if (phase === 'Running') {
+                return { started: startedAt(current), uid: current.metadata?.uid };
+            }
+            const waiting = current.status?.containerStatuses?.[0]?.state?.waiting;
+            if (phase === 'Failed' || phase === 'Succeeded' || Date.now() > deadline) {
+                await this.deletePod(namespace, names.pod).catch(() => undefined);
+                throw new Error(`the pod of the sandbox ${key.id} did not start${waiting?.reason ? `: ${waiting.reason}${waiting.message ? ` ${waiting.message}` : ''}` : ` (${phase})`}`);
+            }
+            await (0, sleep_1.sleep)(500);
+        }
+    }
+    // What the caller's namespace has for sandboxes: no network for the pods of their files,
+    // and the quota on the storage class of sandboxes when they have one of their own.
+    prepare(namespace) {
+        let prepared = this.prepared.get(namespace);
+        if (!prepared) {
+            prepared = this.applyPolicy(namespace);
+            prepared.catch(() => this.prepared.delete(namespace));
+            this.prepared.set(namespace, prepared);
+        }
+        return prepared;
+    }
+    async applyPolicy(namespace) {
+        const metadata = {
+            namespace,
+            labels: MANAGED,
+            annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
+        };
+        const policy = {
+            apiVersion: 'networking.k8s.io/v1',
+            kind: 'NetworkPolicy',
+            metadata: { ...metadata, name: FILES_POLICY },
+            spec: { podSelector: { matchLabels: { [FILES_LABEL]: 'true' } }, policyTypes: ['Ingress', 'Egress'] },
+        };
+        await (0, apply_1.replace)(this.instance, policy);
+        const storageClass = this.backend.sandboxes.storageClass;
+        if (!storageClass) {
+            // one an earlier config had
+            await this.instance.objectApi
+                .delete({ apiVersion: 'v1', kind: 'ResourceQuota', metadata: { ...metadata, name: QUOTA_NAME } })
+                .catch((e) => ((0, apply_1.statusCodeOf)(e) === 404 ? undefined : Promise.reject(e)));
+        }
+        else {
+            const scope = `${storageClass}.storageclass.storage.k8s.io`;
+            const quota = {
+                apiVersion: 'v1',
+                kind: 'ResourceQuota',
+                metadata: { ...metadata, name: QUOTA_NAME },
+                spec: {
+                    hard: {
+                        // two claims of a sandbox, both of `size`
+                        [`${scope}/requests.storage`]: String(2 * this.limits.bytes),
+                        [`${scope}/persistentvolumeclaims`]: String(2 * this.limits.count),
+                    },
+                },
+            };
+            await (0, apply_1.replace)(this.instance, quota);
+        }
+    }
+    // A command in the pod. It fails when the command does, with the last of its error output.
+    async exec(pod, command, stdin, stdout, signal) {
+        let stderr = '';
+        const errors = new stream_1.Writable({
+            write(chunk, _encoding, callback) {
+                stderr = (stderr + chunk.toString('utf8')).slice(-MAX_STDERR);
+                callback();
+            },
+        });
+        let status;
+        // the stdin of the command waits on the connection, which is there once it started
+        const connection = {};
+        const socket = await this.instance.exec.exec(pod.namespace, pod.name, 'files', command, stdout ?? new stream_1.Writable({ write: (_chunk, _encoding, callback) => callback() }), errors, stdin && throttle(stdin, () => connection.socket), false, (received) => {
+            status = received;
+        });
+        connection.socket = socket;
+        if (stdout) {
+            holdBack(socket, stdout);
+        }
+        await new Promise((resolve, reject) => {
+            const abort = () => {
+                socket.close();
+                reject(new Error(`${command[0]} in the pod of the sandbox was cancelled`));
+            };
+            if (signal.aborted) {
+                abort();
+                return;
+            }
+            signal.addEventListener('abort', abort, { once: true });
+            socket.on('error', (e) => {
+                signal.removeEventListener('abort', abort);
+                reject(e);
+            });
+            socket.on('close', () => {
+                signal.removeEventListener('abort', abort);
+                if (status?.status === 'Success') {
+                    resolve();
+                    return;
+                }
+                const reason = stderr.trim().split('\n').pop() || status?.message || 'the connection to the pod closed';
+                reject(new Error(`${command[0]} in the pod of the sandbox failed: ${reason}`));
+            });
+        });
+    }
+}
+exports.KubernetesSandboxStorage = KubernetesSandboxStorage;
+// Why the remote refused the push of `branch`, from what `git push --porcelain` printed
+// (`<flag>\t<from>:<to>\t<summary>`, `!` for a refused ref); none when it took it.
+function getPushRefusal(printed, branch) {
+    const line = printed
+        .split('\n')
+        .map((printedLine) => printedLine.split('\t'))
+        .find((fields) => fields.length >= 3 && fields[1].endsWith(`:refs/heads/${branch}`));
+    if (line && line[0] !== '!') {
+        return undefined;
+    }
+    const reason = line ? line.slice(2).join(' ') : printed.trim() || 'git printed nothing';
+    return `the remote refused the push to ${branch}: ${reason}`;
+}
+exports.getPushRefusal = getPushRefusal;
+function isOwnedBy(object, uid) {
+    return !!uid && (object.metadata?.ownerReferences ?? []).some((reference) => reference.uid === uid);
+}
+// When the pod started, by the cluster's clock as far as it says, else now.
+function startedAt(pod) {
+    const started = pod.status?.startTime ? new Date(pod.status.startTime).getTime() : NaN;
+    return Number.isNaN(started) ? Date.now() : Math.min(started, Date.now());
+}
+function defined(env) {
+    const result = {};
+    for (const [key, value] of Object.entries(env)) {
+        if (value !== undefined) {
+            result[key] = value;
+        }
+    }
+    return result;
+}
+// The client of the cluster sends what stdin has as fast as it comes: this waits while the
+// connection has more than MAX_STDIN_BUFFERED not sent yet.
+function throttle(stdin, connection) {
+    return stream_1.Readable.from((async function* () {
+        for await (const chunk of stdin) {
+            while ((connection()?.bufferedAmount ?? 0) > MAX_STDIN_BUFFERED) {
+                await (0, sleep_1.sleep)(10);
+            }
+            yield chunk;
+        }
+    })());
+}
+// The client of the cluster writes what the pod sends to stdout without waiting for it to
+// be taken: this stops reading from the connection while stdout is full, so a slow reader
+// of a file or of the files of a run holds the pod back instead of the server's memory.
+function holdBack(socket, stdout) {
+    const tcp = socket._socket;
+    if (!tcp) {
+        return;
+    }
+    let paused = false;
+    socket.on('message', () => {
+        if (paused || !stdout.writableNeedDrain) {
+            return;
+        }
+        paused = true;
+        tcp.pause();
+        const resume = () => {
+            paused = false;
+            stdout.off('close', resume);
+            stdout.off('drain', resume);
+            tcp.resume();
+        };
+        stdout.once('drain', resume);
+        stdout.once('close', resume);
+    });
+}
+/**
+ * The answer of the pod: its first line, then what follows. Whatever follows is kept in
+ * memory, unless the first line says it is a stream: then it is passed on as it comes.
+ */
+class AnswerReader extends stream_1.Writable {
+    constructor(body) {
+        super();
+        this.body = body;
+        this.chunks = [];
+        this.size = 0;
+        this.answer = new Promise((resolve, reject) => {
+            this.resolveAnswer = resolve;
+            this.rejectAnswer = reject;
+        });
+        this.answer.catch(() => undefined);
+    }
+    _write(chunk, _encoding, callback) {
+        if (this.stream) {
+            this.stream.write(chunk, callback);
+            return;
+        }
+        this.size += chunk.length;
+        if (this.size > MAX_ANSWER_BYTES) {
+            callback(new Error('the answer of the pod of the sandbox is too large'));
+            return;
+        }
+        this.chunks.push(chunk);
+        if (!this.header) {
+            // what came before has no newline: only this chunk is searched
+            const found = chunk.indexOf(10);
+            if (found < 0) {
+                callback();
+                return;
+            }
+            const all = Buffer.concat(this.chunks);
+            const newline = this.size - chunk.length + found;
+            try {
+                this.header = JSON.parse(all.subarray(0, newline).toString('utf8'));
+            }
+            catch {
+                callback(new Error('the pod of the sandbox did not answer'));
+                return;
+            }
+            this.chunks = [all.subarray(newline + 1)];
+            if (this.header.ok && this.header.stream && this.body) {
+                this.stream = new stream_1.PassThrough();
+                this.resolveAnswer({ header: this.header, stream: this.stream });
+                this.stream.write(this.chunks[0], callback);
+                this.chunks = [];
+                return;
+            }
+        }
+        callback();
+    }
+    // the command ended well: everything it wrote is here once the stream finished
+    async complete() {
+        this.end();
+        try {
+            await (0, promises_2.finished)(this);
+        }
+        catch (e) {
+            this.fail(e);
+            return;
+        }
+        if (this.stream) {
+            this.stream.end();
+        }
+        else if (this.header) {
+            this.resolveAnswer({ header: this.header, body: Buffer.concat(this.chunks) });
+        }
+        else {
+            this.rejectAnswer(new Error('the pod of the sandbox did not answer'));
+        }
+    }
+    fail(e) {
+        const error = e instanceof Error ? e : new Error(String(e));
+        if (this.stream) {
+            this.stream.destroy(error);
+        }
+        this.rejectAnswer(error);
+    }
+}
+// The files of a sandbox on a cluster: each call is a request to the pod of its files.
+class KubernetesSandboxFiles {
+    constructor(request, rawFile) {
+        this.request = request;
+        this.rawFile = rawFile;
+    }
+    async ask(request) {
+        return (await this.request(request)).result;
+    }
+    async stat(path) {
+        return (await this.ask({ op: 'stat', path })) ?? undefined;
+    }
+    tree(query) {
+        return this.ask({ op: 'tree', query });
+    }
+    read(path, query) {
+        return this.ask({ op: 'read', path, query });
+    }
+    raw(path) {
+        return this.rawFile(path);
+    }
+    search(query) {
+        return this.ask({ op: 'search', query });
+    }
+    changes() {
+        return this.ask({ op: 'changes' });
+    }
+    diff(format, author) {
+        return this.ask({ op: 'diff', format, author });
+    }
+    async commit(author) {
+        return (await this.ask({ op: 'commit', author })) ?? undefined;
+    }
+    // Planned in the pod and planned again when written: the sandbox does not change in
+    // between, the registry holds it.
+    async plan(ops) {
+        const planned = await this.ask({ op: 'edit', ops, commit: false });
+        return {
+            ...planned,
+            commit: async () => {
+                await this.ask({ op: 'edit', ops, commit: true });
+            },
+        };
+    }
+}
+//# sourceMappingURL=kubernetes-sandbox-storage.js.map