hammerkit 1.27.0-next.8 → 2.0.0-next.1

package/dist/server/kubernetes-backend.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.awaitRunObjectsGone = exports.removeRunObjects = exports.CallerNamespaces = exports.getVolumeScope = exports.KubernetesExecutor = exports.getRunRoleBinding = exports.getNamespacePolicies = exports.getClusterAccess = exports.isCallerNamespace = exports.getCallerNamespace = void 0;
+exports.awaitRunObjectsGone = exports.removeRunObjects = exports.CallerNamespaces = exports.getVolumeScope = exports.getPrivilegedNamespaceProblem = exports.getCapabilitiesProblem = exports.getPodSecurityLabels = exports.KubernetesExecutor = exports.getRunRoleBinding = exports.removeServiceRunNamespace = exports.getServerIngressPolicy = exports.getNamespacePolicies = exports.getClusterAccess = exports.isCallerNamespace = exports.SERVICE_RUN_LABEL = exports.getServiceRunNamespace = exports.getCallerNamespace = void 0;
 const crypto_1 = require("crypto");
 const fs_1 = require("fs");
 const os_1 = require("os");
@@ -8,6 +8,7 @@ const path_1 = require("path");
 const net_1 = require("net");
 const apply_1 = require("../kubernetes/apply");
 const job_placement_1 = require("../kubernetes/job-placement");
+const capacity_1 = require("./capacity");
 const kubernetes_instance_1 = require("../kubernetes/kubernetes-instance");
 const log_1 = require("../log");
 const network_policy_1 = require("../kubernetes/network-policy");
@@ -22,6 +23,8 @@ const POLICY_NAME = 'hammerkit';
 const MANAGED_LABEL = 'hammerkit.dev/managed';
 // how long the end of a run waits for its pods and claims to be gone
 const RELEASE_TIMEOUT = 120000;
+// how long the setup of a caller's namespace is taken as it is, before it is applied again
+const NAMESPACE_SETUP_REUSE_MS = 5 * 60000;
 // where a pod of the cluster finds the token and certificate of its ServiceAccount
 const IN_CLUSTER_NAMESPACE = '/var/run/secrets/kubernetes.io/serviceaccount/namespace';
 /**
@@ -32,6 +35,20 @@ function getCallerNamespace(prefix, owner) {
     return `${prefix}${(0, crypto_1.createHash)('sha256').update(`${owner.iss}\0${owner.sub}`).digest('hex').slice(0, 16)}`;
 }
 exports.getCallerNamespace = getCallerNamespace;
+/**
+ * The namespace of a service run: one of its own, as the services stay up for hours while
+ * the caller's other runs go on in theirs (one at a time, each removing what the one
+ * before left).
+ */
+function getServiceRunNamespace(prefix, run, wakes = 0) {
+    // a woken run starts in a namespace of its own, the one it slept in may still be removed;
+    // a run whose volumes are kept while it sleeps has one namespace (wakes 0) from start to end
+    const name = wakes > 0 ? `up\0${run}\0${wakes}` : `up\0${run}`;
+    return `${prefix}${(0, crypto_1.createHash)('sha256').update(name).digest('hex').slice(0, 16)}`;
+}
+exports.getServiceRunNamespace = getServiceRunNamespace;
+// the label of the namespace of a service run: the run label value of its run
+exports.SERVICE_RUN_LABEL = 'hammerkit.dev/service-run';
 // Whether a namespace is one getCallerNamespace makes with this prefix: a server named `hk`
 // passes over the namespaces of a server named `hk-test` on the same cluster.
 function isCallerNamespace(prefix, name) {
@@ -143,6 +160,64 @@ function getNamespacePolicies(namespace, deniedEgress, allowedEgress = []) {
     ];
 }
 exports.getNamespacePolicies = getNamespacePolicies;
+/**
+ * What lets the server into the namespace of a service run, to the ports of its endpoints:
+ * policies add up, so this one opens the pods to the server's namespace besides what the
+ * others let in, limited ones too.
+ */
+// The server reaches the ports of the endpoints, from its own namespace, and no other port:
+// what else the run starts (a database) stays for the pods of the run.
+function getServerIngressPolicy(namespace, server, ports) {
+    return {
+        apiVersion: 'networking.k8s.io/v1',
+        kind: 'NetworkPolicy',
+        metadata: {
+            name: 'hammerkit-server-ingress',
+            namespace,
+            annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
+            labels: { [MANAGED_LABEL]: 'true' },
+        },
+        spec: {
+            podSelector: {},
+            policyTypes: ['Ingress'],
+            ingress: ports.length > 0
+                ? [
+                    {
+                        _from: [{ namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': server } } }],
+                        ports: [...new Set(ports)].map((port) => ({ protocol: 'TCP', port })),
+                    },
+                ]
+                : [],
+        },
+    };
+}
+exports.getServerIngressPolicy = getServerIngressPolicy;
+// The namespace of a service run goes with all it holds; waits up to `timeout` for it.
+async function removeServiceRunNamespace(instance, namespace, timeout) {
+    try {
+        await instance.coreApi.deleteNamespace({ name: namespace, propagationPolicy: 'Background' });
+    }
+    catch (e) {
+        if ((0, apply_1.statusCodeOf)(e) === 404) {
+            return;
+        }
+        throw e;
+    }
+    const deadline = Date.now() + timeout;
+    while (Date.now() < deadline) {
+        try {
+            await instance.coreApi.readNamespace({ name: namespace });
+        }
+        catch (e) {
+            if ((0, apply_1.statusCodeOf)(e) === 404) {
+                return;
+            }
+            throw e;
+        }
+        await (0, sleep_1.sleep)(1000);
+    }
+}
+exports.removeServiceRunNamespace = removeServiceRunNamespace;
 /**
  * The server's rights in a caller's namespace: the operator's ClusterRole, bound there
  * only, so the server needs no rights on Secrets or pods anywhere else.
@@ -167,22 +242,33 @@ exports.getRunRoleBinding = getRunRoleBinding;
  * process of the server as on a host (see HostExecutor); what changes is where its
  * container tasks and services go: into the caller's namespace, on the nodes of the
  * operator's pools, with the secrets of providers read in the Job itself. The runs
- * of one caller execute one after the other, since the objects of a task are named
- * by its cache id in the caller's namespace; different callers run side by side.
- * When a run ends, everything made for it is removed, but for the claims of its tasks
- * the server keeps (`backend.keepVolumes`), which go once they expired.
+ * of a caller execute side by side in their namespace, up to `runs.limits.running`:
+ * the objects of a run are named and found by its run label (getItemKey), so two runs
+ * of the same task have their own. What two runs share are the claims the server keeps
+ * (`backend.keepVolumes`) of a scope (getVolumeScope): the runs of one scope go one at
+ * a time (exclusive). When a run ends, everything made for it is removed, but for the
+ * kept claims, which go once they expired.
  */
 class KubernetesExecutor {
     constructor(config, processEnvs, options) {
         this.config = config;
-        // the registry keeps a caller's next run in its queue, not running and timed here
-        this.runningPerCaller = 1;
-        // the run that executes, or the last one waiting, per caller namespace
+        // per caller namespace, what changes objects of the whole namespace: its setup, the
+        // removal of what a stopped server left, the sweeps of kept claims. The last one waiting.
         this.queues = new Map();
+        // per caller namespace, the runs that execute in it: by run label, their volume scope
+        this.active = new Map();
         // per caller namespace, when its kept claims are next looked at
         this.sweeps = new Map();
         // whether the kept claims a server left when it stopped were looked for
         this.swept = false;
+        // the namespaces where no run of this process left anything: what a run makes goes at its
+        // end, unless its release failed
+        this.tidy = new Set();
+        // the namespaces of the service runs that execute, with how many executions each (a run
+        // that moves executes beside itself), and whether those of earlier ones went
+        this.serviceRuns = new Map();
+        this.serviceRunsSwept = false;
+        this.asleep = new Set();
         this.access = getClusterAccess(config.backend, processEnvs);
         this.instance = (0, kubernetes_instance_1.createKubernetesInstances)({
             type: 'kubernetes',
@@ -200,20 +286,143 @@ class KubernetesExecutor {
         this.namespace = namespace;
         this.namespaces = new CallerNamespaces(this.instance, config.backend);
         this.keep = config.backend.keepVolumes === false ? 0 : (0, units_1.parseDuration)(config.backend.keepVolumes);
+        this.keepServiceRuns = config.runs.up.keepVolumes;
         this.log = options.log;
         this.host = new executor_1.HostExecutor({ ...options, placement: this.placement() });
     }
-    execute(run, context) {
-        const namespace = getCallerNamespace(this.config.backend.namespacePrefix, run.owner);
-        return this.enqueue(namespace, () => {
-            if (context.signal.aborted) {
-                throw new Error('the run was cancelled');
+    // what the namespaces of the server's callers and service runs request and use
+    capacity() {
+        const prefix = this.config.backend.namespacePrefix;
+        return new capacity_1.KubernetesCapacity(this.instance, {
+            quotaName: POLICY_NAME,
+            namespaces: async () => (await this.instance.coreApi.listNamespace({ labelSelector: `${MANAGED_LABEL}=true` })).items
+                .filter((namespace) => isCallerNamespace(prefix, namespace.metadata?.name ?? ''))
+                .map((namespace) => ({
+                name: namespace.metadata?.name,
+                kind: namespace.metadata?.labels?.[exports.SERVICE_RUN_LABEL] !== undefined ? 'service-run' : 'caller',
+            })),
+            ownerOf: (namespace, runs) => {
+                for (const run of runs) {
+                    if (run.up && getServiceRunNamespace(prefix, run.id, run.wakes) === namespace) {
+                        return { owner: { iss: run.owner.iss, sub: run.owner.sub }, run: run.id };
+                    }
+                    if (getCallerNamespace(prefix, run.owner) === namespace) {
+                        return { owner: { iss: run.owner.iss, sub: run.owner.sub } };
+                    }
+                }
+                return {};
+            },
+        });
+    }
+    async execute(run, context) {
+        if (run.up) {
+            if (run.source.type === 'sandbox') {
+                throw new Error('on a cluster a service run starts from a branch, push the sandbox and start it from there');
             }
+            // in a namespace of its own, apart from the caller's runs and their claims
             return this.host.execute(run, context);
-        });
+        }
+        const namespace = getCallerNamespace(this.config.backend.namespacePrefix, run.owner);
+        const label = (0, job_placement_1.getRunLabelValue)(run.id);
+        let runs = this.active.get(namespace);
+        if (!runs) {
+            runs = new Map();
+            this.active.set(namespace, runs);
+        }
+        runs.set(label, getVolumeScope(run));
+        try {
+            await this.host.execute(run, context);
+        }
+        finally {
+            runs.delete(label);
+            if (runs.size === 0 && this.active.get(namespace) === runs) {
+                this.active.delete(namespace);
+            }
+        }
+    }
+    // A service run moves to another commit in its namespace: unchanged services stay, the
+    // others roll to their new pods (see kubernetesServiceRuntime)
+    movesBeside(run) {
+        return !!run.up;
+    }
+    namespaceOf(run) {
+        const prefix = this.config.backend.namespacePrefix;
+        return run.up
+            ? getServiceRunNamespace(prefix, run.id, this.keepServiceRuns ? 0 : run.wakes)
+            : getCallerNamespace(prefix, run.owner);
+    }
+    /**
+     * A service run that slept with its volumes ended without waking: its namespace goes, with
+     * the claims in it.
+     */
+    async forget(run) {
+        if (!run.up) {
+            return;
+        }
+        // a namespace is kept only for a run that has one from start to end
+        const namespace = getServiceRunNamespace(this.config.backend.namespacePrefix, run.id);
+        this.asleep.delete(namespace);
+        await this.enqueue(namespace, () => this.removeServiceRun(namespace, (text) => this.log?.(`run=${run.id} ${text}`)));
+    }
+    /**
+     * The service runs of a previous start that sleep with their volumes: their namespaces stay,
+     * without what runs in them (a server that stopped without putting the run to sleep left
+     * it). The namespaces of any other service run go.
+     */
+    restored(runs) {
+        if (this.keepServiceRuns) {
+            for (const run of runs.filter((candidate) => candidate.up)) {
+                const namespace = getServiceRunNamespace(this.config.backend.namespacePrefix, run.id);
+                this.asleep.add(namespace);
+                this.enqueue(namespace, () => removeRunObjects(this.instance, namespace, job_placement_1.RUN_LABEL, RELEASE_TIMEOUT, { claims: false })).catch((e) => this.log?.(`what runs in the namespace ${namespace} of run ${run.id} could not be removed: ${(0, log_1.getErrorMessage)(e)}`));
+            }
+        }
+        this.sweepServiceRuns();
+    }
+    // The end of a service run: its namespace goes with all it holds. A server that may not
+    // delete it removes what runs in it, and looks again at the next service run.
+    async removeServiceRun(namespace, say) {
+        try {
+            await removeServiceRunNamespace(this.instance, namespace, RELEASE_TIMEOUT);
+        }
+        catch (e) {
+            // at least nothing of it runs on: its objects are the server's to remove (runRole)
+            const forbidden = (0, apply_1.statusCodeOf)(e) === 403;
+            const text = forbidden
+                ? `the server may not delete the namespace ${namespace}, its ClusterRole needs delete on namespaces (see the backend docs); removing what runs in it`
+                : `the namespace ${namespace} could not be removed: ${(0, log_1.getErrorMessage)(e)}`;
+            this.log?.(text);
+            say(text);
+            try {
+                await removeRunObjects(this.instance, namespace, job_placement_1.RUN_LABEL, RELEASE_TIMEOUT);
+            }
+            catch (inner) {
+                say(`what the run made in ${namespace} could not be removed: ${(0, log_1.getErrorMessage)(inner)}`);
+            }
+            // the next look removes the namespace
+            this.serviceRunsSwept = false;
+        }
     }
-    // What happens in a caller's namespace happens one at a time: their runs, and the
-    // removal of the claims their runs kept
+    // The runs of a scope share its kept claims, which one of them may change at a time:
+    // what a task finds in one is what the last run left there. Without kept claims, runs
+    // share no object and nothing is held.
+    exclusive(run) {
+        if (this.keep === 0 || run.up) {
+            return undefined;
+        }
+        return `${getCallerNamespace(this.config.backend.namespacePrefix, run.owner)}/${getVolumeScope(run)}`;
+    }
+    // The scopes of the runs that execute in the namespace, but `except`'s: their kept claims
+    // are theirs to mount and stay. A server that keeps none mounts none.
+    scopesInUse(namespace, except) {
+        if (this.keep === 0) {
+            return new Set();
+        }
+        return new Set([...(this.active.get(namespace) ?? new Map()).entries()]
+            .filter(([label]) => label !== except)
+            .map(([, scope]) => scope));
+    }
+    // What changes objects of a whole namespace happens one at a time, not what its runs do
     enqueue(namespace, fn) {
         const previous = this.queues.get(namespace) ?? Promise.resolve();
         const execution = previous.then(fn);
@@ -227,54 +436,107 @@ class KubernetesExecutor {
         return execution;
     }
     placement() {
-        const backend = this.config.backend;
         return {
             prepare: async (run, context, work) => {
-                const namespace = getCallerNamespace(backend.namespacePrefix, run.owner);
-                await this.namespaces.prepare(namespace);
-                // what an earlier run left when the server stopped before its end; the runs
-                // of a caller execute one at a time, so no other run is in the namespace
-                await removeRunObjects(this.instance, namespace, `${job_placement_1.RUN_LABEL},${job_placement_1.RUN_LABEL}!=${(0, job_placement_1.getRunLabelValue)(run.id)}`, RELEASE_TIMEOUT);
-                // the claims earlier runs kept that expired; the others this run may mount
-                const { next } = await (0, kept_claims_1.sweepKeptClaims)(this.instance, namespace, {
-                    now: new Date(),
-                    keep: this.keep > 0,
-                    timeout: RELEASE_TIMEOUT,
-                });
-                if (next) {
-                    this.sweepLater(namespace, Math.max(0, next.getTime() - Date.now()));
+                const namespace = this.namespaceOf(run);
+                if (run.up) {
+                    if (context.signal.aborted) {
+                        throw new Error('the run was cancelled');
+                    }
+                    this.serviceRuns.set(namespace, (this.serviceRuns.get(namespace) ?? 0) + 1);
+                    this.asleep.delete(namespace);
+                    this.sweepServiceRuns();
+                    const up = run.up;
+                    // after what a stop of its sleep left in it went (see restored)
+                    await this.enqueue(namespace, async () => {
+                        if (context.signal.aborted) {
+                            throw new Error('the run was cancelled');
+                        }
+                        // reached by the server at the Services of its endpoints
+                        await this.namespaces.prepare(namespace, {
+                            serviceRun: (0, job_placement_1.getRunLabelValue)(run.id),
+                            server: this.namespace,
+                            ports: (up.endpoints ?? []).map((endpoint) => endpoint.port),
+                            quota: this.config.runs.up.quota,
+                        });
+                    });
+                    context.emit({ type: 'log', task: 'hammerkit', stream: 'stdout', text: `runs in the namespace ${namespace}` });
+                    return { kubernetes: this.environmentOf(run, namespace), env: this.access.env };
                 }
+                const label = (0, job_placement_1.getRunLabelValue)(run.id);
+                let capabilitiesProblem;
+                await this.enqueue(namespace, async () => {
+                    if (context.signal.aborted) {
+                        throw new Error('the run was cancelled');
+                    }
+                    capabilitiesProblem = await this.namespaces.prepare(namespace);
+                    // what an earlier run left when the server stopped before its end, or when its
+                    // release failed: the objects of a run that does not execute. One that starts
+                    // after this waits for it here before it makes any.
+                    if (!this.tidy.has(namespace)) {
+                        const running = [...new Set([label, ...(this.active.get(namespace)?.keys() ?? [])])];
+                        await removeRunObjects(this.instance, namespace, `${job_placement_1.RUN_LABEL},${job_placement_1.RUN_LABEL} notin (${running.join(',')})`, RELEASE_TIMEOUT);
+                        this.tidy.add(namespace);
+                    }
+                    // the claims earlier runs kept that expired; the others this run may mount
+                    const { next } = await (0, kept_claims_1.sweepKeptClaims)(this.instance, namespace, {
+                        now: new Date(),
+                        keep: this.keep > 0,
+                        timeout: RELEASE_TIMEOUT,
+                        inUse: this.scopesInUse(namespace, label),
+                    });
+                    if (next) {
+                        this.sweepLater(namespace, Math.max(0, next.getTime() - Date.now()));
+                    }
+                });
                 this.sweepLeftClaims();
                 context.emit({ type: 'log', task: 'hammerkit', stream: 'stdout', text: `runs in the namespace ${namespace}` });
-                const kubernetes = {
-                    type: 'kubernetes',
-                    namespace,
-                    kubeConfig: this.access.kubeConfig,
-                    context: this.access.context,
-                    httpRoutes: [],
-                    storageClass: backend.storageClass,
-                    volumeSize: backend.volumeSize,
-                    maxVolumeSize: (0, server_config_1.getMaxVolumeSize)(backend),
-                    networkPolicyEnforced: backend.networkPolicy === 'enforced',
-                    server: {
-                        run: run.id,
-                        namespace: this.namespace,
-                        pools: backend.pools,
-                        providers: backend.providers,
-                        accounts: Object.fromEntries(Object.entries(this.config.accounts).flatMap(([name, account]) => account.kubernetes ? [[name, account.kubernetes]] : [])),
-                        ...(work?.claim ? { workspace: getWorkspace(work.root, work.claim, namespace) } : {}),
-                        ...(this.keep > 0 ? { volumeScope: getVolumeScope(run) } : {}),
-                    },
+                return {
+                    kubernetes: this.environmentOf(run, namespace, work, capabilitiesProblem),
+                    env: this.access.env,
                 };
-                return { kubernetes, env: this.access.env };
             },
             release: async (run, context) => {
-                const namespace = getCallerNamespace(backend.namespacePrefix, run.owner);
+                const namespace = this.namespaceOf(run);
+                if (run.up) {
+                    const say = (text) => context.emit({ type: 'log', task: 'hammerkit', stream: 'stderr', text });
+                    try {
+                        if (context.keep?.() === 'everything') {
+                            // the services are another execution's of the run now: the one it moved to, or
+                            // the one a move that did not get there leaves them to
+                            return;
+                        }
+                        if (context.keep?.() === 'volumes' && this.keepServiceRuns) {
+                            // it sleeps: what runs goes, the namespace stays with its claims and policies
+                            this.asleep.add(namespace);
+                            await this.enqueue(namespace, () => removeRunObjects(this.instance, namespace, job_placement_1.RUN_LABEL, RELEASE_TIMEOUT, { claims: false })).catch((e) => {
+                                // the wake applies its services again over what is left, the end removes it
+                                const text = `what runs in the namespace ${namespace} could not be stopped: ${(0, log_1.getErrorMessage)(e)}`;
+                                this.log?.(text);
+                                say(text);
+                            });
+                            return;
+                        }
+                        await this.enqueue(namespace, () => this.removeServiceRun(namespace, say));
+                    }
+                    finally {
+                        const executions = (this.serviceRuns.get(namespace) ?? 1) - 1;
+                        if (executions > 0) {
+                            this.serviceRuns.set(namespace, executions);
+                        }
+                        else {
+                            this.serviceRuns.delete(namespace);
+                        }
+                    }
+                    return;
+                }
                 try {
                     const selector = `${job_placement_1.RUN_LABEL}=${(0, job_placement_1.getRunLabelValue)(run.id)}`;
                     await removeRunObjects(this.instance, namespace, selector, RELEASE_TIMEOUT);
                 }
                 catch (e) {
+                    // the next run removes what is left
+                    this.tidy.delete(namespace);
                     context.emit({
                         type: 'log',
                         task: 'hammerkit',
@@ -300,8 +562,69 @@ class KubernetesExecutor {
             },
         };
     }
-    // The kept claims of a namespace that expired go once no run of the caller is in it: at
-    // the soonest expiry the server knows of
+    // The cluster as the process of a run sees it: the namespace its tasks and services go in.
+    environmentOf(run, namespace, work, capabilitiesProblem) {
+        const backend = this.config.backend;
+        return {
+            type: 'kubernetes',
+            namespace,
+            kubeConfig: this.access.kubeConfig,
+            context: this.access.context,
+            httpRoutes: [],
+            storageClass: backend.storageClass,
+            volumeSize: backend.volumeSize,
+            maxVolumeSize: (0, server_config_1.getMaxVolumeSize)(backend),
+            networkPolicyEnforced: backend.networkPolicy === 'enforced',
+            server: {
+                run: run.id,
+                namespace: this.namespace,
+                pools: backend.pools,
+                providers: backend.providers,
+                accounts: Object.fromEntries(Object.entries(this.config.accounts).flatMap(([name, account]) => account.kubernetes ? [[name, account.kubernetes]] : [])),
+                ...(work?.claim ? { workspace: getWorkspace(work.root, work.claim, namespace, backend.sandboxes.runner) } : {}),
+                // the claims of a service run go with its namespace
+                ...(this.keep > 0 && !run.up ? { volumeScope: getVolumeScope(run) } : {}),
+                ...(run.up ? { serviceRun: true } : {}),
+                ...(backend.capabilities
+                    ? {
+                        capabilities: {
+                            runtimeClass: backend.capabilities.runtimeClass,
+                            ...(capabilitiesProblem ? { unavailable: capabilitiesProblem } : {}),
+                        },
+                    }
+                    : {}),
+            },
+        };
+    }
+    // Once, on the first service run: the namespaces of service runs the server left when it
+    // stopped, whose services went with it
+    // Once when the server starts, and on the next service run after a namespace could not be
+    // removed: the namespaces of service runs that none of the server's runs is in, whose
+    // services went with the process of a server that stopped
+    sweepServiceRuns() {
+        if (this.serviceRunsSwept) {
+            return;
+        }
+        this.serviceRunsSwept = true;
+        Promise.resolve()
+            .then(() => this.instance.coreApi.listNamespace({ labelSelector: `${MANAGED_LABEL}=true,${exports.SERVICE_RUN_LABEL}` }))
+            .then(async (namespaces) => {
+            for (const namespace of namespaces.items) {
+                const name = namespace.metadata?.name;
+                if (name &&
+                    isCallerNamespace(this.config.backend.namespacePrefix, name) &&
+                    !this.serviceRuns.has(name) &&
+                    !this.asleep.has(name) &&
+                    !namespace.metadata?.deletionTimestamp) {
+                    this.log?.(`removing the namespace ${name} of a service run from before the server started`);
+                    await removeServiceRunNamespace(this.instance, name, 0);
+                }
+            }
+        })
+            .catch((e) => this.log?.(`the namespaces of service runs could not be removed: ${(0, log_1.getErrorMessage)(e)}`));
+    }
+    // The kept claims of a namespace that expired go at the soonest expiry the server knows
+    // of, but those of a scope a run of the caller executes in
     sweepLater(namespace, delay) {
         const at = Date.now() + delay + 1000;
         const pending = this.sweeps.get(namespace);
@@ -321,6 +644,7 @@ class KubernetesExecutor {
             now: new Date(),
             keep: this.keep > 0,
             timeout: RELEASE_TIMEOUT,
+            inUse: this.scopesInUse(namespace),
         });
         if (removed.length > 0) {
             this.log?.(`removed ${removed.length} kept claims in ${namespace}`);
@@ -337,7 +661,7 @@ class KubernetesExecutor {
         }
         this.swept = true;
         Promise.resolve()
-            .then(() => this.instance.coreApi.listNamespace({ labelSelector: `${MANAGED_LABEL}=true` }))
+            .then(() => this.instance.coreApi.listNamespace({ labelSelector: `${MANAGED_LABEL}=true,!${exports.SERVICE_RUN_LABEL}` }))
             .then((namespaces) => {
             for (const namespace of namespaces.items) {
                 const name = namespace.metadata?.name;
@@ -352,11 +676,55 @@ class KubernetesExecutor {
     }
 }
 exports.KubernetesExecutor = KubernetesExecutor;
+const ENFORCE_LABEL = 'pod-security.kubernetes.io/enforce';
+const AUDIT_LABEL = 'pod-security.kubernetes.io/audit';
+/**
+ * The Pod Security level a caller's namespace is made with: `baseline`, no privileged pods,
+ * host paths or host network, whatever a build asks. `baseline` also refuses the capabilities
+ * of `backend.capabilities`, sandboxed or not: with them the namespace enforces `privileged`
+ * and audits `baseline`, and the operator's admission policy (see the docs on capabilities)
+ * refuses what `baseline` would to a pod outside the sandbox's RuntimeClass.
+ */
+function getPodSecurityLabels(backend) {
+    return backend.capabilities
+        ? { [ENFORCE_LABEL]: 'privileged', [AUDIT_LABEL]: 'baseline' }
+        : { [ENFORCE_LABEL]: 'baseline' };
+}
+exports.getPodSecurityLabels = getPodSecurityLabels;
+/**
+ * The server never changes the level of a namespace it made (that would take `patch` on every
+ * namespace of the cluster): one made before `backend.capabilities` is relabelled by the
+ * operator. Until then its tasks that add capabilities fail with this, the others run.
+ */
+function getCapabilitiesProblem(namespace, labels, backend) {
+    const enforce = labels?.[ENFORCE_LABEL];
+    if (!backend.capabilities || enforce === 'privileged') {
+        return undefined;
+    }
+    return (`the namespace ${namespace} was made before backend.capabilities and enforces ` +
+        `${enforce ? `Pod Security ${enforce}` : 'no Pod Security level of its own'}, not privileged: kubectl label namespace ${namespace} ` +
+        `${ENFORCE_LABEL}=privileged ${AUDIT_LABEL}=baseline --overwrite`);
+}
+exports.getCapabilitiesProblem = getCapabilitiesProblem;
+/**
+ * A namespace made while the server had `backend.capabilities` admits privileged pods but for
+ * the operator's admission policy. Without capabilities the server runs nothing in it until the
+ * operator sets it back.
+ */
+function getPrivilegedNamespaceProblem(namespace, labels, backend) {
+    if (backend.capabilities || labels?.[ENFORCE_LABEL] !== 'privileged') {
+        return undefined;
+    }
+    return (`the namespace ${namespace} enforces Pod Security privileged, made while the server had ` +
+        `backend.capabilities, and the server runs nothing in it without them: kubectl label namespace ` +
+        `${namespace} ${ENFORCE_LABEL}=baseline ${AUDIT_LABEL}- --overwrite`);
+}
+exports.getPrivilegedNamespaceProblem = getPrivilegedNamespaceProblem;
 // What may mount a claim a run kept: the runs of the same repository, as they share the
 // cache, and on a sandbox the runs of that sandbox only, whose claim is on the same node or
 // zone as the kept ones. The owner is the namespace's.
 function getVolumeScope(run) {
-    if (run.source.type === 'sandbox') {
+    if (run.source.type === 'sandbox' && !run.source.check) {
         return `sandbox-${(0, crypto_1.createHash)('sha256').update(run.source.id).digest('hex').slice(0, 24)}`;
     }
     return (0, executor_1.cacheNamespace)(run);
@@ -370,6 +738,8 @@ class CallerNamespaces {
     constructor(instance, backend) {
         this.instance = instance;
         this.backend = backend;
+        // the namespaces this process set up, by their uid, and when
+        this.applied = new Map();
     }
     name(owner) {
         return getCallerNamespace(this.backend.namespacePrefix, owner);
@@ -377,9 +747,15 @@ class CallerNamespaces {
     // The caller's namespace: made by the server and kept apart from the others. The
     // server binds its rights there, isolates the network of the pods, applies the
     // operator's quota and gives containers without resources default requests, so they
-    // are admitted under a quota on compute. All of it is replaced on every run, so the
-    // namespace follows the config.
-    async prepare(namespace) {
+    // are admitted under a quota on compute. All of it is replaced when the server sets up
+    // a namespace it did not set up in the last `NAMESPACE_SETUP_REUSE_MS` (it reads the
+    // config once, at its start), so the namespace follows the config; a namespace made anew
+    // since (deleted and made again) is set up again at once. Returns what keeps the tasks that
+    // add capabilities from running in it (getCapabilitiesProblem), if anything.
+    async prepare(namespace, 
+    // a service run's own namespace: its label, the namespace of the server that reaches its
+    // endpoints, and its quota
+    serviceRun) {
         const backend = this.backend;
         let existing;
         try {
@@ -393,17 +769,33 @@ class CallerNamespaces {
         if (existing && existing.metadata?.labels?.[MANAGED_LABEL] !== 'true') {
             throw new Error(`the namespace ${namespace} was not made by hammerkit, the server does not run tasks in it`);
         }
+        const privileged = getPrivilegedNamespaceProblem(namespace, existing?.metadata?.labels, backend);
+        if (privileged) {
+            throw new Error(privileged);
+        }
+        // one made by another run of this server at the same time has the labels of its config
+        const problem = () => getCapabilitiesProblem(namespace, existing?.metadata?.labels ?? getPodSecurityLabels(backend), backend);
+        const started = Date.now();
+        const applied = this.applied.get(namespace);
+        if (existing &&
+            !existing.metadata?.deletionTimestamp &&
+            applied &&
+            applied.uid === existing.metadata?.uid &&
+            started - applied.at < NAMESPACE_SETUP_REUSE_MS) {
+            return problem();
+        }
+        this.applied.delete(namespace);
         if (!existing) {
             try {
-                await this.instance.coreApi.createNamespace({
+                existing = await this.instance.coreApi.createNamespace({
                     body: {
                         metadata: {
                             name: namespace,
                             labels: {
                                 [MANAGED_LABEL]: 'true',
                                 'hammerkit.dev/caller': namespace,
-                                // no privileged pods, host paths or host network, whatever a build asks
-                                'pod-security.kubernetes.io/enforce': 'baseline',
+                                ...(serviceRun ? { [exports.SERVICE_RUN_LABEL]: serviceRun.serviceRun } : {}),
+                                ...getPodSecurityLabels(backend),
                             },
                         },
                     },
@@ -419,7 +811,10 @@ class CallerNamespaces {
         const [first, ...policies] = getNamespacePolicies(namespace, backend.deniedEgress, backend.allowedEgress);
         // the API server may refuse what the binding allows for a moment after it was made
         await untilAllowed(() => (0, apply_1.replace)(this.instance, first));
-        for (const policy of policies) {
+        for (const policy of [
+            ...policies,
+            ...(serviceRun ? [getServerIngressPolicy(namespace, serviceRun.server, serviceRun.ports)] : []),
+        ]) {
             await (0, apply_1.replace)(this.instance, policy);
         }
         const metadata = {
@@ -432,7 +827,7 @@ class CallerNamespaces {
             apiVersion: 'v1',
             kind: 'ResourceQuota',
             metadata,
-            spec: { hard: backend.quota },
+            spec: { hard: serviceRun?.quota ?? backend.quota },
         };
         const defaults = backend.defaultResources;
         const limits = {
@@ -450,7 +845,7 @@ class CallerNamespaces {
             },
         };
         for (const [object, wanted] of [
-            [quota, !!backend.quota],
+            [quota, !!(serviceRun?.quota ?? backend.quota)],
             [limits, !!defaults],
         ]) {
             if (wanted) {
@@ -465,6 +860,11 @@ class CallerNamespaces {
                 });
             }
         }
+        const uid = existing?.metadata?.uid;
+        if (uid) {
+            this.applied.set(namespace, { uid, at: started });
+        }
+        return problem();
     }
     whoAmI() {
         if (!this.subject) {
@@ -487,7 +887,7 @@ class CallerNamespaces {
 exports.CallerNamespaces = CallerNamespaces;
 // The claim of a sandbox as the Jobs of its run mount it: the run's copy of the files is
 // `root`, a source below it is the same path below `subPath` in the claim.
-function getWorkspace(root, claim, namespace) {
+function getWorkspace(root, claim, namespace, runner) {
     if (claim.namespace !== namespace) {
         throw new Error(`the sandbox is in ${claim.namespace}, the run in ${namespace}`);
     }
@@ -497,6 +897,7 @@ function getWorkspace(root, claim, namespace) {
         subPath: claim.subPath,
         label: claim.label,
         ...(claim.pod ? { pod: claim.pod } : {}),
+        ...(runner ? { runner: true } : {}),
     };
 }
 // Everything a run made in the caller's namespace, by the label selector of the run:
@@ -505,16 +906,21 @@ function getWorkspace(root, claim, namespace) {
 // run outlives its policy.
 // Deletes what a run made. The NetworkPolicies go last, once the pods are: a limited
 // pod's own policy is all that limits it.
-async function removeRunObjects(instance, namespace, labelSelector, timeout) {
+async function removeRunObjects(instance, namespace, labelSelector, timeout, 
+// `claims: false` leaves the claims: a service run that sleeps keeps its volumes
+options = {}) {
+    const claims = options.claims ?? true;
     const selector = { namespace, labelSelector };
     await instance.batchApi.deleteCollectionNamespacedJob({ ...selector, propagationPolicy: 'Background' });
     await instance.appsApi.deleteCollectionNamespacedDeployment({ ...selector, propagationPolicy: 'Background' });
     await instance.coreApi.deleteCollectionNamespacedPod({ ...selector, gracePeriodSeconds: 0 });
     await instance.coreApi.deleteCollectionNamespacedService(selector);
     await instance.coreApi.deleteCollectionNamespacedSecret(selector);
-    await instance.coreApi.deleteCollectionNamespacedPersistentVolumeClaim(selector);
+    if (claims) {
+        await instance.coreApi.deleteCollectionNamespacedPersistentVolumeClaim(selector);
+    }
     // the next run of the caller also uses the same claim names
-    await awaitRunObjectsGone(instance, namespace, labelSelector, timeout);
+    await awaitRunObjectsGone(instance, namespace, labelSelector, timeout, claims);
     const policies = await instance.objectApi.list('networking.k8s.io/v1', 'NetworkPolicy', namespace, undefined, undefined, undefined, undefined, labelSelector);
     for (const policy of policies.items) {
         // the items of a list do not say their kind
@@ -543,11 +949,13 @@ async function untilAllowed(fn, attempts = 20) {
     }
 }
 // Until the pods and claims of a run are gone: a claim is kept while a pod uses it.
-async function awaitRunObjectsGone(instance, namespace, labelSelector, timeout) {
+async function awaitRunObjectsGone(instance, namespace, labelSelector, timeout, withClaims = true) {
     const deadline = Date.now() + timeout;
     for (;;) {
         const pods = await instance.coreApi.listNamespacedPod({ namespace, labelSelector });
-        const claims = await instance.coreApi.listNamespacedPersistentVolumeClaim({ namespace, labelSelector });
+        const claims = withClaims
+            ? await instance.coreApi.listNamespacedPersistentVolumeClaim({ namespace, labelSelector })
+            : { items: [] };
         const left = [...pods.items, ...claims.items].map((item) => `${item.kind ?? ''} ${item.metadata?.name}`.trim());
         if (left.length === 0) {
             return;