hammerkit 1.27.0-next.2 → 1.27.0-next.21

package/dist/server/executor.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.HostExecutor = exports.volumeScope = exports.cacheNamespace = exports.defaultWorker = void 0;
 const child_process_1 = require("child_process");
 const crypto_1 = require("crypto");
+const checks_1 = require("./checks");
 const fs_1 = require("fs");
 const promises_1 = require("fs/promises");
 const path_1 = require("path");
@@ -35,9 +36,18 @@ exports.defaultWorker = defaultWorker;
 // share one, those of an upload or a sandbox only their owner's. Somebody who can only
 // upload, or an agent on a sandbox, can not leave an entry for a run from a reviewed
 // repository to find.
+//
+// A check on a sandbox runs on a copy of the files a commit of them would hold, so it is a
+// run of the repository like a ref run of that commit, and shares its cache: what its
+// owner may write in the sandbox is not part of it. It leaves entries a run from main can
+// find, keyed by the files it ran on, as a ref run of a branch does: so only a caller who
+// may push a branch of the repository and run from it may check (server.ts).
 function cacheNamespace(run) {
-    const key = run.source.type === 'ref' ? `ref\0${run.source.repo}` : `${run.source.type}\0${run.owner.iss}\0${run.owner.sub}`;
-    return `${run.source.type}-${(0, crypto_1.createHash)('sha256').update(key).digest('hex').slice(0, 24)}`;
+    const repository = run.source.type === 'ref' || (run.source.type === 'sandbox' && run.source.check);
+    const key = repository
+        ? `ref\0${run.source.repo}`
+        : `${run.source.type}\0${run.owner.iss}\0${run.owner.sub}`;
+    return `${repository ? 'ref' : run.source.type}-${(0, crypto_1.createHash)('sha256').update(key).digest('hex').slice(0, 24)}`;
 }
 exports.cacheNamespace = cacheNamespace;
 // The docker volumes of a run's services are its caller's, within the cache's source trust:
@@ -100,7 +110,7 @@ class HostExecutor {
         if (extra.length > 0) {
             throw new Error(`the run would use ${extra.join(', ')}, which it was not admitted for`);
         }
-        const refused = (0, run_plan_1.refuseArguments)(run, planned.plan);
+        const refused = (0, run_plan_1.refuseArguments)(run, planned.plan, policy);
         if (refused) {
             throw new Error(refused);
         }
@@ -121,13 +131,42 @@ class HostExecutor {
             let claim;
             // the files of the sandbox the tasks work on
             let work;
-            if (source.type === 'ref') {
+            if (source.type === 'sandbox' && source.check) {
+                // a check works on the files a push of the sandbox commits, copied out of it: never
+                // on the sandbox, whose ignored files (node_modules, outputs) its owner writes
+                const { registry, network } = sandboxes;
+                cwd = (0, path_1.join)(root, 'source');
+                try {
+                    await registry.exportTree(source.id, source.tree, cwd, abortCtrl.signal);
+                }
+                catch (e) {
+                    throw new Error(`the files of the sandbox ${source.id} could not be copied for the check, submit it again: ${e instanceof Error ? e.message : String(e)}`);
+                }
+                // a repository of its own, as a checkout of the commit is: the root of the project,
+                // which the paths of the tasks' keys are relative to, is the one of git
+                await (0, git_checkout_1.git)(['init', '--quiet', cwd], {}, abortCtrl.signal, `the check of sandbox ${source.id}`, 'init');
+                context.emit({
+                    type: 'log',
+                    task: 'hammerkit',
+                    stream: 'stdout',
+                    text: `check of sandbox ${source.id} of ${source.repo}: files ${source.tree}, fetched at ${source.commit}`,
+                });
+                await this.writeBundle(cwd, bundle.files);
+                input = {
+                    entry: (0, path_1.join)(cwd, version.entry),
+                    ...(planning.policy.network ? { network: (0, work_network_1.toNetworkSchema)(planning.policy.network) } : {}),
+                    sandboxNetwork: network,
+                };
+            }
+            else if (source.type === 'ref') {
                 cwd = (0, path_1.join)(root, 'source');
                 const { repo, ref, commit: pinned } = source;
                 const credentials = await repos.credentials(repo);
                 let commit;
                 try {
-                    commit = await (0, git_checkout_1.checkoutRef)(repos.url(repo), ref, cwd, credentials.env, abortCtrl.signal, pinned);
+                    // a woken service run starts the commit it started first, where the ref is now or not
+                    const fetched = run.wakes ? pinned : ref;
+                    commit = await (0, git_checkout_1.checkoutRef)(repos.url(repo), fetched, cwd, credentials.env, abortCtrl.signal, pinned);
                 }
                 catch (e) {
                     throw repos.scrub(e, credentials.values);
@@ -188,7 +227,16 @@ class HostExecutor {
                     options: run.options,
                     accounts: run.accounts,
                     volumeScope: volumeScope(run),
+                    ...((0, checks_1.isCheckRun)(run) ? { check: true } : {}),
                     catalog: planning.environment.hosted?.catalog,
+                    ...(run.up
+                        ? {
+                            up: {
+                                services: run.up.services,
+                                endpoints: run.up.endpoints.map(({ service, port }) => ({ service, port })),
+                            },
+                        }
+                        : {}),
                     ...(placed ? { kubernetes: placed.kubernetes } : {}),
                 }, cwd, { ...this.processEnvironment(home, cache), ...placed?.env }, context, abortCtrl.signal);
             }
@@ -282,6 +330,9 @@ class HostExecutor {
                     if (message.kind === 'event') {
                         context.emit(message.event);
                     }
+                    else if (message.kind === 'up') {
+                        context.up?.(message.targets);
+                    }
                     else if (message.kind === 'result') {
                         result = message;
                         // nothing left to tell the process

package/dist/server/kubernetes-backend.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.awaitRunObjectsGone = exports.removeRunObjects = exports.CallerNamespaces = exports.getVolumeScope = exports.KubernetesExecutor = exports.getRunRoleBinding = exports.getNamespacePolicies = exports.getClusterAccess = exports.isCallerNamespace = exports.getCallerNamespace = void 0;
+exports.awaitRunObjectsGone = exports.removeRunObjects = exports.CallerNamespaces = exports.getVolumeScope = exports.KubernetesExecutor = exports.getRunRoleBinding = exports.removeServiceRunNamespace = exports.getServerIngressPolicy = exports.getNamespacePolicies = exports.getClusterAccess = exports.isCallerNamespace = exports.SERVICE_RUN_LABEL = exports.getServiceRunNamespace = exports.getCallerNamespace = void 0;
 const crypto_1 = require("crypto");
 const fs_1 = require("fs");
 const os_1 = require("os");
@@ -22,6 +22,8 @@ const POLICY_NAME = 'hammerkit';
 const MANAGED_LABEL = 'hammerkit.dev/managed';
 // how long the end of a run waits for its pods and claims to be gone
 const RELEASE_TIMEOUT = 120000;
+// how long the setup of a caller's namespace is taken as it is, before it is applied again
+const NAMESPACE_SETUP_REUSE_MS = 5 * 60000;
 // where a pod of the cluster finds the token and certificate of its ServiceAccount
 const IN_CLUSTER_NAMESPACE = '/var/run/secrets/kubernetes.io/serviceaccount/namespace';
 /**
@@ -32,6 +34,19 @@ function getCallerNamespace(prefix, owner) {
     return `${prefix}${(0, crypto_1.createHash)('sha256').update(`${owner.iss}\0${owner.sub}`).digest('hex').slice(0, 16)}`;
 }
 exports.getCallerNamespace = getCallerNamespace;
+/**
+ * The namespace of a service run: one of its own, as the services stay up for hours while
+ * the caller's other runs go on in theirs (one at a time, each removing what the one
+ * before left).
+ */
+function getServiceRunNamespace(prefix, run, wakes = 0) {
+    // a woken run starts in a namespace of its own, the one it slept in may still be removed
+    const name = wakes > 0 ? `up\0${run}\0${wakes}` : `up\0${run}`;
+    return `${prefix}${(0, crypto_1.createHash)('sha256').update(name).digest('hex').slice(0, 16)}`;
+}
+exports.getServiceRunNamespace = getServiceRunNamespace;
+// the label of the namespace of a service run: the run label value of its run
+exports.SERVICE_RUN_LABEL = 'hammerkit.dev/service-run';
 // Whether a namespace is one getCallerNamespace makes with this prefix: a server named `hk`
 // passes over the namespaces of a server named `hk-test` on the same cluster.
 function isCallerNamespace(prefix, name) {
@@ -143,6 +158,64 @@ function getNamespacePolicies(namespace, deniedEgress, allowedEgress = []) {
     ];
 }
 exports.getNamespacePolicies = getNamespacePolicies;
+/**
+ * What lets the server into the namespace of a service run, to the ports of its endpoints:
+ * policies add up, so this one opens the pods to the server's namespace besides what the
+ * others let in, limited ones too.
+ */
+// The server reaches the ports of the endpoints, from its own namespace, and no other port:
+// what else the run starts (a database) stays for the pods of the run.
+function getServerIngressPolicy(namespace, server, ports) {
+    return {
+        apiVersion: 'networking.k8s.io/v1',
+        kind: 'NetworkPolicy',
+        metadata: {
+            name: 'hammerkit-server-ingress',
+            namespace,
+            annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
+            labels: { [MANAGED_LABEL]: 'true' },
+        },
+        spec: {
+            podSelector: {},
+            policyTypes: ['Ingress'],
+            ingress: ports.length > 0
+                ? [
+                    {
+                        _from: [{ namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': server } } }],
+                        ports: [...new Set(ports)].map((port) => ({ protocol: 'TCP', port })),
+                    },
+                ]
+                : [],
+        },
+    };
+}
+exports.getServerIngressPolicy = getServerIngressPolicy;
+// The namespace of a service run goes with all it holds; waits up to `timeout` for it.
+async function removeServiceRunNamespace(instance, namespace, timeout) {
+    try {
+        await instance.coreApi.deleteNamespace({ name: namespace, propagationPolicy: 'Background' });
+    }
+    catch (e) {
+        if ((0, apply_1.statusCodeOf)(e) === 404) {
+            return;
+        }
+        throw e;
+    }
+    const deadline = Date.now() + timeout;
+    while (Date.now() < deadline) {
+        try {
+            await instance.coreApi.readNamespace({ name: namespace });
+        }
+        catch (e) {
+            if ((0, apply_1.statusCodeOf)(e) === 404) {
+                return;
+            }
+            throw e;
+        }
+        await (0, sleep_1.sleep)(1000);
+    }
+}
+exports.removeServiceRunNamespace = removeServiceRunNamespace;
 /**
  * The server's rights in a caller's namespace: the operator's ClusterRole, bound there
  * only, so the server needs no rights on Secrets or pods anywhere else.
@@ -167,22 +240,31 @@ exports.getRunRoleBinding = getRunRoleBinding;
  * process of the server as on a host (see HostExecutor); what changes is where its
  * container tasks and services go: into the caller's namespace, on the nodes of the
  * operator's pools, with the secrets of providers read in the Job itself. The runs
- * of one caller execute one after the other, since the objects of a task are named
- * by its cache id in the caller's namespace; different callers run side by side.
- * When a run ends, everything made for it is removed, but for the claims of its tasks
- * the server keeps (`backend.keepVolumes`), which go once they expired.
+ * of a caller execute side by side in their namespace, up to `runs.limits.running`:
+ * the objects of a run are named and found by its run label (getItemKey), so two runs
+ * of the same task have their own. What two runs share are the claims the server keeps
+ * (`backend.keepVolumes`) of a scope (getVolumeScope): the runs of one scope go one at
+ * a time (exclusive). When a run ends, everything made for it is removed, but for the
+ * kept claims, which go once they expired.
  */
 class KubernetesExecutor {
     constructor(config, processEnvs, options) {
         this.config = config;
-        // the registry keeps a caller's next run in its queue, not running and timed here
-        this.runningPerCaller = 1;
-        // the run that executes, or the last one waiting, per caller namespace
+        // per caller namespace, what changes objects of the whole namespace: its setup, the
+        // removal of what a stopped server left, the sweeps of kept claims. The last one waiting.
         this.queues = new Map();
+        // per caller namespace, the runs that execute in it: by run label, their volume scope
+        this.active = new Map();
         // per caller namespace, when its kept claims are next looked at
         this.sweeps = new Map();
         // whether the kept claims a server left when it stopped were looked for
         this.swept = false;
+        // the namespaces where no run of this process left anything: what a run makes goes at its
+        // end, unless its release failed
+        this.tidy = new Set();
+        // the namespaces of the service runs that execute, and whether those of earlier ones went
+        this.serviceRuns = new Set();
+        this.serviceRunsSwept = false;
         this.access = getClusterAccess(config.backend, processEnvs);
         this.instance = (0, kubernetes_instance_1.createKubernetesInstances)({
             type: 'kubernetes',
@@ -203,17 +285,56 @@ class KubernetesExecutor {
         this.log = options.log;
         this.host = new executor_1.HostExecutor({ ...options, placement: this.placement() });
     }
-    execute(run, context) {
-        const namespace = getCallerNamespace(this.config.backend.namespacePrefix, run.owner);
-        return this.enqueue(namespace, () => {
-            if (context.signal.aborted) {
-                throw new Error('the run was cancelled');
+    async execute(run, context) {
+        if (run.up) {
+            if (run.source.type === 'sandbox') {
+                throw new Error('on a cluster a service run starts from a branch, push the sandbox and start it from there');
             }
+            // in a namespace of its own, apart from the caller's runs and their claims
             return this.host.execute(run, context);
-        });
+        }
+        const namespace = getCallerNamespace(this.config.backend.namespacePrefix, run.owner);
+        const label = (0, job_placement_1.getRunLabelValue)(run.id);
+        let runs = this.active.get(namespace);
+        if (!runs) {
+            runs = new Map();
+            this.active.set(namespace, runs);
+        }
+        runs.set(label, getVolumeScope(run));
+        try {
+            await this.host.execute(run, context);
+        }
+        finally {
+            runs.delete(label);
+            if (runs.size === 0 && this.active.get(namespace) === runs) {
+                this.active.delete(namespace);
+            }
+        }
+    }
+    namespaceOf(run) {
+        const prefix = this.config.backend.namespacePrefix;
+        return run.up ? getServiceRunNamespace(prefix, run.id, run.wakes) : getCallerNamespace(prefix, run.owner);
+    }
+    // The runs of a scope share its kept claims, which one of them may change at a time:
+    // what a task finds in one is what the last run left there. Without kept claims, runs
+    // share no object and nothing is held.
+    exclusive(run) {
+        if (this.keep === 0 || run.up) {
+            return undefined;
+        }
+        return `${getCallerNamespace(this.config.backend.namespacePrefix, run.owner)}/${getVolumeScope(run)}`;
+    }
+    // The scopes of the runs that execute in the namespace, but `except`'s: their kept claims
+    // are theirs to mount and stay. A server that keeps none mounts none.
+    scopesInUse(namespace, except) {
+        if (this.keep === 0) {
+            return new Set();
+        }
+        return new Set([...(this.active.get(namespace) ?? new Map()).entries()]
+            .filter(([label]) => label !== except)
+            .map(([, scope]) => scope));
     }
-    // What happens in a caller's namespace happens one at a time: their runs, and the
-    // removal of the claims their runs kept
+    // What changes objects of a whole namespace happens one at a time, not what its runs do
     enqueue(namespace, fn) {
         const previous = this.queues.get(namespace) ?? Promise.resolve();
         const execution = previous.then(fn);
@@ -227,54 +348,94 @@ class KubernetesExecutor {
         return execution;
     }
     placement() {
-        const backend = this.config.backend;
         return {
             prepare: async (run, context, work) => {
-                const namespace = getCallerNamespace(backend.namespacePrefix, run.owner);
-                await this.namespaces.prepare(namespace);
-                // what an earlier run left when the server stopped before its end; the runs
-                // of a caller execute one at a time, so no other run is in the namespace
-                await removeRunObjects(this.instance, namespace, `${job_placement_1.RUN_LABEL},${job_placement_1.RUN_LABEL}!=${(0, job_placement_1.getRunLabelValue)(run.id)}`, RELEASE_TIMEOUT);
-                // the claims earlier runs kept that expired; the others this run may mount
-                const { next } = await (0, kept_claims_1.sweepKeptClaims)(this.instance, namespace, {
-                    now: new Date(),
-                    keep: this.keep > 0,
-                    timeout: RELEASE_TIMEOUT,
-                });
-                if (next) {
-                    this.sweepLater(namespace, Math.max(0, next.getTime() - Date.now()));
+                const namespace = this.namespaceOf(run);
+                if (run.up) {
+                    if (context.signal.aborted) {
+                        throw new Error('the run was cancelled');
+                    }
+                    this.serviceRuns.add(namespace);
+                    this.sweepServiceRuns();
+                    // reached by the server at the Services of its endpoints
+                    await this.namespaces.prepare(namespace, {
+                        serviceRun: (0, job_placement_1.getRunLabelValue)(run.id),
+                        server: this.namespace,
+                        ports: (run.up.endpoints ?? []).map((endpoint) => endpoint.port),
+                        quota: this.config.runs.up.quota,
+                    });
+                    context.emit({ type: 'log', task: 'hammerkit', stream: 'stdout', text: `runs in the namespace ${namespace}` });
+                    return { kubernetes: this.environmentOf(run, namespace), env: this.access.env };
                 }
+                const label = (0, job_placement_1.getRunLabelValue)(run.id);
+                await this.enqueue(namespace, async () => {
+                    if (context.signal.aborted) {
+                        throw new Error('the run was cancelled');
+                    }
+                    await this.namespaces.prepare(namespace);
+                    // what an earlier run left when the server stopped before its end, or when its
+                    // release failed: the objects of a run that does not execute. One that starts
+                    // after this waits for it here before it makes any.
+                    if (!this.tidy.has(namespace)) {
+                        const running = [...new Set([label, ...(this.active.get(namespace)?.keys() ?? [])])];
+                        await removeRunObjects(this.instance, namespace, `${job_placement_1.RUN_LABEL},${job_placement_1.RUN_LABEL} notin (${running.join(',')})`, RELEASE_TIMEOUT);
+                        this.tidy.add(namespace);
+                    }
+                    // the claims earlier runs kept that expired; the others this run may mount
+                    const { next } = await (0, kept_claims_1.sweepKeptClaims)(this.instance, namespace, {
+                        now: new Date(),
+                        keep: this.keep > 0,
+                        timeout: RELEASE_TIMEOUT,
+                        inUse: this.scopesInUse(namespace, label),
+                    });
+                    if (next) {
+                        this.sweepLater(namespace, Math.max(0, next.getTime() - Date.now()));
+                    }
+                });
                 this.sweepLeftClaims();
                 context.emit({ type: 'log', task: 'hammerkit', stream: 'stdout', text: `runs in the namespace ${namespace}` });
-                const kubernetes = {
-                    type: 'kubernetes',
-                    namespace,
-                    kubeConfig: this.access.kubeConfig,
-                    context: this.access.context,
-                    httpRoutes: [],
-                    storageClass: backend.storageClass,
-                    volumeSize: backend.volumeSize,
-                    maxVolumeSize: (0, server_config_1.getMaxVolumeSize)(backend),
-                    networkPolicyEnforced: backend.networkPolicy === 'enforced',
-                    server: {
-                        run: run.id,
-                        namespace: this.namespace,
-                        pools: backend.pools,
-                        providers: backend.providers,
-                        accounts: Object.fromEntries(Object.entries(this.config.accounts).flatMap(([name, account]) => account.kubernetes ? [[name, account.kubernetes]] : [])),
-                        ...(work?.claim ? { workspace: getWorkspace(work.root, work.claim, namespace) } : {}),
-                        ...(this.keep > 0 ? { volumeScope: getVolumeScope(run) } : {}),
-                    },
-                };
-                return { kubernetes, env: this.access.env };
+                return { kubernetes: this.environmentOf(run, namespace, work), env: this.access.env };
             },
             release: async (run, context) => {
-                const namespace = getCallerNamespace(backend.namespacePrefix, run.owner);
+                const namespace = this.namespaceOf(run);
+                if (run.up) {
+                    try {
+                        await removeServiceRunNamespace(this.instance, namespace, RELEASE_TIMEOUT);
+                    }
+                    catch (e) {
+                        // at least nothing of it runs on: its objects are the server's to remove (runRole)
+                        const forbidden = (0, apply_1.statusCodeOf)(e) === 403;
+                        const text = forbidden
+                            ? `the server may not delete the namespace ${namespace}, its ClusterRole needs delete on namespaces (see the backend docs); removing what runs in it`
+                            : `the namespace ${namespace} could not be removed: ${(0, log_1.getErrorMessage)(e)}`;
+                        this.log?.(text);
+                        context.emit({ type: 'log', task: 'hammerkit', stream: 'stderr', text });
+                        try {
+                            await removeRunObjects(this.instance, namespace, job_placement_1.RUN_LABEL, RELEASE_TIMEOUT);
+                        }
+                        catch (inner) {
+                            context.emit({
+                                type: 'log',
+                                task: 'hammerkit',
+                                stream: 'stderr',
+                                text: `what the run made in ${namespace} could not be removed: ${(0, log_1.getErrorMessage)(inner)}`,
+                            });
+                        }
+                        // the next look removes the namespace
+                        this.serviceRunsSwept = false;
+                    }
+                    finally {
+                        this.serviceRuns.delete(namespace);
+                    }
+                    return;
+                }
                 try {
                     const selector = `${job_placement_1.RUN_LABEL}=${(0, job_placement_1.getRunLabelValue)(run.id)}`;
                     await removeRunObjects(this.instance, namespace, selector, RELEASE_TIMEOUT);
                 }
                 catch (e) {
+                    // the next run removes what is left
+                    this.tidy.delete(namespace);
                     context.emit({
                         type: 'log',
                         task: 'hammerkit',
@@ -300,8 +461,59 @@ class KubernetesExecutor {
             },
         };
     }
-    // The kept claims of a namespace that expired go once no run of the caller is in it: at
-    // the soonest expiry the server knows of
+    // The cluster as the process of a run sees it: the namespace its tasks and services go in.
+    environmentOf(run, namespace, work) {
+        const backend = this.config.backend;
+        return {
+            type: 'kubernetes',
+            namespace,
+            kubeConfig: this.access.kubeConfig,
+            context: this.access.context,
+            httpRoutes: [],
+            storageClass: backend.storageClass,
+            volumeSize: backend.volumeSize,
+            maxVolumeSize: (0, server_config_1.getMaxVolumeSize)(backend),
+            networkPolicyEnforced: backend.networkPolicy === 'enforced',
+            server: {
+                run: run.id,
+                namespace: this.namespace,
+                pools: backend.pools,
+                providers: backend.providers,
+                accounts: Object.fromEntries(Object.entries(this.config.accounts).flatMap(([name, account]) => account.kubernetes ? [[name, account.kubernetes]] : [])),
+                ...(work?.claim ? { workspace: getWorkspace(work.root, work.claim, namespace, backend.sandboxes.runner) } : {}),
+                // the claims of a service run go with its namespace
+                ...(this.keep > 0 && !run.up ? { volumeScope: getVolumeScope(run) } : {}),
+            },
+        };
+    }
+    // Once, on the first service run: the namespaces of service runs the server left when it
+    // stopped, whose services went with it
+    // Once when the server starts, and on the next service run after a namespace could not be
+    // removed: the namespaces of service runs that none of the server's runs is in, whose
+    // services went with the process of a server that stopped
+    sweepServiceRuns() {
+        if (this.serviceRunsSwept) {
+            return;
+        }
+        this.serviceRunsSwept = true;
+        Promise.resolve()
+            .then(() => this.instance.coreApi.listNamespace({ labelSelector: `${MANAGED_LABEL}=true,${exports.SERVICE_RUN_LABEL}` }))
+            .then(async (namespaces) => {
+            for (const namespace of namespaces.items) {
+                const name = namespace.metadata?.name;
+                if (name &&
+                    isCallerNamespace(this.config.backend.namespacePrefix, name) &&
+                    !this.serviceRuns.has(name) &&
+                    !namespace.metadata?.deletionTimestamp) {
+                    this.log?.(`removing the namespace ${name} of a service run from before the server started`);
+                    await removeServiceRunNamespace(this.instance, name, 0);
+                }
+            }
+        })
+            .catch((e) => this.log?.(`the namespaces of service runs could not be removed: ${(0, log_1.getErrorMessage)(e)}`));
+    }
+    // The kept claims of a namespace that expired go at the soonest expiry the server knows
+    // of, but those of a scope a run of the caller executes in
     sweepLater(namespace, delay) {
         const at = Date.now() + delay + 1000;
         const pending = this.sweeps.get(namespace);
@@ -321,6 +533,7 @@ class KubernetesExecutor {
             now: new Date(),
             keep: this.keep > 0,
             timeout: RELEASE_TIMEOUT,
+            inUse: this.scopesInUse(namespace),
         });
         if (removed.length > 0) {
             this.log?.(`removed ${removed.length} kept claims in ${namespace}`);
@@ -337,7 +550,7 @@ class KubernetesExecutor {
         }
         this.swept = true;
         Promise.resolve()
-            .then(() => this.instance.coreApi.listNamespace({ labelSelector: `${MANAGED_LABEL}=true` }))
+            .then(() => this.instance.coreApi.listNamespace({ labelSelector: `${MANAGED_LABEL}=true,!${exports.SERVICE_RUN_LABEL}` }))
             .then((namespaces) => {
             for (const namespace of namespaces.items) {
                 const name = namespace.metadata?.name;
@@ -356,7 +569,7 @@ exports.KubernetesExecutor = KubernetesExecutor;
 // cache, and on a sandbox the runs of that sandbox only, whose claim is on the same node or
 // zone as the kept ones. The owner is the namespace's.
 function getVolumeScope(run) {
-    if (run.source.type === 'sandbox') {
+    if (run.source.type === 'sandbox' && !run.source.check) {
         return `sandbox-${(0, crypto_1.createHash)('sha256').update(run.source.id).digest('hex').slice(0, 24)}`;
     }
     return (0, executor_1.cacheNamespace)(run);
@@ -370,6 +583,8 @@ class CallerNamespaces {
     constructor(instance, backend) {
         this.instance = instance;
         this.backend = backend;
+        // the namespaces this process set up, by their uid, and when
+        this.applied = new Map();
     }
     name(owner) {
         return getCallerNamespace(this.backend.namespacePrefix, owner);
@@ -377,9 +592,14 @@ class CallerNamespaces {
     // The caller's namespace: made by the server and kept apart from the others. The
     // server binds its rights there, isolates the network of the pods, applies the
     // operator's quota and gives containers without resources default requests, so they
-    // are admitted under a quota on compute. All of it is replaced on every run, so the
-    // namespace follows the config.
-    async prepare(namespace) {
+    // are admitted under a quota on compute. All of it is replaced when the server sets up
+    // a namespace it did not set up in the last `NAMESPACE_SETUP_REUSE_MS` (it reads the
+    // config once, at its start), so the namespace follows the config; a namespace made anew
+    // since (deleted and made again) is set up again at once.
+    async prepare(namespace, 
+    // a service run's own namespace: its label, the namespace of the server that reaches its
+    // endpoints, and its quota
+    serviceRun) {
         const backend = this.backend;
         let existing;
         try {
@@ -393,15 +613,26 @@ class CallerNamespaces {
         if (existing && existing.metadata?.labels?.[MANAGED_LABEL] !== 'true') {
             throw new Error(`the namespace ${namespace} was not made by hammerkit, the server does not run tasks in it`);
         }
+        const started = Date.now();
+        const applied = this.applied.get(namespace);
+        if (existing &&
+            !existing.metadata?.deletionTimestamp &&
+            applied &&
+            applied.uid === existing.metadata?.uid &&
+            started - applied.at < NAMESPACE_SETUP_REUSE_MS) {
+            return;
+        }
+        this.applied.delete(namespace);
         if (!existing) {
             try {
-                await this.instance.coreApi.createNamespace({
+                existing = await this.instance.coreApi.createNamespace({
                     body: {
                         metadata: {
                             name: namespace,
                             labels: {
                                 [MANAGED_LABEL]: 'true',
                                 'hammerkit.dev/caller': namespace,
+                                ...(serviceRun ? { [exports.SERVICE_RUN_LABEL]: serviceRun.serviceRun } : {}),
                                 // no privileged pods, host paths or host network, whatever a build asks
                                 'pod-security.kubernetes.io/enforce': 'baseline',
                             },
@@ -419,7 +650,10 @@ class CallerNamespaces {
         const [first, ...policies] = getNamespacePolicies(namespace, backend.deniedEgress, backend.allowedEgress);
         // the API server may refuse what the binding allows for a moment after it was made
         await untilAllowed(() => (0, apply_1.replace)(this.instance, first));
-        for (const policy of policies) {
+        for (const policy of [
+            ...policies,
+            ...(serviceRun ? [getServerIngressPolicy(namespace, serviceRun.server, serviceRun.ports)] : []),
+        ]) {
             await (0, apply_1.replace)(this.instance, policy);
         }
         const metadata = {
@@ -432,7 +666,7 @@ class CallerNamespaces {
             apiVersion: 'v1',
             kind: 'ResourceQuota',
             metadata,
-            spec: { hard: backend.quota },
+            spec: { hard: serviceRun?.quota ?? backend.quota },
         };
         const defaults = backend.defaultResources;
         const limits = {
@@ -450,7 +684,7 @@ class CallerNamespaces {
             },
         };
         for (const [object, wanted] of [
-            [quota, !!backend.quota],
+            [quota, !!(serviceRun?.quota ?? backend.quota)],
             [limits, !!defaults],
         ]) {
             if (wanted) {
@@ -465,6 +699,10 @@ class CallerNamespaces {
                 });
             }
         }
+        const uid = existing?.metadata?.uid;
+        if (uid) {
+            this.applied.set(namespace, { uid, at: started });
+        }
     }
     whoAmI() {
         if (!this.subject) {
@@ -487,7 +725,7 @@ class CallerNamespaces {
 exports.CallerNamespaces = CallerNamespaces;
 // The claim of a sandbox as the Jobs of its run mount it: the run's copy of the files is
 // `root`, a source below it is the same path below `subPath` in the claim.
-function getWorkspace(root, claim, namespace) {
+function getWorkspace(root, claim, namespace, runner) {
     if (claim.namespace !== namespace) {
         throw new Error(`the sandbox is in ${claim.namespace}, the run in ${namespace}`);
     }
@@ -497,6 +735,7 @@ function getWorkspace(root, claim, namespace) {
         subPath: claim.subPath,
         label: claim.label,
         ...(claim.pod ? { pod: claim.pod } : {}),
+        ...(runner ? { runner: true } : {}),
     };
 }
 // Everything a run made in the caller's namespace, by the label selector of the run: