hammerkit 1.27.0-next.8 → 1.27.0-next.9

package/dist/kubernetes/apply.js

@@ -36,28 +36,39 @@ exports.apply = apply;
  * of a policy).
  */
 async function replace(instance, spec) {
-    let current;
-    try {
-        current = await instance.objectApi.read(spec);
-    }
-    catch (e) {
-        if (statusCodeOf(e) !== 404) {
-            throw e;
+    for (let attempt = 1;; attempt++) {
+        let current;
+        try {
+            current = await instance.objectApi.read(spec);
+        }
+        catch (e) {
+            if (statusCodeOf(e) !== 404) {
+                throw e;
+            }
+            try {
+                return await instance.objectApi.create(spec);
+            }
+            catch (createError) {
+                if (statusCodeOf(createError) !== 409) {
+                    throw createError;
+                }
+                current = await instance.objectApi.read(spec);
+            }
         }
         try {
-            return await instance.objectApi.create(spec);
+            return await instance.objectApi.replace({
+                ...spec,
+                metadata: { ...spec.metadata, resourceVersion: current.metadata?.resourceVersion },
+            });
         }
-        catch (createError) {
-            if (statusCodeOf(createError) !== 409) {
-                throw createError;
+        catch (e) {
+            // replaced by someone else since it was read (the server sets up a caller's
+            // namespace for a run and for a sandbox at the same time): read it again
+            if (statusCodeOf(e) !== 409 || attempt >= 3) {
+                throw e;
             }
-            current = await instance.objectApi.read(spec);
         }
     }
-    return await instance.objectApi.replace({
-        ...spec,
-        metadata: { ...spec.metadata, resourceVersion: current.metadata?.resourceVersion },
-    });
 }
 exports.replace = replace;
 //# sourceMappingURL=apply.js.map

package/dist/kubernetes/ensure-http-route.js

@@ -36,7 +36,7 @@ async function ensureHttpRoute(instance, env, route, service) {
                     ],
                     backendRefs: [
                         {
-                            name: (0, resources_1.getResourceName)(service),
+                            name: (0, resources_1.getResourceName)(service, env),
                             port: route.servicePort ?? service.data.ports[0].containerPort,
                         },
                     ],

package/dist/kubernetes/ensure-kubernetes-deployment-exists.js

@@ -21,7 +21,7 @@ async function ensureKubernetesDeploymentExists(instance, env, service, persiste
 // the needed services by name, and the proxy of a limited network
 network) {
     const envs = (0, replace_env_variables_1.getEnvironmentVariables)(service.data.envs);
-    const name = (0, resources_1.getResourceName)(service);
+    const name = (0, resources_1.getResourceName)(service, env);
     const healthcheckCommand = (0, work_service_1.getHealthcheckCommand)(service.data);
     const serviceCommand = (0, work_service_1.getServiceCommand)(service.data, { kubernetes: true });
     const probe = healthcheckCommand
@@ -47,25 +47,21 @@ network) {
                 'hammerkit.dev/version': (0, version_1.getVersion)(),
             },
             labels: {
-                'hammerkit.dev/id': service.id(),
+                ...(0, resources_1.getItemLabels)(service, env),
                 'hammerkit.dev/state': stateKey,
-                ...(0, job_placement_1.getRunLabels)(env),
                 [exports.DEFINITION_LABEL]: (0, service_definition_1.getServiceDefinitionHash)(service),
             },
         },
         spec: {
             selector: {
-                matchLabels: {
-                    'hammerkit.dev/id': service.id(),
-                },
+                matchLabels: (0, resources_1.getItemLabels)(service, env),
             },
             replicas: 1,
             template: {
                 metadata: {
                     labels: {
-                        'hammerkit.dev/id': service.id(),
+                        ...(0, resources_1.getItemLabels)(service, env),
                         'hammerkit.dev/state': stateKey,
-                        ...(0, job_placement_1.getRunLabels)(env),
                         ...workspace.labels,
                         ...(0, network_policy_1.getNetworkLabels)(env, service),
                     },
@@ -127,7 +123,7 @@ network) {
         return;
     }
     await (0, await_running_state_1.awaitDeployRunningState)(instance, env, name, {
-        podSelector: `hammerkit.dev/id=${service.id()}`,
+        podSelector: (0, resources_1.getItemSelector)(service, env),
         timeout: service.data.healthcheck?.timeout ?? null,
         timeoutMessage: service.data.healthcheck
             ? (0, work_healthcheck_1.getHealthcheckTimeoutMessage)(service.name, service.data.healthcheck)

package/dist/kubernetes/ensure-kubernetes-service-exists.js

@@ -4,11 +4,10 @@ exports.ensureKubernetesServiceExists = void 0;
 const apply_1 = require("./apply");
 const resources_1 = require("./resources");
 const version_1 = require("../version");
-const job_placement_1 = require("./job-placement");
 // A service without ports gets a headless Service: a ClusterIP one needs a port,
 // and its tasks reach every port of the pod, as on docker.
 async function ensureKubernetesServiceExists(instance, env, service) {
-    const name = (0, resources_1.getResourceName)(service);
+    const name = (0, resources_1.getResourceName)(service, env);
     const headless = service.data.ports.length === 0;
     const svc = {
         kind: 'Service',
@@ -19,17 +18,13 @@ async function ensureKubernetesServiceExists(instance, env, service) {
             annotations: {
                 'hammerkit.dev/version': (0, version_1.getVersion)(),
             },
-            labels: {
-                'hammerkit.dev/id': service.id(),
-                ...(0, job_placement_1.getRunLabels)(env),
-            },
+            labels: (0, resources_1.getItemLabels)(service, env),
         },
         spec: {
             type: 'ClusterIP',
             ...(headless ? { clusterIP: 'None' } : {}),
-            selector: {
-                'hammerkit.dev/id': service.id(),
-            },
+            // the pods of the service of this run, on a server where runs share the namespace
+            selector: (0, resources_1.getItemLabels)(service, env),
             ...(headless
                 ? {}
                 : {

package/dist/kubernetes/ensure-persistent-data.js

@@ -67,17 +67,14 @@ async function getPodForPersistence(instance, env, service, persistence, type, f
     for (const volume of persistence.volumes) {
         await (0, ensure_kubernetes_persistent_volume_claim_exists_1.ensureKubernetesPersistentVolumeClaimExists)(instance, env, volume, service, persistence.sizes[volume.name], persistence.kept.includes(volume.name));
     }
-    const name = (0, resources_1.getResourceName)(service, type === 'write' ? '-upload' : '-download');
+    const name = (0, resources_1.getResourceName)(service, env, type === 'write' ? '-upload' : '-download');
     const podSpec = {
         kind: 'Pod',
         apiVersion: 'v1',
         metadata: {
             namespace: env.namespace,
             name,
-            labels: {
-                'hammerkit.dev/id': service.id(),
-                ...(0, job_placement_1.getRunLabels)(env),
-            },
+            labels: (0, resources_1.getItemLabels)(service, env),
             annotations: {
                 'hammerkit.dev/version': (0, version_1.getVersion)(),
             },

package/dist/kubernetes/get-service-ip.js

@@ -5,7 +5,7 @@ const resources_1 = require("./resources");
 // The address a task reaches the service on: its clusterIP, or for a headless
 // Service (a service without ports) the address of its ready pod.
 async function getServiceIp(instance, env, workService) {
-    const name = (0, resources_1.getResourceName)(workService);
+    const name = (0, resources_1.getResourceName)(workService, env);
     const svc = await instance.coreApi.readNamespacedService({ name, namespace: env.namespace });
     const clusterIP = svc.spec?.clusterIP ?? null;
     if (clusterIP !== 'None') {
@@ -13,7 +13,7 @@ async function getServiceIp(instance, env, workService) {
     }
     const pods = await instance.coreApi.listNamespacedPod({
         namespace: env.namespace,
-        labelSelector: `hammerkit.dev/id=${workService.id()}`,
+        labelSelector: (0, resources_1.getItemSelector)(workService, env),
     });
     const ready = pods.items.find((pod) => !pod.metadata?.deletionTimestamp &&
         pod.status?.podIP &&

package/dist/kubernetes/job-secrets.js

@@ -65,8 +65,8 @@ exports.getSecretHosts = getSecretHosts;
 function accountsOf(secrets) {
     return [...new Set(secrets.map((s) => s.source.account))];
 }
-function getCredentialsName(item) {
-    return (0, resources_1.getResourceName)(item, '-credentials');
+function getCredentialsName(item, kubernetes) {
+    return (0, resources_1.getResourceName)(item, kubernetes, '-credentials');
 }
 exports.getCredentialsName = getCredentialsName;
 // The keys of the credentials Secret: `<account>.token` and `<account>.file.<key>`
@@ -136,9 +136,9 @@ async function ensureJobCredentials(instance, kubernetes, item, secrets, admitte
         apiVersion: 'v1',
         metadata: {
             namespace: kubernetes.namespace,
-            name: getCredentialsName(item),
+            name: getCredentialsName(item, kubernetes),
             annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
-            labels: { 'hammerkit.dev/id': item.id(), ...(0, job_placement_1.getRunLabels)(kubernetes) },
+            labels: (0, resources_1.getItemLabels)(item, kubernetes),
         },
         type: 'Opaque',
         data,
@@ -169,7 +169,10 @@ async function removeJobCredentials(instance, kubernetes, item) {
         return;
     }
     try {
-        await instance.coreApi.deleteNamespacedSecret({ name: getCredentialsName(item), namespace: kubernetes.namespace });
+        await instance.coreApi.deleteNamespacedSecret({
+            name: getCredentialsName(item, kubernetes),
+            namespace: kubernetes.namespace,
+        });
     }
     catch (e) {
         if ((0, apply_1.statusCodeOf)(e) !== 404) {
@@ -213,7 +216,7 @@ proxyEnv = []) {
         }
         const name = `hammerkit-account-${index}`;
         accountVolume.set(account, name);
-        volumes.push({ name, secret: { secretName: getCredentialsName(item), items, defaultMode: 0o444 } });
+        volumes.push({ name, secret: { secretName: getCredentialsName(item, { server }), items, defaultMode: 0o444 } });
     });
     const initContainers = secrets.map(({ index, secret, source }) => {
         const label = `secret ${(0, work_secret_1.getSecretName)(secret)} (${(0, work_secret_1.describeSecretSource)(secret)})`;

package/dist/kubernetes/kept-claims.js

@@ -5,6 +5,7 @@ const crypto_1 = require("crypto");
 const apply_1 = require("./apply");
 const sleep_1 = require("../utils/sleep");
 const job_placement_1 = require("./job-placement");
+const resources_1 = require("./resources");
 // On a server with `backend.keepVolumes`, the claim of a task stays after the run, so the
 // next run of the same source finds the outputs in it and mounts them without restoring
 // them from the cache. A kept claim has no run label (the end of the run removes what has
@@ -18,11 +19,15 @@ exports.STATE_ANNOTATION = 'hammerkit.dev/state';
 // when it goes, set by the end of the run that mounted it last
 exports.EXPIRES_ANNOTATION = 'hammerkit.dev/expires';
 // The claim of a work item: a task's on a server that keeps them carries the scope, so a
-// run of another source never mounts it. 63 characters at most, a pod's volume name.
+// run of another source never mounts it, and the runs of one scope go one at a time
+// (KubernetesExecutor.exclusive). Any other claim on a server is the run's own, as the
+// runs of a caller share their namespace. 63 characters at most, a pod's volume name.
 function getClaimName(work, kubernetes) {
-    const scope = kubernetes?.server?.volumeScope;
-    const name = 'hammerkit-' + work.id();
-    return scope && isTask(work) ? `${name}-${(0, crypto_1.createHash)('sha256').update(scope).digest('hex').slice(0, 12)}` : name;
+    const scope = kubernetes.server?.volumeScope;
+    if (scope && isTask(work)) {
+        return `hammerkit-${work.id()}-${(0, crypto_1.createHash)('sha256').update(scope).digest('hex').slice(0, 12)}`;
+    }
+    return 'hammerkit-' + (0, resources_1.getItemKey)(work, kubernetes);
 }
 exports.getClaimName = getClaimName;
 // whether the claim of the work item stays after the run
@@ -156,14 +161,17 @@ async function expireKeptClaims(instance, namespace, runLabel, expires) {
     }
 }
 exports.expireKeptClaims = expireKeptClaims;
-// While no run of the caller is in the namespace: the kept claims that expired, and those
-// a run left without an expiry because the server stopped before its end (all of them
-// when the server keeps none any more). The names of those removed, and when the next of
-// the others expires.
+// The kept claims that expired, and those a run left without an expiry because the server
+// stopped before its end (all of them when the server keeps none any more), but those of
+// the scopes `inUse`: a run of the scope executes and mounts them, or may in a moment.
+// The names of those removed, and when the next of the others expires.
 async function sweepKeptClaims(instance, namespace, options) {
     const expired = [];
     let next = null;
     for (const claim of await listKeptClaims(instance, namespace)) {
+        if (options.inUse?.has(claim.metadata?.labels?.[exports.VOLUME_SCOPE_LABEL] ?? '')) {
+            continue;
+        }
         const expires = Date.parse(claim.metadata?.annotations?.[exports.EXPIRES_ANNOTATION] ?? '');
         if (!options.keep || !(expires > options.now.getTime())) {
             expired.push(claim);
@@ -178,14 +186,18 @@ async function sweepKeptClaims(instance, namespace, options) {
 exports.sweepKeptClaims = sweepKeptClaims;
 // When a run needs room under the quota of its namespace: the kept claims it does not use
 // go, those of the other sources and those of its own a task of this run did not take (one
-// renamed or changed since). The names of those removed.
+// renamed or changed since). Of another scope, only one its last run released, which has an
+// expiry: one without is mounted by a run of the caller that executes now (or was left by
+// a server that stopped, which the next sweep removes). The names of those removed.
 async function evictKeptClaims(instance, kubernetes, timeout) {
     const server = kubernetes.server;
     if (!server?.volumeScope) {
         return [];
     }
     const unused = await listKeptClaims(instance, kubernetes.namespace, `${exports.VOLUME_SCOPE_LABEL},${exports.KEPT_BY_LABEL}!=${(0, job_placement_1.getRunLabelValue)(server.run)}`);
-    return removeClaims(instance, kubernetes.namespace, unused, timeout);
+    const released = unused.filter((claim) => claim.metadata?.labels?.[exports.VOLUME_SCOPE_LABEL] === server.volumeScope ||
+        !!claim.metadata?.annotations?.[exports.EXPIRES_ANNOTATION]);
+    return removeClaims(instance, kubernetes.namespace, released, timeout);
 }
 exports.evictKeptClaims = evictKeptClaims;
 //# sourceMappingURL=kept-claims.js.map

package/dist/kubernetes/network-policy.js

@@ -17,7 +17,7 @@ const sleep_1 = require("../utils/sleep");
 const network_gate_1 = require("./network-gate");
 const job_placement_1 = require("./job-placement");
 Object.defineProperty(exports, "RUN_LABEL", { enumerable: true, get: function () { return job_placement_1.RUN_LABEL; } });
-const ID_LABEL = 'hammerkit.dev/id';
+const resources_1 = require("./resources");
 const PROXY_LABEL = 'hammerkit.dev/proxy-for';
 const NETWORK_OF_LABEL = 'hammerkit.dev/network-of';
 // on the proxy of a task on a sandbox: the sandbox, whose pod of files it stays with
@@ -86,7 +86,7 @@ function getNetworkPolicyName(item, scope) {
 exports.getNetworkPolicyName = getNetworkPolicyName;
 // the pods of the item, of the run when the scope has one
 function getItemPodLabels(item, scope) {
-    return { [ID_LABEL]: item.id(), ...(scope.run ? { [job_placement_1.RUN_LABEL]: scope.run } : {}) };
+    return { [resources_1.ID_LABEL]: item.id(), ...(scope.run ? { [job_placement_1.RUN_LABEL]: scope.run } : {}) };
 }
 function toSelector(labels) {
     return Object.entries(labels)
@@ -107,7 +107,12 @@ function getIngress(env, item) {
     if (ids.length === 0) {
         return [];
     }
-    const from = { matchExpressions: [{ key: ID_LABEL, operator: 'In', values: ids }] };
+    // on a server, of this run: the runs of a caller share their namespace
+    const run = (0, job_placement_1.getRunLabels)(env);
+    const from = {
+        matchExpressions: [{ key: resources_1.ID_LABEL, operator: 'In', values: ids }],
+        ...(Object.keys(run).length > 0 ? { matchLabels: run } : {}),
+    };
     // the client's name for `from`, a reserved word
     return [{ _from: [{ podSelector: from }] }];
 }
@@ -132,7 +137,7 @@ function buildNetworkPolicy(env, item, network, scope, proxy = getRunProxy(item,
             ingress: getIngress(env, item),
             egress: [
                 ...(needs.length > 0
-                    ? [{ to: needs.map((need) => ({ podSelector: { matchLabels: { [ID_LABEL]: need.service.id() } } })) }]
+                    ? [{ to: needs.map((need) => ({ podSelector: { matchLabels: (0, resources_1.getItemLabels)(need.service, env) } })) }]
                     : []),
                 ...(proxied
                     ? [
@@ -306,12 +311,14 @@ async function releaseKubernetesItemNetwork(instance, env, item, scope) {
 }
 exports.releaseKubernetesItemNetwork = releaseKubernetesItemNetwork;
 // Every policy and proxy of the item, of any run, once none of its pods runs:
-// what `down` and `clean` remove.
+// what `down` and `clean` remove. On a server, those of the server's run only: the
+// runs of a caller share their namespace, and the pods of another run of the same item
+// would run unlimited without their policy.
 async function removeAllKubernetesItemNetworks(instance, env, item) {
-    if (!(await awaitPodsEnded(instance, env, { [ID_LABEL]: item.id() }))) {
+    if (!(await awaitPodsEnded(instance, env, (0, resources_1.getItemLabels)(item, env)))) {
         throw new Error(`pods of ${item.name} are still running, so its network policy stays`);
     }
-    const labelSelector = `${NETWORK_OF_LABEL}=${item.id()}`;
+    const labelSelector = toSelector({ [NETWORK_OF_LABEL]: item.id(), ...(0, job_placement_1.getRunLabels)(env) });
     let policies;
     try {
         const list = await instance.objectApi.list('networking.k8s.io/v1', 'NetworkPolicy', env.namespace, undefined, undefined, undefined, undefined, labelSelector);

package/dist/kubernetes/remove-persistent-data.js

@@ -1,8 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.removePersistentData = void 0;
+const resources_1 = require("./resources");
 async function removePersistentData(instance, kubernetes, task) {
-    const labelSelector = `hammerkit.dev/id=${task.id()}`;
+    // on a server, the run's: a claim it kept has no run label and goes once it expired
+    const labelSelector = (0, resources_1.getItemSelector)(task, kubernetes);
     const configmaps = await instance.coreApi.listNamespacedConfigMap({ namespace: kubernetes.namespace, labelSelector });
     for (const configMap of configmaps.items) {
         if (configMap.metadata?.name) {

package/dist/kubernetes/resources.js

@@ -1,12 +1,34 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.removeInvalidCharacters = exports.getResourceName = void 0;
+exports.removeInvalidCharacters = exports.getResourceName = exports.getItemSelector = exports.getItemLabels = exports.getItemKey = exports.ID_LABEL = void 0;
+const job_placement_1 = require("./job-placement");
 // An object name is a DNS label: lowercase letters, digits and `-`, at most 63
 // characters, starting and ending with a letter or digit. The id makes a name
 // unique; the item's name is there for people and is cut to fit.
 const MAX_NAME = 63;
-function getResourceName(item, suffix) {
-    const tail = `-${item.id()}${suffix ?? ''}`;
+exports.ID_LABEL = 'hammerkit.dev/id';
+// What tells the objects of an item apart in its namespace: its id, and on a server the
+// run's label too. The runs of a caller execute side by side in their namespace, and two
+// of them often run the same task (the same command on two sandboxes of a repository),
+// so each has objects of its own, which nothing of the other run finds or removes. The
+// first 8 digits of the run label tell runs apart, as they do for a run's proxies.
+function getItemKey(item, env) {
+    return env.server ? `${item.id().substring(0, 32)}-${(0, job_placement_1.getRunLabelValue)(env.server.run).substring(0, 8)}` : item.id();
+}
+exports.getItemKey = getItemKey;
+// The labels the objects of an item carry, by which they are found again
+function getItemLabels(item, env) {
+    return { [exports.ID_LABEL]: item.id(), ...(0, job_placement_1.getRunLabels)(env) };
+}
+exports.getItemLabels = getItemLabels;
+function getItemSelector(item, env) {
+    return Object.entries(getItemLabels(item, env))
+        .map(([key, value]) => `${key}=${value}`)
+        .join(',');
+}
+exports.getItemSelector = getItemSelector;
+function getResourceName(item, env, suffix) {
+    const tail = `-${getItemKey(item, env)}${suffix ?? ''}`;
     const name = removeInvalidCharacters(item.name)
         .slice(0, Math.max(0, MAX_NAME - tail.length))
         .replace(/-+$/, '');

package/dist/kubernetes/secrets.js

@@ -5,19 +5,18 @@ const work_secret_1 = require("../planner/work-secret");
 const apply_1 = require("./apply");
 const resources_1 = require("./resources");
 const version_1 = require("../version");
-const job_placement_1 = require("./job-placement");
 const SECRET_VOLUME_NAME = 'hammerkit-secrets';
-function getSecretResourceName(item) {
-    return (0, resources_1.getResourceName)(item, '-secrets');
+function getSecretResourceName(item, kubernetes) {
+    return (0, resources_1.getResourceName)(item, kubernetes, '-secrets');
 }
 exports.getSecretResourceName = getSecretResourceName;
 // The values go into a Secret of the item; the pod only references it, so no
 // value is part of a Job or Deployment spec.
-function getKubernetesSecretRefs(item, secrets) {
+function getKubernetesSecretRefs(item, kubernetes, secrets) {
     if (secrets.length === 0) {
         return { env: [], mounts: [], volumes: [] };
     }
-    const secretName = getSecretResourceName(item);
+    const secretName = getSecretResourceName(item, kubernetes);
     const refs = {
         env: [],
         mounts: [],
@@ -50,14 +49,11 @@ async function ensureKubernetesSecret(instance, kubernetes, item, secrets, envir
         apiVersion: 'v1',
         metadata: {
             namespace: kubernetes.namespace,
-            name: getSecretResourceName(item),
+            name: getSecretResourceName(item, kubernetes),
             annotations: {
                 'hammerkit.dev/version': (0, version_1.getVersion)(),
             },
-            labels: {
-                'hammerkit.dev/id': item.id(),
-                ...(0, job_placement_1.getRunLabels)(kubernetes),
-            },
+            labels: (0, resources_1.getItemLabels)(item, kubernetes),
         },
         type: 'Opaque',
         stringData,
@@ -68,7 +64,7 @@ exports.ensureKubernetesSecret = ensureKubernetesSecret;
 async function removeKubernetesSecret(instance, kubernetes, item) {
     try {
         await instance.coreApi.deleteNamespacedSecret({
-            name: getSecretResourceName(item),
+            name: getSecretResourceName(item, kubernetes),
             namespace: kubernetes.namespace,
         });
     }

package/dist/kubernetes/volumes.js

@@ -51,7 +51,7 @@ async function getKubernetesPersistence(work, kubernetes) {
         mounts: [],
         initCommands: [],
     };
-    const workspace = kubernetes?.server?.workspace;
+    const workspace = kubernetes.server?.workspace;
     for (const mount of work.data.mounts) {
         if (workspace &&
             appendWorkspaceMount(workspace, persistence, mount.containerPath, mount.localPath, mount.isFile, mount.readOnly)) {
@@ -94,7 +94,7 @@ function appendVolume(sourceWork, persistence, containerPath, localPath, isFile,
             },
         });
         persistence.sizes[volumeName] = 'volumeSize' in sourceWork.data ? sourceWork.data.volumeSize : null;
-        if (kubernetes && (0, kept_claims_1.isKeptClaim)(sourceWork, kubernetes)) {
+        if ((0, kept_claims_1.isKeptClaim)(sourceWork, kubernetes)) {
             persistence.kept.push(volumeName);
         }
     }

package/dist/planner/work-runtime-kubernetes.js

@@ -38,7 +38,7 @@ const work_secret_1 = require("./work-secret");
 async function listJobNames(instance, kubernetes, task) {
     const jobs = await instance.batchApi.listNamespacedJob({
         namespace: kubernetes.namespace,
-        labelSelector: `hammerkit.dev/id=${task.id()}`,
+        labelSelector: (0, resources_1.getItemSelector)(task, kubernetes),
     });
     return jobs.items.flatMap((job) => (job.metadata?.name ? [job.metadata.name] : []));
 }
@@ -201,7 +201,7 @@ function kubernetesTaskRuntime(task, kubernetes) {
             }
             const server = kubernetes.server;
             const { delivered, inJob } = (0, job_secrets_1.splitJobSecrets)(task.data.secrets, server);
-            const secrets = (0, secrets_1.getKubernetesSecretRefs)(task, delivered);
+            const secrets = (0, secrets_1.getKubernetesSecretRefs)(task, kubernetes, delivered);
             const admitted = environment.hosted?.accounts;
             let secretHosts;
             try {
@@ -230,7 +230,7 @@ function kubernetesTaskRuntime(task, kubernetes) {
                     : {}),
             };
             const labels = {
-                'hammerkit.dev/id': task.id(),
+                ...(0, resources_1.getItemLabels)(task, kubernetes),
                 'hammerkit.dev/state': options.stateKey,
                 ...workspace.labels,
                 [network_policy_1.RUN_LABEL]: scope.run,
@@ -258,7 +258,7 @@ function kubernetesTaskRuntime(task, kubernetes) {
                 return;
             }
             try {
-                const podName = (0, resources_1.getResourceName)(task, '-job');
+                const podName = (0, resources_1.getResourceName)(task, kubernetes, '-job');
                 let i = 0;
                 for (const cmd of task.data.cmds) {
                     const container = `cmd-${++i}`;
@@ -387,7 +387,7 @@ function kubernetesTaskRuntime(task, kubernetes) {
         async stop() {
             const jobs = await instance.batchApi.listNamespacedJob({
                 namespace: kubernetes.namespace,
-                labelSelector: `hammerkit.dev/id=${task.id()}`,
+                labelSelector: (0, resources_1.getItemSelector)(task, kubernetes),
             });
             for (const job of jobs.items) {
                 if (job.metadata?.name) {
@@ -396,7 +396,7 @@ function kubernetesTaskRuntime(task, kubernetes) {
             }
             const deployments = await instance.appsApi.listNamespacedDeployment({
                 namespace: kubernetes.namespace,
-                labelSelector: `hammerkit.dev/id=${task.id()}`,
+                labelSelector: (0, resources_1.getItemSelector)(task, kubernetes),
             });
             for (const deploy of deployments.items) {
                 if (deploy.metadata?.name) {
@@ -408,7 +408,7 @@ function kubernetesTaskRuntime(task, kubernetes) {
             }
             const pods = await instance.coreApi.listNamespacedPod({
                 namespace: kubernetes.namespace,
-                labelSelector: `hammerkit.dev/id=${task.id()}`,
+                labelSelector: (0, resources_1.getItemSelector)(task, kubernetes),
             });
             for (const pod of pods.items) {
                 if (pod.metadata?.name) {
@@ -430,7 +430,7 @@ function kubernetesTaskRuntime(task, kubernetes) {
             }
             const jobs = await instance.batchApi.listNamespacedJob({
                 namespace: kubernetes.namespace,
-                labelSelector: `hammerkit.dev/id=${task.id()}`,
+                labelSelector: (0, resources_1.getItemSelector)(task, kubernetes),
             });
             const completedStates = jobs.items
                 .filter((j) => j.status?.succeeded && j.metadata?.labels?.['hammerkit.dev/state'])
@@ -467,7 +467,7 @@ function kubernetesServiceRuntime(service, kubernetes) {
         async initialize(state) {
             const deployments = await instance.appsApi.listNamespacedDeployment({
                 namespace: kubernetes.namespace,
-                labelSelector: `hammerkit.dev/id=${service.id()}`,
+                labelSelector: (0, resources_1.getItemSelector)(service, kubernetes),
             });
             const deployment = deployments.items[0];
             if (!deployment?.metadata?.name) {
@@ -494,7 +494,7 @@ function kubernetesServiceRuntime(service, kubernetes) {
         async stop() {
             const deployments = await instance.appsApi.listNamespacedDeployment({
                 namespace: kubernetes.namespace,
-                labelSelector: `hammerkit.dev/id=${service.id()}`,
+                labelSelector: (0, resources_1.getItemSelector)(service, kubernetes),
             });
             for (const deployment of deployments.items) {
                 if (deployment.metadata?.name) {
@@ -514,9 +514,13 @@ function kubernetesServiceRuntime(service, kubernetes) {
             await (0, ensure_kubernetes_service_exists_1.ensureKubernetesServiceExists)(instance, kubernetes, service);
             await (0, ensure_persistent_data_1.ensurePersistentData)(instance, kubernetes, environment, service, persistence);
             await (0, secrets_1.ensureKubernetesSecret)(instance, kubernetes, service, service.data.secrets, environment);
-            const network = await (0, network_policy_1.ensureKubernetesItemNetwork)(instance, kubernetes, service, { followLog: false, run: null });
-            await (0, ensure_kubernetes_deployment_exists_1.ensureKubernetesDeploymentExists)(instance, kubernetes, service, persistence, (0, secrets_1.getKubernetesSecretRefs)(service, service.data.secrets), options.stateKey, network);
-            const name = (0, resources_1.getResourceName)(service);
+            // on a server, the run's: its runs share the namespace, and the service goes with the run
+            const network = await (0, network_policy_1.ensureKubernetesItemNetwork)(instance, kubernetes, service, {
+                followLog: false,
+                run: kubernetes.server ? (0, job_placement_1.getRunLabelValue)(kubernetes.server.run) : null,
+            });
+            await (0, ensure_kubernetes_deployment_exists_1.ensureKubernetesDeploymentExists)(instance, kubernetes, service, persistence, (0, secrets_1.getKubernetesSecretRefs)(service, kubernetes, service.data.secrets), options.stateKey, network);
+            const name = (0, resources_1.getResourceName)(service, kubernetes);
             options.state.set({
                 type: 'running',
                 dns: { containerId: name },
@@ -548,7 +552,7 @@ function kubernetesServiceRuntime(service, kubernetes) {
             }
             const services = await instance.coreApi.listNamespacedService({
                 namespace: kubernetes.namespace,
-                labelSelector: `hammerkit.dev/id=${service.id()}`,
+                labelSelector: (0, resources_1.getItemSelector)(service, kubernetes),
             });
             for (const service of services.items) {
                 if (service.metadata?.name) {
@@ -564,7 +568,7 @@ function kubernetesServiceRuntime(service, kubernetes) {
         async currentStateKey() {
             const deployments = await instance.appsApi.listNamespacedDeployment({
                 namespace: kubernetes.namespace,
-                labelSelector: `hammerkit.dev/id=${service.id()}`,
+                labelSelector: (0, resources_1.getItemSelector)(service, kubernetes),
             });
             const deployment = deployments.items[0];
             return deployment?.metadata?.labels?.['hammerkit.dev/state'] ?? null;

package/dist/server/kubernetes-backend.js

@@ -167,18 +167,21 @@ exports.getRunRoleBinding = getRunRoleBinding;
  * process of the server as on a host (see HostExecutor); what changes is where its
  * container tasks and services go: into the caller's namespace, on the nodes of the
  * operator's pools, with the secrets of providers read in the Job itself. The runs
- * of one caller execute one after the other, since the objects of a task are named
- * by its cache id in the caller's namespace; different callers run side by side.
- * When a run ends, everything made for it is removed, but for the claims of its tasks
- * the server keeps (`backend.keepVolumes`), which go once they expired.
+ * of a caller execute side by side in their namespace, up to `runs.limits.running`:
+ * the objects of a run are named and found by its run label (getItemKey), so two runs
+ * of the same task have their own. What two runs share are the claims the server keeps
+ * (`backend.keepVolumes`) of a scope (getVolumeScope): the runs of one scope go one at
+ * a time (exclusive). When a run ends, everything made for it is removed, but for the
+ * kept claims, which go once they expired.
  */
 class KubernetesExecutor {
     constructor(config, processEnvs, options) {
         this.config = config;
-        // the registry keeps a caller's next run in its queue, not running and timed here
-        this.runningPerCaller = 1;
-        // the run that executes, or the last one waiting, per caller namespace
+        // per caller namespace, what changes objects of the whole namespace: its setup, the
+        // removal of what a stopped server left, the sweeps of kept claims. The last one waiting.
         this.queues = new Map();
+        // per caller namespace, the runs that execute in it: by run label, their volume scope
+        this.active = new Map();
         // per caller namespace, when its kept claims are next looked at
         this.sweeps = new Map();
         // whether the kept claims a server left when it stopped were looked for
@@ -203,17 +206,45 @@ class KubernetesExecutor {
         this.log = options.log;
         this.host = new executor_1.HostExecutor({ ...options, placement: this.placement() });
     }
-    execute(run, context) {
+    async execute(run, context) {
         const namespace = getCallerNamespace(this.config.backend.namespacePrefix, run.owner);
-        return this.enqueue(namespace, () => {
-            if (context.signal.aborted) {
-                throw new Error('the run was cancelled');
+        const label = (0, job_placement_1.getRunLabelValue)(run.id);
+        let runs = this.active.get(namespace);
+        if (!runs) {
+            runs = new Map();
+            this.active.set(namespace, runs);
+        }
+        runs.set(label, getVolumeScope(run));
+        try {
+            await this.host.execute(run, context);
+        }
+        finally {
+            runs.delete(label);
+            if (runs.size === 0 && this.active.get(namespace) === runs) {
+                this.active.delete(namespace);
             }
-            return this.host.execute(run, context);
-        });
+        }
+    }
+    // The runs of a scope share its kept claims, which one of them may change at a time:
+    // what a task finds in one is what the last run left there. Without kept claims, runs
+    // share no object and nothing is held.
+    exclusive(run) {
+        if (this.keep === 0) {
+            return undefined;
+        }
+        return `${getCallerNamespace(this.config.backend.namespacePrefix, run.owner)}/${getVolumeScope(run)}`;
     }
-    // What happens in a caller's namespace happens one at a time: their runs, and the
-    // removal of the claims their runs kept
+    // The scopes of the runs that execute in the namespace, but `except`'s: their kept claims
+    // are theirs to mount and stay. A server that keeps none mounts none.
+    scopesInUse(namespace, except) {
+        if (this.keep === 0) {
+            return new Set();
+        }
+        return new Set([...(this.active.get(namespace) ?? new Map()).entries()]
+            .filter(([label]) => label !== except)
+            .map(([, scope]) => scope));
+    }
+    // What changes objects of a whole namespace happens one at a time, not what its runs do
     enqueue(namespace, fn) {
         const previous = this.queues.get(namespace) ?? Promise.resolve();
         const execution = previous.then(fn);
@@ -231,19 +262,28 @@ class KubernetesExecutor {
         return {
             prepare: async (run, context, work) => {
                 const namespace = getCallerNamespace(backend.namespacePrefix, run.owner);
-                await this.namespaces.prepare(namespace);
-                // what an earlier run left when the server stopped before its end; the runs
-                // of a caller execute one at a time, so no other run is in the namespace
-                await removeRunObjects(this.instance, namespace, `${job_placement_1.RUN_LABEL},${job_placement_1.RUN_LABEL}!=${(0, job_placement_1.getRunLabelValue)(run.id)}`, RELEASE_TIMEOUT);
-                // the claims earlier runs kept that expired; the others this run may mount
-                const { next } = await (0, kept_claims_1.sweepKeptClaims)(this.instance, namespace, {
-                    now: new Date(),
-                    keep: this.keep > 0,
-                    timeout: RELEASE_TIMEOUT,
+                const label = (0, job_placement_1.getRunLabelValue)(run.id);
+                await this.enqueue(namespace, async () => {
+                    if (context.signal.aborted) {
+                        throw new Error('the run was cancelled');
+                    }
+                    await this.namespaces.prepare(namespace);
+                    // what an earlier run left when the server stopped before its end: the objects of
+                    // a run that does not execute. One that starts after this waits for it here
+                    // before it makes any.
+                    const running = [...new Set([label, ...(this.active.get(namespace)?.keys() ?? [])])];
+                    await removeRunObjects(this.instance, namespace, `${job_placement_1.RUN_LABEL},${job_placement_1.RUN_LABEL} notin (${running.join(',')})`, RELEASE_TIMEOUT);
+                    // the claims earlier runs kept that expired; the others this run may mount
+                    const { next } = await (0, kept_claims_1.sweepKeptClaims)(this.instance, namespace, {
+                        now: new Date(),
+                        keep: this.keep > 0,
+                        timeout: RELEASE_TIMEOUT,
+                        inUse: this.scopesInUse(namespace, label),
+                    });
+                    if (next) {
+                        this.sweepLater(namespace, Math.max(0, next.getTime() - Date.now()));
+                    }
                 });
-                if (next) {
-                    this.sweepLater(namespace, Math.max(0, next.getTime() - Date.now()));
-                }
                 this.sweepLeftClaims();
                 context.emit({ type: 'log', task: 'hammerkit', stream: 'stdout', text: `runs in the namespace ${namespace}` });
                 const kubernetes = {
@@ -300,8 +340,8 @@ class KubernetesExecutor {
             },
         };
     }
-    // The kept claims of a namespace that expired go once no run of the caller is in it: at
-    // the soonest expiry the server knows of
+    // The kept claims of a namespace that expired go at the soonest expiry the server knows
+    // of, but those of a scope a run of the caller executes in
     sweepLater(namespace, delay) {
         const at = Date.now() + delay + 1000;
         const pending = this.sweeps.get(namespace);
@@ -321,6 +361,7 @@ class KubernetesExecutor {
             now: new Date(),
             keep: this.keep > 0,
             timeout: RELEASE_TIMEOUT,
+            inUse: this.scopesInUse(namespace),
         });
         if (removed.length > 0) {
             this.log?.(`removed ${removed.length} kept claims in ${namespace}`);

package/dist/server/runs.js

@@ -424,13 +424,20 @@ class RunRegistry {
         return (source.type === 'sandbox' &&
             [...this.entries.values()].some((entry) => entry.run.state === 'running' && onSandbox(entry.run, source.id)));
     }
+    // Whether a run that executes holds what the run would (RunExecutor.exclusive)
+    held(run) {
+        const exclusive = this.options.executor.exclusive?.bind(this.options.executor);
+        const key = exclusive?.(run);
+        return (key !== undefined &&
+            [...this.entries.values()].some((entry) => entry.run.state === 'running' && exclusive?.(entry.run) === key));
+    }
     // The first run in the queue whose caller has a place left: one caller with many runs
     // does not keep the others from the machine. A run on a sandbox another run works on
-    // waits for it.
+    // waits for it, as does one that needs what an executing run holds.
     dispatch() {
         while (this.running < this.maxConcurrent) {
-            const perCaller = Math.min(this.options.limits?.running ?? Infinity, this.options.executor.runningPerCaller ?? Infinity);
-            const index = this.queue.findIndex((entry) => this.runningOf(entry.run.owner) < perCaller && !this.sandboxBusy(entry.run));
+            const perCaller = this.options.limits?.running ?? Infinity;
+            const index = this.queue.findIndex((entry) => this.runningOf(entry.run.owner) < perCaller && !this.sandboxBusy(entry.run) && !this.held(entry.run));
             if (index < 0) {
                 return;
             }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "hammerkit",
   "description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
-  "version": "1.27.0-next.8",
+  "version": "1.27.0-next.9",
   "keywords": [
     "build",
     "build-tool",