hammerkit 1.25.0 → 1.27.0-next.1

package/dist/kubernetes/apply.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.apply = exports.statusCodeOf = void 0;
+exports.replace = exports.apply = exports.statusCodeOf = void 0;
 // The client throws an ApiException carrying the HTTP status as `code`; watch
 // errors carry it as `statusCode`.
 function statusCodeOf(e) {
@@ -30,4 +30,34 @@ async function apply(instance, spec) {
     }
 }
 exports.apply = apply;
+/**
+ * Makes the object exactly as given: replaced as a whole when it exists, so what was
+ * dropped from it goes too, which a merge patch would keep (a key of a quota, a rule
+ * of a policy).
+ */
+async function replace(instance, spec) {
+    let current;
+    try {
+        current = await instance.objectApi.read(spec);
+    }
+    catch (e) {
+        if (statusCodeOf(e) !== 404) {
+            throw e;
+        }
+        try {
+            return await instance.objectApi.create(spec);
+        }
+        catch (createError) {
+            if (statusCodeOf(createError) !== 409) {
+                throw createError;
+            }
+            current = await instance.objectApi.read(spec);
+        }
+    }
+    return await instance.objectApi.replace({
+        ...spec,
+        metadata: { ...spec.metadata, resourceVersion: current.metadata?.resourceVersion },
+    });
+}
+exports.replace = replace;
 //# sourceMappingURL=apply.js.map

package/dist/kubernetes/await-running-state.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.deleteJobAndWait = exports.awaitDeployRunningState = exports.deleteJob = exports.awaitJobCompletion = exports.awaitRunningState = void 0;
+exports.stopJob = exports.deleteJobAndWait = exports.awaitDeployRunningState = exports.deleteJob = exports.awaitJobCompletion = exports.awaitRunningState = void 0;
 const abort_1 = require("../executer/abort");
 const abort_event_1 = require("../utils/abort-event");
 const apply_1 = require("./apply");
@@ -126,4 +126,36 @@ async function deleteJobAndWait(instance, env, name, abort, pollInterval = 500)
     }
 }
 exports.deleteJobAndWait = deleteJobAndWait;
+// Stops a Job at once: its pods go without a grace period, and this returns once they
+// are gone, so what limited them (the item's NetworkPolicy) can go after them.
+async function stopJob(instance, env, name, timeout = 30000, pollInterval = 500) {
+    await deleteJob(instance, env, name);
+    const deadline = Date.now() + timeout;
+    for (;;) {
+        const pods = await instance.coreApi.listNamespacedPod({
+            namespace: env.namespace,
+            labelSelector: `job-name=${name}`,
+        });
+        if (pods.items.length === 0) {
+            return;
+        }
+        if (Date.now() > deadline) {
+            throw new Error(`the pods of ${name} were still there after ${Math.round(timeout / 1000)}s`);
+        }
+        for (const pod of pods.items) {
+            // also one the Job's deletion already ends, with the default grace period
+            if (pod.metadata?.name) {
+                await instance.coreApi
+                    .deleteNamespacedPod({ name: pod.metadata.name, namespace: env.namespace, gracePeriodSeconds: 0 })
+                    .catch((e) => {
+                    if ((0, apply_1.statusCodeOf)(e) !== 404) {
+                        throw e;
+                    }
+                });
+            }
+        }
+        await (0, sleep_1.sleep)(pollInterval);
+    }
+}
+exports.stopJob = stopJob;
 //# sourceMappingURL=await-running-state.js.map

package/dist/kubernetes/ensure-kubernetes-deployment-exists.js

@@ -1,45 +1,42 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ensureKubernetesDeploymentExists = void 0;
-const work_item_1 = require("../planner/work-item");
+exports.ensureKubernetesDeploymentExists = exports.DEFINITION_LABEL = void 0;
+const service_definition_1 = require("../planner/service-definition");
 const work_service_1 = require("../planner/work-service");
+const volumes_1 = require("./volumes");
 const service_hostname_1 = require("../planner/utils/service-hostname");
 const apply_1 = require("./apply");
-const get_service_ip_1 = require("./get-service-ip");
 const version_1 = require("../version");
 const replace_env_variables_1 = require("../environment/replace-env-variables");
 const await_running_state_1 = require("./await-running-state");
 const resources_1 = require("./resources");
 const work_healthcheck_1 = require("../planner/work-healthcheck");
 const container_resources_1 = require("./container-resources");
-async function ensureKubernetesDeploymentExists(instance, env, service, persistence, secrets, stateKey) {
-    const hostAliases = [];
-    for (const need of service.needs) {
-        if (!(0, work_item_1.isContainerWorkServiceItem)(need.service)) {
-            continue;
-        }
-        const ip = await (0, get_service_ip_1.getServiceIp)(instance, env, need.service);
-        if (!ip) {
-            throw new Error(`unable to get service ip for ${need.name}`);
-        }
-        hostAliases.push({
-            ip,
-            hostnames: [need.name],
-        });
-    }
+const job_placement_1 = require("./job-placement");
+const network_policy_1 = require("./network-policy");
+const egress_proxy_1 = require("../network/egress-proxy");
+// what the service was created from, to tell a changed one
+exports.DEFINITION_LABEL = 'hammerkit.dev/definition';
+async function ensureKubernetesDeploymentExists(instance, env, service, persistence, secrets, stateKey, 
+// the needed services by name, and the proxy of a limited network
+network) {
     const envs = (0, replace_env_variables_1.getEnvironmentVariables)(service.data.envs);
     const name = (0, resources_1.getResourceName)(service);
     const healthcheckCommand = (0, work_service_1.getHealthcheckCommand)(service.data);
-    const serviceCommand = (0, work_service_1.getServiceCommand)(service.data);
+    const serviceCommand = (0, work_service_1.getServiceCommand)(service.data, { kubernetes: true });
     const probe = healthcheckCommand
         ? {
             exec: {
-                command: healthcheckCommand,
+                // the service itself, not through the proxy (see getProxylessEnvironment)
+                command: withoutProxy(healthcheckCommand, network.env),
             },
             periodSeconds: 5,
             failureThreshold: 3,
         }
         : undefined;
+    const server = env.server;
+    const placement = server ? (0, job_placement_1.getPodPlacement)(server, undefined) : null;
+    const workspace = (0, volumes_1.getWorkspacePodSpec)(env, persistence);
     const deployment = {
         kind: 'Deployment',
         apiVersion: 'apps/v1',
@@ -52,6 +49,8 @@ async function ensureKubernetesDeploymentExists(instance, env, service, persiste
             labels: {
                 'hammerkit.dev/id': service.id(),
                 'hammerkit.dev/state': stateKey,
+                ...(0, job_placement_1.getRunLabels)(env),
+                [exports.DEFINITION_LABEL]: (0, service_definition_1.getServiceDefinitionHash)(service),
             },
         },
         spec: {
@@ -66,12 +65,26 @@ async function ensureKubernetesDeploymentExists(instance, env, service, persiste
                     labels: {
                         'hammerkit.dev/id': service.id(),
                         'hammerkit.dev/state': stateKey,
+                        ...(0, job_placement_1.getRunLabels)(env),
+                        ...workspace.labels,
+                        ...(0, network_policy_1.getNetworkLabels)(env, service),
                     },
                 },
                 spec: {
+                    ...(workspace.affinity ? { affinity: workspace.affinity } : {}),
                     // the pod resolves its own service name, as with Docker's Hostname
                     hostname: (0, service_hostname_1.getServiceHostname)(service.name),
-                    hostAliases: hostAliases,
+                    hostAliases: network.hostAliases,
+                    initContainers: network.initContainers,
+                    // on a server the pod is the build's: no token of the namespace, and the
+                    // nodes of the first pool
+                    ...(placement
+                        ? {
+                            automountServiceAccountToken: false,
+                            nodeSelector: placement.nodeSelector,
+                            tolerations: placement.tolerations,
+                        }
+                        : {}),
                     containers: [
                         {
                             name: service.name.replace(/:/, '-'),
@@ -83,6 +96,7 @@ async function ensureKubernetesDeploymentExists(instance, env, service, persiste
                                     value: value,
                                 })),
                                 ...secrets.env,
+                                ...network.env,
                             ],
                             ...getKubernetesCommand(serviceCommand),
                             ports: service.data.ports.map((p) => ({
@@ -92,6 +106,7 @@ async function ensureKubernetesDeploymentExists(instance, env, service, persiste
                             livenessProbe: probe,
                             ...(0, container_resources_1.getKubernetesResources)(service.data.resources),
                             volumeMounts: [...persistence.mounts.map((m) => m.mount), ...secrets.mounts],
+                            ...(0, job_placement_1.getContainerSecurity)(env),
                         },
                         {
                             name: 'debug',
@@ -99,9 +114,10 @@ async function ensureKubernetesDeploymentExists(instance, env, service, persiste
                             command: ['sleep'],
                             args: ['3600'],
                             volumeMounts: persistence.mounts.map((m) => m.mount),
+                            ...(0, job_placement_1.getContainerSecurity)(env),
                         },
                     ],
-                    volumes: [...persistence.volumes, ...secrets.volumes],
+                    volumes: [...persistence.volumes, ...persistence.workspace, ...secrets.volumes],
                 },
             },
         },
@@ -131,4 +147,10 @@ function getKubernetesCommand(command) {
     }
     return {};
 }
+// A probe runs with the container's variables, so it empties the proxy ones
+// through `env`, as a Docker healthcheck does with the exec's own variables.
+function withoutProxy(command, env) {
+    const proxyless = (0, egress_proxy_1.getProxylessEnvironment)(Object.fromEntries(env.map((e) => [e.name, e.value ?? ''])));
+    return proxyless ? ['env', ...Object.keys(proxyless).map((key) => `${key}=`), ...command] : command;
+}
 //# sourceMappingURL=ensure-kubernetes-deployment-exists.js.map

package/dist/kubernetes/ensure-kubernetes-persistent-volume-claim-exists.js

@@ -1,9 +1,46 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ensureKubernetesPersistentVolumeClaimExists = void 0;
+exports.ensureKubernetesPersistentVolumeClaimExists = exports.getClaimSize = exports.DEFAULT_VOLUME_SIZE = void 0;
 const apply_1 = require("./apply");
 const version_1 = require("../version");
-async function ensureKubernetesPersistentVolumeClaimExists(instance, env, volume, service) {
+const job_placement_1 = require("./job-placement");
+const log_1 = require("../log");
+const units_1 = require("../utils/units");
+const sleep_1 = require("../utils/sleep");
+const kept_claims_1 = require("./kept-claims");
+// how long a claim that is being removed (a kept one that expired) gets to go before one
+// of its name is made again, and the kept claims of other sources when the quota is full
+const REMOVAL_TIMEOUT = 60000;
+// what the claim of a task or service requests when neither it nor the environment says
+exports.DEFAULT_VOLUME_SIZE = '1Gi';
+// The size of a claim: the work's `volumeSize`, else the environment's, at most what the
+// environment allows (a server's `maxVolumeSize`).
+function getClaimSize(env, size, name) {
+    const requested = size ?? env.volumeSize ?? exports.DEFAULT_VOLUME_SIZE;
+    if (env.maxVolumeSize && (0, units_1.parseSize)(requested) > (0, units_1.parseSize)(env.maxVolumeSize)) {
+        throw new Error(`${name} asks for a claim of ${requested}, the server allows at most ${env.maxVolumeSize}`);
+    }
+    return requested;
+}
+exports.getClaimSize = getClaimSize;
+// The size a claim requests, in bytes, as the cluster may write it back (`1Gi`, bytes)
+function getRequestedBytes(claim) {
+    const storage = claim.spec?.resources?.requests?.storage;
+    try {
+        return storage === undefined ? null : (0, units_1.parseSize)(String(storage));
+    }
+    catch {
+        return null;
+    }
+}
+// The claim of a work item, of `size` (see getClaimSize). One a run before made smaller
+// grows, where its storage class allows that; a larger one stays as it is, since a claim
+// never shrinks. A `kept` claim (see kept-claims) carries the scope of the run instead of
+// its label, and does not expire while the run mounts it.
+async function ensureKubernetesPersistentVolumeClaimExists(instance, env, volume, service, size, kept = false) {
+    const storage = getClaimSize(env, size, service.name);
+    const scope = kept ? env.server?.volumeScope : undefined;
+    const runLabel = env.server ? (0, job_placement_1.getRunLabelValue)(env.server.run) : undefined;
     const pvc = {
         kind: 'PersistentVolumeClaim',
         apiVersion: 'v1',
@@ -15,6 +52,7 @@ async function ensureKubernetesPersistentVolumeClaimExists(instance, env, volume
             },
             labels: {
                 'hammerkit.dev/id': service.id(),
+                ...(scope && runLabel ? { [kept_claims_1.VOLUME_SCOPE_LABEL]: scope, [kept_claims_1.KEPT_BY_LABEL]: runLabel } : (0, job_placement_1.getRunLabels)(env)),
             },
         },
         spec: {
@@ -22,7 +60,7 @@ async function ensureKubernetesPersistentVolumeClaimExists(instance, env, volume
             storageClassName: env.storageClass,
             resources: {
                 requests: {
-                    storage: '1Gi',
+                    storage,
                 },
             },
         },
@@ -34,7 +72,96 @@ async function ensureKubernetesPersistentVolumeClaimExists(instance, env, volume
     // mounts it — and hammerkit creates that pod (upload pod / deployment) only
     // after this call. Waiting here deadlocks until the test times out. k8s binds
     // the claim when the consuming pod is scheduled, so applying it is enough.
-    return await (0, apply_1.apply)(instance, pvc);
+    let existing = await readClaim(instance, pvc);
+    if (existing?.metadata?.deletionTimestamp) {
+        // a kept claim that expired: one of the same name is made once it is gone
+        existing = await untilRemoved(instance, pvc);
+    }
+    if (!existing) {
+        return await createClaim(instance, env, pvc);
+    }
+    if (scope) {
+        if (existing.metadata?.labels?.[kept_claims_1.VOLUME_SCOPE_LABEL] !== scope) {
+            throw new Error(`the claim ${volume.name} is not one this run may mount, it was made for another source`);
+        }
+        if (existing.metadata?.labels?.[kept_claims_1.KEPT_BY_LABEL] !== runLabel ||
+            existing.metadata?.annotations?.[kept_claims_1.EXPIRES_ANNOTATION]) {
+            // mounted by this run now: it expires at the run's end
+            existing = await instance.objectApi.patch({
+                apiVersion: pvc.apiVersion,
+                kind: pvc.kind,
+                metadata: {
+                    name: pvc.metadata.name,
+                    namespace: pvc.metadata.namespace,
+                    labels: { [kept_claims_1.KEPT_BY_LABEL]: runLabel },
+                    // null removes it
+                    annotations: { [kept_claims_1.EXPIRES_ANNOTATION]: null },
+                },
+            });
+        }
+    }
+    const current = getRequestedBytes(existing);
+    if (current === null || current >= (0, units_1.parseSize)(storage)) {
+        return existing;
+    }
+    try {
+        // the requested size is all of a bound claim's spec that may change
+        return await instance.objectApi.patch({
+            ...pvc,
+            spec: { resources: { requests: { storage } } },
+        });
+    }
+    catch (e) {
+        throw new Error(`the claim ${volume.name} has ${(0, units_1.formatSize)(current)} and ${service.name} needs ${storage}, ` +
+            `which its storage class could not grow it to (${(0, log_1.getErrorMessage)(e)}): ` +
+            'remove it with hammerkit clean and run again');
+    }
 }
 exports.ensureKubernetesPersistentVolumeClaimExists = ensureKubernetesPersistentVolumeClaimExists;
+async function createClaim(instance, env, pvc) {
+    for (let evicted = false;; evicted = true) {
+        try {
+            return await instance.objectApi.create(pvc);
+        }
+        catch (e) {
+            // Two work items can make the same claim concurrently (e.g. a volume
+            // shared between a task and a service that inherits its generates)
+            if ((0, apply_1.statusCodeOf)(e) === 409) {
+                return await instance.objectApi.read(pvc);
+            }
+            // the claims a server keeps for other sources make way under the quota
+            if (evicted || (0, apply_1.statusCodeOf)(e) !== 403 || !/exceeded quota/i.test((0, log_1.getErrorMessage)(e))) {
+                throw e;
+            }
+            const removed = await (0, kept_claims_1.evictKeptClaims)(instance, env, REMOVAL_TIMEOUT);
+            if (removed.length === 0) {
+                throw e;
+            }
+        }
+    }
+}
+async function untilRemoved(instance, pvc) {
+    const deadline = Date.now() + REMOVAL_TIMEOUT;
+    for (;;) {
+        const claim = await readClaim(instance, pvc);
+        if (!claim) {
+            return null;
+        }
+        if (Date.now() > deadline) {
+            throw new Error(`the claim ${pvc.metadata.name} is being removed and still there after ${REMOVAL_TIMEOUT / 1000}s`);
+        }
+        await (0, sleep_1.sleep)(1000);
+    }
+}
+async function readClaim(instance, pvc) {
+    try {
+        return await instance.objectApi.read(pvc);
+    }
+    catch (e) {
+        if ((0, apply_1.statusCodeOf)(e) === 404) {
+            return null;
+        }
+        throw e;
+    }
+}
 //# sourceMappingURL=ensure-kubernetes-persistent-volume-claim-exists.js.map

package/dist/kubernetes/ensure-kubernetes-service-exists.js

@@ -4,8 +4,12 @@ exports.ensureKubernetesServiceExists = void 0;
 const apply_1 = require("./apply");
 const resources_1 = require("./resources");
 const version_1 = require("../version");
+const job_placement_1 = require("./job-placement");
+// A service without ports gets a headless Service: a ClusterIP one needs a port,
+// and its tasks reach every port of the pod, as on docker.
 async function ensureKubernetesServiceExists(instance, env, service) {
     const name = (0, resources_1.getResourceName)(service);
+    const headless = service.data.ports.length === 0;
     const svc = {
         kind: 'Service',
         apiVersion: 'v1',
@@ -17,19 +21,36 @@ async function ensureKubernetesServiceExists(instance, env, service) {
             },
             labels: {
                 'hammerkit.dev/id': service.id(),
+                ...(0, job_placement_1.getRunLabels)(env),
             },
         },
         spec: {
             type: 'ClusterIP',
+            ...(headless ? { clusterIP: 'None' } : {}),
             selector: {
                 'hammerkit.dev/id': service.id(),
             },
-            ports: service.data.ports.map((port) => ({
-                port: port.hostPort ?? port.containerPort,
-                targetPort: port.containerPort,
-            })),
+            ...(headless
+                ? {}
+                : {
+                    ports: service.data.ports.map((port) => ({
+                        port: port.hostPort ?? port.containerPort,
+                        targetPort: port.containerPort,
+                    })),
+                }),
         },
     };
+    // the clusterIP of a Service is fixed, so one from before its ports changed goes first
+    const existing = await instance.coreApi
+        .readNamespacedService({ name, namespace: env.namespace })
+        .catch((e) => ((0, apply_1.statusCodeOf)(e) === 404 ? null : Promise.reject(e)));
+    if (existing && (existing.spec?.clusterIP === 'None') !== headless) {
+        await instance.coreApi.deleteNamespacedService({ name, namespace: env.namespace }).catch((e) => {
+            if ((0, apply_1.statusCodeOf)(e) !== 404) {
+                throw e;
+            }
+        });
+    }
     return await (0, apply_1.apply)(instance, svc);
 }
 exports.ensureKubernetesServiceExists = ensureKubernetesServiceExists;

package/dist/kubernetes/ensure-persistent-data.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ensurePersistentData = exports.getPodForPersistence = void 0;
+exports.ensurePersistentData = exports.clearClaim = exports.getPodForPersistence = void 0;
 const apply_1 = require("./apply");
 const path_1 = require("path");
 const tar_1 = require("tar");
@@ -11,6 +11,8 @@ const version_1 = require("../version");
 const log_1 = require("../log");
 const ensure_kubernetes_persistent_volume_claim_exists_1 = require("./ensure-kubernetes-persistent-volume-claim-exists");
 const stream_1 = require("stream");
+const job_placement_1 = require("./job-placement");
+const assert_item_paths_1 = require("../planner/utils/assert-item-paths");
 // A previous run that crashed between creating an upload/download pod and its
 // `finally` cleanup leaves a pod behind under the same deterministic name. apply()
 // would then patch that stale pod in place — and if it is stuck in a terminal
@@ -43,9 +45,12 @@ async function getPodForPersistence(instance, env, service, persistence, type, f
     if (persistence.volumes.length === 0) {
         return;
     }
+    // the claim of a sandbox is mounted by the pods of the work only: it holds no output
+    // and takes no upload
+    const own = persistence.mounts.filter((m) => !m.workspace);
     const mounts = type === 'read'
-        ? persistence.mounts.map((m) => m.mount)
-        : persistence.mounts.reduce((array, v) => {
+        ? own.map((m) => m.mount)
+        : own.reduce((array, v) => {
             if (!array.some((i) => i.name === v.mount.name)) {
                 array.push({
                     mountPath: v.uploadPath,
@@ -60,7 +65,7 @@ async function getPodForPersistence(instance, env, service, persistence, type, f
     // function directly, and a claim shared with another work item can be removed
     // concurrently — without this the pod would hang Pending on "pvc not found".
     for (const volume of persistence.volumes) {
-        await (0, ensure_kubernetes_persistent_volume_claim_exists_1.ensureKubernetesPersistentVolumeClaimExists)(instance, env, volume, service);
+        await (0, ensure_kubernetes_persistent_volume_claim_exists_1.ensureKubernetesPersistentVolumeClaimExists)(instance, env, volume, service, persistence.sizes[volume.name], persistence.kept.includes(volume.name));
     }
     const name = (0, resources_1.getResourceName)(service, type === 'write' ? '-upload' : '-download');
     const podSpec = {
@@ -71,12 +76,21 @@ async function getPodForPersistence(instance, env, service, persistence, type, f
             name,
             labels: {
                 'hammerkit.dev/id': service.id(),
+                ...(0, job_placement_1.getRunLabels)(env),
             },
             annotations: {
                 'hammerkit.dev/version': (0, version_1.getVersion)(),
             },
         },
         spec: {
+            // on a server the claims are bound where the pod of the task can go: the nodes
+            // of its pool
+            ...(env.server
+                ? {
+                    automountServiceAccountToken: false,
+                    ...(0, job_placement_1.getPodPlacement)(env.server, 'pool' in service.data ? service.data.pool : undefined),
+                }
+                : {}),
             containers: [
                 {
                     name: 'volume',
@@ -84,6 +98,7 @@ async function getPodForPersistence(instance, env, service, persistence, type, f
                     command: ['sh'],
                     tty: true,
                     volumeMounts: mounts,
+                    ...(0, job_placement_1.getContainerSecurity)(env),
                 },
             ],
             volumes: persistence.volumes,
@@ -105,18 +120,44 @@ async function getPodForPersistence(instance, env, service, persistence, type, f
     }
 }
 exports.getPodForPersistence = getPodForPersistence;
-async function ensurePersistentData(instance, env, environment, service, persistence) {
+// Empties the claim of `work` in an upload pod: a kept claim an earlier run left, before
+// the task runs on it or outputs are restored into it, so it holds what a new one would
+async function clearClaim(instance, env, work, pod) {
+    const targetPath = `/dev/hammerkit/${work.id()}`;
+    work.status.console('stdout', `clear what an earlier run left in ${targetPath}`);
+    let failure = null;
+    const res = await instance.exec.exec(env.namespace, pod, 'volume', ['find', targetPath, '-mindepth', '1', '-maxdepth', '1', '-exec', 'rm', '-rf', '{}', '+'], process.stdout, process.stderr, null, false, (status) => {
+        if (status.status !== 'Success') {
+            failure = status.message ?? status.reason ?? 'unknown';
+        }
+    });
+    await new Promise((resolve, reject) => {
+        res.on('error', reject);
+        res.on('close', () => failure ? reject(new Error(`unable to clear ${targetPath} in ${pod}: ${failure}`)) : resolve());
+    });
+}
+exports.clearClaim = clearClaim;
+async function ensurePersistentData(instance, env, environment, service, persistence, 
+// the kept claim of the work holds what an earlier run left, which goes first
+options = {}) {
+    // what a run of a server uploads comes from its checkout and nowhere else
+    if (environment.hosted) {
+        await (0, assert_item_paths_1.assertItemPathsWithinRoot)(service, environment.cwd);
+    }
     // Dedupe of already-uploaded state (skip the upload when the persisted state
     // key still matches) needs an in-cluster state marker per volume — deferred
     // post-1.6.0, tracked in FOLLOWUPS.md alongside the volumes.ts stateKey/matcher.
     for (const volume of persistence.volumes) {
-        await (0, ensure_kubernetes_persistent_volume_claim_exists_1.ensureKubernetesPersistentVolumeClaimExists)(instance, env, volume, service);
+        await (0, ensure_kubernetes_persistent_volume_claim_exists_1.ensureKubernetesPersistentVolumeClaimExists)(instance, env, volume, service, persistence.sizes[volume.name], persistence.kept.includes(volume.name));
     }
-    if (persistence.sources.length === 0) {
+    if (persistence.sources.length === 0 && !options.reset) {
         return;
     }
     service.status.console('stdout', 'start upload container');
     await getPodForPersistence(instance, env, service, persistence, 'write', async (name) => {
+        if (options.reset) {
+            await clearClaim(instance, env, service, name);
+        }
         for (const source of persistence.sources) {
             // skip sources that don't exist on disk — nothing to upload
             if (!(await environment.file.exists(source.localPath))) {

package/dist/kubernetes/get-service-ip.js

@@ -2,10 +2,24 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getServiceIp = void 0;
 const resources_1 = require("./resources");
+// The address a task reaches the service on: its clusterIP, or for a headless
+// Service (a service without ports) the address of its ready pod.
 async function getServiceIp(instance, env, workService) {
     const name = (0, resources_1.getResourceName)(workService);
     const svc = await instance.coreApi.readNamespacedService({ name, namespace: env.namespace });
-    return svc.spec?.clusterIP ?? null;
+    const clusterIP = svc.spec?.clusterIP ?? null;
+    if (clusterIP !== 'None') {
+        return clusterIP;
+    }
+    const pods = await instance.coreApi.listNamespacedPod({
+        namespace: env.namespace,
+        labelSelector: `hammerkit.dev/id=${workService.id()}`,
+    });
+    const ready = pods.items.find((pod) => !pod.metadata?.deletionTimestamp &&
+        pod.status?.podIP &&
+        pod.status.conditions?.some((c) => c.type === 'Ready' && c.status === 'True'));
+    const running = ready ?? pods.items.find((pod) => !pod.metadata?.deletionTimestamp && pod.status?.podIP);
+    return running?.status?.podIP ?? null;
 }
 exports.getServiceIp = getServiceIp;
 //# sourceMappingURL=get-service-ip.js.map