hammerkit 1.25.0 → 1.27.0-next.1

package/dist/planner/work-runtime-kubernetes.js

@@ -3,18 +3,18 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.kubernetesForwardRuntime = exports.kubernetesServiceRuntime = exports.kubernetesTaskRuntime = void 0;
+exports.kubernetesForwardRuntime = exports.kubernetesServiceRuntime = exports.kubernetesTaskRuntime = exports.describeFailedJob = void 0;
 const abort_1 = require("../executer/abort");
-const get_service_ip_1 = require("../kubernetes/get-service-ip");
-const work_item_1 = require("./work-item");
 const kubernetes_service_1 = require("../executer/kubernetes-service");
 const kubernetes_instance_1 = require("../kubernetes/kubernetes-instance");
 const replace_env_variables_1 = require("../environment/replace-env-variables");
 const apply_1 = require("../kubernetes/apply");
 const ensure_kubernetes_service_exists_1 = require("../kubernetes/ensure-kubernetes-service-exists");
 const ensure_kubernetes_deployment_exists_1 = require("../kubernetes/ensure-kubernetes-deployment-exists");
+const service_definition_1 = require("./service-definition");
 const ensure_namespace_1 = require("../kubernetes/ensure-namespace");
 const ensure_persistent_data_1 = require("../kubernetes/ensure-persistent-data");
+const network_gate_1 = require("../kubernetes/network-gate");
 const await_running_state_1 = require("../kubernetes/await-running-state");
 const volumes_1 = require("../kubernetes/volumes");
 const resources_1 = require("../kubernetes/resources");
@@ -22,11 +22,19 @@ const ensure_http_route_1 = require("../kubernetes/ensure-http-route");
 const find_process_1 = __importDefault(require("find-process"));
 const log_1 = require("../log");
 const version_1 = require("../version");
+const task_arguments_1 = require("./utils/task-arguments");
 const restore_kubernetes_data_1 = require("../kubernetes/restore-kubernetes-data");
 const store_kubernetes_data_1 = require("../kubernetes/store-kubernetes-data");
 const remove_persistent_data_1 = require("../kubernetes/remove-persistent-data");
+const kept_claims_1 = require("../kubernetes/kept-claims");
 const container_resources_1 = require("../kubernetes/container-resources");
 const secrets_1 = require("../kubernetes/secrets");
+const job_secrets_1 = require("../kubernetes/job-secrets");
+const job_placement_1 = require("../kubernetes/job-placement");
+const network_policy_1 = require("../kubernetes/network-policy");
+const run_id_1 = require("../utils/run-id");
+const work_network_1 = require("./work-network");
+const work_secret_1 = require("./work-secret");
 async function listJobNames(instance, kubernetes, task) {
     const jobs = await instance.batchApi.listNamespacedJob({
         namespace: kubernetes.namespace,
@@ -34,138 +42,334 @@ async function listJobNames(instance, kubernetes, task) {
     });
     return jobs.items.flatMap((job) => (job.metadata?.name ? [job.metadata.name] : []));
 }
+// The Secret of the values the server hands over and the one of the credentials the
+// pod reads its own secrets with
+async function removeTaskSecrets(instance, kubernetes, task) {
+    await (0, secrets_1.removeKubernetesSecret)(instance, kubernetes, task);
+    await (0, job_secrets_1.removeJobCredentials)(instance, kubernetes, task);
+}
+// The newest pod of a Job
+async function getJobPod(instance, kubernetes, job) {
+    const pods = await instance.coreApi.listNamespacedPod({
+        namespace: kubernetes.namespace,
+        labelSelector: `job-name=${job}`,
+    });
+    return pods.items
+        .slice()
+        .sort((a, b) => (a.metadata?.creationTimestamp?.toISOString() ?? '').localeCompare(b.metadata?.creationTimestamp?.toISOString() ?? ''))
+        .pop();
+}
+// a task's log is kept to this, the run's own cap is smaller
+const MAX_JOB_LOG_BYTES = 4 * 1024 * 1024;
+// What the command printed, as the task's output: read from the cluster once the Job
+// has ended. Without a pod (it never started) there is nothing to say.
+async function writeJobLogs(instance, kubernetes, task, job, container) {
+    try {
+        const pod = await getJobPod(instance, kubernetes, job);
+        if (!pod?.metadata?.name) {
+            return;
+        }
+        const log = await instance.coreApi.readNamespacedPodLog({
+            name: pod.metadata.name,
+            namespace: kubernetes.namespace,
+            container,
+            limitBytes: MAX_JOB_LOG_BYTES,
+        });
+        for (const line of String(log ?? '').split(/\r?\n/)) {
+            if (line.length > 0) {
+                task.status.console('stdout', line);
+            }
+        }
+    }
+    catch (e) {
+        task.status.write('debug', `the log of ${job} could not be read: ${(0, log_1.getErrorMessage)(e)}`);
+    }
+}
+// the command of a Job exited with this code, as a crash on docker or a host
+class JobCrash extends Error {
+    constructor(exitCode) {
+        super(`exit code ${exitCode}`);
+        this.exitCode = exitCode;
+    }
+}
+// Why a Job failed: the network gate, an init container that could not read a secret, which
+// names the secret, never its output (which is the provider's and can hold the value), or the
+// exit code of the command.
+async function describeFailedJob(instance, kubernetes, task, job, container, secrets) {
+    const pod = await getJobPod(instance, kubernetes, job).catch(() => undefined);
+    for (const status of pod?.status?.initContainerStatuses ?? []) {
+        const terminated = status.state?.terminated ?? status.lastState?.terminated;
+        if (!terminated || terminated.exitCode === 0) {
+            continue;
+        }
+        if (status.name === network_gate_1.NETWORK_GATE_CONTAINER) {
+            // the gate says why in its termination message
+            return {
+                type: 'error',
+                message: `${task}: ${terminated.message?.trim() || 'the network policy did not take hold'}`,
+            };
+        }
+        const secret = secrets.find((s) => `secret-${s.index}` === status.name);
+        if (secret) {
+            return {
+                type: 'error',
+                message: `secret ${(0, work_secret_1.getSecretName)(secret.secret)} (${(0, work_secret_1.describeSecretSource)(secret.secret)}) could not be read: ${terminated.reason ?? 'exit code'} ${terminated.exitCode}`,
+            };
+        }
+    }
+    const command = pod?.status?.containerStatuses?.find((status) => status.name === container);
+    const terminated = command?.state?.terminated ?? command?.lastState?.terminated;
+    if (terminated && terminated.exitCode !== 0) {
+        return { type: 'crash', exitCode: terminated.exitCode };
+    }
+    return undefined;
+}
+exports.describeFailedJob = describeFailedJob;
 function kubernetesTaskRuntime(task, kubernetes) {
     const instance = (0, kubernetes_instance_1.createKubernetesInstances)(kubernetes);
+    // on a server that keeps them, the task's claim stays after the run with the state of its
+    // outputs, and the next run of the same source mounts them as they are (see kept-claims)
+    const kept = (0, kept_claims_1.isKeptClaim)(task, kubernetes);
+    const claim = (0, volumes_1.getVolumeName)(task, kubernetes);
     return {
         initialize() {
             return Promise.resolve();
         },
-        async restore(environment, path) {
-            await (0, restore_kubernetes_data_1.restoreKubernetesData)(task, kubernetes, instance, environment, path);
+        async restore(environment, path, stateKey) {
+            if (!kept) {
+                await (0, restore_kubernetes_data_1.restoreKubernetesData)(task, kubernetes, instance, environment, path);
+                return;
+            }
+            const reset = await (0, kept_claims_1.forgetClaimState)(instance, kubernetes, claim);
+            await (0, restore_kubernetes_data_1.restoreKubernetesData)(task, kubernetes, instance, environment, path, { reset });
+            if (stateKey) {
+                await (0, kept_claims_1.recordClaimState)(instance, kubernetes, claim, stateKey);
+            }
         },
         async archive(environment, path) {
             await (0, store_kubernetes_data_1.storeKubernetesData)(task, kubernetes, instance, environment, path);
         },
         async execute(environment, options) {
             const envs = (0, replace_env_variables_1.getEnvironmentVariables)(task.data.envs);
-            const persistence = await (0, volumes_1.getKubernetesPersistence)(task);
+            const persistence = await (0, volumes_1.getKubernetesPersistence)(task, kubernetes);
+            const workspace = (0, volumes_1.getWorkspacePodSpec)(kubernetes, persistence, { name: task.name, pool: task.data.pool });
             await (0, ensure_namespace_1.ensureNamespace)(instance, kubernetes.namespace);
-            await (0, ensure_persistent_data_1.ensurePersistentData)(instance, kubernetes, environment, task, persistence);
-            const secrets = (0, secrets_1.getKubernetesSecretRefs)(task, task.data.secrets);
             try {
-                await (0, secrets_1.ensureKubernetesSecret)(instance, kubernetes, task, task.data.secrets, environment);
+                // the task writes to it now: what it holds stops being the outputs of a state, and
+                // goes, as a new claim would hold nothing
+                const reset = kept && (await (0, kept_claims_1.forgetClaimState)(instance, kubernetes, claim));
+                await (0, ensure_persistent_data_1.ensurePersistentData)(instance, kubernetes, environment, task, persistence, { reset });
+            }
+            catch (e) {
+                options.state.set({ stateKey: options.stateKey, type: 'error', errorMessage: (0, log_1.getErrorMessage)(e) });
+                return;
+            }
+            const server = kubernetes.server;
+            const { delivered, inJob } = (0, job_secrets_1.splitJobSecrets)(task.data.secrets, server);
+            const secrets = (0, secrets_1.getKubernetesSecretRefs)(task, delivered);
+            const admitted = environment.hosted?.accounts;
+            try {
+                (0, job_secrets_1.assertSecretsReachable)(task, inJob);
+                if (server) {
+                    (0, job_secrets_1.assertJobAccounts)(inJob, server, admitted);
+                }
+                await (0, secrets_1.ensureKubernetesSecret)(instance, kubernetes, task, delivered, environment);
             }
             catch (e) {
+                await removeTaskSecrets(instance, kubernetes, task).catch(() => undefined);
                 options.state.set({ stateKey: options.stateKey, type: 'error', errorMessage: (0, log_1.getErrorMessage)(e) });
                 return;
             }
+            // the run its pods and limits belong to: on a server, the server's run; on a
+            // sandbox, its proxy stays with the sandbox for the next runs
+            const scope = {
+                run: server ? (0, job_placement_1.getRunLabelValue)(server.run) : (0, run_id_1.getRunId)(),
+                ...(server?.workspace
+                    ? {
+                        sandbox: {
+                            label: server.workspace.label,
+                            ...(server.workspace.pod ? { pod: server.workspace.pod } : {}),
+                        },
+                    }
+                    : {}),
+            };
+            const labels = {
+                'hammerkit.dev/id': task.id(),
+                'hammerkit.dev/state': options.stateKey,
+                ...workspace.labels,
+                [network_policy_1.RUN_LABEL]: scope.run,
+                ...(0, network_policy_1.getNetworkLabels)(kubernetes, task),
+            };
+            const placement = server ? (0, job_placement_1.getPodPlacement)(server, task.data.pool) : null;
             // The last job of a run stays: its state label is what currentStateKey
             // reads. Jobs from an earlier run of this task go before this one starts.
             for (const name of await listJobNames(instance, kubernetes, task)) {
                 await (0, await_running_state_1.deleteJobAndWait)(instance, kubernetes, name, options.abort);
             }
-            // the services the task needs, by the names it lists them under
-            const hostAliases = [];
-            for (const need of task.needs) {
-                if (!(0, work_item_1.isContainerWorkServiceItem)(need.service)) {
-                    continue;
-                }
-                const ip = await (0, get_service_ip_1.getServiceIp)(instance, kubernetes, need.service);
-                if (!ip) {
-                    throw new Error(`unable to get service ip for ${need.name}`);
-                }
-                hostAliases.push({ ip, hostnames: [need.name] });
+            // the services the task needs by the names it lists them under, and the
+            // NetworkPolicy and proxy of a limited network, both of this run
+            let network;
+            try {
+                network = await (0, network_policy_1.ensureKubernetesItemNetwork)(instance, kubernetes, task, { followLog: true, ...scope });
             }
-            const podName = `${task.name}-${options.stateKey}`;
-            let i = 0;
-            for (const cmd of task.data.cmds) {
-                const spec = {
-                    kind: 'Job',
-                    apiVersion: 'batch/v1',
-                    metadata: {
-                        namespace: kubernetes.namespace,
-                        name: podName,
-                        annotations: {
-                            'hammerkit.dev/version': (0, version_1.getVersion)(),
-                        },
-                        labels: {
-                            'hammerkit.dev/id': task.id(),
-                            'hammerkit.dev/state': options.stateKey,
+            catch (e) {
+                await removeTaskSecrets(instance, kubernetes, task).catch(() => undefined);
+                options.state.set({ stateKey: options.stateKey, type: 'error', errorMessage: (0, log_1.getErrorMessage)(e) });
+                return;
+            }
+            try {
+                const podName = (0, resources_1.getResourceName)(task, '-job');
+                let i = 0;
+                for (const cmd of task.data.cmds) {
+                    const container = `cmd-${++i}`;
+                    let secretSpec;
+                    try {
+                        // made for each Job right before it, so a later command's token is as fresh
+                        const credentials = await (0, job_secrets_1.ensureJobCredentials)(instance, kubernetes, task, inJob, admitted);
+                        secretSpec = (0, job_secrets_1.getJobSecretSpec)(task, inJob, server, credentials, environment);
+                    }
+                    catch (e) {
+                        await removeTaskSecrets(instance, kubernetes, task).catch(() => undefined);
+                        options.state.set({ stateKey: options.stateKey, type: 'error', errorMessage: (0, log_1.getErrorMessage)(e) });
+                        return;
+                    }
+                    // the arguments after `--` keep their `$` as text, Kubernetes expands `$(NAME)`
+                    const command = secretSpec.command(task.data.shell, (0, task_arguments_1.getKubernetesCommandText)(cmd));
+                    const spec = {
+                        kind: 'Job',
+                        apiVersion: 'batch/v1',
+                        metadata: {
+                            namespace: kubernetes.namespace,
+                            name: podName,
+                            annotations: {
+                                'hammerkit.dev/version': (0, version_1.getVersion)(),
+                            },
+                            labels,
                         },
-                    },
-                    spec: {
-                        template: {
-                            metadata: {
-                                annotations: {
-                                    'hammerkit.dev/version': (0, version_1.getVersion)(),
+                        spec: {
+                            template: {
+                                metadata: {
+                                    annotations: {
+                                        'hammerkit.dev/version': (0, version_1.getVersion)(),
+                                    },
+                                    labels,
                                 },
-                                labels: {
-                                    'hammerkit.dev/id': task.id(),
-                                    'hammerkit.dev/state': options.stateKey,
+                                spec: {
+                                    // the network gate first: nothing runs before the item's policy holds
+                                    ...(network.initContainers.length + secretSpec.initContainers.length > 0
+                                        ? { initContainers: [...network.initContainers, ...secretSpec.initContainers] }
+                                        : {}),
+                                    containers: [
+                                        {
+                                            image: task.data.image,
+                                            // through the task's shell, as on Docker
+                                            command: command.command,
+                                            args: command.args,
+                                            workingDir: cmd.cwd,
+                                            env: [
+                                                ...Object.entries(envs).map(([key, value]) => ({ name: key, value })),
+                                                ...secrets.env,
+                                                ...network.env,
+                                                ...command.env,
+                                            ],
+                                            name: container,
+                                            volumeMounts: [
+                                                ...persistence.mounts.map((m) => m.mount),
+                                                ...secrets.mounts,
+                                                ...secretSpec.mounts,
+                                            ],
+                                            ...(0, container_resources_1.getKubernetesResources)(task.data.resources),
+                                            ...(0, job_placement_1.getContainerSecurity)(kubernetes),
+                                        },
+                                    ],
+                                    volumes: [
+                                        ...persistence.volumes,
+                                        ...persistence.workspace,
+                                        ...secrets.volumes,
+                                        ...secretSpec.volumes,
+                                    ],
+                                    ...(workspace.affinity ? { affinity: workspace.affinity } : {}),
+                                    hostAliases: network.hostAliases,
+                                    restartPolicy: 'Never',
+                                    // on a server the pod is the build's: no token of the namespace, and the
+                                    // nodes of its pool
+                                    ...(placement
+                                        ? {
+                                            automountServiceAccountToken: false,
+                                            nodeSelector: placement.nodeSelector,
+                                            tolerations: placement.tolerations,
+                                        }
+                                        : {}),
                                 },
                             },
-                            spec: {
-                                containers: [
-                                    {
-                                        image: task.data.image,
-                                        // through the task's shell, as on Docker
-                                        command: [task.data.shell, '-c'],
-                                        args: [cmd.cmd],
-                                        workingDir: cmd.cwd,
-                                        env: [
-                                            ...Object.entries(envs).map(([key, value]) => ({ name: key, value })),
-                                            ...secrets.env,
-                                        ],
-                                        name: `cmd-${++i}`,
-                                        volumeMounts: [...persistence.mounts.map((m) => m.mount), ...secrets.mounts],
-                                        ...(0, container_resources_1.getKubernetesResources)(task.data.resources),
-                                    },
-                                ],
-                                volumes: [...persistence.volumes, ...secrets.volumes],
-                                hostAliases,
-                                restartPolicy: 'Never',
-                            },
+                            backoffLimit: server ? 0 : 1,
+                            parallelism: 1,
                         },
-                        backoffLimit: 1,
-                        parallelism: 1,
-                    },
-                };
-                try {
-                    const pod = await (0, apply_1.apply)(instance, spec);
-                    if (!pod.status?.succeeded) {
-                        await (0, await_running_state_1.awaitJobCompletion)(instance, kubernetes, spec.metadata.name, options.abort);
+                    };
+                    try {
+                        const pod = await (0, apply_1.apply)(instance, spec);
+                        if (!pod.status?.succeeded) {
+                            try {
+                                await (0, await_running_state_1.awaitJobCompletion)(instance, kubernetes, spec.metadata.name, options.abort);
+                            }
+                            catch (e) {
+                                if (!(e instanceof abort_1.AbortError)) {
+                                    await writeJobLogs(instance, kubernetes, task, podName, container);
+                                    const failure = await describeFailedJob(instance, kubernetes, task.name, podName, container, inJob);
+                                    if (failure?.type === 'crash') {
+                                        throw new JobCrash(failure.exitCode);
+                                    }
+                                    throw new Error(failure?.message ?? (0, log_1.getErrorMessage)(e));
+                                }
+                                throw e;
+                            }
+                        }
+                        await writeJobLogs(instance, kubernetes, task, podName, container);
+                        task.status.write('debug', 'pod completed');
+                        // its init containers are done with the credentials
+                        await (0, job_secrets_1.removeJobCredentials)(instance, kubernetes, task);
+                        if (i < task.data.cmds.length) {
+                            await (0, await_running_state_1.deleteJobAndWait)(instance, kubernetes, podName, options.abort);
+                        }
                     }
-                    task.status.write('debug', 'pod completed');
-                    if (i < task.data.cmds.length) {
-                        await (0, await_running_state_1.deleteJobAndWait)(instance, kubernetes, podName, options.abort);
-                    }
-                }
-                catch (e) {
-                    if (e instanceof abort_1.AbortError) {
-                        options.state.set({ stateKey: options.stateKey, type: 'canceled' });
+                    catch (e) {
+                        if (e instanceof abort_1.AbortError) {
+                            // at once, so its policy can go (in finally) once the pod ended
+                            await (0, await_running_state_1.stopJob)(instance, kubernetes, podName).catch((e) => task.status.write('error', `the job could not be stopped: ${(0, log_1.getErrorMessage)(e)}`));
+                            await removeTaskSecrets(instance, kubernetes, task).catch(() => undefined);
+                            options.state.set({ stateKey: options.stateKey, type: 'canceled' });
+                            return;
+                        }
+                        await (0, await_running_state_1.deleteJob)(instance, kubernetes, podName).catch(() => undefined);
+                        options.state.set(e instanceof JobCrash
+                            ? { stateKey: options.stateKey, type: 'crash', exitCode: e.exitCode }
+                            : { stateKey: options.stateKey, type: 'error', errorMessage: (0, log_1.getErrorMessage)(e) });
+                        await removeTaskSecrets(instance, kubernetes, task).catch(() => undefined);
                         return;
                     }
-                    await (0, await_running_state_1.deleteJob)(instance, kubernetes, podName).catch(() => undefined);
-                    options.state.set({
-                        stateKey: options.stateKey,
-                        type: 'error',
-                        errorMessage: (0, log_1.getErrorMessage)(e),
-                    });
-                    await (0, secrets_1.removeKubernetesSecret)(instance, kubernetes, task).catch(() => undefined);
-                    return;
+                }
+                // the finished job stays for its state, the values it read need not
+                await removeTaskSecrets(instance, kubernetes, task);
+                if (kept) {
+                    await (0, kept_claims_1.recordClaimState)(instance, kubernetes, claim, options.stateKey);
+                }
+            }
+            finally {
+                network.close();
+                // once the run's pods ended: a cancelled job's pod is still terminating
+                if ((0, work_network_1.isLimitedNetwork)((0, work_network_1.getEffectiveNetwork)(task.data))) {
+                    await (0, network_policy_1.releaseKubernetesItemNetwork)(instance, kubernetes, task, scope).catch((e) => task.status.write('error', `remove of network policy failed ${(0, log_1.getErrorMessage)(e)}`));
                 }
             }
-            // the finished job stays for its state, the values it read need not
-            await (0, secrets_1.removeKubernetesSecret)(instance, kubernetes, task);
         },
         async stop() {
             const jobs = await instance.batchApi.listNamespacedJob({
                 namespace: kubernetes.namespace,
                 labelSelector: `hammerkit.dev/id=${task.id()}`,
             });
-            for (const pod of jobs.items) {
-                if (pod.metadata?.name) {
-                    await instance.batchApi.deleteNamespacedJob({ name: pod.metadata.name, namespace: kubernetes.namespace });
+            for (const job of jobs.items) {
+                if (job.metadata?.name) {
+                    await (0, await_running_state_1.stopJob)(instance, kubernetes, job.metadata.name);
                 }
             }
             const deployments = await instance.appsApi.listNamespacedDeployment({
@@ -189,13 +393,19 @@ function kubernetesTaskRuntime(task, kubernetes) {
                     await instance.coreApi.deleteNamespacedPod({ name: pod.metadata.name, namespace: kubernetes.namespace });
                 }
             }
+            // the limits go last, once no pod of the task runs
+            await (0, network_policy_1.removeAllKubernetesItemNetworks)(instance, kubernetes, task);
         },
         async remove() {
             await this.stop();
             await (0, remove_persistent_data_1.removePersistentData)(instance, kubernetes, task);
-            await (0, secrets_1.removeKubernetesSecret)(instance, kubernetes, task);
+            await removeTaskSecrets(instance, kubernetes, task);
         },
         async currentStateKey() {
+            if (kept) {
+                // the server removes the Jobs of a run at its end, the claim says
+                return (0, kept_claims_1.useClaimState)(instance, kubernetes, claim);
+            }
             const jobs = await instance.batchApi.listNamespacedJob({
                 namespace: kubernetes.namespace,
                 labelSelector: `hammerkit.dev/id=${task.id()}`,
@@ -210,11 +420,11 @@ function kubernetesTaskRuntime(task, kubernetes) {
             // outputs live in the task's claim; without it the finished job left
             // nothing to reuse
             try {
-                const claim = await instance.coreApi.readNamespacedPersistentVolumeClaim({
-                    name: (0, volumes_1.getVolumeName)(task),
+                const found = await instance.coreApi.readNamespacedPersistentVolumeClaim({
+                    name: claim,
                     namespace: kubernetes.namespace,
                 });
-                return claim.metadata?.deletionTimestamp ? null : stateKey;
+                return found.metadata?.deletionTimestamp ? null : stateKey;
             }
             catch (e) {
                 if ((0, apply_1.statusCodeOf)(e) === 404) {
@@ -229,8 +439,29 @@ exports.kubernetesTaskRuntime = kubernetesTaskRuntime;
 function kubernetesServiceRuntime(service, kubernetes) {
     const instance = (0, kubernetes_instance_1.createKubernetesInstances)(kubernetes);
     return {
-        initialize() {
-            return Promise.resolve();
+        // A Deployment left by an earlier `up` is the running service, so `down`
+        // stops it and a run reuses it, as with a container on Docker. One whose
+        // definition changed goes, with its limits, and is created anew.
+        async initialize(state) {
+            const deployments = await instance.appsApi.listNamespacedDeployment({
+                namespace: kubernetes.namespace,
+                labelSelector: `hammerkit.dev/id=${service.id()}`,
+            });
+            const deployment = deployments.items[0];
+            if (!deployment?.metadata?.name) {
+                return;
+            }
+            if (deployment.metadata.labels?.[ensure_kubernetes_deployment_exists_1.DEFINITION_LABEL] !== (0, service_definition_1.getServiceDefinitionHash)(service)) {
+                service.status.write('info', `${service.name} changed since it was started, recreating it`);
+                await this.stop();
+                return;
+            }
+            state.set({
+                type: 'running',
+                dns: { containerId: deployment.metadata.name },
+                stateKey: deployment.metadata.labels?.['hammerkit.dev/state'] ?? '',
+                remote: null,
+            });
         },
         async restore(environment, path) {
             await (0, restore_kubernetes_data_1.restoreKubernetesData)(service, kubernetes, instance, environment, path);
@@ -252,14 +483,17 @@ function kubernetesServiceRuntime(service, kubernetes) {
                 }
             }
             await (0, secrets_1.removeKubernetesSecret)(instance, kubernetes, service);
+            // the limits go last, once the service's pods ended
+            await (0, network_policy_1.removeAllKubernetesItemNetworks)(instance, kubernetes, service);
         },
         async execute(environment, options) {
-            const persistence = await (0, volumes_1.getKubernetesPersistence)(service);
+            const persistence = await (0, volumes_1.getKubernetesPersistence)(service, kubernetes);
             await (0, ensure_namespace_1.ensureNamespace)(instance, kubernetes.namespace);
             await (0, ensure_kubernetes_service_exists_1.ensureKubernetesServiceExists)(instance, kubernetes, service);
             await (0, ensure_persistent_data_1.ensurePersistentData)(instance, kubernetes, environment, service, persistence);
             await (0, secrets_1.ensureKubernetesSecret)(instance, kubernetes, service, service.data.secrets, environment);
-            await (0, ensure_kubernetes_deployment_exists_1.ensureKubernetesDeploymentExists)(instance, kubernetes, service, persistence, (0, secrets_1.getKubernetesSecretRefs)(service, service.data.secrets), options.stateKey);
+            const network = await (0, network_policy_1.ensureKubernetesItemNetwork)(instance, kubernetes, service, { followLog: false, run: null });
+            await (0, ensure_kubernetes_deployment_exists_1.ensureKubernetesDeploymentExists)(instance, kubernetes, service, persistence, (0, secrets_1.getKubernetesSecretRefs)(service, service.data.secrets), options.stateKey, network);
             const name = (0, resources_1.getResourceName)(service);
             options.state.set({
                 type: 'running',

package/dist/planner/work-runtime-local.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getLocalWorkRuntime = void 0;
+exports.getLocalWorkRuntime = exports.getStateFilename = void 0;
 const tar_1 = require("tar");
 const path_1 = require("path");
 const local_task_1 = require("../executer/local-task");
@@ -11,9 +11,11 @@ const event_cache_1 = require("../executer/event-cache");
 const get_outputs_to_reset_1 = require("./utils/get-outputs-to-reset");
 const find_process_1 = __importDefault(require("find-process"));
 const stream_1 = require("stream");
+const work_state_record_1 = require("./work-state-record");
 function getStateFilename(task) {
-    return (0, path_1.join)(task.data.cwd, '.hammerkit', `${task.id()}`);
+    return (0, path_1.join)(task.data.cwd, '.hammerkit', 'state', (0, work_state_record_1.getStateRecordName)(task));
 }
+exports.getStateFilename = getStateFilename;
 function getPidFilename(task) {
     return (0, path_1.join)(task.data.cwd, '.hammerkit', `${task.id()}.pid`);
 }
@@ -39,6 +41,8 @@ function getLocalWorkRuntime(task) {
             // pidfile check happens in execute (initialize has no Environment in scope)
         },
         async restore(environment, path) {
+            // the restored outputs aren't the ones the record names any more
+            await removeStateFile(environment, task);
             await restoreLocal(environment, task.data, path);
         },
         async archive(environment, path) {
@@ -54,7 +58,7 @@ function getLocalWorkRuntime(task) {
                 });
                 return;
             }
-            await environment.file.createDirectory((0, path_1.join)(task.data.cwd, '.hammerkit'));
+            await environment.file.createDirectory((0, path_1.join)(task.data.cwd, '.hammerkit', 'state'));
             const pidFile = getPidFilename(task);
             await environment.file.writeFile(pidFile, `${process.pid}`);
             try {
@@ -62,16 +66,14 @@ function getLocalWorkRuntime(task) {
                 // fails overwrites outputs, so the previous record goes first, and only
                 // a run that succeeded writes a new one.
                 const stateFile = getStateFilename(task);
-                if (await environment.file.exists(stateFile)) {
-                    await environment.file.remove(stateFile);
-                }
+                await removeStateFile(environment, task);
                 for (const generate of (0, get_outputs_to_reset_1.getOutputsToReset)(task.data)) {
                     await environment.file.remove(generate.path);
                 }
                 await (0, local_task_1.localTask)(task, environment, options);
                 const failed = ['crash', 'error', 'canceled'].includes(options.state.current.type);
                 if (!failed) {
-                    await environment.file.writeFile(stateFile, options.stateKey);
+                    await environment.file.writeFile(stateFile, (0, work_state_record_1.formatStateRecord)(task, options.stateKey));
                 }
             }
             finally {
@@ -95,7 +97,7 @@ function getLocalWorkRuntime(task) {
                     return null;
                 }
             }
-            return await environment.file.read(stateFileName);
+            return (0, work_state_record_1.parseStateRecord)(task, await environment.file.read(stateFileName));
         },
         async remove(environment) {
             for (const generate of task.data.generates) {
@@ -109,6 +111,12 @@ function getLocalWorkRuntime(task) {
     };
 }
 exports.getLocalWorkRuntime = getLocalWorkRuntime;
+async function removeStateFile(environment, task) {
+    const stateFile = getStateFilename(task);
+    if (await environment.file.exists(stateFile)) {
+        await environment.file.remove(stateFile);
+    }
+}
 async function archiveLocal(environment, task, path) {
     for (const generatedArchive of (0, event_cache_1.getArchivePaths)(task.data, path)) {
         await environment.file.writeStream(generatedArchive.filename,