hammerkit 1.27.0-next.3 → 1.27.0-next.31

package/dist/kubernetes/apply.js

@@ -36,28 +36,39 @@ exports.apply = apply;
  * of a policy).
  */
 async function replace(instance, spec) {
-    let current;
-    try {
-        current = await instance.objectApi.read(spec);
-    }
-    catch (e) {
-        if (statusCodeOf(e) !== 404) {
-            throw e;
+    for (let attempt = 1;; attempt++) {
+        let current;
+        try {
+            current = await instance.objectApi.read(spec);
+        }
+        catch (e) {
+            if (statusCodeOf(e) !== 404) {
+                throw e;
+            }
+            try {
+                return await instance.objectApi.create(spec);
+            }
+            catch (createError) {
+                if (statusCodeOf(createError) !== 409) {
+                    throw createError;
+                }
+                current = await instance.objectApi.read(spec);
+            }
         }
         try {
-            return await instance.objectApi.create(spec);
+            return await instance.objectApi.replace({
+                ...spec,
+                metadata: { ...spec.metadata, resourceVersion: current.metadata?.resourceVersion },
+            });
         }
-        catch (createError) {
-            if (statusCodeOf(createError) !== 409) {
-                throw createError;
+        catch (e) {
+            // replaced by someone else since it was read (the server sets up a caller's
+            // namespace for a run and for a sandbox at the same time): read it again
+            if (statusCodeOf(e) !== 409 || attempt >= 3) {
+                throw e;
             }
-            current = await instance.objectApi.read(spec);
         }
     }
-    return await instance.objectApi.replace({
-        ...spec,
-        metadata: { ...spec.metadata, resourceVersion: current.metadata?.resourceVersion },
-    });
 }
 exports.replace = replace;
 //# sourceMappingURL=apply.js.map

package/dist/kubernetes/await-running-state.js

@@ -1,10 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.stopJob = exports.deleteJobAndWait = exports.awaitDeployRunningState = exports.deleteJob = exports.awaitJobCompletion = exports.awaitRunningState = void 0;
+exports.stopJob = exports.deleteJobAndWait = exports.awaitDeployRunningState = exports.PASSING_FAILURE_MS = exports.deleteJob = exports.awaitJobCompletion = exports.awaitRunningState = void 0;
 const abort_1 = require("../executer/abort");
 const abort_event_1 = require("../utils/abort-event");
 const apply_1 = require("./apply");
 const sleep_1 = require("../utils/sleep");
+const cluster_state_1 = require("./cluster-state");
 // The waits below poll instead of using the client's Watch: Watch aborts every
 // request after 30s, which a pod or deployment still pulling its image outlasts.
 async function awaitRunningState(instance, env, name, phase, pollInterval = 1000) {
@@ -76,29 +77,74 @@ async function deleteJob(instance, env, name) {
     }
 }
 exports.deleteJob = deleteJob;
-// Waits for a ready replica. The deadline only starts once a pod runs, so an
-// image pull doesn't count against it (as on docker, where it starts after
-// the container started).
+// how long a reason that may still pass (an image pull a registry refused for now, a
+// container the runtime could not create) may last before it fails the service
+exports.PASSING_FAILURE_MS = 60000;
+// Waits for an available replica: a ready one, for a service without a healthcheck one that
+// stayed ready for a while. The deadline only starts once the container runs, and again
+// at each start of it, so an image pull or a back-off doesn't count against it (as on
+// docker, where it starts after the container started). A pod that won't get there without a change (an image
+// that isn't there, a container that crashed once more than its
+// service's `restarts`, 2 by default) fails it at once, with
+// the reason the cluster gives; one that may still pass fails once it lasted a minute.
+// Only the pods of this Deployment count: not the ones of one deleted before it was
+// created again, which may still be there, terminating or not yet.
 async function awaitDeployRunningState(instance, env, name, readiness, pollInterval = 1000, now = Date.now) {
     let runningSince = null;
+    // how often the container was restarted when last looked at
+    let restarts = 0;
+    let said;
+    // since when a pod has had a failure that may still pass, by pod
+    const failingSince = new Map();
     for (;;) {
         const deployment = await instance.appsApi.readNamespacedDeployment({ name, namespace: env.namespace });
-        if ((deployment.status?.readyReplicas ?? 0) > 0) {
+        // available: ready for the Deployment's minReadySeconds
+        if ((deployment.status?.availableReplicas ?? 0) > 0) {
             return;
         }
+        const created = deployment.metadata?.creationTimestamp;
+        const listed = await instance.coreApi.listNamespacedPod({
+            namespace: env.namespace,
+            labelSelector: readiness.podSelector,
+        });
+        const pods = listed.items.filter((pod) => !pod.metadata?.deletionTimestamp &&
+            !(created && pod.metadata?.creationTimestamp && pod.metadata.creationTimestamp < created));
+        for (const [index, pod] of pods.entries()) {
+            const key = pod.metadata?.name ?? `${index}`;
+            const failure = (0, cluster_state_1.getPodFailure)(pod, readiness.container, readiness.restarts);
+            if (!failure) {
+                failingSince.delete(key);
+                continue;
+            }
+            const since = failingSince.get(key) ?? now();
+            failingSince.set(key, since);
+            if (failure.final || now() - since >= exports.PASSING_FAILURE_MS) {
+                throw new Error(`${readiness.subject ?? name} can not become ready, ${failure.text}`);
+            }
+        }
+        const waiting = pods.map((pod) => (0, cluster_state_1.getPodWaiting)(pod, readiness.container)).find((text) => !!text);
+        if (waiting && waiting !== said) {
+            readiness.onWaiting?.(waiting);
+        }
+        said = waiting;
         if (readiness.timeout !== null) {
-            if (runningSince === null) {
-                const pods = await instance.coreApi.listNamespacedPod({
-                    namespace: env.namespace,
-                    labelSelector: readiness.podSelector,
-                });
-                if (pods.items.some((pod) => pod.status?.phase === 'Running')) {
-                    runningSince = now();
-                }
+            // the deadline bounds each start of the container, as on Docker: a crash is counted by
+            // `restarts`, and the back-off before the next start doesn't count against it
+            const statuses = pods.flatMap((pod) => (pod.status?.containerStatuses ?? []).filter((status) => !readiness.container || status.name === readiness.container));
+            const running = statuses.length > 0
+                ? statuses.some((status) => status.state?.running)
+                : pods.some((pod) => pod.status?.phase === 'Running');
+            const restarted = Math.max(0, ...statuses.map((status) => status.restartCount ?? 0));
+            if (!running) {
+                runningSince = null;
+            }
+            else if (runningSince === null || restarted !== restarts) {
+                runningSince = now();
             }
             else if (now() - runningSince >= readiness.timeout) {
                 throw new Error(readiness.timeoutMessage);
             }
+            restarts = restarted;
         }
         await (0, sleep_1.sleep)(pollInterval);
     }

package/dist/kubernetes/claim-layout.js

@@ -0,0 +1,71 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.hasOutputsFlatteningAlike = exports.hasPathsEndingAlike = exports.byDepth = exports.getClaimSubPath = void 0;
+const crypto_1 = require("crypto");
+const path_1 = require("path");
+const portable_path_1 = require("../planner/utils/portable-path");
+// The paths a work item keeps in its claim. Its consumers mount its outputs by the same
+// paths, so they find the same folders.
+function getClaimPaths(work) {
+    const data = work.data;
+    const paths = [];
+    if ('mounts' in data) {
+        paths.push(...data.mounts.map((mount) => mount.containerPath));
+    }
+    if ('src' in data) {
+        paths.push(...data.src.filter((src) => !src.inherited).map((src) => src.absolutePath));
+    }
+    if ('generates' in data) {
+        paths.push(...data.generates.filter((generate) => !generate.inherited).map((generate) => generate.path));
+    }
+    if ('volumes' in data) {
+        paths.push(...data.volumes.filter((volume) => !volume.inherited).map((volume) => volume.containerPath));
+    }
+    return paths;
+}
+// Where `path` is in the claim of `work`, relative to its root. A path is named after its
+// last segment, so claims without two such paths are laid out as they always were (a
+// service's data among them). Of the paths with the same last segment, the least deep one
+// (then the first in sorted order) keeps the name, and the others also carry a hash of
+// their path. Both go by the path
+// relative to the project, as the task's id does, so a run in another checkout directory
+// (each run of a server has its own) or with its `generates` in another order finds the
+// same folders in a kept claim.
+function getClaimSubPath(work, path) {
+    const name = (0, path_1.basename)(path) || 'root';
+    const alike = getClaimPaths(work).filter((other) => other !== path && (0, path_1.basename)(other) === (0, path_1.basename)(path));
+    if (alike.length === 0) {
+        return name;
+    }
+    const own = (0, portable_path_1.portablePath)(work.data.projectRoot, path);
+    const first = [own, ...alike.map((other) => (0, portable_path_1.portablePath)(work.data.projectRoot, other))].sort(byDepth)[0];
+    if (first === own) {
+        return name;
+    }
+    return `${name}-${(0, crypto_1.createHash)('sha256').update(own).digest('hex').slice(0, 12)}`;
+}
+exports.getClaimSubPath = getClaimSubPath;
+function byDepth(a, b) {
+    return a.split('/').length - b.split('/').length || (a < b ? -1 : a > b ? 1 : 0);
+}
+exports.byDepth = byDepth;
+// Whether two different paths of the claim of `work` end alike. Before getClaimSubPath
+// named them apart, they shared one folder, and outputs made then are mixed.
+function hasPathsEndingAlike(work) {
+    const paths = new Set(getClaimPaths(work));
+    return new Set([...paths].map((path) => (0, path_1.basename)(path))).size < paths.size;
+}
+exports.hasPathsEndingAlike = hasPathsEndingAlike;
+// Whether two outputs of `work` flatten to the same name of a cache archive (`a-b` and
+// `a/b`, see getArchivePaths). Before they were named apart, one archive held both.
+function hasOutputsFlatteningAlike(work) {
+    const data = work.data;
+    if (!('generates' in data) || !('cwd' in data)) {
+        return false;
+    }
+    const paths = new Set(data.generates.filter((generate) => !generate.inherited).map((generate) => generate.path));
+    const names = new Set([...paths].map((path) => (0, path_1.relative)(data.cwd, path).split(path_1.sep).join('-')));
+    return names.size < paths.size;
+}
+exports.hasOutputsFlatteningAlike = hasOutputsFlatteningAlike;
+//# sourceMappingURL=claim-layout.js.map

package/dist/kubernetes/cluster-state.js

@@ -0,0 +1,245 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readClusterState = exports.describeEvent = exports.describePod = exports.getPodWaiting = exports.describeLastExit = exports.getPodFailure = exports.CRASHES_TO_FAIL = void 0;
+const log_1 = require("../log");
+const apply_1 = require("./apply");
+const job_placement_1 = require("./job-placement");
+const network_gate_1 = require("./network-gate");
+const resources_1 = require("./resources");
+// what a pod's or event's message is cut to
+const MAX_MESSAGE = 500;
+// how many events of a run are said, the newest
+const MAX_EVENTS = 20;
+// events the result of a task says better: a Job of a failed command
+const SAID_ELSEWHERE = ['BackoffLimitExceeded'];
+// A container restarted this often (ended three times) is taken as one that keeps crashing,
+// unless its service sets `restarts`: once may be a race with a dependency or a slow first start
+exports.CRASHES_TO_FAIL = 2;
+// what a pull error says when the image is not there to pull, or not for this cluster:
+// waiting does not change it. Others (a registry that limits, a token not there yet) may pass.
+const IMAGE_GONE = /not found|manifest unknown|unauthorized|denied|forbidden|invalid reference|no such host/i;
+/**
+ * Why a pod of a service will not become ready without a change, as the cluster says it:
+ * an image that can not be pulled, a container that keeps crashing or can not be created,
+ * an init container that failed. Null while it may still get there. A container fails once
+ * it was restarted `restarts` times and crashed again.
+ *
+ * Only what the cluster wrote is quoted: the termination message of a container is its
+ * own output (and may hold a value it read), except for the network gate's, which says why
+ * the policy did not take hold.
+ */
+function getPodFailure(pod, container, restarts = exports.CRASHES_TO_FAIL) {
+    for (const status of pod.status?.initContainerStatuses ?? []) {
+        const terminated = status.state?.terminated ?? status.lastState?.terminated;
+        if (!terminated || terminated.exitCode === 0) {
+            continue;
+        }
+        if (status.name === network_gate_1.NETWORK_GATE_CONTAINER) {
+            return {
+                text: `the network policy did not take hold: ${cut(terminated.message?.trim() || 'the gate failed')}`,
+                final: true,
+            };
+        }
+        if (status.restartCount >= exports.CRASHES_TO_FAIL) {
+            return { text: `its init container ${status.name} failed with exit code ${terminated.exitCode}`, final: true };
+        }
+    }
+    for (const status of mainStatuses(pod, container)) {
+        const failure = getContainerFailure(status, restartsEnded(pod, status), restarts);
+        if (failure) {
+            return failure;
+        }
+    }
+    return null;
+}
+exports.getPodFailure = getPodFailure;
+function getContainerFailure(status, restartsEnded, restarts) {
+    const waiting = status.state?.waiting;
+    const reason = waiting?.reason;
+    const said = waiting?.message ? `: ${cut(waiting.message)}` : '';
+    switch (reason) {
+        // the first pull: final only when the registry said the image is not there, as older
+        // kubelets leave that out of the ImagePullBackOff that follows
+        case 'ErrImagePull':
+            return IMAGE_GONE.test(waiting?.message ?? '')
+                ? { text: `its image ${status.image} could not be pulled (${reason})${said}`, final: true }
+                : null;
+        case 'ImagePullBackOff':
+            return {
+                text: `its image ${status.image} could not be pulled (${reason})${said}`,
+                final: IMAGE_GONE.test(waiting?.message ?? ''),
+            };
+        case 'ErrImageNeverPull':
+        case 'InvalidImageName':
+            return { text: `its image ${status.image} could not be pulled (${reason})${said}`, final: true };
+        case 'CreateContainerConfigError':
+        case 'CreateContainerError':
+        case 'RunContainerError':
+            return { text: `its container could not be started (${reason})${said}`, final: false };
+        case 'CrashLoopBackOff':
+            return status.restartCount >= restarts
+                ? { text: `it keeps crashing: ${describeLastExit(status)}`, final: true }
+                : null;
+    }
+    // a kubelet may leave an instance that ended as it is until its back-off is over, and say
+    // CrashLoopBackOff only when it looks at the pod again, a minute later: the same crash
+    if (restartsEnded && status.state?.terminated && status.restartCount >= restarts) {
+        return { text: `it keeps crashing: ${describeLastExit(status)}`, final: true };
+    }
+    return null;
+}
+// How the last instance of a container ended, as the cluster says it.
+function describeLastExit(status) {
+    // the instance that ended now, else the one before the one that waits or runs
+    const last = status.state?.terminated ?? status.lastState?.terminated;
+    const restarts = `, restarted ${status.restartCount} time${status.restartCount === 1 ? '' : 's'}`;
+    if (!last) {
+        return `it ended${restarts}`;
+    }
+    if (last.reason === 'OOMKilled') {
+        return `it was killed for using more memory than its limit (OOMKilled)${restarts}`;
+    }
+    // the kubelet kills a container whose liveness probe (its healthcheck) keeps failing
+    const killed = last.exitCode === 137 || last.exitCode === 143 ? ', killed as when its healthcheck kept failing' : '';
+    return `it exited with code ${last.exitCode}${last.reason ? ` (${last.reason})` : ''}${killed}${restarts}`;
+}
+exports.describeLastExit = describeLastExit;
+/**
+ * What a pod of a service waits for, in a few words, while it is not ready: a node, its
+ * image, its container. Undefined when it waits for nothing the cluster names.
+ */
+function getPodWaiting(pod, container) {
+    const scheduled = pod.status?.conditions?.find((condition) => condition.type === 'PodScheduled');
+    if (scheduled?.status === 'False') {
+        return `waits for a node: ${cut(scheduled.message ?? scheduled.reason ?? 'not scheduled yet')}`;
+    }
+    for (const status of mainStatuses(pod, container)) {
+        const waiting = status.state?.waiting;
+        const said = waiting?.message ? `: ${cut(waiting.message)}` : '';
+        switch (waiting?.reason) {
+            case 'ErrImagePull':
+            case 'ImagePullBackOff':
+                return `can not pull its image ${status.image} yet${said}`;
+            case 'CreateContainerConfigError':
+            case 'CreateContainerError':
+            case 'RunContainerError':
+                return `can not start its container yet (${waiting.reason})${said}`;
+            case 'CrashLoopBackOff':
+                return `crashed and starts again: ${describeLastExit(status)}`;
+        }
+        const ended = status.state?.terminated;
+        if (ended && restartsEnded(pod, status)) {
+            return `${ended.exitCode === 0 ? 'ended' : 'crashed'} and starts again: ${describeLastExit(status)}`;
+        }
+        if (status.restartCount > 0 && status.state?.running && !status.ready) {
+            return `restarted: ${describeLastExit(status)}`;
+        }
+    }
+    return undefined;
+}
+exports.getPodWaiting = getPodWaiting;
+// Whether the kubelet starts a container that ended again: not one of a pod that finished (a
+// task's Job, an evicted pod, which its ReplicaSet replaces) or that never restarts.
+function restartsEnded(pod, status) {
+    const phase = pod.status?.phase;
+    if (phase === 'Succeeded' || phase === 'Failed') {
+        return false;
+    }
+    const policy = pod.spec?.restartPolicy ?? 'Always';
+    return policy === 'Always' || (policy === 'OnFailure' && status.state?.terminated?.exitCode !== 0);
+}
+function mainStatuses(pod, container) {
+    return (pod.status?.containerStatuses ?? []).filter((status) => !container || status.name === container);
+}
+// One line of a pod that is not as it should be, or null for one that runs, ready, or ended well.
+function describePod(pod) {
+    const name = pod.metadata?.name ?? '?';
+    const phase = pod.status?.phase ?? 'Unknown';
+    const failure = getPodFailure(pod)?.text;
+    const waiting = getPodWaiting(pod);
+    const unready = (pod.status?.containerStatuses ?? [])
+        .filter((status) => !status.ready && !status.state?.terminated)
+        .map((status) => status.name);
+    const crashed = (pod.status?.containerStatuses ?? []).filter((status) => (status.state?.terminated && status.state.terminated.exitCode !== 0) || status.restartCount > 0);
+    if (!failure && !waiting && crashed.length === 0 && (phase === 'Succeeded' || unready.length === 0)) {
+        return null;
+    }
+    const said = failure ?? waiting;
+    const parts = [phase, ...(said ? [said] : [])];
+    for (const status of crashed) {
+        if (!said?.includes(describeLastExit(status))) {
+            parts.push(`${status.name}: ${describeLastExit(status)}`);
+        }
+    }
+    if (!said && unready.length > 0) {
+        parts.push(`not ready: ${unready.join(', ')}`);
+    }
+    return `pod ${name}: ${parts.join(', ')}`;
+}
+exports.describePod = describePod;
+/**
+ * The message of a warning event, as far as it is the cluster's: a failed probe says the
+ * output of the command, which is the service's, and is left out.
+ */
+function describeEvent(event) {
+    const object = event.involvedObject;
+    const message = (event.message ?? '').replace(/^((Readiness|Liveness|Startup) probe (failed|errored)).*$/s, '$1');
+    const count = (event.count ?? 1) > 1 ? ` (${event.count} times)` : '';
+    return `${event.reason ?? 'Warning'} ${object.kind?.toLowerCase() ?? 'object'}/${object.name ?? '?'}: ${cut(message.trim())}${count}`;
+}
+exports.describeEvent = describeEvent;
+/**
+ * What the cluster said of a run that failed, into its log before the server removes its
+ * objects: its pods that are not as they should be, and the warning events of its objects.
+ * In the namespace of a service run everything is the run's; in a caller's namespace, which
+ * runs share, only the objects of the run's items (named by their key) are.
+ */
+async function readClusterState(instance, kubernetes, workTree, ownNamespace) {
+    const namespace = kubernetes.namespace;
+    const keys = [...Object.values(workTree.tasks), ...Object.values(workTree.services)].map((item) => (0, resources_1.getItemKey)(item, kubernetes));
+    const ours = (name) => ownNamespace || keys.some((key) => name?.includes(key));
+    const lines = [];
+    try {
+        const labels = (0, job_placement_1.getRunLabels)(kubernetes);
+        const labelSelector = Object.entries(labels)
+            .map(([key, value]) => `${key}=${value}`)
+            .join(',');
+        const pods = await instance.coreApi.listNamespacedPod({
+            namespace,
+            ...(labelSelector && !ownNamespace ? { labelSelector } : {}),
+        });
+        for (const pod of pods.items) {
+            const line = ours(pod.metadata?.name) ? describePod(pod) : null;
+            if (line) {
+                lines.push(line);
+            }
+        }
+    }
+    catch (e) {
+        lines.push(`its pods could not be read: ${(0, log_1.getErrorMessage)(e)}`);
+    }
+    try {
+        const events = await instance.coreApi.listNamespacedEvent({ namespace, fieldSelector: 'type=Warning' });
+        const warnings = events.items
+            .filter((event) => ours(event.involvedObject.name) && !SAID_ELSEWHERE.includes(event.reason ?? ''))
+            .sort((a, b) => eventTime(a) - eventTime(b))
+            .slice(-MAX_EVENTS);
+        lines.push(...warnings.map((event) => `event ${describeEvent(event)}`));
+    }
+    catch (e) {
+        lines.push((0, apply_1.statusCodeOf)(e) === 403
+            ? `its events could not be read: the runRole of the server needs list on events (see the backend docs)`
+            : `its events could not be read: ${(0, log_1.getErrorMessage)(e)}`);
+    }
+    return lines;
+}
+exports.readClusterState = readClusterState;
+function eventTime(event) {
+    const time = event.lastTimestamp ?? event.eventTime ?? event.firstTimestamp ?? event.metadata?.creationTimestamp;
+    return time ? new Date(time).getTime() : 0;
+}
+function cut(text) {
+    const line = text.replace(/\s+/g, ' ');
+    return line.length > MAX_MESSAGE ? `${line.slice(0, MAX_MESSAGE)}…` : line;
+}
+//# sourceMappingURL=cluster-state.js.map

package/dist/kubernetes/ensure-http-route.js

@@ -36,7 +36,7 @@ async function ensureHttpRoute(instance, env, route, service) {
                     ],
                     backendRefs: [
                         {
-                            name: (0, resources_1.getResourceName)(service),
+                            name: (0, resources_1.getResourceName)(service, env),
                             port: route.servicePort ?? service.data.ports[0].containerPort,
                         },
                     ],

package/dist/kubernetes/ensure-kubernetes-deployment-exists.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ensureKubernetesDeploymentExists = exports.DEFINITION_LABEL = void 0;
+exports.getServiceContainerName = exports.MIN_READY_SECONDS = exports.ensureKubernetesDeploymentExists = exports.DEFINITION_LABEL = void 0;
 const service_definition_1 = require("../planner/service-definition");
 const work_service_1 = require("../planner/work-service");
 const volumes_1 = require("./volumes");
@@ -17,11 +17,13 @@ const network_policy_1 = require("./network-policy");
 const egress_proxy_1 = require("../network/egress-proxy");
 // what the service was created from, to tell a changed one
 exports.DEFINITION_LABEL = 'hammerkit.dev/definition';
+// how often a starting container is checked: as the liveness probe
+const STARTUP_PERIOD_SECONDS = 5;
 async function ensureKubernetesDeploymentExists(instance, env, service, persistence, secrets, stateKey, 
 // the needed services by name, and the proxy of a limited network
 network) {
     const envs = (0, replace_env_variables_1.getEnvironmentVariables)(service.data.envs);
-    const name = (0, resources_1.getResourceName)(service);
+    const name = (0, resources_1.getResourceName)(service, env);
     const healthcheckCommand = (0, work_service_1.getHealthcheckCommand)(service.data);
     const serviceCommand = (0, work_service_1.getServiceCommand)(service.data, { kubernetes: true });
     const probe = healthcheckCommand
@@ -34,6 +36,16 @@ network) {
             failureThreshold: 3,
         }
         : undefined;
+    // the healthcheck's `timeout` for each start: until it passed once, the liveness probe
+    // doesn't kill a container that starts slowly, and the wait fails it with the timeout
+    const startupProbe = probe && service.data.healthcheck
+        ? {
+            ...probe,
+            // a check more than the timeout holds, and one more for the first check's offset: the
+            // wait fails it with the timeout before the kubelet restarts it
+            failureThreshold: Math.ceil(service.data.healthcheck.timeout / (STARTUP_PERIOD_SECONDS * 1000)) + 2,
+        }
+        : undefined;
     const server = env.server;
     const placement = server ? (0, job_placement_1.getPodPlacement)(server, undefined) : null;
     const workspace = (0, volumes_1.getWorkspacePodSpec)(env, persistence);
@@ -47,25 +59,24 @@ network) {
                 'hammerkit.dev/version': (0, version_1.getVersion)(),
             },
             labels: {
-                'hammerkit.dev/id': service.id(),
+                ...(0, resources_1.getItemLabels)(service, env),
                 'hammerkit.dev/state': stateKey,
-                ...(0, job_placement_1.getRunLabels)(env),
                 [exports.DEFINITION_LABEL]: (0, service_definition_1.getServiceDefinitionHash)(service),
             },
         },
         spec: {
             selector: {
-                matchLabels: {
-                    'hammerkit.dev/id': service.id(),
-                },
+                matchLabels: (0, resources_1.getItemLabels)(service, env),
             },
             replicas: 1,
+            // without a healthcheck a pod is ready as soon as its container runs, also one that
+            // exits a second later: it counts once it ran this long without a restart
+            minReadySeconds: probe ? 0 : exports.MIN_READY_SECONDS,
             template: {
                 metadata: {
                     labels: {
-                        'hammerkit.dev/id': service.id(),
+                        ...(0, resources_1.getItemLabels)(service, env),
                         'hammerkit.dev/state': stateKey,
-                        ...(0, job_placement_1.getRunLabels)(env),
                         ...workspace.labels,
                         ...(0, network_policy_1.getNetworkLabels)(env, service),
                     },
@@ -87,7 +98,7 @@ network) {
                         : {}),
                     containers: [
                         {
-                            name: service.name.replace(/:/, '-'),
+                            name: getServiceContainerName(service),
                             image: service.data.image,
                             workingDir: (0, work_service_1.getServiceWorkingDir)(service.data),
                             env: [
@@ -104,6 +115,7 @@ network) {
                             })),
                             readinessProbe: probe,
                             livenessProbe: probe,
+                            startupProbe,
                             ...(0, container_resources_1.getKubernetesResources)(service.data.resources),
                             volumeMounts: [...persistence.mounts.map((m) => m.mount), ...secrets.mounts],
                             ...(0, job_placement_1.getContainerSecurity)(env),
@@ -123,18 +135,29 @@ network) {
         },
     };
     const deploy = await (0, apply_1.apply)(instance, deployment);
-    if (deploy.status?.readyReplicas ?? 0 > 0) {
+    if ((deploy.status?.availableReplicas ?? 0) > 0) {
         return;
     }
     await (0, await_running_state_1.awaitDeployRunningState)(instance, env, name, {
-        podSelector: `hammerkit.dev/id=${service.id()}`,
+        podSelector: (0, resources_1.getItemSelector)(service, env),
         timeout: service.data.healthcheck?.timeout ?? null,
         timeoutMessage: service.data.healthcheck
             ? (0, work_healthcheck_1.getHealthcheckTimeoutMessage)(service.name, service.data.healthcheck)
             : '',
+        container: getServiceContainerName(service),
+        subject: `service ${service.name}`,
+        onWaiting: (text) => service.status.write('warn', `${service.name} ${text}`),
+        ...(service.data.restarts !== null ? { restarts: service.data.restarts } : {}),
     });
 }
 exports.ensureKubernetesDeploymentExists = ensureKubernetesDeploymentExists;
+// how long the container of a service without a healthcheck has to run before it counts as up
+exports.MIN_READY_SECONDS = 10;
+// The container of a service's pod that runs its command, beside the `debug` one.
+function getServiceContainerName(service) {
+    return service.name.replace(/:/, '-');
+}
+exports.getServiceContainerName = getServiceContainerName;
 // Kubernetes `command` replaces the image entrypoint and `args` its command.
 // A shell service sets both; an exec-form `cmd` keeps the established mapping
 // (first token as `command`, the rest as `args`).

package/dist/kubernetes/ensure-kubernetes-persistent-volume-claim-exists.js

@@ -65,7 +65,7 @@ async function ensureKubernetesPersistentVolumeClaimExists(instance, env, volume
             },
         },
     };
-    service.status.console('stdout', `ensure volume ${volume.name}`);
+    service.status.write('debug', `ensure volume ${volume.name}`);
     // Don't block on the PVC reaching "Bound". Storage classes with
     // volumeBindingMode WaitForFirstConsumer (docker-desktop's "standard",
     // k3d/k3s local-path, most cloud defaults) keep the claim Pending until a pod

package/dist/kubernetes/ensure-kubernetes-service-exists.js

@@ -4,11 +4,10 @@ exports.ensureKubernetesServiceExists = void 0;
 const apply_1 = require("./apply");
 const resources_1 = require("./resources");
 const version_1 = require("../version");
-const job_placement_1 = require("./job-placement");
 // A service without ports gets a headless Service: a ClusterIP one needs a port,
 // and its tasks reach every port of the pod, as on docker.
 async function ensureKubernetesServiceExists(instance, env, service) {
-    const name = (0, resources_1.getResourceName)(service);
+    const name = (0, resources_1.getResourceName)(service, env);
     const headless = service.data.ports.length === 0;
     const svc = {
         kind: 'Service',
@@ -19,21 +18,19 @@ async function ensureKubernetesServiceExists(instance, env, service) {
             annotations: {
                 'hammerkit.dev/version': (0, version_1.getVersion)(),
             },
-            labels: {
-                'hammerkit.dev/id': service.id(),
-                ...(0, job_placement_1.getRunLabels)(env),
-            },
+            labels: (0, resources_1.getItemLabels)(service, env),
         },
         spec: {
             type: 'ClusterIP',
             ...(headless ? { clusterIP: 'None' } : {}),
-            selector: {
-                'hammerkit.dev/id': service.id(),
-            },
+            // the pods of the service of this run, on a server where runs share the namespace
+            selector: (0, resources_1.getItemLabels)(service, env),
             ...(headless
                 ? {}
                 : {
+                    // a Service of more than one port needs a name for each
                     ports: service.data.ports.map((port) => ({
+                        name: `tcp-${port.hostPort ?? port.containerPort}`,
                         port: port.hostPort ?? port.containerPort,
                         targetPort: port.containerPort,
                     })),