hammerkit 1.27.0-next.28 → 1.27.0-next.29

package/build.schema.json

@@ -651,6 +651,12 @@
                     "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/resources",
                     "description": "cpu and memory requests and limits"
                   },
+                  "restarts": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "maximum": 10,
+                    "description": "crashes restarted before it fails on Kubernetes, 0 to 10, 2 by default"
+                  },
                   "volumeSize": {
                     "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/volumeSize",
                     "description": "size of its claim on Kubernetes, e.g. 10Gi"

package/dist/kubernetes/await-running-state.js

@@ -81,15 +81,18 @@ exports.deleteJob = deleteJob;
 // container the runtime could not create) may last before it fails the service
 exports.PASSING_FAILURE_MS = 60000;
 // Waits for an available replica: a ready one, for a service without a healthcheck one that
-// stayed ready for a while. The deadline only starts once a pod runs, so an
-// image pull doesn't count against it (as on docker, where it starts after
-// the container started). A pod that won't get there without a change (an image
-// that isn't there, a container that crashed twice in a row) fails it at once, with
+// stayed ready for a while. The deadline only starts once the container runs, and again
+// at each start of it, so an image pull or a back-off doesn't count against it (as on
+// docker, where it starts after the container started). A pod that won't get there without a change (an image
+// that isn't there, a container that crashed once more than its
+// service's `restarts`, 2 by default) fails it at once, with
 // the reason the cluster gives; one that may still pass fails once it lasted a minute.
 // Only the pods of this Deployment count: not the ones of one deleted before it was
 // created again, which may still be there, terminating or not yet.
 async function awaitDeployRunningState(instance, env, name, readiness, pollInterval = 1000, now = Date.now) {
     let runningSince = null;
+    // how often the container was restarted when last looked at
+    let restarts = 0;
     let said;
     // since when a pod has had a failure that may still pass, by pod
     const failingSince = new Map();
@@ -108,7 +111,7 @@ async function awaitDeployRunningState(instance, env, name, readiness, pollInter
             !(created && pod.metadata?.creationTimestamp && pod.metadata.creationTimestamp < created));
         for (const [index, pod] of pods.entries()) {
             const key = pod.metadata?.name ?? `${index}`;
-            const failure = (0, cluster_state_1.getPodFailure)(pod, readiness.container);
+            const failure = (0, cluster_state_1.getPodFailure)(pod, readiness.container, readiness.restarts);
             if (!failure) {
                 failingSince.delete(key);
                 continue;
@@ -125,14 +128,23 @@ async function awaitDeployRunningState(instance, env, name, readiness, pollInter
         }
         said = waiting;
         if (readiness.timeout !== null) {
-            if (runningSince === null) {
-                if (pods.some((pod) => pod.status?.phase === 'Running')) {
-                    runningSince = now();
-                }
+            // the deadline bounds each start of the container, as on Docker: a crash is counted by
+            // `restarts`, and the back-off before the next start doesn't count against it
+            const statuses = pods.flatMap((pod) => (pod.status?.containerStatuses ?? []).filter((status) => !readiness.container || status.name === readiness.container));
+            const running = statuses.length > 0
+                ? statuses.some((status) => status.state?.running)
+                : pods.some((pod) => pod.status?.phase === 'Running');
+            const restarted = Math.max(0, ...statuses.map((status) => status.restartCount ?? 0));
+            if (!running) {
+                runningSince = null;
+            }
+            else if (runningSince === null || restarted !== restarts) {
+                runningSince = now();
             }
             else if (now() - runningSince >= readiness.timeout) {
                 throw new Error(readiness.timeoutMessage);
             }
+            restarts = restarted;
         }
         await (0, sleep_1.sleep)(pollInterval);
     }

package/dist/kubernetes/cluster-state.js

@@ -12,8 +12,8 @@ const MAX_MESSAGE = 500;
 const MAX_EVENTS = 20;
 // events the result of a task says better: a Job of a failed command
 const SAID_ELSEWHERE = ['BackoffLimitExceeded'];
-// A container restarted this often in a row is taken as one that keeps crashing: once may be
-// a race with a dependency or a slow first start
+// A container restarted this often (ended three times) is taken as one that keeps crashing,
+// unless its service sets `restarts`: once may be a race with a dependency or a slow first start
 exports.CRASHES_TO_FAIL = 2;
 // what a pull error says when the image is not there to pull, or not for this cluster:
 // waiting does not change it. Others (a registry that limits, a token not there yet) may pass.
@@ -21,13 +21,14 @@ const IMAGE_GONE = /not found|manifest unknown|unauthorized|denied|forbidden|inv
 /**
  * Why a pod of a service will not become ready without a change, as the cluster says it:
  * an image that can not be pulled, a container that keeps crashing or can not be created,
- * an init container that failed. Null while it may still get there.
+ * an init container that failed. Null while it may still get there. A container fails once
+ * it was restarted `restarts` times and crashed again.
  *
  * Only what the cluster wrote is quoted: the termination message of a container is its
  * own output (and may hold a value it read), except for the network gate's, which says why
  * the policy did not take hold.
  */
-function getPodFailure(pod, container) {
+function getPodFailure(pod, container, restarts = exports.CRASHES_TO_FAIL) {
     for (const status of pod.status?.initContainerStatuses ?? []) {
         const terminated = status.state?.terminated ?? status.lastState?.terminated;
         if (!terminated || terminated.exitCode === 0) {
@@ -44,7 +45,7 @@ function getPodFailure(pod, container) {
         }
     }
     for (const status of mainStatuses(pod, container)) {
-        const failure = getContainerFailure(status);
+        const failure = getContainerFailure(status, restartsEnded(pod, status), restarts);
         if (failure) {
             return failure;
         }
@@ -52,7 +53,7 @@ function getPodFailure(pod, container) {
     return null;
 }
 exports.getPodFailure = getPodFailure;
-function getContainerFailure(status) {
+function getContainerFailure(status, restartsEnded, restarts) {
     const waiting = status.state?.waiting;
     const reason = waiting?.reason;
     const said = waiting?.message ? `: ${cut(waiting.message)}` : '';
@@ -76,15 +77,21 @@ function getContainerFailure(status) {
         case 'RunContainerError':
             return { text: `its container could not be started (${reason})${said}`, final: false };
         case 'CrashLoopBackOff':
-            return status.restartCount >= exports.CRASHES_TO_FAIL
+            return status.restartCount >= restarts
                 ? { text: `it keeps crashing: ${describeLastExit(status)}`, final: true }
                 : null;
     }
+    // a kubelet may leave an instance that ended as it is until its back-off is over, and say
+    // CrashLoopBackOff only when it looks at the pod again, a minute later: the same crash
+    if (restartsEnded && status.state?.terminated && status.restartCount >= restarts) {
+        return { text: `it keeps crashing: ${describeLastExit(status)}`, final: true };
+    }
     return null;
 }
 // How the last instance of a container ended, as the cluster says it.
 function describeLastExit(status) {
-    const last = status.lastState?.terminated ?? status.state?.terminated;
+    // the instance that ended now, else the one before the one that waits or runs
+    const last = status.state?.terminated ?? status.lastState?.terminated;
     const restarts = `, restarted ${status.restartCount} time${status.restartCount === 1 ? '' : 's'}`;
     if (!last) {
         return `it ended${restarts}`;
@@ -120,6 +127,10 @@ function getPodWaiting(pod, container) {
             case 'CrashLoopBackOff':
                 return `crashed and starts again: ${describeLastExit(status)}`;
         }
+        const ended = status.state?.terminated;
+        if (ended && restartsEnded(pod, status)) {
+            return `${ended.exitCode === 0 ? 'ended' : 'crashed'} and starts again: ${describeLastExit(status)}`;
+        }
         if (status.restartCount > 0 && status.state?.running && !status.ready) {
             return `restarted: ${describeLastExit(status)}`;
         }
@@ -127,6 +138,16 @@ function getPodWaiting(pod, container) {
     return undefined;
 }
 exports.getPodWaiting = getPodWaiting;
+// Whether the kubelet starts a container that ended again: not one of a pod that finished (a
+// task's Job, an evicted pod, which its ReplicaSet replaces) or that never restarts.
+function restartsEnded(pod, status) {
+    const phase = pod.status?.phase;
+    if (phase === 'Succeeded' || phase === 'Failed') {
+        return false;
+    }
+    const policy = pod.spec?.restartPolicy ?? 'Always';
+    return policy === 'Always' || (policy === 'OnFailure' && status.state?.terminated?.exitCode !== 0);
+}
 function mainStatuses(pod, container) {
     return (pod.status?.containerStatuses ?? []).filter((status) => !container || status.name === container);
 }

package/dist/kubernetes/ensure-kubernetes-deployment-exists.js

@@ -17,6 +17,8 @@ const network_policy_1 = require("./network-policy");
 const egress_proxy_1 = require("../network/egress-proxy");
 // what the service was created from, to tell a changed one
 exports.DEFINITION_LABEL = 'hammerkit.dev/definition';
+// how often a starting container is checked: as the liveness probe
+const STARTUP_PERIOD_SECONDS = 5;
 async function ensureKubernetesDeploymentExists(instance, env, service, persistence, secrets, stateKey, 
 // the needed services by name, and the proxy of a limited network
 network) {
@@ -34,6 +36,16 @@ network) {
             failureThreshold: 3,
         }
         : undefined;
+    // the healthcheck's `timeout` for each start: until it passed once, the liveness probe
+    // doesn't kill a container that starts slowly, and the wait fails it with the timeout
+    const startupProbe = probe && service.data.healthcheck
+        ? {
+            ...probe,
+            // a check more than the timeout holds, and one more for the first check's offset: the
+            // wait fails it with the timeout before the kubelet restarts it
+            failureThreshold: Math.ceil(service.data.healthcheck.timeout / (STARTUP_PERIOD_SECONDS * 1000)) + 2,
+        }
+        : undefined;
     const server = env.server;
     const placement = server ? (0, job_placement_1.getPodPlacement)(server, undefined) : null;
     const workspace = (0, volumes_1.getWorkspacePodSpec)(env, persistence);
@@ -103,6 +115,7 @@ network) {
                             })),
                             readinessProbe: probe,
                             livenessProbe: probe,
+                            startupProbe,
                             ...(0, container_resources_1.getKubernetesResources)(service.data.resources),
                             volumeMounts: [...persistence.mounts.map((m) => m.mount), ...secrets.mounts],
                             ...(0, job_placement_1.getContainerSecurity)(env),
@@ -134,6 +147,7 @@ network) {
         container: getServiceContainerName(service),
         subject: `service ${service.name}`,
         onWaiting: (text) => service.status.write('warn', `${service.name} ${text}`),
+        ...(service.data.restarts !== null ? { restarts: service.data.restarts } : {}),
     });
 }
 exports.ensureKubernetesDeploymentExists = ensureKubernetesDeploymentExists;

package/dist/planner/utils/append-work-service.js

@@ -118,6 +118,7 @@ function parseService(workTree, service, environment, context) {
             secrets: (0, work_secret_1.parseWorkSecrets)(service.cwd, context.projectRoot, service.schema.secrets, envs, environment, context.secrets, service.schema.account),
             resources: (0, work_resources_1.parseWorkResources)(service.schema.resources),
             volumeSize: service.schema.volumeSize ?? null,
+            restarts: service.schema.restarts ?? null,
             network: (0, work_network_1.parseWorkNetwork)(service.schema.network),
             networkCeiling: null,
             args: service.schema.args ?? null,

package/dist/schema/build-file-container-service-schema.js

@@ -41,6 +41,14 @@ exports.buildFileContainerServiceSchema = (0, zod_1.object)({
     // the service account its provider secrets are read as, unless a secret names one
     account: (0, zod_1.string)().optional(),
     resources: resources_schema_1.resourcesSchema.optional(),
+    // on Kubernetes, how often its container may crash and be restarted before the
+    // service fails (2 by default): it fails when it crashes once more
+    restarts: (0, zod_1.number)()
+        .int()
+        .min(0)
+        .max(10)
+        .describe('crashes restarted before it fails on Kubernetes, 0 to 10, 2 by default')
+        .optional(),
     // on Kubernetes, the size of the claim that holds its sources, mounts and volumes
     volumeSize: size_schema_1.sizeSchema.describe('size of its claim on Kubernetes, e.g. 10Gi').optional(),
     // what the service reaches besides its needs; tasks needing it always reach it

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "hammerkit",
   "description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
-  "version": "1.27.0-next.28",
+  "version": "1.27.0-next.29",
   "keywords": [
     "build",
     "build-tool",