hammerkit 1.27.0-next.9 → 2.0.0-next.1

package/dist/kubernetes/cluster-state.js

@@ -0,0 +1,245 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readClusterState = exports.describeEvent = exports.describePod = exports.getPodWaiting = exports.describeLastExit = exports.getPodFailure = exports.CRASHES_TO_FAIL = void 0;
+const log_1 = require("../log");
+const apply_1 = require("./apply");
+const job_placement_1 = require("./job-placement");
+const network_gate_1 = require("./network-gate");
+const resources_1 = require("./resources");
+// what a pod's or event's message is cut to
+const MAX_MESSAGE = 500;
+// how many events of a run are said, the newest
+const MAX_EVENTS = 20;
+// events the result of a task says better: a Job of a failed command
+const SAID_ELSEWHERE = ['BackoffLimitExceeded'];
+// A container restarted this often (ended three times) is taken as one that keeps crashing,
+// unless its service sets `restarts`: once may be a race with a dependency or a slow first start
+exports.CRASHES_TO_FAIL = 2;
+// what a pull error says when the image is not there to pull, or not for this cluster:
+// waiting does not change it. Others (a registry that limits, a token not there yet) may pass.
+const IMAGE_GONE = /not found|manifest unknown|unauthorized|denied|forbidden|invalid reference|no such host/i;
+/**
+ * Why a pod of a service will not become ready without a change, as the cluster says it:
+ * an image that can not be pulled, a container that keeps crashing or can not be created,
+ * an init container that failed. Null while it may still get there. A container fails once
+ * it was restarted `restarts` times and crashed again.
+ *
+ * Only what the cluster wrote is quoted: the termination message of a container is its
+ * own output (and may hold a value it read), except for the network gate's, which says why
+ * the policy did not take hold.
+ */
+function getPodFailure(pod, container, restarts = exports.CRASHES_TO_FAIL) {
+    for (const status of pod.status?.initContainerStatuses ?? []) {
+        const terminated = status.state?.terminated ?? status.lastState?.terminated;
+        if (!terminated || terminated.exitCode === 0) {
+            continue;
+        }
+        if (status.name === network_gate_1.NETWORK_GATE_CONTAINER) {
+            return {
+                text: `the network policy did not take hold: ${cut(terminated.message?.trim() || 'the gate failed')}`,
+                final: true,
+            };
+        }
+        if (status.restartCount >= exports.CRASHES_TO_FAIL) {
+            return { text: `its init container ${status.name} failed with exit code ${terminated.exitCode}`, final: true };
+        }
+    }
+    for (const status of mainStatuses(pod, container)) {
+        const failure = getContainerFailure(status, restartsEnded(pod, status), restarts);
+        if (failure) {
+            return failure;
+        }
+    }
+    return null;
+}
+exports.getPodFailure = getPodFailure;
+function getContainerFailure(status, restartsEnded, restarts) {
+    const waiting = status.state?.waiting;
+    const reason = waiting?.reason;
+    const said = waiting?.message ? `: ${cut(waiting.message)}` : '';
+    switch (reason) {
+        // the first pull: final only when the registry said the image is not there, as older
+        // kubelets leave that out of the ImagePullBackOff that follows
+        case 'ErrImagePull':
+            return IMAGE_GONE.test(waiting?.message ?? '')
+                ? { text: `its image ${status.image} could not be pulled (${reason})${said}`, final: true }
+                : null;
+        case 'ImagePullBackOff':
+            return {
+                text: `its image ${status.image} could not be pulled (${reason})${said}`,
+                final: IMAGE_GONE.test(waiting?.message ?? ''),
+            };
+        case 'ErrImageNeverPull':
+        case 'InvalidImageName':
+            return { text: `its image ${status.image} could not be pulled (${reason})${said}`, final: true };
+        case 'CreateContainerConfigError':
+        case 'CreateContainerError':
+        case 'RunContainerError':
+            return { text: `its container could not be started (${reason})${said}`, final: false };
+        case 'CrashLoopBackOff':
+            return status.restartCount >= restarts
+                ? { text: `it keeps crashing: ${describeLastExit(status)}`, final: true }
+                : null;
+    }
+    // a kubelet may leave an instance that ended as it is until its back-off is over, and say
+    // CrashLoopBackOff only when it looks at the pod again, a minute later: the same crash
+    if (restartsEnded && status.state?.terminated && status.restartCount >= restarts) {
+        return { text: `it keeps crashing: ${describeLastExit(status)}`, final: true };
+    }
+    return null;
+}
+// How the last instance of a container ended, as the cluster says it.
+function describeLastExit(status) {
+    // the instance that ended now, else the one before the one that waits or runs
+    const last = status.state?.terminated ?? status.lastState?.terminated;
+    const restarts = `, restarted ${status.restartCount} time${status.restartCount === 1 ? '' : 's'}`;
+    if (!last) {
+        return `it ended${restarts}`;
+    }
+    if (last.reason === 'OOMKilled') {
+        return `it was killed for using more memory than its limit (OOMKilled)${restarts}`;
+    }
+    // the kubelet kills a container whose liveness probe (its healthcheck) keeps failing
+    const killed = last.exitCode === 137 || last.exitCode === 143 ? ', killed as when its healthcheck kept failing' : '';
+    return `it exited with code ${last.exitCode}${last.reason ? ` (${last.reason})` : ''}${killed}${restarts}`;
+}
+exports.describeLastExit = describeLastExit;
+/**
+ * What a pod of a service waits for, in a few words, while it is not ready: a node, its
+ * image, its container. Undefined when it waits for nothing the cluster names.
+ */
+function getPodWaiting(pod, container) {
+    const scheduled = pod.status?.conditions?.find((condition) => condition.type === 'PodScheduled');
+    if (scheduled?.status === 'False') {
+        return `waits for a node: ${cut(scheduled.message ?? scheduled.reason ?? 'not scheduled yet')}`;
+    }
+    for (const status of mainStatuses(pod, container)) {
+        const waiting = status.state?.waiting;
+        const said = waiting?.message ? `: ${cut(waiting.message)}` : '';
+        switch (waiting?.reason) {
+            case 'ErrImagePull':
+            case 'ImagePullBackOff':
+                return `can not pull its image ${status.image} yet${said}`;
+            case 'CreateContainerConfigError':
+            case 'CreateContainerError':
+            case 'RunContainerError':
+                return `can not start its container yet (${waiting.reason})${said}`;
+            case 'CrashLoopBackOff':
+                return `crashed and starts again: ${describeLastExit(status)}`;
+        }
+        const ended = status.state?.terminated;
+        if (ended && restartsEnded(pod, status)) {
+            return `${ended.exitCode === 0 ? 'ended' : 'crashed'} and starts again: ${describeLastExit(status)}`;
+        }
+        if (status.restartCount > 0 && status.state?.running && !status.ready) {
+            return `restarted: ${describeLastExit(status)}`;
+        }
+    }
+    return undefined;
+}
+exports.getPodWaiting = getPodWaiting;
+// Whether the kubelet starts a container that ended again: not one of a pod that finished (a
+// task's Job, an evicted pod, which its ReplicaSet replaces) or that never restarts.
+function restartsEnded(pod, status) {
+    const phase = pod.status?.phase;
+    if (phase === 'Succeeded' || phase === 'Failed') {
+        return false;
+    }
+    const policy = pod.spec?.restartPolicy ?? 'Always';
+    return policy === 'Always' || (policy === 'OnFailure' && status.state?.terminated?.exitCode !== 0);
+}
+function mainStatuses(pod, container) {
+    return (pod.status?.containerStatuses ?? []).filter((status) => !container || status.name === container);
+}
+// One line of a pod that is not as it should be, or null for one that runs, ready, or ended well.
+function describePod(pod) {
+    const name = pod.metadata?.name ?? '?';
+    const phase = pod.status?.phase ?? 'Unknown';
+    const failure = getPodFailure(pod)?.text;
+    const waiting = getPodWaiting(pod);
+    const unready = (pod.status?.containerStatuses ?? [])
+        .filter((status) => !status.ready && !status.state?.terminated)
+        .map((status) => status.name);
+    const crashed = (pod.status?.containerStatuses ?? []).filter((status) => (status.state?.terminated && status.state.terminated.exitCode !== 0) || status.restartCount > 0);
+    if (!failure && !waiting && crashed.length === 0 && (phase === 'Succeeded' || unready.length === 0)) {
+        return null;
+    }
+    const said = failure ?? waiting;
+    const parts = [phase, ...(said ? [said] : [])];
+    for (const status of crashed) {
+        if (!said?.includes(describeLastExit(status))) {
+            parts.push(`${status.name}: ${describeLastExit(status)}`);
+        }
+    }
+    if (!said && unready.length > 0) {
+        parts.push(`not ready: ${unready.join(', ')}`);
+    }
+    return `pod ${name}: ${parts.join(', ')}`;
+}
+exports.describePod = describePod;
+/**
+ * The message of a warning event, as far as it is the cluster's: a failed probe says the
+ * output of the command, which is the service's, and is left out.
+ */
+function describeEvent(event) {
+    const object = event.involvedObject;
+    const message = (event.message ?? '').replace(/^((Readiness|Liveness|Startup) probe (failed|errored)).*$/s, '$1');
+    const count = (event.count ?? 1) > 1 ? ` (${event.count} times)` : '';
+    return `${event.reason ?? 'Warning'} ${object.kind?.toLowerCase() ?? 'object'}/${object.name ?? '?'}: ${cut(message.trim())}${count}`;
+}
+exports.describeEvent = describeEvent;
+/**
+ * What the cluster said of a run that failed, into its log before the server removes its
+ * objects: its pods that are not as they should be, and the warning events of its objects.
+ * In the namespace of a service run everything is the run's; in a caller's namespace, which
+ * runs share, only the objects of the run's items (named by their key) are.
+ */
+async function readClusterState(instance, kubernetes, workTree, ownNamespace) {
+    const namespace = kubernetes.namespace;
+    const keys = [...Object.values(workTree.tasks), ...Object.values(workTree.services)].map((item) => (0, resources_1.getItemKey)(item, kubernetes));
+    const ours = (name) => ownNamespace || keys.some((key) => name?.includes(key));
+    const lines = [];
+    try {
+        const labels = (0, job_placement_1.getRunLabels)(kubernetes);
+        const labelSelector = Object.entries(labels)
+            .map(([key, value]) => `${key}=${value}`)
+            .join(',');
+        const pods = await instance.coreApi.listNamespacedPod({
+            namespace,
+            ...(labelSelector && !ownNamespace ? { labelSelector } : {}),
+        });
+        for (const pod of pods.items) {
+            const line = ours(pod.metadata?.name) ? describePod(pod) : null;
+            if (line) {
+                lines.push(line);
+            }
+        }
+    }
+    catch (e) {
+        lines.push(`its pods could not be read: ${(0, log_1.getErrorMessage)(e)}`);
+    }
+    try {
+        const events = await instance.coreApi.listNamespacedEvent({ namespace, fieldSelector: 'type=Warning' });
+        const warnings = events.items
+            .filter((event) => ours(event.involvedObject.name) && !SAID_ELSEWHERE.includes(event.reason ?? ''))
+            .sort((a, b) => eventTime(a) - eventTime(b))
+            .slice(-MAX_EVENTS);
+        lines.push(...warnings.map((event) => `event ${describeEvent(event)}`));
+    }
+    catch (e) {
+        lines.push((0, apply_1.statusCodeOf)(e) === 403
+            ? `its events could not be read: the runRole of the server needs list on events (see the backend docs)`
+            : `its events could not be read: ${(0, log_1.getErrorMessage)(e)}`);
+    }
+    return lines;
+}
+exports.readClusterState = readClusterState;
+function eventTime(event) {
+    const time = event.lastTimestamp ?? event.eventTime ?? event.firstTimestamp ?? event.metadata?.creationTimestamp;
+    return time ? new Date(time).getTime() : 0;
+}
+function cut(text) {
+    const line = text.replace(/\s+/g, ' ');
+    return line.length > MAX_MESSAGE ? `${line.slice(0, MAX_MESSAGE)}…` : line;
+}
+//# sourceMappingURL=cluster-state.js.map

package/dist/kubernetes/ensure-kubernetes-deployment-exists.js

@@ -1,9 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ensureKubernetesDeploymentExists = exports.DEFINITION_LABEL = void 0;
+exports.getServiceContainerName = exports.MIN_READY_SECONDS = exports.getPodTemplateHash = exports.POD_ANNOTATION = exports.ROLLOUT_DEADLINE_SECONDS = exports.ensureKubernetesDeploymentExists = exports.DEFINITION_LABEL = void 0;
 const service_definition_1 = require("../planner/service-definition");
 const work_service_1 = require("../planner/work-service");
 const volumes_1 = require("./volumes");
+const crypto_1 = require("crypto");
 const service_hostname_1 = require("../planner/utils/service-hostname");
 const apply_1 = require("./apply");
 const version_1 = require("../version");
@@ -15,11 +16,17 @@ const container_resources_1 = require("./container-resources");
 const job_placement_1 = require("./job-placement");
 const network_policy_1 = require("./network-policy");
 const egress_proxy_1 = require("../network/egress-proxy");
+const service_rollouts_1 = require("./service-rollouts");
 // what the service was created from, to tell a changed one
 exports.DEFINITION_LABEL = 'hammerkit.dev/definition';
+// how often a starting container is checked: as the liveness probe
+const STARTUP_PERIOD_SECONDS = 5;
 async function ensureKubernetesDeploymentExists(instance, env, service, persistence, secrets, stateKey, 
 // the needed services by name, and the proxy of a limited network
-network) {
+network, 
+// a service run (env.server.serviceRun): the Deployment the service has now, put back when
+// the new one does not get ready, and what stops the wait for it
+rollout = {}) {
     const envs = (0, replace_env_variables_1.getEnvironmentVariables)(service.data.envs);
     const name = (0, resources_1.getResourceName)(service, env);
     const healthcheckCommand = (0, work_service_1.getHealthcheckCommand)(service.data);
@@ -34,6 +41,16 @@ network) {
             failureThreshold: 3,
         }
         : undefined;
+    // the healthcheck's `timeout` for each start: until it passed once, the liveness probe
+    // doesn't kill a container that starts slowly, and the wait fails it with the timeout
+    const startupProbe = probe && service.data.healthcheck
+        ? {
+            ...probe,
+            // a check more than the timeout holds, and one more for the first check's offset: the
+            // wait fails it with the timeout before the kubelet restarts it
+            failureThreshold: Math.ceil(service.data.healthcheck.timeout / (STARTUP_PERIOD_SECONDS * 1000)) + 2,
+        }
+        : undefined;
     const server = env.server;
     const placement = server ? (0, job_placement_1.getPodPlacement)(server, undefined) : null;
     const workspace = (0, volumes_1.getWorkspacePodSpec)(env, persistence);
@@ -57,6 +74,9 @@ network) {
                 matchLabels: (0, resources_1.getItemLabels)(service, env),
             },
             replicas: 1,
+            // without a healthcheck a pod is ready as soon as its container runs, also one that
+            // exits a second later: it counts once it ran this long without a restart
+            minReadySeconds: probe ? 0 : exports.MIN_READY_SECONDS,
             template: {
                 metadata: {
                     labels: {
@@ -83,7 +103,7 @@ network) {
                         : {}),
                     containers: [
                         {
-                            name: service.name.replace(/:/, '-'),
+                            name: getServiceContainerName(service),
                             image: service.data.image,
                             workingDir: (0, work_service_1.getServiceWorkingDir)(service.data),
                             env: [
@@ -100,6 +120,7 @@ network) {
                             })),
                             readinessProbe: probe,
                             livenessProbe: probe,
+                            startupProbe,
                             ...(0, container_resources_1.getKubernetesResources)(service.data.resources),
                             volumeMounts: [...persistence.mounts.map((m) => m.mount), ...secrets.mounts],
                             ...(0, job_placement_1.getContainerSecurity)(env),
@@ -118,19 +139,122 @@ network) {
             },
         },
     };
-    const deploy = await (0, apply_1.apply)(instance, deployment);
-    if (deploy.status?.readyReplicas ?? 0 > 0) {
-        return;
-    }
-    await (0, await_running_state_1.awaitDeployRunningState)(instance, env, name, {
+    const readiness = {
         podSelector: (0, resources_1.getItemSelector)(service, env),
         timeout: service.data.healthcheck?.timeout ?? null,
         timeoutMessage: service.data.healthcheck
             ? (0, work_healthcheck_1.getHealthcheckTimeoutMessage)(service.name, service.data.healthcheck)
             : '',
-    });
+        container: getServiceContainerName(service),
+        subject: `service ${service.name}`,
+        onWaiting: (text) => service.status.write('warn', `${service.name} ${text}`),
+        ...(service.data.restarts !== null ? { restarts: service.data.restarts } : {}),
+    };
+    if (env.server?.serviceRun) {
+        await rollOut(instance, env, service, deployment, readiness, rollout);
+        return;
+    }
+    const deploy = await (0, apply_1.apply)(instance, deployment);
+    if ((deploy.status?.availableReplicas ?? 0) > 0) {
+        return;
+    }
+    await (0, await_running_state_1.awaitDeployRunningState)(instance, env, name, readiness);
 }
 exports.ensureKubernetesDeploymentExists = ensureKubernetesDeploymentExists;
+// how long the pods of a rolled service may take to get ready, as Kubernetes counts it
+exports.ROLLOUT_DEADLINE_SECONDS = 600;
+/**
+ * The Deployment of a service of a service run, which keeps it from one commit to the next:
+ * replaced as a whole (a merge would keep what the new one dropped, such as a variable) and
+ * rolled. Its requests go to the pods it has until a new one is ready. A service with
+ * volumes of its own stops first: two pods would write the same data. The new pod is asked
+ * to go where the old one runs (getClaimPodSpec), whose claims are usually attached to one
+ * node at a time. When the execution ends before all its services are up, the worker puts
+ * every Deployment it rolled back as it was (serviceRollouts).
+ */
+async function rollOut(instance, env, service, deployment, readiness, rollout) {
+    const spec = deployment.spec;
+    const template = spec.template;
+    const definition = (0, service_definition_1.getServiceDefinitionHash)(service);
+    const stateKey = template.metadata?.labels?.['hammerkit.dev/state'] ?? '';
+    const ownVolumes = service.data.volumes.some((volume) => !volume.inherited);
+    spec.strategy = ownVolumes
+        ? { type: 'Recreate' }
+        : { type: 'RollingUpdate', rollingUpdate: { maxSurge: 1, maxUnavailable: 0 } };
+    spec.progressDeadlineSeconds = exports.ROLLOUT_DEADLINE_SECONDS;
+    // the pods of this definition and state, apart from the ones it rolls from
+    template.metadata = {
+        ...template.metadata,
+        labels: { ...template.metadata?.labels, [exports.DEFINITION_LABEL]: definition },
+    };
+    const pod = getPodTemplateHash(template, service.data.projectRoot);
+    deployment.metadata.annotations = { ...deployment.metadata.annotations, [exports.POD_ANNOTATION]: pod };
+    const previous = rollout.previous;
+    if (previous &&
+        previous.metadata?.annotations?.[exports.POD_ANNOTATION] === pod &&
+        previous.metadata?.labels?.[exports.DEFINITION_LABEL] === definition &&
+        isAvailable(previous)) {
+        // its pod would be the same but for the state it runs at: a service that watches its
+        // files has them already (syncLiveFiles), and its pod stays
+        service_rollouts_1.serviceRollouts.add(env.namespace, deployment.metadata.name, previous);
+        await relabelDeployment(instance, env.namespace, deployment.metadata.name, stateKey);
+        service.status.write('info', `${service.name} watches its files, it got the changed ones and stays up`);
+        // it goes on running with them: else the move does not get up, and gets them back
+        await (0, await_running_state_1.watchKeptPods)(instance, env, deployment.metadata.name, { ...readiness, signal: rollout.signal });
+        return;
+    }
+    // put back, or removed, by the worker when the execution ends before its services are up
+    service_rollouts_1.serviceRollouts.add(env.namespace, deployment.metadata.name, rollout.previous);
+    const applied = await (0, apply_1.replaceDeployment)(instance, deployment);
+    if (rollout.previous) {
+        service.status.write('info', `${service.name} changed, its new pod takes over once it is ready`);
+    }
+    await (0, await_running_state_1.awaitDeployRollout)(instance, env, deployment.metadata.name, {
+        ...readiness,
+        podSelector: `${readiness.podSelector},hammerkit.dev/state=${stateKey},${exports.DEFINITION_LABEL}=${definition}`,
+        generation: applied.metadata?.generation ?? 0,
+        signal: rollout.signal,
+    });
+}
+// the pod a Deployment of a service run makes, but for the state it runs at
+exports.POD_ANNOTATION = 'hammerkit.dev/pod';
+// A move executes from a checkout of its own (`root`), which the pod's paths name: where the
+// checkout is does not count, as for getServiceDefinitionHash
+function getPodTemplateHash(template, root) {
+    const labels = { ...template.metadata?.labels };
+    delete labels['hammerkit.dev/state'];
+    const json = JSON.stringify({ ...template, metadata: { ...template.metadata, labels } });
+    const escaped = root ? JSON.stringify(root).slice(1, -1) : '';
+    return (0, crypto_1.createHash)('sha256')
+        .update(escaped ? json.split(escaped).join('<checkout>') : json)
+        .digest('hex')
+        .slice(0, 32);
+}
+exports.getPodTemplateHash = getPodTemplateHash;
+// a Deployment whose pods are all of its template and ready
+function isAvailable(deployment) {
+    const status = deployment.status;
+    return (!deployment.metadata?.deletionTimestamp &&
+        (status?.observedGeneration ?? 0) >= (deployment.metadata?.generation ?? 0) &&
+        (status?.updatedReplicas ?? 0) >= 1 &&
+        (status?.availableReplicas ?? 0) >= 1 &&
+        (status?.replicas ?? 0) === (status?.updatedReplicas ?? 0));
+}
+// The Deployment runs at the state `stateKey` now: its template, and so its pods, stay
+async function relabelDeployment(instance, namespace, name, stateKey) {
+    await instance.appsApi.patchNamespacedDeployment({
+        name,
+        namespace,
+        body: [{ op: 'add', path: '/metadata/labels/hammerkit.dev~1state', value: stateKey }],
+    });
+}
+// how long the container of a service without a healthcheck has to run before it counts as up
+exports.MIN_READY_SECONDS = 10;
+// The container of a service's pod that runs its command, beside the `debug` one.
+function getServiceContainerName(service) {
+    return service.name.replace(/:/, '-');
+}
+exports.getServiceContainerName = getServiceContainerName;
 // Kubernetes `command` replaces the image entrypoint and `args` its command.
 // A shell service sets both; an exec-form `cmd` keeps the established mapping
 // (first token as `command`, the rest as `args`).

package/dist/kubernetes/ensure-kubernetes-service-exists.js

@@ -28,7 +28,9 @@ async function ensureKubernetesServiceExists(instance, env, service) {
             ...(headless
                 ? {}
                 : {
+                    // a Service of more than one port needs a name for each
                     ports: service.data.ports.map((port) => ({
+                        name: `tcp-${port.hostPort ?? port.containerPort}`,
                         port: port.hostPort ?? port.containerPort,
                         targetPort: port.containerPort,
                     })),