hammerkit 1.27.0-next.24 → 1.27.0-next.26

package/build.schema.json

@@ -459,6 +459,91 @@
                 },
                 "additionalProperties": false,
                 "description": "local task"
+              },
+              {
+                "type": "object",
+                "properties": {
+                  "deps": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/deps"
+                  },
+                  "needs": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/needs"
+                  },
+                  "description": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/description"
+                  },
+                  "extend": {
+                    "type": "string"
+                  },
+                  "reset": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/reset"
+                  },
+                  "envs": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/envs"
+                  },
+                  "labels": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/labels"
+                  },
+                  "cache": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/cache"
+                  },
+                  "cmds": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/cmds"
+                  },
+                  "src": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/src"
+                  },
+                  "generates": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/generates"
+                  },
+                  "shell": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/shell"
+                  },
+                  "continuous": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/continuous"
+                  },
+                  "timeout": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/timeout"
+                  },
+                  "secrets": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/secrets"
+                  },
+                  "account": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/account"
+                  },
+                  "resources": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/resources"
+                  },
+                  "network": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/network"
+                  },
+                  "args": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/args"
+                  },
+                  "runner": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/runner"
+                  },
+                  "publish": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/publish"
+                  },
+                  "image": {
+                    "type": "string"
+                  },
+                  "mounts": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/mounts"
+                  },
+                  "volumeSize": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/volumeSize"
+                  },
+                  "capAdd": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/capAdd"
+                  }
+                },
+                "required": [
+                  "extend"
+                ],
+                "additionalProperties": false,
+                "description": "container task taking its image from the task it extends"
               }
             ]
           }

package/dist/harness/sandbox-client.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.quotePosix = exports.fetchFailure = exports.toSandboxPath = exports.checkSandboxPath = exports.exitCodeOf = exports.followRun = exports.submitRun = exports.FINAL_RUN_STATES = exports.oneAtATime = exports.SandboxClient = exports.listPublishedTasks = exports.runCheck = exports.CHECK_DESCRIPTION = exports.SandboxClientError = void 0;
+exports.quotePosix = exports.fetchFailure = exports.toSandboxPath = exports.checkSandboxPath = exports.exitCodeOf = exports.readRunLog = exports.getRun = exports.followRun = exports.submitRun = exports.FINAL_RUN_STATES = exports.oneAtATime = exports.SandboxClient = exports.listPublishedTasks = exports.runCheck = exports.CHECK_DESCRIPTION = exports.SandboxClientError = void 0;
 const path_1 = require("path");
 const token_source_1 = require("../remote/token-source");
 // a write carries up to 32 MiB
@@ -323,7 +323,9 @@ async function followRun(connection, submitted, options = {}) {
     const cancel = () => {
         cancelling ?? (cancelling = call(connection, 'POST', `${path}/cancel`).catch(() => undefined));
     };
-    options.signal?.addEventListener('abort', cancel, { once: true });
+    if (!options.detach) {
+        options.signal?.addEventListener('abort', cancel, { once: true });
+    }
     const lines = [];
     let from = 1;
     let truncated = false;
@@ -348,6 +350,9 @@ async function followRun(connection, submitted, options = {}) {
         let run = submitted;
         let waitingSaid = false;
         for (;;) {
+            if (options.signal?.aborted && options.detach) {
+                break;
+            }
             if (options.signal?.aborted) {
                 // the process may exit after this, so the cancel has to reach the server first
                 cancel();
@@ -367,18 +372,7 @@ async function followRun(connection, submitted, options = {}) {
             run = await json(await call(connection, 'GET', path, { query: { wait: POLL_WAIT } }));
         }
         if (truncated) {
-            // the end of a long log is kept in the results
-            const tasks = options.task !== undefined ? [options.task] : Object.keys(run.results ?? {});
-            for (const task of tasks) {
-                const result = run.results?.[task];
-                if (!result) {
-                    continue;
-                }
-                const said = options.task !== undefined ? 'the last lines were' : `the last lines of ${task} were`;
-                for (const text of [`… the server cut the log off at its limit, ${said}:`, ...result.lastLog]) {
-                    add({ line: 0, task, stream: 'stdout', text });
-                }
-            }
+            truncatedEnd(run, options.task).forEach(add);
         }
         return { run, lines, truncated };
     }
@@ -387,6 +381,59 @@ async function followRun(connection, submitted, options = {}) {
     }
 }
 exports.followRun = followRun;
+// The end of a log the server cut off at its limit: the last lines of each task, which its
+// results keep.
+function truncatedEnd(run, only) {
+    const lines = [];
+    const tasks = only !== undefined ? [only] : Object.keys(run.results ?? {});
+    for (const task of tasks) {
+        const result = run.results?.[task];
+        if (!result) {
+            continue;
+        }
+        const said = only !== undefined ? 'the last lines were' : `the last lines of ${task} were`;
+        for (const text of [`… the server cut the log off at its limit, ${said}:`, ...result.lastLog]) {
+            lines.push({ line: 0, task, stream: 'stdout', text });
+        }
+    }
+    return lines;
+}
+// A run as the server keeps it, or null when it keeps none the caller may see.
+async function getRun(connection, id) {
+    const response = await call(connection, 'GET', `/v1/runs/${encodeURIComponent(id)}`, { missing: true });
+    return response.status === 404 ? null : json(response);
+}
+exports.getRun = getRun;
+/**
+ * Reads the log of a run as it is now, every line or the last `tail` of them, of one task or
+ * of all: also of a run that ended, as long as the server keeps it.
+ */
+async function readRunLog(connection, run, query, onLine) {
+    const path = `/v1/runs/${encodeURIComponent(run.id)}/logs`;
+    if (query.tail !== undefined) {
+        const page = await json(await call(connection, 'GET', path, { query: { task: query.task, tail: query.tail } }));
+        page.lines.forEach(onLine);
+        return { truncated: page.truncated };
+    }
+    let from = 1;
+    let truncated = false;
+    for (;;) {
+        const page = await json(await call(connection, 'GET', path, { query: { task: query.task, from, limit: 1000 } }));
+        truncated = page.truncated;
+        for (const line of page.lines) {
+            onLine(line);
+            from = line.line + 1;
+        }
+        if (!page.next) {
+            break;
+        }
+    }
+    if (truncated) {
+        truncatedEnd(run, query.task).forEach(onLine);
+    }
+    return { truncated };
+}
+exports.readRunLog = readRunLog;
 function exitCodeOf(result) {
     if (!result) {
         return null;

package/dist/kubernetes/await-running-state.js

@@ -1,10 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.stopJob = exports.deleteJobAndWait = exports.awaitDeployRunningState = exports.deleteJob = exports.awaitJobCompletion = exports.awaitRunningState = void 0;
+exports.stopJob = exports.deleteJobAndWait = exports.awaitDeployRunningState = exports.PASSING_FAILURE_MS = exports.deleteJob = exports.awaitJobCompletion = exports.awaitRunningState = void 0;
 const abort_1 = require("../executer/abort");
 const abort_event_1 = require("../utils/abort-event");
 const apply_1 = require("./apply");
 const sleep_1 = require("../utils/sleep");
+const cluster_state_1 = require("./cluster-state");
 // The waits below poll instead of using the client's Watch: Watch aborts every
 // request after 30s, which a pod or deployment still pulling its image outlasts.
 async function awaitRunningState(instance, env, name, phase, pollInterval = 1000) {
@@ -76,23 +77,56 @@ async function deleteJob(instance, env, name) {
     }
 }
 exports.deleteJob = deleteJob;
-// Waits for a ready replica. The deadline only starts once a pod runs, so an
+// how long a reason that may still pass (an image pull a registry refused for now, a
+// container the runtime could not create) may last before it fails the service
+exports.PASSING_FAILURE_MS = 60000;
+// Waits for an available replica: a ready one, for a service without a healthcheck one that
+// stayed ready for a while. The deadline only starts once a pod runs, so an
 // image pull doesn't count against it (as on docker, where it starts after
-// the container started).
+// the container started). A pod that won't get there without a change (an image
+// that isn't there, a container that crashed twice in a row) fails it at once, with
+// the reason the cluster gives; one that may still pass fails once it lasted a minute.
+// Only the pods of this Deployment count: not the ones of one deleted before it was
+// created again, which may still be there, terminating or not yet.
 async function awaitDeployRunningState(instance, env, name, readiness, pollInterval = 1000, now = Date.now) {
     let runningSince = null;
+    let said;
+    // since when a pod has had a failure that may still pass, by pod
+    const failingSince = new Map();
     for (;;) {
         const deployment = await instance.appsApi.readNamespacedDeployment({ name, namespace: env.namespace });
-        if ((deployment.status?.readyReplicas ?? 0) > 0) {
+        // available: ready for the Deployment's minReadySeconds
+        if ((deployment.status?.availableReplicas ?? 0) > 0) {
             return;
         }
+        const created = deployment.metadata?.creationTimestamp;
+        const listed = await instance.coreApi.listNamespacedPod({
+            namespace: env.namespace,
+            labelSelector: readiness.podSelector,
+        });
+        const pods = listed.items.filter((pod) => !pod.metadata?.deletionTimestamp &&
+            !(created && pod.metadata?.creationTimestamp && pod.metadata.creationTimestamp < created));
+        for (const [index, pod] of pods.entries()) {
+            const key = pod.metadata?.name ?? `${index}`;
+            const failure = (0, cluster_state_1.getPodFailure)(pod, readiness.container);
+            if (!failure) {
+                failingSince.delete(key);
+                continue;
+            }
+            const since = failingSince.get(key) ?? now();
+            failingSince.set(key, since);
+            if (failure.final || now() - since >= exports.PASSING_FAILURE_MS) {
+                throw new Error(`${readiness.subject ?? name} can not become ready, ${failure.text}`);
+            }
+        }
+        const waiting = pods.map((pod) => (0, cluster_state_1.getPodWaiting)(pod, readiness.container)).find((text) => !!text);
+        if (waiting && waiting !== said) {
+            readiness.onWaiting?.(waiting);
+        }
+        said = waiting;
         if (readiness.timeout !== null) {
             if (runningSince === null) {
-                const pods = await instance.coreApi.listNamespacedPod({
-                    namespace: env.namespace,
-                    labelSelector: readiness.podSelector,
-                });
-                if (pods.items.some((pod) => pod.status?.phase === 'Running')) {
+                if (pods.some((pod) => pod.status?.phase === 'Running')) {
                     runningSince = now();
                 }
             }

package/dist/kubernetes/cluster-state.js

@@ -0,0 +1,224 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readClusterState = exports.describeEvent = exports.describePod = exports.getPodWaiting = exports.describeLastExit = exports.getPodFailure = exports.CRASHES_TO_FAIL = void 0;
+const log_1 = require("../log");
+const apply_1 = require("./apply");
+const job_placement_1 = require("./job-placement");
+const network_gate_1 = require("./network-gate");
+const resources_1 = require("./resources");
+// what a pod's or event's message is cut to
+const MAX_MESSAGE = 500;
+// how many events of a run are said, the newest
+const MAX_EVENTS = 20;
+// events the result of a task says better: a Job of a failed command
+const SAID_ELSEWHERE = ['BackoffLimitExceeded'];
+// A container restarted this often in a row is taken as one that keeps crashing: once may be
+// a race with a dependency or a slow first start
+exports.CRASHES_TO_FAIL = 2;
+// what a pull error says when the image is not there to pull, or not for this cluster:
+// waiting does not change it. Others (a registry that limits, a token not there yet) may pass.
+const IMAGE_GONE = /not found|manifest unknown|unauthorized|denied|forbidden|invalid reference|no such host/i;
+/**
+ * Why a pod of a service will not become ready without a change, as the cluster says it:
+ * an image that can not be pulled, a container that keeps crashing or can not be created,
+ * an init container that failed. Null while it may still get there.
+ *
+ * Only what the cluster wrote is quoted: the termination message of a container is its
+ * own output (and may hold a value it read), except for the network gate's, which says why
+ * the policy did not take hold.
+ */
+function getPodFailure(pod, container) {
+    for (const status of pod.status?.initContainerStatuses ?? []) {
+        const terminated = status.state?.terminated ?? status.lastState?.terminated;
+        if (!terminated || terminated.exitCode === 0) {
+            continue;
+        }
+        if (status.name === network_gate_1.NETWORK_GATE_CONTAINER) {
+            return {
+                text: `the network policy did not take hold: ${cut(terminated.message?.trim() || 'the gate failed')}`,
+                final: true,
+            };
+        }
+        if (status.restartCount >= exports.CRASHES_TO_FAIL) {
+            return { text: `its init container ${status.name} failed with exit code ${terminated.exitCode}`, final: true };
+        }
+    }
+    for (const status of mainStatuses(pod, container)) {
+        const failure = getContainerFailure(status);
+        if (failure) {
+            return failure;
+        }
+    }
+    return null;
+}
+exports.getPodFailure = getPodFailure;
+function getContainerFailure(status) {
+    const waiting = status.state?.waiting;
+    const reason = waiting?.reason;
+    const said = waiting?.message ? `: ${cut(waiting.message)}` : '';
+    switch (reason) {
+        // the first pull: final only when the registry said the image is not there, as older
+        // kubelets leave that out of the ImagePullBackOff that follows
+        case 'ErrImagePull':
+            return IMAGE_GONE.test(waiting?.message ?? '')
+                ? { text: `its image ${status.image} could not be pulled (${reason})${said}`, final: true }
+                : null;
+        case 'ImagePullBackOff':
+            return {
+                text: `its image ${status.image} could not be pulled (${reason})${said}`,
+                final: IMAGE_GONE.test(waiting?.message ?? ''),
+            };
+        case 'ErrImageNeverPull':
+        case 'InvalidImageName':
+            return { text: `its image ${status.image} could not be pulled (${reason})${said}`, final: true };
+        case 'CreateContainerConfigError':
+        case 'CreateContainerError':
+        case 'RunContainerError':
+            return { text: `its container could not be started (${reason})${said}`, final: false };
+        case 'CrashLoopBackOff':
+            return status.restartCount >= exports.CRASHES_TO_FAIL
+                ? { text: `it keeps crashing: ${describeLastExit(status)}`, final: true }
+                : null;
+    }
+    return null;
+}
+// How the last instance of a container ended, as the cluster says it.
+function describeLastExit(status) {
+    const last = status.lastState?.terminated ?? status.state?.terminated;
+    const restarts = `, restarted ${status.restartCount} time${status.restartCount === 1 ? '' : 's'}`;
+    if (!last) {
+        return `it ended${restarts}`;
+    }
+    if (last.reason === 'OOMKilled') {
+        return `it was killed for using more memory than its limit (OOMKilled)${restarts}`;
+    }
+    // the kubelet kills a container whose liveness probe (its healthcheck) keeps failing
+    const killed = last.exitCode === 137 || last.exitCode === 143 ? ', killed as when its healthcheck kept failing' : '';
+    return `it exited with code ${last.exitCode}${last.reason ? ` (${last.reason})` : ''}${killed}${restarts}`;
+}
+exports.describeLastExit = describeLastExit;
+/**
+ * What a pod of a service waits for, in a few words, while it is not ready: a node, its
+ * image, its container. Undefined when it waits for nothing the cluster names.
+ */
+function getPodWaiting(pod, container) {
+    const scheduled = pod.status?.conditions?.find((condition) => condition.type === 'PodScheduled');
+    if (scheduled?.status === 'False') {
+        return `waits for a node: ${cut(scheduled.message ?? scheduled.reason ?? 'not scheduled yet')}`;
+    }
+    for (const status of mainStatuses(pod, container)) {
+        const waiting = status.state?.waiting;
+        const said = waiting?.message ? `: ${cut(waiting.message)}` : '';
+        switch (waiting?.reason) {
+            case 'ErrImagePull':
+            case 'ImagePullBackOff':
+                return `can not pull its image ${status.image} yet${said}`;
+            case 'CreateContainerConfigError':
+            case 'CreateContainerError':
+            case 'RunContainerError':
+                return `can not start its container yet (${waiting.reason})${said}`;
+            case 'CrashLoopBackOff':
+                return `crashed and starts again: ${describeLastExit(status)}`;
+        }
+        if (status.restartCount > 0 && status.state?.running && !status.ready) {
+            return `restarted: ${describeLastExit(status)}`;
+        }
+    }
+    return undefined;
+}
+exports.getPodWaiting = getPodWaiting;
+function mainStatuses(pod, container) {
+    return (pod.status?.containerStatuses ?? []).filter((status) => !container || status.name === container);
+}
+// One line of a pod that is not as it should be, or null for one that runs, ready, or ended well.
+function describePod(pod) {
+    const name = pod.metadata?.name ?? '?';
+    const phase = pod.status?.phase ?? 'Unknown';
+    const failure = getPodFailure(pod)?.text;
+    const waiting = getPodWaiting(pod);
+    const unready = (pod.status?.containerStatuses ?? [])
+        .filter((status) => !status.ready && !status.state?.terminated)
+        .map((status) => status.name);
+    const crashed = (pod.status?.containerStatuses ?? []).filter((status) => (status.state?.terminated && status.state.terminated.exitCode !== 0) || status.restartCount > 0);
+    if (!failure && !waiting && crashed.length === 0 && (phase === 'Succeeded' || unready.length === 0)) {
+        return null;
+    }
+    const said = failure ?? waiting;
+    const parts = [phase, ...(said ? [said] : [])];
+    for (const status of crashed) {
+        if (!said?.includes(describeLastExit(status))) {
+            parts.push(`${status.name}: ${describeLastExit(status)}`);
+        }
+    }
+    if (!said && unready.length > 0) {
+        parts.push(`not ready: ${unready.join(', ')}`);
+    }
+    return `pod ${name}: ${parts.join(', ')}`;
+}
+exports.describePod = describePod;
+/**
+ * The message of a warning event, as far as it is the cluster's: a failed probe says the
+ * output of the command, which is the service's, and is left out.
+ */
+function describeEvent(event) {
+    const object = event.involvedObject;
+    const message = (event.message ?? '').replace(/^((Readiness|Liveness|Startup) probe (failed|errored)).*$/s, '$1');
+    const count = (event.count ?? 1) > 1 ? ` (${event.count} times)` : '';
+    return `${event.reason ?? 'Warning'} ${object.kind?.toLowerCase() ?? 'object'}/${object.name ?? '?'}: ${cut(message.trim())}${count}`;
+}
+exports.describeEvent = describeEvent;
+/**
+ * What the cluster said of a run that failed, into its log before the server removes its
+ * objects: its pods that are not as they should be, and the warning events of its objects.
+ * In the namespace of a service run everything is the run's; in a caller's namespace, which
+ * runs share, only the objects of the run's items (named by their key) are.
+ */
+async function readClusterState(instance, kubernetes, workTree, ownNamespace) {
+    const namespace = kubernetes.namespace;
+    const keys = [...Object.values(workTree.tasks), ...Object.values(workTree.services)].map((item) => (0, resources_1.getItemKey)(item, kubernetes));
+    const ours = (name) => ownNamespace || keys.some((key) => name?.includes(key));
+    const lines = [];
+    try {
+        const labels = (0, job_placement_1.getRunLabels)(kubernetes);
+        const labelSelector = Object.entries(labels)
+            .map(([key, value]) => `${key}=${value}`)
+            .join(',');
+        const pods = await instance.coreApi.listNamespacedPod({
+            namespace,
+            ...(labelSelector && !ownNamespace ? { labelSelector } : {}),
+        });
+        for (const pod of pods.items) {
+            const line = ours(pod.metadata?.name) ? describePod(pod) : null;
+            if (line) {
+                lines.push(line);
+            }
+        }
+    }
+    catch (e) {
+        lines.push(`its pods could not be read: ${(0, log_1.getErrorMessage)(e)}`);
+    }
+    try {
+        const events = await instance.coreApi.listNamespacedEvent({ namespace, fieldSelector: 'type=Warning' });
+        const warnings = events.items
+            .filter((event) => ours(event.involvedObject.name) && !SAID_ELSEWHERE.includes(event.reason ?? ''))
+            .sort((a, b) => eventTime(a) - eventTime(b))
+            .slice(-MAX_EVENTS);
+        lines.push(...warnings.map((event) => `event ${describeEvent(event)}`));
+    }
+    catch (e) {
+        lines.push((0, apply_1.statusCodeOf)(e) === 403
+            ? `its events could not be read: the runRole of the server needs list on events (see the backend docs)`
+            : `its events could not be read: ${(0, log_1.getErrorMessage)(e)}`);
+    }
+    return lines;
+}
+exports.readClusterState = readClusterState;
+function eventTime(event) {
+    const time = event.lastTimestamp ?? event.eventTime ?? event.firstTimestamp ?? event.metadata?.creationTimestamp;
+    return time ? new Date(time).getTime() : 0;
+}
+function cut(text) {
+    const line = text.replace(/\s+/g, ' ');
+    return line.length > MAX_MESSAGE ? `${line.slice(0, MAX_MESSAGE)}…` : line;
+}
+//# sourceMappingURL=cluster-state.js.map

package/dist/kubernetes/ensure-kubernetes-deployment-exists.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ensureKubernetesDeploymentExists = exports.DEFINITION_LABEL = void 0;
+exports.getServiceContainerName = exports.MIN_READY_SECONDS = exports.ensureKubernetesDeploymentExists = exports.DEFINITION_LABEL = void 0;
 const service_definition_1 = require("../planner/service-definition");
 const work_service_1 = require("../planner/work-service");
 const volumes_1 = require("./volumes");
@@ -57,6 +57,9 @@ network) {
                 matchLabels: (0, resources_1.getItemLabels)(service, env),
             },
             replicas: 1,
+            // without a healthcheck a pod is ready as soon as its container runs, also one that
+            // exits a second later: it counts once it ran this long without a restart
+            minReadySeconds: probe ? 0 : exports.MIN_READY_SECONDS,
             template: {
                 metadata: {
                     labels: {
@@ -83,7 +86,7 @@ network) {
                         : {}),
                     containers: [
                         {
-                            name: service.name.replace(/:/, '-'),
+                            name: getServiceContainerName(service),
                             image: service.data.image,
                             workingDir: (0, work_service_1.getServiceWorkingDir)(service.data),
                             env: [
@@ -119,7 +122,7 @@ network) {
         },
     };
     const deploy = await (0, apply_1.apply)(instance, deployment);
-    if (deploy.status?.readyReplicas ?? 0 > 0) {
+    if ((deploy.status?.availableReplicas ?? 0) > 0) {
         return;
     }
     await (0, await_running_state_1.awaitDeployRunningState)(instance, env, name, {
@@ -128,9 +131,19 @@ network) {
         timeoutMessage: service.data.healthcheck
             ? (0, work_healthcheck_1.getHealthcheckTimeoutMessage)(service.name, service.data.healthcheck)
             : '',
+        container: getServiceContainerName(service),
+        subject: `service ${service.name}`,
+        onWaiting: (text) => service.status.write('warn', `${service.name} ${text}`),
     });
 }
 exports.ensureKubernetesDeploymentExists = ensureKubernetesDeploymentExists;
+// how long the container of a service without a healthcheck has to run before it counts as up
+exports.MIN_READY_SECONDS = 10;
+// The container of a service's pod that runs its command, beside the `debug` one.
+function getServiceContainerName(service) {
+    return service.name.replace(/:/, '-');
+}
+exports.getServiceContainerName = getServiceContainerName;
 // Kubernetes `command` replaces the image entrypoint and `args` its command.
 // A shell service sets both; an exec-form `cmd` keeps the established mapping
 // (first token as `command`, the rest as `args`).

package/dist/kubernetes/follow-pod-logs.js

@@ -0,0 +1,182 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.followPodLogs = void 0;
+const stream_1 = require("stream");
+const readline_1 = require("readline");
+const log_1 = require("../log");
+// what is read of an instance of a container that already ended
+const MAX_ENDED_LOG_BYTES = 1024 * 1024;
+/**
+ * Follows the log of a container in the pods of a selector, as `docker attach` does for a
+ * service on Docker: the instance that runs as it writes, and an instance that ended before
+ * it was seen (one that crashed and was restarted) once, from its start. Each instance is
+ * read once, by its container id.
+ */
+function followPodLogs(instance, namespace, labelSelector, container, onLine, options = {}) {
+    const seen = new Set();
+    const following = [];
+    const reading = new Set();
+    let stopped = false;
+    let failed = false;
+    let timer;
+    let polling = Promise.resolve();
+    let busy = false;
+    let again = false;
+    // the logs followed now, which end when their instance does
+    let streaming = 0;
+    const fail = (e) => {
+        if (!failed && !stopped) {
+            failed = true;
+            options.onError?.(`the log of ${container} could not be read: ${(0, log_1.getErrorMessage)(e)}`);
+        }
+    };
+    const track = (promise) => {
+        const tracked = promise.catch(fail).finally(() => reading.delete(tracked));
+        reading.add(tracked);
+    };
+    const readEnded = async (pod, previous) => {
+        const log = await instance.coreApi.readNamespacedPodLog({
+            name: pod,
+            namespace,
+            container,
+            previous,
+            limitBytes: MAX_ENDED_LOG_BYTES,
+        });
+        for (const line of String(log ?? '').split(/\r?\n/)) {
+            if (line.length > 0) {
+                onLine(line);
+            }
+        }
+    };
+    const follow = async (pod) => {
+        const output = new stream_1.PassThrough();
+        let ended = false;
+        // the instance ended or the request broke: a restarted one is looked for at once
+        const end = () => {
+            if (!ended) {
+                ended = true;
+                streaming--;
+                wake();
+            }
+        };
+        // stop() aborts the request, which errors the stream the client pipes in here
+        output.on('pipe', (source) => source.on('error', () => {
+            end();
+        }));
+        output.on('end', end);
+        (0, readline_1.createInterface)({ input: output }).on('line', (line) => onLine(line));
+        streaming++;
+        let controller;
+        try {
+            controller = await instance.log.log(namespace, pod, container, output, { follow: true });
+        }
+        catch (e) {
+            end();
+            throw e;
+        }
+        if (stopped) {
+            controller.abort();
+        }
+        else {
+            following.push(controller);
+        }
+    };
+    const look = (pod) => {
+        const name = pod.metadata?.name;
+        const status = pod.status?.containerStatuses?.find((s) => s.name === container);
+        // a pod that goes is of a Deployment before this one, or of a rollout that replaced it
+        if (!name || !status || pod.metadata?.deletionTimestamp) {
+            return;
+        }
+        // an instance that ended before this one, not seen while it ran: its lines go first
+        let before = Promise.resolve();
+        const last = status.lastState?.terminated?.containerID;
+        if (last && !seen.has(last)) {
+            seen.add(last);
+            before = readEnded(name, true).catch(fail);
+            track(before);
+        }
+        const current = status.containerID;
+        if (!current || seen.has(current)) {
+            return;
+        }
+        if (status.state?.running) {
+            seen.add(current);
+            track(before.then(() => follow(name)));
+        }
+        else if (status.state?.terminated) {
+            seen.add(current);
+            track(before.then(() => readEnded(name, false)));
+        }
+    };
+    const poll = async () => {
+        try {
+            const pods = await instance.coreApi.listNamespacedPod({ namespace, labelSelector });
+            if (!stopped) {
+                pods.items.forEach(look);
+            }
+        }
+        catch (e) {
+            fail(e);
+        }
+    };
+    const loop = () => {
+        if (stopped) {
+            return;
+        }
+        busy = true;
+        polling = poll().finally(() => {
+            busy = false;
+            if (stopped) {
+                return;
+            }
+            if (again) {
+                again = false;
+                loop();
+                return;
+            }
+            const interval = streaming > 0 ? options.followingInterval ?? 15000 : options.pollInterval ?? 1000;
+            timer = setTimeout(loop, interval);
+            timer.unref?.();
+        });
+    };
+    // looks now, not at the next interval
+    function wake() {
+        if (stopped) {
+            return;
+        }
+        if (busy) {
+            again = true;
+            return;
+        }
+        clearTimeout(timer);
+        loop();
+    }
+    loop();
+    return {
+        async settle() {
+            await polling;
+            await poll();
+            // what follows a running instance never ends by itself
+            const ended = [...reading];
+            let wait;
+            await Promise.race([
+                Promise.all(ended),
+                new Promise((resolve) => {
+                    wait = setTimeout(resolve, 5000);
+                    wait.unref?.();
+                }),
+            ]);
+            clearTimeout(wait);
+        },
+        stop() {
+            stopped = true;
+            clearTimeout(timer);
+            for (const controller of following.splice(0)) {
+                controller.abort();
+            }
+        },
+    };
+}
+exports.followPodLogs = followPodLogs;
+//# sourceMappingURL=follow-pod-logs.js.map

package/dist/planner/utils/append-work-service.js

@@ -143,7 +143,11 @@ function appendServiceInit(workTree, service, item, environment, context) {
         task.needs[selfNeed] = { name: task.needs[selfNeed].name, service: view };
     }
     else {
-        task.needs.push({ name: item.name, service: view });
+        const name = getInitNeedName(service);
+        if (task.needs.some((need) => need.name === name)) {
+            throw new Error(`${task.name} needs two services as ${name}, give one another name with name:`);
+        }
+        task.needs.push({ name, service: view });
     }
     removeRequirer(item, task.name);
     // explicit, so neither the default method nor `--cache` turns it back on
@@ -153,6 +157,18 @@ function appendServiceInit(workTree, service, item, environment, context) {
     }
     item.data.init = { task, state };
 }
+// The host name the init reaches its service by: the name its task declares the
+// need by, else the service's name in its own build file. Planning the task
+// skips that need (it is the init's), and the full name of a service of an
+// include, `services:db`, is no host name: Docker and DNS take no colon.
+function getInitNeedName(service) {
+    const declared = service.init?.task.needs.find((need) => need.service.relativeName === service.relativeName);
+    if (declared) {
+        return declared.relativeName;
+    }
+    const prefix = service.scope.namePrefix;
+    return prefix.length > 0 ? service.relativeName.slice(prefix.length + 1) : service.relativeName;
+}
 const initViews = new WeakMap();
 // The service itself for the view its init task needs, else the item.
 function getPlannedService(item) {

package/dist/planner/work-runtime-kubernetes.js

@@ -11,6 +11,7 @@ const replace_env_variables_1 = require("../environment/replace-env-variables");
 const apply_1 = require("../kubernetes/apply");
 const ensure_kubernetes_service_exists_1 = require("../kubernetes/ensure-kubernetes-service-exists");
 const ensure_kubernetes_deployment_exists_1 = require("../kubernetes/ensure-kubernetes-deployment-exists");
+const follow_pod_logs_1 = require("../kubernetes/follow-pod-logs");
 const service_definition_1 = require("./service-definition");
 const ensure_namespace_1 = require("../kubernetes/ensure-namespace");
 const ensure_persistent_data_1 = require("../kubernetes/ensure-persistent-data");
@@ -544,7 +545,31 @@ function kubernetesServiceRuntime(service, kubernetes) {
                 followLog: false,
                 run: kubernetes.server ? (0, job_placement_1.getRunLabelValue)(kubernetes.server.run) : null,
             });
-            await (0, ensure_kubernetes_deployment_exists_1.ensureKubernetesDeploymentExists)(instance, kubernetes, service, persistence, (0, secrets_1.getKubernetesSecretRefs)(service, kubernetes, service.data.secrets), options.stateKey, network);
+            // its output, as on Docker: while it starts, and while it runs unless it runs on
+            // without hammerkit (`up --daemon`)
+            const logs = (0, follow_pod_logs_1.followPodLogs)(instance, kubernetes.namespace, (0, resources_1.getItemSelector)(service, kubernetes), (0, ensure_kubernetes_deployment_exists_1.getServiceContainerName)(service), (line) => service.status.console('stdout', line), { onError: (text) => service.status.write('debug', text) });
+            try {
+                await (0, ensure_kubernetes_deployment_exists_1.ensureKubernetesDeploymentExists)(instance, kubernetes, service, persistence, (0, secrets_1.getKubernetesSecretRefs)(service, kubernetes, service.data.secrets), options.stateKey, network);
+            }
+            catch (e) {
+                // the last words of a container that crashed, before why it failed
+                await logs.settle().catch(() => undefined);
+                logs.stop();
+                throw e;
+            }
+            if (options.daemon || options.abort.aborted || environment.abortCtrl.signal.aborted) {
+                logs.stop();
+            }
+            else {
+                // either ends it, and takes the other listener with it
+                const listening = new AbortController();
+                const stop = () => {
+                    logs.stop();
+                    listening.abort();
+                };
+                options.abort.addEventListener('abort', stop, { once: true, signal: listening.signal });
+                environment.abortCtrl.signal.addEventListener('abort', stop, { once: true, signal: listening.signal });
+            }
             const name = (0, resources_1.getResourceName)(service, kubernetes);
             options.state.set({
                 type: 'running',

package/dist/remote/remote-work-commands.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.registerWorkCommands = exports.describePlan = exports.describeApproval = void 0;
+exports.registerWorkCommands = exports.describePlan = exports.describeApproval = exports.describeListedRun = exports.describeRunEnd = void 0;
 const commander_1 = require("commander");
 const colors_1 = __importDefault(require("colors"));
 const token_source_1 = require("./token-source");
@@ -67,6 +67,31 @@ function printableLine(text) {
     return text.split('\t').map(server_client_1.printableWhole).join('\t');
 }
 const collectOption = (value, previous = []) => [...previous, value];
+/**
+ * Why a run that did not succeed ended: its error, then each task that failed with its own
+ * error or exit code, where the error of the run does not say it already.
+ */
+function describeRunEnd(run) {
+    const lines = [`${(0, server_client_1.printable)(run.id)} ${(0, server_client_1.printable)(run.state)}${run.error ? `: ${(0, server_client_1.printableWhole)(run.error)}` : ''}`];
+    for (const [task, result] of Object.entries(run.results ?? {})) {
+        if (result.status !== 'failed') {
+            continue;
+        }
+        const why = result.error ?? (result.exitCode !== undefined ? `exit code ${result.exitCode}` : undefined);
+        if (why && !run.error?.includes(`${task}: ${why}`)) {
+            lines.push(`  ${(0, server_client_1.printable)(task)}: ${(0, server_client_1.printableWhole)(why)}`);
+        }
+    }
+    return lines;
+}
+exports.describeRunEnd = describeRunEnd;
+// What `remote runs` shows of a run, on one line.
+function describeListedRun(run) {
+    const what = run.up ? `up ${run.up.services.join(', ')}` : run.tasks.join(', ') || 'by labels';
+    const until = run.up && (run.state === 'up' || run.state === 'asleep') && run.expiresAt ? ` until ${run.expiresAt}` : '';
+    return `${(0, server_client_1.printable)(run.id)}  ${(0, server_client_1.printable)(run.createdAt)}  ${(0, server_client_1.printable)(run.state)}${(0, server_client_1.printable)(until)}  ${(0, server_client_1.printable)(what)}${run.error ? `: ${(0, server_client_1.printable)(run.error)}` : ''}`;
+}
+exports.describeListedRun = describeListedRun;
 // What a person who approves is shown of a run: everything the decision rests on.
 function describeApproval(run) {
     const source = run.source.type === 'ref'
@@ -241,7 +266,7 @@ function registerWorkCommands(remote, context) {
         const at = submitted.source.type === 'ref' ? ` at ${(0, server_client_1.printable)(submitted.source.commit)}` : '';
         environment.stdout.write(`${(0, server_client_1.printable)(submitted.id)} on ${serverName}: ${tasks.map(server_client_1.printable).join(', ')} from ${(0, server_client_1.printable)(ref)}${at}\n`);
         if (options.detach) {
-            environment.stdout.write(`follow it with: hammerkit remote show ${(0, server_client_1.printable)(submitted.id)}\n`);
+            environment.stdout.write(`follow it with: hammerkit remote logs ${(0, server_client_1.printable)(submitted.id)} --follow\n`);
             return;
         }
         const prefixed = tasks.length > 1;
@@ -267,7 +292,12 @@ function registerWorkCommands(remote, context) {
             .concat(Object.values(run.results ?? {}))
             .find((result) => result?.status === 'failed');
         const exitCode = failed?.exitCode && failed.exitCode > 0 ? failed.exitCode : 1;
-        fail(`${(0, server_client_1.printable)(run.id)} ${run.state}${run.error ? `: ${(0, server_client_1.printable)(run.error)}` : ''}`, exitCode);
+        const [said, ...why] = describeRunEnd(run);
+        for (const line of why) {
+            environment.stdout.write(`${line}\n`);
+        }
+        environment.stdout.write(`read its log again with: hammerkit remote logs ${(0, server_client_1.printable)(run.id)}\n`);
+        fail(said, exitCode);
     }));
     remote
         .command('up <services...>')
@@ -317,7 +347,12 @@ function registerWorkCommands(remote, context) {
         });
         const run = followed.run;
         if (run.state !== 'up') {
-            fail(`${(0, server_client_1.printable)(run.id)} ${run.state}${run.error ? `: ${(0, server_client_1.printable)(run.error)}` : ''}`, 1);
+            const [said, ...why] = describeRunEnd(run);
+            for (const line of why) {
+                environment.stdout.write(`${line}\n`);
+            }
+            environment.stdout.write(`read its log again with: hammerkit remote logs ${(0, server_client_1.printable)(run.id)}\n`);
+            fail(said, 1);
         }
         environment.stdout.write(`${(0, server_client_1.printable)(run.id)} is up until ${(0, server_client_1.printable)(run.expiresAt ?? '-')}${run.up?.sleepTtl ? `, asleep after ${(0, units_1.formatDuration)(run.up.sleepTtl)} without a request` : ''}\n`);
         for (const endpoint of run.up?.endpoints ?? []) {
@@ -325,6 +360,78 @@ function registerWorkCommands(remote, context) {
         }
         environment.stdout.write(`extend it with: hammerkit remote extend ${(0, server_client_1.printable)(run.id)} 8h, stop it with: hammerkit remote stop ${(0, server_client_1.printable)(run.id)}\n`);
     }));
+    remote
+        .command('runs [server]')
+        .description('list your runs on a server, newest first: their state, and why one failed')
+        .addOption(new commander_1.Option('--limit <count>', 'show at most this many').default('20'))
+        .action(guarded('Remote runs was not successful', async (name, options) => {
+        const limit = Number(options.limit);
+        if (!Number.isInteger(limit) || limit < 1) {
+            throw new Error('--limit is a number of runs, 1 or more');
+        }
+        const { name: serverName, server, token } = await connect(name);
+        const { runs } = (await (0, server_client_1.callServer)(server, token, 'GET', '/v1/runs')).body;
+        if (runs.length === 0) {
+            environment.stdout.write(`${serverName}: you have no runs it still keeps\n`);
+            return;
+        }
+        for (const run of runs.slice(0, limit)) {
+            environment.stdout.write(`${describeListedRun(run)}\n`);
+        }
+        if (runs.length > limit) {
+            environment.stdout.write(`… and ${runs.length - limit} older, see them with --limit ${runs.length}\n`);
+        }
+    }));
+    remote
+        .command('logs <run> [server]')
+        .description('print the log of a run, also of one that ended: what its tasks and services printed, and what the server and the cluster said')
+        .addOption(new commander_1.Option('--task <name>', 'only the lines of this task or service'))
+        .addOption(new commander_1.Option('--tail <lines>', 'only the last lines, at most 1000').argParser((value) => {
+        if (!/^\d+$/.test(value) || Number(value) < 1 || Number(value) > 1000) {
+            throw new commander_1.InvalidArgumentError('a number of lines from 1 to 1000');
+        }
+        return Number(value);
+    }))
+        .addOption(new commander_1.Option('-f, --follow', 'keep printing what comes until the run ends; ctrl-c stops following, not the run'))
+        .action(guarded('Remote logs was not successful', async (id, name, options) => {
+        if (!RUN_ID.test(id)) {
+            throw new Error(`${(0, server_client_1.printable)(id)} is not a run id (run_...), list them with: hammerkit remote runs`);
+        }
+        if (options.follow && options.tail !== undefined) {
+            throw new Error('--follow prints the whole log, leave --tail out');
+        }
+        const { name: serverName, server } = await getServer(name);
+        const connection = { url: server.url, ...(0, token_source_1.getTokenSource)(environment, serverName, server) };
+        const found = await (0, sandbox_client_1.getRun)(connection, id);
+        if (!found) {
+            throw new Error(`${serverName} keeps no run ${(0, server_client_1.printable)(id)} you may see, list them with: hammerkit remote runs`);
+        }
+        let run = found;
+        const task = options.task;
+        const print = (line) => environment.stdout.write(`${task ? '' : `[${(0, server_client_1.printable)(line.task)}] `}${printableLine(line.text)}\n`);
+        if (options.follow) {
+            run = (await (0, sandbox_client_1.followRun)(connection, run, {
+                task,
+                signal: environment.abortCtrl.signal,
+                detach: true,
+                onLine: print,
+            })).run;
+        }
+        else {
+            await (0, sandbox_client_1.readRunLog)(connection, run, { task, tail: options.tail }, print);
+        }
+        if (run.state === 'succeeded' || run.state === 'up' || run.state === 'asleep') {
+            environment.stdout.write(`${(0, server_client_1.printable)(run.id)} ${(0, server_client_1.printable)(run.state)}\n`);
+        }
+        else if (sandbox_client_1.FINAL_RUN_STATES.includes(run.state)) {
+            for (const line of describeRunEnd(run)) {
+                environment.stdout.write(`${line}\n`);
+            }
+        }
+        else {
+            environment.stdout.write(`${(0, server_client_1.printable)(run.id)} is ${(0, server_client_1.printable)(run.state)}, follow it with --follow\n`);
+        }
+    }));
     remote
         .command('extend <run> <ttl> [server]')
         .description('keep a service run that is up for this long from now, such as 8h, at most the maxTtl of the server')

package/dist/schema/build-file-container-task-schema.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.buildFileContainerTaskSchema = void 0;
+exports.CONTAINER_TASK_PROPERTIES = exports.buildFileExtendingContainerTaskSchema = exports.buildFileContainerTaskSchema = void 0;
 const build_file_base_task_schema_1 = require("./build-file-base-task-schema");
 const zod_1 = require("zod");
 const size_schema_1 = require("./size-schema");
@@ -17,4 +17,15 @@ exports.buildFileContainerTaskSchema = build_file_base_task_schema_1.buildFileBa
 })
     .strict()
     .describe('container task');
+// A task that extends another takes the image of its base, so it may set the
+// container properties without an image of its own; once extended, a task whose
+// base has no image either is refused them (applySchemaExtension).
+exports.buildFileExtendingContainerTaskSchema = exports.buildFileContainerTaskSchema
+    .extend({
+    image: (0, zod_1.string)().optional(),
+    extend: (0, zod_1.string)(),
+})
+    .strict()
+    .describe('container task taking its image from the task it extends');
+exports.CONTAINER_TASK_PROPERTIES = ['mounts', 'volumeSize', 'capAdd'];
 //# sourceMappingURL=build-file-container-task-schema.js.map

package/dist/schema/build-file-task-schema.js

@@ -4,7 +4,11 @@ exports.isBuildFileLocalTaskSchema = exports.isBuildFileContainerTaskSchema = ex
 const zod_1 = require("zod");
 const build_file_container_task_schema_1 = require("./build-file-container-task-schema");
 const build_file_local_task_schema_1 = require("./build-file-local-task-schema");
-exports.buildFileTaskSchema = (0, zod_1.union)([build_file_container_task_schema_1.buildFileContainerTaskSchema, build_file_local_task_schema_1.buildFileLocalTaskSchema]);
+exports.buildFileTaskSchema = (0, zod_1.union)([
+    build_file_container_task_schema_1.buildFileContainerTaskSchema,
+    build_file_local_task_schema_1.buildFileLocalTaskSchema,
+    build_file_container_task_schema_1.buildFileExtendingContainerTaskSchema,
+]);
 const isBuildFileContainerTaskSchema = (val) => 'image' in val;
 exports.isBuildFileContainerTaskSchema = isBuildFileContainerTaskSchema;
 const isBuildFileLocalTaskSchema = (val) => !('image' in val);

package/dist/schema/reference-parser.js

@@ -7,6 +7,7 @@ const split_name_1 = require("../planner/utils/split-name");
 const read_env_file_1 = require("../parser/read-env-file");
 const label_values_1 = require("../executer/label-values");
 const merge_environment_variables_1 = require("../environment/merge-environment-variables");
+const build_file_container_task_schema_1 = require("./build-file-container-task-schema");
 const resolve_cache_1 = require("../cache/resolve-cache");
 const project_root_1 = require("../planner/project-root");
 const secret_catalog_1 = require("../secrets/secret-catalog");
@@ -222,6 +223,7 @@ function applySchemaExtension(reference, task) {
         if (!reset.has('mounts'))
             taskSchema.mounts = extendArray(taskSchema.mounts, extend.schema.mounts);
     }
+    rejectContainerPropertiesWithoutImage(task);
     // deps and needs resolve in the file that declares them, so every base up the
     // chain contributes its own, not only the nearest; what a task resets is not
     // passed on to the tasks extending it either
@@ -236,6 +238,16 @@ function applySchemaExtension(reference, task) {
     }
     task.extension = 'applied';
 }
+function rejectContainerPropertiesWithoutImage(task) {
+    if ((0, build_file_task_schema_1.isBuildFileContainerTaskSchema)(task.schema)) {
+        return;
+    }
+    const schema = task.schema;
+    const property = build_file_container_task_schema_1.CONTAINER_TASK_PROPERTIES.find((key) => schema[key] !== undefined);
+    if (property) {
+        throw new Error(`task ${task.relativeName} sets ${property}, but neither it nor its base has an image`);
+    }
+}
 function extendArray(base, extend) {
     if (!extend) {
         return base;

package/dist/server/run-worker.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.workerMain = exports.readInput = exports.taskCauses = exports.runTasks = exports.getEndpointTargets = exports.publishEndpoints = exports.getRunNetwork = exports.workerInputSchema = void 0;
+exports.workerMain = exports.readInput = exports.taskCauses = exports.describeFailures = exports.runTasks = exports.getEndpointTargets = exports.publishEndpoints = exports.getRunNetwork = exports.workerInputSchema = void 0;
 const readline_1 = require("readline");
 const zod_1 = require("zod");
 const cli_1 = require("../cli");
@@ -22,6 +22,8 @@ const work_network_1 = require("../planner/work-network");
 const run_plan_1 = require("./run-plan");
 const runs_1 = require("./runs");
 const resources_1 = require("../kubernetes/resources");
+const cluster_state_1 = require("../kubernetes/cluster-state");
+const kubernetes_instance_1 = require("../kubernetes/kubernetes-instance");
 const check_keys_1 = require("../optimizer/check-keys");
 // one log line of a task is cut here, the registry caps how many events a run keeps
 const MAX_LOG_TEXT = 8 * 1024;
@@ -234,9 +236,11 @@ up) {
                 ? cli.up({ ...execOptions, workers: cli.defaultWorkers(), daemon: false, wait: 'ready' })
                 : cli.exec({ ...execOptions, workers: cli.defaultWorkers() });
             const seen = new Map();
+            const failedServices = new Set();
             let ready = false;
             const report = () => {
                 reportTasks(emit, execution.state.current, seen, (text) => secrets.redact(text));
+                reportServiceFailures(emit, execution.state.current, failedServices, (text) => secrets.redact(text));
                 if (input.up && up && !ready) {
                     const targets = getEndpointTargets(execution.state.current, input.up, input.kubernetes);
                     if (targets) {
@@ -262,7 +266,16 @@ up) {
                     return;
                 }
                 if (!result.success) {
-                    throw new Error(cancelled() ? 'the run was cancelled' : `${name} failed`);
+                    if (cancelled()) {
+                        throw new Error('the run was cancelled');
+                    }
+                    if (input.kubernetes) {
+                        // what the cluster said, before the server removes what the run made
+                        await reportClusterState(input.kubernetes, execution.state.current, !!input.up, emit, (text) => secrets.redact(text));
+                    }
+                    // a service run names what failed and why: its services have no results of their own
+                    const failures = input.up ? describeFailures(execution.state.current, (text) => secrets.redact(text)) : [];
+                    throw new Error(failures.length > 0 ? failures.join('; ').slice(0, runs_1.MAX_TASK_ERROR) : `${name} failed`);
                 }
             }
             finally {
@@ -275,6 +288,82 @@ up) {
     }
 }
 exports.runTasks = runTasks;
+// What the cluster said of a run that failed, into its log: the server removes its pods
+// once it ended.
+async function reportClusterState(kubernetes, workTree, ownNamespace, emit, redact) {
+    let lines;
+    try {
+        lines = await (0, cluster_state_1.readClusterState)((0, kubernetes_instance_1.createKubernetesInstances)(kubernetes), kubernetes, workTree, ownNamespace);
+    }
+    catch (e) {
+        lines = [`it could not be read: ${(0, log_1.getErrorMessage)(e)}`];
+    }
+    if (lines.length === 0) {
+        return;
+    }
+    for (const text of [`what the cluster said in ${kubernetes.namespace}:`, ...lines.map((line) => `  ${line}`)]) {
+        emit({ type: 'log', task: 'hammerkit', stream: 'stderr', text: redact(text).slice(0, MAX_LOG_TEXT) });
+    }
+}
+// A service has no result of its own in the run: why it failed goes into the log, under
+// its name, once.
+function reportServiceFailures(emit, workTree, said, redact) {
+    for (const service of (0, plan_work_tasks_1.iterateWorkServices)(workTree)) {
+        const state = service.state.current;
+        if (state.type === 'error' && !said.has(service.name)) {
+            said.add(service.name);
+            if (lastError(service) === state.errorMessage) {
+                // the runtime said it already
+                continue;
+            }
+            emit({
+                type: 'log',
+                task: service.name,
+                stream: 'stderr',
+                text: redact(state.errorMessage).slice(0, MAX_LOG_TEXT),
+            });
+        }
+    }
+}
+// The last error the runtime of an item wrote to its status.
+function lastError(item) {
+    let last;
+    for (const message of item.status.read()) {
+        if (message.level === 'error') {
+            last = message.message;
+        }
+    }
+    return last;
+}
+/**
+ * What failed in a run, one line each, for its error: the services and tasks that failed,
+ * with their reason. Those that only stopped because another failed are left out.
+ */
+function describeFailures(workTree, redact) {
+    const failures = [];
+    for (const service of (0, plan_work_tasks_1.iterateWorkServices)(workTree)) {
+        const state = service.state.current;
+        if (state.type === 'error') {
+            failures.push(`${service.name}: ${redact(state.errorMessage)}`);
+        }
+        else if (state.type === 'end' && state.reason === 'crash') {
+            // on Docker a service that does not get ready ends, and its runtime says why
+            const why = lastError(service);
+            failures.push(`${service.name}: ${why ? redact(why) : 'it stopped on its own'}`);
+        }
+    }
+    for (const task of (0, plan_work_tasks_1.iterateWorkTasks)(workTree)) {
+        const state = task.state.current;
+        if (state.type === 'error') {
+            failures.push(`${task.name}: ${redact(state.errorMessage)}`);
+        }
+        else if (state.type === 'crash') {
+            failures.push(`${task.name}: exit code ${state.exitCode}`);
+        }
+    }
+    return failures;
+}
+exports.describeFailures = describeFailures;
 // Why a task ran, as a task event carries it: the first causes, masked and cut short.
 function taskCauses(causes, redact) {
     return causes?.length

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "hammerkit",
   "description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
-  "version": "1.27.0-next.24",
+  "version": "1.27.0-next.26",
   "keywords": [
     "build",
     "build-tool",