@cat-factory/executor-harness 1.145.1 → 1.147.0

package/dist/docker-command.js

@@ -0,0 +1,91 @@
+import { spawn } from 'node:child_process';
+import { log } from './logger.js';
+import { killChildProcess, spawnDetached } from './process.js';
+// ---------------------------------------------------------------------------
+// How the harness runs one `docker …` command ON ITS OWN BEHALF, bounded and abortable.
+//
+// This is NOT a second `runCapturedCommand` (captured-command.ts), which stays the one way the
+// harness runs a DECLARED shell command: that one takes a shell string, merges both streams into
+// one rolling tail and answers with a conventional exit code, because its two callers report a
+// pass/fail plus a tail to a model. The docker checks need the three things it deliberately does
+// not offer: an argv (no shell, so nothing quotes an image tag), a STDIN body (the probe archive
+// is piped to `docker load`), and stdout kept APART from stderr, since the whole evidence that a
+// container ran is a marker on stdout while the evidence of why it did not is on stderr.
+//
+// What it does NOT re-decide is how a child dies: `killChildProcess` owns the SIGTERM→SIGKILL
+// escalation for every process this harness spawns, and a bespoke `SIGKILL` here would be one
+// path whose kill semantics drift from the rest with no test able to see it.
+// ---------------------------------------------------------------------------
+/** How much of each stream is buffered. The TAIL is kept: that is where a failure prints. */
+const OUTPUT_CAP_CHARS = 64 * 1024;
+/**
+ * The real runner: spawn docker, feed it `stdin` when there is any, and report what happened.
+ *
+ * Never rejects. Every way a spawn can go wrong is one of the two outcomes, because the caller
+ * classifies them differently and an exception would collapse that distinction into whichever
+ * `catch` caught it first.
+ */
+export const spawnDockerCommand = (args, opts) => new Promise((resolve) => {
+    const logger = opts.logger ?? log;
+    if (opts.signal?.aborted) {
+        resolve({ outcome: 'failed', reason: abandonedReason(args) });
+        return;
+    }
+    const child = spawn('docker', args, {
+        stdio: ['pipe', 'pipe', 'pipe'],
+        detached: spawnDetached,
+        windowsHide: true,
+    });
+    let stdout = '';
+    let stderr = '';
+    let settled = false;
+    const finish = (result) => {
+        if (settled)
+            return;
+        settled = true;
+        clearTimeout(timer);
+        opts.signal?.removeEventListener('abort', onAbort);
+        resolve(result);
+    };
+    const timer = setTimeout(() => {
+        logger.warn('docker: command did not answer in time, killing it', {
+            command: args[0] ?? '',
+            timeoutMs: opts.timeoutMs,
+        });
+        killChildProcess(child, undefined, logger);
+        finish({
+            outcome: 'failed',
+            reason: `\`docker ${args[0] ?? ''}\` did not answer within ${Math.round(opts.timeoutMs / 1000)}s`,
+        });
+    }, opts.timeoutMs);
+    timer.unref?.();
+    const onAbort = () => {
+        killChildProcess(child, undefined, logger);
+        finish({ outcome: 'failed', reason: abandonedReason(args) });
+    };
+    opts.signal?.addEventListener('abort', onAbort, { once: true });
+    // The tail, not the head: a `docker run` that failed says why in its last lines, and the
+    // marker a passing one prints is the whole of its output anyway.
+    child.stdout.on('data', (chunk) => {
+        stdout = (stdout + chunk.toString('utf8')).slice(-OUTPUT_CAP_CHARS);
+    });
+    child.stderr.on('data', (chunk) => {
+        stderr = (stderr + chunk.toString('utf8')).slice(-OUTPUT_CAP_CHARS);
+    });
+    child.on('error', (err) => {
+        finish({
+            outcome: 'failed',
+            reason: err.code === 'ENOENT'
+                ? 'the docker CLI is not on PATH'
+                : `the docker CLI could not be spawned (${err.code ?? err.message})`,
+        });
+    });
+    child.on('close', (code) => finish({ outcome: 'ran', code: code ?? -1, stdout, stderr }));
+    // A daemon that dies mid-load closes the pipe under us; `close` above already reports that,
+    // so the EPIPE here has nothing to add and must not become an unhandled error event.
+    child.stdin.on('error', () => { });
+    child.stdin.end(opts.stdin);
+});
+function abandonedReason(args) {
+    return `the job was cancelled before \`docker ${args[0] ?? ''}\` answered`;
+}

package/dist/docker-probe-image.d.ts

@@ -0,0 +1,36 @@
+/** The tag the probe image is loaded under. Removed again once the check has answered. */
+export declare const PROBE_IMAGE_TAG = "cat-factory-docker-probe:1";
+/**
+ * What the container must print for the check to pass.
+ *
+ * A marker on stdout rather than a zero exit status: the point of the check is that a process
+ * inside the container actually ran, and only output it produced proves that. An exit status is
+ * the daemon's word for it.
+ */
+export declare const PROBE_SENTINEL = "cat-factory-docker-probe-ok";
+/** The argv the probe container runs. `busybox` dispatches on its own name, so this is an echo. */
+export declare const PROBE_COMMAND: readonly string[];
+/** The docker name for the architecture THIS process's payload is built for, when there is one. */
+export declare function payloadArchitecture(arch?: string): string | undefined;
+interface TarEntry {
+    name: string;
+    content: Buffer;
+    mode: number;
+}
+/** A whole tar stream: the members, then the two zero blocks that terminate one. */
+export declare function tarArchive(entries: readonly TarEntry[]): Buffer;
+/**
+ * Assemble the docker-archive `docker load` reads, from one statically linked binary.
+ *
+ * Classic (v1) docker-archive rather than OCI layout: `docker load` accepts both on every engine
+ * this image can run against, and the v1 shape is three files with no blob directory to get
+ * wrong. The layer digest is the sha256 of the UNCOMPRESSED layer tar, which is what
+ * `rootfs.diff_ids` means; an engine that disagrees with it refuses the load, which the caller
+ * reads as could-not-determine rather than as a broken daemon.
+ *
+ * `architecture` is the DAEMON's own word for its architecture, in docker's vocabulary, so
+ * nothing here decides it (see {@link PAYLOAD_ARCHITECTURES}). The result is byte-stable for one
+ * `(payload, architecture)` pair, which is what lets the caller build it once per container.
+ */
+export declare function buildProbeArchive(payload: Buffer, architecture: string): Buffer;
+export {};

package/dist/docker-probe-image.js

@@ -0,0 +1,148 @@
+import { createHash } from 'node:crypto';
+// ---------------------------------------------------------------------------
+// The one-container image the platform runs to find out whether this machine's Docker daemon
+// can actually run a container, built here in memory rather than pulled.
+//
+// It exists because `docker info` answers a different question from the one every caller
+// actually asks. A daemon that ANSWERS is not a daemon that WORKS: this container's rootless
+// daemon runs inside whatever sandbox the deployment gave it, and a nested user namespace
+// routinely refuses the overlay mount every image materialisation needs. The daemon serves
+// happily, `docker version` reports a server, and `docker pull` of a multi-layer image,
+// `docker run` of a single-layer one and `docker build` all fail with the same EINVAL. Issue
+// #2120 is three agents in one run each discovering that for themselves, against a system
+// prompt that told them, as stated fact, that Docker worked here.
+//
+// Why the payload is BUILT and not pulled: a probe that needs the network answers a question
+// about the registry as much as about the daemon, cannot run in a sandbox with no egress, and
+// costs the job its first turn. This one is a single layer holding one statically linked
+// binary already in the image, assembled into a docker-archive tar and handed to `docker load`
+// on stdin, so the whole check is local and takes about as long as starting one container.
+//
+// ONE layer, deliberately. The reported failure kills `docker run` of a single-layer image too
+// (the container's own writable layer is already a second overlay lower dir), so one layer is
+// enough to detect it, and it keeps `docker load` (the step this file could plausibly get WRONG)
+// as small as it can be. That matters because the caller reads a load failure as "could
+// not determine" and a RUN failure as "this daemon cannot run containers": a bug in the archive
+// below must never be able to tell an agent that a working daemon is broken.
+// ---------------------------------------------------------------------------
+/** The tag the probe image is loaded under. Removed again once the check has answered. */
+export const PROBE_IMAGE_TAG = 'cat-factory-docker-probe:1';
+/** Where the payload binary lands inside the probe image, and what the container is asked to run. */
+const PROBE_BINARY_PATH = 'busybox';
+/**
+ * What the container must print for the check to pass.
+ *
+ * A marker on stdout rather than a zero exit status: the point of the check is that a process
+ * inside the container actually ran, and only output it produced proves that. An exit status is
+ * the daemon's word for it.
+ */
+export const PROBE_SENTINEL = 'cat-factory-docker-probe-ok';
+/** The argv the probe container runs. `busybox` dispatches on its own name, so this is an echo. */
+export const PROBE_COMMAND = [`/${PROBE_BINARY_PATH}`, 'echo', PROBE_SENTINEL];
+/**
+ * Node's architecture names mapped onto the docker name for the same machine.
+ *
+ * This names THE PAYLOAD, never the daemon. `process.arch` is the architecture of the harness
+ * process, and the binary it hands over is built for that; the daemon it is measured against
+ * answers for itself (`docker version --format {{.Server.Arch}}`), and the caller compares the
+ * two rather than assuming they agree. An external `DOCKER_HOST` is a first-class path here, and
+ * an arm64 harness talking to an amd64 sidecar shares nothing with it but the socket.
+ *
+ * That comparison is also what makes two of these entries safe. `process.arch` reports `ppc64` on
+ * both endiannesses and `arm` with no variant, so those rows are a HYPOTHESIS about the payload,
+ * not a claim: a machine the guess is wrong about answers with a different name and the check
+ * reports that it could not be carried out. Nothing here may produce a verdict about the daemon.
+ * An architecture nothing maps does the same, which is why `386` was worth adding rather than
+ * leaving to a fallback: the mapping is unambiguous and its absence cost a real check.
+ */
+const PAYLOAD_ARCHITECTURES = {
+    x64: 'amd64',
+    ia32: '386',
+    arm64: 'arm64',
+    arm: 'arm',
+    s390x: 's390x',
+    ppc64: 'ppc64le',
+    riscv64: 'riscv64',
+};
+/** The docker name for the architecture THIS process's payload is built for, when there is one. */
+export function payloadArchitecture(arch = process.arch) {
+    return PAYLOAD_ARCHITECTURES[arch];
+}
+const TAR_BLOCK = 512;
+/** A fixed timestamp everywhere a tar or an image config wants one, so the archive is byte-stable. */
+const EPOCH = '1970-01-01T00:00:00Z';
+/**
+ * One ustar header block.
+ *
+ * The checksum is summed with its own field read as eight SPACES and only then written back
+ * over it, which is the format's own rule and the one detail a hand-rolled writer gets wrong. A
+ * header whose checksum covers its own checksum bytes is rejected by every reader, and
+ * `docker load` reports that as an unreadable archive, which this module's caller would then
+ * have to decide was not the daemon's fault.
+ */
+function tarHeader(name, size, mode) {
+    const header = Buffer.alloc(TAR_BLOCK);
+    header.write(name, 0, 100, 'utf8');
+    header.write(octalField(mode, 8), 100, 8, 'latin1');
+    header.write(octalField(0, 8), 108, 8, 'latin1'); // uid
+    header.write(octalField(0, 8), 116, 8, 'latin1'); // gid
+    header.write(octalField(size, 12), 124, 12, 'latin1');
+    header.write(octalField(0, 12), 136, 12, 'latin1'); // mtime
+    header.write('        ', 148, 8, 'latin1');
+    header.write('0', 156, 1, 'latin1'); // typeflag: a regular file
+    header.write('ustar\0', 257, 6, 'latin1');
+    header.write('00', 263, 2, 'latin1');
+    let sum = 0;
+    for (const byte of header)
+        sum += byte;
+    header.write(`${sum.toString(8).padStart(6, '0')}\0 `, 148, 8, 'latin1');
+    return header;
+}
+/** A numeric tar field: zero-padded octal in `width - 1` characters, then a NUL. */
+function octalField(value, width) {
+    return `${value.toString(8).padStart(width - 1, '0')}\0`;
+}
+/** One tar member: its header, its content, and the padding up to the next 512-byte block. */
+function tarMember(entry) {
+    const padding = (TAR_BLOCK - (entry.content.length % TAR_BLOCK)) % TAR_BLOCK;
+    return Buffer.concat([
+        tarHeader(entry.name, entry.content.length, entry.mode),
+        entry.content,
+        Buffer.alloc(padding),
+    ]);
+}
+/** A whole tar stream: the members, then the two zero blocks that terminate one. */
+export function tarArchive(entries) {
+    return Buffer.concat([...entries.map(tarMember), Buffer.alloc(TAR_BLOCK * 2)]);
+}
+/**
+ * Assemble the docker-archive `docker load` reads, from one statically linked binary.
+ *
+ * Classic (v1) docker-archive rather than OCI layout: `docker load` accepts both on every engine
+ * this image can run against, and the v1 shape is three files with no blob directory to get
+ * wrong. The layer digest is the sha256 of the UNCOMPRESSED layer tar, which is what
+ * `rootfs.diff_ids` means; an engine that disagrees with it refuses the load, which the caller
+ * reads as could-not-determine rather than as a broken daemon.
+ *
+ * `architecture` is the DAEMON's own word for its architecture, in docker's vocabulary, so
+ * nothing here decides it (see {@link PAYLOAD_ARCHITECTURES}). The result is byte-stable for one
+ * `(payload, architecture)` pair, which is what lets the caller build it once per container.
+ */
+export function buildProbeArchive(payload, architecture) {
+    const layer = tarArchive([{ name: PROBE_BINARY_PATH, content: payload, mode: 0o755 }]);
+    const diffId = `sha256:${createHash('sha256').update(layer).digest('hex')}`;
+    const config = Buffer.from(JSON.stringify({
+        architecture,
+        os: 'linux',
+        created: EPOCH,
+        config: {},
+        rootfs: { type: 'layers', diff_ids: [diffId] },
+        history: [{ created: EPOCH, created_by: 'cat-factory docker capability probe' }],
+    }));
+    const manifest = Buffer.from(JSON.stringify([{ Config: 'config.json', RepoTags: [PROBE_IMAGE_TAG], Layers: ['layer.tar'] }]));
+    return tarArchive([
+        { name: 'config.json', content: config, mode: 0o644 },
+        { name: 'layer.tar', content: layer, mode: 0o644 },
+        { name: 'manifest.json', content: manifest, mode: 0o644 },
+    ]);
+}

package/dist/docker-status.d.ts

@@ -1,3 +1,5 @@
+import { type DockerWorkload } from './docker-capability.js';
+import { type Logger } from './logger.js';
 /**
  * Where `entrypoint.sh` records its verdict. The two halves of one contract: change this and the
  * `DOCKER_STATUS_FILE` default in `entrypoint.sh` together. `HARNESS_DOCKER_STATUS_FILE`
@@ -58,32 +60,97 @@ export declare function readDockerStatus(path?: string): Promise<DockerStatus>;
  * adding a source without a sentence stops building.
  */
 export declare function describeDockerAbsence(status: DockerStatus): string;
-/** Whether a daemon is answering RIGHT NOW. Injected so the unit suite can state either answer. */
-export type DockerProbe = () => Promise<boolean>;
 /**
- * The default {@link DockerProbe}: `docker version` talks to the SERVER, unlike the client-only
- * `docker --version`, which answers happily with no daemon at all.
+ * What a daemon can do RIGHT NOW. Injected so the unit suite can state every answer.
+ *
+ * It answers with a WORKLOAD rather than with a boolean, and that is the correction this type
+ * carries. It used to be `docker version`, which proves the daemon is serving; a stand-up needs
+ * a daemon that can materialise an image, and a sandboxed rootless daemon routinely serves while
+ * being unable to (issue #2120). Running compose against that one produced a mount error the
+ * agent had to interpret, from the one mechanism whose entire job is to say why infra did not
+ * come up.
+ *
+ * The weaker fact did not go away, though: it rides `daemonAnswered` on the `unknown` arm, since
+ * the workload check establishes it on its way past and {@link resolveDockerVerdict} still needs
+ * it. Takes the job's signal, because this is a live check on the critical path of a run that can
+ * be cancelled under it.
+ */
+export type DockerProbe = (signal?: AbortSignal) => Promise<DockerWorkload>;
+/**
+ * The default {@link DockerProbe}: the process-wide workload probe, which loads a one-layer
+ * image and runs a container from it, memoised per container.
+ *
+ * Named for what it answers. It was `probeDockerServing`, which is the fact this module exists to
+ * say is not enough.
  */
-export declare const probeDockerServing: DockerProbe;
+export declare const probeLiveDockerCapability: DockerProbe;
+/**
+ * The sentence for a daemon that is serving and cannot run anything. It names what was tried,
+ * because "docker is unavailable" against a daemon the agent can see answering reads as a bug in
+ * the platform rather than as the sandbox limit it is.
+ */
+export declare function describeDockerUnusable(workload: {
+    detail: string;
+}): string;
 /** What a stand-up is entitled to conclude about the daemon at the moment it is about to run. */
 export interface DockerVerdict {
-    /** Three-valued exactly as {@link DockerStatus.available}, and read the same way. */
+    /**
+     * Whether a stand-up may PROCEED, three-valued exactly as {@link DockerStatus.available} and
+     * read the same way. It is the decision, not a description of the daemon: a daemon that is
+     * answering and cannot run a container is `false` here and `daemon: true` below, and a record
+     * that reported the first as the second would send an operator to restart a daemon that is
+     * already up.
+     */
     available: boolean | undefined;
-    /** Set only for a CONFIRMED absence: the sentence to refuse with. Absent means proceed. */
+    /**
+     * Set only for a CONFIRMED negative: the sentence to refuse with. Absent means proceed.
+     *
+     * Two causes reach it and they read differently on purpose. Nothing is answering here, and
+     * something is answering here but cannot run a container: an operator sent to restart a daemon
+     * that is already up would find nothing wrong with it.
+     */
     refusal?: string;
+    /**
+     * Whether a daemon ANSWERED the live check. Absent when nothing was checked (an undecided
+     * record probes nothing) or when nothing answered, so it is never read as a decided `false`.
+     */
+    daemon?: boolean;
+    /** What a real container did on it, when one was tried. Absent when nothing was measured. */
+    workload?: DockerWorkload;
 }
 /**
- * Resolve what to do now, from what boot recorded plus what a daemon says today.
+ * Resolve what to do now, from what boot recorded plus what the daemon can do today.
+ *
+ * `entrypoint.sh` probes once, at boot, within a bounded wait, and it probes for a SOCKET. Two
+ * things follow, and the branches below are one each.
+ *
+ * A recorded absence is a HYPOTHESIS. A container outlives its boot: a warm pool serves many jobs
+ * from one, and a sidecar daemon that took longer than the wait allows is serving perfectly well
+ * by the second job. Refusing off the record alone latches that container into refusing local
+ * infra that works, for its whole life, with a stale sentence explaining why. The record is still
+ * what supplies the cause and the daemon's own log tail, which no probe can reconstruct.
+ *
+ * A recorded PRESENCE is a hypothesis too, and that half was missing. `serving` is not `usable`:
+ * a rootless daemon in a sandbox answers while unable to mount any image layer, so compose ran
+ * and died on a mount error the agent then had to interpret. So the probe is consulted in both
+ * directions, and it runs a real container rather than asking the daemon about itself.
  *
- * `entrypoint.sh` probes once, at boot, within a bounded wait. A container outlives that: a warm
- * pool serves many jobs from one, and a sidecar daemon that took longer than the wait allows is
- * serving perfectly well by the second job. Refusing off the recorded verdict alone latches that
- * container into refusing local infra that in fact works, for its whole life, with a stale
- * sentence explaining why. So a recorded absence is a HYPOTHESIS here, and the live probe settles
- * it; the recorded verdict is still what supplies the cause and the daemon's own log tail, which
- * no probe can reconstruct.
+ * A check that could not be CARRIED OUT settles nothing, and the cheap fact is what decides
+ * there. Falling straight back to the boot record would re-latch the very refusal the paragraph
+ * above rules out: the four ways the workload check can come back undeterminable (no payload in
+ * this image variant, an architecture it is not built for, a `docker load` the engine refuses, a
+ * timeout) have nothing to do with whether a daemon is up, so a warm container whose sidecar
+ * arrived late would be denied local infra for the rest of its life over a stale sentence. So a
+ * daemon that ANSWERED contradicts a recorded absence exactly as the old `docker version` probe
+ * did, and only a check that never reached a daemon at all leaves the record to decide.
  *
- * Only a recorded `false` is re-confirmed. "Not decided" keeps attempting exactly as before: the
- * point of the third value is that nothing turns it into a refusal, and a probe here would.
+ * "Not decided" still keeps attempting, untouched. The point of the third value is that NOTHING
+ * turns it into a refusal: the entrypoint's bounded wait may still be running, and a workload
+ * probe against a daemon that has not finished starting fails for a reason that says nothing
+ * about what it will do a second later.
  */
-export declare function resolveDockerVerdict(status: DockerStatus, probe?: DockerProbe): Promise<DockerVerdict>;
+export declare function resolveDockerVerdict(status: DockerStatus, opts?: {
+    probe?: DockerProbe;
+    signal?: AbortSignal;
+    logger?: Logger;
+}): Promise<DockerVerdict>;

package/dist/docker-status.js

@@ -1,6 +1,6 @@
-import { execFile } from 'node:child_process';
 import { readFile } from 'node:fs/promises';
-import { promisify } from 'node:util';
+import { probeDockerWorkload } from './docker-capability.js';
+import { log } from './logger.js';
 // What this container knows about its own Docker daemon, as recorded by `entrypoint.sh`.
 //
 // The Tester's local-mode infra stand-up (`docker compose up --wait`) is the only thing in the
@@ -13,8 +13,12 @@ import { promisify } from 'node:util';
 // daemon reads it instead.
 //
 // The recorded verdict describes BOOT, and a container outlives its boot, so nothing refuses on
-// it unconfirmed: `resolveDockerVerdict` re-checks a recorded absence against a live daemon and
-// keeps the record for what only the record holds, the cause and the daemon's own log tail.
+// it unconfirmed: `resolveDockerVerdict` re-checks it against a live daemon and keeps the record
+// for what only the record holds, the cause and the daemon's own log tail. What it records is
+// also only that a SOCKET answered, which is a weaker fact than any caller wants, so the live
+// check RUNS A CONTAINER (docker-capability.ts) rather than settling for the daemon's word about
+// itself. It still reports that weaker fact alongside, since a check that could not be carried
+// out is what the boot record has to be read against, and nothing else establishes it.
 //
 // The three-valued shape is deliberate and is the point (CLAUDE.md, "Degrade loudly"): a daemon
 // that FAILED and a daemon nobody asked about are different facts with different correct
@@ -106,42 +110,95 @@ function absenceCause(source) {
 function unnamedSource(source) {
     return `no Docker daemon answered in this container (unrecognised source ${JSON.stringify(source)})`;
 }
-/** A live probe may not outlast the thing it is guarding; a hung socket is an absent daemon here. */
-const PROBE_TIMEOUT_MS = 10_000;
-const execFileAsync = promisify(execFile);
 /**
- * The default {@link DockerProbe}: `docker version` talks to the SERVER, unlike the client-only
- * `docker --version`, which answers happily with no daemon at all.
+ * The default {@link DockerProbe}: the process-wide workload probe, which loads a one-layer
+ * image and runs a container from it, memoised per container.
+ *
+ * Named for what it answers. It was `probeDockerServing`, which is the fact this module exists to
+ * say is not enough.
  */
-export const probeDockerServing = async () => {
-    try {
-        await execFileAsync('docker', ['version', '--format', '{{.Server.Version}}'], {
-            timeout: PROBE_TIMEOUT_MS,
-        });
-        return true;
-    }
-    catch {
-        return false;
-    }
-};
+export const probeLiveDockerCapability = (signal) => probeDockerWorkload(signal);
 /**
- * Resolve what to do now, from what boot recorded plus what a daemon says today.
+ * The sentence for a daemon that is serving and cannot run anything. It names what was tried,
+ * because "docker is unavailable" against a daemon the agent can see answering reads as a bug in
+ * the platform rather than as the sandbox limit it is.
+ */
+export function describeDockerUnusable(workload) {
+    return `this container's Docker daemon is reachable but cannot run a container (${workload.detail})`;
+}
+/**
+ * Resolve what to do now, from what boot recorded plus what the daemon can do today.
+ *
+ * `entrypoint.sh` probes once, at boot, within a bounded wait, and it probes for a SOCKET. Two
+ * things follow, and the branches below are one each.
  *
- * `entrypoint.sh` probes once, at boot, within a bounded wait. A container outlives that: a warm
- * pool serves many jobs from one, and a sidecar daemon that took longer than the wait allows is
- * serving perfectly well by the second job. Refusing off the recorded verdict alone latches that
- * container into refusing local infra that in fact works, for its whole life, with a stale
- * sentence explaining why. So a recorded absence is a HYPOTHESIS here, and the live probe settles
- * it; the recorded verdict is still what supplies the cause and the daemon's own log tail, which
- * no probe can reconstruct.
+ * A recorded absence is a HYPOTHESIS. A container outlives its boot: a warm pool serves many jobs
+ * from one, and a sidecar daemon that took longer than the wait allows is serving perfectly well
+ * by the second job. Refusing off the record alone latches that container into refusing local
+ * infra that works, for its whole life, with a stale sentence explaining why. The record is still
+ * what supplies the cause and the daemon's own log tail, which no probe can reconstruct.
  *
- * Only a recorded `false` is re-confirmed. "Not decided" keeps attempting exactly as before: the
- * point of the third value is that nothing turns it into a refusal, and a probe here would.
+ * A recorded PRESENCE is a hypothesis too, and that half was missing. `serving` is not `usable`:
+ * a rootless daemon in a sandbox answers while unable to mount any image layer, so compose ran
+ * and died on a mount error the agent then had to interpret. So the probe is consulted in both
+ * directions, and it runs a real container rather than asking the daemon about itself.
+ *
+ * A check that could not be CARRIED OUT settles nothing, and the cheap fact is what decides
+ * there. Falling straight back to the boot record would re-latch the very refusal the paragraph
+ * above rules out: the four ways the workload check can come back undeterminable (no payload in
+ * this image variant, an architecture it is not built for, a `docker load` the engine refuses, a
+ * timeout) have nothing to do with whether a daemon is up, so a warm container whose sidecar
+ * arrived late would be denied local infra for the rest of its life over a stale sentence. So a
+ * daemon that ANSWERED contradicts a recorded absence exactly as the old `docker version` probe
+ * did, and only a check that never reached a daemon at all leaves the record to decide.
+ *
+ * "Not decided" still keeps attempting, untouched. The point of the third value is that NOTHING
+ * turns it into a refusal: the entrypoint's bounded wait may still be running, and a workload
+ * probe against a daemon that has not finished starting fails for a reason that says nothing
+ * about what it will do a second later.
  */
-export async function resolveDockerVerdict(status, probe = probeDockerServing) {
-    if (status.available !== false)
-        return { available: status.available };
-    if (await probe())
-        return { available: true };
-    return { available: false, refusal: describeDockerAbsence(status) };
+export async function resolveDockerVerdict(status, opts = {}) {
+    if (status.available === undefined)
+        return { available: undefined };
+    const workload = await askTotally(opts.probe ?? probeLiveDockerCapability, opts.signal, opts.logger);
+    if (workload.status === 'usable')
+        return { available: true, daemon: true, workload };
+    if (workload.status === 'unusable') {
+        return {
+            available: false,
+            refusal: describeDockerUnusable(workload),
+            daemon: true,
+            workload,
+        };
+    }
+    if (workload.daemonAnswered)
+        return { available: true, daemon: true, workload };
+    return status.available
+        ? { available: true, workload }
+        : { available: false, refusal: describeDockerAbsence(status), workload };
+}
+/**
+ * Call the probe and answer even if it throws.
+ *
+ * The default probe is total by construction and says so, but this is the seam an injected one
+ * arrives through, and the caller is a stand-up documented as best-effort: a throw here would
+ * fail a job over the mechanism whose whole purpose is to make a failure legible. A throw settles
+ * nothing about the daemon, so it becomes the same value as any other check that could not be
+ * carried out and the boot record decides, exactly as it did before the probe existed.
+ */
+async function askTotally(probe, signal, logger) {
+    try {
+        return await probe(signal);
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        (logger ?? log).warn('docker: the live daemon check threw; falling back to the boot record', {
+            error: message,
+        });
+        return {
+            status: 'unknown',
+            reason: `the live Docker check could not be carried out (${message})`,
+            daemonAnswered: false,
+        };
+    }
 }

package/dist/environment-inventory.d.ts

@@ -1,4 +1,5 @@
 import { type Logger } from './logger.js';
+import { type DockerWorkload } from './docker-capability.js';
 /**
  * What running one probe did, kept as RAW as the spawn: whether the binary ran, and with what.
  * Classification into presence lives in pure code below, because the two callers classify the
@@ -35,6 +36,38 @@ export type ToolPresence = {
     status: 'unknown';
     reason: string;
 };
+/**
+ * What this machine's Docker daemon is good for, which is FIVE answers and not three.
+ *
+ * `absent` and `unknown` mean for the daemon what they mean for any other tool. The three that
+ * are particular to Docker split the case where a daemon ANSWERED, because answering is not the
+ * question anyone is asking:
+ *
+ *   - `usable`:   a container was built and run on it here. `docker build` / `run` / `compose`
+ *                   work, and this is the only state that may say so.
+ *   - `unusable`: a container could NOT be run, with the daemon serving throughout. The state
+ *                   issue #2120 is about; stated as a prohibition, with the cause.
+ *   - `serving`:  it answered, and the workload check could not be carried out (no payload on
+ *                   this machine, an unmapped architecture, a timeout). Neither of the other two,
+ *                   and rendered as "try it if you need it".
+ */
+export type DockerCapability = {
+    status: 'usable';
+    server?: string;
+} | {
+    status: 'unusable';
+    server?: string;
+    detail: string;
+} | {
+    status: 'serving';
+    server?: string;
+    reason: string;
+} | {
+    status: 'absent';
+} | {
+    status: 'unknown';
+    reason: string;
+};
 /** One probed entry: the name the agent would type, and what came back. */
 export interface ProbedTool {
     name: string;
@@ -46,12 +79,13 @@ export interface ProbedTool {
 export interface EnvironmentInventory {
     tools: ProbedTool[];
     /**
-     * Whether a Docker daemon actually answered: the readiness fact, not the CLI's presence. The
-     * image's `entrypoint.sh` starts a rootless daemon BEST-EFFORT and execs the server without
-     * waiting for it, so at job start this probe is the only thing that knows how that went, and
-     * "has not answered yet" is one of its answers (see {@link probeDockerDaemon}).
+     * What the Docker daemon is good for: not the CLI's presence, and not merely whether the
+     * daemon answered. The image's `entrypoint.sh` starts a rootless daemon BEST-EFFORT and execs
+     * the server without waiting for it, so at job start this probe is the only thing that knows
+     * how that went; "has not answered yet" is one of its answers (see {@link probeDockerDaemon})
+     * and "answered, but cannot run a container" is another (see {@link DockerCapability}).
      */
-    dockerDaemon: ToolPresence;
+    dockerDaemon: DockerCapability;
     /**
      * The port the harness's own job server holds in this network namespace. Not probed: the
      * process reads its own {@link harnessListenPort}, which is the only honest answer when a
@@ -124,6 +158,18 @@ export interface ProbeEnvironmentOptions {
      * only so the suite can assert the rendered line without an ambient `PORT` deciding its text.
      */
     harnessPort?: number;
+    /**
+     * Whether the daemon can actually RUN a container, defaulting to the process-wide probe
+     * (docker-capability.ts). Asked only once a daemon has answered, since there is nothing to run
+     * a workload on otherwise, and memoised per container so a warm pool pays for it once.
+     */
+    workload?: (signal?: AbortSignal) => Promise<DockerWorkload>;
+    /**
+     * The job's signal, forwarded to the probes that spawn something. The workload check starts a
+     * CONTAINER, so a cancelled job must stop paying for it rather than hold the daemon for the
+     * rest of its budget.
+     */
+    signal?: AbortSignal;
 }
 /**
  * Probe the machine. EVERYTHING runs concurrently, the daemon included.
@@ -163,7 +209,6 @@ export declare function renderEnvironmentInventory(inventory: EnvironmentInvento
  * the pass itself, and it says so in a log line rather than silently shortening the prompt.
  */
 export declare function appendEnvironmentInventory(systemPrompt: string, opts?: {
-    signal?: AbortSignal;
     log?: Logger;
     run?: ProbeRunner;
 } & ProbeEnvironmentOptions): Promise<string>;