@cat-factory/executor-harness 1.145.1 → 1.147.0

package/src/docker-probe-image.ts

@@ -0,0 +1,171 @@
+import { createHash } from 'node:crypto'
+
+// ---------------------------------------------------------------------------
+// The one-container image the platform runs to find out whether this machine's Docker daemon
+// can actually run a container, built here in memory rather than pulled.
+//
+// It exists because `docker info` answers a different question from the one every caller
+// actually asks. A daemon that ANSWERS is not a daemon that WORKS: this container's rootless
+// daemon runs inside whatever sandbox the deployment gave it, and a nested user namespace
+// routinely refuses the overlay mount every image materialisation needs. The daemon serves
+// happily, `docker version` reports a server, and `docker pull` of a multi-layer image,
+// `docker run` of a single-layer one and `docker build` all fail with the same EINVAL. Issue
+// #2120 is three agents in one run each discovering that for themselves, against a system
+// prompt that told them, as stated fact, that Docker worked here.
+//
+// Why the payload is BUILT and not pulled: a probe that needs the network answers a question
+// about the registry as much as about the daemon, cannot run in a sandbox with no egress, and
+// costs the job its first turn. This one is a single layer holding one statically linked
+// binary already in the image, assembled into a docker-archive tar and handed to `docker load`
+// on stdin, so the whole check is local and takes about as long as starting one container.
+//
+// ONE layer, deliberately. The reported failure kills `docker run` of a single-layer image too
+// (the container's own writable layer is already a second overlay lower dir), so one layer is
+// enough to detect it, and it keeps `docker load` (the step this file could plausibly get WRONG)
+// as small as it can be. That matters because the caller reads a load failure as "could
+// not determine" and a RUN failure as "this daemon cannot run containers": a bug in the archive
+// below must never be able to tell an agent that a working daemon is broken.
+// ---------------------------------------------------------------------------
+
+/** The tag the probe image is loaded under. Removed again once the check has answered. */
+export const PROBE_IMAGE_TAG = 'cat-factory-docker-probe:1'
+
+/** Where the payload binary lands inside the probe image, and what the container is asked to run. */
+const PROBE_BINARY_PATH = 'busybox'
+
+/**
+ * What the container must print for the check to pass.
+ *
+ * A marker on stdout rather than a zero exit status: the point of the check is that a process
+ * inside the container actually ran, and only output it produced proves that. An exit status is
+ * the daemon's word for it.
+ */
+export const PROBE_SENTINEL = 'cat-factory-docker-probe-ok'
+
+/** The argv the probe container runs. `busybox` dispatches on its own name, so this is an echo. */
+export const PROBE_COMMAND: readonly string[] = [`/${PROBE_BINARY_PATH}`, 'echo', PROBE_SENTINEL]
+
+/**
+ * Node's architecture names mapped onto the docker name for the same machine.
+ *
+ * This names THE PAYLOAD, never the daemon. `process.arch` is the architecture of the harness
+ * process, and the binary it hands over is built for that; the daemon it is measured against
+ * answers for itself (`docker version --format {{.Server.Arch}}`), and the caller compares the
+ * two rather than assuming they agree. An external `DOCKER_HOST` is a first-class path here, and
+ * an arm64 harness talking to an amd64 sidecar shares nothing with it but the socket.
+ *
+ * That comparison is also what makes two of these entries safe. `process.arch` reports `ppc64` on
+ * both endiannesses and `arm` with no variant, so those rows are a HYPOTHESIS about the payload,
+ * not a claim: a machine the guess is wrong about answers with a different name and the check
+ * reports that it could not be carried out. Nothing here may produce a verdict about the daemon.
+ * An architecture nothing maps does the same, which is why `386` was worth adding rather than
+ * leaving to a fallback: the mapping is unambiguous and its absence cost a real check.
+ */
+const PAYLOAD_ARCHITECTURES: Readonly<Record<string, string>> = {
+  x64: 'amd64',
+  ia32: '386',
+  arm64: 'arm64',
+  arm: 'arm',
+  s390x: 's390x',
+  ppc64: 'ppc64le',
+  riscv64: 'riscv64',
+}
+
+/** The docker name for the architecture THIS process's payload is built for, when there is one. */
+export function payloadArchitecture(arch: string = process.arch): string | undefined {
+  return PAYLOAD_ARCHITECTURES[arch]
+}
+
+const TAR_BLOCK = 512
+
+/** A fixed timestamp everywhere a tar or an image config wants one, so the archive is byte-stable. */
+const EPOCH = '1970-01-01T00:00:00Z'
+
+interface TarEntry {
+  name: string
+  content: Buffer
+  mode: number
+}
+
+/**
+ * One ustar header block.
+ *
+ * The checksum is summed with its own field read as eight SPACES and only then written back
+ * over it, which is the format's own rule and the one detail a hand-rolled writer gets wrong. A
+ * header whose checksum covers its own checksum bytes is rejected by every reader, and
+ * `docker load` reports that as an unreadable archive, which this module's caller would then
+ * have to decide was not the daemon's fault.
+ */
+function tarHeader(name: string, size: number, mode: number): Buffer {
+  const header = Buffer.alloc(TAR_BLOCK)
+  header.write(name, 0, 100, 'utf8')
+  header.write(octalField(mode, 8), 100, 8, 'latin1')
+  header.write(octalField(0, 8), 108, 8, 'latin1') // uid
+  header.write(octalField(0, 8), 116, 8, 'latin1') // gid
+  header.write(octalField(size, 12), 124, 12, 'latin1')
+  header.write(octalField(0, 12), 136, 12, 'latin1') // mtime
+  header.write('        ', 148, 8, 'latin1')
+  header.write('0', 156, 1, 'latin1') // typeflag: a regular file
+  header.write('ustar\0', 257, 6, 'latin1')
+  header.write('00', 263, 2, 'latin1')
+  let sum = 0
+  for (const byte of header) sum += byte
+  header.write(`${sum.toString(8).padStart(6, '0')}\0 `, 148, 8, 'latin1')
+  return header
+}
+
+/** A numeric tar field: zero-padded octal in `width - 1` characters, then a NUL. */
+function octalField(value: number, width: number): string {
+  return `${value.toString(8).padStart(width - 1, '0')}\0`
+}
+
+/** One tar member: its header, its content, and the padding up to the next 512-byte block. */
+function tarMember(entry: TarEntry): Buffer {
+  const padding = (TAR_BLOCK - (entry.content.length % TAR_BLOCK)) % TAR_BLOCK
+  return Buffer.concat([
+    tarHeader(entry.name, entry.content.length, entry.mode),
+    entry.content,
+    Buffer.alloc(padding),
+  ])
+}
+
+/** A whole tar stream: the members, then the two zero blocks that terminate one. */
+export function tarArchive(entries: readonly TarEntry[]): Buffer {
+  return Buffer.concat([...entries.map(tarMember), Buffer.alloc(TAR_BLOCK * 2)])
+}
+
+/**
+ * Assemble the docker-archive `docker load` reads, from one statically linked binary.
+ *
+ * Classic (v1) docker-archive rather than OCI layout: `docker load` accepts both on every engine
+ * this image can run against, and the v1 shape is three files with no blob directory to get
+ * wrong. The layer digest is the sha256 of the UNCOMPRESSED layer tar, which is what
+ * `rootfs.diff_ids` means; an engine that disagrees with it refuses the load, which the caller
+ * reads as could-not-determine rather than as a broken daemon.
+ *
+ * `architecture` is the DAEMON's own word for its architecture, in docker's vocabulary, so
+ * nothing here decides it (see {@link PAYLOAD_ARCHITECTURES}). The result is byte-stable for one
+ * `(payload, architecture)` pair, which is what lets the caller build it once per container.
+ */
+export function buildProbeArchive(payload: Buffer, architecture: string): Buffer {
+  const layer = tarArchive([{ name: PROBE_BINARY_PATH, content: payload, mode: 0o755 }])
+  const diffId = `sha256:${createHash('sha256').update(layer).digest('hex')}`
+  const config = Buffer.from(
+    JSON.stringify({
+      architecture,
+      os: 'linux',
+      created: EPOCH,
+      config: {},
+      rootfs: { type: 'layers', diff_ids: [diffId] },
+      history: [{ created: EPOCH, created_by: 'cat-factory docker capability probe' }],
+    }),
+  )
+  const manifest = Buffer.from(
+    JSON.stringify([{ Config: 'config.json', RepoTags: [PROBE_IMAGE_TAG], Layers: ['layer.tar'] }]),
+  )
+  return tarArchive([
+    { name: 'config.json', content: config, mode: 0o644 },
+    { name: 'layer.tar', content: layer, mode: 0o644 },
+    { name: 'manifest.json', content: manifest, mode: 0o644 },
+  ])
+}

package/src/docker-status.ts

@@ -1,6 +1,6 @@
-import { execFile } from 'node:child_process'
 import { readFile } from 'node:fs/promises'
-import { promisify } from 'node:util'
+import { probeDockerWorkload, type DockerWorkload } from './docker-capability.js'
+import { log, type Logger } from './logger.js'
 
 // What this container knows about its own Docker daemon, as recorded by `entrypoint.sh`.
 //
@@ -14,8 +14,12 @@ import { promisify } from 'node:util'
 // daemon reads it instead.
 //
 // The recorded verdict describes BOOT, and a container outlives its boot, so nothing refuses on
-// it unconfirmed: `resolveDockerVerdict` re-checks a recorded absence against a live daemon and
-// keeps the record for what only the record holds, the cause and the daemon's own log tail.
+// it unconfirmed: `resolveDockerVerdict` re-checks it against a live daemon and keeps the record
+// for what only the record holds, the cause and the daemon's own log tail. What it records is
+// also only that a SOCKET answered, which is a weaker fact than any caller wants, so the live
+// check RUNS A CONTAINER (docker-capability.ts) rather than settling for the daemon's word about
+// itself. It still reports that weaker fact alongside, since a check that could not be carried
+// out is what the boot record has to be read against, and nothing else establishes it.
 //
 // The three-valued shape is deliberate and is the point (CLAUDE.md, "Degrade loudly"): a daemon
 // that FAILED and a daemon nobody asked about are different facts with different correct
@@ -146,56 +150,149 @@ function unnamedSource(source: never): string {
   return `no Docker daemon answered in this container (unrecognised source ${JSON.stringify(source)})`
 }
 
-/** Whether a daemon is answering RIGHT NOW. Injected so the unit suite can state either answer. */
-export type DockerProbe = () => Promise<boolean>
-
-/** A live probe may not outlast the thing it is guarding; a hung socket is an absent daemon here. */
-const PROBE_TIMEOUT_MS = 10_000
+/**
+ * What a daemon can do RIGHT NOW. Injected so the unit suite can state every answer.
+ *
+ * It answers with a WORKLOAD rather than with a boolean, and that is the correction this type
+ * carries. It used to be `docker version`, which proves the daemon is serving; a stand-up needs
+ * a daemon that can materialise an image, and a sandboxed rootless daemon routinely serves while
+ * being unable to (issue #2120). Running compose against that one produced a mount error the
+ * agent had to interpret, from the one mechanism whose entire job is to say why infra did not
+ * come up.
+ *
+ * The weaker fact did not go away, though: it rides `daemonAnswered` on the `unknown` arm, since
+ * the workload check establishes it on its way past and {@link resolveDockerVerdict} still needs
+ * it. Takes the job's signal, because this is a live check on the critical path of a run that can
+ * be cancelled under it.
+ */
+export type DockerProbe = (signal?: AbortSignal) => Promise<DockerWorkload>
 
-const execFileAsync = promisify(execFile)
+/**
+ * The default {@link DockerProbe}: the process-wide workload probe, which loads a one-layer
+ * image and runs a container from it, memoised per container.
+ *
+ * Named for what it answers. It was `probeDockerServing`, which is the fact this module exists to
+ * say is not enough.
+ */
+export const probeLiveDockerCapability: DockerProbe = (signal) => probeDockerWorkload(signal)
 
 /**
- * The default {@link DockerProbe}: `docker version` talks to the SERVER, unlike the client-only
- * `docker --version`, which answers happily with no daemon at all.
+ * The sentence for a daemon that is serving and cannot run anything. It names what was tried,
+ * because "docker is unavailable" against a daemon the agent can see answering reads as a bug in
+ * the platform rather than as the sandbox limit it is.
  */
-export const probeDockerServing: DockerProbe = async () => {
-  try {
-    await execFileAsync('docker', ['version', '--format', '{{.Server.Version}}'], {
-      timeout: PROBE_TIMEOUT_MS,
-    })
-    return true
-  } catch {
-    return false
-  }
+export function describeDockerUnusable(workload: { detail: string }): string {
+  return `this container's Docker daemon is reachable but cannot run a container (${workload.detail})`
 }
 
 /** What a stand-up is entitled to conclude about the daemon at the moment it is about to run. */
 export interface DockerVerdict {
-  /** Three-valued exactly as {@link DockerStatus.available}, and read the same way. */
+  /**
+   * Whether a stand-up may PROCEED, three-valued exactly as {@link DockerStatus.available} and
+   * read the same way. It is the decision, not a description of the daemon: a daemon that is
+   * answering and cannot run a container is `false` here and `daemon: true` below, and a record
+   * that reported the first as the second would send an operator to restart a daemon that is
+   * already up.
+   */
   available: boolean | undefined
-  /** Set only for a CONFIRMED absence: the sentence to refuse with. Absent means proceed. */
+  /**
+   * Set only for a CONFIRMED negative: the sentence to refuse with. Absent means proceed.
+   *
+   * Two causes reach it and they read differently on purpose. Nothing is answering here, and
+   * something is answering here but cannot run a container: an operator sent to restart a daemon
+   * that is already up would find nothing wrong with it.
+   */
   refusal?: string
+  /**
+   * Whether a daemon ANSWERED the live check. Absent when nothing was checked (an undecided
+   * record probes nothing) or when nothing answered, so it is never read as a decided `false`.
+   */
+  daemon?: boolean
+  /** What a real container did on it, when one was tried. Absent when nothing was measured. */
+  workload?: DockerWorkload
 }
 
 /**
- * Resolve what to do now, from what boot recorded plus what a daemon says today.
+ * Resolve what to do now, from what boot recorded plus what the daemon can do today.
+ *
+ * `entrypoint.sh` probes once, at boot, within a bounded wait, and it probes for a SOCKET. Two
+ * things follow, and the branches below are one each.
  *
- * `entrypoint.sh` probes once, at boot, within a bounded wait. A container outlives that: a warm
- * pool serves many jobs from one, and a sidecar daemon that took longer than the wait allows is
- * serving perfectly well by the second job. Refusing off the recorded verdict alone latches that
- * container into refusing local infra that in fact works, for its whole life, with a stale
- * sentence explaining why. So a recorded absence is a HYPOTHESIS here, and the live probe settles
- * it; the recorded verdict is still what supplies the cause and the daemon's own log tail, which
- * no probe can reconstruct.
+ * A recorded absence is a HYPOTHESIS. A container outlives its boot: a warm pool serves many jobs
+ * from one, and a sidecar daemon that took longer than the wait allows is serving perfectly well
+ * by the second job. Refusing off the record alone latches that container into refusing local
+ * infra that works, for its whole life, with a stale sentence explaining why. The record is still
+ * what supplies the cause and the daemon's own log tail, which no probe can reconstruct.
  *
- * Only a recorded `false` is re-confirmed. "Not decided" keeps attempting exactly as before: the
- * point of the third value is that nothing turns it into a refusal, and a probe here would.
+ * A recorded PRESENCE is a hypothesis too, and that half was missing. `serving` is not `usable`:
+ * a rootless daemon in a sandbox answers while unable to mount any image layer, so compose ran
+ * and died on a mount error the agent then had to interpret. So the probe is consulted in both
+ * directions, and it runs a real container rather than asking the daemon about itself.
+ *
+ * A check that could not be CARRIED OUT settles nothing, and the cheap fact is what decides
+ * there. Falling straight back to the boot record would re-latch the very refusal the paragraph
+ * above rules out: the four ways the workload check can come back undeterminable (no payload in
+ * this image variant, an architecture it is not built for, a `docker load` the engine refuses, a
+ * timeout) have nothing to do with whether a daemon is up, so a warm container whose sidecar
+ * arrived late would be denied local infra for the rest of its life over a stale sentence. So a
+ * daemon that ANSWERED contradicts a recorded absence exactly as the old `docker version` probe
+ * did, and only a check that never reached a daemon at all leaves the record to decide.
+ *
+ * "Not decided" still keeps attempting, untouched. The point of the third value is that NOTHING
+ * turns it into a refusal: the entrypoint's bounded wait may still be running, and a workload
+ * probe against a daemon that has not finished starting fails for a reason that says nothing
+ * about what it will do a second later.
  */
 export async function resolveDockerVerdict(
   status: DockerStatus,
-  probe: DockerProbe = probeDockerServing,
+  opts: { probe?: DockerProbe; signal?: AbortSignal; logger?: Logger } = {},
 ): Promise<DockerVerdict> {
-  if (status.available !== false) return { available: status.available }
-  if (await probe()) return { available: true }
-  return { available: false, refusal: describeDockerAbsence(status) }
+  if (status.available === undefined) return { available: undefined }
+  const workload = await askTotally(
+    opts.probe ?? probeLiveDockerCapability,
+    opts.signal,
+    opts.logger,
+  )
+  if (workload.status === 'usable') return { available: true, daemon: true, workload }
+  if (workload.status === 'unusable') {
+    return {
+      available: false,
+      refusal: describeDockerUnusable(workload),
+      daemon: true,
+      workload,
+    }
+  }
+  if (workload.daemonAnswered) return { available: true, daemon: true, workload }
+  return status.available
+    ? { available: true, workload }
+    : { available: false, refusal: describeDockerAbsence(status), workload }
+}
+
+/**
+ * Call the probe and answer even if it throws.
+ *
+ * The default probe is total by construction and says so, but this is the seam an injected one
+ * arrives through, and the caller is a stand-up documented as best-effort: a throw here would
+ * fail a job over the mechanism whose whole purpose is to make a failure legible. A throw settles
+ * nothing about the daemon, so it becomes the same value as any other check that could not be
+ * carried out and the boot record decides, exactly as it did before the probe existed.
+ */
+async function askTotally(
+  probe: DockerProbe,
+  signal: AbortSignal | undefined,
+  logger: Logger | undefined,
+): Promise<DockerWorkload> {
+  try {
+    return await probe(signal)
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err)
+    ;(logger ?? log).warn('docker: the live daemon check threw; falling back to the boot record', {
+      error: message,
+    })
+    return {
+      status: 'unknown',
+      reason: `the live Docker check could not be carried out (${message})`,
+      daemonAnswered: false,
+    }
+  }
 }

package/src/environment-inventory.ts

@@ -2,6 +2,7 @@ import { execFile } from 'node:child_process'
 import { promisify } from 'node:util'
 import { log, type Logger } from './logger.js'
 import { harnessListenPort } from './harness-port.js'
+import { probeDockerWorkload, type DockerWorkload } from './docker-capability.js'
 
 // ---------------------------------------------------------------------------
 // What this machine actually has, probed ONCE per job and stated to the agent.
@@ -32,6 +33,12 @@ import { harnessListenPort } from './harness-port.js'
 //     a daemon this machine is CONFIGURED for but which has not answered yet is a fourth state
 //     that resolves to `unknown`, never to the absence a refused connection looks like: the
 //     image's daemon is started in the background and the job begins before it is ready.
+//   - A daemon that ANSWERS is still not a daemon that WORKS, which is the same mistake one level
+//     in. A rootless daemon nested in a sandbox serves while its snapshotter cannot mount any
+//     image layer, so `docker info` succeeds and `docker build` / `docker run` / `docker pull`
+//     all fail (issue #2120). Only a container that RAN settles that, so the reachable case is
+//     split by a real workload (docker-capability.ts) into `usable`, `unusable`, and a daemon
+//     that answered while the check itself could not be carried out.
 //
 // Deliberately NOT here: the agent's own tools (web search, file tools, MCP servers). Those are
 // the CLI's, they differ per harness, and each is already stated where it is true. Claiming one
@@ -68,6 +75,28 @@ export type ToolPresence =
   | { status: 'absent' }
   | { status: 'unknown'; reason: string }
 
+/**
+ * What this machine's Docker daemon is good for, which is FIVE answers and not three.
+ *
+ * `absent` and `unknown` mean for the daemon what they mean for any other tool. The three that
+ * are particular to Docker split the case where a daemon ANSWERED, because answering is not the
+ * question anyone is asking:
+ *
+ *   - `usable`:   a container was built and run on it here. `docker build` / `run` / `compose`
+ *                   work, and this is the only state that may say so.
+ *   - `unusable`: a container could NOT be run, with the daemon serving throughout. The state
+ *                   issue #2120 is about; stated as a prohibition, with the cause.
+ *   - `serving`:  it answered, and the workload check could not be carried out (no payload on
+ *                   this machine, an unmapped architecture, a timeout). Neither of the other two,
+ *                   and rendered as "try it if you need it".
+ */
+export type DockerCapability =
+  | { status: 'usable'; server?: string }
+  | { status: 'unusable'; server?: string; detail: string }
+  | { status: 'serving'; server?: string; reason: string }
+  | { status: 'absent' }
+  | { status: 'unknown'; reason: string }
+
 /** One probed entry: the name the agent would type, and what came back. */
 export interface ProbedTool {
   name: string
@@ -80,12 +109,13 @@ export interface ProbedTool {
 export interface EnvironmentInventory {
   tools: ProbedTool[]
   /**
-   * Whether a Docker daemon actually answered: the readiness fact, not the CLI's presence. The
-   * image's `entrypoint.sh` starts a rootless daemon BEST-EFFORT and execs the server without
-   * waiting for it, so at job start this probe is the only thing that knows how that went, and
-   * "has not answered yet" is one of its answers (see {@link probeDockerDaemon}).
+   * What the Docker daemon is good for: not the CLI's presence, and not merely whether the
+   * daemon answered. The image's `entrypoint.sh` starts a rootless daemon BEST-EFFORT and execs
+   * the server without waiting for it, so at job start this probe is the only thing that knows
+   * how that went; "has not answered yet" is one of its answers (see {@link probeDockerDaemon})
+   * and "answered, but cannot run a container" is another (see {@link DockerCapability}).
    */
-  dockerDaemon: ToolPresence
+  dockerDaemon: DockerCapability
   /**
    * The port the harness's own job server holds in this network namespace. Not probed: the
    * process reads its own {@link harnessListenPort}, which is the only honest answer when a
@@ -310,6 +340,18 @@ export interface ProbeEnvironmentOptions {
    * only so the suite can assert the rendered line without an ambient `PORT` deciding its text.
    */
   harnessPort?: number
+  /**
+   * Whether the daemon can actually RUN a container, defaulting to the process-wide probe
+   * (docker-capability.ts). Asked only once a daemon has answered, since there is nothing to run
+   * a workload on otherwise, and memoised per container so a warm pool pays for it once.
+   */
+  workload?: (signal?: AbortSignal) => Promise<DockerWorkload>
+  /**
+   * The job's signal, forwarded to the probes that spawn something. The workload check starts a
+   * CONTAINER, so a cancelled job must stop paying for it rather than hold the daemon for the
+   * rest of its budget.
+   */
+  signal?: AbortSignal
 }
 
 /**
@@ -346,11 +388,35 @@ export async function probeEnvironment(
         presence: toolPresence(await run(probe.command, probe.args)),
       })),
     ),
-    probeDockerDaemon(run, opts),
+    probeDockerCapability(run, opts),
   ])
   return { tools, dockerDaemon, harnessPort: opts.harnessPort ?? harnessListenPort() }
 }
 
+/**
+ * The daemon's full answer: whether one is reachable, and then whether it can run a container.
+ *
+ * The two steps are kept apart because they fail for unrelated reasons and only the FIRST has a
+ * cheap answer. A daemon nobody can reach has no workload to run, so the check that costs a
+ * container start is asked only where there is something to ask it of; a daemon that answered
+ * carries its server version into every one of the three states that follow it, because the
+ * agent reading the line is entitled to know which daemon the verdict is about.
+ */
+async function probeDockerCapability(
+  run: ProbeRunner,
+  opts: ProbeEnvironmentOptions,
+): Promise<DockerCapability> {
+  const daemon = await probeDockerDaemon(run, opts)
+  if (daemon.status === 'absent') return { status: 'absent' }
+  if (daemon.status === 'unknown') return { status: 'unknown', reason: daemon.reason }
+  const server = daemon.version ? { server: daemon.version } : {}
+  const workload = await (opts.workload ?? probeDockerWorkload)(opts.signal)
+  if (workload.status === 'usable') return { status: 'usable', ...server }
+  if (workload.status === 'unusable')
+    return { status: 'unusable', ...server, detail: workload.detail }
+  return { status: 'serving', ...server, reason: workload.reason }
+}
+
 /**
  * Ask the daemon itself, and do not mistake a daemon that is STARTING for one that is not there.
  *
@@ -482,26 +548,57 @@ function harnessPortLine(port: number): string {
   )
 }
 
-/** The Docker line, which says something different in each of the three cases. */
-function dockerDaemonLine(daemon: ToolPresence): string {
-  if (daemon.status === 'present') {
-    const server = daemon.version ? ` (server ${daemon.version})` : ''
-    return (
-      `A Docker daemon is reachable${server}: \`docker build\`, \`docker run\` and ` +
-      '`docker compose up` work here.'
-    )
-  }
-  if (daemon.status === 'unknown') {
-    return (
-      'Whether a Docker daemon is reachable could not be determined ' +
-      `(${daemon.reason}): try it if you need it, and do not read a failure as a defect in the work.`
-    )
+/**
+ * The Docker line, which says something different in each of the five cases, and is TOTAL over
+ * them: adding a state without deciding what an agent should do about it stops the build.
+ *
+ * Only `usable` may claim the commands work, and it may only be reached by having RUN one. The
+ * line that used to stand here made that claim off `docker info` alone, which is how every agent
+ * in a run was told, as fact, that a daemon which could not mount a single image layer would
+ * build and run one.
+ */
+function dockerDaemonLine(daemon: DockerCapability): string {
+  const server = 'server' in daemon && daemon.server ? ` (server ${daemon.server})` : ''
+  switch (daemon.status) {
+    case 'usable':
+      return (
+        `A Docker daemon is reachable${server} and the platform ran a container on it: ` +
+        '`docker build`, `docker run` and `docker compose up` work here.'
+      )
+    case 'unusable':
+      return (
+        `A Docker daemon is reachable${server} but it CANNOT run a container: the platform ` +
+        `built a one-layer image and tried to run it here, and that failed (${daemon.detail}). ` +
+        '`docker build`, `docker run`, `docker pull` of a multi-layer image and ' +
+        '`docker compose up` all fail for the same reason, so there is nothing to retry and no ' +
+        'flag that works around it. Produce the Dockerfile or compose file you were asked for, ' +
+        'say in one line that it could not be built or run here, and move on.'
+      )
+    case 'serving':
+      return (
+        `A Docker daemon is reachable${server}, but whether it can actually build or run an ` +
+        `image was NOT established (${daemon.reason}). Reaching the daemon is not the same fact: ` +
+        'a sandboxed one answers while being unable to mount any image layer. Try it if you need ' +
+        'it, and do not read a failure as a defect in the work.'
+      )
+    case 'unknown':
+      return (
+        'Whether a Docker daemon is reachable could not be determined ' +
+        `(${daemon.reason}): try it if you need it, and do not read a failure as a defect in the work.`
+      )
+    case 'absent':
+      return (
+        'NO Docker daemon is reachable: `docker build`, `docker run` and `docker compose up` ' +
+        'will fail here whatever the CLI reports. Produce the Dockerfile or compose file you ' +
+        'were asked for, say in one line that you could not build it here, and move on.'
+      )
+    default:
+      return unnamedCapability(daemon)
   }
-  return (
-    'NO Docker daemon is reachable: `docker build`, `docker run` and `docker compose up` will ' +
-    'fail here whatever the CLI reports. Produce the Dockerfile or compose file you were asked ' +
-    'for, say in one line that you could not build it here, and move on.'
-  )
+}
+
+function unnamedCapability(daemon: never): string {
+  return `Whether a Docker daemon is reachable could not be determined (the platform reported an unrecognised verdict ${JSON.stringify(daemon)}): try it if you need it.`
 }
 
 /**
@@ -520,15 +617,16 @@ function dockerDaemonLine(daemon: ToolPresence): string {
  */
 export async function appendEnvironmentInventory(
   systemPrompt: string,
-  opts: { signal?: AbortSignal; log?: Logger; run?: ProbeRunner } & ProbeEnvironmentOptions = {},
+  opts: { log?: Logger; run?: ProbeRunner } & ProbeEnvironmentOptions = {},
 ): Promise<string> {
   const logger = opts.log ?? log
+  // Everything that is not this function's OWN is forwarded by construction, rather than key by
+  // key. The list of copied keys silently dropped `workload`, whose whole point is that a suite
+  // can inject one: a test driving THIS entry point (the only one `handleAgent` uses) got the
+  // real probe instead, which starts a container on whatever machine the suite runs on.
+  const { log: _log, run, ...probeOptions } = opts
   try {
-    const inventory = await probeEnvironment(opts.run ?? spawnProbeRunner(opts.signal), {
-      ...(opts.sleep ? { sleep: opts.sleep } : {}),
-      ...(opts.daemonExpected === undefined ? {} : { daemonExpected: opts.daemonExpected }),
-      ...(opts.harnessPort === undefined ? {} : { harnessPort: opts.harnessPort }),
-    })
+    const inventory = await probeEnvironment(run ?? spawnProbeRunner(opts.signal), probeOptions)
     logger.info('agent: probed the environment', {
       installed: inventory.tools
         .filter((t) => t.presence.status === 'present')

package/src/harness-server.ts

@@ -6,6 +6,7 @@ import { handleAgent } from './agent.js'
 import { handleInline } from './inline.js'
 import { redactSecrets } from './git.js'
 import { readDockerStatus } from './docker-status.js'
+import { reportedDockerWorkload } from './docker-capability.js'
 import { harnessListenPort } from './harness-port.js'
 import { JobRegistry, loadRunnerLimits, type JobResultBase, type RunOptions } from './runner.js'
 import { log } from './logger.js'
@@ -141,11 +142,16 @@ const server = createServer((req, res) => {
       // would spawn a process per poll to answer a question this endpoint is not the one to act
       // on; the stand-up re-confirms a recorded absence at the moment it matters
       // (`resolveDockerVerdict`), so a stale negative here never becomes a stale refusal there.
+      //
+      // `workload` is the other half, and the reason the block used to mislead: what the record
+      // says is `serving`, and serving is not usable. It reports the last measurement any job
+      // took (docker-capability.ts) and NEVER takes one itself, for the same polling reason,
+      // which is why `unmeasured` is one of the words it can answer.
       return send(res, 200, {
         status: 'ok',
         ...(HARNESS_VERSION ? { version: HARNESS_VERSION } : {}),
         capabilities: HARNESS_BODY_CAPABILITIES,
-        docker: await readDockerStatus(),
+        docker: { ...(await readDockerStatus()), workload: reportedDockerWorkload() },
       })
     }
     // All non-health endpoints are gated by the optional shared secret.