@cat-factory/executor-harness 1.145.1 → 1.147.0

package/dist/environment-inventory.js

@@ -2,6 +2,7 @@ import { execFile } from 'node:child_process';
 import { promisify } from 'node:util';
 import { log } from './logger.js';
 import { harnessListenPort } from './harness-port.js';
+import { probeDockerWorkload } from './docker-capability.js';
 // ---------------------------------------------------------------------------
 // What this machine actually has, probed ONCE per job and stated to the agent.
 //
@@ -31,6 +32,12 @@ import { harnessListenPort } from './harness-port.js';
 //     a daemon this machine is CONFIGURED for but which has not answered yet is a fourth state
 //     that resolves to `unknown`, never to the absence a refused connection looks like: the
 //     image's daemon is started in the background and the job begins before it is ready.
+//   - A daemon that ANSWERS is still not a daemon that WORKS, which is the same mistake one level
+//     in. A rootless daemon nested in a sandbox serves while its snapshotter cannot mount any
+//     image layer, so `docker info` succeeds and `docker build` / `docker run` / `docker pull`
+//     all fail (issue #2120). Only a container that RAN settles that, so the reachable case is
+//     split by a real workload (docker-capability.ts) into `usable`, `unusable`, and a daemon
+//     that answered while the check itself could not be carried out.
 //
 // Deliberately NOT here: the agent's own tools (web search, file tools, MCP servers). Those are
 // the CLI's, they differ per harness, and each is already stated where it is true. Claiming one
@@ -260,10 +267,33 @@ export async function probeEnvironment(run, opts = {}) {
             showVersion: probe.showVersion,
             presence: toolPresence(await run(probe.command, probe.args)),
         }))),
-        probeDockerDaemon(run, opts),
+        probeDockerCapability(run, opts),
     ]);
     return { tools, dockerDaemon, harnessPort: opts.harnessPort ?? harnessListenPort() };
 }
+/**
+ * The daemon's full answer: whether one is reachable, and then whether it can run a container.
+ *
+ * The two steps are kept apart because they fail for unrelated reasons and only the FIRST has a
+ * cheap answer. A daemon nobody can reach has no workload to run, so the check that costs a
+ * container start is asked only where there is something to ask it of; a daemon that answered
+ * carries its server version into every one of the three states that follow it, because the
+ * agent reading the line is entitled to know which daemon the verdict is about.
+ */
+async function probeDockerCapability(run, opts) {
+    const daemon = await probeDockerDaemon(run, opts);
+    if (daemon.status === 'absent')
+        return { status: 'absent' };
+    if (daemon.status === 'unknown')
+        return { status: 'unknown', reason: daemon.reason };
+    const server = daemon.version ? { server: daemon.version } : {};
+    const workload = await (opts.workload ?? probeDockerWorkload)(opts.signal);
+    if (workload.status === 'usable')
+        return { status: 'usable', ...server };
+    if (workload.status === 'unusable')
+        return { status: 'unusable', ...server, detail: workload.detail };
+    return { status: 'serving', ...server, reason: workload.reason };
+}
 /**
  * Ask the daemon itself, and do not mistake a daemon that is STARTING for one that is not there.
  *
@@ -377,20 +407,46 @@ function harnessPortLine(port) {
         'check aimed at it passes without your service ever having run. Bind anything you start ' +
         'somewhere else.');
 }
-/** The Docker line, which says something different in each of the three cases. */
+/**
+ * The Docker line, which says something different in each of the five cases, and is TOTAL over
+ * them: adding a state without deciding what an agent should do about it stops the build.
+ *
+ * Only `usable` may claim the commands work, and it may only be reached by having RUN one. The
+ * line that used to stand here made that claim off `docker info` alone, which is how every agent
+ * in a run was told, as fact, that a daemon which could not mount a single image layer would
+ * build and run one.
+ */
 function dockerDaemonLine(daemon) {
-    if (daemon.status === 'present') {
-        const server = daemon.version ? ` (server ${daemon.version})` : '';
-        return (`A Docker daemon is reachable${server}: \`docker build\`, \`docker run\` and ` +
-            '`docker compose up` work here.');
-    }
-    if (daemon.status === 'unknown') {
-        return ('Whether a Docker daemon is reachable could not be determined ' +
-            `(${daemon.reason}): try it if you need it, and do not read a failure as a defect in the work.`);
+    const server = 'server' in daemon && daemon.server ? ` (server ${daemon.server})` : '';
+    switch (daemon.status) {
+        case 'usable':
+            return (`A Docker daemon is reachable${server} and the platform ran a container on it: ` +
+                '`docker build`, `docker run` and `docker compose up` work here.');
+        case 'unusable':
+            return (`A Docker daemon is reachable${server} but it CANNOT run a container: the platform ` +
+                `built a one-layer image and tried to run it here, and that failed (${daemon.detail}). ` +
+                '`docker build`, `docker run`, `docker pull` of a multi-layer image and ' +
+                '`docker compose up` all fail for the same reason, so there is nothing to retry and no ' +
+                'flag that works around it. Produce the Dockerfile or compose file you were asked for, ' +
+                'say in one line that it could not be built or run here, and move on.');
+        case 'serving':
+            return (`A Docker daemon is reachable${server}, but whether it can actually build or run an ` +
+                `image was NOT established (${daemon.reason}). Reaching the daemon is not the same fact: ` +
+                'a sandboxed one answers while being unable to mount any image layer. Try it if you need ' +
+                'it, and do not read a failure as a defect in the work.');
+        case 'unknown':
+            return ('Whether a Docker daemon is reachable could not be determined ' +
+                `(${daemon.reason}): try it if you need it, and do not read a failure as a defect in the work.`);
+        case 'absent':
+            return ('NO Docker daemon is reachable: `docker build`, `docker run` and `docker compose up` ' +
+                'will fail here whatever the CLI reports. Produce the Dockerfile or compose file you ' +
+                'were asked for, say in one line that you could not build it here, and move on.');
+        default:
+            return unnamedCapability(daemon);
     }
-    return ('NO Docker daemon is reachable: `docker build`, `docker run` and `docker compose up` will ' +
-        'fail here whatever the CLI reports. Produce the Dockerfile or compose file you were asked ' +
-        'for, say in one line that you could not build it here, and move on.');
+}
+function unnamedCapability(daemon) {
+    return `Whether a Docker daemon is reachable could not be determined (the platform reported an unrecognised verdict ${JSON.stringify(daemon)}): try it if you need it.`;
 }
 /**
  * Probe the machine and fold the inventory onto `systemPrompt`. THE composition point: the harness
@@ -408,12 +464,13 @@ function dockerDaemonLine(daemon) {
  */
 export async function appendEnvironmentInventory(systemPrompt, opts = {}) {
     const logger = opts.log ?? log;
+    // Everything that is not this function's OWN is forwarded by construction, rather than key by
+    // key. The list of copied keys silently dropped `workload`, whose whole point is that a suite
+    // can inject one: a test driving THIS entry point (the only one `handleAgent` uses) got the
+    // real probe instead, which starts a container on whatever machine the suite runs on.
+    const { log: _log, run, ...probeOptions } = opts;
     try {
-        const inventory = await probeEnvironment(opts.run ?? spawnProbeRunner(opts.signal), {
-            ...(opts.sleep ? { sleep: opts.sleep } : {}),
-            ...(opts.daemonExpected === undefined ? {} : { daemonExpected: opts.daemonExpected }),
-            ...(opts.harnessPort === undefined ? {} : { harnessPort: opts.harnessPort }),
-        });
+        const inventory = await probeEnvironment(run ?? spawnProbeRunner(opts.signal), probeOptions);
         logger.info('agent: probed the environment', {
             installed: inventory.tools
                 .filter((t) => t.presence.status === 'present')

package/dist/harness-server.js

@@ -6,6 +6,7 @@ import { handleAgent } from './agent.js';
 import { handleInline } from './inline.js';
 import { redactSecrets } from './git.js';
 import { readDockerStatus } from './docker-status.js';
+import { reportedDockerWorkload } from './docker-capability.js';
 import { harnessListenPort } from './harness-port.js';
 import { JobRegistry, loadRunnerLimits } from './runner.js';
 import { log } from './logger.js';
@@ -122,11 +123,16 @@ const server = createServer((req, res) => {
             // would spawn a process per poll to answer a question this endpoint is not the one to act
             // on; the stand-up re-confirms a recorded absence at the moment it matters
             // (`resolveDockerVerdict`), so a stale negative here never becomes a stale refusal there.
+            //
+            // `workload` is the other half, and the reason the block used to mislead: what the record
+            // says is `serving`, and serving is not usable. It reports the last measurement any job
+            // took (docker-capability.ts) and NEVER takes one itself, for the same polling reason,
+            // which is why `unmeasured` is one of the words it can answer.
             return send(res, 200, {
                 status: 'ok',
                 ...(HARNESS_VERSION ? { version: HARNESS_VERSION } : {}),
                 capabilities: HARNESS_BODY_CAPABILITIES,
-                docker: await readDockerStatus(),
+                docker: { ...(await readDockerStatus()), workload: reportedDockerWorkload() },
             });
         }
         // All non-health endpoints are gated by the optional shared secret.

package/dist/infra-standup.d.ts

@@ -9,19 +9,22 @@ import type { Logger } from './logger.js';
  * still run unit-level tests and report what it could. A no-op for ephemeral / no-infra /
  * no-compose-path runs.
  *
- * A CONFIRMED absence of a Docker daemon short-circuits it: the container's own probe
- * ({@link readDockerStatus}, recorded by `entrypoint.sh`) already knows there is nothing to
- * talk to, so running compose against it would only turn a fact this container holds into a
- * connection error the agent has to interpret. The record then carries `dockerAvailable: false`
- * and the stated cause, which is what makes the Tester step say why it ran no infra instead of
- * looking like a Tester that simply chose not to. Anything OTHER than a confirmed absence
- * attempts as before (`DockerStatus.available` in docker-status.ts states why "undecided" is its
- * own value).
+ * A CONFIRMED absence of a USABLE Docker daemon short-circuits it: the container already knows
+ * compose cannot work, so running it would only turn a fact this container holds into an error
+ * the agent has to interpret. The record then carries the stated cause plus the two facts that
+ * decide where a human should look (`dockerAvailable`: was anything answering, `dockerWorkload`:
+ * what a container did on it), which is what makes the Tester step say why it ran no infra
+ * instead of looking like a Tester that simply chose not to. Anything OTHER than a confirmed
+ * negative attempts as before (`DockerStatus.available` in docker-status.ts states why
+ * "undecided" is its own value).
  *
- * "Confirmed", not merely recorded: {@link resolveDockerVerdict} re-checks a recorded absence
- * against a live daemon first, so a warm-pool container whose sidecar came up late is not
- * latched into refusing infra that works. `probe` is that check, injected so the unit suite can
- * state both answers on a machine that has its own daemon either way.
+ * "Confirmed", not merely recorded: {@link resolveDockerVerdict} re-checks the boot record
+ * against a live daemon first, so a warm-pool container whose sidecar came up late is not latched
+ * into refusing infra that works. It re-checks a recorded PRESENCE too, by running an actual
+ * container: a rootless daemon in a sandbox answers `docker version` while being unable to mount
+ * an image, and compose against that one died on a mount error inside the very mechanism that
+ * exists to explain why infra did not come up. `probe` is that check, injected so the unit suite
+ * can state every answer on a machine that has its own daemon either way.
  *
  * Whether it succeeds or fails, the (redacted, bounded) command output is captured into a
  * {@link InfraSetupRecord} returned alongside the prompt `note`, so the backend can surface

package/dist/infra-standup.js

@@ -4,7 +4,7 @@
 // guarantees the matching teardown. `manageInfra` is the one entry point a mode calls.
 import { execFile } from 'node:child_process';
 import { promisify } from 'node:util';
-import { probeDockerServing, readDockerStatus, resolveDockerVerdict, } from './docker-status.js';
+import { probeLiveDockerCapability, readDockerStatus, resolveDockerVerdict, } from './docker-status.js';
 import { standUpFrontend, tearDownFrontend } from './frontend-infra.js';
 import { captureRedactedOutput, redactSecrets } from './redact.js';
 const exec = promisify(execFile);
@@ -15,19 +15,22 @@ const exec = promisify(execFile);
  * still run unit-level tests and report what it could. A no-op for ephemeral / no-infra /
  * no-compose-path runs.
  *
- * A CONFIRMED absence of a Docker daemon short-circuits it: the container's own probe
- * ({@link readDockerStatus}, recorded by `entrypoint.sh`) already knows there is nothing to
- * talk to, so running compose against it would only turn a fact this container holds into a
- * connection error the agent has to interpret. The record then carries `dockerAvailable: false`
- * and the stated cause, which is what makes the Tester step say why it ran no infra instead of
- * looking like a Tester that simply chose not to. Anything OTHER than a confirmed absence
- * attempts as before (`DockerStatus.available` in docker-status.ts states why "undecided" is its
- * own value).
+ * A CONFIRMED absence of a USABLE Docker daemon short-circuits it: the container already knows
+ * compose cannot work, so running it would only turn a fact this container holds into an error
+ * the agent has to interpret. The record then carries the stated cause plus the two facts that
+ * decide where a human should look (`dockerAvailable`: was anything answering, `dockerWorkload`:
+ * what a container did on it), which is what makes the Tester step say why it ran no infra
+ * instead of looking like a Tester that simply chose not to. Anything OTHER than a confirmed
+ * negative attempts as before (`DockerStatus.available` in docker-status.ts states why
+ * "undecided" is its own value).
  *
- * "Confirmed", not merely recorded: {@link resolveDockerVerdict} re-checks a recorded absence
- * against a live daemon first, so a warm-pool container whose sidecar came up late is not
- * latched into refusing infra that works. `probe` is that check, injected so the unit suite can
- * state both answers on a machine that has its own daemon either way.
+ * "Confirmed", not merely recorded: {@link resolveDockerVerdict} re-checks the boot record
+ * against a live daemon first, so a warm-pool container whose sidecar came up late is not latched
+ * into refusing infra that works. It re-checks a recorded PRESENCE too, by running an actual
+ * container: a rootless daemon in a sandbox answers `docker version` while being unable to mount
+ * an image, and compose against that one died on a mount error inside the very mechanism that
+ * exists to explain why infra did not come up. `probe` is that check, injected so the unit suite
+ * can state every answer on a machine that has its own daemon either way.
  *
  * Whether it succeeds or fails, the (redacted, bounded) command output is captured into a
  * {@link InfraSetupRecord} returned alongside the prompt `note`, so the backend can surface
@@ -38,26 +41,42 @@ const exec = promisify(execFile);
  * this container makes about itself, and the acceptance suite can only exercise it on a machine
  * where the daemon genuinely fails.
  */
-export async function standUpInfra(dir, infra, signal, logger, probe = probeDockerServing) {
+export async function standUpInfra(dir, infra, signal, logger, probe = probeLiveDockerCapability) {
     if (infra.environment !== 'local' || infra.noInfraDependencies || !infra.composePath) {
         return { started: false };
     }
     const startedAt = Date.now();
     const recorded = await readDockerStatus();
-    const docker = await resolveDockerVerdict(recorded, probe);
+    const docker = await resolveDockerVerdict(recorded, {
+        probe,
+        ...(signal ? { signal } : {}),
+        logger,
+    });
     if (docker.refusal) {
         const note = `the dependencies could not be started: ${docker.refusal}`;
-        logger.warn('agent(explore): infra stand-up refused, no docker daemon', {
+        logger.warn('agent(explore): infra stand-up refused, no usable docker daemon', {
             composePath: infra.composePath,
             dockerSource: recorded.source,
             dockerReason: recorded.reason,
+            // What the LIVE check found, which is the only place the second refusal cause exists: the
+            // boot record's own words for a daemon that answers and cannot run anything are `serving`
+            // and nothing else, so a log line carrying the record alone describes the wrong failure.
+            dockerWorkload: docker.workload?.status ?? 'unmeasured',
+            ...(docker.workload?.status === 'unusable' ? { dockerDetail: docker.workload.detail } : {}),
         });
         return {
             started: false,
             note,
             record: {
                 started: false,
-                dockerAvailable: false,
+                // NOT a flat `false`. Two refusals reach this branch and they have opposite fixes: with
+                // nothing answering, the executor image or the sandbox running it is what to go and look
+                // at; with a daemon that answers and cannot run a container, that daemon is up and an
+                // operator sent to restart it finds nothing wrong. `dockerAvailable` answers only the
+                // first question and `dockerWorkload` the second, so neither has to carry the other's
+                // fact (the same rule the compose-failure branch below states for its own `false`).
+                dockerAvailable: docker.daemon === true,
+                ...workloadRecord(docker.workload),
                 composePath: infra.composePath,
                 at: Date.now(),
                 durationMs: Date.now() - startedAt,
@@ -76,6 +95,7 @@ export async function standUpInfra(dir, infra, signal, logger, probe = probeDock
             record: {
                 started: true,
                 dockerAvailable: true,
+                ...workloadRecord(docker.workload),
                 composePath: infra.composePath,
                 at: Date.now(),
                 durationMs: Date.now() - startedAt,
@@ -96,12 +116,13 @@ export async function standUpInfra(dir, infra, signal, logger, probe = probeDock
             note,
             record: {
                 started: false,
-                // A compose failure with a REACHABLE daemon: the two `false`s above and here are
-                // different diagnoses (nothing to talk to vs the stack itself did not come up), and
-                // only stating both keeps the second from being read as the first. Read off the
-                // RESOLVED verdict, so a container whose daemon came up after boot claims the daemon it
-                // actually reached rather than the one its boot record still denies.
-                ...(docker.available === true ? { dockerAvailable: true } : {}),
+                // A compose failure with a REACHABLE daemon, which is a third diagnosis again: the stack
+                // itself did not come up. Read off the RESOLVED verdict, so a container whose daemon came
+                // up after boot claims the daemon it actually reached rather than the one its boot record
+                // still denies, and OMITTED rather than `false` when nothing answered the live check,
+                // because the boot record's word for that is a hypothesis and not a measurement.
+                ...(docker.daemon === true ? { dockerAvailable: true } : {}),
+                ...workloadRecord(docker.workload),
                 composePath: infra.composePath,
                 at: Date.now(),
                 durationMs: Date.now() - startedAt,
@@ -111,6 +132,20 @@ export async function standUpInfra(dir, infra, signal, logger, probe = probeDock
         };
     }
 }
+/**
+ * What the live check measured, for the record the Tester step shows.
+ *
+ * Its own field beside `dockerAvailable` because the two answer different questions and only one
+ * of them has a boolean's worth of answers: a daemon either answered or it did not, while what a
+ * container DID on it is `usable`, `unusable`, or a check that could not be carried out. Absent
+ * when nothing was measured at all (an undecided boot record probes nothing), which is not the
+ * same as a check that ran and could not tell.
+ */
+function workloadRecord(workload) {
+    if (!workload)
+        return {};
+    return { dockerWorkload: workload.status === 'unknown' ? 'undetermined' : workload.status };
+}
 /**
  * Stand the run's infra up and return a single cleanup handle, dispatching on the spec's
  * `kind`: the frontend UI-test flow (`kind: 'frontend'`) builds/serves the app + WireMock as

package/dist/job.d.ts

@@ -556,6 +556,16 @@ export interface InfraSetupRecord {
      * mistake that let a daemon-less image read as an ordinary infra failure for months.
      */
     dockerAvailable?: boolean;
+    /**
+     * What a real container DID on that daemon, when the platform measured it.
+     *
+     * The third diagnosis, and the one `dockerAvailable` structurally cannot carry: a rootless
+     * daemon nested in a sandbox answers throughout while unable to mount any image layer, so it is
+     * `dockerAvailable: true` and no stack can come up on it (issue #2120). Reporting that as an
+     * absent daemon sends a human to restart one that is already up. `undetermined` is a check that
+     * ran and could not tell; ABSENT means nothing was measured at all.
+     */
+    dockerWorkload?: 'usable' | 'unusable' | 'undetermined';
     /** The repo-relative compose file that was stood up. */
     composePath?: string;
     /** Epoch ms the stand-up attempt finished. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cat-factory/executor-harness",
-  "version": "1.145.1",
+  "version": "1.147.0",
   "description": "Container payload: a thin TypeScript wrapper that runs the Pi coding agent against a cloned repo and opens a PR. Runs in the Cloudflare Container (and, in local native mode, as a host process); carries no secrets.",
   "repository": {
     "type": "git",
@@ -25,10 +25,10 @@
     "access": "public"
   },
   "devDependencies": {
-    "@cat-factory/contracts": "0.334.0",
-    "@cat-factory/kernel": "0.323.2",
-    "@cat-factory/server": "0.307.9",
-    "@cat-factory/spend": "0.16.26",
+    "@cat-factory/contracts": "0.338.0",
+    "@cat-factory/kernel": "0.327.0",
+    "@cat-factory/server": "0.310.1",
+    "@cat-factory/spend": "0.17.3",
     "@hono/node-server": "^2.1.1",
     "@types/node": "^26.4.0",
     "hono": "^4.13.5",

package/src/agent.ts

@@ -187,8 +187,11 @@ export async function handleAgent(job: AgentJob, opts: RunOptions = {}): Promise
     // This sits on the critical path AHEAD of the clone, which is the cost of having one
     // composition point instead of one per mode (each mode owns its own clone, so there is no
     // single post-clone place to put this). The pass is sized for that: everything in it runs
-    // concurrently, every probe is a call that answers in milliseconds or is wedged, and the one
-    // deliberate wait is a single short retry for a daemon that is still starting.
+    // concurrently, and every probe either answers in milliseconds or is bounded. Two of them are
+    // deliberate waits rather than instant answers: one short retry for a daemon that is still
+    // starting, and, only once a daemon has answered, the container the platform runs to find out
+    // whether it can run one at all (`docker-capability.ts`, budgeted and memoised per container
+    // for a positive). Both take the job's signal, so an abandoned run stops paying at once.
 
     const staged: AgentJob = {
       ...job,