@cat-factory/executor-harness 1.145.1 → 1.149.0

package/src/infra-standup.ts

@@ -6,9 +6,10 @@
 import { execFile } from 'node:child_process'
 import { promisify } from 'node:util'
 import type { AgentInfraSpec, InfraSetupRecord, ServiceInfraSpec } from './job.js'
+import type { DockerWorkload } from './docker-capability.js'
 import {
   type DockerProbe,
-  probeDockerServing,
+  probeLiveDockerCapability,
   readDockerStatus,
   resolveDockerVerdict,
 } from './docker-status.js'
@@ -26,19 +27,22 @@ const exec = promisify(execFile)
  * still run unit-level tests and report what it could. A no-op for ephemeral / no-infra /
  * no-compose-path runs.
  *
- * A CONFIRMED absence of a Docker daemon short-circuits it: the container's own probe
- * ({@link readDockerStatus}, recorded by `entrypoint.sh`) already knows there is nothing to
- * talk to, so running compose against it would only turn a fact this container holds into a
- * connection error the agent has to interpret. The record then carries `dockerAvailable: false`
- * and the stated cause, which is what makes the Tester step say why it ran no infra instead of
- * looking like a Tester that simply chose not to. Anything OTHER than a confirmed absence
- * attempts as before (`DockerStatus.available` in docker-status.ts states why "undecided" is its
- * own value).
+ * A CONFIRMED absence of a USABLE Docker daemon short-circuits it: the container already knows
+ * compose cannot work, so running it would only turn a fact this container holds into an error
+ * the agent has to interpret. The record then carries the stated cause plus the two facts that
+ * decide where a human should look (`dockerAvailable`: was anything answering, `dockerWorkload`:
+ * what a container did on it), which is what makes the Tester step say why it ran no infra
+ * instead of looking like a Tester that simply chose not to. Anything OTHER than a confirmed
+ * negative attempts as before (`DockerStatus.available` in docker-status.ts states why
+ * "undecided" is its own value).
  *
- * "Confirmed", not merely recorded: {@link resolveDockerVerdict} re-checks a recorded absence
- * against a live daemon first, so a warm-pool container whose sidecar came up late is not
- * latched into refusing infra that works. `probe` is that check, injected so the unit suite can
- * state both answers on a machine that has its own daemon either way.
+ * "Confirmed", not merely recorded: {@link resolveDockerVerdict} re-checks the boot record
+ * against a live daemon first, so a warm-pool container whose sidecar came up late is not latched
+ * into refusing infra that works. It re-checks a recorded PRESENCE too, by running an actual
+ * container: a rootless daemon in a sandbox answers `docker version` while being unable to mount
+ * an image, and compose against that one died on a mount error inside the very mechanism that
+ * exists to explain why infra did not come up. `probe` is that check, injected so the unit suite
+ * can state every answer on a machine that has its own daemon either way.
  *
  * Whether it succeeds or fails, the (redacted, bounded) command output is captured into a
  * {@link InfraSetupRecord} returned alongside the prompt `note`, so the backend can surface
@@ -54,27 +58,46 @@ export async function standUpInfra(
   infra: ServiceInfraSpec,
   signal: AbortSignal | undefined,
   logger: Logger,
-  probe: DockerProbe = probeDockerServing,
+  probe: DockerProbe = probeLiveDockerCapability,
 ): Promise<{ started: boolean; note?: string; record?: InfraSetupRecord }> {
   if (infra.environment !== 'local' || infra.noInfraDependencies || !infra.composePath) {
     return { started: false }
   }
   const startedAt = Date.now()
   const recorded = await readDockerStatus()
-  const docker = await resolveDockerVerdict(recorded, probe)
+  const docker = await resolveDockerVerdict(recorded, {
+    probe,
+    ...(signal ? { signal } : {}),
+    logger,
+  })
   if (docker.refusal) {
     const note = `the dependencies could not be started: ${docker.refusal}`
-    logger.warn('agent(explore): infra stand-up refused, no docker daemon', {
+    logger.warn('agent(explore): infra stand-up refused, no usable docker daemon', {
       composePath: infra.composePath,
       dockerSource: recorded.source,
       dockerReason: recorded.reason,
+      // What the LIVE check found, which is the only place the second refusal cause exists: the
+      // boot record's own words for a daemon that answers and cannot run anything are `serving`
+      // and nothing else, so a log line carrying the record alone describes the wrong failure.
+      dockerWorkload: docker.workload?.status ?? 'unmeasured',
+      ...(docker.workload?.status === 'usable'
+        ? { dockerEgress: docker.workload.egress.status }
+        : {}),
+      ...(docker.workload?.status === 'unusable' ? { dockerDetail: docker.workload.detail } : {}),
     })
     return {
       started: false,
       note,
       record: {
         started: false,
-        dockerAvailable: false,
+        // NOT a flat `false`. Two refusals reach this branch and they have opposite fixes: with
+        // nothing answering, the executor image or the sandbox running it is what to go and look
+        // at; with a daemon that answers and cannot run a container, that daemon is up and an
+        // operator sent to restart it finds nothing wrong. `dockerAvailable` answers only the
+        // first question and `dockerWorkload` the second, so neither has to carry the other's
+        // fact (the same rule the compose-failure branch below states for its own `false`).
+        dockerAvailable: docker.daemon === true,
+        ...workloadRecord(docker.workload),
         composePath: infra.composePath,
         at: Date.now(),
         durationMs: Date.now() - startedAt,
@@ -97,6 +120,7 @@ export async function standUpInfra(
       record: {
         started: true,
         dockerAvailable: true,
+        ...workloadRecord(docker.workload),
         composePath: infra.composePath,
         at: Date.now(),
         durationMs: Date.now() - startedAt,
@@ -116,12 +140,13 @@ export async function standUpInfra(
       note,
       record: {
         started: false,
-        // A compose failure with a REACHABLE daemon: the two `false`s above and here are
-        // different diagnoses (nothing to talk to vs the stack itself did not come up), and
-        // only stating both keeps the second from being read as the first. Read off the
-        // RESOLVED verdict, so a container whose daemon came up after boot claims the daemon it
-        // actually reached rather than the one its boot record still denies.
-        ...(docker.available === true ? { dockerAvailable: true } : {}),
+        // A compose failure with a REACHABLE daemon, which is a third diagnosis again: the stack
+        // itself did not come up. Read off the RESOLVED verdict, so a container whose daemon came
+        // up after boot claims the daemon it actually reached rather than the one its boot record
+        // still denies, and OMITTED rather than `false` when nothing answered the live check,
+        // because the boot record's word for that is a hypothesis and not a measurement.
+        ...(docker.daemon === true ? { dockerAvailable: true } : {}),
+        ...workloadRecord(docker.workload),
         composePath: infra.composePath,
         at: Date.now(),
         durationMs: Date.now() - startedAt,
@@ -132,6 +157,35 @@ export async function standUpInfra(
   }
 }
 
+/**
+ * What the live check measured, for the record the Tester step shows.
+ *
+ * Its own field beside `dockerAvailable` because the two answer different questions and only one
+ * of them has a boolean's worth of answers: a daemon either answered or it did not, while what a
+ * container DID on it is `usable`, `unusable`, or a check that could not be carried out. Absent
+ * when nothing was measured at all (an undecided boot record probes nothing), which is not the
+ * same as a check that ran and could not tell.
+ *
+ * THREE fields rather than two, because `usable` is not one fact. A daemon running with
+ * `--iptables=false` runs containers perfectly and gives them no network, so it is `usable` and
+ * every `docker build` that fetches anything on it is guaranteed to fail. The agent's prompt and
+ * `GET /health` both learn that; without `dockerEgress` the record a human reads on the Tester
+ * step, and every backend consumer of it, sees the same undifferentiated `usable` as a sandbox
+ * where the stack actually works. Present only on `usable`, which is the one verdict that has an
+ * egress half at all: on any other, a word here would report "no measurement" and "measured, and
+ * it cannot get out" in the same field.
+ */
+function workloadRecord(workload: DockerWorkload | undefined): {
+  dockerWorkload?: InfraSetupRecord['dockerWorkload']
+  dockerEgress?: InfraSetupRecord['dockerEgress']
+} {
+  if (!workload) return {}
+  if (workload.status === 'usable') {
+    return { dockerWorkload: 'usable', dockerEgress: workload.egress.status }
+  }
+  return { dockerWorkload: workload.status === 'unknown' ? 'undetermined' : workload.status }
+}
+
 /**
  * Stand the run's infra up and return a single cleanup handle, dispatching on the spec's
  * `kind`: the frontend UI-test flow (`kind: 'frontend'`) builds/serves the app + WireMock as

package/src/job.ts

@@ -955,6 +955,26 @@ export interface InfraSetupRecord {
    * mistake that let a daemon-less image read as an ordinary infra failure for months.
    */
   dockerAvailable?: boolean
+  /**
+   * What a real container DID on that daemon, when the platform measured it.
+   *
+   * The third diagnosis, and the one `dockerAvailable` structurally cannot carry: a rootless
+   * daemon nested in a sandbox answers throughout while unable to mount any image layer, so it is
+   * `dockerAvailable: true` and no stack can come up on it (issue #2120). Reporting that as an
+   * absent daemon sends a human to restart one that is already up. `undetermined` is a check that
+   * ran and could not tell; ABSENT means nothing was measured at all.
+   */
+  dockerWorkload?: 'usable' | 'unusable' | 'undetermined'
+  /**
+   * What a container started ON that daemon could REACH, when the platform measured it.
+   *
+   * The fourth diagnosis, and the one `dockerWorkload: 'usable'` structurally cannot carry: a
+   * rootless daemon started with `--iptables=false` installs no MASQUERADE rule for its bridge,
+   * so it runs containers perfectly and none of them has a route out. The stack comes up and
+   * every `docker build` that fetches a dependency fails, slowly. Present only alongside
+   * `usable`, which is the one verdict with an egress half; absent means nothing measured it.
+   */
+  dockerEgress?: 'reachable' | 'blocked' | 'undetermined'
   /** The repo-relative compose file that was stood up. */
   composePath?: string
   /** Epoch ms the stand-up attempt finished. */

package/src/redact.ts

@@ -72,6 +72,25 @@ export function redactSecrets(input: string): string {
   return redact(input)
 }
 
+/**
+ * A scrubbed, length-bounded excerpt of a string that is about to be QUOTED at a human or a
+ * model: a failing command's output, a rejected setting, a thrown message.
+ *
+ * One helper rather than a private `bounded()` per module, which is what this replaced. Two of
+ * them had drifted: `docker-probe-image.ts` echoed a rejected `HARNESS_DOCKER_EGRESS_TARGET`
+ * verbatim into a string that reaches every agent's system prompt and `GET /health`, while its
+ * same-named neighbour in `docker-capability.ts` scrubbed first. A proxy URL with an embedded
+ * token in that setting is the ordinary way that becomes a leak, and two helpers with one name
+ * in sibling files is what kept the divergence invisible.
+ *
+ * Scrub BEFORE bounding, so the cut cannot land inside a credential and leave half of it
+ * quotable, and mark a trimmed value so a reader never takes the head for the whole.
+ */
+export function scrubbedExcerpt(text: string, maxChars: number): string {
+  const scrubbed = redactSecrets(text)
+  return scrubbed.length > maxChars ? `${scrubbed.slice(0, maxChars)}…` : scrubbed
+}
+
 /** Cap on captured command output kept on an infra record (tail-biased — failures show last). */
 export const MAX_CAPTURED_OUTPUT_CHARS = 16_000