@cat-factory/executor-harness 1.145.1 → 1.149.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +106 -8
- package/dist/agent.js +8 -2
- package/dist/docker-capability.d.ts +158 -0
- package/dist/docker-capability.js +510 -0
- package/dist/docker-command.d.ts +30 -0
- package/dist/docker-command.js +91 -0
- package/dist/docker-probe-image.d.ts +97 -0
- package/dist/docker-probe-image.js +283 -0
- package/dist/docker-status.d.ts +99 -19
- package/dist/docker-status.js +93 -36
- package/dist/environment-inventory.d.ts +57 -6
- package/dist/environment-inventory.js +149 -17
- package/dist/harness-server.js +7 -1
- package/dist/infra-standup.d.ts +15 -12
- package/dist/infra-standup.js +73 -23
- package/dist/job.d.ts +20 -0
- package/dist/redact.d.ts +15 -0
- package/dist/redact.js +18 -0
- package/package.json +5 -5
- package/src/agent.ts +8 -2
- package/src/docker-capability.ts +767 -0
- package/src/docker-command.ts +117 -0
- package/src/docker-probe-image.ts +328 -0
- package/src/docker-status.ts +148 -38
- package/src/environment-inventory.ts +216 -30
- package/src/harness-server.ts +7 -1
- package/src/infra-standup.ts +77 -23
- package/src/job.ts +20 -0
- package/src/redact.ts +19 -0
package/src/docker-status.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { execFile } from 'node:child_process'
|
|
2
1
|
import { readFile } from 'node:fs/promises'
|
|
3
|
-
import {
|
|
2
|
+
import { probeDockerWorkload, type DockerWorkload } from './docker-capability.js'
|
|
3
|
+
import { log, type Logger } from './logger.js'
|
|
4
4
|
|
|
5
5
|
// What this container knows about its own Docker daemon, as recorded by `entrypoint.sh`.
|
|
6
6
|
//
|
|
@@ -14,8 +14,12 @@ import { promisify } from 'node:util'
|
|
|
14
14
|
// daemon reads it instead.
|
|
15
15
|
//
|
|
16
16
|
// The recorded verdict describes BOOT, and a container outlives its boot, so nothing refuses on
|
|
17
|
-
// it unconfirmed: `resolveDockerVerdict` re-checks
|
|
18
|
-
//
|
|
17
|
+
// it unconfirmed: `resolveDockerVerdict` re-checks it against a live daemon and keeps the record
|
|
18
|
+
// for what only the record holds, the cause and the daemon's own log tail. What it records is
|
|
19
|
+
// also only that a SOCKET answered, which is a weaker fact than any caller wants, so the live
|
|
20
|
+
// check RUNS A CONTAINER (docker-capability.ts) rather than settling for the daemon's word about
|
|
21
|
+
// itself. It still reports that weaker fact alongside, since a check that could not be carried
|
|
22
|
+
// out is what the boot record has to be read against, and nothing else establishes it.
|
|
19
23
|
//
|
|
20
24
|
// The three-valued shape is deliberate and is the point (CLAUDE.md, "Degrade loudly"): a daemon
|
|
21
25
|
// that FAILED and a daemon nobody asked about are different facts with different correct
|
|
@@ -55,7 +59,20 @@ export type DockerSource =
|
|
|
55
59
|
export interface DockerStatus {
|
|
56
60
|
available: boolean | undefined
|
|
57
61
|
source: DockerSource
|
|
58
|
-
/**
|
|
62
|
+
/**
|
|
63
|
+
* Why, in the entrypoint's own closed vocabulary (`serving`, `serving-without-nat`,
|
|
64
|
+
* `still-starting`, `failed`, `missing`, `unreachable`, `probing`).
|
|
65
|
+
*
|
|
66
|
+
* Reported and never branched on, which is what lets the entrypoint add a word without anything
|
|
67
|
+
* here having to know it. Two are worth knowing about. `serving-without-nat`: the daemon that
|
|
68
|
+
* manages its own firewall rules exited without serving, so the one that came up runs with
|
|
69
|
+
* `--iptables=false` and its NESTED containers have no egress. That is a CAUSE, and the only
|
|
70
|
+
* place one exists; what MEASURES the consequence is the egress half of the workload check
|
|
71
|
+
* (docker-capability.ts), from inside a container, with no way to learn why. `still-starting`
|
|
72
|
+
* is a daemon that had not answered when the boot budget ran out and is STILL RUNNING, which is
|
|
73
|
+
* the one absence that routinely stops being true: it is exactly what the live re-probe below
|
|
74
|
+
* exists to catch.
|
|
75
|
+
*/
|
|
59
76
|
reason: string
|
|
60
77
|
/** A human detail for the failing cases: the dockerd log tail, or what was unreachable. */
|
|
61
78
|
detail?: string
|
|
@@ -146,56 +163,149 @@ function unnamedSource(source: never): string {
|
|
|
146
163
|
return `no Docker daemon answered in this container (unrecognised source ${JSON.stringify(source)})`
|
|
147
164
|
}
|
|
148
165
|
|
|
149
|
-
/**
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
166
|
+
/**
|
|
167
|
+
* What a daemon can do RIGHT NOW. Injected so the unit suite can state every answer.
|
|
168
|
+
*
|
|
169
|
+
* It answers with a WORKLOAD rather than with a boolean, and that is the correction this type
|
|
170
|
+
* carries. It used to be `docker version`, which proves the daemon is serving; a stand-up needs
|
|
171
|
+
* a daemon that can materialise an image, and a sandboxed rootless daemon routinely serves while
|
|
172
|
+
* being unable to (issue #2120). Running compose against that one produced a mount error the
|
|
173
|
+
* agent had to interpret, from the one mechanism whose entire job is to say why infra did not
|
|
174
|
+
* come up.
|
|
175
|
+
*
|
|
176
|
+
* The weaker fact did not go away, though: it rides `daemonAnswered` on the `unknown` arm, since
|
|
177
|
+
* the workload check establishes it on its way past and {@link resolveDockerVerdict} still needs
|
|
178
|
+
* it. Takes the job's signal, because this is a live check on the critical path of a run that can
|
|
179
|
+
* be cancelled under it.
|
|
180
|
+
*/
|
|
181
|
+
export type DockerProbe = (signal?: AbortSignal) => Promise<DockerWorkload>
|
|
154
182
|
|
|
155
|
-
|
|
183
|
+
/**
|
|
184
|
+
* The default {@link DockerProbe}: the process-wide workload probe, which loads a one-layer
|
|
185
|
+
* image and runs a container from it, memoised per container.
|
|
186
|
+
*
|
|
187
|
+
* Named for what it answers. It was `probeDockerServing`, which is the fact this module exists to
|
|
188
|
+
* say is not enough.
|
|
189
|
+
*/
|
|
190
|
+
export const probeLiveDockerCapability: DockerProbe = (signal) => probeDockerWorkload(signal)
|
|
156
191
|
|
|
157
192
|
/**
|
|
158
|
-
* The
|
|
159
|
-
*
|
|
193
|
+
* The sentence for a daemon that is serving and cannot run anything. It names what was tried,
|
|
194
|
+
* because "docker is unavailable" against a daemon the agent can see answering reads as a bug in
|
|
195
|
+
* the platform rather than as the sandbox limit it is.
|
|
160
196
|
*/
|
|
161
|
-
export
|
|
162
|
-
|
|
163
|
-
await execFileAsync('docker', ['version', '--format', '{{.Server.Version}}'], {
|
|
164
|
-
timeout: PROBE_TIMEOUT_MS,
|
|
165
|
-
})
|
|
166
|
-
return true
|
|
167
|
-
} catch {
|
|
168
|
-
return false
|
|
169
|
-
}
|
|
197
|
+
export function describeDockerUnusable(workload: { detail: string }): string {
|
|
198
|
+
return `this container's Docker daemon is reachable but cannot run a container (${workload.detail})`
|
|
170
199
|
}
|
|
171
200
|
|
|
172
201
|
/** What a stand-up is entitled to conclude about the daemon at the moment it is about to run. */
|
|
173
202
|
export interface DockerVerdict {
|
|
174
|
-
/**
|
|
203
|
+
/**
|
|
204
|
+
* Whether a stand-up may PROCEED, three-valued exactly as {@link DockerStatus.available} and
|
|
205
|
+
* read the same way. It is the decision, not a description of the daemon: a daemon that is
|
|
206
|
+
* answering and cannot run a container is `false` here and `daemon: true` below, and a record
|
|
207
|
+
* that reported the first as the second would send an operator to restart a daemon that is
|
|
208
|
+
* already up.
|
|
209
|
+
*/
|
|
175
210
|
available: boolean | undefined
|
|
176
|
-
/**
|
|
211
|
+
/**
|
|
212
|
+
* Set only for a CONFIRMED negative: the sentence to refuse with. Absent means proceed.
|
|
213
|
+
*
|
|
214
|
+
* Two causes reach it and they read differently on purpose. Nothing is answering here, and
|
|
215
|
+
* something is answering here but cannot run a container: an operator sent to restart a daemon
|
|
216
|
+
* that is already up would find nothing wrong with it.
|
|
217
|
+
*/
|
|
177
218
|
refusal?: string
|
|
219
|
+
/**
|
|
220
|
+
* Whether a daemon ANSWERED the live check. Absent when nothing was checked (an undecided
|
|
221
|
+
* record probes nothing) or when nothing answered, so it is never read as a decided `false`.
|
|
222
|
+
*/
|
|
223
|
+
daemon?: boolean
|
|
224
|
+
/** What a real container did on it, when one was tried. Absent when nothing was measured. */
|
|
225
|
+
workload?: DockerWorkload
|
|
178
226
|
}
|
|
179
227
|
|
|
180
228
|
/**
|
|
181
|
-
* Resolve what to do now, from what boot recorded plus what
|
|
229
|
+
* Resolve what to do now, from what boot recorded plus what the daemon can do today.
|
|
182
230
|
*
|
|
183
|
-
* `entrypoint.sh` probes once, at boot, within a bounded wait
|
|
184
|
-
*
|
|
185
|
-
* serving perfectly well by the second job. Refusing off the recorded verdict alone latches that
|
|
186
|
-
* container into refusing local infra that in fact works, for its whole life, with a stale
|
|
187
|
-
* sentence explaining why. So a recorded absence is a HYPOTHESIS here, and the live probe settles
|
|
188
|
-
* it; the recorded verdict is still what supplies the cause and the daemon's own log tail, which
|
|
189
|
-
* no probe can reconstruct.
|
|
231
|
+
* `entrypoint.sh` probes once, at boot, within a bounded wait, and it probes for a SOCKET. Two
|
|
232
|
+
* things follow, and the branches below are one each.
|
|
190
233
|
*
|
|
191
|
-
*
|
|
192
|
-
*
|
|
234
|
+
* A recorded absence is a HYPOTHESIS. A container outlives its boot: a warm pool serves many jobs
|
|
235
|
+
* from one, and a sidecar daemon that took longer than the wait allows is serving perfectly well
|
|
236
|
+
* by the second job. Refusing off the record alone latches that container into refusing local
|
|
237
|
+
* infra that works, for its whole life, with a stale sentence explaining why. The record is still
|
|
238
|
+
* what supplies the cause and the daemon's own log tail, which no probe can reconstruct.
|
|
239
|
+
*
|
|
240
|
+
* A recorded PRESENCE is a hypothesis too, and that half was missing. `serving` is not `usable`:
|
|
241
|
+
* a rootless daemon in a sandbox answers while unable to mount any image layer, so compose ran
|
|
242
|
+
* and died on a mount error the agent then had to interpret. So the probe is consulted in both
|
|
243
|
+
* directions, and it runs a real container rather than asking the daemon about itself.
|
|
244
|
+
*
|
|
245
|
+
* A check that could not be CARRIED OUT settles nothing, and the cheap fact is what decides
|
|
246
|
+
* there. Falling straight back to the boot record would re-latch the very refusal the paragraph
|
|
247
|
+
* above rules out: the four ways the workload check can come back undeterminable (no payload in
|
|
248
|
+
* this image variant, an architecture it is not built for, a `docker load` the engine refuses, a
|
|
249
|
+
* timeout) have nothing to do with whether a daemon is up, so a warm container whose sidecar
|
|
250
|
+
* arrived late would be denied local infra for the rest of its life over a stale sentence. So a
|
|
251
|
+
* daemon that ANSWERED contradicts a recorded absence exactly as the old `docker version` probe
|
|
252
|
+
* did, and only a check that never reached a daemon at all leaves the record to decide.
|
|
253
|
+
*
|
|
254
|
+
* "Not decided" still keeps attempting, untouched. The point of the third value is that NOTHING
|
|
255
|
+
* turns it into a refusal: the entrypoint's bounded wait may still be running, and a workload
|
|
256
|
+
* probe against a daemon that has not finished starting fails for a reason that says nothing
|
|
257
|
+
* about what it will do a second later.
|
|
193
258
|
*/
|
|
194
259
|
export async function resolveDockerVerdict(
|
|
195
260
|
status: DockerStatus,
|
|
196
|
-
|
|
261
|
+
opts: { probe?: DockerProbe; signal?: AbortSignal; logger?: Logger } = {},
|
|
197
262
|
): Promise<DockerVerdict> {
|
|
198
|
-
if (status.available
|
|
199
|
-
|
|
200
|
-
|
|
263
|
+
if (status.available === undefined) return { available: undefined }
|
|
264
|
+
const workload = await askTotally(
|
|
265
|
+
opts.probe ?? probeLiveDockerCapability,
|
|
266
|
+
opts.signal,
|
|
267
|
+
opts.logger,
|
|
268
|
+
)
|
|
269
|
+
if (workload.status === 'usable') return { available: true, daemon: true, workload }
|
|
270
|
+
if (workload.status === 'unusable') {
|
|
271
|
+
return {
|
|
272
|
+
available: false,
|
|
273
|
+
refusal: describeDockerUnusable(workload),
|
|
274
|
+
daemon: true,
|
|
275
|
+
workload,
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
if (workload.daemonAnswered) return { available: true, daemon: true, workload }
|
|
279
|
+
return status.available
|
|
280
|
+
? { available: true, workload }
|
|
281
|
+
: { available: false, refusal: describeDockerAbsence(status), workload }
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* Call the probe and answer even if it throws.
|
|
286
|
+
*
|
|
287
|
+
* The default probe is total by construction and says so, but this is the seam an injected one
|
|
288
|
+
* arrives through, and the caller is a stand-up documented as best-effort: a throw here would
|
|
289
|
+
* fail a job over the mechanism whose whole purpose is to make a failure legible. A throw settles
|
|
290
|
+
* nothing about the daemon, so it becomes the same value as any other check that could not be
|
|
291
|
+
* carried out and the boot record decides, exactly as it did before the probe existed.
|
|
292
|
+
*/
|
|
293
|
+
async function askTotally(
|
|
294
|
+
probe: DockerProbe,
|
|
295
|
+
signal: AbortSignal | undefined,
|
|
296
|
+
logger: Logger | undefined,
|
|
297
|
+
): Promise<DockerWorkload> {
|
|
298
|
+
try {
|
|
299
|
+
return await probe(signal)
|
|
300
|
+
} catch (err) {
|
|
301
|
+
const message = err instanceof Error ? err.message : String(err)
|
|
302
|
+
;(logger ?? log).warn('docker: the live daemon check threw; falling back to the boot record', {
|
|
303
|
+
error: message,
|
|
304
|
+
})
|
|
305
|
+
return {
|
|
306
|
+
status: 'unknown',
|
|
307
|
+
reason: `the live Docker check could not be carried out (${message})`,
|
|
308
|
+
daemonAnswered: false,
|
|
309
|
+
}
|
|
310
|
+
}
|
|
201
311
|
}
|
|
@@ -2,6 +2,11 @@ import { execFile } from 'node:child_process'
|
|
|
2
2
|
import { promisify } from 'node:util'
|
|
3
3
|
import { log, type Logger } from './logger.js'
|
|
4
4
|
import { harnessListenPort } from './harness-port.js'
|
|
5
|
+
import {
|
|
6
|
+
probeDockerWorkload,
|
|
7
|
+
type ContainerEgress,
|
|
8
|
+
type DockerWorkload,
|
|
9
|
+
} from './docker-capability.js'
|
|
5
10
|
|
|
6
11
|
// ---------------------------------------------------------------------------
|
|
7
12
|
// What this machine actually has, probed ONCE per job and stated to the agent.
|
|
@@ -32,6 +37,18 @@ import { harnessListenPort } from './harness-port.js'
|
|
|
32
37
|
// a daemon this machine is CONFIGURED for but which has not answered yet is a fourth state
|
|
33
38
|
// that resolves to `unknown`, never to the absence a refused connection looks like: the
|
|
34
39
|
// image's daemon is started in the background and the job begins before it is ready.
|
|
40
|
+
// - A daemon that ANSWERS is still not a daemon that WORKS, which is the same mistake one level
|
|
41
|
+
// in. A rootless daemon nested in a sandbox serves while its snapshotter cannot mount any
|
|
42
|
+
// image layer, so `docker info` succeeds and `docker build` / `docker run` / `docker pull`
|
|
43
|
+
// all fail (issue #2120). Only a container that RAN settles that, so the reachable case is
|
|
44
|
+
// split by a real workload (docker-capability.ts) into `usable`, `unusable`, and a daemon
|
|
45
|
+
// that answered while the check itself could not be carried out.
|
|
46
|
+
// - And a daemon that runs containers is still not one whose containers have a NETWORK, which
|
|
47
|
+
// is the same mistake one level in again. Loading and running a local image needs no network,
|
|
48
|
+
// so a daemon started with `--iptables=false` passes the workload check while every nested
|
|
49
|
+
// container is cut off (issue #2174): what an agent then hits is a `docker build` whose
|
|
50
|
+
// `RUN npm ci` sits in retry backoff for about seven minutes before failing. So `usable`
|
|
51
|
+
// carries its own egress verdict, and the line below says something different for each.
|
|
35
52
|
//
|
|
36
53
|
// Deliberately NOT here: the agent's own tools (web search, file tools, MCP servers). Those are
|
|
37
54
|
// the CLI's, they differ per harness, and each is already stated where it is true. Claiming one
|
|
@@ -68,6 +85,33 @@ export type ToolPresence =
|
|
|
68
85
|
| { status: 'absent' }
|
|
69
86
|
| { status: 'unknown'; reason: string }
|
|
70
87
|
|
|
88
|
+
/**
|
|
89
|
+
* What this machine's Docker daemon is good for, which is FIVE answers and not three.
|
|
90
|
+
*
|
|
91
|
+
* `absent` and `unknown` mean for the daemon what they mean for any other tool. The three that
|
|
92
|
+
* are particular to Docker split the case where a daemon ANSWERED, because answering is not the
|
|
93
|
+
* question anyone is asking:
|
|
94
|
+
*
|
|
95
|
+
* - `usable`: a container was built and run on it here. `docker build` / `run` / `compose`
|
|
96
|
+
* work, and this is the only state that may say so.
|
|
97
|
+
* - `unusable`: a container could NOT be run, with the daemon serving throughout. The state
|
|
98
|
+
* issue #2120 is about; stated as a prohibition, with the cause.
|
|
99
|
+
* - `serving`: it answered, and the workload check could not be carried out (no payload on
|
|
100
|
+
* this machine, an unmapped architecture, a timeout). Neither of the other two,
|
|
101
|
+
* and rendered as "try it if you need it".
|
|
102
|
+
*
|
|
103
|
+
* `usable` then carries what a nested container could REACH, which is a second fact and not a
|
|
104
|
+
* sixth state: what an agent may do with the daemon and what its containers can fetch are
|
|
105
|
+
* different questions, and each of the three egress answers changes the advice without changing
|
|
106
|
+
* the verdict on the daemon.
|
|
107
|
+
*/
|
|
108
|
+
export type DockerCapability =
|
|
109
|
+
| { status: 'usable'; server?: string; egress: ContainerEgress }
|
|
110
|
+
| { status: 'unusable'; server?: string; detail: string }
|
|
111
|
+
| { status: 'serving'; server?: string; reason: string }
|
|
112
|
+
| { status: 'absent' }
|
|
113
|
+
| { status: 'unknown'; reason: string }
|
|
114
|
+
|
|
71
115
|
/** One probed entry: the name the agent would type, and what came back. */
|
|
72
116
|
export interface ProbedTool {
|
|
73
117
|
name: string
|
|
@@ -80,12 +124,13 @@ export interface ProbedTool {
|
|
|
80
124
|
export interface EnvironmentInventory {
|
|
81
125
|
tools: ProbedTool[]
|
|
82
126
|
/**
|
|
83
|
-
*
|
|
84
|
-
* image's `entrypoint.sh` starts a rootless daemon BEST-EFFORT and execs
|
|
85
|
-
* waiting for it, so at job start this probe is the only thing that knows
|
|
86
|
-
* "has not answered yet" is one of its answers (see {@link probeDockerDaemon})
|
|
127
|
+
* What the Docker daemon is good for: not the CLI's presence, and not merely whether the
|
|
128
|
+
* daemon answered. The image's `entrypoint.sh` starts a rootless daemon BEST-EFFORT and execs
|
|
129
|
+
* the server without waiting for it, so at job start this probe is the only thing that knows
|
|
130
|
+
* how that went; "has not answered yet" is one of its answers (see {@link probeDockerDaemon})
|
|
131
|
+
* and "answered, but cannot run a container" is another (see {@link DockerCapability}).
|
|
87
132
|
*/
|
|
88
|
-
dockerDaemon:
|
|
133
|
+
dockerDaemon: DockerCapability
|
|
89
134
|
/**
|
|
90
135
|
* The port the harness's own job server holds in this network namespace. Not probed: the
|
|
91
136
|
* process reads its own {@link harnessListenPort}, which is the only honest answer when a
|
|
@@ -310,6 +355,18 @@ export interface ProbeEnvironmentOptions {
|
|
|
310
355
|
* only so the suite can assert the rendered line without an ambient `PORT` deciding its text.
|
|
311
356
|
*/
|
|
312
357
|
harnessPort?: number
|
|
358
|
+
/**
|
|
359
|
+
* Whether the daemon can actually RUN a container, defaulting to the process-wide probe
|
|
360
|
+
* (docker-capability.ts). Asked only once a daemon has answered, since there is nothing to run
|
|
361
|
+
* a workload on otherwise, and memoised per container so a warm pool pays for it once.
|
|
362
|
+
*/
|
|
363
|
+
workload?: (signal?: AbortSignal) => Promise<DockerWorkload>
|
|
364
|
+
/**
|
|
365
|
+
* The job's signal, forwarded to the probes that spawn something. The workload check starts a
|
|
366
|
+
* CONTAINER, so a cancelled job must stop paying for it rather than hold the daemon for the
|
|
367
|
+
* rest of its budget.
|
|
368
|
+
*/
|
|
369
|
+
signal?: AbortSignal
|
|
313
370
|
}
|
|
314
371
|
|
|
315
372
|
/**
|
|
@@ -346,11 +403,35 @@ export async function probeEnvironment(
|
|
|
346
403
|
presence: toolPresence(await run(probe.command, probe.args)),
|
|
347
404
|
})),
|
|
348
405
|
),
|
|
349
|
-
|
|
406
|
+
probeDockerCapability(run, opts),
|
|
350
407
|
])
|
|
351
408
|
return { tools, dockerDaemon, harnessPort: opts.harnessPort ?? harnessListenPort() }
|
|
352
409
|
}
|
|
353
410
|
|
|
411
|
+
/**
|
|
412
|
+
* The daemon's full answer: whether one is reachable, and then whether it can run a container.
|
|
413
|
+
*
|
|
414
|
+
* The two steps are kept apart because they fail for unrelated reasons and only the FIRST has a
|
|
415
|
+
* cheap answer. A daemon nobody can reach has no workload to run, so the check that costs a
|
|
416
|
+
* container start is asked only where there is something to ask it of; a daemon that answered
|
|
417
|
+
* carries its server version into every one of the three states that follow it, because the
|
|
418
|
+
* agent reading the line is entitled to know which daemon the verdict is about.
|
|
419
|
+
*/
|
|
420
|
+
async function probeDockerCapability(
|
|
421
|
+
run: ProbeRunner,
|
|
422
|
+
opts: ProbeEnvironmentOptions,
|
|
423
|
+
): Promise<DockerCapability> {
|
|
424
|
+
const daemon = await probeDockerDaemon(run, opts)
|
|
425
|
+
if (daemon.status === 'absent') return { status: 'absent' }
|
|
426
|
+
if (daemon.status === 'unknown') return { status: 'unknown', reason: daemon.reason }
|
|
427
|
+
const server = daemon.version ? { server: daemon.version } : {}
|
|
428
|
+
const workload = await (opts.workload ?? probeDockerWorkload)(opts.signal)
|
|
429
|
+
if (workload.status === 'usable') return { status: 'usable', ...server, egress: workload.egress }
|
|
430
|
+
if (workload.status === 'unusable')
|
|
431
|
+
return { status: 'unusable', ...server, detail: workload.detail }
|
|
432
|
+
return { status: 'serving', ...server, reason: workload.reason }
|
|
433
|
+
}
|
|
434
|
+
|
|
354
435
|
/**
|
|
355
436
|
* Ask the daemon itself, and do not mistake a daemon that is STARTING for one that is not there.
|
|
356
437
|
*
|
|
@@ -482,26 +563,124 @@ function harnessPortLine(port: number): string {
|
|
|
482
563
|
)
|
|
483
564
|
}
|
|
484
565
|
|
|
485
|
-
/**
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
566
|
+
/**
|
|
567
|
+
* The Docker line, which says something different in each of the five cases, and is TOTAL over
|
|
568
|
+
* them: adding a state without deciding what an agent should do about it stops the build.
|
|
569
|
+
*
|
|
570
|
+
* Only `usable` may claim the commands work, and it may only be reached by having RUN one. The
|
|
571
|
+
* line that used to stand here made that claim off `docker info` alone, which is how every agent
|
|
572
|
+
* in a run was told, as fact, that a daemon which could not mount a single image layer would
|
|
573
|
+
* build and run one.
|
|
574
|
+
*/
|
|
575
|
+
function dockerDaemonLine(daemon: DockerCapability): string {
|
|
576
|
+
const server = 'server' in daemon && daemon.server ? ` (server ${daemon.server})` : ''
|
|
577
|
+
switch (daemon.status) {
|
|
578
|
+
case 'usable':
|
|
579
|
+
return `A Docker daemon is reachable${server} and the platform ran a container on it: ${usableCommands(daemon.egress)} work here. ${egressSentence(daemon.egress)}`
|
|
580
|
+
case 'unusable':
|
|
581
|
+
return (
|
|
582
|
+
`A Docker daemon is reachable${server} but it CANNOT run a container: the platform ` +
|
|
583
|
+
`built a one-layer image and tried to run it here, and that failed (${daemon.detail}). ` +
|
|
584
|
+
'`docker build`, `docker run`, `docker pull` of a multi-layer image and ' +
|
|
585
|
+
'`docker compose up` all fail for the same reason, so there is nothing to retry and no ' +
|
|
586
|
+
'flag that works around it. Produce the Dockerfile or compose file you were asked for, ' +
|
|
587
|
+
'say in one line that it could not be built or run here, and move on.'
|
|
588
|
+
)
|
|
589
|
+
case 'serving':
|
|
590
|
+
return (
|
|
591
|
+
`A Docker daemon is reachable${server}, but whether it can actually build or run an ` +
|
|
592
|
+
`image was NOT established (${daemon.reason}). Reaching the daemon is not the same fact: ` +
|
|
593
|
+
'a sandboxed one answers while being unable to mount any image layer. Try it if you need ' +
|
|
594
|
+
'it, and do not read a failure as a defect in the work.'
|
|
595
|
+
)
|
|
596
|
+
case 'unknown':
|
|
597
|
+
return (
|
|
598
|
+
'Whether a Docker daemon is reachable could not be determined ' +
|
|
599
|
+
`(${daemon.reason}): try it if you need it, and do not read a failure as a defect in the work.`
|
|
600
|
+
)
|
|
601
|
+
case 'absent':
|
|
602
|
+
return (
|
|
603
|
+
'NO Docker daemon is reachable: `docker build`, `docker run` and `docker compose up` ' +
|
|
604
|
+
'will fail here whatever the CLI reports. Produce the Dockerfile or compose file you ' +
|
|
605
|
+
'were asked for, say in one line that you could not build it here, and move on.'
|
|
606
|
+
)
|
|
607
|
+
default:
|
|
608
|
+
return unnamedCapability(daemon)
|
|
493
609
|
}
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
function unnamedCapability(daemon: never): string {
|
|
613
|
+
return `Whether a Docker daemon is reachable could not be determined (the platform reported an unrecognised verdict ${JSON.stringify(daemon)}): try it if you need it.`
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
/**
|
|
617
|
+
* Which commands the `usable` line may claim, which is the EGRESS verdict's business and not the
|
|
618
|
+
* daemon's.
|
|
619
|
+
*
|
|
620
|
+
* Split out because the line used to open with the full list and then, one sentence later, tell
|
|
621
|
+
* the agent that every `RUN` line which fetches anything fails. A block that also says not to
|
|
622
|
+
* spend turns re-checking it cannot afford to state and then retract the same fact: an agent
|
|
623
|
+
* reading the first sentence has already been told `docker build` works here, which is the exact
|
|
624
|
+
* shape of the lie issue #2174 is about. TOTAL over {@link ContainerEgress}, like its sibling.
|
|
625
|
+
*/
|
|
626
|
+
function usableCommands(egress: ContainerEgress): string {
|
|
627
|
+
switch (egress.status) {
|
|
628
|
+
case 'blocked':
|
|
629
|
+
// Deliberately omits `docker build`: it is the one the missing NAT rule actually breaks,
|
|
630
|
+
// and the sentence that follows explains which part of it and why.
|
|
631
|
+
return '`docker run` and `docker compose up` of images that are already built'
|
|
632
|
+
case 'reachable':
|
|
633
|
+
case 'undetermined':
|
|
634
|
+
return '`docker build`, `docker run` and `docker compose up`'
|
|
635
|
+
default:
|
|
636
|
+
return unnamedEgressCommands(egress)
|
|
499
637
|
}
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
)
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
/** The commands claimed for an egress verdict this build does not know: none of them. */
|
|
641
|
+
function unnamedEgressCommands(egress: never): string {
|
|
642
|
+
return `the docker commands the platform could name for its verdict ${JSON.stringify(egress)}`
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
/**
|
|
646
|
+
* The second half of the `usable` line: what a container started HERE can reach, and what to do
|
|
647
|
+
* about it. TOTAL over {@link ContainerEgress} for the same reason the line above is over the
|
|
648
|
+
* daemon's states.
|
|
649
|
+
*
|
|
650
|
+
* The `blocked` arm is the one this exists for, and it is precise about WHICH commands break,
|
|
651
|
+
* because "docker has no network" is not true and an agent that believed it would skip work it
|
|
652
|
+
* could have done. The daemon has a network: it pulls base images and compose pulls its services
|
|
653
|
+
* normally. What has none is the container each `RUN` line executes in. The seven minutes are
|
|
654
|
+
* named because the failure does not look like a failure from inside the agent's loop: npm turns
|
|
655
|
+
* "no route" into `EAI_AGAIN` only once its retry backoff gives up, so the build reads as a hang
|
|
656
|
+
* and the natural response is to wait longer.
|
|
657
|
+
*/
|
|
658
|
+
function egressSentence(egress: ContainerEgress): string {
|
|
659
|
+
switch (egress.status) {
|
|
660
|
+
case 'reachable':
|
|
661
|
+
return 'A container started here also reaches the network, so a build that installs dependencies works.'
|
|
662
|
+
case 'blocked':
|
|
663
|
+
return (
|
|
664
|
+
'A container started here could reach NOTHING the platform tried, and that is a fact ' +
|
|
665
|
+
`about this sandbox rather than about your work (${egress.detail}). The daemon itself is ` +
|
|
666
|
+
'fine: it pulls base images, and `docker compose up` of pre-built images works. What ' +
|
|
667
|
+
'fails is every `RUN` line in a `docker build` that fetches from the public internet ' +
|
|
668
|
+
'(`npm ci`, `apk add`, `pip install`), and it fails SLOWLY: npm reports `EAI_AGAIN` only ' +
|
|
669
|
+
'after some seven minutes of retry backoff, so it reads as a hang. Do not wait it out ' +
|
|
670
|
+
'and do not retry. Vendor what you need, skip the image build, verify some other way, or ' +
|
|
671
|
+
'say in one line that you could not verify it here. If this project already builds ' +
|
|
672
|
+
'against a mirror inside this network, that is not one of the addresses tried above and ' +
|
|
673
|
+
'is worth one attempt.'
|
|
674
|
+
)
|
|
675
|
+
case 'undetermined':
|
|
676
|
+
return `Whether a container started here can reach the network was NOT established (${egress.reason}), so try it if you need it and do not read a failure as a defect in the work.`
|
|
677
|
+
default:
|
|
678
|
+
return unnamedEgress(egress)
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
function unnamedEgress(egress: never): string {
|
|
683
|
+
return `Whether a container started here can reach the network could not be determined (the platform reported an unrecognised verdict ${JSON.stringify(egress)}).`
|
|
505
684
|
}
|
|
506
685
|
|
|
507
686
|
/**
|
|
@@ -520,21 +699,28 @@ function dockerDaemonLine(daemon: ToolPresence): string {
|
|
|
520
699
|
*/
|
|
521
700
|
export async function appendEnvironmentInventory(
|
|
522
701
|
systemPrompt: string,
|
|
523
|
-
opts: {
|
|
702
|
+
opts: { log?: Logger; run?: ProbeRunner } & ProbeEnvironmentOptions = {},
|
|
524
703
|
): Promise<string> {
|
|
525
704
|
const logger = opts.log ?? log
|
|
705
|
+
// Everything that is not this function's OWN is forwarded by construction, rather than key by
|
|
706
|
+
// key. The list of copied keys silently dropped `workload`, whose whole point is that a suite
|
|
707
|
+
// can inject one: a test driving THIS entry point (the only one `handleAgent` uses) got the
|
|
708
|
+
// real probe instead, which starts a container on whatever machine the suite runs on.
|
|
709
|
+
const { log: _log, run, ...probeOptions } = opts
|
|
526
710
|
try {
|
|
527
|
-
const inventory = await probeEnvironment(
|
|
528
|
-
...(opts.sleep ? { sleep: opts.sleep } : {}),
|
|
529
|
-
...(opts.daemonExpected === undefined ? {} : { daemonExpected: opts.daemonExpected }),
|
|
530
|
-
...(opts.harnessPort === undefined ? {} : { harnessPort: opts.harnessPort }),
|
|
531
|
-
})
|
|
711
|
+
const inventory = await probeEnvironment(run ?? spawnProbeRunner(opts.signal), probeOptions)
|
|
532
712
|
logger.info('agent: probed the environment', {
|
|
533
713
|
installed: inventory.tools
|
|
534
714
|
.filter((t) => t.presence.status === 'present')
|
|
535
715
|
.map((t) => t.name)
|
|
536
716
|
.join(','),
|
|
537
717
|
dockerDaemon: inventory.dockerDaemon.status,
|
|
718
|
+
// Present only when a nested container was actually asked, which is the one state that has
|
|
719
|
+
// an egress answer at all. A word here for every other state would report "no measurement"
|
|
720
|
+
// and "measured, and it cannot get out" in the same field.
|
|
721
|
+
...(inventory.dockerDaemon.status === 'usable'
|
|
722
|
+
? { dockerEgress: inventory.dockerDaemon.egress.status }
|
|
723
|
+
: {}),
|
|
538
724
|
// The unknowns, by NAME: the block tells the agent a probe failed, and this is the only place
|
|
539
725
|
// an operator can see WHICH, since the reason the agent reads is deliberately wordy prose.
|
|
540
726
|
unknown: inventory.tools
|
package/src/harness-server.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { handleAgent } from './agent.js'
|
|
|
6
6
|
import { handleInline } from './inline.js'
|
|
7
7
|
import { redactSecrets } from './git.js'
|
|
8
8
|
import { readDockerStatus } from './docker-status.js'
|
|
9
|
+
import { reportedDockerWorkload } from './docker-capability.js'
|
|
9
10
|
import { harnessListenPort } from './harness-port.js'
|
|
10
11
|
import { JobRegistry, loadRunnerLimits, type JobResultBase, type RunOptions } from './runner.js'
|
|
11
12
|
import { log } from './logger.js'
|
|
@@ -141,11 +142,16 @@ const server = createServer((req, res) => {
|
|
|
141
142
|
// would spawn a process per poll to answer a question this endpoint is not the one to act
|
|
142
143
|
// on; the stand-up re-confirms a recorded absence at the moment it matters
|
|
143
144
|
// (`resolveDockerVerdict`), so a stale negative here never becomes a stale refusal there.
|
|
145
|
+
//
|
|
146
|
+
// `workload` is the other half, and the reason the block used to mislead: what the record
|
|
147
|
+
// says is `serving`, and serving is not usable. It reports the last measurement any job
|
|
148
|
+
// took (docker-capability.ts) and NEVER takes one itself, for the same polling reason,
|
|
149
|
+
// which is why `unmeasured` is one of the words it can answer.
|
|
144
150
|
return send(res, 200, {
|
|
145
151
|
status: 'ok',
|
|
146
152
|
...(HARNESS_VERSION ? { version: HARNESS_VERSION } : {}),
|
|
147
153
|
capabilities: HARNESS_BODY_CAPABILITIES,
|
|
148
|
-
docker: await readDockerStatus(),
|
|
154
|
+
docker: { ...(await readDockerStatus()), workload: reportedDockerWorkload() },
|
|
149
155
|
})
|
|
150
156
|
}
|
|
151
157
|
// All non-health endpoints are gated by the optional shared secret.
|