@cat-factory/executor-harness 1.145.1 → 1.149.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +106 -8
- package/dist/agent.js +8 -2
- package/dist/docker-capability.d.ts +158 -0
- package/dist/docker-capability.js +510 -0
- package/dist/docker-command.d.ts +30 -0
- package/dist/docker-command.js +91 -0
- package/dist/docker-probe-image.d.ts +97 -0
- package/dist/docker-probe-image.js +283 -0
- package/dist/docker-status.d.ts +99 -19
- package/dist/docker-status.js +93 -36
- package/dist/environment-inventory.d.ts +57 -6
- package/dist/environment-inventory.js +149 -17
- package/dist/harness-server.js +7 -1
- package/dist/infra-standup.d.ts +15 -12
- package/dist/infra-standup.js +73 -23
- package/dist/job.d.ts +20 -0
- package/dist/redact.d.ts +15 -0
- package/dist/redact.js +18 -0
- package/package.json +5 -5
- package/src/agent.ts +8 -2
- package/src/docker-capability.ts +767 -0
- package/src/docker-command.ts +117 -0
- package/src/docker-probe-image.ts +328 -0
- package/src/docker-status.ts +148 -38
- package/src/environment-inventory.ts +216 -30
- package/src/harness-server.ts +7 -1
- package/src/infra-standup.ts +77 -23
- package/src/job.ts +20 -0
- package/src/redact.ts +19 -0
|
@@ -0,0 +1,510 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { spawnDockerCommand, } from './docker-command.js';
|
|
3
|
+
import { buildEgressCommand, buildProbeArchive, EGRESS_DNS_MARKER, EGRESS_TCP_MARKER, parseEgressTarget, payloadArchitecture, PROBE_COMMAND, PROBE_IMAGE_TAG, PROBE_SENTINEL, } from './docker-probe-image.js';
|
|
4
|
+
import { log } from './logger.js';
|
|
5
|
+
import { scrubbedExcerpt } from './redact.js';
|
|
6
|
+
/**
|
|
7
|
+
* The statically linked binary the probe image is built from, overridable for an image variant
|
|
8
|
+
* that ships it elsewhere. Absent is a supported answer, not a failure: under
|
|
9
|
+
* `LOCAL_NATIVE_AGENTS` the harness runs on a developer's machine that never saw this image.
|
|
10
|
+
*/
|
|
11
|
+
const PAYLOAD_PATH = process.env.HARNESS_DOCKER_PROBE_BINARY?.trim() || '/bin/busybox';
|
|
12
|
+
/**
|
|
13
|
+
* Where the egress check aims, overridable for a deployment whose network permits something else.
|
|
14
|
+
*
|
|
15
|
+
* A raw IPv4 address rather than a name, so the connect answers a question about ROUTING alone:
|
|
16
|
+
* pointing it at a hostname would make every verdict depend on DNS, which is the other half and
|
|
17
|
+
* is measured separately. `1.1.1.1:443` is an anycast address that answers TLS from everywhere
|
|
18
|
+
* and belongs to no API this repo calls; the name is npm's because npm is what the outage
|
|
19
|
+
* actually broke. Neither is validated here (see `parseEgressTarget`), so a rejected setting is
|
|
20
|
+
* reported rather than replaced.
|
|
21
|
+
*
|
|
22
|
+
* Both defaults aim at the PUBLIC internet, and a deployment that deliberately has none should
|
|
23
|
+
* point these at what it does run (an internal registry mirror and its own DNS zone). Left at
|
|
24
|
+
* the defaults there, the measurement is honest but narrow: it establishes that a container
|
|
25
|
+
* cannot reach these two, which is why the prompt built from a `blocked` verdict says which
|
|
26
|
+
* targets were tried rather than that nothing at all is reachable.
|
|
27
|
+
*/
|
|
28
|
+
const EGRESS_TARGET = process.env.HARNESS_DOCKER_EGRESS_TARGET?.trim() || '1.1.1.1:443';
|
|
29
|
+
const EGRESS_DNS_NAME = process.env.HARNESS_DOCKER_EGRESS_DNS_NAME?.trim() || 'registry.npmjs.org';
|
|
30
|
+
/**
|
|
31
|
+
* The ceiling on ONE WHOLE measurement, shared out across the docker commands it makes: each
|
|
32
|
+
* gets what is left of it, down to {@link MIN_COMMAND_MS}.
|
|
33
|
+
*
|
|
34
|
+
* One budget rather than a per-command ceiling, because a per-command one multiplies: three
|
|
35
|
+
* commands at 30s each is a minute and a half of dead time on a wedged daemon, on the critical
|
|
36
|
+
* path ahead of the clone. Sized for a WEDGED daemon and not for a slow one: the payload is a
|
|
37
|
+
* couple of megabytes already on local disk, so a daemon that works answers in about the time it
|
|
38
|
+
* takes to start one container, and a daemon that cannot mount fails immediately.
|
|
39
|
+
*
|
|
40
|
+
* What it does NOT do is bound the cost per JOB, and the comment that once claimed so was wrong.
|
|
41
|
+
* A POSITIVE verdict is memoised for the container's life; a negative is deliberately
|
|
42
|
+
* re-measured (see {@link createDockerWorkloadProbe}), and two independent sites ask per job (the
|
|
43
|
+
* environment inventory and the compose stand-up), so a serving-but-wedged daemon costs this
|
|
44
|
+
* twice per job. That is the price of not latching a warm container into a stale refusal, which
|
|
45
|
+
* is why the budget is the size it is and why an abandoned job stops paying it at once.
|
|
46
|
+
*/
|
|
47
|
+
const WORKLOAD_BUDGET_MS = 20_000;
|
|
48
|
+
/** The floor on one command's share of the budget, so an exhausted budget still gets an answer. */
|
|
49
|
+
const MIN_COMMAND_MS = 1_000;
|
|
50
|
+
/**
|
|
51
|
+
* The ceiling on the egress container, which gets its OWN budget rather than a share of the one
|
|
52
|
+
* above.
|
|
53
|
+
*
|
|
54
|
+
* The argument for a single shared budget is that a per-command ceiling multiplies on a WEDGED
|
|
55
|
+
* daemon, and that argument does not reach here: this container is started only after another one
|
|
56
|
+
* has already run to completion, so the daemon is known to work by the time it is spawned. What
|
|
57
|
+
* it does have to allow for is a check that is SUPPOSED to be slow in the failing case, since a
|
|
58
|
+
* blocked route is silent rather than refused and both in-container timeouts have to expire.
|
|
59
|
+
* Taking that out of the workload budget would have starved the step this whole module exists
|
|
60
|
+
* for; leaving it unbounded would hand a wedged network the whole job.
|
|
61
|
+
*/
|
|
62
|
+
const EGRESS_BUDGET_MS = 20_000;
|
|
63
|
+
/**
|
|
64
|
+
* The ceiling on removing the probe image again.
|
|
65
|
+
*
|
|
66
|
+
* Its own, and deliberately NOT given the caller's abort signal: the image is the platform's, and
|
|
67
|
+
* a cancelled job is the one case where nobody is left to clean up after it. Bounded separately
|
|
68
|
+
* so a wedged daemon cannot turn the cleanup into a second full budget.
|
|
69
|
+
*/
|
|
70
|
+
const CLEANUP_TIMEOUT_MS = 5_000;
|
|
71
|
+
/** How much of a failing command's output is kept. It is quoted into an agent's system prompt. */
|
|
72
|
+
const DETAIL_CHARS = 300;
|
|
73
|
+
/** Build a {@link ProbeArchiveMemo}. Supplied only by {@link realDeps}, so a test memoises nothing. */
|
|
74
|
+
export function oneSlotArchiveMemo() {
|
|
75
|
+
let held;
|
|
76
|
+
return {
|
|
77
|
+
read: (key) => (held?.key === key ? held.archive : undefined),
|
|
78
|
+
write: (key, archive) => {
|
|
79
|
+
held = { key, archive };
|
|
80
|
+
},
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
const realDeps = {
|
|
84
|
+
readPayload: (path) => readFile(path),
|
|
85
|
+
payloadPath: PAYLOAD_PATH,
|
|
86
|
+
runDocker: spawnDockerCommand,
|
|
87
|
+
arch: process.arch,
|
|
88
|
+
egress: { target: EGRESS_TARGET, dnsName: EGRESS_DNS_NAME },
|
|
89
|
+
archives: oneSlotArchiveMemo(),
|
|
90
|
+
};
|
|
91
|
+
/**
|
|
92
|
+
* Carry out one measurement. Pure of caching, so the suite states every branch directly.
|
|
93
|
+
*
|
|
94
|
+
* TOTAL: it never rejects, whatever happens inside it. The thing it replaced was total by
|
|
95
|
+
* construction (a `try/catch` around one `execFile`), and it is consulted from a stand-up path
|
|
96
|
+
* documented as best-effort, so a throw here would fail a job over a probe whose whole purpose is
|
|
97
|
+
* to make a failure legible. A throw is also, by definition, the platform's own machinery
|
|
98
|
+
* breaking, which is the `unknown` disposition and never the `unusable` one.
|
|
99
|
+
*
|
|
100
|
+
* That asymmetry is the design. Only the RUN produces `unusable`; everything before it produces
|
|
101
|
+
* `unknown`, because everything before it is the platform's own machinery and a bug in it must be
|
|
102
|
+
* able to say "I could not tell" and never "your daemon is broken". The load step in particular is
|
|
103
|
+
* the one this repo wrote itself.
|
|
104
|
+
*/
|
|
105
|
+
export async function measureDockerWorkload(deps = realDeps, signal) {
|
|
106
|
+
const seen = { daemonAnswered: false };
|
|
107
|
+
try {
|
|
108
|
+
return await measure(deps, seen, signal);
|
|
109
|
+
}
|
|
110
|
+
catch (err) {
|
|
111
|
+
const cause = describeThrown(err);
|
|
112
|
+
(deps.logger ?? log).warn('docker capability: the container check itself fell over', {
|
|
113
|
+
error: cause,
|
|
114
|
+
});
|
|
115
|
+
return undeterminable(`the platform's own container check could not be completed (${cause})`, seen.daemonAnswered);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
async function measure(deps, seen, signal) {
|
|
119
|
+
const deadline = Date.now() + WORKLOAD_BUDGET_MS;
|
|
120
|
+
const share = () => Math.max(MIN_COMMAND_MS, deadline - Date.now());
|
|
121
|
+
const command = (args, stdin) => deps.runDocker(args, {
|
|
122
|
+
...(stdin ? { stdin } : {}),
|
|
123
|
+
...(signal ? { signal } : {}),
|
|
124
|
+
timeoutMs: share(),
|
|
125
|
+
...(deps.logger ? { logger: deps.logger } : {}),
|
|
126
|
+
});
|
|
127
|
+
// Ask the DAEMON which architecture it runs, rather than assuming it shares this process's.
|
|
128
|
+
// An external `DOCKER_HOST` is a supported path, and an arm64 harness against an amd64 sidecar
|
|
129
|
+
// (or a remote x86_64 daemon reached from an arm64 laptop) shares nothing with it but a socket:
|
|
130
|
+
// declaring the wrong one in the image config gets the run refused, which would report a
|
|
131
|
+
// perfectly good daemon as one that cannot run containers. It is also the cheapest proof that a
|
|
132
|
+
// daemon is answering at all, which is the fact `resolveDockerVerdict` reads back.
|
|
133
|
+
const asked = await command(['version', '--format', '{{.Server.Arch}}']);
|
|
134
|
+
if (asked.outcome !== 'ran' || asked.code !== 0) {
|
|
135
|
+
return undeterminable(`no Docker daemon answered the platform's container check (${describeOutcome(asked)})`, false);
|
|
136
|
+
}
|
|
137
|
+
seen.daemonAnswered = true;
|
|
138
|
+
const daemonArch = asked.stdout.trim();
|
|
139
|
+
if (!/^[a-z0-9_]+$/.test(daemonArch)) {
|
|
140
|
+
return undeterminable(`the Docker daemon did not name an architecture the platform can build an image for (${scrubbedExcerpt(daemonArch, 40) || 'it answered nothing'})`, true);
|
|
141
|
+
}
|
|
142
|
+
const payloadArch = payloadArchitecture(deps.arch);
|
|
143
|
+
if (!payloadArch) {
|
|
144
|
+
return undeterminable(`the platform has no container check for the ${deps.arch} architecture`, true);
|
|
145
|
+
}
|
|
146
|
+
if (payloadArch !== daemonArch) {
|
|
147
|
+
return undeterminable(`the platform's container check is built for ${payloadArch} and this daemon runs ${daemonArch}`, true);
|
|
148
|
+
}
|
|
149
|
+
const assembled = await assembleArchive(deps, daemonArch);
|
|
150
|
+
if ('reason' in assembled)
|
|
151
|
+
return undeterminable(assembled.reason, true);
|
|
152
|
+
const load = await command(['load'], assembled.archive);
|
|
153
|
+
if (load.outcome !== 'ran' || load.code !== 0) {
|
|
154
|
+
return undeterminable(`the platform could not load its own probe image (${describeOutcome(load)})`, true);
|
|
155
|
+
}
|
|
156
|
+
const run = await command([
|
|
157
|
+
'run',
|
|
158
|
+
'--rm',
|
|
159
|
+
'--pull',
|
|
160
|
+
'never',
|
|
161
|
+
'--network',
|
|
162
|
+
'none',
|
|
163
|
+
PROBE_IMAGE_TAG,
|
|
164
|
+
...PROBE_COMMAND,
|
|
165
|
+
]);
|
|
166
|
+
// A daemon that ran that container has answered the first question, and only then is there a
|
|
167
|
+
// second one worth asking. An `unusable` daemon cannot run the egress container either, and a
|
|
168
|
+
// check that could not be carried out has nothing to measure egress against.
|
|
169
|
+
const verdict = classifyRun(run);
|
|
170
|
+
const measured = verdict.status === 'usable'
|
|
171
|
+
? { status: 'usable', egress: await measureEgress(deps, signal) }
|
|
172
|
+
: verdict;
|
|
173
|
+
// After both runs, whatever the verdict is: the probe image is the platform's, and an agent
|
|
174
|
+
// that runs `docker images` should not have to wonder whose it is.
|
|
175
|
+
await removeProbeImage(deps);
|
|
176
|
+
return measured;
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Run the second container and read what it reached.
|
|
180
|
+
*
|
|
181
|
+
* On the DEFAULT network, deliberately, which is the one thing that separates it from the marker
|
|
182
|
+
* run above (`--network none`). What an agent's own `docker build` and `docker run` get is the
|
|
183
|
+
* bridge, and the bridge is exactly what a daemon started with `--iptables=false` fails to NAT.
|
|
184
|
+
*
|
|
185
|
+
* Never concludes anything about the DAEMON. Every failure here is either evidence about the
|
|
186
|
+
* network or evidence about this check, and the caller has already established that the daemon
|
|
187
|
+
* runs containers.
|
|
188
|
+
*/
|
|
189
|
+
async function measureEgress(deps, signal) {
|
|
190
|
+
const setting = parseEgressTarget(deps.egress.target, deps.egress.dnsName);
|
|
191
|
+
// A rejected setting is read from this container's own environment, so it answers the same way
|
|
192
|
+
// on every job: latched rather than re-measured, which would otherwise spend two container
|
|
193
|
+
// starts per job re-reading one unchanged string.
|
|
194
|
+
if ('invalid' in setting) {
|
|
195
|
+
return { status: 'undetermined', reason: setting.invalid, recheck: false };
|
|
196
|
+
}
|
|
197
|
+
const run = await deps.runDocker(['run', '--rm', '--pull', 'never', PROBE_IMAGE_TAG, ...buildEgressCommand(setting.target)], {
|
|
198
|
+
...(signal ? { signal } : {}),
|
|
199
|
+
timeoutMs: EGRESS_BUDGET_MS,
|
|
200
|
+
...(deps.logger ? { logger: deps.logger } : {}),
|
|
201
|
+
});
|
|
202
|
+
return classifyEgress(run, setting.target);
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* What the egress container's output proves, over the four combinations its two markers can
|
|
206
|
+
* carry.
|
|
207
|
+
*
|
|
208
|
+
* Read off the STATUS each command printed rather than off the run's own exit code, because the
|
|
209
|
+
* two failures that look alike from outside need opposite answers: a refused connection is
|
|
210
|
+
* evidence about the network, and a 126/127 is busybox saying the image has no such applet, which
|
|
211
|
+
* is evidence about the platform's own payload and may never be reported as a network that is not
|
|
212
|
+
* there.
|
|
213
|
+
*/
|
|
214
|
+
function classifyEgress(run, target) {
|
|
215
|
+
const where = `${target.host}:${target.port}`;
|
|
216
|
+
if (run.outcome === 'failed') {
|
|
217
|
+
return {
|
|
218
|
+
status: 'undetermined',
|
|
219
|
+
reason: `the platform's egress check did not run (${run.reason})`,
|
|
220
|
+
// A spawn failure or a timeout is about this attempt and not about the container.
|
|
221
|
+
recheck: true,
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
const tcp = readMarker(run.stdout, EGRESS_TCP_MARKER);
|
|
225
|
+
const dns = readMarker(run.stdout, EGRESS_DNS_MARKER);
|
|
226
|
+
if (tcp === undefined || dns === undefined) {
|
|
227
|
+
// Nothing was measured, and the two ways to get here need different words. `docker run`
|
|
228
|
+
// failing at the DAEMON level (125, or its own "no such image") means the container never
|
|
229
|
+
// started, so the network was not the thing that did not answer; a container that started
|
|
230
|
+
// and printed something unrecognisable is the platform's own payload misbehaving. Reporting
|
|
231
|
+
// the first as "printed no verdict" tells an operator to go looking at the check's output
|
|
232
|
+
// for a container that produced none.
|
|
233
|
+
const refused = platformSideRunFailure(run) ?? (run.code === 125 ? daemonRefusedEgressRun : undefined);
|
|
234
|
+
return refused
|
|
235
|
+
? {
|
|
236
|
+
status: 'undetermined',
|
|
237
|
+
reason: `the platform's egress container did not start (${refused}: ${describeOutcome(run)})`,
|
|
238
|
+
// The daemon just ran the marker container, so a bridge it could not attach now is
|
|
239
|
+
// the sort of thing that can differ on the next job.
|
|
240
|
+
recheck: true,
|
|
241
|
+
}
|
|
242
|
+
: {
|
|
243
|
+
status: 'undetermined',
|
|
244
|
+
reason: `the platform's egress check printed no verdict (${describeOutcome(run)})`,
|
|
245
|
+
// Same image, same argv, same output: asking again re-reads the same non-answer.
|
|
246
|
+
recheck: false,
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
if ([tcp, dns].some((code) => code === 126 || code === 127)) {
|
|
250
|
+
return {
|
|
251
|
+
status: 'undetermined',
|
|
252
|
+
reason: "the platform's egress check could not run inside its own probe container (the payload " +
|
|
253
|
+
'has no `nc` or `nslookup` applet)',
|
|
254
|
+
// A fact about the image this repo builds, which does not change under a running container.
|
|
255
|
+
recheck: false,
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
if (tcp === 0 && dns === 0)
|
|
259
|
+
return { status: 'reachable' };
|
|
260
|
+
if (tcp === 0) {
|
|
261
|
+
return {
|
|
262
|
+
status: 'blocked',
|
|
263
|
+
detail: `a container reached ${where} but could not resolve ${target.dnsName}: the route out works and DNS does not`,
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
if (dns === 0) {
|
|
267
|
+
// A resolved name proves a path out of the container exists, so the connect failing is far
|
|
268
|
+
// more likely to be about the ADDRESS than about the network. Saying "blocked" here would
|
|
269
|
+
// condemn a working sandbox over a target it happens to filter.
|
|
270
|
+
return {
|
|
271
|
+
status: 'undetermined',
|
|
272
|
+
reason: `a container resolved ${target.dnsName} but could not connect to ${where}, so this ` +
|
|
273
|
+
'deployment probably filters that address; point HARNESS_DOCKER_EGRESS_TARGET at one it permits',
|
|
274
|
+
// A filtered address is a standing property of the network this container sits in.
|
|
275
|
+
recheck: false,
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
return {
|
|
279
|
+
status: 'blocked',
|
|
280
|
+
detail: `a container could reach neither ${where} nor ${target.dnsName}, the two the platform is configured to try`,
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
/** What a 125 from the egress run means, kept beside the other platform-side run messages. */
|
|
284
|
+
const daemonRefusedEgressRun = 'the daemon refused to create or start it';
|
|
285
|
+
/**
|
|
286
|
+
* The exit status printed after `marker`, or undefined when the container never printed one.
|
|
287
|
+
*
|
|
288
|
+
* The LAST occurrence wins, so a marker that somehow reached the stream twice is read at its
|
|
289
|
+
* final value rather than at whichever came first.
|
|
290
|
+
*/
|
|
291
|
+
function readMarker(stdout, marker) {
|
|
292
|
+
const status = [...stdout.matchAll(new RegExp(`${marker}(\\d{1,3})`, 'g'))].pop()?.[1];
|
|
293
|
+
return status === undefined ? undefined : Number(status);
|
|
294
|
+
}
|
|
295
|
+
function classifyRun(run) {
|
|
296
|
+
if (run.outcome === 'failed') {
|
|
297
|
+
return undeterminable(`the platform's container check did not run (${run.reason})`, true);
|
|
298
|
+
}
|
|
299
|
+
if (run.code === 0) {
|
|
300
|
+
if (run.stdout.includes(PROBE_SENTINEL))
|
|
301
|
+
return { status: 'usable' };
|
|
302
|
+
// Nothing explains this: the container was reported as having run cleanly and produced none
|
|
303
|
+
// of the output it exists to produce. That is a fact about the check, not about the daemon.
|
|
304
|
+
return undeterminable("the platform's probe container exited cleanly without printing its marker", true);
|
|
305
|
+
}
|
|
306
|
+
const ours = platformSideRunFailure(run);
|
|
307
|
+
return ours
|
|
308
|
+
? undeterminable(`${ours} (${describeOutcome(run)})`, true)
|
|
309
|
+
: { status: 'unusable', detail: describeOutcome(run) };
|
|
310
|
+
}
|
|
311
|
+
/** Messages that name the PLATFORM's half of a failed run rather than the daemon's. */
|
|
312
|
+
const PLATFORM_SIDE_RUN_MESSAGES = [
|
|
313
|
+
{
|
|
314
|
+
pattern: /no such image|unable to find image/i,
|
|
315
|
+
cause: "the platform's probe image was not there to be run",
|
|
316
|
+
},
|
|
317
|
+
{
|
|
318
|
+
pattern: /exec format error/i,
|
|
319
|
+
cause: "the platform's probe binary cannot be executed on this daemon's machine",
|
|
320
|
+
},
|
|
321
|
+
];
|
|
322
|
+
function platformSideRunFailure(run) {
|
|
323
|
+
if (run.code === 126 || run.code === 127) {
|
|
324
|
+
return "the platform's probe binary could not be invoked inside the container";
|
|
325
|
+
}
|
|
326
|
+
const said = `${run.stderr}\n${run.stdout}`;
|
|
327
|
+
return PLATFORM_SIDE_RUN_MESSAGES.find((m) => m.pattern.test(said))?.cause;
|
|
328
|
+
}
|
|
329
|
+
/**
|
|
330
|
+
* Assemble the archive for the daemon's architecture, reusing the last one built.
|
|
331
|
+
*
|
|
332
|
+
* A read that fails is classified rather than asserted away: `HARNESS_DOCKER_PROBE_BINARY`
|
|
333
|
+
* pointing at a directory, at a path this user may not read, or at a failing mount is a
|
|
334
|
+
* misconfiguration an operator can fix, and "this machine does not have it" states the opposite
|
|
335
|
+
* fact. The sentence goes into an agent's system prompt and into `GET /health`, so a discarded
|
|
336
|
+
* cause is a cause nobody ever sees.
|
|
337
|
+
*/
|
|
338
|
+
async function assembleArchive(deps, architecture) {
|
|
339
|
+
const key = `${deps.payloadPath}::${architecture}`;
|
|
340
|
+
const held = deps.archives?.read(key);
|
|
341
|
+
if (held)
|
|
342
|
+
return { archive: held };
|
|
343
|
+
let payload;
|
|
344
|
+
try {
|
|
345
|
+
payload = await deps.readPayload(deps.payloadPath);
|
|
346
|
+
}
|
|
347
|
+
catch (err) {
|
|
348
|
+
return { reason: describePayloadFailure(err, deps.payloadPath) };
|
|
349
|
+
}
|
|
350
|
+
const archive = buildProbeArchive(payload, architecture);
|
|
351
|
+
deps.archives?.write(key, archive);
|
|
352
|
+
return { archive };
|
|
353
|
+
}
|
|
354
|
+
function describePayloadFailure(err, path) {
|
|
355
|
+
const needs = `the platform's own container check needs ${path}`;
|
|
356
|
+
switch (err.code) {
|
|
357
|
+
case 'ENOENT':
|
|
358
|
+
case 'ENOTDIR':
|
|
359
|
+
return `${needs}, which this machine does not have`;
|
|
360
|
+
case 'EACCES':
|
|
361
|
+
case 'EPERM':
|
|
362
|
+
return `${needs}, which it is not permitted to read`;
|
|
363
|
+
case 'EISDIR':
|
|
364
|
+
return `${needs} to be a file, and it is a directory`;
|
|
365
|
+
default:
|
|
366
|
+
return `${needs}, which could not be read (${describeThrown(err)})`;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
/**
|
|
370
|
+
* Remove the probe image, and SAY SO when that did not work.
|
|
371
|
+
*
|
|
372
|
+
* The one line above it promises an agent will never find a `cat-factory-docker-probe` and wonder
|
|
373
|
+
* whose it is, and the daemon has two ordinary ways to refuse: a `--rm` teardown still in flight
|
|
374
|
+
* holds the image ("image is being used by stopped container"), and a wedged daemon does not
|
|
375
|
+
* answer at all. Discarding the outcome left both silent, so the promise was unverifiable in
|
|
376
|
+
* exactly the states that break it.
|
|
377
|
+
*/
|
|
378
|
+
async function removeProbeImage(deps) {
|
|
379
|
+
const removed = await deps.runDocker(['image', 'rm', '-f', PROBE_IMAGE_TAG], {
|
|
380
|
+
timeoutMs: CLEANUP_TIMEOUT_MS,
|
|
381
|
+
...(deps.logger ? { logger: deps.logger } : {}),
|
|
382
|
+
});
|
|
383
|
+
if (removed.outcome === 'ran' && removed.code === 0)
|
|
384
|
+
return;
|
|
385
|
+
(deps.logger ?? log).warn('docker capability: the probe image could not be removed', {
|
|
386
|
+
image: PROBE_IMAGE_TAG,
|
|
387
|
+
error: describeOutcome(removed),
|
|
388
|
+
});
|
|
389
|
+
}
|
|
390
|
+
function undeterminable(reason, daemonAnswered) {
|
|
391
|
+
return { status: 'unknown', reason, daemonAnswered };
|
|
392
|
+
}
|
|
393
|
+
/** A bounded, scrubbed one-line summary of what a command said, for a prompt or a log field. */
|
|
394
|
+
function describeOutcome(outcome) {
|
|
395
|
+
if (outcome.outcome === 'failed')
|
|
396
|
+
return outcome.reason;
|
|
397
|
+
const said = `${outcome.stderr}\n${outcome.stdout}`
|
|
398
|
+
.split('\n')
|
|
399
|
+
.map((line) => line.trim())
|
|
400
|
+
.filter(Boolean)
|
|
401
|
+
.join('; ');
|
|
402
|
+
return bounded(said) || `docker exited ${outcome.code} without saying why`;
|
|
403
|
+
}
|
|
404
|
+
/** The one describer for a thrown value here: scrubbed and bounded, like any other detail. */
|
|
405
|
+
function describeThrown(err) {
|
|
406
|
+
return bounded(err instanceof Error ? err.message : String(err)) || 'it said nothing';
|
|
407
|
+
}
|
|
408
|
+
function bounded(text) {
|
|
409
|
+
return scrubbedExcerpt(text, DETAIL_CHARS);
|
|
410
|
+
}
|
|
411
|
+
/**
|
|
412
|
+
* Build a probe that measures at most once per container for a POSITIVE answer.
|
|
413
|
+
*
|
|
414
|
+
* A daemon that has run a container has proved something that does not stop being true, so that
|
|
415
|
+
* verdict is kept and every later job reads it for free. A negative is NOT kept, for the reason
|
|
416
|
+
* `resolveDockerVerdict` gives about the boot record: a container outlives its boot, a warm pool
|
|
417
|
+
* serves many jobs from one, and a daemon that was not ready for the first job must not latch
|
|
418
|
+
* the whole container into saying so. Re-measuring a negative is cheap; a daemon that cannot
|
|
419
|
+
* mount fails at once.
|
|
420
|
+
*
|
|
421
|
+
* A `usable` verdict whose EGRESS could not be determined is re-measured on the same rule and for
|
|
422
|
+
* the same reason. Whether the bridge is NATed is settled once and for the daemon's life, so a
|
|
423
|
+
* measured `reachable` or `blocked` is kept; a check that timed out measured nothing, and latching
|
|
424
|
+
* that would leave the container permanently unable to say which of the two it is.
|
|
425
|
+
*
|
|
426
|
+
* But only where asking again could ANSWER differently, which is what `ContainerEgress.recheck`
|
|
427
|
+
* carries. Most ways to reach `undetermined` are standing facts about this container: a rejected
|
|
428
|
+
* target setting, a payload with no `nc`, an address the deployment filters. Re-running the whole
|
|
429
|
+
* measurement on those never converges, and it is not cheap: it is `docker version`, the archive,
|
|
430
|
+
* `docker load`, two container starts and an `image rm`, per job, ahead of the clone.
|
|
431
|
+
*
|
|
432
|
+
* Concurrent callers share one in-flight measurement rather than each starting a container, and
|
|
433
|
+
* the measurement is cancelled when the LAST of them has abandoned it. Neither half is optional:
|
|
434
|
+
* one job's abort may not kill a measurement a sibling job is still waiting on (the local native
|
|
435
|
+
* transport serves every concurrent job from one process), and a measurement nobody is waiting
|
|
436
|
+
* for is a container start no job will read, which is what an abandoned run should stop paying
|
|
437
|
+
* for the moment it is abandoned.
|
|
438
|
+
*/
|
|
439
|
+
export function createDockerWorkloadProbe(deps = realDeps) {
|
|
440
|
+
let latest;
|
|
441
|
+
let inFlight;
|
|
442
|
+
const begin = () => {
|
|
443
|
+
const cancel = new AbortController();
|
|
444
|
+
const measurement = {
|
|
445
|
+
cancel,
|
|
446
|
+
waiters: 0,
|
|
447
|
+
result: measureDockerWorkload(deps, cancel.signal).then((verdict) => {
|
|
448
|
+
latest = verdict;
|
|
449
|
+
if (inFlight === measurement)
|
|
450
|
+
inFlight = undefined;
|
|
451
|
+
return verdict;
|
|
452
|
+
}),
|
|
453
|
+
};
|
|
454
|
+
return measurement;
|
|
455
|
+
};
|
|
456
|
+
const probe = (async (signal) => {
|
|
457
|
+
if (latest?.status === 'usable' && !isWorthReMeasuring(latest.egress))
|
|
458
|
+
return latest;
|
|
459
|
+
const measurement = (inFlight ??= begin());
|
|
460
|
+
measurement.waiters += 1;
|
|
461
|
+
const watch = signal ? watchAbandonment(signal) : undefined;
|
|
462
|
+
try {
|
|
463
|
+
return watch
|
|
464
|
+
? await Promise.race([measurement.result, watch.abandoned])
|
|
465
|
+
: await measurement.result;
|
|
466
|
+
}
|
|
467
|
+
finally {
|
|
468
|
+
watch?.dispose();
|
|
469
|
+
measurement.waiters -= 1;
|
|
470
|
+
if (measurement.waiters === 0 && inFlight === measurement)
|
|
471
|
+
measurement.cancel.abort();
|
|
472
|
+
}
|
|
473
|
+
});
|
|
474
|
+
probe.last = () => latest;
|
|
475
|
+
return probe;
|
|
476
|
+
}
|
|
477
|
+
/** Whether a kept verdict's egress half is one that asking again could still settle. */
|
|
478
|
+
function isWorthReMeasuring(egress) {
|
|
479
|
+
return egress.status === 'undetermined' && egress.recheck;
|
|
480
|
+
}
|
|
481
|
+
/**
|
|
482
|
+
* A verdict for the caller whose job was cancelled while it waited, and the listener teardown
|
|
483
|
+
* that keeps a long-lived native-transport process from accumulating one per job.
|
|
484
|
+
*/
|
|
485
|
+
function watchAbandonment(signal) {
|
|
486
|
+
let give = () => { };
|
|
487
|
+
const abandoned = new Promise((resolve) => {
|
|
488
|
+
give = () => resolve(undeterminable("the job was cancelled before the platform's container check answered", false));
|
|
489
|
+
if (signal.aborted)
|
|
490
|
+
give();
|
|
491
|
+
else
|
|
492
|
+
signal.addEventListener('abort', give, { once: true });
|
|
493
|
+
});
|
|
494
|
+
return { abandoned, dispose: () => signal.removeEventListener('abort', give) };
|
|
495
|
+
}
|
|
496
|
+
/** The process-wide probe. One per container, which is what makes the positive memo worth having. */
|
|
497
|
+
export const probeDockerWorkload = createDockerWorkloadProbe();
|
|
498
|
+
/**
|
|
499
|
+
* What `GET /health` reports about the workload check.
|
|
500
|
+
*
|
|
501
|
+
* `unmeasured` is its own word rather than an omitted key or a `null`: this endpoint is polled
|
|
502
|
+
* from boot, so "nothing has needed the daemon yet" is the normal early answer and it must not
|
|
503
|
+
* read as either a broken daemon or a build that cannot report one.
|
|
504
|
+
*/
|
|
505
|
+
export function reportedDockerWorkload(probe = probeDockerWorkload) {
|
|
506
|
+
return (probe.last() ?? {
|
|
507
|
+
status: 'unmeasured',
|
|
508
|
+
reason: 'nothing in this container has needed the docker daemon yet',
|
|
509
|
+
});
|
|
510
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { type Logger } from './logger.js';
|
|
2
|
+
/** What running one docker command did, kept as raw as the spawn. */
|
|
3
|
+
export type CommandOutcome = {
|
|
4
|
+
outcome: 'ran';
|
|
5
|
+
code: number;
|
|
6
|
+
stdout: string;
|
|
7
|
+
stderr: string;
|
|
8
|
+
} | {
|
|
9
|
+
outcome: 'failed';
|
|
10
|
+
reason: string;
|
|
11
|
+
};
|
|
12
|
+
/** What one docker invocation is given. `timeoutMs` is required: an unbounded one has no caller. */
|
|
13
|
+
export interface DockerCommandOptions {
|
|
14
|
+
/** Piped to the command's stdin and closed. */
|
|
15
|
+
stdin?: Buffer;
|
|
16
|
+
/** The job's signal. An abandoned job's command is killed rather than left running. */
|
|
17
|
+
signal?: AbortSignal;
|
|
18
|
+
timeoutMs: number;
|
|
19
|
+
logger?: Logger;
|
|
20
|
+
}
|
|
21
|
+
/** Run one `docker …` command. Injected so the suite drives every branch with no daemon. */
|
|
22
|
+
export type DockerCommandRunner = (args: string[], opts: DockerCommandOptions) => Promise<CommandOutcome>;
|
|
23
|
+
/**
|
|
24
|
+
* The real runner: spawn docker, feed it `stdin` when there is any, and report what happened.
|
|
25
|
+
*
|
|
26
|
+
* Never rejects. Every way a spawn can go wrong is one of the two outcomes, because the caller
|
|
27
|
+
* classifies them differently and an exception would collapse that distinction into whichever
|
|
28
|
+
* `catch` caught it first.
|
|
29
|
+
*/
|
|
30
|
+
export declare const spawnDockerCommand: DockerCommandRunner;
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { log } from './logger.js';
|
|
3
|
+
import { killChildProcess, spawnDetached } from './process.js';
|
|
4
|
+
// ---------------------------------------------------------------------------
|
|
5
|
+
// How the harness runs one `docker …` command ON ITS OWN BEHALF, bounded and abortable.
|
|
6
|
+
//
|
|
7
|
+
// This is NOT a second `runCapturedCommand` (captured-command.ts), which stays the one way the
|
|
8
|
+
// harness runs a DECLARED shell command: that one takes a shell string, merges both streams into
|
|
9
|
+
// one rolling tail and answers with a conventional exit code, because its two callers report a
|
|
10
|
+
// pass/fail plus a tail to a model. The docker checks need the three things it deliberately does
|
|
11
|
+
// not offer: an argv (no shell, so nothing quotes an image tag), a STDIN body (the probe archive
|
|
12
|
+
// is piped to `docker load`), and stdout kept APART from stderr, since the whole evidence that a
|
|
13
|
+
// container ran is a marker on stdout while the evidence of why it did not is on stderr.
|
|
14
|
+
//
|
|
15
|
+
// What it does NOT re-decide is how a child dies: `killChildProcess` owns the SIGTERM→SIGKILL
|
|
16
|
+
// escalation for every process this harness spawns, and a bespoke `SIGKILL` here would be one
|
|
17
|
+
// path whose kill semantics drift from the rest with no test able to see it.
|
|
18
|
+
// ---------------------------------------------------------------------------
|
|
19
|
+
/** How much of each stream is buffered. The TAIL is kept: that is where a failure prints. */
|
|
20
|
+
const OUTPUT_CAP_CHARS = 64 * 1024;
|
|
21
|
+
/**
|
|
22
|
+
* The real runner: spawn docker, feed it `stdin` when there is any, and report what happened.
|
|
23
|
+
*
|
|
24
|
+
* Never rejects. Every way a spawn can go wrong is one of the two outcomes, because the caller
|
|
25
|
+
* classifies them differently and an exception would collapse that distinction into whichever
|
|
26
|
+
* `catch` caught it first.
|
|
27
|
+
*/
|
|
28
|
+
export const spawnDockerCommand = (args, opts) => new Promise((resolve) => {
|
|
29
|
+
const logger = opts.logger ?? log;
|
|
30
|
+
if (opts.signal?.aborted) {
|
|
31
|
+
resolve({ outcome: 'failed', reason: abandonedReason(args) });
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
const child = spawn('docker', args, {
|
|
35
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
36
|
+
detached: spawnDetached,
|
|
37
|
+
windowsHide: true,
|
|
38
|
+
});
|
|
39
|
+
let stdout = '';
|
|
40
|
+
let stderr = '';
|
|
41
|
+
let settled = false;
|
|
42
|
+
const finish = (result) => {
|
|
43
|
+
if (settled)
|
|
44
|
+
return;
|
|
45
|
+
settled = true;
|
|
46
|
+
clearTimeout(timer);
|
|
47
|
+
opts.signal?.removeEventListener('abort', onAbort);
|
|
48
|
+
resolve(result);
|
|
49
|
+
};
|
|
50
|
+
const timer = setTimeout(() => {
|
|
51
|
+
logger.warn('docker: command did not answer in time, killing it', {
|
|
52
|
+
command: args[0] ?? '',
|
|
53
|
+
timeoutMs: opts.timeoutMs,
|
|
54
|
+
});
|
|
55
|
+
killChildProcess(child, undefined, logger);
|
|
56
|
+
finish({
|
|
57
|
+
outcome: 'failed',
|
|
58
|
+
reason: `\`docker ${args[0] ?? ''}\` did not answer within ${Math.round(opts.timeoutMs / 1000)}s`,
|
|
59
|
+
});
|
|
60
|
+
}, opts.timeoutMs);
|
|
61
|
+
timer.unref?.();
|
|
62
|
+
const onAbort = () => {
|
|
63
|
+
killChildProcess(child, undefined, logger);
|
|
64
|
+
finish({ outcome: 'failed', reason: abandonedReason(args) });
|
|
65
|
+
};
|
|
66
|
+
opts.signal?.addEventListener('abort', onAbort, { once: true });
|
|
67
|
+
// The tail, not the head: a `docker run` that failed says why in its last lines, and the
|
|
68
|
+
// marker a passing one prints is the whole of its output anyway.
|
|
69
|
+
child.stdout.on('data', (chunk) => {
|
|
70
|
+
stdout = (stdout + chunk.toString('utf8')).slice(-OUTPUT_CAP_CHARS);
|
|
71
|
+
});
|
|
72
|
+
child.stderr.on('data', (chunk) => {
|
|
73
|
+
stderr = (stderr + chunk.toString('utf8')).slice(-OUTPUT_CAP_CHARS);
|
|
74
|
+
});
|
|
75
|
+
child.on('error', (err) => {
|
|
76
|
+
finish({
|
|
77
|
+
outcome: 'failed',
|
|
78
|
+
reason: err.code === 'ENOENT'
|
|
79
|
+
? 'the docker CLI is not on PATH'
|
|
80
|
+
: `the docker CLI could not be spawned (${err.code ?? err.message})`,
|
|
81
|
+
});
|
|
82
|
+
});
|
|
83
|
+
child.on('close', (code) => finish({ outcome: 'ran', code: code ?? -1, stdout, stderr }));
|
|
84
|
+
// A daemon that dies mid-load closes the pipe under us; `close` above already reports that,
|
|
85
|
+
// so the EPIPE here has nothing to add and must not become an unhandled error event.
|
|
86
|
+
child.stdin.on('error', () => { });
|
|
87
|
+
child.stdin.end(opts.stdin);
|
|
88
|
+
});
|
|
89
|
+
function abandonedReason(args) {
|
|
90
|
+
return `the job was cancelled before \`docker ${args[0] ?? ''}\` answered`;
|
|
91
|
+
}
|