@celilo/e2e 0.11.3 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/bin/e2e-status +1 -1
- package/bin/e2e-up +2 -2
- package/config/dns/tangohost.com.zone +1 -1
- package/config/dns/templates/example.net.zone +3 -3
- package/config/dns/templates/iamtheinternet.org.zone +2 -2
- package/config/resolver/unbound.conf +4 -0
- package/config/routing/fw-ext-routes.sh +8 -8
- package/config/routing/fw-isp-routes.sh +2 -2
- package/config/routing/fw-main-routes.sh +5 -5
- package/config/routing/management-routes.sh +1 -1
- package/config/routing/observer-setup.sh +1 -1
- package/config/routing/public-resolver-routes.sh +33 -0
- package/config/routing/public-sim-entrypoint.sh +1 -1
- package/config/routing/resolver-internal-routes.sh +7 -2
- package/config/routing/resolver-routes.sh +1 -1
- package/config/routing/target-routes.sh +1 -1
- package/config/routing/target-setup.sh +1 -1
- package/config/socks/startup.sh +2 -2
- package/docker/Dockerfile.ip-echo +20 -0
- package/docker/Dockerfile.resolver +5 -1
- package/package.json +5 -4
- package/simulators/greenwave/server.ts +35 -0
- package/simulators/greenwave/state.ts +79 -7
- package/simulators/ip-echo/server.ts +76 -0
- package/src/address-plan.test.ts +4 -4
- package/src/cli/build.ts +30 -0
- package/src/cli/command-registry.ts +19 -0
- package/src/cli/completion.ts +16 -1
- package/src/cli/index.ts +85 -4
- package/src/container-manager.ts +75 -12
- package/src/docker-compose-generator.ts +97 -29
- package/src/doctor.test.ts +279 -0
- package/src/doctor.ts +421 -0
- package/src/extract-failure.ts +41 -0
- package/src/index.ts +12 -0
- package/src/last-run.test.ts +62 -0
- package/src/last-run.ts +54 -0
- package/src/network-builder.ts +11 -0
- package/src/observer.test.ts +2 -2
- package/src/observer.ts +1 -1
- package/src/router-swap.ts +121 -0
- package/src/run-lock.test.ts +73 -2
- package/src/run-lock.ts +110 -14
- package/src/runner.ts +99 -5
- package/src/simulator-ips.ts +21 -0
- package/src/socks-proxy.ts +2 -2
- package/src/types.ts +34 -4
- package/src/vantage.test.ts +1 -1
- package/src/zone-classifier.test.ts +11 -16
- package/src/zone-classifier.ts +16 -31
package/src/doctor.ts
ADDED
|
@@ -0,0 +1,421 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* cele2e doctor — "is my environment sane?" as one mechanical answer.
|
|
3
|
+
*
|
|
4
|
+
* Every check here exists because its absence was diagnosed as a product bug
|
|
5
|
+
* first. They share a shape: the mistake is silent, and the failure surfaces
|
|
6
|
+
* later and somewhere unrelated. A missing bake shows up as an SSH error
|
|
7
|
+
* against a firewall IP; a pruned base image shows up as a TLS handshake
|
|
8
|
+
* timeout 16 images into a build. Each one is cheap to detect BEFORE a run
|
|
9
|
+
* starts, so `run` calls this as its implicit preflight and refuses rather
|
|
10
|
+
* than starting something that cannot succeed.
|
|
11
|
+
*
|
|
12
|
+
* Docker access is behind DoctorProbe (Rule 2.3) so every check is unit-
|
|
13
|
+
* testable with the condition deliberately broken — which is the point: a gate
|
|
14
|
+
* nobody has seen fail is not a gate (Rule 7.6).
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { execFileSync } from 'node:child_process';
|
|
18
|
+
import { existsSync, readFileSync, readdirSync } from 'node:fs';
|
|
19
|
+
import { join } from 'node:path';
|
|
20
|
+
import {
|
|
21
|
+
type LockHolder,
|
|
22
|
+
type LockStatus,
|
|
23
|
+
formatAge,
|
|
24
|
+
heartbeatAgeMs,
|
|
25
|
+
isSameSession,
|
|
26
|
+
isSuspect,
|
|
27
|
+
lockStatus,
|
|
28
|
+
} from './run-lock';
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Images the per-test compose references by `image:` alone — it has no `build:`
|
|
32
|
+
* for them, so docker CANNOT produce them on demand. `cele2e build-infra` bakes
|
|
33
|
+
* both (install.sh, then `docker commit`). Missing → the run is doomed.
|
|
34
|
+
*/
|
|
35
|
+
export const BAKED_MANAGEMENT_IMAGES = [
|
|
36
|
+
'celilo-e2e/management:latest',
|
|
37
|
+
'celilo-e2e/management:vanilla',
|
|
38
|
+
] as const;
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* The bake commits a container that was started with `sleep infinity`, so it
|
|
42
|
+
* must override Cmd back to the management image's real entrypoint. When that
|
|
43
|
+
* override is lost the container comes up without ever running /startup.sh, the
|
|
44
|
+
* ssh-keys volume is never populated, and every machine the test adds fails
|
|
45
|
+
* with "Cannot connect to root@<ip> with provided SSH key" — an error that
|
|
46
|
+
* reads as a network or firewall bug and is neither (ce-um6).
|
|
47
|
+
*/
|
|
48
|
+
export const EXPECTED_MANAGEMENT_CMD = '/startup.sh';
|
|
49
|
+
|
|
50
|
+
/** Warn above this much reclaimable image space — the pressure that makes people prune. */
|
|
51
|
+
const RECLAIMABLE_WARN_BYTES = 20 * 1024 ** 3;
|
|
52
|
+
|
|
53
|
+
export type CheckStatus = 'ok' | 'warn' | 'fail';
|
|
54
|
+
|
|
55
|
+
export interface DoctorCheck {
|
|
56
|
+
/** Stable machine-readable id, e.g. 'management-image'. */
|
|
57
|
+
name: string;
|
|
58
|
+
status: CheckStatus;
|
|
59
|
+
/** What was actually found. */
|
|
60
|
+
detail: string;
|
|
61
|
+
/** The exact command that fixes it. Present whenever status is not 'ok'. */
|
|
62
|
+
remedy?: string;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export interface DoctorReport {
|
|
66
|
+
checks: DoctorCheck[];
|
|
67
|
+
/** True when no check failed. Warnings do not block a run. */
|
|
68
|
+
ok: boolean;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Every Docker fact doctor needs, injectable so the checks are testable. */
|
|
72
|
+
export interface DoctorProbe {
|
|
73
|
+
/** Is this image present in the local store? An untagged ref means `:latest`. */
|
|
74
|
+
imageExists(ref: string): boolean;
|
|
75
|
+
/** The image's configured Cmd, or null when the image is absent. */
|
|
76
|
+
imageCmd(ref: string): string[] | null;
|
|
77
|
+
/** Names of leftover `celilo-e2e-*` containers. */
|
|
78
|
+
staleContainers(): string[];
|
|
79
|
+
/** Bytes docker reports as reclaimable image space. */
|
|
80
|
+
reclaimableImageBytes(): number;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// ─── Dockerfile parsing ──────────────────────────────────────────────
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* External base images a Dockerfile pulls from, ignoring references to its own
|
|
87
|
+
* earlier build stages (`FROM x AS fetch` … `COPY --from=fetch`). Pure so the
|
|
88
|
+
* multi-stage and `--platform=` forms both stay covered by a unit test.
|
|
89
|
+
*/
|
|
90
|
+
export function parseBaseImages(dockerfile: string): string[] {
|
|
91
|
+
const stages = new Set<string>();
|
|
92
|
+
const bases: string[] = [];
|
|
93
|
+
for (const raw of dockerfile.split('\n')) {
|
|
94
|
+
const line = raw.trim();
|
|
95
|
+
const match = line.match(/^FROM\s+(.*)$/i);
|
|
96
|
+
if (!match) continue;
|
|
97
|
+
const tokens = match[1].split(/\s+/).filter((t) => !t.startsWith('--'));
|
|
98
|
+
const ref = tokens[0];
|
|
99
|
+
if (!ref) continue;
|
|
100
|
+
const asIdx = tokens.findIndex((t) => t.toLowerCase() === 'as');
|
|
101
|
+
if (asIdx >= 0 && tokens[asIdx + 1]) stages.add(tokens[asIdx + 1]);
|
|
102
|
+
if (!stages.has(ref) && !bases.includes(ref)) bases.push(ref);
|
|
103
|
+
}
|
|
104
|
+
return bases;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Map `docker/Dockerfile.observer` → `celilo-e2e/observer` (docker-compose-generator's imageTag). */
|
|
108
|
+
function derivedImageTag(dockerfileName: string): string {
|
|
109
|
+
return `celilo-e2e/${dockerfileName.replace(/^Dockerfile\./, '')}`;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export interface DockerfileBases {
|
|
113
|
+
/** The `celilo-e2e/<name>` tag this Dockerfile produces. */
|
|
114
|
+
derived: string;
|
|
115
|
+
bases: string[];
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** Read every `docker/Dockerfile.*` and pair its produced tag with its base images. */
|
|
119
|
+
export function readDockerfileBases(pkgDir: string): DockerfileBases[] {
|
|
120
|
+
const dir = join(pkgDir, 'docker');
|
|
121
|
+
if (!existsSync(dir)) return [];
|
|
122
|
+
return readdirSync(dir)
|
|
123
|
+
.filter((f) => f.startsWith('Dockerfile.'))
|
|
124
|
+
.sort()
|
|
125
|
+
.map((f) => ({
|
|
126
|
+
derived: derivedImageTag(f),
|
|
127
|
+
bases: parseBaseImages(readFileSync(join(dir, f), 'utf-8')),
|
|
128
|
+
}));
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// ─── Checks ──────────────────────────────────────────────────────────
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* The run-lock, including the two states `status` used to render identically to
|
|
135
|
+
* a healthy one: a stale-heartbeat holder (suspect) and this session's own kept
|
|
136
|
+
* stack (friction, not contention — the next run clears it).
|
|
137
|
+
*/
|
|
138
|
+
export function checkRunLock(lock: LockStatus): DoctorCheck {
|
|
139
|
+
if (lock.free || !lock.holder) {
|
|
140
|
+
return { name: 'run-lock', status: 'ok', detail: 'free' };
|
|
141
|
+
}
|
|
142
|
+
const h = lock.holder;
|
|
143
|
+
const age = formatAge(lock.heartbeatAgeMs ?? 0);
|
|
144
|
+
if (lock.suspect) {
|
|
145
|
+
return {
|
|
146
|
+
name: 'run-lock',
|
|
147
|
+
status: 'fail',
|
|
148
|
+
detail: `SUSPECT — ${h.session} running ${h.test} (pid ${h.pid}) has not beaten its heartbeat for ${age}; the process is alive but is not progressing`,
|
|
149
|
+
remedy: `Inspect it (\`docker ps\`, \`kill -0 ${h.pid}\`); if it is wedged, kill ${h.pid} then \`cele2e down\``,
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
if (lock.ownKept) {
|
|
153
|
+
return {
|
|
154
|
+
name: 'run-lock',
|
|
155
|
+
status: 'warn',
|
|
156
|
+
detail: `held by THIS session's own kept stack (${h.test}) — the next run auto-releases it`,
|
|
157
|
+
remedy: 'cele2e release (or `cele2e down` to also tear the stack down)',
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
if (h.state === 'kept') {
|
|
161
|
+
return {
|
|
162
|
+
name: 'run-lock',
|
|
163
|
+
status: 'fail',
|
|
164
|
+
detail: `held by another session's kept stack: ${h.session} (${h.test})`,
|
|
165
|
+
remedy: 'Wait for that session, or `cele2e release` if you know it is abandoned',
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
return {
|
|
169
|
+
name: 'run-lock',
|
|
170
|
+
status: 'fail',
|
|
171
|
+
detail: `busy — ${h.session} running ${h.test} (pid ${h.pid}), heartbeat ${age} ago`,
|
|
172
|
+
remedy: 'Poll with `cele2e status --json` (exit 0 = free, 3 = busy) and wait',
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** The baked management images exist AND still run /startup.sh. */
|
|
177
|
+
export function checkManagementImage(probe: DoctorProbe): DoctorCheck {
|
|
178
|
+
const missing = BAKED_MANAGEMENT_IMAGES.filter((ref) => !probe.imageExists(ref));
|
|
179
|
+
if (missing.length > 0) {
|
|
180
|
+
return {
|
|
181
|
+
name: 'management-image',
|
|
182
|
+
status: 'fail',
|
|
183
|
+
detail: `baked image missing: ${missing.join(', ')} — the test compose references these by tag and cannot build them`,
|
|
184
|
+
remedy: 'cele2e build-infra',
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
const unbaked = BAKED_MANAGEMENT_IMAGES.filter(
|
|
188
|
+
(ref) => !(probe.imageCmd(ref) ?? []).includes(EXPECTED_MANAGEMENT_CMD),
|
|
189
|
+
);
|
|
190
|
+
if (unbaked.length > 0) {
|
|
191
|
+
return {
|
|
192
|
+
name: 'management-image',
|
|
193
|
+
status: 'fail',
|
|
194
|
+
detail: `${unbaked.join(', ')} does not run ${EXPECTED_MANAGEMENT_CMD} — the bake commit lost its Cmd, so the ssh-keys volume will stay empty and every \`machine add\` will fail with "Cannot connect to root@<ip> with provided SSH key"`,
|
|
195
|
+
remedy: 'cele2e build-infra',
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
return {
|
|
199
|
+
name: 'management-image',
|
|
200
|
+
status: 'ok',
|
|
201
|
+
detail: `${BAKED_MANAGEMENT_IMAGES.length} baked images present, both running ${EXPECTED_MANAGEMENT_CMD}`,
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Base images the next build genuinely has to fetch.
|
|
207
|
+
*
|
|
208
|
+
* A base image absent from the local store is only a problem when the image
|
|
209
|
+
* built FROM it is also absent — with the derived tag present, buildkit serves
|
|
210
|
+
* the build from cache and never resolves the reference. Requiring every base
|
|
211
|
+
* unconditionally would refuse environments that work today, so the check is
|
|
212
|
+
* the conjunction: no derived image AND no base image means the build must go
|
|
213
|
+
* to a registry, which is where `failed to solve: ubuntu:22.04: net/http: TLS
|
|
214
|
+
* handshake timeout` comes from — 16 images deep, blamed on the network.
|
|
215
|
+
*/
|
|
216
|
+
export function checkBaseImages(probe: DoctorProbe, dockerfiles: DockerfileBases[]): DoctorCheck {
|
|
217
|
+
const needPull = new Set<string>();
|
|
218
|
+
for (const { derived, bases } of dockerfiles) {
|
|
219
|
+
if (probe.imageExists(derived)) continue;
|
|
220
|
+
for (const base of bases) {
|
|
221
|
+
if (!probe.imageExists(base)) needPull.add(base);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
if (needPull.size === 0) {
|
|
225
|
+
return { name: 'base-images', status: 'ok', detail: 'every image the build needs is local' };
|
|
226
|
+
}
|
|
227
|
+
const refs = [...needPull].sort();
|
|
228
|
+
return {
|
|
229
|
+
name: 'base-images',
|
|
230
|
+
status: 'fail',
|
|
231
|
+
detail: `${refs.length} base image(s) absent locally and needed by an image that must be rebuilt: ${refs.join(', ')}`,
|
|
232
|
+
remedy: refs.map((r) => `docker pull ${r}`).join('\n '),
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/** Leftover containers from a crashed run, which start-of-run cleanup will wipe. */
|
|
237
|
+
export function checkStaleContainers(probe: DoctorProbe, lock: LockStatus): DoctorCheck {
|
|
238
|
+
const names = probe.staleContainers();
|
|
239
|
+
if (names.length === 0) {
|
|
240
|
+
return {
|
|
241
|
+
name: 'stale-containers',
|
|
242
|
+
status: 'ok',
|
|
243
|
+
detail: 'no leftover celilo-e2e-* containers',
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
// With a live holder these are simply that run's containers, not debris.
|
|
247
|
+
if (!lock.free && !lock.ownKept) {
|
|
248
|
+
return {
|
|
249
|
+
name: 'stale-containers',
|
|
250
|
+
status: 'ok',
|
|
251
|
+
detail: `${names.length} celilo-e2e-* container(s) belonging to the current lock holder`,
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
return {
|
|
255
|
+
name: 'stale-containers',
|
|
256
|
+
status: 'warn',
|
|
257
|
+
detail: `${names.length} celilo-e2e-* container(s) with no live run: ${names.slice(0, 4).join(', ')}${names.length > 4 ? ', …' : ''}`,
|
|
258
|
+
remedy: 'cele2e down (start-of-run cleanup also removes these)',
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* Reclaimable image space. A warning, never a failure — the point is to steer
|
|
264
|
+
* the reflex: under disk pressure people reach for `docker image prune`, which
|
|
265
|
+
* deletes the base images the next build needs. Pruning e2e containers by name
|
|
266
|
+
* frees space without costing a 27-image rebuild.
|
|
267
|
+
*/
|
|
268
|
+
export function checkDiskPressure(probe: DoctorProbe): DoctorCheck {
|
|
269
|
+
const bytes = probe.reclaimableImageBytes();
|
|
270
|
+
const gib = (bytes / 1024 ** 3).toFixed(1);
|
|
271
|
+
if (bytes < RECLAIMABLE_WARN_BYTES) {
|
|
272
|
+
return { name: 'disk', status: 'ok', detail: `${gib} GiB reclaimable image space` };
|
|
273
|
+
}
|
|
274
|
+
return {
|
|
275
|
+
name: 'disk',
|
|
276
|
+
status: 'warn',
|
|
277
|
+
detail: `${gib} GiB reclaimable image space — do NOT run \`docker image prune\`; it deletes the base images the next build needs and costs a full rebuild`,
|
|
278
|
+
remedy:
|
|
279
|
+
'docker ps -aq --filter name=celilo-e2e | xargs -r docker rm -f && docker volume prune -f',
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* Turn buildkit's `failed to solve: <ref>: … TLS handshake timeout` into the
|
|
285
|
+
* one sentence that identifies the cause.
|
|
286
|
+
*
|
|
287
|
+
* That message names the network because buildkit was, technically, on the
|
|
288
|
+
* network — but it only went there because the base image was not in the local
|
|
289
|
+
* store, which is what `docker image prune` (the reflex under disk pressure)
|
|
290
|
+
* removes. Reading it as a connectivity problem cost a full 27-image rebuild.
|
|
291
|
+
* checkBaseImages preflights for this, so reaching here means the image went
|
|
292
|
+
* missing mid-run; either way, say what to pull.
|
|
293
|
+
*
|
|
294
|
+
* Returns '' when the failure names a RUN step rather than an image reference —
|
|
295
|
+
* a genuine build error, which needs no translation.
|
|
296
|
+
*/
|
|
297
|
+
export function explainBuildFailure(stderr: string): string {
|
|
298
|
+
const ref = stderr.match(/failed to solve:\s*([^\s:"]+(?::[^\s:"]+)?)/)?.[1];
|
|
299
|
+
if (!ref || ref === 'process') return '';
|
|
300
|
+
return [
|
|
301
|
+
'',
|
|
302
|
+
'',
|
|
303
|
+
`This names "${ref}" — buildkit had to fetch it, which means it is not in the local image store.`,
|
|
304
|
+
`A \`docker image prune\` removes exactly these. Fix: docker pull ${ref}`,
|
|
305
|
+
'Then re-run. `cele2e doctor` checks for this before a run starts.',
|
|
306
|
+
].join('\n');
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
// ─── Real probe ──────────────────────────────────────────────────────
|
|
310
|
+
|
|
311
|
+
function docker(args: string[]): string {
|
|
312
|
+
return execFileSync('docker', args, {
|
|
313
|
+
encoding: 'utf-8',
|
|
314
|
+
stdio: ['ignore', 'pipe', 'ignore'],
|
|
315
|
+
timeout: 15_000,
|
|
316
|
+
}).trim();
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
export function createDockerProbe(): DoctorProbe {
|
|
320
|
+
// One `docker images` listing answers every existence question; per-image
|
|
321
|
+
// inspect calls would be ~40 process spawns on a preflight that must be fast.
|
|
322
|
+
let tags: Set<string> | null = null;
|
|
323
|
+
const knownTags = (): Set<string> => {
|
|
324
|
+
if (tags) return tags;
|
|
325
|
+
try {
|
|
326
|
+
tags = new Set(
|
|
327
|
+
docker(['images', '--format', '{{.Repository}}:{{.Tag}}'])
|
|
328
|
+
.split('\n')
|
|
329
|
+
.map((l) => l.trim())
|
|
330
|
+
.filter(Boolean),
|
|
331
|
+
);
|
|
332
|
+
} catch {
|
|
333
|
+
tags = new Set();
|
|
334
|
+
}
|
|
335
|
+
return tags;
|
|
336
|
+
};
|
|
337
|
+
|
|
338
|
+
return {
|
|
339
|
+
imageExists(ref) {
|
|
340
|
+
const withTag = ref.includes(':') ? ref : `${ref}:latest`;
|
|
341
|
+
return knownTags().has(withTag);
|
|
342
|
+
},
|
|
343
|
+
imageCmd(ref) {
|
|
344
|
+
try {
|
|
345
|
+
return JSON.parse(docker(['image', 'inspect', ref, '--format', '{{json .Config.Cmd}}']));
|
|
346
|
+
} catch {
|
|
347
|
+
return null;
|
|
348
|
+
}
|
|
349
|
+
},
|
|
350
|
+
staleContainers() {
|
|
351
|
+
try {
|
|
352
|
+
return docker(['ps', '-a', '--filter', 'name=celilo-e2e', '--format', '{{.Names}}'])
|
|
353
|
+
.split('\n')
|
|
354
|
+
.map((l) => l.trim())
|
|
355
|
+
.filter(Boolean);
|
|
356
|
+
} catch {
|
|
357
|
+
return [];
|
|
358
|
+
}
|
|
359
|
+
},
|
|
360
|
+
reclaimableImageBytes() {
|
|
361
|
+
try {
|
|
362
|
+
for (const line of docker(['system', 'df', '--format', '{{json .}}']).split('\n')) {
|
|
363
|
+
const row = JSON.parse(line) as { Type?: string; Reclaimable?: string };
|
|
364
|
+
if (row.Type !== 'Images') continue;
|
|
365
|
+
// Reclaimable looks like "12.3GB (48%)".
|
|
366
|
+
const m = row.Reclaimable?.match(/^([\d.]+)\s*([KMGT]?)B/i);
|
|
367
|
+
if (!m) return 0;
|
|
368
|
+
const scale = { '': 1, K: 1024, M: 1024 ** 2, G: 1024 ** 3, T: 1024 ** 4 }[
|
|
369
|
+
m[2].toUpperCase()
|
|
370
|
+
];
|
|
371
|
+
return Number.parseFloat(m[1]) * (scale ?? 1);
|
|
372
|
+
}
|
|
373
|
+
} catch {}
|
|
374
|
+
return 0;
|
|
375
|
+
},
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
// ─── Report ──────────────────────────────────────────────────────────
|
|
380
|
+
|
|
381
|
+
export interface DiagnoseOptions {
|
|
382
|
+
pkgDir: string;
|
|
383
|
+
probe?: DoctorProbe;
|
|
384
|
+
lock?: LockStatus;
|
|
385
|
+
/** Skip the run-lock check — `run` acquires the lock itself and reports contention its own way. */
|
|
386
|
+
skipLock?: boolean;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
export function diagnose(options: DiagnoseOptions): DoctorReport {
|
|
390
|
+
const probe = options.probe ?? createDockerProbe();
|
|
391
|
+
const lock = options.lock ?? lockStatus();
|
|
392
|
+
const checks: DoctorCheck[] = [];
|
|
393
|
+
if (!options.skipLock) checks.push(checkRunLock(lock));
|
|
394
|
+
checks.push(checkManagementImage(probe));
|
|
395
|
+
checks.push(checkBaseImages(probe, readDockerfileBases(options.pkgDir)));
|
|
396
|
+
checks.push(checkStaleContainers(probe, lock));
|
|
397
|
+
checks.push(checkDiskPressure(probe));
|
|
398
|
+
return { checks, ok: !checks.some((c) => c.status === 'fail') };
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
const ICONS: Record<CheckStatus, string> = { ok: '✓', warn: '!', fail: '✗' };
|
|
402
|
+
|
|
403
|
+
/** Render a report as the lines `doctor` prints and `run` prints on refusal. */
|
|
404
|
+
export function formatReport(report: DoctorReport): string[] {
|
|
405
|
+
const lines: string[] = [];
|
|
406
|
+
for (const c of report.checks) {
|
|
407
|
+
lines.push(` ${ICONS[c.status]} ${c.name.padEnd(18)} ${c.detail}`);
|
|
408
|
+
if (c.remedy) lines.push(` fix: ${c.remedy}`);
|
|
409
|
+
}
|
|
410
|
+
return lines;
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
/** The holder line `status` prints, with heartbeat age always visible. */
|
|
414
|
+
export function formatHolderLine(h: LockHolder): string {
|
|
415
|
+
const flags = [
|
|
416
|
+
isSuspect(h) ? 'SUSPECT' : null,
|
|
417
|
+
h.state === 'kept' && isSameSession(h) ? 'yours' : null,
|
|
418
|
+
].filter(Boolean);
|
|
419
|
+
const suffix = flags.length ? ` [${flags.join(', ')}]` : '';
|
|
420
|
+
return `${h.session} — ${h.state} ${h.test} (pid ${h.pid}), heartbeat ${formatAge(heartbeatAgeMs(h))} ago${suffix}`;
|
|
421
|
+
}
|
package/src/extract-failure.ts
CHANGED
|
@@ -12,6 +12,47 @@ export function stripAnsi(s: string): string {
|
|
|
12
12
|
return s.replace(ANSI_RE, '');
|
|
13
13
|
}
|
|
14
14
|
|
|
15
|
+
export interface StageTally {
|
|
16
|
+
/** Stages that failed on their own merits. */
|
|
17
|
+
failed: number;
|
|
18
|
+
/** Stages that never ran because an earlier stage had already failed. */
|
|
19
|
+
skipped: number;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Split a staged suite's failures into real ones and cascade-skips.
|
|
24
|
+
*
|
|
25
|
+
* Staged e2e suites guard every stage with `requireStage`, which throws
|
|
26
|
+
* `Skipped: <stage> — <earlier error>` once anything upstream has failed. bun
|
|
27
|
+
* counts each of those as a failure, so ONE bad fixture line in stage 1 reports
|
|
28
|
+
* as "9 failed" in a 10-stage suite — nine counts of a defect that does not
|
|
29
|
+
* exist, and a summary that buries the one that does.
|
|
30
|
+
*
|
|
31
|
+
* The `Skipped:` marker is the semantic signal, so that is what we key on; the
|
|
32
|
+
* sub-millisecond durations those stages show are a symptom, not the contract.
|
|
33
|
+
*/
|
|
34
|
+
export function tallyStages(lines: string[]): StageTally {
|
|
35
|
+
const plain = lines.map(stripAnsi);
|
|
36
|
+
const isFailure = (l: string): boolean => /^\(fail\)\s+\S/.test(l);
|
|
37
|
+
let failed = 0;
|
|
38
|
+
let skipped = 0;
|
|
39
|
+
for (let i = 0; i < plain.length; i++) {
|
|
40
|
+
if (!isFailure(plain[i])) continue;
|
|
41
|
+
// bun prints the reason on the following lines, indented under the failure.
|
|
42
|
+
// The window MUST stop at the next failure: cascade-skips come in runs, and
|
|
43
|
+
// a window that reads past the boundary attributes the next stage's
|
|
44
|
+
// "Skipped:" to this one — which misreads the single real defect at the top
|
|
45
|
+
// of a cascade as just another skip, losing the only line worth reading.
|
|
46
|
+
const reason: string[] = [];
|
|
47
|
+
for (let j = i + 1; j < plain.length && j <= i + 4 && !isFailure(plain[j]); j++) {
|
|
48
|
+
reason.push(plain[j]);
|
|
49
|
+
}
|
|
50
|
+
if (/(?:^|\W)Skipped:\s/.test(reason.join('\n'))) skipped++;
|
|
51
|
+
else failed++;
|
|
52
|
+
}
|
|
53
|
+
return { failed, skipped };
|
|
54
|
+
}
|
|
55
|
+
|
|
15
56
|
export function extractFailureMessage(lines: string[], exitCode: number): string {
|
|
16
57
|
const plain = lines.map(stripAnsi);
|
|
17
58
|
|
package/src/index.ts
CHANGED
|
@@ -102,3 +102,15 @@ export {
|
|
|
102
102
|
SHARED_PROJECT_NAME,
|
|
103
103
|
SHARED_NETWORKS,
|
|
104
104
|
} from './docker-compose-generator';
|
|
105
|
+
|
|
106
|
+
// Runtime router-device swap — the harness half of "the ISP replaced the box"
|
|
107
|
+
// (openspec/changes/module-pause-lifecycle task 7.7).
|
|
108
|
+
export {
|
|
109
|
+
ROUTER_DEVICES,
|
|
110
|
+
type RouterDevice,
|
|
111
|
+
type RouterSwapResult,
|
|
112
|
+
forwardedPorts,
|
|
113
|
+
listRouterForwards,
|
|
114
|
+
routerDhcpDnsServers,
|
|
115
|
+
swapRouterDevice,
|
|
116
|
+
} from './router-swap';
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { afterEach, beforeEach, expect, test } from 'bun:test';
|
|
2
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { type LastRun, readLastRun, writeLastRun } from './last-run';
|
|
6
|
+
|
|
7
|
+
let dir: string;
|
|
8
|
+
|
|
9
|
+
beforeEach(() => {
|
|
10
|
+
dir = mkdtempSync(join(tmpdir(), 'e2e-last-'));
|
|
11
|
+
process.env.CELILO_E2E_LAST_RUN_PATH = join(dir, 'last-run.json');
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
afterEach(() => {
|
|
15
|
+
rmSync(dir, { recursive: true, force: true });
|
|
16
|
+
delete process.env.CELILO_E2E_LAST_RUN_PATH;
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
const RUN: LastRun = {
|
|
20
|
+
runId: 'abc-123',
|
|
21
|
+
resultsDir: '/repo/e2e/results/2026-08-12T10-00-00',
|
|
22
|
+
startedAt: '2026-08-12T17:00:00.000Z',
|
|
23
|
+
status: 'completed',
|
|
24
|
+
total: 3,
|
|
25
|
+
passed: 3,
|
|
26
|
+
failed: 0,
|
|
27
|
+
skipped: 0,
|
|
28
|
+
durationMs: 600_000,
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
test('with no run recorded, readLastRun is null rather than throwing', () => {
|
|
32
|
+
expect(readLastRun()).toBeNull();
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
test('a recorded run round-trips every field `cele2e last --json` promises', () => {
|
|
36
|
+
writeLastRun(RUN);
|
|
37
|
+
expect(readLastRun()).toEqual(RUN);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test('the record is overwritten, so `last` never returns a stale earlier run', () => {
|
|
41
|
+
// The whole point: inferring "my results dir" from `ls -t results | head -1`
|
|
42
|
+
// hands back the PREVIOUS run's numbers when a run refuses to start, and a
|
|
43
|
+
// suite that never executed then reads as a clean pass.
|
|
44
|
+
writeLastRun(RUN);
|
|
45
|
+
writeLastRun({ ...RUN, runId: 'def-456', resultsDir: '/repo/e2e/results/later', failed: 2 });
|
|
46
|
+
const last = readLastRun();
|
|
47
|
+
expect(last?.runId).toBe('def-456');
|
|
48
|
+
expect(last?.resultsDir).toBe('/repo/e2e/results/later');
|
|
49
|
+
expect(last?.failed).toBe(2);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
test('a run still in flight is recorded as running, so its logs are findable', () => {
|
|
53
|
+
writeLastRun({ ...RUN, status: 'running', passed: 0, total: 3 });
|
|
54
|
+
expect(readLastRun()?.status).toBe('running');
|
|
55
|
+
expect(readLastRun()?.resultsDir).toBe(RUN.resultsDir);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test('a corrupt record is null, not fatal', () => {
|
|
59
|
+
writeLastRun(RUN);
|
|
60
|
+
Bun.write(process.env.CELILO_E2E_LAST_RUN_PATH as string, 'not json{');
|
|
61
|
+
expect(readLastRun()).toBeNull();
|
|
62
|
+
});
|
package/src/last-run.ts
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A machine-global pointer to the most recent cele2e run.
|
|
3
|
+
*
|
|
4
|
+
* Without one, "how did my run go?" is answered by `ls -t e2e/results | head -1`
|
|
5
|
+
* — which returns the newest directory, not YOUR run. When a run refuses to
|
|
6
|
+
* start (a held lock, a failed preflight) that inference silently hands back the
|
|
7
|
+
* PREVIOUS run's numbers, and a suite that never executed reads as a clean pass.
|
|
8
|
+
*
|
|
9
|
+
* Written at run start (so a run that dies mid-way still has a findable results
|
|
10
|
+
* dir) and again at the end with the counts. Lives beside the run-lock rather
|
|
11
|
+
* than in a worktree, because which checkout started the run is not something
|
|
12
|
+
* the reader knows.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
16
|
+
import { homedir } from 'node:os';
|
|
17
|
+
import { dirname, join } from 'node:path';
|
|
18
|
+
|
|
19
|
+
export interface LastRun {
|
|
20
|
+
runId: string;
|
|
21
|
+
resultsDir: string;
|
|
22
|
+
startedAt: string;
|
|
23
|
+
/** 'running' until the suite finishes; then 'completed' or 'failed'. */
|
|
24
|
+
status: 'running' | 'completed' | 'failed';
|
|
25
|
+
total: number;
|
|
26
|
+
passed: number;
|
|
27
|
+
failed: number;
|
|
28
|
+
/** Stages a failing earlier stage blocked. Distinct from real failures. */
|
|
29
|
+
skipped: number;
|
|
30
|
+
durationMs: number;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function lastRunPath(): string {
|
|
34
|
+
return (
|
|
35
|
+
process.env.CELILO_E2E_LAST_RUN_PATH || join(homedir(), '.cache', 'celilo-e2e', 'last-run.json')
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function writeLastRun(run: LastRun): void {
|
|
40
|
+
try {
|
|
41
|
+
mkdirSync(dirname(lastRunPath()), { recursive: true });
|
|
42
|
+
writeFileSync(lastRunPath(), `${JSON.stringify(run, null, 2)}\n`);
|
|
43
|
+
} catch {
|
|
44
|
+
// ponytail: a bookkeeping pointer must never be the thing that fails a run.
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function readLastRun(): LastRun | null {
|
|
49
|
+
try {
|
|
50
|
+
return JSON.parse(readFileSync(lastRunPath(), 'utf-8')) as LastRun;
|
|
51
|
+
} catch {
|
|
52
|
+
return null;
|
|
53
|
+
}
|
|
54
|
+
}
|
package/src/network-builder.ts
CHANGED
|
@@ -59,6 +59,17 @@ export class NetworkBuilder {
|
|
|
59
59
|
return this;
|
|
60
60
|
}
|
|
61
61
|
|
|
62
|
+
/**
|
|
63
|
+
* Make the `fw-isp` simulator stand in for an Axon Networks Q1000K rather
|
|
64
|
+
* than the default GreenWave C4000XG. The two speak an identical CGI
|
|
65
|
+
* protocol; only the TR-181 vendor extension prefix differs, so this swaps
|
|
66
|
+
* `X_GWS_Via`/`X_LANTIQ_COM_INTERFACE` for `X_AXON_Via`/`X_AXON_INTERFACE`.
|
|
67
|
+
*/
|
|
68
|
+
axonRouter(): this {
|
|
69
|
+
this.config.routerVendorPrefix = 'X_AXON_';
|
|
70
|
+
return this;
|
|
71
|
+
}
|
|
72
|
+
|
|
62
73
|
/**
|
|
63
74
|
* Place the celilo management container in a given zone. Defaults to
|
|
64
75
|
* `internal`; `secure-mgmt` gives celilo-mgr its own control-plane network
|
package/src/observer.test.ts
CHANGED
|
@@ -44,14 +44,14 @@ describe('placement faithfulness (the load-bearing invariants)', () => {
|
|
|
44
44
|
});
|
|
45
45
|
|
|
46
46
|
test('publicInternet uses only the public resolver (never the internal split-horizon view)', () => {
|
|
47
|
-
expect(OBSERVER_PLACEMENTS.publicInternet.resolvers).toEqual(['
|
|
47
|
+
expect(OBSERVER_PLACEMENTS.publicInternet.resolvers).toEqual(['203.0.113.1']);
|
|
48
48
|
});
|
|
49
49
|
|
|
50
50
|
test('observerEnv serializes the routing profile for the setup script', () => {
|
|
51
51
|
const env = observerEnv(OBSERVER_PLACEMENTS.internalDevice);
|
|
52
52
|
expect(env.OBSERVER_INTERZONE).toBe('0');
|
|
53
53
|
expect(env.OBSERVER_GATEWAY).toBe(OBSERVER_PLACEMENTS.internalDevice.gateway);
|
|
54
|
-
expect(env.OBSERVER_RESOLVERS).toContain('
|
|
54
|
+
expect(env.OBSERVER_RESOLVERS).toContain('203.0.113.1');
|
|
55
55
|
});
|
|
56
56
|
});
|
|
57
57
|
|
package/src/observer.ts
CHANGED
|
@@ -47,7 +47,7 @@ export interface ObserverPlacement {
|
|
|
47
47
|
// NO route to the segmented zones, so a dmz/app/secure container IP is unreachable.
|
|
48
48
|
const HOME_ROUTER = '10.226.1.1';
|
|
49
49
|
const INTERNAL_RESOLVER = '10.226.1.10';
|
|
50
|
-
const PUBLIC_RESOLVER = '
|
|
50
|
+
const PUBLIC_RESOLVER = '203.0.113.1';
|
|
51
51
|
const INTERNET_GATEWAY = '100.64.0.1'; // fw-ext, on internet-external
|
|
52
52
|
|
|
53
53
|
/** Derive an observer host IP in a zone from its gateway (no new literal subnets). */
|