@celilo/e2e 0.20.5 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/e2e-bake-management +362 -10
- package/config/routing/management-routes.sh +11 -0
- package/config/routing/minio-startup.sh +27 -32
- package/config/routing/resolver-internal-routes.sh +14 -6
- package/config/socks/startup.sh +46 -0
- package/config/ssh/sshd-celilo-api.conf +6 -0
- package/docker/Dockerfile.management +47 -0
- package/docker/Dockerfile.minio +34 -16
- package/docker/Dockerfile.socks-proxy +14 -0
- package/package.json +2 -2
- package/registry-server/src/bootstrap.ts +43 -5
- package/registry-server/src/index.ts +1 -0
- package/registry-server/src/server.ts +18 -1
- package/simulators/greenwave/state.ts +85 -4
- package/src/bake-probe-explanation.test.ts +43 -0
- package/src/bake-probe-fails-closed.test.ts +35 -0
- package/src/bake-probe-no-shell-expansion.test.ts +40 -0
- package/src/cli/build.test.ts +50 -0
- package/src/cli/build.ts +36 -1
- package/src/cli/host.ts +71 -2
- package/src/docker-compose-generator.ts +7 -3
- package/src/doctor.test.ts +87 -6
- package/src/doctor.ts +112 -23
- package/src/greenwave-external-interface.test.ts +63 -0
- package/src/index.ts +1 -0
- package/src/installed-cli-digest.test.ts +113 -0
- package/src/installed-cli-digest.ts +102 -0
- package/src/parse-line.ts +8 -2
- package/src/resolver-internal-config.test.ts +65 -0
- package/src/shared-infra-pebble-root.test.ts +26 -0
- package/src/shared-infra-self-contained.test.ts +53 -0
- package/src/shared-infra.ts +61 -0
- package/src/socks-proxy.ts +45 -11
- package/src/stale-acme-root.ts +66 -0
- package/src/types.ts +8 -0
package/src/cli/build.ts
CHANGED
|
@@ -34,7 +34,7 @@ import { basename, join, resolve } from 'node:path';
|
|
|
34
34
|
import { gunzipSync } from 'node:zlib';
|
|
35
35
|
import { stageAptRepo } from '../../scripts/stage-apt-repo';
|
|
36
36
|
import { stageLibsignal } from '../../scripts/stage-libsignal';
|
|
37
|
-
import { explainBuildFailure } from '../doctor';
|
|
37
|
+
import { explainBuildFailure, readDockerfileBases } from '../doctor';
|
|
38
38
|
import { isNetappCurrent } from '../netapp-staleness';
|
|
39
39
|
import { ensureRegistryServerBundle, ensureTerraformFakeBundle } from '../registry-bundle';
|
|
40
40
|
import { findMonorepoRoot } from '../repo-root';
|
|
@@ -582,6 +582,37 @@ export async function stageNetappsFromRegistry(netappsDir: string): Promise<void
|
|
|
582
582
|
console.log('');
|
|
583
583
|
}
|
|
584
584
|
|
|
585
|
+
/**
|
|
586
|
+
* Pull every base image a `docker/Dockerfile.*` names into the local image
|
|
587
|
+
* store, so buildkit resolves each FROM locally instead of against a registry.
|
|
588
|
+
*
|
|
589
|
+
* This is the one-command fix for the base-images warn (celilo#1244). It is
|
|
590
|
+
* cheap — the layers are usually already in buildkit's cache, so a pull is ~2s
|
|
591
|
+
* and reports `Already exists` — and it removes the failure mode where a
|
|
592
|
+
* registry blip kills a build 30 seconds in with an error naming the network.
|
|
593
|
+
*
|
|
594
|
+
* Best-effort per image: a pull that fails leaves the build to try the same
|
|
595
|
+
* resolution itself, which is exactly today's behaviour, so refusing here would
|
|
596
|
+
* only make build-infra more brittle than the thing it is protecting.
|
|
597
|
+
*/
|
|
598
|
+
export function pullDockerfileBases(pkgDir: string): string[] {
|
|
599
|
+
const bases = [...new Set(readDockerfileBases(pkgDir).flatMap((d) => d.bases))].sort();
|
|
600
|
+
if (bases.length === 0) return [];
|
|
601
|
+
console.log(`${bold}Pulling ${bases.length} Docker base images...${reset}`);
|
|
602
|
+
const failed: string[] = [];
|
|
603
|
+
for (const ref of bases) {
|
|
604
|
+
const result = spawnSync('docker', ['pull', '--quiet', ref], { stdio: 'ignore' });
|
|
605
|
+
if (result.status !== 0) failed.push(ref);
|
|
606
|
+
}
|
|
607
|
+
if (failed.length > 0) {
|
|
608
|
+
console.error(
|
|
609
|
+
`${yellow}⚠ could not pull ${failed.join(', ')} — the build will try the same resolution itself${reset}`,
|
|
610
|
+
);
|
|
611
|
+
}
|
|
612
|
+
console.log('');
|
|
613
|
+
return failed;
|
|
614
|
+
}
|
|
615
|
+
|
|
585
616
|
function buildDockerImages(pkgDir: string): void {
|
|
586
617
|
// Refresh the bundled registry-server source before docker build —
|
|
587
618
|
// Dockerfile.registry copies from <pkgDir>/registry-server, which is
|
|
@@ -900,6 +931,10 @@ export async function runBuild(options: BuildOptions): Promise<void> {
|
|
|
900
931
|
process.exit(1);
|
|
901
932
|
}
|
|
902
933
|
|
|
934
|
+
// Before any build: put every base in the image store, so buildkit never has
|
|
935
|
+
// to ask a registry mid-build (celilo#1244).
|
|
936
|
+
pullDockerfileBases(pkgDir);
|
|
937
|
+
|
|
903
938
|
console.log(`${bold}Building E2E Docker images...${reset}\n`);
|
|
904
939
|
buildDockerImages(pkgDir);
|
|
905
940
|
|
package/src/cli/host.ts
CHANGED
|
@@ -75,6 +75,68 @@ export function colimaStartArgs(budget: HostVmBudget, profile: string): string[]
|
|
|
75
75
|
];
|
|
76
76
|
}
|
|
77
77
|
|
|
78
|
+
/**
|
|
79
|
+
* The binfmt_misc handler amd64 emulation registers in the VM kernel.
|
|
80
|
+
* `tonistiigi/binfmt --install amd64` writes exactly this name.
|
|
81
|
+
*/
|
|
82
|
+
const AMD64_BINFMT_HANDLER = '/proc/sys/fs/binfmt_misc/qemu-x86_64';
|
|
83
|
+
|
|
84
|
+
/** Is amd64 emulation registered in the VM right now? */
|
|
85
|
+
function haveAmd64Emulation(profile: string): boolean {
|
|
86
|
+
// stdio ignored on purpose: colima ssh writes config warnings to stderr on
|
|
87
|
+
// every invocation, and only the exit status answers the question.
|
|
88
|
+
return (
|
|
89
|
+
spawnSync('colima', ['ssh', '-p', profile, '--', 'test', '-f', AMD64_BINFMT_HANDLER], {
|
|
90
|
+
stdio: 'ignore',
|
|
91
|
+
}).status === 0
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Register amd64 emulation in the VM, idempotently.
|
|
97
|
+
*
|
|
98
|
+
* WHY THIS LIVES IN `host up` AND NOT IN A README. binfmt_misc registrations
|
|
99
|
+
* are kernel state inside the colima VM, so they do NOT survive a VM restart —
|
|
100
|
+
* and nothing about the resulting failure names emulation. `build-infra` dies
|
|
101
|
+
* building the signal image, whose Dockerfile pins `--platform=linux/amd64`,
|
|
102
|
+
* with an exec-format error that reads like a broken base image. It only ever
|
|
103
|
+
* appeared to work because CI builds on an amd64 runner and a warm local cache
|
|
104
|
+
* hid it on this arm64 Mac.
|
|
105
|
+
*
|
|
106
|
+
* Installing costs one small privileged container. The check above costs an
|
|
107
|
+
* ssh, so the install runs only when the handler is genuinely absent — which,
|
|
108
|
+
* on a VM that has just been created or restarted, it always is.
|
|
109
|
+
*
|
|
110
|
+
* Non-fatal by design: a host with no amd64 module still runs every suite that
|
|
111
|
+
* does not build the signal image, and failing `host up` over it would be a
|
|
112
|
+
* worse trade than the warning.
|
|
113
|
+
*/
|
|
114
|
+
function ensureAmd64Emulation(profile: string): void {
|
|
115
|
+
// On an x64 host amd64 is native, so no handler is registered and none is
|
|
116
|
+
// wanted — without this guard the install would re-run on every `host up`.
|
|
117
|
+
if (process.arch === 'x64') return;
|
|
118
|
+
if (haveAmd64Emulation(profile)) return;
|
|
119
|
+
|
|
120
|
+
console.log(
|
|
121
|
+
`${dim}Registering amd64 emulation (binfmt_misc does not survive a VM restart)${reset}`,
|
|
122
|
+
);
|
|
123
|
+
const result = spawnSync(
|
|
124
|
+
'docker',
|
|
125
|
+
['run', '--privileged', '--rm', 'tonistiigi/binfmt', '--install', 'amd64'],
|
|
126
|
+
{ stdio: 'ignore' },
|
|
127
|
+
);
|
|
128
|
+
if (result.status === 0 && haveAmd64Emulation(profile)) {
|
|
129
|
+
console.log(`${green}amd64 emulation registered.${reset}`);
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
console.error(
|
|
133
|
+
`${yellow}!${reset} Could not register amd64 emulation. Suites that build an amd64 image`,
|
|
134
|
+
);
|
|
135
|
+
console.error(
|
|
136
|
+
` (signal) will fail with an exec-format error. Retry with: ${bold}docker run --privileged --rm tonistiigi/binfmt --install amd64${reset}`,
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
|
|
78
140
|
function reportStatus(): number {
|
|
79
141
|
const profile = activeProfile();
|
|
80
142
|
const host = readHostFacts();
|
|
@@ -124,7 +186,9 @@ function bringUp(): number {
|
|
|
124
186
|
`${bold}Creating the colima VM${reset} ${dim}(${budget.cpus} CPU, ${budget.memoryGiB} GiB, ${budget.mountType} mounts)${reset}`,
|
|
125
187
|
);
|
|
126
188
|
const result = spawnSync('colima', colimaStartArgs(budget, profile), { stdio: 'inherit' });
|
|
127
|
-
return result.status ?? 1;
|
|
189
|
+
if (result.status !== 0) return result.status ?? 1;
|
|
190
|
+
ensureAmd64Emulation(profile);
|
|
191
|
+
return 0;
|
|
128
192
|
}
|
|
129
193
|
|
|
130
194
|
const { problems, needsRecreate } = evaluateHostVm(existing, host, budget);
|
|
@@ -142,6 +206,9 @@ function bringUp(): number {
|
|
|
142
206
|
|
|
143
207
|
if (colimaRunning(profile) && problems.length === 0) {
|
|
144
208
|
console.log(`${green}Docker host already up and in policy.${reset}`);
|
|
209
|
+
// Still check: a VM someone started with a bare `colima start`, or one
|
|
210
|
+
// that restarted under us, is in policy and has no emulation registered.
|
|
211
|
+
ensureAmd64Emulation(profile);
|
|
145
212
|
return 0;
|
|
146
213
|
}
|
|
147
214
|
|
|
@@ -155,7 +222,9 @@ function bringUp(): number {
|
|
|
155
222
|
['start', '-p', profile, '--cpu', String(budget.cpus), '--memory', String(budget.memoryGiB)],
|
|
156
223
|
{ stdio: 'inherit' },
|
|
157
224
|
);
|
|
158
|
-
return result.status ?? 1;
|
|
225
|
+
if (result.status !== 0) return result.status ?? 1;
|
|
226
|
+
ensureAmd64Emulation(profile);
|
|
227
|
+
return 0;
|
|
159
228
|
}
|
|
160
229
|
|
|
161
230
|
/**
|
|
@@ -450,7 +450,11 @@ export function generateSharedInfraYaml(): string {
|
|
|
450
450
|
dockerfile: 'docker/Dockerfile.pebble',
|
|
451
451
|
networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.PEBBLE } },
|
|
452
452
|
cap_add: ['NET_ADMIN'],
|
|
453
|
-
|
|
453
|
+
// -dnsserver is the resolver Pebble asks when validating a challenge. It
|
|
454
|
+
// MUST be a shared-infra resolver: comcast-resolver (203.0.113.1) is
|
|
455
|
+
// per-test, so naming it here left shared infra unable to finish ACME
|
|
456
|
+
// until a test stack existed (celilo#1365).
|
|
457
|
+
command: `-config /config/pebble-config.json -dnsserver ${SIMULATOR_IPS.PUBLIC_RESOLVER}:53`,
|
|
454
458
|
volumes: [
|
|
455
459
|
'./config/pebble/pebble-config.json:/config/pebble-config.json:ro',
|
|
456
460
|
'./config/pebble/pebble-tls.crt:/config/pebble-tls.crt:ro',
|
|
@@ -511,7 +515,7 @@ export function generateSharedInfraYaml(): string {
|
|
|
511
515
|
dockerfile: 'docker/Dockerfile.isitup',
|
|
512
516
|
networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.ISITUP } },
|
|
513
517
|
cap_add: ['NET_ADMIN'],
|
|
514
|
-
dns: [
|
|
518
|
+
dns: [SIMULATOR_IPS.PUBLIC_RESOLVER],
|
|
515
519
|
});
|
|
516
520
|
|
|
517
521
|
// celilo.computer static site simulator — serves install.sh and the docs
|
|
@@ -523,7 +527,7 @@ export function generateSharedInfraYaml(): string {
|
|
|
523
527
|
dockerfile: 'docker/Dockerfile.celilo-website-sim',
|
|
524
528
|
networks: { 'internet-external': { ipv4_address: SIMULATOR_IPS.WEBSITE } },
|
|
525
529
|
cap_add: ['NET_ADMIN'],
|
|
526
|
-
dns: [
|
|
530
|
+
dns: [SIMULATOR_IPS.PUBLIC_RESOLVER],
|
|
527
531
|
});
|
|
528
532
|
|
|
529
533
|
// npm-compat registry simulator — serves @celilo/* tarballs to install.sh's
|
package/src/doctor.test.ts
CHANGED
|
@@ -29,7 +29,9 @@ import {
|
|
|
29
29
|
checkStaleContainers,
|
|
30
30
|
diagnose,
|
|
31
31
|
explainBuildFailure,
|
|
32
|
+
imageLookupKey,
|
|
32
33
|
parseBaseImages,
|
|
34
|
+
readDockerfileBases,
|
|
33
35
|
stackStartedAt,
|
|
34
36
|
} from './doctor';
|
|
35
37
|
import type { LeakedStack } from './doctor';
|
|
@@ -133,25 +135,36 @@ describe('base images (problem 3: a missing image reads as a network failure)',
|
|
|
133
135
|
{ derived: 'celilo-e2e/pebble', bases: ['ghcr.io/letsencrypt/pebble:latest'] },
|
|
134
136
|
];
|
|
135
137
|
|
|
136
|
-
test('a base image missing with its derived image ALSO missing fails,
|
|
138
|
+
test('a base image missing with its derived image ALSO missing fails, naming the fix', () => {
|
|
137
139
|
// Post `docker image prune`: nothing local, so the build must go to a
|
|
138
140
|
// registry — the exact state that produced "failed to solve: ubuntu:22.04:
|
|
139
141
|
// net/http: TLS handshake timeout" 16 images into a rebuild.
|
|
140
142
|
const check = checkBaseImages(healthyProbe({ imageExists: () => false }), dockerfiles);
|
|
141
143
|
expect(check.status).toBe('fail');
|
|
142
144
|
expect(check.detail).toContain('ubuntu:22.04');
|
|
143
|
-
expect(check.remedy).toContain('
|
|
144
|
-
expect(check.remedy).toContain('docker pull ghcr.io/letsencrypt/pebble:latest');
|
|
145
|
+
expect(check.remedy).toContain('cele2e build-infra');
|
|
145
146
|
});
|
|
146
147
|
|
|
147
|
-
test('a base
|
|
148
|
-
//
|
|
149
|
-
//
|
|
148
|
+
test('a base absent with its derived image PRESENT warns — it used to be silent', () => {
|
|
149
|
+
// The measured case this check existed and could not see (celilo#1244).
|
|
150
|
+
// 2026-09-04: all twelve bases absent, 29 derived tags present, doctor said
|
|
151
|
+
// `ok`, and buildkit went to the registry anyway — 30 of 32 suites died in
|
|
152
|
+
// 1-2s each. Not a `fail`: ten of twelve bases are routinely absent on a
|
|
153
|
+
// host where e2e passes, so failing would refuse a working machine.
|
|
150
154
|
const check = checkBaseImages(
|
|
151
155
|
healthyProbe({ imageExists: (ref) => ref.startsWith('celilo-e2e/') }),
|
|
152
156
|
dockerfiles,
|
|
153
157
|
);
|
|
158
|
+
expect(check.status).toBe('warn');
|
|
159
|
+
expect(check.detail).toContain('ubuntu:22.04');
|
|
160
|
+
expect(check.detail).toContain('cache key');
|
|
161
|
+
expect(check.remedy).toContain('cele2e build-infra');
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
test('every base present is the only state reported ok, and it says so', () => {
|
|
165
|
+
const check = checkBaseImages(healthyProbe(), dockerfiles);
|
|
154
166
|
expect(check.status).toBe('ok');
|
|
167
|
+
expect(check.detail).toContain('2 base image(s)');
|
|
155
168
|
});
|
|
156
169
|
|
|
157
170
|
test('only the images that actually need fetching are named', () => {
|
|
@@ -165,6 +178,64 @@ describe('base images (problem 3: a missing image reads as a network failure)',
|
|
|
165
178
|
expect(check.detail).toContain('ubuntu:22.04');
|
|
166
179
|
expect(check.detail).not.toContain('pebble');
|
|
167
180
|
});
|
|
181
|
+
|
|
182
|
+
// ─── Reach probe ───────────────────────────────────────────────────
|
|
183
|
+
//
|
|
184
|
+
// The set this check walks is COMPUTED (every `docker/Dockerfile.*` crossed
|
|
185
|
+
// with a store query), which is the shape apps/celilo/CLAUDE.md says to
|
|
186
|
+
// measure rather than reason about. So derive it from the REAL docker/ dir
|
|
187
|
+
// and plant one absence at a time: whichever base the check names is its
|
|
188
|
+
// actual reach. Hand-writing the Dockerfile list here would give confident
|
|
189
|
+
// reach data about a directory that does not exist.
|
|
190
|
+
describe('reach: every base a real Dockerfile names is examined', () => {
|
|
191
|
+
const real = readDockerfileBases(join(import.meta.dir, '..'));
|
|
192
|
+
const realBases = [...new Set(real.flatMap((d) => d.bases))].sort();
|
|
193
|
+
|
|
194
|
+
test('the probe has a set to measure', () => {
|
|
195
|
+
expect(real.length).toBeGreaterThan(0);
|
|
196
|
+
expect(realBases.length).toBeGreaterThan(0);
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
for (const planted of realBases) {
|
|
200
|
+
test(`${planted} absent from the image store is reported, not skipped`, () => {
|
|
201
|
+
// Every derived tag present — the state the old short-circuit treated
|
|
202
|
+
// as proof the base was irrelevant. It is the state that broke a run.
|
|
203
|
+
const check = checkBaseImages(
|
|
204
|
+
healthyProbe({ imageExists: (ref) => ref !== planted }),
|
|
205
|
+
real,
|
|
206
|
+
);
|
|
207
|
+
expect(check.status).not.toBe('ok');
|
|
208
|
+
expect(check.detail).toContain(planted);
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
});
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
describe('image lookup key (a pinned base read absent forever)', () => {
|
|
215
|
+
// Every e2e Dockerfile pins its bases by digest. `docker images` lists such an
|
|
216
|
+
// image as `nginx:<none>` plus `nginx@sha256:…`, so a lookup by `repo:tag`
|
|
217
|
+
// could never match and `base-images` would warn about all of them for good.
|
|
218
|
+
test('a digest-pinned ref collapses to repo@digest, dropping the tag', () => {
|
|
219
|
+
expect(imageLookupKey('nginx:alpine@sha256:abc')).toBe('nginx@sha256:abc');
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
test('a registry path with a digest keeps the whole repo', () => {
|
|
223
|
+
expect(imageLookupKey('ghcr.io/letsencrypt/pebble:latest@sha256:abc')).toBe(
|
|
224
|
+
'ghcr.io/letsencrypt/pebble@sha256:abc',
|
|
225
|
+
);
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
test('an untagged ref gets :latest, the way docker does', () => {
|
|
229
|
+
expect(imageLookupKey('cznic/knot')).toBe('cznic/knot:latest');
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
test('a host:port registry is not mistaken for a tag', () => {
|
|
233
|
+
expect(imageLookupKey('localhost:5000/thing')).toBe('localhost:5000/thing:latest');
|
|
234
|
+
});
|
|
235
|
+
|
|
236
|
+
test('a plain tagged ref is unchanged', () => {
|
|
237
|
+
expect(imageLookupKey('ubuntu:22.04')).toBe('ubuntu:22.04');
|
|
238
|
+
});
|
|
168
239
|
});
|
|
169
240
|
|
|
170
241
|
describe('run-lock (problems 2 and 5)', () => {
|
|
@@ -367,6 +438,16 @@ describe('build failure translation', () => {
|
|
|
367
438
|
expect(text).toContain('docker pull ubuntu:22.04');
|
|
368
439
|
});
|
|
369
440
|
|
|
441
|
+
test('an exec-format error is explained as missing amd64 emulation', () => {
|
|
442
|
+
const text = explainBuildFailure(
|
|
443
|
+
'ERROR: failed to solve: process "/bin/sh -c apt-get update" did not complete ' +
|
|
444
|
+
'successfully: exec /bin/sh: exec format error',
|
|
445
|
+
);
|
|
446
|
+
expect(text).toContain('exec format error');
|
|
447
|
+
expect(text).toContain('binfmt_misc');
|
|
448
|
+
expect(text).toContain('cele2e host up');
|
|
449
|
+
});
|
|
450
|
+
|
|
370
451
|
test('a genuine RUN-step failure is left alone', () => {
|
|
371
452
|
expect(
|
|
372
453
|
explainBuildFailure(
|
package/src/doctor.ts
CHANGED
|
@@ -99,7 +99,14 @@ export interface LeakedStack {
|
|
|
99
99
|
|
|
100
100
|
/** Every Docker fact doctor needs, injectable so the checks are testable. */
|
|
101
101
|
export interface DoctorProbe {
|
|
102
|
-
/**
|
|
102
|
+
/**
|
|
103
|
+
* Is this image present in docker's IMAGE STORE (the `docker images` tag
|
|
104
|
+
* list)? An untagged ref means `:latest`.
|
|
105
|
+
*
|
|
106
|
+
* This is not buildkit's build cache. buildkit can build from a base absent
|
|
107
|
+
* here, so `false` means "may reach a registry", not "the build will fail" —
|
|
108
|
+
* see `checkBaseImages`, which is where that asymmetry decides a verdict.
|
|
109
|
+
*/
|
|
103
110
|
imageExists(ref: string): boolean;
|
|
104
111
|
/** The image's configured Cmd, or null when the image is absent. */
|
|
105
112
|
imageCmd(ref: string): string[] | null;
|
|
@@ -143,6 +150,29 @@ export function parseBaseImages(dockerfile: string): string[] {
|
|
|
143
150
|
return bases;
|
|
144
151
|
}
|
|
145
152
|
|
|
153
|
+
/**
|
|
154
|
+
* The key a ref is found under in docker's image listing.
|
|
155
|
+
*
|
|
156
|
+
* A digest-pinned ref (`nginx:alpine@sha256:…`, which is how every e2e
|
|
157
|
+
* Dockerfile pins its bases) is listed by REPO AND DIGEST, never by the tag it
|
|
158
|
+
* was pulled with — `docker images` shows `nginx:<none>` for it. So looking a
|
|
159
|
+
* pinned ref up by `repo:tag` never matches and every pinned base reads absent
|
|
160
|
+
* forever. Pinned refs collapse to `repo@sha256:…`; an unpinned one gets
|
|
161
|
+
* `:latest` the way docker does.
|
|
162
|
+
*/
|
|
163
|
+
export function imageLookupKey(ref: string): string {
|
|
164
|
+
const at = ref.indexOf('@');
|
|
165
|
+
if (at >= 0) {
|
|
166
|
+
const digest = ref.slice(at + 1);
|
|
167
|
+
const name = ref.slice(0, at);
|
|
168
|
+
const lastSlash = name.lastIndexOf('/');
|
|
169
|
+
const colon = name.indexOf(':', lastSlash + 1);
|
|
170
|
+
return `${colon >= 0 ? name.slice(0, colon) : name}@${digest}`;
|
|
171
|
+
}
|
|
172
|
+
const lastSlash = ref.lastIndexOf('/');
|
|
173
|
+
return ref.includes(':', lastSlash + 1) ? ref : `${ref}:latest`;
|
|
174
|
+
}
|
|
175
|
+
|
|
146
176
|
/** Map `docker/Dockerfile.observer` → `celilo-e2e/observer` (docker-compose-generator's imageTag). */
|
|
147
177
|
function derivedImageTag(dockerfileName: string): string {
|
|
148
178
|
return `celilo-e2e/${dockerfileName.replace(/^Dockerfile\./, '')}`;
|
|
@@ -256,33 +286,70 @@ export function checkManagementImage(probe: DoctorProbe): DoctorCheck {
|
|
|
256
286
|
}
|
|
257
287
|
|
|
258
288
|
/**
|
|
259
|
-
* Base images the next build
|
|
289
|
+
* Base images the next build must resolve, measured against docker's image
|
|
290
|
+
* store — which is NOT the store buildkit builds from.
|
|
291
|
+
*
|
|
292
|
+
* `imageExists` reads `docker images`, the image-store tag list. buildkit keeps
|
|
293
|
+
* its own cache, so a base absent from the tag list may still build fine. That
|
|
294
|
+
* asymmetry decides the verdict here, in one direction only:
|
|
295
|
+
*
|
|
296
|
+
* base tag PRESENT -> buildkit resolves it locally. Guaranteed safe.
|
|
297
|
+
* base tag ABSENT -> buildkit MAY reach a registry. Not knowably safe.
|
|
260
298
|
*
|
|
261
|
-
*
|
|
262
|
-
*
|
|
263
|
-
*
|
|
264
|
-
*
|
|
265
|
-
* the
|
|
266
|
-
*
|
|
267
|
-
*
|
|
299
|
+
* The old check turned the second case into silence whenever the derived image
|
|
300
|
+
* was present, on the premise that "buildkit serves the build from cache and
|
|
301
|
+
* never resolves the reference". That premise is false, measured twice: on
|
|
302
|
+
* 2026-09-03 three suites died on `failed to solve: debian:trixie-slim` with
|
|
303
|
+
* the derived tag present, and on 2026-09-04 all twelve bases were absent, all
|
|
304
|
+
* 29 derived tags were present, doctor said `ok`, and 30 of 32 suites died in
|
|
305
|
+
* 1-2s each. buildkit resolves a FROM reference to compute its cache key, and a
|
|
306
|
+
* tag is mutable, so that resolution can go to the registry.
|
|
307
|
+
*
|
|
308
|
+
* So the absent case is a WARN, never silence, and never a `fail` either: ten
|
|
309
|
+
* of twelve bases are routinely absent from this host's tag list while e2e
|
|
310
|
+
* passes on it, so failing would refuse a working machine ten times over. The
|
|
311
|
+
* exception is a base whose derived image is ALSO absent — then the build has
|
|
312
|
+
* to run and has nothing local to run from, which is a certainty, not a risk.
|
|
313
|
+
*
|
|
314
|
+
* `cele2e build-infra` now pulls every base a Dockerfile names, which is the
|
|
315
|
+
* one-command way out of the warn.
|
|
268
316
|
*/
|
|
269
317
|
export function checkBaseImages(probe: DoctorProbe, dockerfiles: DockerfileBases[]): DoctorCheck {
|
|
270
|
-
|
|
318
|
+
// Every base a Dockerfile names, and whether anything consuming it must be
|
|
319
|
+
// rebuilt from scratch. No short-circuit: the derived tag's presence changes
|
|
320
|
+
// the SEVERITY, never whether the base gets looked at.
|
|
321
|
+
const absent = new Map<string, boolean>();
|
|
271
322
|
for (const { derived, bases } of dockerfiles) {
|
|
272
|
-
|
|
323
|
+
const mustBuild = !probe.imageExists(derived);
|
|
273
324
|
for (const base of bases) {
|
|
274
|
-
if (
|
|
325
|
+
if (probe.imageExists(base)) continue;
|
|
326
|
+
absent.set(base, (absent.get(base) ?? false) || mustBuild);
|
|
275
327
|
}
|
|
276
328
|
}
|
|
277
|
-
|
|
278
|
-
|
|
329
|
+
const total = new Set(dockerfiles.flatMap((d) => d.bases)).size;
|
|
330
|
+
if (absent.size === 0) {
|
|
331
|
+
return {
|
|
332
|
+
name: 'base-images',
|
|
333
|
+
status: 'ok',
|
|
334
|
+
detail: `all ${total} base image(s) are in the local image store, so buildkit resolves every FROM without reaching a registry`,
|
|
335
|
+
};
|
|
336
|
+
}
|
|
337
|
+
const refs = [...absent.keys()].sort();
|
|
338
|
+
const certain = refs.filter((r) => absent.get(r));
|
|
339
|
+
const remedy = 'cele2e build-infra (it pulls every base a Dockerfile names)';
|
|
340
|
+
if (certain.length > 0) {
|
|
341
|
+
return {
|
|
342
|
+
name: 'base-images',
|
|
343
|
+
status: 'fail',
|
|
344
|
+
detail: `${certain.length} base image(s) absent from the image store AND needed by an image that must be rebuilt, so the build has nothing local to build from: ${certain.join(', ')}`,
|
|
345
|
+
remedy,
|
|
346
|
+
};
|
|
279
347
|
}
|
|
280
|
-
const refs = [...needPull].sort();
|
|
281
348
|
return {
|
|
282
349
|
name: 'base-images',
|
|
283
|
-
status: '
|
|
284
|
-
detail: `${refs.length} base image(s) absent
|
|
285
|
-
remedy
|
|
350
|
+
status: 'warn',
|
|
351
|
+
detail: `${refs.length} of ${total} base image(s) absent from the image store: ${refs.join(', ')}. Their derived images are present, but buildkit still resolves each FROM to compute its cache key, so a registry blip fails the build ~30s in and blames the network (celilo#1244)`,
|
|
352
|
+
remedy,
|
|
286
353
|
};
|
|
287
354
|
}
|
|
288
355
|
|
|
@@ -518,6 +585,22 @@ export function checkDiskPressure(probe: DoctorProbe): DoctorCheck {
|
|
|
518
585
|
* a genuine build error, which needs no translation.
|
|
519
586
|
*/
|
|
520
587
|
export function explainBuildFailure(stderr: string): string {
|
|
588
|
+
// Missing amd64 emulation, which names neither amd64 nor emulation. The
|
|
589
|
+
// handler lives in binfmt_misc inside the colima VM and does NOT survive a
|
|
590
|
+
// VM restart, so a rig that built the signal image yesterday fails today
|
|
591
|
+
// with what reads like a corrupt base image. `cele2e host up` registers it.
|
|
592
|
+
if (/exec format error/i.test(stderr)) {
|
|
593
|
+
return [
|
|
594
|
+
'',
|
|
595
|
+
'',
|
|
596
|
+
'This is an "exec format error" — the image is a different architecture than',
|
|
597
|
+
'this host and the kernel has no emulator registered for it. The signal image',
|
|
598
|
+
'pins `--platform=linux/amd64`; on an arm64 Mac that needs binfmt_misc, which',
|
|
599
|
+
'lives in the colima VM kernel and is LOST on every VM restart.',
|
|
600
|
+
'Fix: cele2e host up (registers it, idempotently)',
|
|
601
|
+
].join('\n');
|
|
602
|
+
}
|
|
603
|
+
|
|
521
604
|
const ref = stderr.match(/failed to solve:\s*([^\s:"]+(?::[^\s:"]+)?)/)?.[1];
|
|
522
605
|
if (!ref || ref === 'process') return '';
|
|
523
606
|
return [
|
|
@@ -542,15 +625,22 @@ function docker(args: string[]): string {
|
|
|
542
625
|
export function createDockerProbe(): DoctorProbe {
|
|
543
626
|
// One `docker images` listing answers every existence question; per-image
|
|
544
627
|
// inspect calls would be ~40 process spawns on a preflight that must be fast.
|
|
628
|
+
// Both keys, because a digest-pinned base is listed only by digest — see
|
|
629
|
+
// imageLookupKey. Indexing tags alone made every pinned base read absent.
|
|
545
630
|
let tags: Set<string> | null = null;
|
|
546
631
|
const knownTags = (): Set<string> => {
|
|
547
632
|
if (tags) return tags;
|
|
548
633
|
try {
|
|
549
634
|
tags = new Set(
|
|
550
|
-
docker([
|
|
551
|
-
|
|
635
|
+
docker([
|
|
636
|
+
'images',
|
|
637
|
+
'--digests',
|
|
638
|
+
'--format',
|
|
639
|
+
'{{.Repository}}:{{.Tag}}\t{{.Repository}}@{{.Digest}}',
|
|
640
|
+
])
|
|
641
|
+
.split(/[\n\t]/)
|
|
552
642
|
.map((l) => l.trim())
|
|
553
|
-
.filter(
|
|
643
|
+
.filter((l) => l && !l.endsWith(':<none>') && !l.endsWith('@<none>')),
|
|
554
644
|
);
|
|
555
645
|
} catch {
|
|
556
646
|
tags = new Set();
|
|
@@ -560,8 +650,7 @@ export function createDockerProbe(): DoctorProbe {
|
|
|
560
650
|
|
|
561
651
|
return {
|
|
562
652
|
imageExists(ref) {
|
|
563
|
-
|
|
564
|
-
return knownTags().has(withTag);
|
|
653
|
+
return knownTags().has(imageLookupKey(ref));
|
|
565
654
|
},
|
|
566
655
|
imageCmd(ref) {
|
|
567
656
|
try {
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
/**
|
|
3
|
+
* A port forward must bind the interface that actually carries the public IP.
|
|
4
|
+
*
|
|
5
|
+
* celilo#1436. The greenwave simulator defaulted EXTERNAL_INTERFACE to 'eth1'
|
|
6
|
+
* and nothing set it, while on fw-isp eth0 carries 203.0.113.100 and eth1 is
|
|
7
|
+
* the internal leg. Every DNAT was therefore bound to the interface facing away
|
|
8
|
+
* from the internet: hairpin traffic from inside matched, inbound internet
|
|
9
|
+
* traffic was refused, and no per-test domain could obtain a certificate.
|
|
10
|
+
*
|
|
11
|
+
* This reads the REAL simulator source rather than re-implementing its logic —
|
|
12
|
+
* a test that restates the derivation would pass against a file that still
|
|
13
|
+
* hardcodes the wrong default.
|
|
14
|
+
*/
|
|
15
|
+
import { join } from 'node:path';
|
|
16
|
+
|
|
17
|
+
// Resolved from THIS FILE, not from the cwd. `bun run test` runs each
|
|
18
|
+
// workspace package's suite with that package's own cwd, so a repo-root
|
|
19
|
+
// relative path resolves to packages/e2e/packages/e2e/... and Bun.file()
|
|
20
|
+
// throws ENOENT -- while the same path works when run from the repo root,
|
|
21
|
+
// which is where it gets tried by hand. A test that cannot reach the file it
|
|
22
|
+
// reads is the shape celilo's CLAUDE.md calls "a check that cannot reach the
|
|
23
|
+
// thing it is checking", and it is exactly what this test exists to avoid.
|
|
24
|
+
const SRC = join(import.meta.dir, '..', 'simulators', 'greenwave', 'state.ts');
|
|
25
|
+
|
|
26
|
+
describe('celilo#1436: the greenwave sim does not guess interface names', () => {
|
|
27
|
+
test('the port-forward DNAT is scoped by destination address, not by interface', async () => {
|
|
28
|
+
const src = await Bun.file(SRC).text();
|
|
29
|
+
|
|
30
|
+
// Docker hands out interface names in a non-deterministic order, so
|
|
31
|
+
// DERIVING the right one only chooses which direction breaks. The
|
|
32
|
+
// destination address is the discriminator that cannot be raced: only
|
|
33
|
+
// traffic actually addressed to PUBLIC_IP can match, whichever leg it
|
|
34
|
+
// arrived on.
|
|
35
|
+
expect(src).toMatch(/PREROUTING -d \$\{PUBLIC_IP\}/);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
test('no port-forward rule binds an interface at all', async () => {
|
|
39
|
+
const src = await Bun.file(SRC).text();
|
|
40
|
+
|
|
41
|
+
// The control, and the reason this file exists: re-introducing `-i` on a
|
|
42
|
+
// forwarding rule is exactly the regression, and it would otherwise pass
|
|
43
|
+
// the assertion above unnoticed.
|
|
44
|
+
const forwardRules = src
|
|
45
|
+
.split('\n')
|
|
46
|
+
.filter((l) => l.includes('PREROUTING') && l.includes('DNAT'));
|
|
47
|
+
expect(forwardRules.length).toBeGreaterThan(0);
|
|
48
|
+
for (const rule of forwardRules) {
|
|
49
|
+
expect(rule).not.toMatch(/\s-i\s/);
|
|
50
|
+
}
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test('the DHCP interface is derived from the address it serves, never hardcoded', async () => {
|
|
54
|
+
const src = await Bun.file(SRC).text();
|
|
55
|
+
|
|
56
|
+
// dnsmasq's `interface=` DOES still need a name -- it binds a socket, so
|
|
57
|
+
// an address cannot stand in for it. That one is derived from the router
|
|
58
|
+
// address it is about to serve.
|
|
59
|
+
expect(src).toContain('deriveDhcpInterface');
|
|
60
|
+
expect(src).toContain('interfaceCarrying');
|
|
61
|
+
expect(src).not.toMatch(/interface=eth[0-9]/);
|
|
62
|
+
});
|
|
63
|
+
});
|