@celilo/e2e 0.20.1 → 0.20.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/e2e-bake-management +17 -1
- package/npm-registry-server/tsconfig.json +1 -10
- package/package.json +4 -3
- package/registry-server/tsconfig.json +3 -12
- package/src/block-timing.ts +24 -3
- package/src/cli/build.test.ts +60 -2
- package/src/cli/build.ts +16 -1
- package/src/container-manager.ts +47 -1
- package/src/docker-compose-generator.ts +33 -3
- package/src/extract-failure.ts +36 -14
- package/src/live-stack.test.ts +52 -1
- package/src/live-stack.ts +59 -1
- package/src/module-host.test.ts +55 -1
- package/src/module-host.ts +31 -0
- package/src/netapp-staleness.test.ts +139 -0
- package/src/netapp-staleness.ts +62 -0
- package/src/network-builder.ts +19 -3
- package/src/registry-bundle.ts +14 -2
- package/src/runner-keep.test.ts +72 -0
- package/src/runner.ts +19 -2
- package/src/types.ts +21 -7
- package/registry-server/src/auth.test.ts +0 -76
- package/registry-server/src/bootstrap-packaging.test.ts +0 -71
- package/registry-server/src/introspection.test.ts +0 -243
- package/registry-server/src/module-owner-store.test.ts +0 -85
- package/registry-server/src/rate-limit.test.ts +0 -62
- package/registry-server/src/scoped-token-store.test.ts +0 -93
- package/registry-server/src/server.test.ts +0 -991
- package/registry-server/src/storage.test.ts +0 -152
- package/registry-server/src/sweep.test.ts +0 -326
- package/registry-server/src/validation.test.ts +0 -86
package/bin/e2e-bake-management
CHANGED
|
@@ -38,6 +38,7 @@ import {
|
|
|
38
38
|
SOURCE_LABEL,
|
|
39
39
|
} from '../src/source-fingerprint';
|
|
40
40
|
import { readFileSync } from 'node:fs';
|
|
41
|
+
import { ZONE_GATEWAYS, greenwaveRouterIp } from '../src/types';
|
|
41
42
|
|
|
42
43
|
const PACKAGE_ROOT = join(import.meta.dir, '..');
|
|
43
44
|
const COMPOSE_FILE = join(PACKAGE_ROOT, 'docker-compose.test.yml');
|
|
@@ -356,8 +357,23 @@ async function bakeViaSim(): Promise<void> {
|
|
|
356
357
|
throw new Error('Could not resolve management container id');
|
|
357
358
|
}
|
|
358
359
|
const superseded = imageIdOnTag('celilo-e2e/management:latest');
|
|
360
|
+
// The committed container runs under the sim topology's compose env, and
|
|
361
|
+
// commit captures its Env. Since the default managementZone became
|
|
362
|
+
// secure-mgmt, that baked FW_MAIN_HOP=10.226.120.1 (and the control-plane
|
|
363
|
+
// default gateway) into :latest — every internal-topology suite then
|
|
364
|
+
// inherited a nexthop that is not on-link from the internal LAN and the
|
|
365
|
+
// mgmt box crash-looped in management-routes.sh (celilo#1351). Reset the
|
|
366
|
+
// per-topology routing vars to the internal-topology defaults at commit
|
|
367
|
+
// time, so the image is topology-neutral; every compose sets both anyway.
|
|
359
368
|
run(
|
|
360
|
-
|
|
369
|
+
[
|
|
370
|
+
'docker commit',
|
|
371
|
+
`--change ${JSON.stringify(`ENV DEFAULT_GATEWAY=${greenwaveRouterIp()}`)}`,
|
|
372
|
+
`--change ${JSON.stringify(`ENV FW_MAIN_HOP=${ZONE_GATEWAYS.internal}`)}`,
|
|
373
|
+
`--change ${JSON.stringify(`LABEL ${SOURCE_LABEL}=${sourceStamp(false, version)}`)}`,
|
|
374
|
+
containerId,
|
|
375
|
+
'celilo-e2e/management:latest',
|
|
376
|
+
].join(' '),
|
|
361
377
|
);
|
|
362
378
|
console.log(`✔ ${Math.round((Date.now() - t5) / 1000)}s`);
|
|
363
379
|
removeSupersededImage(superseded, 'celilo-e2e/management:latest');
|
|
@@ -1,12 +1,3 @@
|
|
|
1
1
|
{
|
|
2
|
-
"
|
|
3
|
-
"target": "ESNext",
|
|
4
|
-
"module": "ESNext",
|
|
5
|
-
"moduleResolution": "bundler",
|
|
6
|
-
"strict": true,
|
|
7
|
-
"esModuleInterop": true,
|
|
8
|
-
"skipLibCheck": true,
|
|
9
|
-
"types": ["bun-types"]
|
|
10
|
-
},
|
|
11
|
-
"include": ["src/**/*.ts"]
|
|
2
|
+
"extends": "../../../tsconfig.base.json"
|
|
12
3
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@celilo/e2e",
|
|
3
|
-
"version": "0.20.
|
|
3
|
+
"version": "0.20.3",
|
|
4
4
|
"description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./src/index.ts",
|
|
@@ -20,7 +20,8 @@
|
|
|
20
20
|
"scripts": {
|
|
21
21
|
"test": "bun test --timeout 30000 ./tests/ ./src/ ./npm-registry-server/ ./simulators/",
|
|
22
22
|
"test:completion": "bun test tests/completion",
|
|
23
|
-
"test:integration": "bun test tests-integration/"
|
|
23
|
+
"test:integration": "bun test tests-integration/",
|
|
24
|
+
"test:weekly": "bun test --timeout 30000 ./weekly/"
|
|
24
25
|
},
|
|
25
26
|
"files": [
|
|
26
27
|
"src/",
|
|
@@ -37,7 +38,7 @@
|
|
|
37
38
|
"README.md"
|
|
38
39
|
],
|
|
39
40
|
"dependencies": {
|
|
40
|
-
"@celilo/capabilities": "^
|
|
41
|
+
"@celilo/capabilities": "^5.0.0",
|
|
41
42
|
"@celilo/cli-display": "^0.2.0",
|
|
42
43
|
"@celilo/event-bus": "^0.6.0",
|
|
43
44
|
"@celilo/terraform-fake": "^0.3.1",
|
|
@@ -1,15 +1,6 @@
|
|
|
1
1
|
{
|
|
2
|
+
"extends": "../../tsconfig.base.json",
|
|
2
3
|
"compilerOptions": {
|
|
3
|
-
"
|
|
4
|
-
|
|
5
|
-
"moduleResolution": "bundler",
|
|
6
|
-
"strict": true,
|
|
7
|
-
"skipLibCheck": true,
|
|
8
|
-
"types": ["bun-types"],
|
|
9
|
-
"allowImportingTsExtensions": true,
|
|
10
|
-
"noEmit": true,
|
|
11
|
-
"resolveJsonModule": true,
|
|
12
|
-
"isolatedModules": true
|
|
13
|
-
},
|
|
14
|
-
"include": ["src/**/*"]
|
|
4
|
+
"allowImportingTsExtensions": true
|
|
5
|
+
}
|
|
15
6
|
}
|
package/src/block-timing.ts
CHANGED
|
@@ -35,10 +35,31 @@ export type BlockTiming = Record<string, Record<string, number>>;
|
|
|
35
35
|
/**
|
|
36
36
|
* A failed block below this did no work, so its duration measures nothing.
|
|
37
37
|
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
38
|
+
* Measured, not guessed. Over the 58 recorded runs in `e2e/results/`, every
|
|
39
|
+
* failed block falls in one of two clumps with nothing in between:
|
|
40
|
+
*
|
|
41
|
+
* did no work (a `requireStage` throw) 0.00ms .. 43.6ms
|
|
42
|
+
* did real work over 2s, up to 140s
|
|
43
|
+
*
|
|
44
|
+
* The gap is narrower than those two clumps suggest, because the fastest real
|
|
45
|
+
* work seen anywhere is not 2s. `CASCADE_STDOUT` in the test file pins a real
|
|
46
|
+
* failure at 123.54ms, and celilo#1281 measured real blocks from 145ms. So the
|
|
47
|
+
* floor has to sit between roughly 44ms and 123ms, and where it sits decides
|
|
48
|
+
* which of two mistakes this file makes.
|
|
49
|
+
*
|
|
50
|
+
* At 50 the clearance over the worst observed skip is 1.15x. That is thin
|
|
51
|
+
* enough that one slow skip enters the duration record as a healthy fast
|
|
52
|
+
* block, which is the mistake that matters: it pollutes the baseline every
|
|
53
|
+
* later run is compared against. At 100 the clearance is 2.3x, and the cost is
|
|
54
|
+
* that a REAL failure under 100ms would be dropped. That is the cheaper
|
|
55
|
+
* mistake. This file exists to find blocks running OUT of budget, and one
|
|
56
|
+
* using 0.03% of its cap is not a candidate.
|
|
57
|
+
*
|
|
58
|
+
* Nothing observed is reclassified by the move. No failed block in the recorded
|
|
59
|
+
* runs sits between 43.6ms and 2s, and no fixture sits between 39.19ms and
|
|
60
|
+
* 123.54ms, so both keep their current verdict.
|
|
40
61
|
*/
|
|
41
|
-
const NOT_A_MEASUREMENT_MS =
|
|
62
|
+
const NOT_A_MEASUREMENT_MS = 100;
|
|
42
63
|
|
|
43
64
|
export function parseBlockDurations(lines: string[], junitXml?: string): Record<string, number> {
|
|
44
65
|
const failed = failedBlocks(lines);
|
package/src/cli/build.test.ts
CHANGED
|
@@ -6,14 +6,16 @@ import {
|
|
|
6
6
|
readFileSync,
|
|
7
7
|
readdirSync,
|
|
8
8
|
rmSync,
|
|
9
|
+
utimesSync,
|
|
9
10
|
writeFileSync,
|
|
10
11
|
} from 'node:fs';
|
|
11
12
|
import { tmpdir } from 'node:os';
|
|
12
|
-
import { join } from 'node:path';
|
|
13
|
+
import { basename, join } from 'node:path';
|
|
13
14
|
import { gzipSync } from 'node:zlib';
|
|
14
15
|
import {
|
|
15
16
|
assertGzipValid,
|
|
16
17
|
bakeManagement,
|
|
18
|
+
packageNetapp,
|
|
17
19
|
reportBakeChildFailure,
|
|
18
20
|
stageNetappsFromRegistry,
|
|
19
21
|
verifyNetapp,
|
|
@@ -31,7 +33,7 @@ beforeEach(() => {
|
|
|
31
33
|
afterEach(() => {
|
|
32
34
|
globalThis.fetch = realFetch;
|
|
33
35
|
rmSync(dir, { recursive: true, force: true });
|
|
34
|
-
process.env.CELILO_REGISTRY_URL
|
|
36
|
+
delete process.env.CELILO_REGISTRY_URL;
|
|
35
37
|
});
|
|
36
38
|
|
|
37
39
|
test('fetches each module latest version to <name>.netapp via the download endpoint', async () => {
|
|
@@ -139,6 +141,62 @@ function captureTerminalExit(run: () => void): { exitCodes: unknown[]; stderr: s
|
|
|
139
141
|
// celilo#1302: a lock-free live stack on the builder made the bake child's
|
|
140
142
|
// startup cleanup refuse (exit 3), and the parent reported "Bake step failed"
|
|
141
143
|
// with three install.sh causes for a step that never executed.
|
|
144
|
+
/**
|
|
145
|
+
* Stub console.log/console.error around `run`, returning what each captured.
|
|
146
|
+
*/
|
|
147
|
+
function captureConsole(run: () => void): { stdout: string; stderr: string } {
|
|
148
|
+
const realLog = console.log;
|
|
149
|
+
const realError = console.error;
|
|
150
|
+
const out: string[] = [];
|
|
151
|
+
const err: string[] = [];
|
|
152
|
+
console.log = (...args: unknown[]) => {
|
|
153
|
+
out.push(args.map(String).join(' '));
|
|
154
|
+
};
|
|
155
|
+
console.error = (...args: unknown[]) => {
|
|
156
|
+
err.push(args.map(String).join(' '));
|
|
157
|
+
};
|
|
158
|
+
try {
|
|
159
|
+
run();
|
|
160
|
+
} finally {
|
|
161
|
+
console.log = realLog;
|
|
162
|
+
console.error = realError;
|
|
163
|
+
}
|
|
164
|
+
return { stdout: out.join('\n'), stderr: err.join('\n') };
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// celilo#1258: build-infra is mandatory after cele2e down, and it used to
|
|
168
|
+
// repackage all 37 modules from scratch every run (about 5 minutes to reach a
|
|
169
|
+
// 38 second test) because nothing reused a current .netapp.
|
|
170
|
+
test('a second build-infra with no source change skips repackaging', () => {
|
|
171
|
+
const moduleDir = mkdtempSync(join(tmpdir(), 'module-fixture-'));
|
|
172
|
+
const netappsDir = mkdtempSync(join(tmpdir(), 'netapps-fixture-'));
|
|
173
|
+
writeFileSync(join(moduleDir, 'main.sh'), 'echo hello');
|
|
174
|
+
// First run's output: staged after the source, so it is current.
|
|
175
|
+
const staged = join(netappsDir, `${basename(moduleDir)}.netapp`);
|
|
176
|
+
writeFileSync(staged, 'netapp-bytes');
|
|
177
|
+
|
|
178
|
+
const { stdout, stderr } = captureConsole(() => packageNetapp(moduleDir, netappsDir));
|
|
179
|
+
expect(stdout).toContain('current, skipped');
|
|
180
|
+
// The skip returns before any CLI lookup: no packaging attempt happened.
|
|
181
|
+
expect(stderr).not.toContain('celilo CLI not found');
|
|
182
|
+
|
|
183
|
+
// Recurrence gate: touching a shipped source file flips the answer, and the
|
|
184
|
+
// same call proceeds toward packaging. In a tmpdir there is no monorepo CLI,
|
|
185
|
+
// so getting past the skip surfaces as the CLI-not-found report; the point
|
|
186
|
+
// is that the skip did not fire.
|
|
187
|
+
const later = new Date(Date.now() + 60_000);
|
|
188
|
+
const touched = join(moduleDir, 'main.sh');
|
|
189
|
+
utimesSync(touched, later, later);
|
|
190
|
+
const { stdout: stdoutAfterTouch, stderr: stderrAfterTouch } = captureConsole(() =>
|
|
191
|
+
packageNetapp(moduleDir, netappsDir),
|
|
192
|
+
);
|
|
193
|
+
expect(stderrAfterTouch).toContain('celilo CLI not found');
|
|
194
|
+
expect(stdoutAfterTouch).not.toContain('current, skipped');
|
|
195
|
+
|
|
196
|
+
rmSync(moduleDir, { recursive: true, force: true });
|
|
197
|
+
rmSync(netappsDir, { recursive: true, force: true });
|
|
198
|
+
});
|
|
199
|
+
|
|
142
200
|
test('a refusal exit (3) from the bake child reports the bake did not run, not a bake failure', () => {
|
|
143
201
|
const { exitCodes, stderr } = captureTerminalExit(() => reportBakeChildFailure(3, 0));
|
|
144
202
|
expect(exitCodes).toEqual([3]);
|
package/src/cli/build.ts
CHANGED
|
@@ -35,6 +35,7 @@ import { gunzipSync } from 'node:zlib';
|
|
|
35
35
|
import { stageAptRepo } from '../../scripts/stage-apt-repo';
|
|
36
36
|
import { stageLibsignal } from '../../scripts/stage-libsignal';
|
|
37
37
|
import { explainBuildFailure } from '../doctor';
|
|
38
|
+
import { isNetappCurrent } from '../netapp-staleness';
|
|
38
39
|
import { ensureRegistryServerBundle, ensureTerraformFakeBundle } from '../registry-bundle';
|
|
39
40
|
import { findMonorepoRoot } from '../repo-root';
|
|
40
41
|
import { packNpmRegistryTarballs, stageWebsiteDist } from '../stage-simulator-inputs';
|
|
@@ -433,7 +434,12 @@ export async function fetchSiteFile(
|
|
|
433
434
|
writeFileSync(join(destDir, name), Buffer.from(await res.arrayBuffer()));
|
|
434
435
|
}
|
|
435
436
|
|
|
436
|
-
|
|
437
|
+
/**
|
|
438
|
+
* Package one module directory into `netappsDir` as `<name>.netapp`, skipping
|
|
439
|
+
* the work entirely when the staged netapp is already current over the source
|
|
440
|
+
* (celilo#1258). Exported for its recurrence test.
|
|
441
|
+
*/
|
|
442
|
+
export function packageNetapp(moduleDir: string, netappsDir: string): void {
|
|
437
443
|
const absDir = resolve(moduleDir);
|
|
438
444
|
if (!existsSync(absDir)) {
|
|
439
445
|
console.error(` ${red}skip${reset} ${moduleDir} ${dim}(not found)${reset}`);
|
|
@@ -443,6 +449,15 @@ function packageNetapp(moduleDir: string, netappsDir: string): void {
|
|
|
443
449
|
const name = basename(absDir);
|
|
444
450
|
const out = join(netappsDir, `${name}.netapp`);
|
|
445
451
|
|
|
452
|
+
// Skip a module whose staged .netapp is newer than every source file
|
|
453
|
+
// (celilo#1258): repackaging it again would reproduce the same bytes, and
|
|
454
|
+
// doing that for all 37 modules cost about 5 minutes on every build-infra
|
|
455
|
+
// run. A source edit flips the newest mtime and the module repackages.
|
|
456
|
+
if (isNetappCurrent(out, absDir)) {
|
|
457
|
+
console.log(` ${String(name).padEnd(20)} ${dim}· current, skipped${reset}`);
|
|
458
|
+
return;
|
|
459
|
+
}
|
|
460
|
+
|
|
446
461
|
process.stdout.write(` ${String(name).padEnd(20)} `);
|
|
447
462
|
const start = Date.now();
|
|
448
463
|
|
package/src/container-manager.ts
CHANGED
|
@@ -22,7 +22,7 @@ import {
|
|
|
22
22
|
referencedImages,
|
|
23
23
|
registryUploadsHostDir,
|
|
24
24
|
} from './docker-compose-generator';
|
|
25
|
-
import { type ModuleHost, parseModuleHost } from './module-host';
|
|
25
|
+
import { type ModuleHost, parseModuleHost, parseModuleWhere } from './module-host';
|
|
26
26
|
import { CONTAINER_PREFIX, GUEST_PROJECT_LABEL } from './proxmox-provisioner';
|
|
27
27
|
import { ensureSharedInfra } from './shared-infra';
|
|
28
28
|
import { SIMULATOR_IPS } from './simulator-ips';
|
|
@@ -1110,6 +1110,23 @@ function buildNetworkHandle(
|
|
|
1110
1110
|
return host;
|
|
1111
1111
|
},
|
|
1112
1112
|
|
|
1113
|
+
async targetIp(moduleId: string): Promise<string> {
|
|
1114
|
+
const where = dockerExec(
|
|
1115
|
+
projectName,
|
|
1116
|
+
composeDir,
|
|
1117
|
+
'management',
|
|
1118
|
+
`celilo module where ${moduleId} --json`,
|
|
1119
|
+
);
|
|
1120
|
+
const addresses = parseModuleWhere(where.stdout);
|
|
1121
|
+
if (addresses.length === 0) {
|
|
1122
|
+
throw new Error(
|
|
1123
|
+
`Could not resolve an address for '${moduleId}'. Its deploy did not record one in the inventory\n` +
|
|
1124
|
+
`(or the CLI answered something unparsable). Raw:\n${where.stdout.slice(0, 400)}`,
|
|
1125
|
+
);
|
|
1126
|
+
}
|
|
1127
|
+
return addresses[0];
|
|
1128
|
+
},
|
|
1129
|
+
|
|
1113
1130
|
async execOnModuleHost(moduleId, cmd, timeoutMs = 60_000): Promise<ExecResult> {
|
|
1114
1131
|
const host = await this.moduleHost(moduleId);
|
|
1115
1132
|
return host.reach === 'plain'
|
|
@@ -1280,6 +1297,35 @@ function buildNetworkHandle(
|
|
|
1280
1297
|
await handle.celilo(`system config set network.${zone}.gateway ${ZONE_GATEWAYS[zone]}`);
|
|
1281
1298
|
}
|
|
1282
1299
|
|
|
1300
|
+
// When fw-main carries the secure-mgmt leg, the management box sits on the
|
|
1301
|
+
// control-plane network behind this firewall, and the firewall must TRUST
|
|
1302
|
+
// that subnet or default-DROP drops the SSH `machine add` and every later
|
|
1303
|
+
// hook/converge needs (celilo#1353: 19 suites died at machine add with a
|
|
1304
|
+
// misleading key-mismatch message; the live stack showed the packet
|
|
1305
|
+
// timing out in fw-main's FORWARD chain, policy DROP, trusted sources =
|
|
1306
|
+
// internal only).
|
|
1307
|
+
//
|
|
1308
|
+
// The firewall DERIVES control-plane trust from where the celilo-mgmt
|
|
1309
|
+
// module is deployed (apps/celilo/src/hooks/capability-loader.ts
|
|
1310
|
+
// loadControlPlaneSubnet), falling back to the internal subnet. Suites
|
|
1311
|
+
// that machine-add before any celilo-mgmt deploy have neither, so the
|
|
1312
|
+
// declared control-plane subnet is trusted by nothing. The operator in
|
|
1313
|
+
// this topology declares it explicitly — `firewall.trusted_subnets` is the
|
|
1314
|
+
// product surface for exactly that (composeTrustedSubnets origin:
|
|
1315
|
+
// operator-override) — and the harness models that operator rather than
|
|
1316
|
+
// widening the product's fallback, which the approved design
|
|
1317
|
+
// (openspec/changes/recognize-management-network, D2) deliberately kept
|
|
1318
|
+
// as "previous behaviour + report".
|
|
1319
|
+
//
|
|
1320
|
+
// Idempotent with the derived path: composeTrustedSubnets dedupes by
|
|
1321
|
+
// subnet, so a suite that later deploys celilo-mgmt on secure-mgmt renders
|
|
1322
|
+
// the same ruleset.
|
|
1323
|
+
if (fwMainHasSecureMgmtLeg(join(composeDir, COMPOSE_FILE))) {
|
|
1324
|
+
await handle.celilo(
|
|
1325
|
+
`system config set firewall.trusted_subnets ${ZONE_SUBNETS['secure-mgmt']}`,
|
|
1326
|
+
);
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1283
1329
|
// fw-main is registered as an internal-zone machine; iptables deploys to it.
|
|
1284
1330
|
await handle.celilo(
|
|
1285
1331
|
`machine add ${firewallIp} --ssh-user root --ssh-key-file /root/.ssh/id_ed25519 --zone internal`,
|
|
@@ -625,8 +625,9 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
|
|
|
625
625
|
dmz: zoneNetworkDef('dmz'),
|
|
626
626
|
app: zoneNetworkDef('app'),
|
|
627
627
|
secure: zoneNetworkDef('secure'),
|
|
628
|
-
// Only when
|
|
629
|
-
//
|
|
628
|
+
// Only when the control plane is in use (celilo-mgr, a secure-mgmt
|
|
629
|
+
// machine, or the proxmox simulator). With the default managementZone of
|
|
630
|
+
// `secure-mgmt` this is now present in every generated compose.
|
|
630
631
|
...(needsSecureMgmt ? { 'secure-mgmt': zoneNetworkDef('secure-mgmt') } : {}),
|
|
631
632
|
'isp-external': networkDef('203.0.113.0/24', '203.0.113.250'),
|
|
632
633
|
// internet-external is owned by shared infra; real-internet is per-test
|
|
@@ -747,8 +748,27 @@ export function generateTestComposeYaml(config: NetworkConfig, celiloRoot?: stri
|
|
|
747
748
|
// On its own control-plane network the only router in reach is fw-main's
|
|
748
749
|
// leg there — both for the default route and for the segmented zones.
|
|
749
750
|
DEFAULT_GATEWAY: zone === 'secure-mgmt' ? SECURE_MGMT_GATEWAY : managementDefaultGw,
|
|
750
|
-
|
|
751
|
+
// BOTH branches always set the hop, never let the image's baked value
|
|
752
|
+
// through: the bake `docker commit`s the management container, so a
|
|
753
|
+
// bake from the secure-mgmt default baked FW_MAIN_HOP=10.226.120.1
|
|
754
|
+
// into :latest, and every internal-topology suite then inherited a
|
|
755
|
+
// nexthop that is not on-link from the internal LAN. The mgmt box
|
|
756
|
+
// exited 2 at the segmented-zone routes in management-routes.sh and
|
|
757
|
+
// crash-looped (celilo#1351). Compose env overrides image env, so an
|
|
758
|
+
// always-present FW_MAIN_HOP immunizes every topology against any
|
|
759
|
+
// baked value, current and future.
|
|
760
|
+
FW_MAIN_HOP: zone === 'secure-mgmt' ? SECURE_MGMT_GATEWAY : ZONE_GATEWAYS.internal,
|
|
751
761
|
CELILO_REGISTRY_URL: 'http://e2e-registry.lab',
|
|
762
|
+
// The hook jail is REQUIRED in the e2e, not `auto` (peba, 2026-09-08).
|
|
763
|
+
// ce-rez7 made an unset policy resolve to `off`, which silently turned
|
|
764
|
+
// hook-jail-trespass into a test of nothing: its stage 2 (no policy set)
|
|
765
|
+
// and its stage 3 control (explicit `off`) became the same experiment.
|
|
766
|
+
// celilo#1329. `required` beats `auto` here because a missing bubblewrap
|
|
767
|
+
// backend is then a hard failure instead of a silent drop to unjailed, so
|
|
768
|
+
// it cannot rot the same way twice. A per-command `CELILO_HOOK_JAIL=off`
|
|
769
|
+
// prefix still overrides this, which is how the deliberate unjailed
|
|
770
|
+
// controls in both jail suites keep working.
|
|
771
|
+
CELILO_HOOK_JAIL: 'required',
|
|
752
772
|
// `cele2e run --source-cli` sets this, and the image's shim reads it to
|
|
753
773
|
// run the mounted workspace instead of the CLI install.sh installed.
|
|
754
774
|
// Off by default: the installed CLI is the artifact under test, and
|
|
@@ -979,6 +999,16 @@ export function generateComposeYaml(config: NetworkConfig, celiloRoot = '..'): s
|
|
|
979
999
|
environment: {
|
|
980
1000
|
DEFAULT_GATEWAY: managementDefaultGw,
|
|
981
1001
|
CELILO_REGISTRY_URL: 'http://e2e-registry.lab',
|
|
1002
|
+
// The hook jail is REQUIRED in the e2e, not `auto` (peba, 2026-09-08).
|
|
1003
|
+
// ce-rez7 made an unset policy resolve to `off`, which silently turned
|
|
1004
|
+
// hook-jail-trespass into a test of nothing: its stage 2 (no policy set)
|
|
1005
|
+
// and its stage 3 control (explicit `off`) became the same experiment.
|
|
1006
|
+
// celilo#1329. `required` beats `auto` here because a missing bubblewrap
|
|
1007
|
+
// backend is then a hard failure instead of a silent drop to unjailed, so
|
|
1008
|
+
// it cannot rot the same way twice. A per-command `CELILO_HOOK_JAIL=off`
|
|
1009
|
+
// prefix still overrides this, which is how the deliberate unjailed
|
|
1010
|
+
// controls in both jail suites keep working.
|
|
1011
|
+
CELILO_HOOK_JAIL: 'required',
|
|
982
1012
|
},
|
|
983
1013
|
});
|
|
984
1014
|
|
package/src/extract-failure.ts
CHANGED
|
@@ -28,26 +28,48 @@ export interface StageTally {
|
|
|
28
28
|
* as "9 failed" in a 10-stage suite — nine counts of a defect that does not
|
|
29
29
|
* exist, and a summary that buries the one that does.
|
|
30
30
|
*
|
|
31
|
-
* The `Skipped:`
|
|
32
|
-
*
|
|
31
|
+
* The `error: Skipped:` line is the semantic signal, and bun prints it in BOTH
|
|
32
|
+
* orders relative to the `(fail)` marker: a thrown error's block (source
|
|
33
|
+
* excerpt, error line, stack) lands BEFORE its marker, while a top-level error
|
|
34
|
+
* print and bun's own timeout pointer land AFTER it. A one-directional window
|
|
35
|
+
* misattributes both ways — measured as ce-59f9: a window after a timeout
|
|
36
|
+
* marker catches the NEXT test's excerpt quoting the `Skipped:` template (and
|
|
37
|
+
* calls the timeout a skip), and a skip's reason pushed past the window calls
|
|
38
|
+
* the skip a failure. So each reason line is attributed to its NEAREST marker
|
|
39
|
+
* instead, ties to the earlier one (a thrown-error block sits closest to the
|
|
40
|
+
* marker that follows it), and a marker no reason line claims is a failure on
|
|
41
|
+
* its own merits.
|
|
33
42
|
*/
|
|
34
43
|
export function tallyStages(lines: string[]): StageTally {
|
|
35
44
|
const plain = lines.map(stripAnsi);
|
|
36
45
|
const isFailure = (l: string): boolean => /^\(fail\)\s+\S/.test(l);
|
|
37
|
-
|
|
38
|
-
|
|
46
|
+
const isSkippedReason = (l: string): boolean => /^\s*error:\s*Skipped:\s/.test(l);
|
|
47
|
+
|
|
48
|
+
const markers: number[] = [];
|
|
49
|
+
for (let i = 0; i < plain.length; i++) {
|
|
50
|
+
if (isFailure(plain[i])) markers.push(i);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const skippedMarkers = new Set<number>();
|
|
54
|
+
const MAX_REASON_DISTANCE = 8;
|
|
39
55
|
for (let i = 0; i < plain.length; i++) {
|
|
40
|
-
if (!
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
56
|
+
if (!isSkippedReason(plain[i])) continue;
|
|
57
|
+
let nearest = -1;
|
|
58
|
+
let nearestDist = Number.POSITIVE_INFINITY;
|
|
59
|
+
for (const m of markers) {
|
|
60
|
+
const dist = Math.abs(m - i);
|
|
61
|
+
if (dist <= MAX_REASON_DISTANCE && dist < nearestDist) {
|
|
62
|
+
nearest = m;
|
|
63
|
+
nearestDist = dist;
|
|
64
|
+
}
|
|
49
65
|
}
|
|
50
|
-
if (
|
|
66
|
+
if (nearest >= 0) skippedMarkers.add(nearest);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
let failed = 0;
|
|
70
|
+
let skipped = 0;
|
|
71
|
+
for (const m of markers) {
|
|
72
|
+
if (skippedMarkers.has(m)) skipped++;
|
|
51
73
|
else failed++;
|
|
52
74
|
}
|
|
53
75
|
return { failed, skipped };
|
package/src/live-stack.test.ts
CHANGED
|
@@ -8,7 +8,10 @@ import {
|
|
|
8
8
|
LiveStackError,
|
|
9
9
|
type ProcessProbe,
|
|
10
10
|
SHARED_ORPHAN_MIN_AGE_MS,
|
|
11
|
+
UNIT_ONLY_ENV_ENTRY,
|
|
12
|
+
environMarksUnitOnly,
|
|
11
13
|
findLiveE2eStack,
|
|
14
|
+
isUnitOnlyProcess,
|
|
12
15
|
looksLikeE2eRunCommand,
|
|
13
16
|
} from './live-stack';
|
|
14
17
|
import type { LockStatus } from './run-lock';
|
|
@@ -74,7 +77,13 @@ describe('findLiveE2eStack', () => {
|
|
|
74
77
|
const { docker } = fakeDocker(
|
|
75
78
|
'celilo-e2e-shared_namecheap-dns\trunning\ncelilo-e2e-1788769175864_fw-main\texited\n',
|
|
76
79
|
);
|
|
77
|
-
|
|
80
|
+
// The default probe reads the REAL host process table. Under another
|
|
81
|
+
// session's live e2e run (celilo#1332: the bun 1.4.2 grind) the
|
|
82
|
+
// shared-only + live-runner branch fires instead of the unreadable-age
|
|
83
|
+
// branch pinned here, and the refusal names pid counts instead of the
|
|
84
|
+
// container. A test pinning a message shape injects its inputs: an empty
|
|
85
|
+
// probe makes the pinned shape deterministic.
|
|
86
|
+
const refusal = findLiveE2eStack(docker, noLock, () => []);
|
|
78
87
|
expect(refusal).not.toBeNull();
|
|
79
88
|
expect(refusal?.reason).toContain('celilo-e2e-shared_namecheap-dns');
|
|
80
89
|
expect(refusal?.reason).not.toContain('celilo-e2e-1788769175864_fw-main');
|
|
@@ -145,6 +154,48 @@ describe('findLiveE2eStack', () => {
|
|
|
145
154
|
});
|
|
146
155
|
});
|
|
147
156
|
|
|
157
|
+
describe('unit-only exclusion (celilo#1320)', () => {
|
|
158
|
+
const environOf =
|
|
159
|
+
(byPid: Record<number, string | null>) =>
|
|
160
|
+
(pid: number): string | null =>
|
|
161
|
+
byPid[pid] ?? null;
|
|
162
|
+
|
|
163
|
+
test('an environ carrying the marker is unit-only', () => {
|
|
164
|
+
expect(
|
|
165
|
+
environMarksUnitOnly(['PATH=/usr/bin', UNIT_ONLY_ENV_ENTRY, 'HOME=/root'].join('\0')),
|
|
166
|
+
).toBe(true);
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
test('the marker must be a whole entry, not a substring', () => {
|
|
170
|
+
expect(environMarksUnitOnly(['PATH=/x', 'SOMETHING_CELILO_UNIT_ONLY=12'].join('\0'))).toBe(
|
|
171
|
+
false,
|
|
172
|
+
);
|
|
173
|
+
expect(environMarksUnitOnly('CELILO_UNIT_ONLY=12')).toBe(false);
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
test('an unreadable environ is NOT unit-only (inconclusive never reaps)', () => {
|
|
177
|
+
expect(environMarksUnitOnly(null)).toBe(false);
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
test('isUnitOnlyProcess reads the environ of the matched pid', () => {
|
|
181
|
+
const environ = environOf({
|
|
182
|
+
101: ['PATH=/x', UNIT_ONLY_ENV_ENTRY].join('\0'),
|
|
183
|
+
102: 'PATH=/x',
|
|
184
|
+
103: null,
|
|
185
|
+
});
|
|
186
|
+
expect(isUnitOnlyProcess(101, environ)).toBe(true);
|
|
187
|
+
expect(isUnitOnlyProcess(102, environ)).toBe(false);
|
|
188
|
+
expect(isUnitOnlyProcess(103, environ)).toBe(false);
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
test('the marker matches what test:unit actually exports', () => {
|
|
192
|
+
const rootPackageJson = JSON.parse(
|
|
193
|
+
readFileSync(join(DIR, '../../../package.json'), 'utf-8'),
|
|
194
|
+
) as { scripts: Record<string, string> };
|
|
195
|
+
expect(rootPackageJson.scripts['test:unit']).toContain('CELILO_UNIT_ONLY=1');
|
|
196
|
+
});
|
|
197
|
+
});
|
|
198
|
+
|
|
148
199
|
describe('looksLikeE2eRunCommand', () => {
|
|
149
200
|
test('matches the cele2e CLI by any argv mention', () => {
|
|
150
201
|
expect(looksLikeE2eRunCommand('cele2e run smoke')).toBe(true);
|
package/src/live-stack.ts
CHANGED
|
@@ -40,10 +40,21 @@
|
|
|
40
40
|
* binary only exists inside a checkout), and every refusal reason carries the
|
|
41
41
|
* `[infra-refusal]` marker with its exit code, so a log search distinguishes
|
|
42
42
|
* an environment problem from a check failure (celilo#1314 directions 2+3).
|
|
43
|
+
*
|
|
44
|
+
* The unit-only exclusion (celilo#1320). ci/validate runs `bun run test:unit`
|
|
45
|
+
* on the same persistent builder as npm-consumer-smoke, and every `bun test`
|
|
46
|
+
* child it spawns matched the probe's bun-test arm while managing no Docker
|
|
47
|
+
* at all — the smoke job's teardown then declined to clear a finished run's
|
|
48
|
+
* shared stack, and the next run inside the orphan hour refused. A process
|
|
49
|
+
* whose environ carries CELILO_UNIT_ONLY=1 (the root script exports it; every
|
|
50
|
+
* child inherits) is a unit-test run and is not run evidence. An unreadable
|
|
51
|
+
* environ keeps the match: the exclusion follows the same bias as the
|
|
52
|
+
* evidence test itself, inconclusive never reaps.
|
|
43
53
|
*/
|
|
44
54
|
|
|
45
55
|
import type { ExecFileSyncOptions } from 'node:child_process';
|
|
46
56
|
import { execFileSync } from 'node:child_process';
|
|
57
|
+
import { readFileSync } from 'node:fs';
|
|
47
58
|
import { SHARED_PROJECT_NAME } from './docker-compose-generator';
|
|
48
59
|
import {
|
|
49
60
|
type LockHolder,
|
|
@@ -103,6 +114,48 @@ export const SHARED_ORPHAN_MIN_AGE_MS = 60 * 60_000;
|
|
|
103
114
|
*/
|
|
104
115
|
export type ProcessProbe = () => number[];
|
|
105
116
|
|
|
117
|
+
/**
|
|
118
|
+
* The env entry `bun run test:unit` exports (root package.json). A process
|
|
119
|
+
* carrying it is a unit-test run: the e2e unit suites exercise Docker only
|
|
120
|
+
* through injected readers, so it never manages the shared stack.
|
|
121
|
+
*/
|
|
122
|
+
export const UNIT_ONLY_ENV_ENTRY = 'CELILO_UNIT_ONLY=1';
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Raw environment of one pid (NUL-separated entries), or null when it cannot
|
|
126
|
+
* be read — /proc is Linux-only, and a foreign-uid environ is unreadable even
|
|
127
|
+
* where /proc exists.
|
|
128
|
+
*/
|
|
129
|
+
export type EnvironReader = (pid: number) => string | null;
|
|
130
|
+
|
|
131
|
+
export const realEnvironReader: EnvironReader = (pid) => {
|
|
132
|
+
try {
|
|
133
|
+
return readFileSync(`/proc/${pid}/environ`, 'utf8');
|
|
134
|
+
} catch {
|
|
135
|
+
return null;
|
|
136
|
+
}
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Does a raw environ block mark its process as unit-only? Pure so the
|
|
141
|
+
* probe's exclusion is testable without a filesystem. A null (unreadable)
|
|
142
|
+
* environ is NOT unit-only: an unreadable answer is a missing answer, and the
|
|
143
|
+
* probe biases to refusing — a false refusal costs the operator one docker
|
|
144
|
+
* command, a false miss reaps a live run.
|
|
145
|
+
*/
|
|
146
|
+
export function environMarksUnitOnly(environ: string | null): boolean {
|
|
147
|
+
if (environ === null) return false;
|
|
148
|
+
return environ.split('\0').includes(UNIT_ONLY_ENV_ENTRY);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** The exclusion realProcessProbe applies to its matches; injectable for tests. */
|
|
152
|
+
export function isUnitOnlyProcess(
|
|
153
|
+
pid: number,
|
|
154
|
+
environ: EnvironReader = realEnvironReader,
|
|
155
|
+
): boolean {
|
|
156
|
+
return environMarksUnitOnly(environ(pid));
|
|
157
|
+
}
|
|
158
|
+
|
|
106
159
|
/**
|
|
107
160
|
* Does this process command line look like an e2e run? Pure so the probe's
|
|
108
161
|
* reach is testable. Matches the two ways a run actually exists:
|
|
@@ -171,7 +224,12 @@ export const realProcessProbe: ProcessProbe = (): number[] => {
|
|
|
171
224
|
if (sep <= 0) continue;
|
|
172
225
|
const pid = Number.parseInt(trimmed.slice(0, sep), 10);
|
|
173
226
|
if (!Number.isFinite(pid) || family.has(pid)) continue;
|
|
174
|
-
if (looksLikeE2eRunCommand(trimmed.slice(sep + 1)))
|
|
227
|
+
if (!looksLikeE2eRunCommand(trimmed.slice(sep + 1))) continue;
|
|
228
|
+
// celilo#1320: a unit-test run matches the command probe but manages no
|
|
229
|
+
// Docker. An unreadable environ keeps the match — inconclusive never
|
|
230
|
+
// reaps.
|
|
231
|
+
if (isUnitOnlyProcess(pid)) continue;
|
|
232
|
+
pids.push(pid);
|
|
175
233
|
}
|
|
176
234
|
return pids;
|
|
177
235
|
};
|