@celilo/e2e 0.19.3 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -13
- package/bin/e2e-bake-management +196 -12
- package/bin/e2e-infra +0 -1
- package/bin/e2e-up +14 -3
- package/docker/Dockerfile.observer +12 -1
- package/docker/Dockerfile.target-machine +22 -1
- package/npm-registry-server/package.json +1 -1
- package/package.json +3 -3
- package/registry-server/package.json +1 -1
- package/scripts/pack-celilo-packages.ts +15 -0
- package/src/block-timing.test.ts +559 -0
- package/src/block-timing.ts +366 -0
- package/src/cli/build.test.ts +135 -4
- package/src/cli/build.ts +237 -94
- package/src/cli/command-registry.ts +21 -0
- package/src/cli/command-tree-parser.ts +11 -2
- package/src/cli/completion.ts +9 -0
- package/src/cli/host.ts +252 -0
- package/src/cli/index.ts +78 -51
- package/src/cli/module-discovery.ts +108 -13
- package/src/cli/scaffold.ts +18 -26
- package/src/container-manager.cleanup.test.ts +284 -0
- package/src/container-manager.runner.test.ts +351 -0
- package/src/container-manager.test.ts +84 -0
- package/src/container-manager.ts +721 -185
- package/src/docker-compose-generator.ts +135 -61
- package/src/doctor.test.ts +259 -4
- package/src/doctor.ts +276 -3
- package/src/exit-cleanup.test.ts +83 -1
- package/src/fleet-nameserver-gate.test.ts +45 -0
- package/src/host-vm.test.ts +156 -0
- package/src/host-vm.ts +230 -0
- package/src/index.ts +11 -0
- package/src/live-stack.test.ts +300 -0
- package/src/live-stack.ts +355 -0
- package/src/no-unjustified-sleep.test.ts +90 -0
- package/src/proxmox-provisioner.test.ts +18 -2
- package/src/proxmox-provisioner.ts +22 -0
- package/src/public-sim-routes.test.ts +9 -2
- package/src/repo-root.ts +33 -0
- package/src/run-args.test.ts +76 -0
- package/src/run-args.ts +89 -0
- package/src/runner.ts +213 -8
- package/src/shared-infra.ts +88 -32
- package/src/socks-proxy.ts +2 -0
- package/src/source-fingerprint.test.ts +213 -0
- package/src/source-fingerprint.ts +212 -0
- package/src/stage-simulator-inputs.ts +93 -0
- package/src/stages.ts +133 -0
- package/src/wait-for-run.ts +1 -0
package/src/runner.ts
CHANGED
|
@@ -22,6 +22,16 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from
|
|
|
22
22
|
import { basename, dirname, join, resolve } from 'node:path';
|
|
23
23
|
import { type DisplayMode, ProgressDisplay } from '@celilo/cli-display';
|
|
24
24
|
import { parse as parseYaml } from 'yaml';
|
|
25
|
+
import {
|
|
26
|
+
type BlockTiming,
|
|
27
|
+
type OverBudgetBlock,
|
|
28
|
+
declaredBlockCaps,
|
|
29
|
+
mergeBlockTiming,
|
|
30
|
+
overBudgetBlocks,
|
|
31
|
+
parseBlockDurations,
|
|
32
|
+
serializeBlockTiming,
|
|
33
|
+
tightBlocks,
|
|
34
|
+
} from './block-timing';
|
|
25
35
|
import {
|
|
26
36
|
emitRunCompleted,
|
|
27
37
|
emitRunFailed,
|
|
@@ -33,6 +43,7 @@ import { diagnose, formatReport } from './doctor';
|
|
|
33
43
|
import { type StageTally, extractFailureMessage, stripAnsi, tallyStages } from './extract-failure';
|
|
34
44
|
import { writeLastRun } from './last-run';
|
|
35
45
|
import { parseLine } from './parse-line';
|
|
46
|
+
import { KNOWN_RUN_FLAGS, filterTestFilesByPatterns, parseRunArgs } from './run-args';
|
|
36
47
|
import { E2eBusyError, acquireRunLock, markKept } from './run-lock';
|
|
37
48
|
import { SIMULATOR_IPS } from './simulator-ips';
|
|
38
49
|
|
|
@@ -68,6 +79,8 @@ const testsPath = moduleDir
|
|
|
68
79
|
/** Set by `cele2e run --all` so a module-dirs run ALSO picks up the repo's top-level e2e/tests/. */
|
|
69
80
|
const topLevelTestsPath = process.env.E2E_TOP_LEVEL_TESTS;
|
|
70
81
|
const timingFile = join(testDir, '.e2e-timing.json');
|
|
82
|
+
/** Per-test-block durations. The 300s cap is per BLOCK, and nothing else records that. */
|
|
83
|
+
const blockTimingFile = join(testDir, '.e2e-block-timing.json');
|
|
71
84
|
const persistentFile = join(testDir, '.e2e-persistent.json');
|
|
72
85
|
|
|
73
86
|
// Parse flags out of argv
|
|
@@ -76,6 +89,14 @@ const flagKeep = rawArgs.includes('--keep');
|
|
|
76
89
|
const flagReuse = rawArgs.includes('--reuse');
|
|
77
90
|
const flagLive = rawArgs.includes('--live');
|
|
78
91
|
const flagPublished = rawArgs.includes('--published');
|
|
92
|
+
// --source-cli: run the celilo CLI from the mounted workspace instead of the one
|
|
93
|
+
// baked into the management image. Off by default because the baked CLI is the
|
|
94
|
+
// artifact under test, and running the source over the bind mount roughly
|
|
95
|
+
// doubles every celilo command's start-up (measured 0.14s vs 0.31s on virtiofs,
|
|
96
|
+
// 0.16s vs 0.56s on sshfs), which a 25-40 command test pays each time. The
|
|
97
|
+
// image's shim reads this from the container environment; the compose generator
|
|
98
|
+
// forwards it. Use it to iterate without a build-infra.
|
|
99
|
+
const flagSourceCli = rawArgs.includes('--source-cli');
|
|
79
100
|
const flagVerbose = rawArgs.includes('--verbose');
|
|
80
101
|
// CI mode: no animated spinner, no [progress:*] markers — just one clean
|
|
81
102
|
// ✔/✗ line per step (plus sub-events) written straight to the log. Forgejo's
|
|
@@ -88,7 +109,40 @@ const flagVerbose = rawArgs.includes('--verbose');
|
|
|
88
109
|
const flagCi = rawArgs.includes('--ci');
|
|
89
110
|
const flagNoCi = rawArgs.includes('--no-ci');
|
|
90
111
|
const ci = flagCi || (!flagNoCi && !!process.env.CI);
|
|
91
|
-
const
|
|
112
|
+
const RUN_USAGE = `
|
|
113
|
+
Usage: cele2e run [pattern...] [flags]
|
|
114
|
+
|
|
115
|
+
Runs e2e suites. Each pattern is a substring matched against test file
|
|
116
|
+
names; no pattern (bare) runs every suite in the current scope.
|
|
117
|
+
|
|
118
|
+
Flags:
|
|
119
|
+
--keep Keep network running after tests
|
|
120
|
+
--reuse Reuse existing network if running
|
|
121
|
+
--live Use live (non-simulated) internet
|
|
122
|
+
--published Use published .netapp packages
|
|
123
|
+
--source-cli Run the celilo CLI from the mounted workspace
|
|
124
|
+
--verbose Verbose output
|
|
125
|
+
--ci / --no-ci Force plain CI log output on/off (auto-on when CI env var set)
|
|
126
|
+
--complete Include quarantined (cele2e-ci-unsafe) tests
|
|
127
|
+
--notify Desktop notification when the run finishes
|
|
128
|
+
--shuffle [--seed=<n>] Randomize test order; replay an order with its seed
|
|
129
|
+
|
|
130
|
+
Full CLI help: cele2e --help (run --all is a cele2e-level alias, not a runner flag)
|
|
131
|
+
`;
|
|
132
|
+
|
|
133
|
+
const parsedArgs = parseRunArgs(rawArgs);
|
|
134
|
+
if (parsedArgs.helpRequested) {
|
|
135
|
+
console.log(RUN_USAGE);
|
|
136
|
+
process.exit(0);
|
|
137
|
+
}
|
|
138
|
+
if (parsedArgs.unknownFlags.length > 0) {
|
|
139
|
+
console.error(`Unknown flag(s): ${parsedArgs.unknownFlags.join(' ')}`);
|
|
140
|
+
console.error(
|
|
141
|
+
`Known flags: ${[...KNOWN_RUN_FLAGS, '--seed=<n>'].join(' ')} (--help for run usage)`,
|
|
142
|
+
);
|
|
143
|
+
process.exit(2);
|
|
144
|
+
}
|
|
145
|
+
const patterns = parsedArgs.patterns;
|
|
92
146
|
// Quarantine: tests whose file contains the marker `cele2e-ci-unsafe` (a comment
|
|
93
147
|
// naming the reason + ISS) are known-failing / builder-flaky and tracked
|
|
94
148
|
// separately, so they must never turn a broad run red. They are SKIPPED by
|
|
@@ -161,6 +215,7 @@ const bold = '\x1b[1m';
|
|
|
161
215
|
const dim = '\x1b[2m';
|
|
162
216
|
const green = '\x1b[32m';
|
|
163
217
|
const red = '\x1b[31m';
|
|
218
|
+
const yellow = '\x1b[33m';
|
|
164
219
|
const reset = '\x1b[0m';
|
|
165
220
|
|
|
166
221
|
const interactive =
|
|
@@ -182,6 +237,18 @@ function saveTiming(history: TimingHistory): void {
|
|
|
182
237
|
writeFileSync(timingFile, `${JSON.stringify(history, Object.keys(history).sort(), 2)}\n`);
|
|
183
238
|
}
|
|
184
239
|
|
|
240
|
+
function loadBlockTiming(): BlockTiming {
|
|
241
|
+
try {
|
|
242
|
+
return JSON.parse(readFileSync(blockTimingFile, 'utf-8'));
|
|
243
|
+
} catch {
|
|
244
|
+
return {};
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function saveBlockTiming(timing: BlockTiming): void {
|
|
249
|
+
writeFileSync(blockTimingFile, serializeBlockTiming(timing));
|
|
250
|
+
}
|
|
251
|
+
|
|
185
252
|
function formatDuration(secs: number): string {
|
|
186
253
|
if (secs >= 60) return `${Math.floor(secs / 60)}m${Math.abs(secs) % 60}s`;
|
|
187
254
|
return `${secs}s`;
|
|
@@ -202,12 +269,22 @@ interface TestResult {
|
|
|
202
269
|
rawTail?: string;
|
|
203
270
|
hadDebugPause?: boolean;
|
|
204
271
|
projectName?: string;
|
|
272
|
+
/** Per-`test()` durations in ms, so the per-block cap has a durable record. */
|
|
273
|
+
blocks?: Record<string, number>;
|
|
205
274
|
/**
|
|
206
275
|
* Real stage failures vs stages a failing earlier stage blocked. Without the
|
|
207
276
|
* split, one bad fixture line in stage 1 of a 10-stage suite reports as "9
|
|
208
277
|
* failed" — nine counts of a defect that does not exist.
|
|
209
278
|
*/
|
|
210
279
|
stages?: StageTally;
|
|
280
|
+
/**
|
|
281
|
+
* The test child exited 3: the live-stack guard refused to clean up because
|
|
282
|
+
* another run's stack is in the way (celilo#1297). Bun itself never exits 3
|
|
283
|
+
* — it uses 1 for failures — so the code is unambiguous. The run stops here:
|
|
284
|
+
* every remaining suite would hit the same refusal, and continuing would
|
|
285
|
+
* only pile more noise onto one cause.
|
|
286
|
+
*/
|
|
287
|
+
refused?: boolean;
|
|
211
288
|
}
|
|
212
289
|
|
|
213
290
|
async function runTest(
|
|
@@ -219,6 +296,7 @@ async function runTest(
|
|
|
219
296
|
): Promise<TestResult> {
|
|
220
297
|
const start = Date.now();
|
|
221
298
|
const logFile = join(logDir, 'output.log');
|
|
299
|
+
const junitFile = join(logDir, 'junit.xml');
|
|
222
300
|
|
|
223
301
|
display.reset(start, expectedDuration);
|
|
224
302
|
let hadDebugPause = false;
|
|
@@ -238,6 +316,7 @@ async function runTest(
|
|
|
238
316
|
if (flagPublished) {
|
|
239
317
|
testEnv.CELILO_E2E_INFRA_ROOT = ''; // empty = force published mode
|
|
240
318
|
}
|
|
319
|
+
if (flagSourceCli) testEnv.CELILO_E2E_SOURCE_CLI = '1';
|
|
241
320
|
if (flagReuse) {
|
|
242
321
|
// Read the persistent project and pass it to the test
|
|
243
322
|
try {
|
|
@@ -252,12 +331,22 @@ async function runTest(
|
|
|
252
331
|
}
|
|
253
332
|
}
|
|
254
333
|
|
|
255
|
-
// 1 hour timeout per test (debug sessions can be long)
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
334
|
+
// 1 hour timeout per test (debug sessions can be long). Each test() block's
|
|
335
|
+
// own third argument overrides this — measured on bun 1.3.3, including when
|
|
336
|
+
// the body swallows its own errors — so this is a backstop, not the budget.
|
|
337
|
+
//
|
|
338
|
+
// The JUnit report is the ONLY source of a duration for a block that PASSED:
|
|
339
|
+
// bun's console names a block only when it fails. It does not change the
|
|
340
|
+
// console output, so the display and the failure parsers are untouched.
|
|
341
|
+
const proc = spawn(
|
|
342
|
+
'bun',
|
|
343
|
+
['test', '--timeout', '3600000', '--reporter=junit', `--reporter-outfile=${junitFile}`, file],
|
|
344
|
+
{
|
|
345
|
+
cwd: PKG_DIR,
|
|
346
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
347
|
+
env: testEnv,
|
|
348
|
+
},
|
|
349
|
+
);
|
|
261
350
|
|
|
262
351
|
let stdout = '';
|
|
263
352
|
let stderr = '';
|
|
@@ -363,6 +452,7 @@ async function runTest(
|
|
|
363
452
|
|
|
364
453
|
const expectMatch = stdout.match(/(\d+) expect\(\) calls/);
|
|
365
454
|
const expectCount = expectMatch ? Number.parseInt(expectMatch[1], 10) : undefined;
|
|
455
|
+
const blocks = parseBlockDurations(lines, readJunit(junitFile));
|
|
366
456
|
|
|
367
457
|
if (code === 0) {
|
|
368
458
|
if (expectCount === 0) {
|
|
@@ -370,6 +460,7 @@ async function runTest(
|
|
|
370
460
|
name,
|
|
371
461
|
status: 'suspicious',
|
|
372
462
|
duration,
|
|
463
|
+
blocks,
|
|
373
464
|
hadDebugPause,
|
|
374
465
|
projectName: capturedProjectName,
|
|
375
466
|
error: '0 assertions — test likely exited before reaching assertions',
|
|
@@ -379,6 +470,7 @@ async function runTest(
|
|
|
379
470
|
name,
|
|
380
471
|
status: 'pass',
|
|
381
472
|
duration,
|
|
473
|
+
blocks,
|
|
382
474
|
hadDebugPause,
|
|
383
475
|
projectName: capturedProjectName,
|
|
384
476
|
});
|
|
@@ -403,17 +495,99 @@ async function runTest(
|
|
|
403
495
|
name,
|
|
404
496
|
status: 'fail',
|
|
405
497
|
duration,
|
|
498
|
+
blocks,
|
|
406
499
|
hadDebugPause,
|
|
407
500
|
projectName: capturedProjectName,
|
|
408
501
|
error: extractFailureMessage(lines, code ?? 1),
|
|
409
502
|
rawTail,
|
|
410
503
|
stages: tallyStages(lines),
|
|
504
|
+
refused: code === 3,
|
|
411
505
|
});
|
|
412
506
|
}
|
|
413
507
|
});
|
|
414
508
|
});
|
|
415
509
|
}
|
|
416
510
|
|
|
511
|
+
/** bun writes this only if it got far enough to; a crash leaves nothing. */
|
|
512
|
+
function readJunit(path: string): string | undefined {
|
|
513
|
+
try {
|
|
514
|
+
return readFileSync(path, 'utf-8');
|
|
515
|
+
} catch {
|
|
516
|
+
return undefined;
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
/**
|
|
521
|
+
* Print any block that finished with less than 20% of its own budget left.
|
|
522
|
+
*
|
|
523
|
+
* This runs HERE, in the runner, because it is the only place that sees real
|
|
524
|
+
* durations under real load. A gate in CI cannot see this failure mode at all:
|
|
525
|
+
* on a quiet runner every block fits, and the run is green (celilo#1268,
|
|
526
|
+
* acceptance 4).
|
|
527
|
+
*
|
|
528
|
+
* It reports rather than fails. A block at 90% has not broken anything yet, and
|
|
529
|
+
* failing a passing suite would stop the grind that produces the measurements.
|
|
530
|
+
* The line is the durable signal — it lands in `output.log`, so `classify.sh`
|
|
531
|
+
* can promote a PASS whose blocks are out of room, instead of a later timeout
|
|
532
|
+
* arriving with nothing to distinguish "the host starved this" from "this never
|
|
533
|
+
* had room".
|
|
534
|
+
*/
|
|
535
|
+
/**
|
|
536
|
+
* Fail a suite whose block overran the cap that block itself declares.
|
|
537
|
+
*
|
|
538
|
+
* Reads the JUnit report directly — `overBudgetBlocks` bypasses the skip filter
|
|
539
|
+
* for the reason stated there (celilo#1291: the filter deleted the one block
|
|
540
|
+
* that mattered). bun's per-test timeout is not the enforcer; this is. A block
|
|
541
|
+
* that overruns while the suite still PASSES is the dangerous case — an
|
|
542
|
+
* unbudgeted lease on the rig that reads as green — so a pass with an overrun
|
|
543
|
+
* becomes a fail naming the block, its duration and its declaration. A failed
|
|
544
|
+
* suite keeps its own failure; the overruns are printed either way so the
|
|
545
|
+
* output.log carries the durable line. Debug sessions are exempt: a
|
|
546
|
+
* `net.debug()` pause inside a block inflates its junit time without the suite
|
|
547
|
+
* having run at all.
|
|
548
|
+
*/
|
|
549
|
+
function enforceBlockCaps(result: TestResult, suite: string, logDir: string, file: string): void {
|
|
550
|
+
if (result.hadDebugPause) return;
|
|
551
|
+
const junitXml = readJunit(join(logDir, 'junit.xml'));
|
|
552
|
+
if (!junitXml) return;
|
|
553
|
+
let overruns: OverBudgetBlock[];
|
|
554
|
+
try {
|
|
555
|
+
overruns = overBudgetBlocks(junitXml, readFileSync(file, 'utf-8'));
|
|
556
|
+
} catch {
|
|
557
|
+
return;
|
|
558
|
+
}
|
|
559
|
+
if (overruns.length === 0) return;
|
|
560
|
+
for (const t of overruns) {
|
|
561
|
+
console.log(
|
|
562
|
+
`${red}[block:over-budget]${reset} ${suite} — "${t.block}" ran ${Math.round(t.ms / 1000)}s against its ${Math.round(t.capMs / 1000)}s declaration (${Math.round(t.fraction * 100)}%)`,
|
|
563
|
+
);
|
|
564
|
+
}
|
|
565
|
+
if (result.status === 'pass') {
|
|
566
|
+
result.status = 'fail';
|
|
567
|
+
result.error = overruns
|
|
568
|
+
.map(
|
|
569
|
+
(t) =>
|
|
570
|
+
`"${t.block}" ran ${Math.round(t.ms / 1000)}s against its ${Math.round(t.capMs / 1000)}s cap`,
|
|
571
|
+
)
|
|
572
|
+
.join('; ');
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
function reportTightBlocks(suite: string, file: string, blocks: Record<string, number>): void {
|
|
577
|
+
let caps: Record<string, number>;
|
|
578
|
+
try {
|
|
579
|
+
caps = declaredBlockCaps(readFileSync(file, 'utf-8'));
|
|
580
|
+
} catch {
|
|
581
|
+
return;
|
|
582
|
+
}
|
|
583
|
+
for (const t of tightBlocks(blocks, caps)) {
|
|
584
|
+
const pct = Math.round(t.fraction * 100);
|
|
585
|
+
console.log(
|
|
586
|
+
`${yellow}[block:tight]${reset} ${suite} — "${t.block}" used ${Math.round(t.ms / 1000)}s of its ${Math.round(t.capMs / 1000)}s budget (${pct}%)`,
|
|
587
|
+
);
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
|
|
417
591
|
/**
|
|
418
592
|
* " — 1 stage failed, 8 skipped (blocked by an earlier stage)". Rendered only
|
|
419
593
|
* when a cascade actually happened, so an ordinary single-stage failure stays
|
|
@@ -615,7 +789,7 @@ async function main() {
|
|
|
615
789
|
}
|
|
616
790
|
|
|
617
791
|
if (patterns.length > 0) {
|
|
618
|
-
testFiles = testFiles
|
|
792
|
+
testFiles = filterTestFilesByPatterns(testFiles, patterns);
|
|
619
793
|
}
|
|
620
794
|
|
|
621
795
|
// Drop quarantined tests (file contains `cele2e-ci-unsafe`) from broad runs so
|
|
@@ -669,6 +843,7 @@ async function main() {
|
|
|
669
843
|
}
|
|
670
844
|
|
|
671
845
|
const history = loadTiming();
|
|
846
|
+
let blockTiming = loadBlockTiming();
|
|
672
847
|
const testNames = testFiles.map((f) => basename(f, '.test.ts'));
|
|
673
848
|
const hasHistory = testNames.some((n) => history[n]);
|
|
674
849
|
const suiteExpected = testNames.reduce((sum, n) => sum + (history[n] || 0), 0);
|
|
@@ -758,6 +933,7 @@ async function main() {
|
|
|
758
933
|
console.log(`\n${red}Interrupted. Stopping shared infrastructure...${reset}`);
|
|
759
934
|
stopSharedInfra();
|
|
760
935
|
saveTiming(history);
|
|
936
|
+
saveBlockTiming(blockTiming);
|
|
761
937
|
emitRunFailed({
|
|
762
938
|
runId,
|
|
763
939
|
durationMs: Date.now() - suiteStart,
|
|
@@ -802,6 +978,8 @@ async function main() {
|
|
|
802
978
|
result.error = `cross-test pollution (#254): this test left shared DNS dirty — ${pollution}`;
|
|
803
979
|
}
|
|
804
980
|
|
|
981
|
+
enforceBlockCaps(result, name, logDir, file);
|
|
982
|
+
|
|
805
983
|
emitTestCompleted({
|
|
806
984
|
runId,
|
|
807
985
|
name,
|
|
@@ -817,6 +995,13 @@ async function main() {
|
|
|
817
995
|
if (result.duration >= 10 && !result.hadDebugPause) {
|
|
818
996
|
history[name] = result.duration;
|
|
819
997
|
}
|
|
998
|
+
// Recorded even for a short or failed run: a block that BLEW its cap is
|
|
999
|
+
// exactly the measurement this file exists for, and a debug pause inflates
|
|
1000
|
+
// the suite total without touching the blocks that ran before it.
|
|
1001
|
+
if (result.blocks) {
|
|
1002
|
+
blockTiming = mergeBlockTiming(blockTiming, name, result.blocks);
|
|
1003
|
+
reportTightBlocks(name, file, result.blocks);
|
|
1004
|
+
}
|
|
820
1005
|
|
|
821
1006
|
// Write persistent network info if --keep was set and we captured a project
|
|
822
1007
|
if (flagKeep && result.projectName) {
|
|
@@ -864,6 +1049,24 @@ async function main() {
|
|
|
864
1049
|
}
|
|
865
1050
|
console.log();
|
|
866
1051
|
|
|
1052
|
+
// The live-stack guard refused (celilo#1297): another run's stack is in
|
|
1053
|
+
// the way. Stop the whole run with exit 3 — the same code the lock
|
|
1054
|
+
// refusal uses — instead of grinding every remaining suite into the same
|
|
1055
|
+
// wall. Nothing of ours is running, so there is nothing to tear down.
|
|
1056
|
+
if (result.refused) {
|
|
1057
|
+
console.error(
|
|
1058
|
+
`\n${red}✗ startup cleanup refused — a live e2e stack is in the way:${reset}\n`,
|
|
1059
|
+
);
|
|
1060
|
+
if (result.rawTail) console.error(`${dim}${result.rawTail}${reset}\n`);
|
|
1061
|
+
console.error(`${dim}Clear the live stack with: cele2e down${reset}\n`);
|
|
1062
|
+
emitRunFailed({
|
|
1063
|
+
runId,
|
|
1064
|
+
durationMs: Date.now() - suiteStart,
|
|
1065
|
+
error: result.error ?? 'startup cleanup refused: live e2e stack in the way',
|
|
1066
|
+
});
|
|
1067
|
+
process.exit(3);
|
|
1068
|
+
}
|
|
1069
|
+
|
|
867
1070
|
writeFileSync(
|
|
868
1071
|
join(logDir, 'result.json'),
|
|
869
1072
|
JSON.stringify({ status: result.status, duration: result.duration, error: result.error }),
|
|
@@ -875,6 +1078,7 @@ async function main() {
|
|
|
875
1078
|
}
|
|
876
1079
|
|
|
877
1080
|
saveTiming(history);
|
|
1081
|
+
saveBlockTiming(blockTiming);
|
|
878
1082
|
|
|
879
1083
|
console.log(`${dim}Stopping shared infrastructure...${reset}`);
|
|
880
1084
|
stopSharedInfra();
|
|
@@ -947,6 +1151,7 @@ async function main() {
|
|
|
947
1151
|
const adds: string[] = [];
|
|
948
1152
|
if (!content.includes('results/')) adds.push('results/');
|
|
949
1153
|
if (!content.includes('.e2e-timing.json')) adds.push('.e2e-timing.json');
|
|
1154
|
+
if (!content.includes('.e2e-block-timing.json')) adds.push('.e2e-block-timing.json');
|
|
950
1155
|
if (adds.length) writeFileSync(gitignore, `${content.trimEnd()}\n${adds.join('\n')}\n`);
|
|
951
1156
|
|
|
952
1157
|
emitRunCompleted({
|
package/src/shared-infra.ts
CHANGED
|
@@ -19,7 +19,17 @@ import {
|
|
|
19
19
|
generateSharedInfraYaml,
|
|
20
20
|
prepareRegistryDropZone,
|
|
21
21
|
} from './docker-compose-generator';
|
|
22
|
+
import {
|
|
23
|
+
type DockerReader,
|
|
24
|
+
LiveStackError,
|
|
25
|
+
type ProcessProbe,
|
|
26
|
+
findLiveE2eStack,
|
|
27
|
+
realDocker,
|
|
28
|
+
realProcessProbe,
|
|
29
|
+
refuseOnLiveStack,
|
|
30
|
+
} from './live-stack';
|
|
22
31
|
import { ensureRegistryServerBundle, ensureTerraformFakeBundle } from './registry-bundle';
|
|
32
|
+
import { type LockStatus, lockStatus } from './run-lock';
|
|
23
33
|
|
|
24
34
|
const SHARED_COMPOSE_FILE = 'docker-compose.shared.yml';
|
|
25
35
|
const COMPOSE_TIMEOUT = 120_000;
|
|
@@ -35,40 +45,74 @@ function run(cmd: string, opts?: { cwd?: string; timeout?: number }): string {
|
|
|
35
45
|
|
|
36
46
|
/**
|
|
37
47
|
* Remove ALL celilo-e2e-* Docker resources by name prefix (#212). Used at
|
|
38
|
-
* start-of-run, where the
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
48
|
+
* start-of-run, where the live-stack guard (findLiveE2eStack, called by
|
|
49
|
+
* startupCleanup before this runs) has verified nothing live remains to
|
|
50
|
+
* remove. Graceful compose-down of the shared project first (clean network
|
|
51
|
+
* detach), then a force sweep that catches orphans from any prior invocation
|
|
52
|
+
* regardless of compose-project name or the path that created them.
|
|
53
|
+
*
|
|
54
|
+
* Docker access goes through the injected runner (ce-h4no seam) so tests can
|
|
55
|
+
* prove the guard fires BEFORE any removal command runs.
|
|
42
56
|
*/
|
|
43
|
-
function nukeE2eResources(e2eDir: string): void {
|
|
57
|
+
function nukeE2eResources(e2eDir: string, docker: DockerReader): void {
|
|
44
58
|
try {
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
59
|
+
docker(
|
|
60
|
+
[
|
|
61
|
+
'compose',
|
|
62
|
+
'-f',
|
|
63
|
+
SHARED_COMPOSE_FILE,
|
|
64
|
+
'-p',
|
|
65
|
+
SHARED_PROJECT_NAME,
|
|
66
|
+
'down',
|
|
67
|
+
'--volumes',
|
|
68
|
+
'--remove-orphans',
|
|
69
|
+
],
|
|
70
|
+
{ cwd: e2eDir, timeoutMs: 30_000 },
|
|
48
71
|
);
|
|
49
72
|
} catch {}
|
|
50
73
|
try {
|
|
51
|
-
const ids =
|
|
52
|
-
if (ids.trim())
|
|
74
|
+
const ids = docker(['ps', '-aq', '--filter', 'name=celilo-e2e'], { timeoutMs: 10_000 });
|
|
75
|
+
if (ids.trim()) docker(['rm', '-f', ...ids.split('\n').filter(Boolean)], { timeoutMs: 60_000 });
|
|
53
76
|
} catch {}
|
|
54
77
|
try {
|
|
55
|
-
const nets =
|
|
78
|
+
const nets = docker(['network', 'ls', '--format', '{{.Name}}'])
|
|
56
79
|
.split('\n')
|
|
57
80
|
.filter((n) => n.startsWith('celilo-e2e'));
|
|
58
81
|
for (const net of nets) {
|
|
59
82
|
try {
|
|
60
|
-
|
|
83
|
+
docker(['network', 'rm', net], { timeoutMs: 5_000 });
|
|
61
84
|
} catch {}
|
|
62
85
|
}
|
|
63
86
|
} catch {}
|
|
64
87
|
try {
|
|
65
|
-
|
|
88
|
+
docker(['network', 'prune', '-f'], { timeoutMs: 10_000 });
|
|
66
89
|
} catch {}
|
|
67
90
|
try {
|
|
68
|
-
|
|
91
|
+
docker(['volume', 'prune', '-f'], { timeoutMs: 10_000 });
|
|
69
92
|
} catch {}
|
|
70
93
|
}
|
|
71
94
|
|
|
95
|
+
/**
|
|
96
|
+
* The start-of-run cleanup: refuse if a live stack is in the way
|
|
97
|
+
* (celilo#1297), otherwise sweep every celilo-e2e-* resource. The refusal
|
|
98
|
+
* throws LiveStackError — ensureSharedInfra turns it into exit 3 at its two
|
|
99
|
+
* call sites — so tests can drive this with a fake runner and assert the
|
|
100
|
+
* sweep removes nothing behind a refusal.
|
|
101
|
+
*/
|
|
102
|
+
export function startupCleanup(
|
|
103
|
+
e2eDir: string,
|
|
104
|
+
docker: DockerReader = realDocker,
|
|
105
|
+
lock: () => LockStatus = lockStatus,
|
|
106
|
+
processes: ProcessProbe = realProcessProbe,
|
|
107
|
+
): void {
|
|
108
|
+
const refusal = findLiveE2eStack(docker, lock, processes);
|
|
109
|
+
if (refusal) throw new LiveStackError(refusal);
|
|
110
|
+
console.log(
|
|
111
|
+
'[progress:start] cleaning up stale shared infrastructure | shared infra cleanup complete',
|
|
112
|
+
);
|
|
113
|
+
nukeE2eResources(e2eDir, docker);
|
|
114
|
+
}
|
|
115
|
+
|
|
72
116
|
/**
|
|
73
117
|
* Check if the shared infrastructure is already running and healthy.
|
|
74
118
|
*
|
|
@@ -129,26 +173,40 @@ export async function ensureSharedInfra(): Promise<void> {
|
|
|
129
173
|
return; // Shared infra healthy, nothing to do
|
|
130
174
|
}
|
|
131
175
|
} catch {
|
|
132
|
-
// DNS check failed — restart shared infra
|
|
176
|
+
// DNS check failed — restart shared infra. The restart tears down the
|
|
177
|
+
// running stack, so the live-stack guard fires first (celilo#1297): a
|
|
178
|
+
// stack that is up is never force-removed, even when its DNS is wedged.
|
|
179
|
+
// The operator clears it with the docker removal named in the refusal
|
|
180
|
+
// (a bare `cele2e down` does not exist on ephemeral CI hosts —
|
|
181
|
+
// celilo#1314) and re-runs. This retires
|
|
182
|
+
// the automatic DNS-restart self-heal by decision (peba, 2026-09-07):
|
|
183
|
+
// fail loudly naming the holder beats silently clobbering a stack that
|
|
184
|
+
// might be another session's — the #1297 incident rode exactly this
|
|
185
|
+
// branch.
|
|
133
186
|
console.log(
|
|
134
187
|
'[progress:start] shared infra DNS check failed, restarting | shared infra restarted',
|
|
135
188
|
);
|
|
189
|
+
refuseOnLiveStack();
|
|
136
190
|
await stopSharedInfra();
|
|
137
191
|
}
|
|
138
192
|
}
|
|
139
193
|
|
|
140
194
|
const e2eDir = getE2eDir();
|
|
141
195
|
|
|
142
|
-
// Start-of-run cleanup
|
|
143
|
-
// build path before any docker mutation)
|
|
144
|
-
//
|
|
145
|
-
//
|
|
146
|
-
//
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
196
|
+
// Start-of-run cleanup, behind the live-stack guard. The run-lock held here
|
|
197
|
+
// (acquired in the runner / build path before any docker mutation) is no
|
|
198
|
+
// longer treated as proof that no other session's stack is live — the #1297
|
|
199
|
+
// incident got past it. startupCleanup checks Docker and the lock file at
|
|
200
|
+
// the removal site and refuses with exit 3 when anything live remains.
|
|
201
|
+
try {
|
|
202
|
+
startupCleanup(e2eDir);
|
|
203
|
+
} catch (err) {
|
|
204
|
+
if (err instanceof LiveStackError) {
|
|
205
|
+
console.error(`\n${err.message}\n`);
|
|
206
|
+
process.exit(3);
|
|
207
|
+
}
|
|
208
|
+
throw err;
|
|
209
|
+
}
|
|
152
210
|
|
|
153
211
|
// Refresh the bundled registry-server source so Dockerfile.registry's
|
|
154
212
|
// COPY resolves whether we're in the monorepo (regenerated from the
|
|
@@ -184,13 +242,10 @@ export async function ensureSharedInfra(): Promise<void> {
|
|
|
184
242
|
copyFileSync(tpl, live);
|
|
185
243
|
}
|
|
186
244
|
|
|
187
|
-
//
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
timeout: 300_000,
|
|
192
|
-
});
|
|
193
|
-
|
|
245
|
+
// Start. The generated compose carries `image:` only (never `build:`), so
|
|
246
|
+
// `up` cannot build and cannot resolve a FROM from docker.io; a missing
|
|
247
|
+
// baked tag fails right here, naming the image. startNetwork checks the
|
|
248
|
+
// full referenced set first and names the remedy (cele2e build-infra).
|
|
194
249
|
console.log('[progress:start] starting shared infrastructure | shared infra started');
|
|
195
250
|
run(`docker compose -f ${SHARED_COMPOSE_FILE} -p ${SHARED_PROJECT_NAME} up -d`, {
|
|
196
251
|
cwd: e2eDir,
|
|
@@ -232,6 +287,7 @@ export async function ensureSharedInfra(): Promise<void> {
|
|
|
232
287
|
} catch {}
|
|
233
288
|
lastRestart = Date.now();
|
|
234
289
|
}
|
|
290
|
+
// e2e-sleep-ok: poll cadence; convergence is re-checked each iteration, diagnostics on timeout.
|
|
235
291
|
await new Promise((r) => setTimeout(r, 2000));
|
|
236
292
|
}
|
|
237
293
|
// Diagnostics before failing, so the log shows WHY it didn't converge.
|
package/src/socks-proxy.ts
CHANGED
|
@@ -190,6 +190,7 @@ async function waitForTcpAccept(port: number, timeoutMs = 10_000): Promise<void>
|
|
|
190
190
|
return false;
|
|
191
191
|
});
|
|
192
192
|
if (ok) return;
|
|
193
|
+
// e2e-sleep-ok: poll cadence; tryTcpConnect is re-checked each iteration.
|
|
193
194
|
await new Promise((r) => setTimeout(r, 100));
|
|
194
195
|
}
|
|
195
196
|
throw new Error(
|
|
@@ -340,6 +341,7 @@ async function waitForContainerListening(name: string): Promise<void> {
|
|
|
340
341
|
`Container logs:\n${logs || '(none)'}`,
|
|
341
342
|
);
|
|
342
343
|
}
|
|
344
|
+
// e2e-sleep-ok: poll cadence; the container inspect is re-checked each iteration.
|
|
343
345
|
await new Promise((r) => setTimeout(r, 200));
|
|
344
346
|
}
|
|
345
347
|
throw new Error(
|