@celilo/e2e 0.19.2 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -13
- package/bin/e2e-bake-management +171 -12
- package/bin/e2e-infra +0 -1
- package/bin/e2e-up +14 -3
- package/docker/Dockerfile.observer +12 -1
- package/docker/Dockerfile.target-machine +22 -1
- package/npm-registry-server/package.json +1 -1
- package/package.json +3 -3
- package/registry-server/package.json +1 -1
- package/scripts/pack-celilo-packages.ts +15 -0
- package/src/block-timing.test.ts +559 -0
- package/src/block-timing.ts +366 -0
- package/src/cli/build.test.ts +54 -4
- package/src/cli/build.ts +204 -88
- package/src/cli/command-registry.ts +21 -0
- package/src/cli/command-tree-parser.ts +11 -2
- package/src/cli/completion.ts +9 -0
- package/src/cli/host.ts +252 -0
- package/src/cli/index.ts +78 -51
- package/src/cli/module-discovery.ts +108 -13
- package/src/cli/scaffold.ts +18 -26
- package/src/container-manager.cleanup.test.ts +284 -0
- package/src/container-manager.runner.test.ts +351 -0
- package/src/container-manager.test.ts +84 -0
- package/src/container-manager.ts +721 -185
- package/src/docker-compose-generator.ts +135 -61
- package/src/doctor.test.ts +259 -4
- package/src/doctor.ts +276 -3
- package/src/exit-cleanup.test.ts +83 -1
- package/src/fleet-nameserver-gate.test.ts +45 -0
- package/src/host-vm.test.ts +156 -0
- package/src/host-vm.ts +230 -0
- package/src/index.ts +11 -0
- package/src/live-stack.test.ts +184 -0
- package/src/live-stack.ts +145 -0
- package/src/no-unjustified-sleep.test.ts +90 -0
- package/src/proxmox-provisioner.test.ts +18 -2
- package/src/proxmox-provisioner.ts +22 -0
- package/src/public-sim-routes.test.ts +9 -2
- package/src/repo-root.ts +33 -0
- package/src/run-args.test.ts +76 -0
- package/src/run-args.ts +89 -0
- package/src/runner.ts +213 -8
- package/src/shared-infra.ts +83 -32
- package/src/socks-proxy.ts +2 -0
- package/src/source-fingerprint.test.ts +213 -0
- package/src/source-fingerprint.ts +201 -0
- package/src/stage-simulator-inputs.ts +93 -0
- package/src/stages.ts +133 -0
- package/src/wait-for-run.ts +1 -0
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import { readFileSync } from 'node:fs';
|
|
3
|
+
import { hostname } from 'node:os';
|
|
4
|
+
import { dirname, join } from 'node:path';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
6
|
+
import { type DockerReader, LiveStackError, findLiveE2eStack } from './live-stack';
|
|
7
|
+
import type { LockStatus } from './run-lock';
|
|
8
|
+
import { startupCleanup } from './shared-infra';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* The live-stack guard for the startup cleanup (celilo#1297, ce-h04y).
|
|
12
|
+
*
|
|
13
|
+
* The #1297 incident: a second cele2e invocation got past the host-global run
|
|
14
|
+
* lock and nukeE2eResources force-removed another run's mid-flight stack. The
|
|
15
|
+
* guard checks Docker and the lock file at the removal site. These tests prove
|
|
16
|
+
* the wiring through the injected runner: a live stack refuses with NOTHING
|
|
17
|
+
* removed, a dead one sweeps. Delete the guard call and the first test goes
|
|
18
|
+
* red — the removal commands run behind what should have been a refusal.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
const DIR = dirname(fileURLToPath(import.meta.url));
|
|
22
|
+
const SHARED_INFRA_SRC = readFileSync(join(DIR, 'shared-infra.ts'), 'utf-8');
|
|
23
|
+
|
|
24
|
+
/** Recording fake: answers `docker ps` from `psOutput`, records every command. */
|
|
25
|
+
function fakeDocker(psOutput: string): { docker: DockerReader; commands: string[] } {
|
|
26
|
+
const commands: string[] = [];
|
|
27
|
+
return {
|
|
28
|
+
commands,
|
|
29
|
+
docker: (args) => {
|
|
30
|
+
commands.push(args.join(' '));
|
|
31
|
+
if (args[0] === 'ps') return psOutput;
|
|
32
|
+
return '';
|
|
33
|
+
},
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const noLock: () => LockStatus = () => ({
|
|
38
|
+
free: true,
|
|
39
|
+
holder: null,
|
|
40
|
+
heartbeatAgeMs: null,
|
|
41
|
+
suspect: false,
|
|
42
|
+
ownKept: false,
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
function foreignLock(overrides: Partial<LockStatus> = {}): () => LockStatus {
|
|
46
|
+
return () => ({
|
|
47
|
+
free: false,
|
|
48
|
+
holder: {
|
|
49
|
+
pid: 1,
|
|
50
|
+
hostname: hostname(),
|
|
51
|
+
session: 'polecat/ce-9999 (some other worktree)',
|
|
52
|
+
test: 'crew-alerting',
|
|
53
|
+
runId: 'run-abc',
|
|
54
|
+
startedAt: new Date(Date.now() - 5 * 60_000).toISOString(),
|
|
55
|
+
beatAt: Date.now() - 45_000,
|
|
56
|
+
state: 'running',
|
|
57
|
+
},
|
|
58
|
+
heartbeatAgeMs: 45_000,
|
|
59
|
+
suspect: false,
|
|
60
|
+
ownKept: false,
|
|
61
|
+
...overrides,
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
describe('findLiveE2eStack', () => {
|
|
66
|
+
test('refuses when a celilo-e2e-* container is running, naming it', () => {
|
|
67
|
+
const { docker } = fakeDocker(
|
|
68
|
+
'celilo-e2e-shared_namecheap-dns\trunning\ncelilo-e2e-1788769175864_fw-main\texited\n',
|
|
69
|
+
);
|
|
70
|
+
const refusal = findLiveE2eStack(docker, noLock);
|
|
71
|
+
expect(refusal).not.toBeNull();
|
|
72
|
+
expect(refusal?.reason).toContain('celilo-e2e-shared_namecheap-dns');
|
|
73
|
+
expect(refusal?.reason).not.toContain('celilo-e2e-1788769175864_fw-main');
|
|
74
|
+
expect(refusal?.reason).toContain('cele2e down');
|
|
75
|
+
expect(refusal?.runningContainers).toEqual(['celilo-e2e-shared_namecheap-dns']);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
test('proceeds when every container is exited', () => {
|
|
79
|
+
const { docker } = fakeDocker(
|
|
80
|
+
'celilo-e2e-1788769175864_fw-main\texited\ncelilo-e2e-shared_registry\tdead\n',
|
|
81
|
+
);
|
|
82
|
+
expect(findLiveE2eStack(docker, noLock)).toBeNull();
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
test('treats paused and restarting containers as live — they hold real state', () => {
|
|
86
|
+
for (const state of ['paused', 'restarting', 'running']) {
|
|
87
|
+
const { docker } = fakeDocker(`celilo-e2e-shared_registry\t${state}\n`);
|
|
88
|
+
const refusal = findLiveE2eStack(docker, noLock);
|
|
89
|
+
expect(refusal?.runningContainers).toEqual(['celilo-e2e-shared_registry']);
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test('refuses on a foreign lock with a fresh heartbeat, naming session, test and heartbeat age', () => {
|
|
94
|
+
const { docker } = fakeDocker('');
|
|
95
|
+
const refusal = findLiveE2eStack(docker, foreignLock());
|
|
96
|
+
expect(refusal).not.toBeNull();
|
|
97
|
+
expect(refusal?.reason).toContain('polecat/ce-9999');
|
|
98
|
+
expect(refusal?.reason).toContain('crew-alerting');
|
|
99
|
+
expect(refusal?.reason).toContain('heartbeat 45s old');
|
|
100
|
+
expect(refusal?.holder?.runId).toBe('run-abc');
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test('exempts our own lock — the caller holds it for its whole run', () => {
|
|
104
|
+
const { docker } = fakeDocker('');
|
|
105
|
+
const own: () => LockStatus = () => ({
|
|
106
|
+
free: false,
|
|
107
|
+
holder: {
|
|
108
|
+
pid: process.pid,
|
|
109
|
+
hostname: hostname(),
|
|
110
|
+
session: 'whatever (this worktree)',
|
|
111
|
+
test: 'caddy-internal-private',
|
|
112
|
+
runId: 'run-own',
|
|
113
|
+
startedAt: new Date().toISOString(),
|
|
114
|
+
beatAt: Date.now(),
|
|
115
|
+
state: 'running',
|
|
116
|
+
},
|
|
117
|
+
heartbeatAgeMs: 0,
|
|
118
|
+
suspect: false,
|
|
119
|
+
ownKept: false,
|
|
120
|
+
});
|
|
121
|
+
expect(findLiveE2eStack(docker, own)).toBeNull();
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
test('the container check has no own-lock exemption — a stolen lock says "ours" while another run owns the containers', () => {
|
|
125
|
+
// The #1297 suspected path: the second invocation ends up holding the lock
|
|
126
|
+
// while the first run's containers are still up. Only Docker can tell that
|
|
127
|
+
// truth, so running containers refuse even when the lock is ours.
|
|
128
|
+
const { docker } = fakeDocker('celilo-e2e-shared_namecheap-dns\trunning\n');
|
|
129
|
+
const refusal = findLiveE2eStack(docker, foreignLock({ holder: null }));
|
|
130
|
+
expect(refusal?.runningContainers.length).toBe(1);
|
|
131
|
+
});
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
describe('startupCleanup', () => {
|
|
135
|
+
test('refuses behind a live stack and removes NOTHING (celilo#1297)', () => {
|
|
136
|
+
const { docker, commands } = fakeDocker('celilo-e2e-shared_namecheap-dns\trunning\n');
|
|
137
|
+
expect(() => startupCleanup('/tmp/e2e', docker, noLock)).toThrow(LiveStackError);
|
|
138
|
+
// Only the ps read happened. No compose down, no rm, no prune.
|
|
139
|
+
expect(commands).toEqual([expect.stringContaining('ps -a --filter name=celilo-e2e')]);
|
|
140
|
+
expect(commands.some((c) => c.includes(' down '))).toBe(false);
|
|
141
|
+
expect(commands.some((c) => c.includes('rm -f'))).toBe(false);
|
|
142
|
+
expect(commands.some((c) => c.includes('prune'))).toBe(false);
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
test('sweeps when the environment is provably dead, compose down before the force sweep', () => {
|
|
146
|
+
const { docker, commands } = fakeDocker('celilo-e2e-1788769175864_fw-main\texited\n');
|
|
147
|
+
startupCleanup('/tmp/e2e', docker, noLock);
|
|
148
|
+
const down = commands.findIndex((c) => c.includes('compose -f docker-compose.shared.yml'));
|
|
149
|
+
const rm = commands.findIndex((c) => c.includes('rm -f'));
|
|
150
|
+
const netPrune = commands.findIndex((c) => c.includes('network prune'));
|
|
151
|
+
const volPrune = commands.findIndex((c) => c.includes('volume prune'));
|
|
152
|
+
expect(down).toBeGreaterThanOrEqual(0);
|
|
153
|
+
expect(rm).toBeGreaterThan(down);
|
|
154
|
+
expect(netPrune).toBeGreaterThan(rm);
|
|
155
|
+
expect(volPrune).toBeGreaterThan(netPrune);
|
|
156
|
+
// The sweep keys off the celilo-e2e prefix; it must never name the shared
|
|
157
|
+
// project in a force removal (the graceful compose down above owns that).
|
|
158
|
+
for (const command of commands.filter((c) => c.includes('rm -f'))) {
|
|
159
|
+
expect(command).not.toContain('celilo-e2e-shared');
|
|
160
|
+
}
|
|
161
|
+
});
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
// The guard is only real if it is WIRED — a perfect guard nobody calls is the
|
|
165
|
+
// same failure as no guard. These source assertions pin the two removal sites
|
|
166
|
+
// in ensureSharedInfra; the behavioral tests above pin the guard itself.
|
|
167
|
+
describe('wiring in shared-infra.ts', () => {
|
|
168
|
+
test('the start-of-run nuke runs through startupCleanup, which guards first', () => {
|
|
169
|
+
expect(SHARED_INFRA_SRC).toContain('startupCleanup(e2eDir)');
|
|
170
|
+
const cleanupBody = SHARED_INFRA_SRC.slice(
|
|
171
|
+
SHARED_INFRA_SRC.indexOf('export function startupCleanup'),
|
|
172
|
+
SHARED_INFRA_SRC.indexOf('nukeE2eResources(e2eDir, docker)'),
|
|
173
|
+
);
|
|
174
|
+
expect(cleanupBody).toContain('findLiveE2eStack(docker, lock)');
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
test('the DNS-restart branch guards before tearing down the running stack', () => {
|
|
178
|
+
const branchBody = SHARED_INFRA_SRC.slice(
|
|
179
|
+
SHARED_INFRA_SRC.indexOf('DNS check failed'),
|
|
180
|
+
SHARED_INFRA_SRC.indexOf('await stopSharedInfra()'),
|
|
181
|
+
);
|
|
182
|
+
expect(branchBody).toContain('refuseOnLiveStack()');
|
|
183
|
+
});
|
|
184
|
+
});
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Live-stack guard for the e2e startup cleanup (celilo#1297, ce-h04y).
|
|
3
|
+
*
|
|
4
|
+
* nukeE2eResources tears down EVERY celilo-e2e-* container by name prefix, on
|
|
5
|
+
* the assumption that holding the run-lock proves no other session's stack is
|
|
6
|
+
* live. The #1297 incident broke that assumption: a second invocation got past
|
|
7
|
+
* the lock anyway and force-removed another run's mid-flight stack, nine
|
|
8
|
+
* containers, the suite dead at 12.95s with exit 137. Which lock path let it
|
|
9
|
+
* through is not reconstructable, and does not need to be — the guard defends
|
|
10
|
+
* every path by checking ground truth (Docker and the lock file) at the
|
|
11
|
+
* removal site instead of trusting the lock's own verdict.
|
|
12
|
+
*
|
|
13
|
+
* Decided (peba, 2026-09-07, option a): before the startup cleanup removes
|
|
14
|
+
* anything, it refuses when any celilo-e2e-* container is still live or the
|
|
15
|
+
* run-lock heartbeat is fresh and foreign. It removes only what is provably
|
|
16
|
+
* dead. There is no --force: an operator who wants a live stack gone uses
|
|
17
|
+
* `cele2e down`.
|
|
18
|
+
*
|
|
19
|
+
* Deliberately NOT guarded: the runner's end-of-run stopSharedInfra teardown.
|
|
20
|
+
* The owner tearing down its own live stack is the normal exit path, and the
|
|
21
|
+
* guard's container check would refuse it by definition.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import type { ExecFileSyncOptions } from 'node:child_process';
|
|
25
|
+
import { execFileSync } from 'node:child_process';
|
|
26
|
+
import {
|
|
27
|
+
type LockHolder,
|
|
28
|
+
type LockStatus,
|
|
29
|
+
formatAge,
|
|
30
|
+
formatBusy,
|
|
31
|
+
heartbeatAgeMs,
|
|
32
|
+
isSameSession,
|
|
33
|
+
lockStatus,
|
|
34
|
+
} from './run-lock';
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Docker access, injected so the guard and the sweep it protects are
|
|
38
|
+
* unit-testable without a daemon (the seam lane from ce-h4no). Unlike
|
|
39
|
+
* proxmox-provisioner's DockerRunner this carries per-call timeouts and cwd,
|
|
40
|
+
* which the compose sweep needs.
|
|
41
|
+
*/
|
|
42
|
+
export type DockerReader = (args: string[], opts?: { timeoutMs?: number; cwd?: string }) => string;
|
|
43
|
+
|
|
44
|
+
export const realDocker: DockerReader = (args, opts) =>
|
|
45
|
+
execFileSync('docker', args, {
|
|
46
|
+
encoding: 'utf-8',
|
|
47
|
+
timeout: opts?.timeoutMs ?? 60_000,
|
|
48
|
+
cwd: opts?.cwd,
|
|
49
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
50
|
+
} satisfies ExecFileSyncOptions).trim();
|
|
51
|
+
|
|
52
|
+
/** What the guard found, and the human-readable refusal naming it. */
|
|
53
|
+
export interface LiveStackRefusal {
|
|
54
|
+
reason: string;
|
|
55
|
+
/** Container names still live, when the refusal is about containers. */
|
|
56
|
+
runningContainers: string[];
|
|
57
|
+
/** The foreign lock holder, when the refusal is about the lock. */
|
|
58
|
+
holder: LockHolder | null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export class LiveStackError extends Error {
|
|
62
|
+
constructor(readonly refusal: LiveStackRefusal) {
|
|
63
|
+
super(refusal.reason);
|
|
64
|
+
this.name = 'LiveStackError';
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Container states that mean the container cannot be holding a live stack.
|
|
70
|
+
* Everything else (running, paused, restarting) is live: paused containers
|
|
71
|
+
* keep their memory, restarting ones own their networks, and `docker rm -f`
|
|
72
|
+
* on either kills real work.
|
|
73
|
+
*/
|
|
74
|
+
const DEAD_STATES = new Set(['exited', 'created', 'dead']);
|
|
75
|
+
|
|
76
|
+
function isOwnHolder(h: LockHolder): boolean {
|
|
77
|
+
return h.pid === process.pid || isSameSession(h);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Inspect the machine for a live e2e stack. Returns a refusal when the caller
|
|
82
|
+
* must not remove anything, null when the environment is provably dead.
|
|
83
|
+
*
|
|
84
|
+
* The lock check exempts our own process (the caller holds the lock for its
|
|
85
|
+
* whole run; without the exemption every run would refuse its own cleanup).
|
|
86
|
+
* The container check has NO exemption: a stolen lock (the #1297 suspected
|
|
87
|
+
* path) makes the lock file say "ours" while another run's containers are
|
|
88
|
+
* still up, so only Docker itself can tell that truth.
|
|
89
|
+
*/
|
|
90
|
+
export function findLiveE2eStack(
|
|
91
|
+
docker: DockerReader = realDocker,
|
|
92
|
+
lock: () => LockStatus = lockStatus,
|
|
93
|
+
): LiveStackRefusal | null {
|
|
94
|
+
const status = lock();
|
|
95
|
+
if (!status.free && status.holder && !isOwnHolder(status.holder)) {
|
|
96
|
+
const h = status.holder;
|
|
97
|
+
const heartbeat =
|
|
98
|
+
h.state === 'running' ? `, heartbeat ${formatAge(heartbeatAgeMs(h))} old` : '';
|
|
99
|
+
return {
|
|
100
|
+
reason: `refusing to clean up: the run lock is held by another session — ${formatBusy(h)}${heartbeat}`,
|
|
101
|
+
runningContainers: [],
|
|
102
|
+
holder: h,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const out = docker([
|
|
107
|
+
'ps',
|
|
108
|
+
'-a',
|
|
109
|
+
'--filter',
|
|
110
|
+
'name=celilo-e2e',
|
|
111
|
+
'--format',
|
|
112
|
+
'{{.Names}}\t{{.State}}',
|
|
113
|
+
]);
|
|
114
|
+
const running = out
|
|
115
|
+
.split('\n')
|
|
116
|
+
.filter(Boolean)
|
|
117
|
+
.map((line) => line.split('\t'))
|
|
118
|
+
.filter((parts) => parts.length === 2 && !DEAD_STATES.has(parts[1] ?? ''))
|
|
119
|
+
.map((parts) => parts[0] ?? '');
|
|
120
|
+
if (running.length > 0) {
|
|
121
|
+
return {
|
|
122
|
+
reason: `refusing to clean up: ${running.length} live celilo-e2e-* container(s):\n${running.map((n) => ` - ${n}`).join('\n')}\nA live stack owns these. Clear it with: cele2e down`,
|
|
123
|
+
runningContainers: running,
|
|
124
|
+
holder: null,
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* The startup-cleanup boundary: refuse with exit 3 when a live stack is in
|
|
132
|
+
* the way, return when the caller may remove. Exiting here (rather than
|
|
133
|
+
* throwing) is what makes the refusal survive every caller shape: the run
|
|
134
|
+
* path's bun test child, `cele2e up`, and the build paths all surface a
|
|
135
|
+
* process exit code without each one needing its own handling.
|
|
136
|
+
*/
|
|
137
|
+
export function refuseOnLiveStack(
|
|
138
|
+
docker: DockerReader = realDocker,
|
|
139
|
+
lock: () => LockStatus = lockStatus,
|
|
140
|
+
): void {
|
|
141
|
+
const refusal = findLiveE2eStack(docker, lock);
|
|
142
|
+
if (!refusal) return;
|
|
143
|
+
console.error(`\n${refusal.reason}\n`);
|
|
144
|
+
process.exit(3);
|
|
145
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Recurrence gate for celilo#1267: readiness waits in packages/e2e must poll
|
|
3
|
+
* an observable condition (waitFor or a poll loop), never sleep a fixed
|
|
4
|
+
* duration.
|
|
5
|
+
*
|
|
6
|
+
* apps/celilo/CLAUDE.md prohibits sleeping to mask race conditions, but until
|
|
7
|
+
* now nothing enforced it in this package. respondWith slept 500ms for the
|
|
8
|
+
* responder to boot; on a loaded host that was not enough, the next command
|
|
9
|
+
* hit an unanswered interview, and the error read as a missing fixture value
|
|
10
|
+
* rather than as harness timing.
|
|
11
|
+
*
|
|
12
|
+
* The rule here: every fixed sleep in packages/e2e/src must carry an inline
|
|
13
|
+
* `e2e-sleep-ok:` justification on the same line or the line above. A poll
|
|
14
|
+
* loop's cadence sleep carries one honestly; a bare readiness gate cannot, so
|
|
15
|
+
* it fails this gate instead of failing five suites under load.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { describe, expect, test } from 'bun:test';
|
|
19
|
+
import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
|
|
20
|
+
import { join, resolve } from 'node:path';
|
|
21
|
+
|
|
22
|
+
const SLEEP_PATTERN = /new Promise\(\s*\(?r\)?\s*=>\s*setTimeout\(r\s*,/;
|
|
23
|
+
|
|
24
|
+
function srcDir(): string {
|
|
25
|
+
let dir = import.meta.dir;
|
|
26
|
+
for (let i = 0; i < 8; i++) {
|
|
27
|
+
if (existsSync(join(dir, 'apps')) && existsSync(join(dir, 'modules')))
|
|
28
|
+
return join(dir, 'packages', 'e2e', 'src');
|
|
29
|
+
dir = resolve(dir, '..');
|
|
30
|
+
}
|
|
31
|
+
throw new Error('could not locate repo root (no ancestor with apps/ + modules/)');
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function tsFiles(dir: string): string[] {
|
|
35
|
+
const out: string[] = [];
|
|
36
|
+
for (const entry of readdirSync(dir)) {
|
|
37
|
+
const full = join(dir, entry);
|
|
38
|
+
if (statSync(full).isDirectory()) out.push(...tsFiles(full));
|
|
39
|
+
else if (entry.endsWith('.ts') && !entry.endsWith('.test.ts')) out.push(full);
|
|
40
|
+
}
|
|
41
|
+
return out;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
interface Violation {
|
|
45
|
+
file: string;
|
|
46
|
+
line: number;
|
|
47
|
+
text: string;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function unjustifiedSleeps(): Violation[] {
|
|
51
|
+
const violations: Violation[] = [];
|
|
52
|
+
for (const file of tsFiles(srcDir())) {
|
|
53
|
+
const lines = readFileSync(file, 'utf-8').split('\n');
|
|
54
|
+
lines.forEach((text, i) => {
|
|
55
|
+
if (!SLEEP_PATTERN.test(text)) return;
|
|
56
|
+
const justified = text.includes('e2e-sleep-ok:') || lines[i - 1]?.includes('e2e-sleep-ok:');
|
|
57
|
+
if (!justified) violations.push({ file, line: i + 1, text: text.trim() });
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
return violations;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
describe('recurrence gate: no bare readiness sleeps in packages/e2e (celilo#1267)', () => {
|
|
64
|
+
test('the scan reaches a non-trivial set of files (sanity — it actually ran)', () => {
|
|
65
|
+
const files = tsFiles(srcDir());
|
|
66
|
+
expect(files.length).toBeGreaterThan(10);
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
test('every fixed sleep carries an e2e-sleep-ok justification', () => {
|
|
70
|
+
const violations = unjustifiedSleeps();
|
|
71
|
+
const report = violations
|
|
72
|
+
.map(
|
|
73
|
+
(v) =>
|
|
74
|
+
`${v.file}:${v.line}: ${v.text}\n Add an inline \`e2e-sleep-ok: <reason>\` comment on this line or the one above, or poll an observable condition with waitFor.`,
|
|
75
|
+
)
|
|
76
|
+
.join('\n');
|
|
77
|
+
expect(violations, `Unjustified fixed sleeps:\n${report}`).toEqual([]);
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
test('the known poll-cadence sleeps are still present and justified (the rule engages)', () => {
|
|
81
|
+
// Reach probe, not reasoning: if the justified count drops to zero the
|
|
82
|
+
// scan is matching nothing and the gate above proves nothing.
|
|
83
|
+
const violations = unjustifiedSleeps();
|
|
84
|
+
const total = tsFiles(srcDir()).reduce((count, file) => {
|
|
85
|
+
const lines = readFileSync(file, 'utf-8').split('\n');
|
|
86
|
+
return count + lines.filter((text) => SLEEP_PATTERN.test(text)).length;
|
|
87
|
+
}, 0);
|
|
88
|
+
expect(total - violations.length).toBeGreaterThan(3);
|
|
89
|
+
});
|
|
90
|
+
});
|
|
@@ -9,6 +9,7 @@ import { describe, expect, test } from 'bun:test';
|
|
|
9
9
|
import type { GuestRecord } from '@celilo/terraform-fake';
|
|
10
10
|
import {
|
|
11
11
|
CONTAINER_PREFIX,
|
|
12
|
+
GUEST_PROJECT_LABEL,
|
|
12
13
|
containerNameFor,
|
|
13
14
|
createDockerProvisioner,
|
|
14
15
|
parseNet,
|
|
@@ -60,8 +61,10 @@ describe('createGuest', () => {
|
|
|
60
61
|
});
|
|
61
62
|
|
|
62
63
|
test('names the container so the debris sweep can find it', async () => {
|
|
63
|
-
// cele2e doctor
|
|
64
|
-
//
|
|
64
|
+
// cele2e doctor keys off this prefix; an unprefixed container leaks
|
|
65
|
+
// silently between runs. Project membership is a separate concern — the
|
|
66
|
+
// per-test sweeps cannot match this name (no timestamp in it), so the
|
|
67
|
+
// label test below is what teardown actually keys off.
|
|
65
68
|
const { runner, find } = stubDocker();
|
|
66
69
|
await provisionerWith(runner).createGuest(guest());
|
|
67
70
|
|
|
@@ -70,6 +73,19 @@ describe('createGuest', () => {
|
|
|
70
73
|
expect(name).toBe('celilo-e2e-lxc-203');
|
|
71
74
|
});
|
|
72
75
|
|
|
76
|
+
test('labels the guest with its compose project so teardown can find it (celilo#1247)', async () => {
|
|
77
|
+
// The name cannot carry project membership (no timestamp), and compose's
|
|
78
|
+
// own labels cannot be borrowed (the guest is not a compose service, and
|
|
79
|
+
// compose down removes orphans only on the full label set — verified:
|
|
80
|
+
// a bare com.docker.compose.project label is not enough). The explicit
|
|
81
|
+
// label is what projectTeardownCommands lists guests by.
|
|
82
|
+
const { runner, find } = stubDocker();
|
|
83
|
+
await provisionerWith(runner).createGuest(guest());
|
|
84
|
+
|
|
85
|
+
const run = find('run') ?? [];
|
|
86
|
+
expect(run[run.indexOf('--label') + 1]).toBe(`${GUEST_PROJECT_LABEL}=celilo-e2e-test`);
|
|
87
|
+
});
|
|
88
|
+
|
|
73
89
|
test('passes the gateway through, since target-setup routes from it', async () => {
|
|
74
90
|
const { runner, find } = stubDocker();
|
|
75
91
|
await provisionerWith(runner).createGuest(guest());
|
|
@@ -30,6 +30,20 @@ import { ZONE_GATEWAYS, type Zone } from './types';
|
|
|
30
30
|
*/
|
|
31
31
|
export const CONTAINER_PREFIX = 'celilo-e2e-';
|
|
32
32
|
|
|
33
|
+
/**
|
|
34
|
+
* Marks a sim-created guest as belonging to a compose project.
|
|
35
|
+
*
|
|
36
|
+
* The name prefix alone is not enough for cleanup: a guest is named
|
|
37
|
+
* `celilo-e2e-lxc-<vmid>` with no timestamp in it, so every by-name filter the
|
|
38
|
+
* sweeps use (`celilo-e2e-1<ts>`, `celilo-e2e-<ts>`) misses it. A guest left
|
|
39
|
+
* behind keeps its zone network's endpoint alive, the network rm fails with
|
|
40
|
+
* "has active endpoints", and the subnet stays allocated — every later suite
|
|
41
|
+
* in the run then dies creating that network (celilo#1247). The label is what
|
|
42
|
+
* `projectTeardownCommands` filters on; it carries the project value, so
|
|
43
|
+
* teardown finds exactly the guests of the project it is tearing down.
|
|
44
|
+
*/
|
|
45
|
+
export const GUEST_PROJECT_LABEL = 'celilo-e2e.project';
|
|
46
|
+
|
|
33
47
|
/** Shells out to `docker`. Injected so the logic is testable without a daemon. */
|
|
34
48
|
export type DockerRunner = (args: string[]) => string;
|
|
35
49
|
|
|
@@ -71,6 +85,7 @@ export function zoneForGateway(gateway: string): Zone | undefined {
|
|
|
71
85
|
export function createDockerProvisioner(options: DockerProvisionerOptions) {
|
|
72
86
|
const docker = options.docker ?? realDocker;
|
|
73
87
|
const readyTimeoutMs = options.readyTimeoutMs ?? 60_000;
|
|
88
|
+
// e2e-sleep-ok: injectable test seam; real callers poll a condition between sleeps.
|
|
74
89
|
const sleep = options.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms)));
|
|
75
90
|
|
|
76
91
|
const assertOurs = (name: string): void => {
|
|
@@ -96,6 +111,12 @@ export function createDockerProvisioner(options: DockerProvisionerOptions) {
|
|
|
96
111
|
const name = containerNameFor(guest.vmid);
|
|
97
112
|
assertOurs(name);
|
|
98
113
|
|
|
114
|
+
// Project membership must be discoverable without the name: teardown
|
|
115
|
+
// matches on this label because the guest's name carries no timestamp.
|
|
116
|
+
// Added before the readiness wait, so even a guest that never finishes
|
|
117
|
+
// booting is still findable and removable.
|
|
118
|
+
const projectLabel = ['--label', `${GUEST_PROJECT_LABEL}=${options.project}`] as const;
|
|
119
|
+
|
|
99
120
|
// Same image selection rule the compose generator uses for machines: the
|
|
100
121
|
// app zone needs a dockerd-capable box.
|
|
101
122
|
const image =
|
|
@@ -106,6 +127,7 @@ export function createDockerProvisioner(options: DockerProvisionerOptions) {
|
|
|
106
127
|
'-d',
|
|
107
128
|
'--name',
|
|
108
129
|
name,
|
|
130
|
+
...projectLabel,
|
|
109
131
|
'--hostname',
|
|
110
132
|
guest.hostname,
|
|
111
133
|
'--network',
|
|
@@ -34,7 +34,7 @@ const ISP_EDGE = '100.64.0.1';
|
|
|
34
34
|
const EDGE_SERVICE = 'fw-ext';
|
|
35
35
|
|
|
36
36
|
interface ComposeService {
|
|
37
|
-
|
|
37
|
+
image?: string;
|
|
38
38
|
networks?: Record<string, unknown>;
|
|
39
39
|
}
|
|
40
40
|
|
|
@@ -61,7 +61,14 @@ function publicSimulators(yaml: string): Array<{ name: string; dockerfile: strin
|
|
|
61
61
|
const compose = parse(yaml) as { services?: Record<string, ComposeService> };
|
|
62
62
|
return Object.entries(compose.services ?? {})
|
|
63
63
|
.filter(([name, svc]) => name !== EDGE_SERVICE && svc.networks?.['internet-external'])
|
|
64
|
-
.map(([name, svc]) => ({
|
|
64
|
+
.map(([name, svc]) => ({
|
|
65
|
+
name,
|
|
66
|
+
// The compose carries the tag, not the Dockerfile: the bake convention
|
|
67
|
+
// is tag `celilo-e2e/<name>` <-> `docker/Dockerfile.<name>` (build-infra
|
|
68
|
+
// tags every Dockerfile exactly that way). A drift in the convention
|
|
69
|
+
// makes the reads below fail loudly, which is what we want.
|
|
70
|
+
dockerfile: `docker/Dockerfile.${(svc.image ?? '').replace('celilo-e2e/', '').replace(/:.*/, '')}`,
|
|
71
|
+
}));
|
|
65
72
|
}
|
|
66
73
|
|
|
67
74
|
/**
|
package/src/repo-root.ts
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where the celilo monorepo is, if we are inside one.
|
|
3
|
+
*
|
|
4
|
+
* Its own module because two unrelated consumers need it and neither may
|
|
5
|
+
* import the other: `cli/build.ts` uses it to find modules to package, and
|
|
6
|
+
* `doctor.ts` uses it to fingerprint the source a baked image came from —
|
|
7
|
+
* and `build.ts` already imports `doctor.ts`, so the reverse edge would be a
|
|
8
|
+
* cycle. Re-exported from `cli/build.ts` so its existing callers and its
|
|
9
|
+
* recurrence test are unchanged.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { existsSync } from 'node:fs';
|
|
13
|
+
import { join, resolve } from 'node:path';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Walk up from `startDir` to the celilo checkout, or null.
|
|
17
|
+
*
|
|
18
|
+
* The marker is `apps/celilo/package.json` and must stay that specific
|
|
19
|
+
* (ce-dbc): keying on a bare `modules/` directory made a stray empty
|
|
20
|
+
* `node_modules/modules` from a botched `bun add -g` read as the monorepo
|
|
21
|
+
* root, which skipped the published-staging path and crashed the packer for
|
|
22
|
+
* every npm consumer.
|
|
23
|
+
*/
|
|
24
|
+
export function findMonorepoRoot(startDir: string): string | null {
|
|
25
|
+
let dir = startDir;
|
|
26
|
+
for (let i = 0; i < 10; i++) {
|
|
27
|
+
if (existsSync(join(dir, 'apps', 'celilo', 'package.json'))) return dir;
|
|
28
|
+
const parent = resolve(dir, '..');
|
|
29
|
+
if (parent === dir) break;
|
|
30
|
+
dir = parent;
|
|
31
|
+
}
|
|
32
|
+
return null;
|
|
33
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import { KNOWN_RUN_FLAGS, filterTestFilesByPatterns, parseRunArgs } from './run-args';
|
|
3
|
+
|
|
4
|
+
describe('parseRunArgs', () => {
|
|
5
|
+
test('--help is recognized as help, never discarded into run-everything', () => {
|
|
6
|
+
const parsed = parseRunArgs(['--help']);
|
|
7
|
+
expect(parsed.helpRequested).toBe(true);
|
|
8
|
+
expect(parsed.unknownFlags).toEqual([]);
|
|
9
|
+
expect(parsed.patterns).toEqual([]);
|
|
10
|
+
});
|
|
11
|
+
|
|
12
|
+
test('every known flag is accepted and yields no patterns', () => {
|
|
13
|
+
const parsed = parseRunArgs([...KNOWN_RUN_FLAGS, '--seed=42']);
|
|
14
|
+
expect(parsed.unknownFlags).toEqual([]);
|
|
15
|
+
expect(parsed.patterns).toEqual([]);
|
|
16
|
+
expect(parsed.helpRequested).toBe(false);
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
test('positional patterns survive flag parsing in order', () => {
|
|
20
|
+
const parsed = parseRunArgs(['deploy', '--keep', 'caddy', '--verbose']);
|
|
21
|
+
expect(parsed.patterns).toEqual(['deploy', 'caddy']);
|
|
22
|
+
expect(parsed.unknownFlags).toEqual([]);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test('an unrecognised --flag is reported, not silently discarded', () => {
|
|
26
|
+
// The old runner stripped every --arg it did not know, leaving zero
|
|
27
|
+
// patterns: run everything. `--al` must error, not launch the catalogue.
|
|
28
|
+
const parsed = parseRunArgs(['--al', '--dry-run']);
|
|
29
|
+
expect(parsed.unknownFlags).toEqual(['--al', '--dry-run']);
|
|
30
|
+
expect(parsed.patterns).toEqual([]);
|
|
31
|
+
expect(parsed.helpRequested).toBe(false);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test('the CLI discovery aliases are unknown to the runner', () => {
|
|
35
|
+
// `cele2e run` strips --all/--all-modules before exec. If one ever
|
|
36
|
+
// reaches the runner directly it must error, not run everything.
|
|
37
|
+
const parsed = parseRunArgs(['--all', '--all-modules']);
|
|
38
|
+
expect(parsed.unknownFlags).toEqual(['--all', '--all-modules']);
|
|
39
|
+
});
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
describe('filterTestFilesByPatterns', () => {
|
|
43
|
+
const files = [
|
|
44
|
+
'/mods/knot/e2e/aspect-fanout.test.ts',
|
|
45
|
+
'/top/tests/aspect-fanout-new-systems.test.ts',
|
|
46
|
+
'/top/tests/caddy-internal-private.test.ts',
|
|
47
|
+
];
|
|
48
|
+
|
|
49
|
+
test('an exact pattern selects only its own file, not the sibling it prefixes', () => {
|
|
50
|
+
// The old substring-OR filter dragged aspect-fanout-new-systems into
|
|
51
|
+
// `cele2e run aspect-fanout caddy-internal-private` whenever the
|
|
52
|
+
// top-level dir was in the collected set.
|
|
53
|
+
const picked = filterTestFilesByPatterns(files, ['aspect-fanout', 'caddy-internal-private']);
|
|
54
|
+
expect(picked).toEqual([
|
|
55
|
+
'/mods/knot/e2e/aspect-fanout.test.ts',
|
|
56
|
+
'/top/tests/caddy-internal-private.test.ts',
|
|
57
|
+
]);
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
test('a pattern with no exact match keeps substring semantics', () => {
|
|
61
|
+
const picked = filterTestFilesByPatterns(files, ['fanout-new']);
|
|
62
|
+
expect(picked).toEqual(['/top/tests/aspect-fanout-new-systems.test.ts']);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
test('several patterns union their matches', () => {
|
|
66
|
+
const picked = filterTestFilesByPatterns(files, ['aspect-fanout-new-systems', 'aspect-fanout']);
|
|
67
|
+
expect(picked).toEqual([
|
|
68
|
+
'/mods/knot/e2e/aspect-fanout.test.ts',
|
|
69
|
+
'/top/tests/aspect-fanout-new-systems.test.ts',
|
|
70
|
+
]);
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test('no patterns selects everything', () => {
|
|
74
|
+
expect(filterTestFilesByPatterns(files, [])).toEqual(files);
|
|
75
|
+
});
|
|
76
|
+
});
|