@omnitron-dev/testing 0.4.5 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -0
- package/dist/async/events.js.map +1 -1
- package/dist/async/retry.js.map +1 -1
- package/dist/async/wait.js.map +1 -1
- package/dist/docker/database-manager.js +13 -15
- package/dist/docker/database-manager.js.map +1 -1
- package/dist/docker/docker-test-manager.d.ts +12 -3
- package/dist/docker/docker-test-manager.js +286 -255
- package/dist/docker/docker-test-manager.js.map +1 -1
- package/dist/docker/redis-manager.js +68 -94
- package/dist/docker/redis-manager.js.map +1 -1
- package/dist/docker/types.d.ts +7 -2
- package/dist/env.d.ts +11 -0
- package/dist/env.js +21 -0
- package/dist/env.js.map +1 -0
- package/dist/errors.js.map +1 -1
- package/dist/helpers/assertions.js +3 -3
- package/dist/helpers/assertions.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/runtime/bun-adapter.d.ts +23 -1
- package/dist/runtime/bun-adapter.js +14 -1
- package/dist/runtime/bun-adapter.js.map +1 -1
- package/dist/runtime/deno-adapter.d.ts +72 -15
- package/dist/runtime/deno-adapter.js +93 -78
- package/dist/runtime/deno-adapter.js.map +1 -1
- package/dist/runtime/node-adapter.d.ts +15 -3
- package/dist/runtime/node-adapter.js +1 -1
- package/dist/runtime/node-adapter.js.map +1 -1
- package/dist/runtime/test-adapter.d.ts +1 -1
- package/dist/runtime/test-adapter.js +12 -14
- package/dist/runtime/test-adapter.js.map +1 -1
- package/dist/titan/database/database-testing.module.d.ts +7 -6
- package/dist/titan/database/database-testing.module.js +39 -18
- package/dist/titan/database/database-testing.module.js.map +1 -1
- package/dist/titan/database/index.d.ts +1 -1
- package/dist/titan/database/index.js +1 -1
- package/dist/titan/database/index.js.map +1 -1
- package/dist/titan/database/test-utilities.d.ts +2 -2
- package/dist/titan/database/test-utilities.js +3 -2
- package/dist/titan/database/test-utilities.js.map +1 -1
- package/dist/titan/index.d.ts +4 -4
- package/dist/titan/index.js +3 -3
- package/dist/titan/index.js.map +1 -1
- package/dist/titan/nexus/mock-provider.js +6 -6
- package/dist/titan/nexus/mock-provider.js.map +1 -1
- package/dist/titan/nexus/test-container.js +10 -10
- package/dist/titan/nexus/test-container.js.map +1 -1
- package/dist/titan/nexus/test-utilities.js +9 -9
- package/dist/titan/nexus/test-utilities.js.map +1 -1
- package/dist/titan/pm/mock-process-spawner.d.ts +4 -3
- package/dist/titan/pm/mock-process-spawner.js +33 -14
- package/dist/titan/pm/mock-process-spawner.js.map +1 -1
- package/dist/titan/pm/mock-spawner.d.ts +2 -1
- package/dist/titan/pm/mock-spawner.js +3 -1
- package/dist/titan/pm/mock-spawner.js.map +1 -1
- package/dist/titan/pm/test-process-manager.d.ts +4 -4
- package/dist/titan/pm/test-process-manager.js +7 -3
- package/dist/titan/pm/test-process-manager.js.map +1 -1
- package/dist/titan/redis-test-utils.d.ts +8 -2
- package/dist/titan/redis-test-utils.js +147 -92
- package/dist/titan/redis-test-utils.js.map +1 -1
- package/dist/titan/test-application.d.ts +1 -1
- package/dist/titan/test-application.js +1 -1
- package/dist/titan/test-application.js.map +1 -1
- package/dist/titan/test-fixtures.js.map +1 -1
- package/dist/titan/test-mocks.d.ts +12 -11
- package/dist/titan/test-mocks.js +49 -49
- package/dist/titan/test-mocks.js.map +1 -1
- package/dist/titan/test-module.js.map +1 -1
- package/package.json +66 -40
- package/src/async/README.md +18 -328
- package/src/async/events.ts +1 -1
- package/src/docker/a-port-the-network-could-reach.spec.ts +66 -0
- package/src/docker/database-manager.ts +13 -15
- package/src/docker/docker-orphan-reaper.spec.ts +175 -0
- package/src/docker/docker-port-collision.spec.ts +97 -0
- package/src/docker/docker-test-manager.spec.ts +122 -0
- package/src/docker/docker-test-manager.ts +420 -326
- package/src/docker/docker-volume-cleanup.spec.ts +113 -0
- package/src/docker/redis-cluster-creation.spec.ts +52 -0
- package/src/docker/redis-manager.ts +80 -115
- package/src/docker/types.ts +11 -4
- package/src/env.spec.ts +20 -0
- package/src/env.ts +33 -0
- package/src/globals.d.ts +2 -2
- package/src/helpers/assertions.ts +3 -3
- package/src/index.ts +1 -1
- package/src/runtime/adapter-surface.spec.ts +83 -0
- package/src/runtime/bun-adapter.ts +36 -5
- package/src/runtime/deno-adapter.ts +167 -108
- package/src/runtime/node-adapter.ts +5 -5
- package/src/runtime/test-adapter.test.ts +1 -1
- package/src/runtime/test-adapter.ts +15 -17
- package/src/titan/database/database-testing.module.ts +63 -22
- package/src/titan/database/index.ts +2 -2
- package/src/titan/database/test-utilities.ts +6 -3
- package/src/titan/index.ts +8 -1
- package/src/titan/nexus/mock-provider.ts +12 -12
- package/src/titan/nexus/test-container.ts +12 -12
- package/src/titan/nexus/test-utilities.ts +13 -11
- package/src/titan/pm/mock-process-spawner.ts +74 -26
- package/src/titan/pm/mock-spawner.ts +10 -4
- package/src/titan/pm/test-process-manager.ts +26 -9
- package/src/titan/redis-test-utils.test.ts +102 -0
- package/src/titan/redis-test-utils.ts +191 -109
- package/src/titan/test-application.ts +2 -2
- package/src/titan/test-fixtures.ts +0 -1
- package/src/titan/test-mocks.ts +50 -50
- package/runtime/bun.d.ts +0 -2
- package/runtime/bun.js +0 -5
- package/runtime/bun.ts +0 -7
- package/runtime/deno.d.ts +0 -2
- package/runtime/deno.js +0 -4
- package/runtime/jest.d.ts +0 -2
- package/runtime/jest.js +0 -16
- package/runtime/jest.ts +0 -7
|
@@ -13,12 +13,70 @@
|
|
|
13
13
|
* - Network management
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
|
-
import { execSync, execFileSync } from 'child_process';
|
|
16
|
+
import { execSync, execFileSync, execFile } from 'child_process';
|
|
17
17
|
import { randomBytes } from 'crypto';
|
|
18
|
+
import { hostname } from 'os';
|
|
18
19
|
import * as net from 'net';
|
|
19
20
|
import { EventEmitter } from 'events';
|
|
20
21
|
import type { DockerContainer, DockerContainerStatus, DockerTestManagerOptions, ContainerOptions } from './types.js';
|
|
21
22
|
|
|
23
|
+
/**
|
|
24
|
+
* `docker ps --format '{{.Labels}}'` renders labels as `k=v,k=v`. Values here
|
|
25
|
+
* are manager ids and hostnames, neither of which contains a comma.
|
|
26
|
+
*/
|
|
27
|
+
function parseDockerLabels(labels: string): Record<string, string> {
|
|
28
|
+
const out: Record<string, string> = {};
|
|
29
|
+
for (const pair of labels.split(',')) {
|
|
30
|
+
const eq = pair.indexOf('=');
|
|
31
|
+
if (eq > 0) out[pair.slice(0, eq)] = pair.slice(eq + 1);
|
|
32
|
+
}
|
|
33
|
+
return out;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Docker's CLI evaluates --health-cmd in a shell. Preserve literal CMD arguments. */
|
|
37
|
+
function dockerHealthCommand(test: string | string[]): string {
|
|
38
|
+
if (typeof test === 'string') return test;
|
|
39
|
+
const [mode, ...args] = test;
|
|
40
|
+
if (mode === 'CMD-SHELL') return args.join(' ');
|
|
41
|
+
const argv = mode === 'CMD' ? args : test;
|
|
42
|
+
return argv.map(arg => "'" + arg.replaceAll("'", "'\\''") + "'").join(' ');
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** The pid the manager id starts with, or null when it is not one. */
|
|
46
|
+
function pidFromManagerId(managerId: string): number | null {
|
|
47
|
+
const pid = Number.parseInt(managerId.split('-')[0] ?? '', 10);
|
|
48
|
+
return Number.isInteger(pid) && pid > 0 ? pid : null;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Signal 0 asks the kernel whether the process exists without delivering
|
|
53
|
+
* anything. EPERM means it exists and belongs to another user, which is still
|
|
54
|
+
* alive; only ESRCH means gone.
|
|
55
|
+
*/
|
|
56
|
+
function isPidAlive(pid: number): boolean {
|
|
57
|
+
try {
|
|
58
|
+
process.kill(pid, 0);
|
|
59
|
+
return true;
|
|
60
|
+
} catch (error) {
|
|
61
|
+
return (error as NodeJS.ErrnoException).code === 'EPERM';
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* `docker ps --format '{{.CreatedAt}}'` prints `2026-09-06 01:07:22 +0300 MSK`
|
|
67
|
+
* — not ISO 8601, and the trailing abbreviation is ambiguous between zones.
|
|
68
|
+
* The numeric offset in front of it is not, so the abbreviation is dropped and
|
|
69
|
+
* the rest is rewritten into a form `Date.parse` is specified to accept.
|
|
70
|
+
* Returns null when the shape is anything else, and a null age is never old
|
|
71
|
+
* enough to reap.
|
|
72
|
+
*/
|
|
73
|
+
function parseDockerTimestamp(createdAt: string): number | null {
|
|
74
|
+
const m = /^(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2})(?:\.\d+)? ([+-]\d{2})(\d{2})/.exec(createdAt.trim());
|
|
75
|
+
if (!m) return null;
|
|
76
|
+
const parsed = Date.parse(`${m[1]}T${m[2]}${m[3]}:${m[4]}`);
|
|
77
|
+
return Number.isNaN(parsed) ? null : parsed;
|
|
78
|
+
}
|
|
79
|
+
|
|
22
80
|
/**
|
|
23
81
|
* Docker Test Manager
|
|
24
82
|
*
|
|
@@ -46,10 +104,18 @@ export class DockerTestManager extends EventEmitter {
|
|
|
46
104
|
private static instance: DockerTestManager;
|
|
47
105
|
private containers: Map<string, DockerContainer> = new Map();
|
|
48
106
|
private usedPorts: Set<number> = new Set();
|
|
49
|
-
private networks
|
|
50
|
-
private
|
|
107
|
+
private networks = new Map<string, string>();
|
|
108
|
+
private activeCleanup?: Promise<void>;
|
|
51
109
|
|
|
52
110
|
private dockerPath: string;
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* How many times a `docker run` may be re-attempted with freshly drawn `auto`
|
|
114
|
+
* host ports after docker refuses the published port. Small on purpose: this
|
|
115
|
+
* covers losing a race, not a machine with no free ports.
|
|
116
|
+
*/
|
|
117
|
+
private static readonly PORT_COLLISION_RETRIES = 3;
|
|
118
|
+
|
|
53
119
|
private basePort: number;
|
|
54
120
|
private maxRetries: number;
|
|
55
121
|
private startupTimeout: number;
|
|
@@ -59,18 +125,55 @@ export class DockerTestManager extends EventEmitter {
|
|
|
59
125
|
private verbose: boolean;
|
|
60
126
|
private defaultNetwork?: string;
|
|
61
127
|
|
|
128
|
+
/**
|
|
129
|
+
* Identifies the resources THIS manager created.
|
|
130
|
+
*
|
|
131
|
+
* Cleanup used to select by the shared `test.cleanup=true` label, i.e. every
|
|
132
|
+
* test container on the host. Vitest gives each spec file its own worker
|
|
133
|
+
* process, each with its own manager and its own `process.on('exit')` hook,
|
|
134
|
+
* so the first worker to finish stopped and removed containers the others
|
|
135
|
+
* were still using. `docker stop` sends SIGTERM and Redis exits 0 on SIGTERM
|
|
136
|
+
* — which is exactly the "Container exited with status 'exited' and exit code
|
|
137
|
+
* 0" that made titan-redis, titan-database and titan-notifications flaky
|
|
138
|
+
* under parallel runs.
|
|
139
|
+
*/
|
|
140
|
+
private readonly managerId = `${process.pid}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* The machine the pid in `managerId` belongs to. Tests that talk to a docker
|
|
144
|
+
* daemon on another host would otherwise read that host's pid table as their
|
|
145
|
+
* own. Containers created before this label existed carry no host, which is
|
|
146
|
+
* absence of information rather than a mismatch, so they fall through to the
|
|
147
|
+
* pid and age rules below.
|
|
148
|
+
*/
|
|
149
|
+
private static readonly hostId = hostname();
|
|
150
|
+
|
|
151
|
+
/** Orphan reaping is a property of the process, not of a manager instance. */
|
|
152
|
+
private static orphanReapDone = false;
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* How old an unclaimed container must be before it is reaped.
|
|
156
|
+
*
|
|
157
|
+
* Pids are reused. A container created seconds ago belongs to a run that is
|
|
158
|
+
* starting, whatever the pid table now says about the process that created
|
|
159
|
+
* it, and this is what makes the reaper safe to run while other test
|
|
160
|
+
* processes are working.
|
|
161
|
+
*/
|
|
162
|
+
private static readonly orphanGraceMs = 10 * 60 * 1000;
|
|
163
|
+
|
|
62
164
|
private constructor(options: DockerTestManagerOptions = {}) {
|
|
63
165
|
super();
|
|
64
166
|
this.dockerPath = options.dockerPath || this.findDockerPath();
|
|
65
167
|
|
|
66
168
|
// Worker-aware port allocation to prevent conflicts in parallel test execution
|
|
67
|
-
// Each
|
|
68
|
-
const workerId = parseInt(process.env['
|
|
69
|
-
const
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
this.
|
|
169
|
+
// Each vitest worker gets its own port range within valid TCP port space (10000-65000)
|
|
170
|
+
const workerId = parseInt(process.env['VITEST_POOL_ID'] || process.env['VITEST_WORKER_ID'] || '1', 10);
|
|
171
|
+
const maxBasePort = 55000; // Leave room for 10k range below 65535
|
|
172
|
+
const basePortOffset = ((workerId - 1) * 10000) % (maxBasePort - 10000);
|
|
173
|
+
this.basePort = options.basePort || Math.min(10000 + basePortOffset, maxBasePort);
|
|
174
|
+
|
|
175
|
+
this.maxRetries = options.maxRetries || 30;
|
|
176
|
+
this.startupTimeout = options.startupTimeout || 60000;
|
|
74
177
|
this.gracefulShutdownTimeout = options.gracefulShutdownTimeout || 10000;
|
|
75
178
|
this.maxCleanupRetries = options.maxCleanupRetries || 3;
|
|
76
179
|
this.cleanup = options.cleanup !== false;
|
|
@@ -78,7 +181,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
78
181
|
this.defaultNetwork = options.network;
|
|
79
182
|
|
|
80
183
|
if (this.verbose) {
|
|
81
|
-
this.log(`Initialized for
|
|
184
|
+
this.log(`Initialized for Vitest worker ${workerId} with port range ${this.basePort}-${this.basePort + 10000}`);
|
|
82
185
|
}
|
|
83
186
|
|
|
84
187
|
// Verify Docker is available
|
|
@@ -90,13 +193,17 @@ export class DockerTestManager extends EventEmitter {
|
|
|
90
193
|
// This is safe because signal handlers should be idempotent and cleanup is managed by the singleton pattern
|
|
91
194
|
process.setMaxListeners(Math.max(20, process.getMaxListeners()));
|
|
92
195
|
|
|
196
|
+
// Runs before any container of this process exists, so a reaped port is
|
|
197
|
+
// free by the time this manager draws one.
|
|
198
|
+
this.reapOrphanedResources();
|
|
199
|
+
|
|
93
200
|
process.on('exit', () => this.cleanupSync());
|
|
94
201
|
process.on('SIGINT', () => {
|
|
95
202
|
this.log('SIGINT received - initiating graceful shutdown');
|
|
96
203
|
this.cleanupAllAsync()
|
|
97
204
|
.then(() => {
|
|
98
205
|
this.log('Cleanup completed, exiting');
|
|
99
|
-
process.exit(
|
|
206
|
+
process.exit(130);
|
|
100
207
|
})
|
|
101
208
|
.catch((error) => {
|
|
102
209
|
this.logError('Cleanup failed during SIGINT', error);
|
|
@@ -108,7 +215,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
108
215
|
this.cleanupAllAsync()
|
|
109
216
|
.then(() => {
|
|
110
217
|
this.log('Cleanup completed, exiting');
|
|
111
|
-
process.exit(
|
|
218
|
+
process.exit(143);
|
|
112
219
|
})
|
|
113
220
|
.catch((error) => {
|
|
114
221
|
this.logError('Cleanup failed during SIGTERM', error);
|
|
@@ -275,13 +382,148 @@ export class DockerTestManager extends EventEmitter {
|
|
|
275
382
|
}
|
|
276
383
|
}
|
|
277
384
|
|
|
385
|
+
/**
|
|
386
|
+
* Remove containers and networks left behind by test processes that no
|
|
387
|
+
* longer exist.
|
|
388
|
+
*
|
|
389
|
+
* `cleanupSync` and the signal handlers cover every ending the process gets
|
|
390
|
+
* to observe. SIGKILL is not one of them, and it is the ending vitest uses
|
|
391
|
+
* for a worker that overruns teardown — so a killed run leaves its
|
|
392
|
+
* containers running, holding their ports, their memory and their anonymous
|
|
393
|
+
* volumes, with nothing left on the machine that refers to them.
|
|
394
|
+
*
|
|
395
|
+
* A container is reaped only when BOTH hold:
|
|
396
|
+
*
|
|
397
|
+
* - the pid recorded in `test.manager` is not alive. Pids are reused, so
|
|
398
|
+
* this alone can name a live manager's container as an orphan;
|
|
399
|
+
* - it is older than `orphanGraceMs`. A container seconds old belongs to a
|
|
400
|
+
* run that is starting, whatever the pid table says.
|
|
401
|
+
*
|
|
402
|
+
* Each condition failing leaves a container behind for the next run to
|
|
403
|
+
* collect. Neither can remove a container that is in use.
|
|
404
|
+
*
|
|
405
|
+
* Errors are swallowed: this runs from a constructor, and a machine whose
|
|
406
|
+
* docker is unreachable should fail on the first container command with that
|
|
407
|
+
* command's message, not on construction.
|
|
408
|
+
*
|
|
409
|
+
* @private
|
|
410
|
+
*/
|
|
411
|
+
private reapOrphanedResources(): void {
|
|
412
|
+
if (DockerTestManager.orphanReapDone) return;
|
|
413
|
+
DockerTestManager.orphanReapDone = true;
|
|
414
|
+
|
|
415
|
+
try {
|
|
416
|
+
const listed = execFileSync(
|
|
417
|
+
this.dockerPath,
|
|
418
|
+
['ps', '-a', '--filter', 'label=test.cleanup=true', '--format', '{{.ID}}\t{{.CreatedAt}}\t{{.Labels}}'],
|
|
419
|
+
{ encoding: 'utf8', stdio: ['pipe', 'pipe', 'ignore'] }
|
|
420
|
+
).trim();
|
|
421
|
+
|
|
422
|
+
const orphans: string[] = [];
|
|
423
|
+
for (const line of listed ? listed.split('\n') : []) {
|
|
424
|
+
const [id, createdAt, labels] = line.split('\t');
|
|
425
|
+
if (!id || !createdAt || !labels) continue;
|
|
426
|
+
if (!this.isOrphaned(labels, createdAt)) continue;
|
|
427
|
+
orphans.push(id);
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
if (orphans.length > 0) {
|
|
431
|
+
this.log(`Reaping ${orphans.length} container(s) left by processes that no longer exist`);
|
|
432
|
+
// One call, not one per container: a stalled machine can hold dozens,
|
|
433
|
+
// and every worker of a parallel run pays this before its first test.
|
|
434
|
+
try {
|
|
435
|
+
execFileSync(this.dockerPath, ['rm', '-f', '-v', ...orphans], { stdio: 'ignore' });
|
|
436
|
+
} catch (error) {
|
|
437
|
+
// A concurrent worker reaping the same container makes `docker rm`
|
|
438
|
+
// exit non-zero for that id and continue with the rest. Both wanted
|
|
439
|
+
// it gone.
|
|
440
|
+
this.logError('Some orphaned containers could not be removed', error);
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
this.reapOrphanedNetworks();
|
|
445
|
+
} catch (error) {
|
|
446
|
+
this.logError('Could not scan for orphaned test containers', error);
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* The same rule for networks, minus the age condition: `docker network rm`
|
|
452
|
+
* refuses a network that still has an endpoint attached, so an in-use
|
|
453
|
+
* network is protected by docker itself rather than by a grace period. A
|
|
454
|
+
* leaked one is not free — the address pool it comes from is what
|
|
455
|
+
* `createNetwork` already has to recover from when it runs out.
|
|
456
|
+
*
|
|
457
|
+
* @private
|
|
458
|
+
*/
|
|
459
|
+
private reapOrphanedNetworks(): void {
|
|
460
|
+
try {
|
|
461
|
+
const listed = execFileSync(
|
|
462
|
+
this.dockerPath,
|
|
463
|
+
['network', 'ls', '--filter', 'label=test.cleanup=true', '--format', '{{.ID}}\t{{.Labels}}'],
|
|
464
|
+
{ encoding: 'utf8', stdio: ['pipe', 'pipe', 'ignore'] }
|
|
465
|
+
).trim();
|
|
466
|
+
|
|
467
|
+
const orphans: string[] = [];
|
|
468
|
+
for (const line of listed ? listed.split('\n') : []) {
|
|
469
|
+
const [id, labels] = line.split('\t');
|
|
470
|
+
if (!id || !labels) continue;
|
|
471
|
+
if (!this.isOrphaned(labels, null)) continue;
|
|
472
|
+
orphans.push(id);
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
if (orphans.length > 0) {
|
|
476
|
+
this.log(`Reaping ${orphans.length} network(s) left by processes that no longer exist`);
|
|
477
|
+
try {
|
|
478
|
+
execFileSync(this.dockerPath, ['network', 'rm', ...orphans], { stdio: 'ignore' });
|
|
479
|
+
} catch (error) {
|
|
480
|
+
// Refused for a network still in use, which is the answer we want.
|
|
481
|
+
this.logError('Some orphaned test networks could not be removed', error);
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
} catch (error) {
|
|
485
|
+
this.logError('Could not scan for orphaned test networks', error);
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
/**
|
|
490
|
+
* Whether a labelled resource belongs to a test process that is gone.
|
|
491
|
+
*
|
|
492
|
+
* @param createdAt - `{{.CreatedAt}}`, or null to skip the age condition.
|
|
493
|
+
* @private
|
|
494
|
+
*/
|
|
495
|
+
private isOrphaned(labels: string, createdAt: string | null): boolean {
|
|
496
|
+
const parsed = parseDockerLabels(labels);
|
|
497
|
+
|
|
498
|
+
const owner = parsed['test.manager'];
|
|
499
|
+
// No owner recorded: created by a manager older than the label, or by
|
|
500
|
+
// something else entirely. Either way this reaper cannot say whose it is.
|
|
501
|
+
if (!owner || owner === this.managerId) return false;
|
|
502
|
+
|
|
503
|
+
const host = parsed['test.host'];
|
|
504
|
+
if (host && host !== DockerTestManager.hostId) return false;
|
|
505
|
+
|
|
506
|
+
const pid = pidFromManagerId(owner);
|
|
507
|
+
if (pid === null || isPidAlive(pid)) return false;
|
|
508
|
+
|
|
509
|
+
if (createdAt !== null) {
|
|
510
|
+
const created = parseDockerTimestamp(createdAt);
|
|
511
|
+
if (created === null) return false;
|
|
512
|
+
if (Date.now() - created < DockerTestManager.orphanGraceMs) return false;
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
return true;
|
|
516
|
+
}
|
|
517
|
+
|
|
278
518
|
/**
|
|
279
519
|
* Find an available port for container mapping
|
|
280
520
|
* @private
|
|
281
521
|
*/
|
|
282
522
|
private async findAvailablePort(): Promise<number> {
|
|
523
|
+
const portRangeSize = Math.min(10000, 65535 - this.basePort);
|
|
283
524
|
for (let i = 0; i < this.maxRetries; i++) {
|
|
284
|
-
const port = this.basePort + Math.floor(Math.random() *
|
|
525
|
+
const port = this.basePort + Math.floor(Math.random() * portRangeSize);
|
|
526
|
+
if (port >= 65536) continue; // Safety guard
|
|
285
527
|
if (!this.usedPorts.has(port) && (await this.isPortAvailable(port))) {
|
|
286
528
|
this.usedPorts.add(port);
|
|
287
529
|
return port;
|
|
@@ -342,7 +584,18 @@ export class DockerTestManager extends EventEmitter {
|
|
|
342
584
|
[
|
|
343
585
|
'inspect',
|
|
344
586
|
'--format',
|
|
345
|
-
|
|
587
|
+
// `.State.Health` is absent on a container with no HEALTHCHECK, and
|
|
588
|
+
// Go templates fail hard on a missing map key rather than rendering
|
|
589
|
+
// empty: `docker inspect` exited 1 with "map has no entry for key
|
|
590
|
+
// \"Health\"". stderr is ignored here and the catch below reports a
|
|
591
|
+
// default, so every such container was read as NOT RUNNING — and
|
|
592
|
+
// `waitForContainer` rejected with "Container exited with status
|
|
593
|
+
// 'undefined' and exit code undefined" for a container that was
|
|
594
|
+
// running perfectly. That made `waitFor` unusable with any image
|
|
595
|
+
// lacking a healthcheck, which is precisely the case
|
|
596
|
+
// `waitFor.healthcheck: false` exists to serve; callers worked around
|
|
597
|
+
// it by always declaring one.
|
|
598
|
+
'{{.State.Running}}\t{{.State.Status}}\t{{if .State.Health}}{{.State.Health.Status}}{{end}}\t{{.State.ExitCode}}',
|
|
346
599
|
name,
|
|
347
600
|
],
|
|
348
601
|
{
|
|
@@ -407,23 +660,31 @@ export class DockerTestManager extends EventEmitter {
|
|
|
407
660
|
* ```
|
|
408
661
|
*/
|
|
409
662
|
async createContainer(options: ContainerOptions): Promise<DockerContainer> {
|
|
663
|
+
const resources = options.resources;
|
|
664
|
+
if (resources?.memoryBytes !== undefined && (!Number.isSafeInteger(resources.memoryBytes) || resources.memoryBytes < 6 * 1024 * 1024))
|
|
665
|
+
throw new RangeError('memoryBytes must be a safe integer of at least 6 MiB');
|
|
666
|
+
if (resources?.cpus !== undefined && (!Number.isFinite(resources.cpus) || resources.cpus <= 0))
|
|
667
|
+
throw new RangeError('cpus must be a finite positive number');
|
|
668
|
+
if (resources?.pidsLimit !== undefined && (!Number.isSafeInteger(resources.pidsLimit) || resources.pidsLimit < 1))
|
|
669
|
+
throw new RangeError('pidsLimit must be a positive safe integer');
|
|
410
670
|
const id = `test-${randomBytes(8).toString('hex')}`; // 8 bytes for better uniqueness in parallel execution
|
|
411
671
|
const name = options.name || `container-${id}`;
|
|
412
672
|
const host = '127.0.0.1';
|
|
413
673
|
|
|
414
|
-
//
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
674
|
+
// Process port mappings.
|
|
675
|
+
//
|
|
676
|
+
// `auto` ports are re-drawn if docker refuses the run: findAvailablePort()
|
|
677
|
+
// binds, reads the port and releases it, so between that check and
|
|
678
|
+
// `docker run` anything else on the machine can take it — another suite in
|
|
679
|
+
// this repo, another session, a container someone restarted. The window is
|
|
680
|
+
// small and the machine is shared, which is exactly the combination that
|
|
681
|
+
// produces a red test blaming the code under test.
|
|
682
|
+
const autoPorts = new Set<number>();
|
|
423
683
|
const portMappings = new Map<number, number>();
|
|
424
684
|
if (options.ports) {
|
|
425
685
|
for (const [containerPort, hostPort] of Object.entries(options.ports)) {
|
|
426
686
|
const cPort = parseInt(containerPort);
|
|
687
|
+
if (hostPort === 'auto') autoPorts.add(cPort);
|
|
427
688
|
const hPort = hostPort === 'auto' ? await this.findAvailablePort() : hostPort;
|
|
428
689
|
portMappings.set(cPort, hPort);
|
|
429
690
|
}
|
|
@@ -431,11 +692,30 @@ export class DockerTestManager extends EventEmitter {
|
|
|
431
692
|
|
|
432
693
|
// Build docker run command
|
|
433
694
|
const dockerArgs: string[] = ['run', '-d', '--name', name];
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
695
|
+
if (resources?.memoryBytes !== undefined) dockerArgs.push('--memory', String(resources.memoryBytes));
|
|
696
|
+
if (resources?.cpus !== undefined) dockerArgs.push('--cpus', String(resources.cpus));
|
|
697
|
+
if (resources?.pidsLimit !== undefined) dockerArgs.push('--pids-limit', String(resources.pidsLimit));
|
|
698
|
+
|
|
699
|
+
// Add ports. The slice is remembered so a collision retry can rewrite just
|
|
700
|
+
// these entries instead of rebuilding the whole command.
|
|
701
|
+
const portArgsStart = dockerArgs.length;
|
|
702
|
+
const writePortArgs = () => {
|
|
703
|
+
const args: string[] = [];
|
|
704
|
+
portMappings.forEach((hostPort, containerPort) => {
|
|
705
|
+
// The loopback only: a bare `-p host:container` publishes on 0.0.0.0,
|
|
706
|
+
// past the host firewall, and a test database with its default
|
|
707
|
+
// credentials is then a database the LAN can log into
|
|
708
|
+
// (scripts/a-port-the-network-could-reach.mjs). `isPortAvailable`
|
|
709
|
+
// probes the same address.
|
|
710
|
+
args.push('-p', `127.0.0.1:${hostPort}:${containerPort}`);
|
|
711
|
+
});
|
|
712
|
+
return args;
|
|
713
|
+
};
|
|
714
|
+
dockerArgs.push(...writePortArgs());
|
|
715
|
+
const portArgsLength = dockerArgs.length - portArgsStart;
|
|
716
|
+
const rebuildPortArgs = () => {
|
|
717
|
+
dockerArgs.splice(portArgsStart, portArgsLength, ...writePortArgs());
|
|
718
|
+
};
|
|
439
719
|
|
|
440
720
|
// Add environment variables
|
|
441
721
|
const environment = options.environment || {};
|
|
@@ -445,9 +725,13 @@ export class DockerTestManager extends EventEmitter {
|
|
|
445
725
|
|
|
446
726
|
// Add labels
|
|
447
727
|
const labels = {
|
|
728
|
+
...options.labels,
|
|
448
729
|
'test.id': id,
|
|
449
730
|
'test.cleanup': 'true',
|
|
450
|
-
|
|
731
|
+
// Ownership, so cleanup can select this manager's containers instead of
|
|
732
|
+
// every test container on the machine.
|
|
733
|
+
'test.manager': this.managerId,
|
|
734
|
+
'test.host': DockerTestManager.hostId,
|
|
451
735
|
};
|
|
452
736
|
for (const [key, value] of Object.entries(labels)) {
|
|
453
737
|
dockerArgs.push('--label', `${key}=${value}`);
|
|
@@ -470,29 +754,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
470
754
|
|
|
471
755
|
// Add healthcheck if provided
|
|
472
756
|
if (options.healthcheck) {
|
|
473
|
-
|
|
474
|
-
// The healthcheck.test array format from Docker Compose needs to be converted:
|
|
475
|
-
// - ['CMD', 'arg1', 'arg2'] -> execute command directly (no shell)
|
|
476
|
-
// - ['CMD-SHELL', 'command'] -> execute via shell
|
|
477
|
-
let healthCmd: string;
|
|
478
|
-
if (Array.isArray(options.healthcheck.test)) {
|
|
479
|
-
if (options.healthcheck.test[0] === 'CMD-SHELL') {
|
|
480
|
-
// CMD-SHELL format: join everything after the first element
|
|
481
|
-
healthCmd = options.healthcheck.test.slice(1).join(' ');
|
|
482
|
-
} else if (options.healthcheck.test[0] === 'CMD') {
|
|
483
|
-
// CMD format: join command parts without shell
|
|
484
|
-
// Docker CLI expects this as a single string that will be word-split
|
|
485
|
-
// For example: ['CMD', 'redis-cli', 'ping'] -> 'redis-cli ping'
|
|
486
|
-
healthCmd = options.healthcheck.test.slice(1).join(' ');
|
|
487
|
-
} else {
|
|
488
|
-
// No format specifier, join all parts
|
|
489
|
-
healthCmd = options.healthcheck.test.join(' ');
|
|
490
|
-
}
|
|
491
|
-
} else {
|
|
492
|
-
// String format
|
|
493
|
-
healthCmd = options.healthcheck.test;
|
|
494
|
-
}
|
|
495
|
-
dockerArgs.push('--health-cmd', healthCmd);
|
|
757
|
+
dockerArgs.push('--health-cmd', dockerHealthCommand(options.healthcheck.test));
|
|
496
758
|
if (options.healthcheck.interval) {
|
|
497
759
|
dockerArgs.push('--health-interval', options.healthcheck.interval);
|
|
498
760
|
}
|
|
@@ -510,68 +772,59 @@ export class DockerTestManager extends EventEmitter {
|
|
|
510
772
|
// Add image and command
|
|
511
773
|
dockerArgs.push(options.image);
|
|
512
774
|
if (options.command) {
|
|
513
|
-
dockerArgs.push(
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
if (this.verbose) {
|
|
517
|
-
console.log(`Starting container ${name}: docker ${dockerArgs.join(' ')}`);
|
|
775
|
+
if (Array.isArray(options.command)) dockerArgs.push(...options.command);
|
|
776
|
+
else dockerArgs.push('sh', '-c', options.command);
|
|
518
777
|
}
|
|
519
778
|
|
|
520
|
-
|
|
521
|
-
|
|
779
|
+
let containerId: string;
|
|
780
|
+
for (let attempt = 0; ; attempt++) {
|
|
522
781
|
this.log(`Starting container: ${name}`, { image: options.image });
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
782
|
+
// A synchronous Docker start blocks every peer's readiness timer while
|
|
783
|
+
// Docker creates the next container. Keep cluster startup concurrent.
|
|
784
|
+
const result = await new Promise<{ status: number | null; stdout: string; stderr: string; error?: { code?: string | number | null } }>(resolve => {
|
|
785
|
+
execFile(this.dockerPath, dockerArgs, { encoding: 'utf8', timeout: this.startupTimeout, maxBuffer: 2 * 1024 * 1024 },
|
|
786
|
+
(error, stdout, stderr) => resolve({ status: error ? (typeof error.code === 'number' ? error.code : null) : 0,
|
|
787
|
+
stdout, stderr, ...(error && typeof error.code !== 'number' ? { error } : {}) }));
|
|
526
788
|
});
|
|
527
|
-
|
|
528
|
-
|
|
789
|
+
containerId = (result.stdout ?? '').trim();
|
|
790
|
+
if (result.status === 0 && /^[a-f0-9]{64}$/.test(containerId)) break;
|
|
791
|
+
portMappings.forEach(port => this.usedPorts.delete(port));
|
|
792
|
+
let diagnostic = (result.stderr ?? '').trim();
|
|
793
|
+
for (const value of Object.values(environment)) {
|
|
794
|
+
if (value) diagnostic = diagnostic.replaceAll(value, '[redacted]');
|
|
529
795
|
}
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
portMappings.forEach((port) => {
|
|
533
|
-
this.usedPorts.delete(port);
|
|
534
|
-
});
|
|
535
|
-
const execError = error as {
|
|
536
|
-
stderr?: Buffer | string;
|
|
537
|
-
stdout?: Buffer | string;
|
|
538
|
-
message?: string;
|
|
539
|
-
status?: number;
|
|
540
|
-
};
|
|
541
|
-
const stderr = execError.stderr ? String(execError.stderr).trim() : '';
|
|
542
|
-
const stdout = execError.stdout ? String(execError.stdout).trim() : '';
|
|
543
|
-
const message = execError.message || String(error);
|
|
544
|
-
|
|
545
|
-
// Check if container already exists
|
|
546
|
-
if (stderr.includes('already in use') || stderr.includes('name is already') || stderr.includes('Conflict')) {
|
|
547
|
-
throw new Error(
|
|
548
|
-
`Container name conflict: ${name} already exists.\n` +
|
|
549
|
-
`Please ensure previous test containers are cleaned up.\n` +
|
|
550
|
-
`Stderr: ${stderr}`
|
|
551
|
-
);
|
|
796
|
+
if (/container name .*already in use|name is already in use/i.test(diagnostic)) {
|
|
797
|
+
throw new Error(`Container name conflict: ${name} already exists`);
|
|
552
798
|
}
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
);
|
|
799
|
+
// A failed bind can leave a container in "created". Resolve this
|
|
800
|
+
// attempt's labels and remove only the Docker ID they identify.
|
|
801
|
+
const residue = execFileSync(this.dockerPath, ['ps', '-aq', '--no-trunc',
|
|
802
|
+
'--filter', `label=test.id=${id}`, '--filter', `label=test.manager=${this.managerId}`],
|
|
803
|
+
{ encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
|
804
|
+
if (/^[a-f0-9]{64}$/.test(residue)) {
|
|
805
|
+
execFileSync(this.dockerPath, ['rm', '-f', '-v', residue], { stdio: 'ignore' });
|
|
806
|
+
}
|
|
807
|
+
if (result.error) throw new Error(`Could not execute Docker for ${name}: ${result.error.code ?? 'unknown'}`);
|
|
808
|
+
if (!/port is already allocated|address already in use|Bind for /.test(diagnostic)
|
|
809
|
+
|| !autoPorts.size || attempt >= DockerTestManager.PORT_COLLISION_RETRIES) {
|
|
810
|
+
throw new Error(`Failed to start container ${name} (${options.image}), exit ${result.status ?? 'unknown'}: ${diagnostic.slice(0, 2000) || 'Docker did not confirm creation'}`);
|
|
811
|
+
}
|
|
812
|
+
for (const port of autoPorts) portMappings.set(port, await this.findAvailablePort());
|
|
813
|
+
rebuildPortArgs();
|
|
814
|
+
this.log(`Host port collision for ${name}, retrying with fresh ports`, { attempt: attempt + 1 });
|
|
562
815
|
}
|
|
563
816
|
|
|
564
817
|
// Wait for container to be ready
|
|
565
818
|
if (options.waitFor) {
|
|
566
819
|
try {
|
|
567
|
-
await this.waitForContainer(
|
|
820
|
+
await this.waitForContainer(containerId, options.waitFor, portMappings);
|
|
568
821
|
this.log(`Container ready: ${name}`);
|
|
569
822
|
} catch (error) {
|
|
570
823
|
// Container failed to start, attempt cleanup
|
|
571
824
|
this.logError(`Container ${name} failed to become ready, cleaning up`, error);
|
|
572
825
|
try {
|
|
573
|
-
execFileSync(this.dockerPath, ['stop', '-t', '5',
|
|
574
|
-
execFileSync(this.dockerPath, ['rm', '-f',
|
|
826
|
+
execFileSync(this.dockerPath, ['stop', '-t', '5', containerId], { stdio: 'ignore' });
|
|
827
|
+
execFileSync(this.dockerPath, ['rm', '-f', '-v', containerId], { stdio: 'ignore' });
|
|
575
828
|
} catch {
|
|
576
829
|
// Ignore cleanup errors during rollback
|
|
577
830
|
}
|
|
@@ -617,10 +870,10 @@ export class DockerTestManager extends EventEmitter {
|
|
|
617
870
|
networks,
|
|
618
871
|
createdAt: new Date(),
|
|
619
872
|
internalIp,
|
|
620
|
-
getStatus: async () => this.getContainerStatus(
|
|
621
|
-
isHealthy: async () => this.isContainerHealthy(
|
|
873
|
+
getStatus: async () => this.getContainerStatus(containerId),
|
|
874
|
+
isHealthy: async () => this.isContainerHealthy(containerId),
|
|
622
875
|
cleanup: async () => {
|
|
623
|
-
await this.cleanupContainerWithRetry(
|
|
876
|
+
await this.cleanupContainerWithRetry(containerId, id, portMappings);
|
|
624
877
|
},
|
|
625
878
|
};
|
|
626
879
|
|
|
@@ -632,155 +885,67 @@ export class DockerTestManager extends EventEmitter {
|
|
|
632
885
|
* Clean up a container with retry logic
|
|
633
886
|
* @private
|
|
634
887
|
*/
|
|
635
|
-
private async cleanupContainerWithRetry(
|
|
636
|
-
this.
|
|
637
|
-
|
|
638
|
-
let lastError: Error | null = null;
|
|
639
|
-
|
|
640
|
-
for (let attempt = 1; attempt <= this.maxCleanupRetries; attempt++) {
|
|
641
|
-
try {
|
|
642
|
-
// Check if container exists
|
|
643
|
-
try {
|
|
644
|
-
const status = await this.getContainerStatus(name);
|
|
645
|
-
if (!status.isRunning) {
|
|
646
|
-
this.log(`Container ${name} is not running, force removing`, { attempt });
|
|
647
|
-
}
|
|
648
|
-
} catch {
|
|
649
|
-
// Container doesn't exist, nothing to clean up
|
|
650
|
-
this.log(`Container ${name} doesn't exist, skipping stop`, { attempt });
|
|
651
|
-
break;
|
|
652
|
-
}
|
|
653
|
-
|
|
654
|
-
// Gracefully stop the container first
|
|
655
|
-
try {
|
|
656
|
-
const stopTimeoutSecs = Math.ceil(this.gracefulShutdownTimeout / 1000).toString();
|
|
657
|
-
this.log(`Stopping container ${name}`, { attempt, timeout: this.gracefulShutdownTimeout });
|
|
658
|
-
execFileSync(this.dockerPath, ['stop', '-t', stopTimeoutSecs, name], {
|
|
659
|
-
stdio: 'ignore',
|
|
660
|
-
timeout: this.gracefulShutdownTimeout + 5000, // Add buffer to docker timeout
|
|
661
|
-
});
|
|
662
|
-
this.log(`Container stopped: ${name}`, { attempt });
|
|
663
|
-
} catch (error) {
|
|
664
|
-
this.logError(`Failed to gracefully stop container ${name}`, error);
|
|
665
|
-
// Continue to force remove
|
|
666
|
-
}
|
|
667
|
-
|
|
668
|
-
// Remove container
|
|
669
|
-
try {
|
|
670
|
-
this.log(`Removing container ${name}`, { attempt });
|
|
671
|
-
execFileSync(this.dockerPath, ['rm', '-f', name], { stdio: 'ignore' });
|
|
672
|
-
this.log(`Container removed: ${name}`, { attempt });
|
|
673
|
-
lastError = null; // Clear error on success
|
|
674
|
-
break;
|
|
675
|
-
} catch (error) {
|
|
676
|
-
lastError = error instanceof Error ? error : new Error(String(error));
|
|
677
|
-
if (attempt === this.maxCleanupRetries) {
|
|
678
|
-
throw lastError;
|
|
679
|
-
}
|
|
680
|
-
// Wait before retrying
|
|
681
|
-
await new Promise((resolve) => setTimeout(resolve, 1000 * attempt));
|
|
682
|
-
}
|
|
683
|
-
} catch (error) {
|
|
684
|
-
lastError = error instanceof Error ? error : new Error(String(error));
|
|
685
|
-
if (attempt === this.maxCleanupRetries) {
|
|
686
|
-
this.logError(`Final cleanup attempt failed for container ${name}`, lastError);
|
|
687
|
-
// Don't throw, continue with port cleanup
|
|
688
|
-
} else {
|
|
689
|
-
this.logError(`Cleanup attempt ${attempt} failed for container ${name}, retrying`, lastError);
|
|
690
|
-
await new Promise((resolve) => setTimeout(resolve, 1000 * attempt));
|
|
691
|
-
}
|
|
692
|
-
}
|
|
693
|
-
}
|
|
694
|
-
|
|
695
|
-
// Always release ports, even if cleanup failed
|
|
888
|
+
private async cleanupContainerWithRetry(containerId: string, id: string, portMappings: Map<number, number>): Promise<void> {
|
|
889
|
+
if (!this.containers.has(id)) return;
|
|
890
|
+
const seconds = Math.ceil(this.gracefulShutdownTimeout / 1000).toString();
|
|
696
891
|
try {
|
|
697
|
-
|
|
698
|
-
this.
|
|
699
|
-
this.log(`Released port: ${port}`);
|
|
892
|
+
execFileSync(this.dockerPath, ['stop', '-t', seconds, containerId], {
|
|
893
|
+
stdio: 'ignore', timeout: this.gracefulShutdownTimeout + 5000,
|
|
700
894
|
});
|
|
701
895
|
} catch (error) {
|
|
702
|
-
this.logError(
|
|
896
|
+
this.logError(`Could not stop test container ${containerId} gracefully`, error);
|
|
703
897
|
}
|
|
704
|
-
|
|
705
|
-
// Remove from tracking
|
|
706
|
-
this.containers.delete(id);
|
|
707
|
-
this.log(`Container cleanup completed: ${name}`, { attempts: this.maxCleanupRetries });
|
|
708
|
-
}
|
|
709
|
-
|
|
710
|
-
/**
|
|
711
|
-
* Ensure a Docker network exists
|
|
712
|
-
* @private
|
|
713
|
-
*/
|
|
714
|
-
private async ensureNetwork(network: string): Promise<void> {
|
|
715
|
-
if (!this.networks.has(network)) {
|
|
898
|
+
for (let attempt = 1; ; attempt++) {
|
|
716
899
|
try {
|
|
717
|
-
execFileSync(this.dockerPath, ['
|
|
718
|
-
|
|
719
|
-
encoding: 'utf8',
|
|
720
|
-
});
|
|
721
|
-
this.networks.add(network);
|
|
722
|
-
this.log(`Created network: ${network}`);
|
|
900
|
+
execFileSync(this.dockerPath, ['rm', '-f', '-v', containerId], { stdio: ['ignore', 'ignore', 'pipe'] });
|
|
901
|
+
break;
|
|
723
902
|
} catch (error) {
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
if (stderr.includes('already exists') || stderr.includes('network with name')) {
|
|
728
|
-
this.log(`Network already exists: ${network}`);
|
|
729
|
-
this.networks.add(network);
|
|
730
|
-
} else if (stderr.includes('all predefined address pools have been fully subnetted')) {
|
|
731
|
-
// Docker network pool exhausted - try to clean up unused networks
|
|
732
|
-
this.log('Docker network pool exhausted, attempting cleanup...');
|
|
733
|
-
try {
|
|
734
|
-
// Prune unused networks
|
|
735
|
-
execFileSync(this.dockerPath, ['network', 'prune', '-f'], {
|
|
736
|
-
stdio: 'ignore',
|
|
737
|
-
});
|
|
738
|
-
this.log('Cleaned up unused networks, retrying network creation...');
|
|
739
|
-
// Retry network creation after cleanup
|
|
740
|
-
execFileSync(this.dockerPath, ['network', 'create', network, '--label', 'test.cleanup=true'], {
|
|
741
|
-
stdio: 'pipe',
|
|
742
|
-
encoding: 'utf8',
|
|
743
|
-
});
|
|
744
|
-
this.networks.add(network);
|
|
745
|
-
this.log(`Created network after cleanup: ${network}`);
|
|
746
|
-
} catch (retryError) {
|
|
747
|
-
throw new Error(`Failed to create network ${network} even after cleanup: ${stderr}`);
|
|
748
|
-
}
|
|
749
|
-
} else {
|
|
750
|
-
// Real error - throw it
|
|
751
|
-
throw new Error(`Failed to create network ${network}: ${stderr}`);
|
|
752
|
-
}
|
|
903
|
+
if (/no such (container|object)/i.test(String((error as { stderr?: Buffer }).stderr))) break;
|
|
904
|
+
if (attempt >= this.maxCleanupRetries) throw error;
|
|
905
|
+
await new Promise(resolve => setTimeout(resolve, 1000 * attempt));
|
|
753
906
|
}
|
|
754
907
|
}
|
|
908
|
+
portMappings.forEach(port => this.usedPorts.delete(port));
|
|
909
|
+
this.containers.delete(id);
|
|
755
910
|
}
|
|
756
911
|
|
|
757
912
|
/**
|
|
758
|
-
*
|
|
759
|
-
* @private
|
|
913
|
+
* Create a labelled network or join an existing network under its current owner.
|
|
760
914
|
*/
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
915
|
+
async ensureNetwork(network: string): Promise<void> {
|
|
916
|
+
const create = () => {
|
|
917
|
+
const networkId = execFileSync(this.dockerPath, ['network', 'create', network,
|
|
918
|
+
'--label', 'test.cleanup=true', '--label', `test.manager=${this.managerId}`,
|
|
919
|
+
'--label', `test.host=${DockerTestManager.hostId}`], { stdio: 'pipe', encoding: 'utf8' }).trim();
|
|
920
|
+
this.networks.set(network, networkId);
|
|
921
|
+
};
|
|
922
|
+
try { create(); }
|
|
923
|
+
catch (error) {
|
|
924
|
+
const reason = String((error as { stderr?: string }).stderr ?? error);
|
|
925
|
+
if (reason.includes('already exists')) return;
|
|
926
|
+
if (!reason.includes('all predefined address pools have been fully subnetted')) throw error;
|
|
927
|
+
execFileSync(this.dockerPath, ['network', 'prune', '-f', '--filter', `label=test.manager=${this.managerId}`], { stdio: 'ignore' });
|
|
928
|
+
create();
|
|
764
929
|
}
|
|
930
|
+
}
|
|
765
931
|
|
|
932
|
+
/** Remove a network created by this manager, even if its name has since been reused. */
|
|
933
|
+
async removeNetwork(network: string): Promise<void> {
|
|
934
|
+
const id = this.networks.get(network);
|
|
935
|
+
if (!id) return;
|
|
766
936
|
try {
|
|
767
|
-
execFileSync(this.dockerPath, ['network', 'rm',
|
|
768
|
-
stdio: 'ignore',
|
|
769
|
-
});
|
|
770
|
-
this.networks.delete(network);
|
|
771
|
-
this.log(`Removed network: ${network}`);
|
|
937
|
+
execFileSync(this.dockerPath, ['network', 'rm', id], { stdio: ['ignore', 'ignore', 'pipe'] });
|
|
772
938
|
} catch (error) {
|
|
773
|
-
|
|
774
|
-
this.networks.delete(network); // Remove from tracking anyway
|
|
775
|
-
this.logError(`Failed to remove network ${network}`, error);
|
|
939
|
+
if (!/not found|no such network/i.test(String((error as { stderr?: Buffer }).stderr))) throw error;
|
|
776
940
|
}
|
|
941
|
+
this.networks.delete(network);
|
|
777
942
|
}
|
|
778
943
|
|
|
779
944
|
/**
|
|
780
945
|
* Wait for a container to be ready
|
|
781
946
|
* @private
|
|
782
947
|
*/
|
|
783
|
-
private async waitForContainer(name: string, options: ContainerOptions['waitFor']): Promise<void> {
|
|
948
|
+
private async waitForContainer(name: string, options: ContainerOptions['waitFor'], ports: Map<number, number>): Promise<void> {
|
|
784
949
|
const startTime = Date.now();
|
|
785
950
|
const timeout = options?.timeout || this.startupTimeout;
|
|
786
951
|
|
|
@@ -790,50 +955,22 @@ export class DockerTestManager extends EventEmitter {
|
|
|
790
955
|
try {
|
|
791
956
|
// First check if container is still running
|
|
792
957
|
const status = await this.getContainerStatus(name);
|
|
958
|
+
if (!status.state) throw new Error('Container status is unavailable');
|
|
793
959
|
if (!status.isRunning) {
|
|
794
960
|
throw new Error(
|
|
795
|
-
`Container exited with status '${status.state}' and exit code ${status.exitCode}
|
|
796
|
-
`Check container logs with 'docker logs ${name}'`
|
|
961
|
+
`Container ${name} exited with status '${status.state}' and exit code ${status.exitCode}`
|
|
797
962
|
);
|
|
798
963
|
}
|
|
799
964
|
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
}
|
|
806
|
-
|
|
807
|
-
if (status.health === 'unhealthy') {
|
|
808
|
-
throw new Error(
|
|
809
|
-
`Container ${name} health check failed. ` + `Check container logs with 'docker logs ${name}'`
|
|
810
|
-
);
|
|
811
|
-
}
|
|
812
|
-
|
|
813
|
-
// Health check is still starting
|
|
814
|
-
if (status.health === 'starting') {
|
|
815
|
-
this.log(`Waiting for health check on ${name}...`, { elapsed: Date.now() - startTime });
|
|
816
|
-
}
|
|
817
|
-
}
|
|
818
|
-
|
|
819
|
-
// Check if port is accessible
|
|
820
|
-
if (options?.port) {
|
|
821
|
-
const container = this.containers.get(name);
|
|
822
|
-
if (container) {
|
|
823
|
-
const hostPort = container.ports.get(options.port);
|
|
824
|
-
if (hostPort && (await this.isPortListening('127.0.0.1', hostPort))) {
|
|
825
|
-
this.log(`Port ${hostPort} is listening on ${name}`);
|
|
826
|
-
return;
|
|
827
|
-
}
|
|
828
|
-
}
|
|
829
|
-
}
|
|
830
|
-
|
|
831
|
-
// If no specific wait condition, just check container is running
|
|
832
|
-
if (!options?.healthcheck && !options?.port) {
|
|
833
|
-
this.log(`Container ${name} is running`);
|
|
834
|
-
return;
|
|
965
|
+
if (options?.healthcheck && status.health === 'unhealthy') {
|
|
966
|
+
const probes = execFileSync(this.dockerPath, ['inspect', '--format',
|
|
967
|
+
'{{with .State.Health}}{{.FailingStreak}} failed probes{{range .Log}}; exit={{.ExitCode}} at {{.End}}{{end}}{{end}}', name],
|
|
968
|
+
{ encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
|
969
|
+
throw new Error(`Container ${name} health check failed after ${Date.now() - startTime} ms (${probes})`);
|
|
835
970
|
}
|
|
836
|
-
|
|
971
|
+
const hostPort = options?.port && ports.get(options.port);
|
|
972
|
+
if ((!options?.healthcheck || status.health === 'healthy') &&
|
|
973
|
+
(!options?.port || (hostPort && await this.isPortListening('127.0.0.1', hostPort)))) return;
|
|
837
974
|
lastError = undefined;
|
|
838
975
|
} catch (error) {
|
|
839
976
|
lastError = error instanceof Error ? error : new Error(String(error));
|
|
@@ -907,61 +1044,18 @@ export class DockerTestManager extends EventEmitter {
|
|
|
907
1044
|
/**
|
|
908
1045
|
* Clean up all containers managed by this instance
|
|
909
1046
|
*/
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
return;
|
|
914
|
-
}
|
|
915
|
-
|
|
916
|
-
this.cleanupInProgress = true;
|
|
917
|
-
|
|
918
|
-
try {
|
|
919
|
-
this.log(`Starting cleanup of ${this.containers.size} containers`);
|
|
920
|
-
|
|
921
|
-
// Cleanup containers in parallel with Promise.allSettled to handle failures
|
|
922
|
-
const cleanupPromises = Array.from(this.containers.values()).map((container) =>
|
|
923
|
-
container
|
|
924
|
-
.cleanup()
|
|
925
|
-
.then(() => {
|
|
926
|
-
this.log(`Cleanup succeeded: ${container.name}`);
|
|
927
|
-
})
|
|
928
|
-
.catch((error) => {
|
|
929
|
-
this.logError(`Cleanup failed for ${container.name}`, error);
|
|
930
|
-
// Still continue cleaning up other containers
|
|
931
|
-
})
|
|
932
|
-
);
|
|
933
|
-
|
|
934
|
-
const results = await Promise.allSettled(cleanupPromises);
|
|
935
|
-
|
|
936
|
-
const failed = results.filter((r) => r.status === 'rejected').length;
|
|
937
|
-
if (failed > 0) {
|
|
938
|
-
this.log(`Cleanup completed with ${failed} failures`);
|
|
939
|
-
} else {
|
|
940
|
-
this.log('All containers cleaned up successfully');
|
|
941
|
-
}
|
|
942
|
-
|
|
943
|
-
// Cleanup networks
|
|
944
|
-
this.log(`Cleaning up ${this.networks.size} networks`);
|
|
945
|
-
const networkArray = Array.from(this.networks);
|
|
946
|
-
for (const network of networkArray) {
|
|
947
|
-
try {
|
|
948
|
-
this.log(`Removing network: ${network}`);
|
|
949
|
-
execFileSync(this.dockerPath, ['network', 'rm', network], { stdio: 'ignore' });
|
|
950
|
-
this.log(`Network removed: ${network}`);
|
|
951
|
-
} catch (error) {
|
|
952
|
-
this.logError(`Failed to remove network ${network}`, error);
|
|
953
|
-
// Continue with other networks
|
|
954
|
-
}
|
|
955
|
-
}
|
|
956
|
-
|
|
957
|
-
this.containers.clear();
|
|
958
|
-
this.usedPorts.clear();
|
|
959
|
-
this.networks.clear();
|
|
1047
|
+
cleanupAll(): Promise<void> {
|
|
1048
|
+
return this.activeCleanup ??= this.cleanupOwnedResources().finally(() => { this.activeCleanup = undefined; });
|
|
1049
|
+
}
|
|
960
1050
|
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
1051
|
+
private async cleanupOwnedResources(): Promise<void> {
|
|
1052
|
+
const results = await Promise.allSettled(Array.from(this.containers.values(), container => container.cleanup()));
|
|
1053
|
+
const errors = results.flatMap(result => result.status === 'rejected' ? [result.reason] : []);
|
|
1054
|
+
for (const network of this.networks.keys()) {
|
|
1055
|
+
try { await this.removeNetwork(network); }
|
|
1056
|
+
catch (error) { errors.push(error); }
|
|
964
1057
|
}
|
|
1058
|
+
if (errors.length) throw new AggregateError(errors, 'Some Docker test resources could not be removed');
|
|
965
1059
|
}
|
|
966
1060
|
|
|
967
1061
|
/**
|
|
@@ -1005,7 +1099,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
1005
1099
|
private cleanupSyncWindows(): void {
|
|
1006
1100
|
try {
|
|
1007
1101
|
// Get container IDs with test.cleanup label
|
|
1008
|
-
const containerIds = execFileSync(this.dockerPath, ['ps', '-a', '--filter',
|
|
1102
|
+
const containerIds = execFileSync(this.dockerPath, ['ps', '-a', '--filter', `label=test.manager=${this.managerId}`, '-q'], {
|
|
1009
1103
|
encoding: 'utf8',
|
|
1010
1104
|
stdio: ['pipe', 'pipe', 'ignore'],
|
|
1011
1105
|
}).trim();
|
|
@@ -1025,7 +1119,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
1025
1119
|
}
|
|
1026
1120
|
|
|
1027
1121
|
this.log(`Removing container: ${trimmedId}`);
|
|
1028
|
-
execFileSync(this.dockerPath, ['rm', '-f', trimmedId], { stdio: 'ignore' });
|
|
1122
|
+
execFileSync(this.dockerPath, ['rm', '-f', '-v', trimmedId], { stdio: 'ignore' });
|
|
1029
1123
|
this.log(`Container removed: ${trimmedId}`);
|
|
1030
1124
|
} catch (error) {
|
|
1031
1125
|
this.logError(`Failed to cleanup container ${id}`, error);
|
|
@@ -1038,7 +1132,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
1038
1132
|
try {
|
|
1039
1133
|
const networkIds = execFileSync(
|
|
1040
1134
|
this.dockerPath,
|
|
1041
|
-
['network', 'ls', '--filter',
|
|
1135
|
+
['network', 'ls', '--filter', `label=test.manager=${this.managerId}`, '-q'],
|
|
1042
1136
|
{
|
|
1043
1137
|
encoding: 'utf8',
|
|
1044
1138
|
stdio: ['pipe', 'pipe', 'ignore'],
|
|
@@ -1074,7 +1168,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
1074
1168
|
private cleanupSyncUnix(): void {
|
|
1075
1169
|
try {
|
|
1076
1170
|
// Remove containers
|
|
1077
|
-
const containerCmd = `"${this.dockerPath}" ps -a --filter "label=test.
|
|
1171
|
+
const containerCmd = `"${this.dockerPath}" ps -a --filter "label=test.manager=${this.managerId}" -q`;
|
|
1078
1172
|
try {
|
|
1079
1173
|
const ids = execSync(containerCmd, { encoding: 'utf8', stdio: ['pipe', 'pipe', 'ignore'] }).trim();
|
|
1080
1174
|
if (ids) {
|
|
@@ -1092,7 +1186,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
1092
1186
|
}
|
|
1093
1187
|
|
|
1094
1188
|
this.log(`Removing container: ${trimmedId}`);
|
|
1095
|
-
execFileSync(this.dockerPath, ['rm', '-f', trimmedId], { stdio: 'ignore' });
|
|
1189
|
+
execFileSync(this.dockerPath, ['rm', '-f', '-v', trimmedId], { stdio: 'ignore' });
|
|
1096
1190
|
this.log(`Container removed: ${trimmedId}`);
|
|
1097
1191
|
} catch (error) {
|
|
1098
1192
|
this.logError(`Failed to cleanup container ${id}`, error);
|
|
@@ -1105,7 +1199,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
1105
1199
|
|
|
1106
1200
|
// Remove test networks
|
|
1107
1201
|
try {
|
|
1108
|
-
const networkCmd = `"${this.dockerPath}" network ls --filter "label=test.
|
|
1202
|
+
const networkCmd = `"${this.dockerPath}" network ls --filter "label=test.manager=${this.managerId}" -q`;
|
|
1109
1203
|
const ids = execSync(networkCmd, { encoding: 'utf8', stdio: ['pipe', 'pipe', 'ignore'] }).trim();
|
|
1110
1204
|
if (ids) {
|
|
1111
1205
|
const networkIds = ids.split('\n').filter((id) => id.trim());
|
|
@@ -1127,7 +1221,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
1127
1221
|
|
|
1128
1222
|
// Remove test volumes
|
|
1129
1223
|
try {
|
|
1130
|
-
const volumeCmd = `"${this.dockerPath}" volume ls --filter "label=test.
|
|
1224
|
+
const volumeCmd = `"${this.dockerPath}" volume ls --filter "label=test.manager=${this.managerId}" -q`;
|
|
1131
1225
|
const ids = execSync(volumeCmd, { encoding: 'utf8', stdio: ['pipe', 'pipe', 'ignore'] }).trim();
|
|
1132
1226
|
if (ids) {
|
|
1133
1227
|
const volumeIds = ids.split('\n').filter((id) => id.trim());
|