@omnitron-dev/testing 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/async/events.js.map +1 -1
- package/dist/async/retry.js.map +1 -1
- package/dist/async/wait.js.map +1 -1
- package/dist/docker/database-manager.js.map +1 -1
- package/dist/docker/docker-test-manager.d.ts +8 -0
- package/dist/docker/docker-test-manager.js +194 -45
- package/dist/docker/docker-test-manager.js.map +1 -1
- package/dist/docker/redis-manager.js +24 -50
- package/dist/docker/redis-manager.js.map +1 -1
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/runtime/bun-adapter.d.ts +23 -1
- package/dist/runtime/bun-adapter.js +3 -0
- package/dist/runtime/bun-adapter.js.map +1 -1
- package/dist/runtime/deno-adapter.d.ts +72 -16
- package/dist/runtime/deno-adapter.js +79 -77
- package/dist/runtime/deno-adapter.js.map +1 -1
- package/dist/runtime/node-adapter.d.ts +14 -2
- package/dist/titan/database/database-testing.module.d.ts +1 -1
- package/dist/titan/database/database-testing.module.js.map +1 -1
- package/dist/titan/database/test-utilities.d.ts +1 -1
- package/dist/titan/database/test-utilities.js +1 -1
- package/dist/titan/database/test-utilities.js.map +1 -1
- package/dist/titan/index.d.ts +2 -2
- package/dist/titan/index.js +1 -1
- package/dist/titan/index.js.map +1 -1
- package/dist/titan/nexus/mock-provider.js.map +1 -1
- package/dist/titan/nexus/test-container.js.map +1 -1
- package/dist/titan/nexus/test-utilities.js.map +1 -1
- package/dist/titan/pm/mock-process-spawner.js.map +1 -1
- package/dist/titan/pm/mock-spawner.js.map +1 -1
- package/dist/titan/pm/test-process-manager.js.map +1 -1
- package/dist/titan/redis-test-utils.d.ts +6 -0
- package/dist/titan/redis-test-utils.js +29 -0
- package/dist/titan/redis-test-utils.js.map +1 -1
- package/dist/titan/test-application.js.map +1 -1
- package/dist/titan/test-fixtures.js.map +1 -1
- package/dist/titan/test-mocks.js.map +1 -1
- package/dist/titan/test-module.js.map +1 -1
- package/package.json +36 -29
- package/src/docker/a-port-the-network-could-reach.spec.ts +66 -0
- package/src/docker/docker-orphan-reaper.spec.ts +175 -0
- package/src/docker/docker-port-collision.spec.ts +97 -0
- package/src/docker/docker-test-manager.spec.ts +122 -0
- package/src/docker/docker-test-manager.ts +329 -20
- package/src/docker/docker-volume-cleanup.spec.ts +66 -0
- package/src/docker/redis-manager.ts +41 -58
- package/src/docker/types.ts +1 -0
- package/src/runtime/adapter-surface.spec.ts +83 -0
- package/src/runtime/bun-adapter.ts +21 -0
- package/src/runtime/deno-adapter.ts +149 -104
- package/src/titan/database/database-testing.module.ts +1 -1
- package/src/titan/database/test-utilities.ts +1 -1
- package/src/titan/index.ts +6 -1
- package/src/titan/nexus/test-utilities.ts +1 -0
- package/src/titan/pm/test-process-manager.ts +12 -2
- package/src/titan/redis-test-utils.ts +67 -0
- package/runtime/bun.d.ts +0 -2
- package/runtime/bun.js +0 -5
- package/runtime/bun.ts +0 -7
- package/runtime/deno.d.ts +0 -2
- package/runtime/deno.js +0 -4
- package/runtime/vitest.d.ts +0 -2
- package/runtime/vitest.js +0 -16
- package/runtime/vitest.ts +0 -7
|
@@ -15,10 +15,59 @@
|
|
|
15
15
|
|
|
16
16
|
import { execSync, execFileSync } from 'child_process';
|
|
17
17
|
import { randomBytes } from 'crypto';
|
|
18
|
+
import { hostname } from 'os';
|
|
18
19
|
import * as net from 'net';
|
|
19
20
|
import { EventEmitter } from 'events';
|
|
20
21
|
import type { DockerContainer, DockerContainerStatus, DockerTestManagerOptions, ContainerOptions } from './types.js';
|
|
21
22
|
|
|
23
|
+
/**
|
|
24
|
+
* `docker ps --format '{{.Labels}}'` renders labels as `k=v,k=v`. Values here
|
|
25
|
+
* are manager ids and hostnames, neither of which contains a comma.
|
|
26
|
+
*/
|
|
27
|
+
function parseDockerLabels(labels: string): Record<string, string> {
|
|
28
|
+
const out: Record<string, string> = {};
|
|
29
|
+
for (const pair of labels.split(',')) {
|
|
30
|
+
const eq = pair.indexOf('=');
|
|
31
|
+
if (eq > 0) out[pair.slice(0, eq)] = pair.slice(eq + 1);
|
|
32
|
+
}
|
|
33
|
+
return out;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** The pid the manager id starts with, or null when it is not one. */
|
|
37
|
+
function pidFromManagerId(managerId: string): number | null {
|
|
38
|
+
const pid = Number.parseInt(managerId.split('-')[0] ?? '', 10);
|
|
39
|
+
return Number.isInteger(pid) && pid > 0 ? pid : null;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Signal 0 asks the kernel whether the process exists without delivering
|
|
44
|
+
* anything. EPERM means it exists and belongs to another user, which is still
|
|
45
|
+
* alive; only ESRCH means gone.
|
|
46
|
+
*/
|
|
47
|
+
function isPidAlive(pid: number): boolean {
|
|
48
|
+
try {
|
|
49
|
+
process.kill(pid, 0);
|
|
50
|
+
return true;
|
|
51
|
+
} catch (error) {
|
|
52
|
+
return (error as NodeJS.ErrnoException).code === 'EPERM';
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* `docker ps --format '{{.CreatedAt}}'` prints `2026-09-06 01:07:22 +0300 MSK`
|
|
58
|
+
* — not ISO 8601, and the trailing abbreviation is ambiguous between zones.
|
|
59
|
+
* The numeric offset in front of it is not, so the abbreviation is dropped and
|
|
60
|
+
* the rest is rewritten into a form `Date.parse` is specified to accept.
|
|
61
|
+
* Returns null when the shape is anything else, and a null age is never old
|
|
62
|
+
* enough to reap.
|
|
63
|
+
*/
|
|
64
|
+
function parseDockerTimestamp(createdAt: string): number | null {
|
|
65
|
+
const m = /^(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2})(?:\.\d+)? ([+-]\d{2})(\d{2})/.exec(createdAt.trim());
|
|
66
|
+
if (!m) return null;
|
|
67
|
+
const parsed = Date.parse(`${m[1]}T${m[2]}${m[3]}:${m[4]}`);
|
|
68
|
+
return Number.isNaN(parsed) ? null : parsed;
|
|
69
|
+
}
|
|
70
|
+
|
|
22
71
|
/**
|
|
23
72
|
* Docker Test Manager
|
|
24
73
|
*
|
|
@@ -50,6 +99,14 @@ export class DockerTestManager extends EventEmitter {
|
|
|
50
99
|
private cleanupInProgress = false;
|
|
51
100
|
|
|
52
101
|
private dockerPath: string;
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* How many times a `docker run` may be re-attempted with freshly drawn `auto`
|
|
105
|
+
* host ports after docker refuses the published port. Small on purpose: this
|
|
106
|
+
* covers losing a race, not a machine with no free ports.
|
|
107
|
+
*/
|
|
108
|
+
private static readonly PORT_COLLISION_RETRIES = 3;
|
|
109
|
+
|
|
53
110
|
private basePort: number;
|
|
54
111
|
private maxRetries: number;
|
|
55
112
|
private startupTimeout: number;
|
|
@@ -59,6 +116,42 @@ export class DockerTestManager extends EventEmitter {
|
|
|
59
116
|
private verbose: boolean;
|
|
60
117
|
private defaultNetwork?: string;
|
|
61
118
|
|
|
119
|
+
/**
|
|
120
|
+
* Identifies the resources THIS manager created.
|
|
121
|
+
*
|
|
122
|
+
* Cleanup used to select by the shared `test.cleanup=true` label, i.e. every
|
|
123
|
+
* test container on the host. Vitest gives each spec file its own worker
|
|
124
|
+
* process, each with its own manager and its own `process.on('exit')` hook,
|
|
125
|
+
* so the first worker to finish stopped and removed containers the others
|
|
126
|
+
* were still using. `docker stop` sends SIGTERM and Redis exits 0 on SIGTERM
|
|
127
|
+
* — which is exactly the "Container exited with status 'exited' and exit code
|
|
128
|
+
* 0" that made titan-redis, titan-database and titan-notifications flaky
|
|
129
|
+
* under parallel runs.
|
|
130
|
+
*/
|
|
131
|
+
private readonly managerId = `${process.pid}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* The machine the pid in `managerId` belongs to. Tests that talk to a docker
|
|
135
|
+
* daemon on another host would otherwise read that host's pid table as their
|
|
136
|
+
* own. Containers created before this label existed carry no host, which is
|
|
137
|
+
* absence of information rather than a mismatch, so they fall through to the
|
|
138
|
+
* pid and age rules below.
|
|
139
|
+
*/
|
|
140
|
+
private static readonly hostId = hostname();
|
|
141
|
+
|
|
142
|
+
/** Orphan reaping is a property of the process, not of a manager instance. */
|
|
143
|
+
private static orphanReapDone = false;
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* How old an unclaimed container must be before it is reaped.
|
|
147
|
+
*
|
|
148
|
+
* Pids are reused. A container created seconds ago belongs to a run that is
|
|
149
|
+
* starting, whatever the pid table now says about the process that created
|
|
150
|
+
* it, and this is what makes the reaper safe to run while other test
|
|
151
|
+
* processes are working.
|
|
152
|
+
*/
|
|
153
|
+
private static readonly orphanGraceMs = 10 * 60 * 1000;
|
|
154
|
+
|
|
62
155
|
private constructor(options: DockerTestManagerOptions = {}) {
|
|
63
156
|
super();
|
|
64
157
|
this.dockerPath = options.dockerPath || this.findDockerPath();
|
|
@@ -91,6 +184,10 @@ export class DockerTestManager extends EventEmitter {
|
|
|
91
184
|
// This is safe because signal handlers should be idempotent and cleanup is managed by the singleton pattern
|
|
92
185
|
process.setMaxListeners(Math.max(20, process.getMaxListeners()));
|
|
93
186
|
|
|
187
|
+
// Runs before any container of this process exists, so a reaped port is
|
|
188
|
+
// free by the time this manager draws one.
|
|
189
|
+
this.reapOrphanedResources();
|
|
190
|
+
|
|
94
191
|
process.on('exit', () => this.cleanupSync());
|
|
95
192
|
process.on('SIGINT', () => {
|
|
96
193
|
this.log('SIGINT received - initiating graceful shutdown');
|
|
@@ -276,6 +373,139 @@ export class DockerTestManager extends EventEmitter {
|
|
|
276
373
|
}
|
|
277
374
|
}
|
|
278
375
|
|
|
376
|
+
/**
|
|
377
|
+
* Remove containers and networks left behind by test processes that no
|
|
378
|
+
* longer exist.
|
|
379
|
+
*
|
|
380
|
+
* `cleanupSync` and the signal handlers cover every ending the process gets
|
|
381
|
+
* to observe. SIGKILL is not one of them, and it is the ending vitest uses
|
|
382
|
+
* for a worker that overruns teardown — so a killed run leaves its
|
|
383
|
+
* containers running, holding their ports, their memory and their anonymous
|
|
384
|
+
* volumes, with nothing left on the machine that refers to them.
|
|
385
|
+
*
|
|
386
|
+
* A container is reaped only when BOTH hold:
|
|
387
|
+
*
|
|
388
|
+
* - the pid recorded in `test.manager` is not alive. Pids are reused, so
|
|
389
|
+
* this alone can name a live manager's container as an orphan;
|
|
390
|
+
* - it is older than `orphanGraceMs`. A container seconds old belongs to a
|
|
391
|
+
* run that is starting, whatever the pid table says.
|
|
392
|
+
*
|
|
393
|
+
* Each condition failing leaves a container behind for the next run to
|
|
394
|
+
* collect. Neither can remove a container that is in use.
|
|
395
|
+
*
|
|
396
|
+
* Errors are swallowed: this runs from a constructor, and a machine whose
|
|
397
|
+
* docker is unreachable should fail on the first container command with that
|
|
398
|
+
* command's message, not on construction.
|
|
399
|
+
*
|
|
400
|
+
* @private
|
|
401
|
+
*/
|
|
402
|
+
private reapOrphanedResources(): void {
|
|
403
|
+
if (DockerTestManager.orphanReapDone) return;
|
|
404
|
+
DockerTestManager.orphanReapDone = true;
|
|
405
|
+
|
|
406
|
+
try {
|
|
407
|
+
const listed = execFileSync(
|
|
408
|
+
this.dockerPath,
|
|
409
|
+
['ps', '-a', '--filter', 'label=test.cleanup=true', '--format', '{{.ID}}\t{{.CreatedAt}}\t{{.Labels}}'],
|
|
410
|
+
{ encoding: 'utf8', stdio: ['pipe', 'pipe', 'ignore'] }
|
|
411
|
+
).trim();
|
|
412
|
+
|
|
413
|
+
const orphans: string[] = [];
|
|
414
|
+
for (const line of listed ? listed.split('\n') : []) {
|
|
415
|
+
const [id, createdAt, labels] = line.split('\t');
|
|
416
|
+
if (!id || !createdAt || !labels) continue;
|
|
417
|
+
if (!this.isOrphaned(labels, createdAt)) continue;
|
|
418
|
+
orphans.push(id);
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
if (orphans.length > 0) {
|
|
422
|
+
this.log(`Reaping ${orphans.length} container(s) left by processes that no longer exist`);
|
|
423
|
+
// One call, not one per container: a stalled machine can hold dozens,
|
|
424
|
+
// and every worker of a parallel run pays this before its first test.
|
|
425
|
+
try {
|
|
426
|
+
execFileSync(this.dockerPath, ['rm', '-f', '-v', ...orphans], { stdio: 'ignore' });
|
|
427
|
+
} catch (error) {
|
|
428
|
+
// A concurrent worker reaping the same container makes `docker rm`
|
|
429
|
+
// exit non-zero for that id and continue with the rest. Both wanted
|
|
430
|
+
// it gone.
|
|
431
|
+
this.logError('Some orphaned containers could not be removed', error);
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
this.reapOrphanedNetworks();
|
|
436
|
+
} catch (error) {
|
|
437
|
+
this.logError('Could not scan for orphaned test containers', error);
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
/**
|
|
442
|
+
* The same rule for networks, minus the age condition: `docker network rm`
|
|
443
|
+
* refuses a network that still has an endpoint attached, so an in-use
|
|
444
|
+
* network is protected by docker itself rather than by a grace period. A
|
|
445
|
+
* leaked one is not free — the address pool it comes from is what
|
|
446
|
+
* `createNetwork` already has to recover from when it runs out.
|
|
447
|
+
*
|
|
448
|
+
* @private
|
|
449
|
+
*/
|
|
450
|
+
private reapOrphanedNetworks(): void {
|
|
451
|
+
try {
|
|
452
|
+
const listed = execFileSync(
|
|
453
|
+
this.dockerPath,
|
|
454
|
+
['network', 'ls', '--filter', 'label=test.cleanup=true', '--format', '{{.ID}}\t{{.Labels}}'],
|
|
455
|
+
{ encoding: 'utf8', stdio: ['pipe', 'pipe', 'ignore'] }
|
|
456
|
+
).trim();
|
|
457
|
+
|
|
458
|
+
const orphans: string[] = [];
|
|
459
|
+
for (const line of listed ? listed.split('\n') : []) {
|
|
460
|
+
const [id, labels] = line.split('\t');
|
|
461
|
+
if (!id || !labels) continue;
|
|
462
|
+
if (!this.isOrphaned(labels, null)) continue;
|
|
463
|
+
orphans.push(id);
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
if (orphans.length > 0) {
|
|
467
|
+
this.log(`Reaping ${orphans.length} network(s) left by processes that no longer exist`);
|
|
468
|
+
try {
|
|
469
|
+
execFileSync(this.dockerPath, ['network', 'rm', ...orphans], { stdio: 'ignore' });
|
|
470
|
+
} catch (error) {
|
|
471
|
+
// Refused for a network still in use, which is the answer we want.
|
|
472
|
+
this.logError('Some orphaned test networks could not be removed', error);
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
} catch (error) {
|
|
476
|
+
this.logError('Could not scan for orphaned test networks', error);
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
/**
|
|
481
|
+
* Whether a labelled resource belongs to a test process that is gone.
|
|
482
|
+
*
|
|
483
|
+
* @param createdAt - `{{.CreatedAt}}`, or null to skip the age condition.
|
|
484
|
+
* @private
|
|
485
|
+
*/
|
|
486
|
+
private isOrphaned(labels: string, createdAt: string | null): boolean {
|
|
487
|
+
const parsed = parseDockerLabels(labels);
|
|
488
|
+
|
|
489
|
+
const owner = parsed['test.manager'];
|
|
490
|
+
// No owner recorded: created by a manager older than the label, or by
|
|
491
|
+
// something else entirely. Either way this reaper cannot say whose it is.
|
|
492
|
+
if (!owner || owner === this.managerId) return false;
|
|
493
|
+
|
|
494
|
+
const host = parsed['test.host'];
|
|
495
|
+
if (host && host !== DockerTestManager.hostId) return false;
|
|
496
|
+
|
|
497
|
+
const pid = pidFromManagerId(owner);
|
|
498
|
+
if (pid === null || isPidAlive(pid)) return false;
|
|
499
|
+
|
|
500
|
+
if (createdAt !== null) {
|
|
501
|
+
const created = parseDockerTimestamp(createdAt);
|
|
502
|
+
if (created === null) return false;
|
|
503
|
+
if (Date.now() - created < DockerTestManager.orphanGraceMs) return false;
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
return true;
|
|
507
|
+
}
|
|
508
|
+
|
|
279
509
|
/**
|
|
280
510
|
* Find an available port for container mapping
|
|
281
511
|
* @private
|
|
@@ -345,7 +575,18 @@ export class DockerTestManager extends EventEmitter {
|
|
|
345
575
|
[
|
|
346
576
|
'inspect',
|
|
347
577
|
'--format',
|
|
348
|
-
|
|
578
|
+
// `.State.Health` is absent on a container with no HEALTHCHECK, and
|
|
579
|
+
// Go templates fail hard on a missing map key rather than rendering
|
|
580
|
+
// empty: `docker inspect` exited 1 with "map has no entry for key
|
|
581
|
+
// \"Health\"". stderr is ignored here and the catch below reports a
|
|
582
|
+
// default, so every such container was read as NOT RUNNING — and
|
|
583
|
+
// `waitForContainer` rejected with "Container exited with status
|
|
584
|
+
// 'undefined' and exit code undefined" for a container that was
|
|
585
|
+
// running perfectly. That made `waitFor` unusable with any image
|
|
586
|
+
// lacking a healthcheck, which is precisely the case
|
|
587
|
+
// `waitFor.healthcheck: false` exists to serve; callers worked around
|
|
588
|
+
// it by always declaring one.
|
|
589
|
+
'{{.State.Running}}\t{{.State.Status}}\t{{if .State.Health}}{{.State.Health.Status}}{{end}}\t{{.State.ExitCode}}',
|
|
349
590
|
name,
|
|
350
591
|
],
|
|
351
592
|
{
|
|
@@ -416,17 +657,26 @@ export class DockerTestManager extends EventEmitter {
|
|
|
416
657
|
|
|
417
658
|
// Remove any existing container with the same name to avoid conflicts
|
|
418
659
|
try {
|
|
419
|
-
execFileSync(this.dockerPath, ['rm', '-f', name], { stdio: 'ignore' });
|
|
660
|
+
execFileSync(this.dockerPath, ['rm', '-f', '-v', name], { stdio: 'ignore' });
|
|
420
661
|
this.log(`Removed existing container: ${name}`);
|
|
421
662
|
} catch {
|
|
422
663
|
// Container doesn't exist, which is fine
|
|
423
664
|
}
|
|
424
665
|
|
|
425
|
-
// Process port mappings
|
|
666
|
+
// Process port mappings.
|
|
667
|
+
//
|
|
668
|
+
// `auto` ports are re-drawn if docker refuses the run: findAvailablePort()
|
|
669
|
+
// binds, reads the port and releases it, so between that check and
|
|
670
|
+
// `docker run` anything else on the machine can take it — another suite in
|
|
671
|
+
// this repo, another session, a container someone restarted. The window is
|
|
672
|
+
// small and the machine is shared, which is exactly the combination that
|
|
673
|
+
// produces a red test blaming the code under test.
|
|
674
|
+
const autoPorts = new Set<number>();
|
|
426
675
|
const portMappings = new Map<number, number>();
|
|
427
676
|
if (options.ports) {
|
|
428
677
|
for (const [containerPort, hostPort] of Object.entries(options.ports)) {
|
|
429
678
|
const cPort = parseInt(containerPort);
|
|
679
|
+
if (hostPort === 'auto') autoPorts.add(cPort);
|
|
430
680
|
const hPort = hostPort === 'auto' ? await this.findAvailablePort() : hostPort;
|
|
431
681
|
portMappings.set(cPort, hPort);
|
|
432
682
|
}
|
|
@@ -435,10 +685,26 @@ export class DockerTestManager extends EventEmitter {
|
|
|
435
685
|
// Build docker run command
|
|
436
686
|
const dockerArgs: string[] = ['run', '-d', '--name', name];
|
|
437
687
|
|
|
438
|
-
// Add ports
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
688
|
+
// Add ports. The slice is remembered so a collision retry can rewrite just
|
|
689
|
+
// these entries instead of rebuilding the whole command.
|
|
690
|
+
const portArgsStart = dockerArgs.length;
|
|
691
|
+
const writePortArgs = () => {
|
|
692
|
+
const args: string[] = [];
|
|
693
|
+
portMappings.forEach((hostPort, containerPort) => {
|
|
694
|
+
// The loopback only: a bare `-p host:container` publishes on 0.0.0.0,
|
|
695
|
+
// past the host firewall, and a test database with its default
|
|
696
|
+
// credentials is then a database the LAN can log into
|
|
697
|
+
// (scripts/a-port-the-network-could-reach.mjs). `isPortAvailable`
|
|
698
|
+
// probes the same address.
|
|
699
|
+
args.push('-p', `127.0.0.1:${hostPort}:${containerPort}`);
|
|
700
|
+
});
|
|
701
|
+
return args;
|
|
702
|
+
};
|
|
703
|
+
dockerArgs.push(...writePortArgs());
|
|
704
|
+
const portArgsLength = dockerArgs.length - portArgsStart;
|
|
705
|
+
const rebuildPortArgs = () => {
|
|
706
|
+
dockerArgs.splice(portArgsStart, portArgsLength, ...writePortArgs());
|
|
707
|
+
};
|
|
442
708
|
|
|
443
709
|
// Add environment variables
|
|
444
710
|
const environment = options.environment || {};
|
|
@@ -450,6 +716,10 @@ export class DockerTestManager extends EventEmitter {
|
|
|
450
716
|
const labels = {
|
|
451
717
|
'test.id': id,
|
|
452
718
|
'test.cleanup': 'true',
|
|
719
|
+
// Ownership, so cleanup can select this manager's containers instead of
|
|
720
|
+
// every test container on the machine.
|
|
721
|
+
'test.manager': this.managerId,
|
|
722
|
+
'test.host': DockerTestManager.hostId,
|
|
453
723
|
...options.labels,
|
|
454
724
|
};
|
|
455
725
|
for (const [key, value] of Object.entries(labels)) {
|
|
@@ -521,6 +791,9 @@ export class DockerTestManager extends EventEmitter {
|
|
|
521
791
|
}
|
|
522
792
|
|
|
523
793
|
// Start container using execFileSync to properly handle arguments with spaces
|
|
794
|
+
// The loop only ever repeats for a host-port collision on an `auto` port;
|
|
795
|
+
// every other failure throws on the first pass, unchanged.
|
|
796
|
+
for (let portAttempt = 0; ; portAttempt++) {
|
|
524
797
|
try {
|
|
525
798
|
this.log(`Starting container: ${name}`, { image: options.image });
|
|
526
799
|
const result = execFileSync(this.dockerPath, dockerArgs, {
|
|
@@ -530,6 +803,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
530
803
|
if (this.verbose && result) {
|
|
531
804
|
console.log(result);
|
|
532
805
|
}
|
|
806
|
+
break;
|
|
533
807
|
} catch (error) {
|
|
534
808
|
// Clean up allocated ports before throwing
|
|
535
809
|
portMappings.forEach((port) => {
|
|
@@ -555,6 +829,35 @@ export class DockerTestManager extends EventEmitter {
|
|
|
555
829
|
);
|
|
556
830
|
}
|
|
557
831
|
|
|
832
|
+
// Somebody took the host port between findAvailablePort() and here.
|
|
833
|
+
// Re-draw the `auto` ones and try again; explicit host ports are the
|
|
834
|
+
// caller's contract and are never silently changed.
|
|
835
|
+
const portTaken =
|
|
836
|
+
stderr.includes('port is already allocated') ||
|
|
837
|
+
stderr.includes('address already in use') ||
|
|
838
|
+
stderr.includes('Bind for ');
|
|
839
|
+
if (portTaken && autoPorts.size > 0 && portAttempt < DockerTestManager.PORT_COLLISION_RETRIES) {
|
|
840
|
+
// A run that fails on the port bind still LEAVES the container behind
|
|
841
|
+
// in `created` state, so re-running the same `--name` collides with
|
|
842
|
+
// itself. Remove it before drawing new ports — otherwise the retry
|
|
843
|
+
// reports a name conflict and the port collision it was fixing never
|
|
844
|
+
// appears in the error at all.
|
|
845
|
+
try {
|
|
846
|
+
execFileSync(this.dockerPath, ['rm', '-f', '-v', name], { stdio: 'ignore' });
|
|
847
|
+
} catch {
|
|
848
|
+
// Nothing to remove.
|
|
849
|
+
}
|
|
850
|
+
for (const cPort of autoPorts) {
|
|
851
|
+
const hPort = await this.findAvailablePort();
|
|
852
|
+
portMappings.set(cPort, hPort);
|
|
853
|
+
}
|
|
854
|
+
rebuildPortArgs();
|
|
855
|
+
this.log(`Host port collision for ${name}, retrying with fresh ports`, {
|
|
856
|
+
attempt: portAttempt + 1,
|
|
857
|
+
});
|
|
858
|
+
continue;
|
|
859
|
+
}
|
|
860
|
+
|
|
558
861
|
throw new Error(
|
|
559
862
|
`Failed to start container ${name}:\n` +
|
|
560
863
|
`Command: docker ${dockerArgs.join(' ')}\n` +
|
|
@@ -565,6 +868,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
565
868
|
{ cause: error }
|
|
566
869
|
);
|
|
567
870
|
}
|
|
871
|
+
}
|
|
568
872
|
|
|
569
873
|
// Wait for container to be ready
|
|
570
874
|
if (options.waitFor) {
|
|
@@ -576,7 +880,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
576
880
|
this.logError(`Container ${name} failed to become ready, cleaning up`, error);
|
|
577
881
|
try {
|
|
578
882
|
execFileSync(this.dockerPath, ['stop', '-t', '5', name], { stdio: 'ignore' });
|
|
579
|
-
execFileSync(this.dockerPath, ['rm', '-f', name], { stdio: 'ignore' });
|
|
883
|
+
execFileSync(this.dockerPath, ['rm', '-f', '-v', name], { stdio: 'ignore' });
|
|
580
884
|
} catch {
|
|
581
885
|
// Ignore cleanup errors during rollback
|
|
582
886
|
}
|
|
@@ -673,7 +977,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
673
977
|
// Remove container
|
|
674
978
|
try {
|
|
675
979
|
this.log(`Removing container ${name}`, { attempt });
|
|
676
|
-
execFileSync(this.dockerPath, ['rm', '-f', name], { stdio: 'ignore' });
|
|
980
|
+
execFileSync(this.dockerPath, ['rm', '-f', '-v', name], { stdio: 'ignore' });
|
|
677
981
|
this.log(`Container removed: ${name}`, { attempt });
|
|
678
982
|
lastError = null; // Clear error on success
|
|
679
983
|
break;
|
|
@@ -719,7 +1023,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
719
1023
|
private async ensureNetwork(network: string): Promise<void> {
|
|
720
1024
|
if (!this.networks.has(network)) {
|
|
721
1025
|
try {
|
|
722
|
-
execFileSync(this.dockerPath, ['network', 'create', network, '--label', 'test.cleanup=true'], {
|
|
1026
|
+
execFileSync(this.dockerPath, ['network', 'create', network, '--label', 'test.cleanup=true', '--label', `test.manager=${this.managerId}`, '--label', `test.host=${DockerTestManager.hostId}`], {
|
|
723
1027
|
stdio: 'pipe',
|
|
724
1028
|
encoding: 'utf8',
|
|
725
1029
|
});
|
|
@@ -736,13 +1040,18 @@ export class DockerTestManager extends EventEmitter {
|
|
|
736
1040
|
// Docker network pool exhausted - try to clean up unused networks
|
|
737
1041
|
this.log('Docker network pool exhausted, attempting cleanup...');
|
|
738
1042
|
try {
|
|
739
|
-
// Prune unused networks
|
|
740
|
-
|
|
1043
|
+
// Prune unused networks THAT WE CREATED. A bare `network prune -f`
|
|
1044
|
+
// is host-wide: it removes every unused network on the machine,
|
|
1045
|
+
// including ones belonging to compose stacks and other projects
|
|
1046
|
+
// that happen to have no running container at that instant. Every
|
|
1047
|
+
// other operation in this manager is scoped by the
|
|
1048
|
+
// `test.cleanup=true` label; this one was not.
|
|
1049
|
+
execFileSync(this.dockerPath, ['network', 'prune', '-f', '--filter', `label=test.manager=${this.managerId}`], {
|
|
741
1050
|
stdio: 'ignore',
|
|
742
1051
|
});
|
|
743
1052
|
this.log('Cleaned up unused networks, retrying network creation...');
|
|
744
1053
|
// Retry network creation after cleanup
|
|
745
|
-
execFileSync(this.dockerPath, ['network', 'create', network, '--label', 'test.cleanup=true'], {
|
|
1054
|
+
execFileSync(this.dockerPath, ['network', 'create', network, '--label', 'test.cleanup=true', '--label', `test.manager=${this.managerId}`, '--label', `test.host=${DockerTestManager.hostId}`], {
|
|
746
1055
|
stdio: 'pipe',
|
|
747
1056
|
encoding: 'utf8',
|
|
748
1057
|
});
|
|
@@ -1020,7 +1329,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
1020
1329
|
private cleanupSyncWindows(): void {
|
|
1021
1330
|
try {
|
|
1022
1331
|
// Get container IDs with test.cleanup label
|
|
1023
|
-
const containerIds = execFileSync(this.dockerPath, ['ps', '-a', '--filter',
|
|
1332
|
+
const containerIds = execFileSync(this.dockerPath, ['ps', '-a', '--filter', `label=test.manager=${this.managerId}`, '-q'], {
|
|
1024
1333
|
encoding: 'utf8',
|
|
1025
1334
|
stdio: ['pipe', 'pipe', 'ignore'],
|
|
1026
1335
|
}).trim();
|
|
@@ -1040,7 +1349,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
1040
1349
|
}
|
|
1041
1350
|
|
|
1042
1351
|
this.log(`Removing container: ${trimmedId}`);
|
|
1043
|
-
execFileSync(this.dockerPath, ['rm', '-f', trimmedId], { stdio: 'ignore' });
|
|
1352
|
+
execFileSync(this.dockerPath, ['rm', '-f', '-v', trimmedId], { stdio: 'ignore' });
|
|
1044
1353
|
this.log(`Container removed: ${trimmedId}`);
|
|
1045
1354
|
} catch (error) {
|
|
1046
1355
|
this.logError(`Failed to cleanup container ${id}`, error);
|
|
@@ -1053,7 +1362,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
1053
1362
|
try {
|
|
1054
1363
|
const networkIds = execFileSync(
|
|
1055
1364
|
this.dockerPath,
|
|
1056
|
-
['network', 'ls', '--filter',
|
|
1365
|
+
['network', 'ls', '--filter', `label=test.manager=${this.managerId}`, '-q'],
|
|
1057
1366
|
{
|
|
1058
1367
|
encoding: 'utf8',
|
|
1059
1368
|
stdio: ['pipe', 'pipe', 'ignore'],
|
|
@@ -1089,7 +1398,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
1089
1398
|
private cleanupSyncUnix(): void {
|
|
1090
1399
|
try {
|
|
1091
1400
|
// Remove containers
|
|
1092
|
-
const containerCmd = `"${this.dockerPath}" ps -a --filter "label=test.
|
|
1401
|
+
const containerCmd = `"${this.dockerPath}" ps -a --filter "label=test.manager=${this.managerId}" -q`;
|
|
1093
1402
|
try {
|
|
1094
1403
|
const ids = execSync(containerCmd, { encoding: 'utf8', stdio: ['pipe', 'pipe', 'ignore'] }).trim();
|
|
1095
1404
|
if (ids) {
|
|
@@ -1107,7 +1416,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
1107
1416
|
}
|
|
1108
1417
|
|
|
1109
1418
|
this.log(`Removing container: ${trimmedId}`);
|
|
1110
|
-
execFileSync(this.dockerPath, ['rm', '-f', trimmedId], { stdio: 'ignore' });
|
|
1419
|
+
execFileSync(this.dockerPath, ['rm', '-f', '-v', trimmedId], { stdio: 'ignore' });
|
|
1111
1420
|
this.log(`Container removed: ${trimmedId}`);
|
|
1112
1421
|
} catch (error) {
|
|
1113
1422
|
this.logError(`Failed to cleanup container ${id}`, error);
|
|
@@ -1120,7 +1429,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
1120
1429
|
|
|
1121
1430
|
// Remove test networks
|
|
1122
1431
|
try {
|
|
1123
|
-
const networkCmd = `"${this.dockerPath}" network ls --filter "label=test.
|
|
1432
|
+
const networkCmd = `"${this.dockerPath}" network ls --filter "label=test.manager=${this.managerId}" -q`;
|
|
1124
1433
|
const ids = execSync(networkCmd, { encoding: 'utf8', stdio: ['pipe', 'pipe', 'ignore'] }).trim();
|
|
1125
1434
|
if (ids) {
|
|
1126
1435
|
const networkIds = ids.split('\n').filter((id) => id.trim());
|
|
@@ -1142,7 +1451,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
1142
1451
|
|
|
1143
1452
|
// Remove test volumes
|
|
1144
1453
|
try {
|
|
1145
|
-
const volumeCmd = `"${this.dockerPath}" volume ls --filter "label=test.
|
|
1454
|
+
const volumeCmd = `"${this.dockerPath}" volume ls --filter "label=test.manager=${this.managerId}" -q`;
|
|
1146
1455
|
const ids = execSync(volumeCmd, { encoding: 'utf8', stdio: ['pipe', 'pipe', 'ignore'] }).trim();
|
|
1147
1456
|
if (ids) {
|
|
1148
1457
|
const volumeIds = ids.split('\n').filter((id) => id.trim());
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Removing a test container must remove the anonymous volumes it created.
|
|
3
|
+
*
|
|
4
|
+
* Every `docker rm` here was `rm -f <name>` with no `-v`. The database and
|
|
5
|
+
* cache images this manager starts all declare `VOLUME` in their Dockerfiles —
|
|
6
|
+
* postgres `/var/lib/postgresql/data`, mysql `/var/lib/mysql`, redis `/data` —
|
|
7
|
+
* so each container came with an anonymous volume, and `rm` without `-v` leaves
|
|
8
|
+
* it behind with nothing pointing at it. One per container, forever.
|
|
9
|
+
*
|
|
10
|
+
* Measured on this machine before the fix: 4940 dangling volumes out of 5012,
|
|
11
|
+
* 188 GB in `Local Volumes` with 72 in use. A leak that fired sometimes would
|
|
12
|
+
* leave an intermediate ratio; 98.6% is a missing flag. It filled the Docker
|
|
13
|
+
* VM's disk and took the user's Postgres down with it.
|
|
14
|
+
*
|
|
15
|
+
* `-v` removes only the container's own ANONYMOUS volumes. Named volumes are
|
|
16
|
+
* untouched, which is why this is safe to do unconditionally: it removes
|
|
17
|
+
* exactly what the container created.
|
|
18
|
+
*/
|
|
19
|
+
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
20
|
+
|
|
21
|
+
const execFileSyncMock = vi.fn(() => '');
|
|
22
|
+
|
|
23
|
+
vi.mock('node:child_process', async (importOriginal) => {
|
|
24
|
+
const actual = await importOriginal<typeof import('node:child_process')>();
|
|
25
|
+
return { ...actual, execFileSync: (...args: unknown[]) => execFileSyncMock(...args) };
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
const { DockerTestManager } = await import('./docker-test-manager.js');
|
|
29
|
+
|
|
30
|
+
describe('DockerTestManager container removal', () => {
|
|
31
|
+
beforeEach(() => {
|
|
32
|
+
execFileSyncMock.mockReset();
|
|
33
|
+
execFileSyncMock.mockReturnValue('');
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
it('passes -v everywhere it removes a container', async () => {
|
|
37
|
+
const manager = new DockerTestManager();
|
|
38
|
+
|
|
39
|
+
// Any path that removes a container will do; createContainer clears a stale
|
|
40
|
+
// one by name before starting, which is the first of the six call sites.
|
|
41
|
+
await manager
|
|
42
|
+
.createContainer({ name: 'volume-flag-probe', image: 'redis:7-alpine' })
|
|
43
|
+
.catch(() => undefined);
|
|
44
|
+
|
|
45
|
+
const removals = execFileSyncMock.mock.calls
|
|
46
|
+
.map((c) => c[1] as string[])
|
|
47
|
+
.filter((args) => Array.isArray(args) && args[0] === 'rm');
|
|
48
|
+
|
|
49
|
+
expect(removals.length, 'no container removal was issued').toBeGreaterThan(0);
|
|
50
|
+
for (const args of removals) {
|
|
51
|
+
expect(args, `container removal without -v: ${args.join(' ')}`).toContain('-v');
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it('never passes -v to a network removal', () => {
|
|
56
|
+
// `docker network rm` has no such flag; the blanket edit must not have
|
|
57
|
+
// reached it.
|
|
58
|
+
const source = readFileSync(
|
|
59
|
+
new URL('./docker-test-manager.ts', import.meta.url),
|
|
60
|
+
'utf-8'
|
|
61
|
+
);
|
|
62
|
+
expect(source).not.toMatch(/\['network', 'rm', '-v'/);
|
|
63
|
+
});
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
import { readFileSync } from 'node:fs';
|