@omnitron-dev/testing 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -0
- package/dist/docker/database-manager.js +4 -6
- package/dist/docker/database-manager.js.map +1 -1
- package/dist/docker/docker-test-manager.d.ts +4 -3
- package/dist/docker/docker-test-manager.js +126 -268
- package/dist/docker/docker-test-manager.js.map +1 -1
- package/dist/docker/redis-manager.js +38 -36
- package/dist/docker/redis-manager.js.map +1 -1
- package/dist/docker/types.d.ts +2 -2
- package/dist/env.js +6 -3
- package/dist/env.js.map +1 -1
- package/dist/runtime/deno-adapter.js +1 -1
- package/dist/runtime/deno-adapter.js.map +1 -1
- package/dist/titan/redis-test-utils.js +80 -54
- package/dist/titan/redis-test-utils.js.map +1 -1
- package/package.json +33 -31
- package/src/docker/a-port-the-network-could-reach.spec.ts +1 -1
- package/src/docker/database-manager.ts +4 -6
- package/src/docker/docker-test-manager.ts +113 -353
- package/src/docker/docker-volume-cleanup.spec.ts +62 -15
- package/src/docker/redis-cluster-creation.spec.ts +52 -0
- package/src/docker/redis-manager.ts +33 -49
- package/src/docker/types.ts +4 -4
- package/src/env.spec.ts +20 -0
- package/src/env.ts +5 -3
- package/src/index.ts +1 -1
- package/src/runtime/deno-adapter.ts +1 -1
- package/src/titan/redis-test-utils.test.ts +102 -0
- package/src/titan/redis-test-utils.ts +86 -71
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* - Network management
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
|
-
import { execSync, execFileSync } from 'child_process';
|
|
16
|
+
import { execSync, execFileSync, execFile } from 'child_process';
|
|
17
17
|
import { randomBytes } from 'crypto';
|
|
18
18
|
import { hostname } from 'os';
|
|
19
19
|
import * as net from 'net';
|
|
@@ -33,6 +33,15 @@ function parseDockerLabels(labels: string): Record<string, string> {
|
|
|
33
33
|
return out;
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
+
/** Docker's CLI evaluates --health-cmd in a shell. Preserve literal CMD arguments. */
|
|
37
|
+
function dockerHealthCommand(test: string | string[]): string {
|
|
38
|
+
if (typeof test === 'string') return test;
|
|
39
|
+
const [mode, ...args] = test;
|
|
40
|
+
if (mode === 'CMD-SHELL') return args.join(' ');
|
|
41
|
+
const argv = mode === 'CMD' ? args : test;
|
|
42
|
+
return argv.map(arg => "'" + arg.replaceAll("'", "'\\''") + "'").join(' ');
|
|
43
|
+
}
|
|
44
|
+
|
|
36
45
|
/** The pid the manager id starts with, or null when it is not one. */
|
|
37
46
|
function pidFromManagerId(managerId: string): number | null {
|
|
38
47
|
const pid = Number.parseInt(managerId.split('-')[0] ?? '', 10);
|
|
@@ -95,8 +104,8 @@ export class DockerTestManager extends EventEmitter {
|
|
|
95
104
|
private static instance: DockerTestManager;
|
|
96
105
|
private containers: Map<string, DockerContainer> = new Map();
|
|
97
106
|
private usedPorts: Set<number> = new Set();
|
|
98
|
-
private networks
|
|
99
|
-
private
|
|
107
|
+
private networks = new Map<string, string>();
|
|
108
|
+
private activeCleanup?: Promise<void>;
|
|
100
109
|
|
|
101
110
|
private dockerPath: string;
|
|
102
111
|
|
|
@@ -194,7 +203,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
194
203
|
this.cleanupAllAsync()
|
|
195
204
|
.then(() => {
|
|
196
205
|
this.log('Cleanup completed, exiting');
|
|
197
|
-
process.exit(
|
|
206
|
+
process.exit(130);
|
|
198
207
|
})
|
|
199
208
|
.catch((error) => {
|
|
200
209
|
this.logError('Cleanup failed during SIGINT', error);
|
|
@@ -206,7 +215,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
206
215
|
this.cleanupAllAsync()
|
|
207
216
|
.then(() => {
|
|
208
217
|
this.log('Cleanup completed, exiting');
|
|
209
|
-
process.exit(
|
|
218
|
+
process.exit(143);
|
|
210
219
|
})
|
|
211
220
|
.catch((error) => {
|
|
212
221
|
this.logError('Cleanup failed during SIGTERM', error);
|
|
@@ -655,14 +664,6 @@ export class DockerTestManager extends EventEmitter {
|
|
|
655
664
|
const name = options.name || `container-${id}`;
|
|
656
665
|
const host = '127.0.0.1';
|
|
657
666
|
|
|
658
|
-
// Remove any existing container with the same name to avoid conflicts
|
|
659
|
-
try {
|
|
660
|
-
execFileSync(this.dockerPath, ['rm', '-f', '-v', name], { stdio: 'ignore' });
|
|
661
|
-
this.log(`Removed existing container: ${name}`);
|
|
662
|
-
} catch {
|
|
663
|
-
// Container doesn't exist, which is fine
|
|
664
|
-
}
|
|
665
|
-
|
|
666
667
|
// Process port mappings.
|
|
667
668
|
//
|
|
668
669
|
// `auto` ports are re-drawn if docker refuses the run: findAvailablePort()
|
|
@@ -714,13 +715,13 @@ export class DockerTestManager extends EventEmitter {
|
|
|
714
715
|
|
|
715
716
|
// Add labels
|
|
716
717
|
const labels = {
|
|
718
|
+
...options.labels,
|
|
717
719
|
'test.id': id,
|
|
718
720
|
'test.cleanup': 'true',
|
|
719
721
|
// Ownership, so cleanup can select this manager's containers instead of
|
|
720
722
|
// every test container on the machine.
|
|
721
723
|
'test.manager': this.managerId,
|
|
722
724
|
'test.host': DockerTestManager.hostId,
|
|
723
|
-
...options.labels,
|
|
724
725
|
};
|
|
725
726
|
for (const [key, value] of Object.entries(labels)) {
|
|
726
727
|
dockerArgs.push('--label', `${key}=${value}`);
|
|
@@ -743,29 +744,7 @@ export class DockerTestManager extends EventEmitter {
|
|
|
743
744
|
|
|
744
745
|
// Add healthcheck if provided
|
|
745
746
|
if (options.healthcheck) {
|
|
746
|
-
|
|
747
|
-
// The healthcheck.test array format from Docker Compose needs to be converted:
|
|
748
|
-
// - ['CMD', 'arg1', 'arg2'] -> execute command directly (no shell)
|
|
749
|
-
// - ['CMD-SHELL', 'command'] -> execute via shell
|
|
750
|
-
let healthCmd: string;
|
|
751
|
-
if (Array.isArray(options.healthcheck.test)) {
|
|
752
|
-
if (options.healthcheck.test[0] === 'CMD-SHELL') {
|
|
753
|
-
// CMD-SHELL format: join everything after the first element
|
|
754
|
-
healthCmd = options.healthcheck.test.slice(1).join(' ');
|
|
755
|
-
} else if (options.healthcheck.test[0] === 'CMD') {
|
|
756
|
-
// CMD format: join command parts without shell
|
|
757
|
-
// Docker CLI expects this as a single string that will be word-split
|
|
758
|
-
// For example: ['CMD', 'redis-cli', 'ping'] -> 'redis-cli ping'
|
|
759
|
-
healthCmd = options.healthcheck.test.slice(1).join(' ');
|
|
760
|
-
} else {
|
|
761
|
-
// No format specifier, join all parts
|
|
762
|
-
healthCmd = options.healthcheck.test.join(' ');
|
|
763
|
-
}
|
|
764
|
-
} else {
|
|
765
|
-
// String format
|
|
766
|
-
healthCmd = options.healthcheck.test;
|
|
767
|
-
}
|
|
768
|
-
dockerArgs.push('--health-cmd', healthCmd);
|
|
747
|
+
dockerArgs.push('--health-cmd', dockerHealthCommand(options.healthcheck.test));
|
|
769
748
|
if (options.healthcheck.interval) {
|
|
770
749
|
dockerArgs.push('--health-interval', options.healthcheck.interval);
|
|
771
750
|
}
|
|
@@ -783,104 +762,59 @@ export class DockerTestManager extends EventEmitter {
|
|
|
783
762
|
// Add image and command
|
|
784
763
|
dockerArgs.push(options.image);
|
|
785
764
|
if (options.command) {
|
|
786
|
-
dockerArgs.push(
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
if (this.verbose) {
|
|
790
|
-
console.log(`Starting container ${name}: docker ${dockerArgs.join(' ')}`);
|
|
765
|
+
if (Array.isArray(options.command)) dockerArgs.push(...options.command);
|
|
766
|
+
else dockerArgs.push('sh', '-c', options.command);
|
|
791
767
|
}
|
|
792
768
|
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
// every other failure throws on the first pass, unchanged.
|
|
796
|
-
for (let portAttempt = 0; ; portAttempt++) {
|
|
797
|
-
try {
|
|
769
|
+
let containerId: string;
|
|
770
|
+
for (let attempt = 0; ; attempt++) {
|
|
798
771
|
this.log(`Starting container: ${name}`, { image: options.image });
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
772
|
+
// A synchronous Docker start blocks every peer's readiness timer while
|
|
773
|
+
// Docker creates the next container. Keep cluster startup concurrent.
|
|
774
|
+
const result = await new Promise<{ status: number | null; stdout: string; stderr: string; error?: { code?: string | number | null } }>(resolve => {
|
|
775
|
+
execFile(this.dockerPath, dockerArgs, { encoding: 'utf8', timeout: this.startupTimeout, maxBuffer: 2 * 1024 * 1024 },
|
|
776
|
+
(error, stdout, stderr) => resolve({ status: error ? (typeof error.code === 'number' ? error.code : null) : 0,
|
|
777
|
+
stdout, stderr, ...(error && typeof error.code !== 'number' ? { error } : {}) }));
|
|
802
778
|
});
|
|
803
|
-
|
|
804
|
-
|
|
779
|
+
containerId = (result.stdout ?? '').trim();
|
|
780
|
+
if (result.status === 0 && /^[a-f0-9]{64}$/.test(containerId)) break;
|
|
781
|
+
portMappings.forEach(port => this.usedPorts.delete(port));
|
|
782
|
+
let diagnostic = (result.stderr ?? '').trim();
|
|
783
|
+
for (const value of Object.values(environment)) {
|
|
784
|
+
if (value) diagnostic = diagnostic.replaceAll(value, '[redacted]');
|
|
805
785
|
}
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
// Clean up allocated ports before throwing
|
|
809
|
-
portMappings.forEach((port) => {
|
|
810
|
-
this.usedPorts.delete(port);
|
|
811
|
-
});
|
|
812
|
-
const execError = error as {
|
|
813
|
-
stderr?: Buffer | string;
|
|
814
|
-
stdout?: Buffer | string;
|
|
815
|
-
message?: string;
|
|
816
|
-
status?: number;
|
|
817
|
-
};
|
|
818
|
-
const stderr = execError.stderr ? String(execError.stderr).trim() : '';
|
|
819
|
-
const stdout = execError.stdout ? String(execError.stdout).trim() : '';
|
|
820
|
-
const message = execError.message || String(error);
|
|
821
|
-
|
|
822
|
-
// Check if container already exists
|
|
823
|
-
if (stderr.includes('already in use') || stderr.includes('name is already') || stderr.includes('Conflict')) {
|
|
824
|
-
throw new Error(
|
|
825
|
-
`Container name conflict: ${name} already exists.\n` +
|
|
826
|
-
`Please ensure previous test containers are cleaned up.\n` +
|
|
827
|
-
`Stderr: ${stderr}`,
|
|
828
|
-
{ cause: error }
|
|
829
|
-
);
|
|
786
|
+
if (/container name .*already in use|name is already in use/i.test(diagnostic)) {
|
|
787
|
+
throw new Error(`Container name conflict: ${name} already exists`);
|
|
830
788
|
}
|
|
831
|
-
|
|
832
|
-
//
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
stderr.includes('Bind for ');
|
|
839
|
-
if (portTaken && autoPorts.size > 0 && portAttempt < DockerTestManager.PORT_COLLISION_RETRIES) {
|
|
840
|
-
// A run that fails on the port bind still LEAVES the container behind
|
|
841
|
-
// in `created` state, so re-running the same `--name` collides with
|
|
842
|
-
// itself. Remove it before drawing new ports — otherwise the retry
|
|
843
|
-
// reports a name conflict and the port collision it was fixing never
|
|
844
|
-
// appears in the error at all.
|
|
845
|
-
try {
|
|
846
|
-
execFileSync(this.dockerPath, ['rm', '-f', '-v', name], { stdio: 'ignore' });
|
|
847
|
-
} catch {
|
|
848
|
-
// Nothing to remove.
|
|
849
|
-
}
|
|
850
|
-
for (const cPort of autoPorts) {
|
|
851
|
-
const hPort = await this.findAvailablePort();
|
|
852
|
-
portMappings.set(cPort, hPort);
|
|
853
|
-
}
|
|
854
|
-
rebuildPortArgs();
|
|
855
|
-
this.log(`Host port collision for ${name}, retrying with fresh ports`, {
|
|
856
|
-
attempt: portAttempt + 1,
|
|
857
|
-
});
|
|
858
|
-
continue;
|
|
789
|
+
// A failed bind can leave a container in "created". Resolve this
|
|
790
|
+
// attempt's labels and remove only the Docker ID they identify.
|
|
791
|
+
const residue = execFileSync(this.dockerPath, ['ps', '-aq', '--no-trunc',
|
|
792
|
+
'--filter', `label=test.id=${id}`, '--filter', `label=test.manager=${this.managerId}`],
|
|
793
|
+
{ encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
|
794
|
+
if (/^[a-f0-9]{64}$/.test(residue)) {
|
|
795
|
+
execFileSync(this.dockerPath, ['rm', '-f', '-v', residue], { stdio: 'ignore' });
|
|
859
796
|
}
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
{ cause: error }
|
|
869
|
-
);
|
|
870
|
-
}
|
|
797
|
+
if (result.error) throw new Error(`Could not execute Docker for ${name}: ${result.error.code ?? 'unknown'}`);
|
|
798
|
+
if (!/port is already allocated|address already in use|Bind for /.test(diagnostic)
|
|
799
|
+
|| !autoPorts.size || attempt >= DockerTestManager.PORT_COLLISION_RETRIES) {
|
|
800
|
+
throw new Error(`Failed to start container ${name} (${options.image}), exit ${result.status ?? 'unknown'}: ${diagnostic.slice(0, 2000) || 'Docker did not confirm creation'}`);
|
|
801
|
+
}
|
|
802
|
+
for (const port of autoPorts) portMappings.set(port, await this.findAvailablePort());
|
|
803
|
+
rebuildPortArgs();
|
|
804
|
+
this.log(`Host port collision for ${name}, retrying with fresh ports`, { attempt: attempt + 1 });
|
|
871
805
|
}
|
|
872
806
|
|
|
873
807
|
// Wait for container to be ready
|
|
874
808
|
if (options.waitFor) {
|
|
875
809
|
try {
|
|
876
|
-
await this.waitForContainer(
|
|
810
|
+
await this.waitForContainer(containerId, options.waitFor, portMappings);
|
|
877
811
|
this.log(`Container ready: ${name}`);
|
|
878
812
|
} catch (error) {
|
|
879
813
|
// Container failed to start, attempt cleanup
|
|
880
814
|
this.logError(`Container ${name} failed to become ready, cleaning up`, error);
|
|
881
815
|
try {
|
|
882
|
-
execFileSync(this.dockerPath, ['stop', '-t', '5',
|
|
883
|
-
execFileSync(this.dockerPath, ['rm', '-f', '-v',
|
|
816
|
+
execFileSync(this.dockerPath, ['stop', '-t', '5', containerId], { stdio: 'ignore' });
|
|
817
|
+
execFileSync(this.dockerPath, ['rm', '-f', '-v', containerId], { stdio: 'ignore' });
|
|
884
818
|
} catch {
|
|
885
819
|
// Ignore cleanup errors during rollback
|
|
886
820
|
}
|
|
@@ -926,10 +860,10 @@ export class DockerTestManager extends EventEmitter {
|
|
|
926
860
|
networks,
|
|
927
861
|
createdAt: new Date(),
|
|
928
862
|
internalIp,
|
|
929
|
-
getStatus: async () => this.getContainerStatus(
|
|
930
|
-
isHealthy: async () => this.isContainerHealthy(
|
|
863
|
+
getStatus: async () => this.getContainerStatus(containerId),
|
|
864
|
+
isHealthy: async () => this.isContainerHealthy(containerId),
|
|
931
865
|
cleanup: async () => {
|
|
932
|
-
await this.cleanupContainerWithRetry(
|
|
866
|
+
await this.cleanupContainerWithRetry(containerId, id, portMappings);
|
|
933
867
|
},
|
|
934
868
|
};
|
|
935
869
|
|
|
@@ -941,160 +875,67 @@ export class DockerTestManager extends EventEmitter {
|
|
|
941
875
|
* Clean up a container with retry logic
|
|
942
876
|
* @private
|
|
943
877
|
*/
|
|
944
|
-
private async cleanupContainerWithRetry(
|
|
945
|
-
this.
|
|
946
|
-
|
|
947
|
-
let lastError!: Error | null;
|
|
948
|
-
|
|
949
|
-
for (let attempt = 1; attempt <= this.maxCleanupRetries; attempt++) {
|
|
950
|
-
try {
|
|
951
|
-
// Check if container exists
|
|
952
|
-
try {
|
|
953
|
-
const status = await this.getContainerStatus(name);
|
|
954
|
-
if (!status.isRunning) {
|
|
955
|
-
this.log(`Container ${name} is not running, force removing`, { attempt });
|
|
956
|
-
}
|
|
957
|
-
} catch {
|
|
958
|
-
// Container doesn't exist, nothing to clean up
|
|
959
|
-
this.log(`Container ${name} doesn't exist, skipping stop`, { attempt });
|
|
960
|
-
break;
|
|
961
|
-
}
|
|
962
|
-
|
|
963
|
-
// Gracefully stop the container first
|
|
964
|
-
try {
|
|
965
|
-
const stopTimeoutSecs = Math.ceil(this.gracefulShutdownTimeout / 1000).toString();
|
|
966
|
-
this.log(`Stopping container ${name}`, { attempt, timeout: this.gracefulShutdownTimeout });
|
|
967
|
-
execFileSync(this.dockerPath, ['stop', '-t', stopTimeoutSecs, name], {
|
|
968
|
-
stdio: 'ignore',
|
|
969
|
-
timeout: this.gracefulShutdownTimeout + 5000, // Add buffer to docker timeout
|
|
970
|
-
});
|
|
971
|
-
this.log(`Container stopped: ${name}`, { attempt });
|
|
972
|
-
} catch (error) {
|
|
973
|
-
this.logError(`Failed to gracefully stop container ${name}`, error);
|
|
974
|
-
// Continue to force remove
|
|
975
|
-
}
|
|
976
|
-
|
|
977
|
-
// Remove container
|
|
978
|
-
try {
|
|
979
|
-
this.log(`Removing container ${name}`, { attempt });
|
|
980
|
-
execFileSync(this.dockerPath, ['rm', '-f', '-v', name], { stdio: 'ignore' });
|
|
981
|
-
this.log(`Container removed: ${name}`, { attempt });
|
|
982
|
-
lastError = null; // Clear error on success
|
|
983
|
-
break;
|
|
984
|
-
} catch (error) {
|
|
985
|
-
lastError = error instanceof Error ? error : new Error(String(error));
|
|
986
|
-
if (attempt === this.maxCleanupRetries) {
|
|
987
|
-
throw lastError;
|
|
988
|
-
}
|
|
989
|
-
// Wait before retrying
|
|
990
|
-
await new Promise((resolve) => setTimeout(resolve, 1000 * attempt));
|
|
991
|
-
}
|
|
992
|
-
} catch (error) {
|
|
993
|
-
lastError = error instanceof Error ? error : new Error(String(error));
|
|
994
|
-
if (attempt === this.maxCleanupRetries) {
|
|
995
|
-
this.logError(`Final cleanup attempt failed for container ${name}`, lastError);
|
|
996
|
-
// Don't throw, continue with port cleanup
|
|
997
|
-
} else {
|
|
998
|
-
this.logError(`Cleanup attempt ${attempt} failed for container ${name}, retrying`, lastError);
|
|
999
|
-
await new Promise((resolve) => setTimeout(resolve, 1000 * attempt));
|
|
1000
|
-
}
|
|
1001
|
-
}
|
|
1002
|
-
}
|
|
1003
|
-
|
|
1004
|
-
// Always release ports, even if cleanup failed
|
|
878
|
+
private async cleanupContainerWithRetry(containerId: string, id: string, portMappings: Map<number, number>): Promise<void> {
|
|
879
|
+
if (!this.containers.has(id)) return;
|
|
880
|
+
const seconds = Math.ceil(this.gracefulShutdownTimeout / 1000).toString();
|
|
1005
881
|
try {
|
|
1006
|
-
|
|
1007
|
-
this.
|
|
1008
|
-
this.log(`Released port: ${port}`);
|
|
882
|
+
execFileSync(this.dockerPath, ['stop', '-t', seconds, containerId], {
|
|
883
|
+
stdio: 'ignore', timeout: this.gracefulShutdownTimeout + 5000,
|
|
1009
884
|
});
|
|
1010
885
|
} catch (error) {
|
|
1011
|
-
this.logError(
|
|
886
|
+
this.logError(`Could not stop test container ${containerId} gracefully`, error);
|
|
1012
887
|
}
|
|
1013
|
-
|
|
1014
|
-
// Remove from tracking
|
|
1015
|
-
this.containers.delete(id);
|
|
1016
|
-
this.log(`Container cleanup completed: ${name}`, { attempts: this.maxCleanupRetries });
|
|
1017
|
-
}
|
|
1018
|
-
|
|
1019
|
-
/**
|
|
1020
|
-
* Ensure a Docker network exists
|
|
1021
|
-
* @private
|
|
1022
|
-
*/
|
|
1023
|
-
private async ensureNetwork(network: string): Promise<void> {
|
|
1024
|
-
if (!this.networks.has(network)) {
|
|
888
|
+
for (let attempt = 1; ; attempt++) {
|
|
1025
889
|
try {
|
|
1026
|
-
execFileSync(this.dockerPath, ['
|
|
1027
|
-
|
|
1028
|
-
encoding: 'utf8',
|
|
1029
|
-
});
|
|
1030
|
-
this.networks.add(network);
|
|
1031
|
-
this.log(`Created network: ${network}`);
|
|
890
|
+
execFileSync(this.dockerPath, ['rm', '-f', '-v', containerId], { stdio: ['ignore', 'ignore', 'pipe'] });
|
|
891
|
+
break;
|
|
1032
892
|
} catch (error) {
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
if (stderr.includes('already exists') || stderr.includes('network with name')) {
|
|
1037
|
-
this.log(`Network already exists: ${network}`);
|
|
1038
|
-
this.networks.add(network);
|
|
1039
|
-
} else if (stderr.includes('all predefined address pools have been fully subnetted')) {
|
|
1040
|
-
// Docker network pool exhausted - try to clean up unused networks
|
|
1041
|
-
this.log('Docker network pool exhausted, attempting cleanup...');
|
|
1042
|
-
try {
|
|
1043
|
-
// Prune unused networks THAT WE CREATED. A bare `network prune -f`
|
|
1044
|
-
// is host-wide: it removes every unused network on the machine,
|
|
1045
|
-
// including ones belonging to compose stacks and other projects
|
|
1046
|
-
// that happen to have no running container at that instant. Every
|
|
1047
|
-
// other operation in this manager is scoped by the
|
|
1048
|
-
// `test.cleanup=true` label; this one was not.
|
|
1049
|
-
execFileSync(this.dockerPath, ['network', 'prune', '-f', '--filter', `label=test.manager=${this.managerId}`], {
|
|
1050
|
-
stdio: 'ignore',
|
|
1051
|
-
});
|
|
1052
|
-
this.log('Cleaned up unused networks, retrying network creation...');
|
|
1053
|
-
// Retry network creation after cleanup
|
|
1054
|
-
execFileSync(this.dockerPath, ['network', 'create', network, '--label', 'test.cleanup=true', '--label', `test.manager=${this.managerId}`, '--label', `test.host=${DockerTestManager.hostId}`], {
|
|
1055
|
-
stdio: 'pipe',
|
|
1056
|
-
encoding: 'utf8',
|
|
1057
|
-
});
|
|
1058
|
-
this.networks.add(network);
|
|
1059
|
-
this.log(`Created network after cleanup: ${network}`);
|
|
1060
|
-
} catch (_retryError) {
|
|
1061
|
-
throw new Error(`Failed to create network ${network} even after cleanup: ${stderr}`, { cause: _retryError });
|
|
1062
|
-
}
|
|
1063
|
-
} else {
|
|
1064
|
-
// Real error - throw it
|
|
1065
|
-
throw new Error(`Failed to create network ${network}: ${stderr}`, { cause: error });
|
|
1066
|
-
}
|
|
893
|
+
if (/no such (container|object)/i.test(String((error as { stderr?: Buffer }).stderr))) break;
|
|
894
|
+
if (attempt >= this.maxCleanupRetries) throw error;
|
|
895
|
+
await new Promise(resolve => setTimeout(resolve, 1000 * attempt));
|
|
1067
896
|
}
|
|
1068
897
|
}
|
|
898
|
+
portMappings.forEach(port => this.usedPorts.delete(port));
|
|
899
|
+
this.containers.delete(id);
|
|
1069
900
|
}
|
|
1070
901
|
|
|
1071
902
|
/**
|
|
1072
|
-
*
|
|
1073
|
-
* @private
|
|
903
|
+
* Create a labelled network or join an existing network under its current owner.
|
|
1074
904
|
*/
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
905
|
+
async ensureNetwork(network: string): Promise<void> {
|
|
906
|
+
const create = () => {
|
|
907
|
+
const networkId = execFileSync(this.dockerPath, ['network', 'create', network,
|
|
908
|
+
'--label', 'test.cleanup=true', '--label', `test.manager=${this.managerId}`,
|
|
909
|
+
'--label', `test.host=${DockerTestManager.hostId}`], { stdio: 'pipe', encoding: 'utf8' }).trim();
|
|
910
|
+
this.networks.set(network, networkId);
|
|
911
|
+
};
|
|
912
|
+
try { create(); }
|
|
913
|
+
catch (error) {
|
|
914
|
+
const reason = String((error as { stderr?: string }).stderr ?? error);
|
|
915
|
+
if (reason.includes('already exists')) return;
|
|
916
|
+
if (!reason.includes('all predefined address pools have been fully subnetted')) throw error;
|
|
917
|
+
execFileSync(this.dockerPath, ['network', 'prune', '-f', '--filter', `label=test.manager=${this.managerId}`], { stdio: 'ignore' });
|
|
918
|
+
create();
|
|
1078
919
|
}
|
|
920
|
+
}
|
|
1079
921
|
|
|
922
|
+
/** Remove a network created by this manager, even if its name has since been reused. */
|
|
923
|
+
async removeNetwork(network: string): Promise<void> {
|
|
924
|
+
const id = this.networks.get(network);
|
|
925
|
+
if (!id) return;
|
|
1080
926
|
try {
|
|
1081
|
-
execFileSync(this.dockerPath, ['network', 'rm',
|
|
1082
|
-
stdio: 'ignore',
|
|
1083
|
-
});
|
|
1084
|
-
this.networks.delete(network);
|
|
1085
|
-
this.log(`Removed network: ${network}`);
|
|
927
|
+
execFileSync(this.dockerPath, ['network', 'rm', id], { stdio: ['ignore', 'ignore', 'pipe'] });
|
|
1086
928
|
} catch (error) {
|
|
1087
|
-
|
|
1088
|
-
this.networks.delete(network); // Remove from tracking anyway
|
|
1089
|
-
this.logError(`Failed to remove network ${network}`, error);
|
|
929
|
+
if (!/not found|no such network/i.test(String((error as { stderr?: Buffer }).stderr))) throw error;
|
|
1090
930
|
}
|
|
931
|
+
this.networks.delete(network);
|
|
1091
932
|
}
|
|
1092
933
|
|
|
1093
934
|
/**
|
|
1094
935
|
* Wait for a container to be ready
|
|
1095
936
|
* @private
|
|
1096
937
|
*/
|
|
1097
|
-
private async waitForContainer(name: string, options: ContainerOptions['waitFor']): Promise<void> {
|
|
938
|
+
private async waitForContainer(name: string, options: ContainerOptions['waitFor'], ports: Map<number, number>): Promise<void> {
|
|
1098
939
|
const startTime = Date.now();
|
|
1099
940
|
const timeout = options?.timeout || this.startupTimeout;
|
|
1100
941
|
|
|
@@ -1104,60 +945,22 @@ export class DockerTestManager extends EventEmitter {
|
|
|
1104
945
|
try {
|
|
1105
946
|
// First check if container is still running
|
|
1106
947
|
const status = await this.getContainerStatus(name);
|
|
948
|
+
if (!status.state) throw new Error('Container status is unavailable');
|
|
1107
949
|
if (!status.isRunning) {
|
|
1108
950
|
throw new Error(
|
|
1109
|
-
`Container exited with status '${status.state}' and exit code ${status.exitCode}
|
|
1110
|
-
`Check container logs with 'docker logs ${name}'`
|
|
951
|
+
`Container ${name} exited with status '${status.state}' and exit code ${status.exitCode}`
|
|
1111
952
|
);
|
|
1112
953
|
}
|
|
1113
954
|
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
}
|
|
1120
|
-
|
|
1121
|
-
if (status.health === 'unhealthy') {
|
|
1122
|
-
const elapsed = Date.now() - startTime;
|
|
1123
|
-
this.logError(
|
|
1124
|
-
`Container ${name} health check failed after ${elapsed}ms`,
|
|
1125
|
-
new Error(`Health status: ${status.health}, State: ${status.state}`)
|
|
1126
|
-
);
|
|
1127
|
-
throw new Error(
|
|
1128
|
-
`Container ${name} health check failed. ` + `Check container logs with 'docker logs ${name}'`
|
|
1129
|
-
);
|
|
1130
|
-
}
|
|
1131
|
-
|
|
1132
|
-
// Health check is still starting
|
|
1133
|
-
if (status.health === 'starting') {
|
|
1134
|
-
const elapsed = Date.now() - startTime;
|
|
1135
|
-
this.log(`Waiting for health check on ${name}...`, {
|
|
1136
|
-
elapsed,
|
|
1137
|
-
remaining: timeout - elapsed,
|
|
1138
|
-
state: status.state,
|
|
1139
|
-
});
|
|
1140
|
-
}
|
|
1141
|
-
}
|
|
1142
|
-
|
|
1143
|
-
// Check if port is accessible
|
|
1144
|
-
if (options?.port) {
|
|
1145
|
-
const container = this.containers.get(name);
|
|
1146
|
-
if (container) {
|
|
1147
|
-
const hostPort = container.ports.get(options.port);
|
|
1148
|
-
if (hostPort && (await this.isPortListening('127.0.0.1', hostPort))) {
|
|
1149
|
-
this.log(`Port ${hostPort} is listening on ${name}`);
|
|
1150
|
-
return;
|
|
1151
|
-
}
|
|
1152
|
-
}
|
|
1153
|
-
}
|
|
1154
|
-
|
|
1155
|
-
// If no specific wait condition, just check container is running
|
|
1156
|
-
if (!options?.healthcheck && !options?.port) {
|
|
1157
|
-
this.log(`Container ${name} is running`);
|
|
1158
|
-
return;
|
|
955
|
+
if (options?.healthcheck && status.health === 'unhealthy') {
|
|
956
|
+
const probes = execFileSync(this.dockerPath, ['inspect', '--format',
|
|
957
|
+
'{{with .State.Health}}{{.FailingStreak}} failed probes{{range .Log}}; exit={{.ExitCode}} at {{.End}}{{end}}{{end}}', name],
|
|
958
|
+
{ encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
|
959
|
+
throw new Error(`Container ${name} health check failed after ${Date.now() - startTime} ms (${probes})`);
|
|
1159
960
|
}
|
|
1160
|
-
|
|
961
|
+
const hostPort = options?.port && ports.get(options.port);
|
|
962
|
+
if ((!options?.healthcheck || status.health === 'healthy') &&
|
|
963
|
+
(!options?.port || (hostPort && await this.isPortListening('127.0.0.1', hostPort)))) return;
|
|
1161
964
|
lastError = undefined;
|
|
1162
965
|
} catch (error) {
|
|
1163
966
|
lastError = error instanceof Error ? error : new Error(String(error));
|
|
@@ -1231,61 +1034,18 @@ export class DockerTestManager extends EventEmitter {
|
|
|
1231
1034
|
/**
|
|
1232
1035
|
* Clean up all containers managed by this instance
|
|
1233
1036
|
*/
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
return;
|
|
1238
|
-
}
|
|
1239
|
-
|
|
1240
|
-
this.cleanupInProgress = true;
|
|
1241
|
-
|
|
1242
|
-
try {
|
|
1243
|
-
this.log(`Starting cleanup of ${this.containers.size} containers`);
|
|
1244
|
-
|
|
1245
|
-
// Cleanup containers in parallel with Promise.allSettled to handle failures
|
|
1246
|
-
const cleanupPromises = Array.from(this.containers.values()).map((container) =>
|
|
1247
|
-
container
|
|
1248
|
-
.cleanup()
|
|
1249
|
-
.then(() => {
|
|
1250
|
-
this.log(`Cleanup succeeded: ${container.name}`);
|
|
1251
|
-
})
|
|
1252
|
-
.catch((error) => {
|
|
1253
|
-
this.logError(`Cleanup failed for ${container.name}`, error);
|
|
1254
|
-
// Still continue cleaning up other containers
|
|
1255
|
-
})
|
|
1256
|
-
);
|
|
1257
|
-
|
|
1258
|
-
const results = await Promise.allSettled(cleanupPromises);
|
|
1259
|
-
|
|
1260
|
-
const failed = results.filter((r) => r.status === 'rejected').length;
|
|
1261
|
-
if (failed > 0) {
|
|
1262
|
-
this.log(`Cleanup completed with ${failed} failures`);
|
|
1263
|
-
} else {
|
|
1264
|
-
this.log('All containers cleaned up successfully');
|
|
1265
|
-
}
|
|
1266
|
-
|
|
1267
|
-
// Cleanup networks
|
|
1268
|
-
this.log(`Cleaning up ${this.networks.size} networks`);
|
|
1269
|
-
const networkArray = Array.from(this.networks);
|
|
1270
|
-
for (const network of networkArray) {
|
|
1271
|
-
try {
|
|
1272
|
-
this.log(`Removing network: ${network}`);
|
|
1273
|
-
execFileSync(this.dockerPath, ['network', 'rm', network], { stdio: 'ignore' });
|
|
1274
|
-
this.log(`Network removed: ${network}`);
|
|
1275
|
-
} catch (error) {
|
|
1276
|
-
this.logError(`Failed to remove network ${network}`, error);
|
|
1277
|
-
// Continue with other networks
|
|
1278
|
-
}
|
|
1279
|
-
}
|
|
1280
|
-
|
|
1281
|
-
this.containers.clear();
|
|
1282
|
-
this.usedPorts.clear();
|
|
1283
|
-
this.networks.clear();
|
|
1037
|
+
cleanupAll(): Promise<void> {
|
|
1038
|
+
return this.activeCleanup ??= this.cleanupOwnedResources().finally(() => { this.activeCleanup = undefined; });
|
|
1039
|
+
}
|
|
1284
1040
|
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1041
|
+
private async cleanupOwnedResources(): Promise<void> {
|
|
1042
|
+
const results = await Promise.allSettled(Array.from(this.containers.values(), container => container.cleanup()));
|
|
1043
|
+
const errors = results.flatMap(result => result.status === 'rejected' ? [result.reason] : []);
|
|
1044
|
+
for (const network of this.networks.keys()) {
|
|
1045
|
+
try { await this.removeNetwork(network); }
|
|
1046
|
+
catch (error) { errors.push(error); }
|
|
1288
1047
|
}
|
|
1048
|
+
if (errors.length) throw new AggregateError(errors, 'Some Docker test resources could not be removed');
|
|
1289
1049
|
}
|
|
1290
1050
|
|
|
1291
1051
|
/**
|