@evandam93/dev-env 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +282 -2
- package/bin/dev-env.js +3 -0
- package/dist/abandoned.d.ts +2 -0
- package/dist/abandoned.js +66 -0
- package/dist/affected/graph.d.ts +43 -0
- package/dist/affected/graph.js +222 -0
- package/dist/affected/index.d.ts +5 -0
- package/dist/affected/index.js +3 -0
- package/dist/affected/layout.d.ts +11 -0
- package/dist/affected/layout.js +34 -0
- package/dist/affected/run.d.ts +10 -0
- package/dist/affected/run.js +96 -0
- package/dist/affected/select.d.ts +27 -0
- package/dist/affected/select.js +79 -0
- package/dist/checkout.d.ts +13 -0
- package/dist/checkout.js +40 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +98 -0
- package/dist/compose.d.ts +18 -0
- package/dist/compose.js +119 -0
- package/dist/environment.d.ts +33 -0
- package/dist/environment.js +387 -0
- package/dist/errors.d.ts +4 -0
- package/dist/errors.js +4 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +2 -0
- package/dist/machine.d.ts +3 -0
- package/dist/machine.js +17 -0
- package/dist/manifest.d.ts +95 -0
- package/dist/manifest.js +91 -0
- package/dist/ports.d.ts +6 -0
- package/dist/ports.js +92 -0
- package/dist/processes.d.ts +13 -0
- package/dist/processes.js +116 -0
- package/dist/readiness.d.ts +6 -0
- package/dist/readiness.js +31 -0
- package/dist/shared.d.ts +9 -0
- package/dist/shared.js +81 -0
- package/dist/slots.d.ts +21 -0
- package/dist/slots.js +165 -0
- package/dist/state.d.ts +21 -0
- package/dist/state.js +36 -0
- package/dist/test-environment.d.ts +36 -0
- package/dist/test-environment.js +67 -0
- package/dist/workspace.d.ts +31 -0
- package/dist/workspace.js +140 -0
- package/package.json +75 -4
package/dist/ports.js
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { readFile, unlink, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { connect, createServer } from 'node:net';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { setTimeout as sleep } from 'node:timers/promises';
|
|
6
|
+
import { DevEnvError } from './errors.js';
|
|
7
|
+
export const LOOPBACK = '127.0.0.1';
|
|
8
|
+
/* Below every platform's ephemeral range, so outbound connections never take one. */
|
|
9
|
+
const FIRST_PORT = 20_000;
|
|
10
|
+
const PORT_COUNT = 10_000;
|
|
11
|
+
const LOCK_TIMEOUT_MS = 15_000;
|
|
12
|
+
const LOCK_RETRY_MS = 50;
|
|
13
|
+
export const preferredPortOf = (seed) => FIRST_PORT +
|
|
14
|
+
(createHash('sha256').update(seed).digest().readUInt32BE(0) % PORT_COUNT);
|
|
15
|
+
export const choosePort = async (preferred, isTaken) => {
|
|
16
|
+
for (let offset = 0; offset < PORT_COUNT; offset += 1) {
|
|
17
|
+
const candidate = FIRST_PORT + ((preferred - FIRST_PORT + offset) % PORT_COUNT);
|
|
18
|
+
if (!(await isTaken(candidate)))
|
|
19
|
+
return candidate;
|
|
20
|
+
}
|
|
21
|
+
throw new DevEnvError(`No free port between ${FIRST_PORT} and ${FIRST_PORT + PORT_COUNT - 1}.`);
|
|
22
|
+
};
|
|
23
|
+
const answersOnPort = (port) => new Promise((resolve) => {
|
|
24
|
+
const socket = connect({ host: LOOPBACK, port, timeout: 250 });
|
|
25
|
+
const settle = (answer) => {
|
|
26
|
+
socket.destroy();
|
|
27
|
+
resolve(answer);
|
|
28
|
+
};
|
|
29
|
+
socket.once('connect', () => settle(true));
|
|
30
|
+
socket.once('timeout', () => settle(false));
|
|
31
|
+
socket.once('error', () => settle(false));
|
|
32
|
+
});
|
|
33
|
+
const canBindPort = (port) => new Promise((resolve) => {
|
|
34
|
+
const server = createServer();
|
|
35
|
+
server.once('error', () => resolve(false));
|
|
36
|
+
server.listen(port, LOOPBACK, () => {
|
|
37
|
+
server.close(() => resolve(true));
|
|
38
|
+
});
|
|
39
|
+
});
|
|
40
|
+
/*
|
|
41
|
+
* Both checks are needed: macOS lets a loopback bind succeed beside a wildcard
|
|
42
|
+
* listener, and a socket in the backlog can refuse a connection yet hold the port.
|
|
43
|
+
*/
|
|
44
|
+
export const isPortInUse = async (port) => (await answersOnPort(port)) || !(await canBindPort(port));
|
|
45
|
+
const isProcessAlive = (pid) => {
|
|
46
|
+
try {
|
|
47
|
+
process.kill(pid, 0);
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
return false;
|
|
52
|
+
}
|
|
53
|
+
};
|
|
54
|
+
const removeLockIfAbandoned = async (path) => {
|
|
55
|
+
const holder = Number(await readFile(path, 'utf8').catch(() => ''));
|
|
56
|
+
if (holder > 0 && !isProcessAlive(holder)) {
|
|
57
|
+
await unlink(path).catch(() => undefined);
|
|
58
|
+
}
|
|
59
|
+
};
|
|
60
|
+
/*
|
|
61
|
+
* A file created exclusively, holding the holder's pid, so a holder that died
|
|
62
|
+
* leaves nothing a waiter cannot remove.
|
|
63
|
+
*/
|
|
64
|
+
export const withLock = async (path, timeoutMs, work) => {
|
|
65
|
+
const deadline = Date.now() + timeoutMs;
|
|
66
|
+
for (;;) {
|
|
67
|
+
try {
|
|
68
|
+
await writeFile(path, String(process.pid), { flag: 'wx' });
|
|
69
|
+
break;
|
|
70
|
+
}
|
|
71
|
+
catch (error) {
|
|
72
|
+
if (error.code !== 'EEXIST')
|
|
73
|
+
throw error;
|
|
74
|
+
if (Date.now() > deadline) {
|
|
75
|
+
throw new DevEnvError(`Could not take the lock at ${path}.`);
|
|
76
|
+
}
|
|
77
|
+
await removeLockIfAbandoned(path);
|
|
78
|
+
await sleep(LOCK_RETRY_MS);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
try {
|
|
82
|
+
return await work();
|
|
83
|
+
}
|
|
84
|
+
finally {
|
|
85
|
+
await unlink(path).catch(() => undefined);
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
/*
|
|
89
|
+
* Lives in the git common directory because that is the one place every
|
|
90
|
+
* worktree of a repository shares.
|
|
91
|
+
*/
|
|
92
|
+
export const withAllocationLock = (gitCommonDirectory, work) => withLock(join(gitCommonDirectory, 'dev-env-allocation.lock'), LOCK_TIMEOUT_MS, work);
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { RecordedProcess } from './state.ts';
|
|
2
|
+
export declare const startTimeOf: (pid: number) => Promise<string | undefined>;
|
|
3
|
+
export declare const isLeaderLive: (recorded: RecordedProcess) => Promise<boolean>;
|
|
4
|
+
export declare const isRecordedProcessLive: (recorded: RecordedProcess) => Promise<boolean>;
|
|
5
|
+
export interface StartProcessOptions {
|
|
6
|
+
readonly command: readonly string[];
|
|
7
|
+
readonly directory: string;
|
|
8
|
+
readonly environment: Readonly<Record<string, string>>;
|
|
9
|
+
readonly logPath: string;
|
|
10
|
+
}
|
|
11
|
+
export declare const startProcess: (options: StartProcessOptions) => Promise<RecordedProcess>;
|
|
12
|
+
export declare const stopProcess: (recorded: RecordedProcess) => Promise<void>;
|
|
13
|
+
export declare const tailOf: (path: string, lines: number) => string;
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { execFile, spawn } from 'node:child_process';
|
|
2
|
+
import { closeSync, mkdirSync, openSync, readFileSync } from 'node:fs';
|
|
3
|
+
import { dirname } from 'node:path';
|
|
4
|
+
import { setTimeout as sleep } from 'node:timers/promises';
|
|
5
|
+
import { promisify } from 'node:util';
|
|
6
|
+
import { DevEnvError } from './errors.js';
|
|
7
|
+
const run = promisify(execFile);
|
|
8
|
+
const STOP_GRACE_MS = 5_000;
|
|
9
|
+
const POLL_MS = 100;
|
|
10
|
+
export const startTimeOf = async (pid) => {
|
|
11
|
+
try {
|
|
12
|
+
const { stdout } = await run('ps', ['-o', 'lstart=', '-p', String(pid)], {
|
|
13
|
+
env: { ...process.env, LC_ALL: 'C' },
|
|
14
|
+
});
|
|
15
|
+
return stdout.trim() || undefined;
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
return undefined;
|
|
19
|
+
}
|
|
20
|
+
};
|
|
21
|
+
/*
|
|
22
|
+
* The start time is what stops a recycled pid from being mistaken for this
|
|
23
|
+
* process, so an unrelated program is never signalled.
|
|
24
|
+
*/
|
|
25
|
+
export const isLeaderLive = async (recorded) => (await startTimeOf(recorded.pid)) === recorded.startedAt;
|
|
26
|
+
/*
|
|
27
|
+
* A wrapper such as pnpm can die and leave its server running. A group id alone
|
|
28
|
+
* could be a stranger's after a reboot, so a survivor must also hold this
|
|
29
|
+
* process's log open, which it inherited at spawn. Without lsof this finds no
|
|
30
|
+
* survivors, leaving a process running rather than signalling the wrong one.
|
|
31
|
+
*/
|
|
32
|
+
const hasSurvivors = async (recorded) => {
|
|
33
|
+
try {
|
|
34
|
+
const { stdout } = await run('lsof', [
|
|
35
|
+
'-a',
|
|
36
|
+
'-g',
|
|
37
|
+
String(recorded.pid),
|
|
38
|
+
'-t',
|
|
39
|
+
'--',
|
|
40
|
+
recorded.logPath,
|
|
41
|
+
]);
|
|
42
|
+
return stdout.trim() !== '';
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
return false;
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
export const isRecordedProcessLive = async (recorded) => {
|
|
49
|
+
const startedAt = await startTimeOf(recorded.pid);
|
|
50
|
+
if (startedAt !== undefined)
|
|
51
|
+
return startedAt === recorded.startedAt;
|
|
52
|
+
return hasSurvivors(recorded);
|
|
53
|
+
};
|
|
54
|
+
export const startProcess = async (options) => {
|
|
55
|
+
const [program, ...args] = options.command;
|
|
56
|
+
if (program === undefined) {
|
|
57
|
+
throw new DevEnvError('A process service needs a command.');
|
|
58
|
+
}
|
|
59
|
+
mkdirSync(dirname(options.logPath), { recursive: true });
|
|
60
|
+
const log = openSync(options.logPath, 'w');
|
|
61
|
+
const child = spawn(program, args, {
|
|
62
|
+
cwd: options.directory,
|
|
63
|
+
detached: true,
|
|
64
|
+
env: { ...process.env, ...options.environment },
|
|
65
|
+
stdio: ['ignore', log, log],
|
|
66
|
+
});
|
|
67
|
+
closeSync(log);
|
|
68
|
+
const pid = await new Promise((resolve, reject) => {
|
|
69
|
+
child.once('spawn', () => resolve(child.pid ?? 0));
|
|
70
|
+
child.once('error', (error) => reject(new DevEnvError(`Could not run ${program}: ${error.message}`)));
|
|
71
|
+
});
|
|
72
|
+
child.unref();
|
|
73
|
+
const startedAt = await startTimeOf(pid);
|
|
74
|
+
if (startedAt === undefined) {
|
|
75
|
+
throw new DevEnvError(`${program} exited as soon as it started.`);
|
|
76
|
+
}
|
|
77
|
+
return { pid, startedAt, logPath: options.logPath };
|
|
78
|
+
};
|
|
79
|
+
const isGroupAlive = (pid) => {
|
|
80
|
+
try {
|
|
81
|
+
process.kill(-pid, 0);
|
|
82
|
+
return true;
|
|
83
|
+
}
|
|
84
|
+
catch {
|
|
85
|
+
return false;
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
const signalGroup = (pid, signal) => {
|
|
89
|
+
try {
|
|
90
|
+
process.kill(-pid, signal);
|
|
91
|
+
}
|
|
92
|
+
catch (error) {
|
|
93
|
+
// macOS answers EPERM rather than ESRCH once every member has exited but is not yet reaped.
|
|
94
|
+
const { code } = error;
|
|
95
|
+
if (code !== 'ESRCH' && code !== 'EPERM')
|
|
96
|
+
throw error;
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
export const stopProcess = async (recorded) => {
|
|
100
|
+
if (!(await isRecordedProcessLive(recorded)))
|
|
101
|
+
return;
|
|
102
|
+
signalGroup(recorded.pid, 'SIGTERM');
|
|
103
|
+
const deadline = Date.now() + STOP_GRACE_MS;
|
|
104
|
+
while (isGroupAlive(recorded.pid) && Date.now() < deadline) {
|
|
105
|
+
await sleep(POLL_MS);
|
|
106
|
+
}
|
|
107
|
+
signalGroup(recorded.pid, 'SIGKILL');
|
|
108
|
+
};
|
|
109
|
+
export const tailOf = (path, lines) => {
|
|
110
|
+
try {
|
|
111
|
+
return readFileSync(path, 'utf8').split('\n').slice(-lines).join('\n');
|
|
112
|
+
}
|
|
113
|
+
catch {
|
|
114
|
+
return '';
|
|
115
|
+
}
|
|
116
|
+
};
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { setTimeout as sleep } from 'node:timers/promises';
|
|
2
|
+
const POLL_MS = 250;
|
|
3
|
+
// Long enough for a rendered page on a busy machine, short enough to ask again if a request was lost while the server started.
|
|
4
|
+
const ATTEMPT_MS = 30_000;
|
|
5
|
+
/*
|
|
6
|
+
* Asks the URL until it answers below 400, the process exits or the time is
|
|
7
|
+
* up, one request at a time. Each request is given time to finish: a page
|
|
8
|
+
* slower than the wait would never pass, and a server keeps rendering a page
|
|
9
|
+
* whose request was given up on, so asking again piles up its work until the
|
|
10
|
+
* database runs out of connections.
|
|
11
|
+
*/
|
|
12
|
+
export const waitUntilAnswering = async ({ url, timeoutMs, isLive, }) => {
|
|
13
|
+
const deadline = Date.now() + timeoutMs;
|
|
14
|
+
while (Date.now() < deadline) {
|
|
15
|
+
if (!(await isLive()))
|
|
16
|
+
return 'exited';
|
|
17
|
+
const attemptMs = Math.min(ATTEMPT_MS, deadline - Date.now());
|
|
18
|
+
try {
|
|
19
|
+
const response = await fetch(url, {
|
|
20
|
+
signal: AbortSignal.timeout(Math.max(attemptMs, 1)),
|
|
21
|
+
});
|
|
22
|
+
if (response.status < 400)
|
|
23
|
+
return 'ready';
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
// Refused or timed out: the server is still starting.
|
|
27
|
+
}
|
|
28
|
+
await sleep(POLL_MS);
|
|
29
|
+
}
|
|
30
|
+
return 'timed out';
|
|
31
|
+
};
|
package/dist/shared.d.ts
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { ComposeProject } from './compose.ts';
|
|
2
|
+
import type { Target } from './manifest.ts';
|
|
3
|
+
export declare const sharedServicesOf: (target: Target) => readonly string[];
|
|
4
|
+
export declare const sharedPortsOf: (target: Target) => Readonly<Record<string, number>>;
|
|
5
|
+
/** Every port a shared service on this machine holds, which no environment may take. */
|
|
6
|
+
export declare const portsSharedOnMachine: () => ReadonlySet<number>;
|
|
7
|
+
export declare const sharedProjectOf: (target: Target, ports?: Readonly<Record<string, number>>) => ComposeProject;
|
|
8
|
+
export declare const runningSharedServicesOf: (target: Target) => Promise<ReadonlyMap<string, number | undefined>>;
|
|
9
|
+
export declare const ensureSharedServices: (target: Target, claimed: ReadonlySet<number>) => Promise<Readonly<Record<string, number>>>;
|
package/dist/shared.js
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync, } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { assertServicesAreSafe, composeFileOf, composeUp, runningServicesOf, } from './compose.js';
|
|
4
|
+
import { machineDirectoryOf, machineIdOf } from './machine.js';
|
|
5
|
+
import { portVariablesOf } from './manifest.js';
|
|
6
|
+
import { choosePort, isPortInUse, preferredPortOf, withLock } from './ports.js';
|
|
7
|
+
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
8
|
+
// A waiter queues behind a start that can take as long as the slowest service.
|
|
9
|
+
const LOCK_MARGIN_MS = 60_000;
|
|
10
|
+
export const sharedServicesOf = (target) => Object.entries(target.manifest.services)
|
|
11
|
+
.filter(([, service]) => service.kind === 'compose' && service.shared)
|
|
12
|
+
.map(([name]) => name);
|
|
13
|
+
const sharedDirectory = () => {
|
|
14
|
+
const directory = join(machineDirectoryOf(), 'shared');
|
|
15
|
+
mkdirSync(directory, { recursive: true });
|
|
16
|
+
return directory;
|
|
17
|
+
};
|
|
18
|
+
/* Not tied to a checkout: every checkout of the target on the machine uses it. */
|
|
19
|
+
const projectNameOf = (target) => `dev-env-shared-${machineIdOf(machineDirectoryOf())}-${target.name}`;
|
|
20
|
+
const statePathOf = (target) => join(sharedDirectory(), `${projectNameOf(target)}.json`);
|
|
21
|
+
const readPorts = (path) => existsSync(path)
|
|
22
|
+
? JSON.parse(readFileSync(path, 'utf8')).ports
|
|
23
|
+
: {};
|
|
24
|
+
export const sharedPortsOf = (target) => readPorts(statePathOf(target));
|
|
25
|
+
/** Every port a shared service on this machine holds, which no environment may take. */
|
|
26
|
+
export const portsSharedOnMachine = () => new Set(readdirSync(sharedDirectory())
|
|
27
|
+
.filter((file) => file.endsWith('.json'))
|
|
28
|
+
.flatMap((file) => Object.values(readPorts(join(sharedDirectory(), file)))));
|
|
29
|
+
/*
|
|
30
|
+
* The base compose file only: a test override trades durability for speed in
|
|
31
|
+
* one environment, and this one serves them all.
|
|
32
|
+
*/
|
|
33
|
+
export const sharedProjectOf = (target, ports = sharedPortsOf(target)) => ({
|
|
34
|
+
name: projectNameOf(target),
|
|
35
|
+
directory: target.directory,
|
|
36
|
+
files: [composeFileOf(target.directory)],
|
|
37
|
+
variables: portVariablesOf({
|
|
38
|
+
...Object.fromEntries(Object.keys(target.manifest.services).map((service) => [service, 0])),
|
|
39
|
+
...ports,
|
|
40
|
+
}),
|
|
41
|
+
});
|
|
42
|
+
export const runningSharedServicesOf = async (target) => sharedServicesOf(target).length === 0
|
|
43
|
+
? new Map()
|
|
44
|
+
: runningServicesOf(sharedProjectOf(target));
|
|
45
|
+
/*
|
|
46
|
+
* Starts the target's shared services unless they already run, and returns
|
|
47
|
+
* their ports. A running stack is never recreated, even when this checkout's
|
|
48
|
+
* compose file defines it differently, because other runs are using it; the
|
|
49
|
+
* first definition stays until someone removes the project.
|
|
50
|
+
*/
|
|
51
|
+
export const ensureSharedServices = async (target, claimed) => {
|
|
52
|
+
const services = sharedServicesOf(target);
|
|
53
|
+
if (services.length === 0)
|
|
54
|
+
return {};
|
|
55
|
+
const timeoutMs = Math.max(...services.map((service) => target.manifest.services[service]?.timeoutMs ?? DEFAULT_TIMEOUT_MS));
|
|
56
|
+
return withLock(join(machineDirectoryOf(), 'shared.lock'), timeoutMs + LOCK_MARGIN_MS, async () => {
|
|
57
|
+
const recorded = sharedPortsOf(target);
|
|
58
|
+
const running = Object.keys(recorded).length > 0
|
|
59
|
+
? await runningServicesOf(sharedProjectOf(target, recorded))
|
|
60
|
+
: new Map();
|
|
61
|
+
if (services.every((service) => recorded[service] !== undefined &&
|
|
62
|
+
running.get(service) === recorded[service]))
|
|
63
|
+
return recorded;
|
|
64
|
+
const ports = {};
|
|
65
|
+
for (const service of services) {
|
|
66
|
+
const previous = recorded[service];
|
|
67
|
+
ports[service] = await choosePort(previous ??
|
|
68
|
+
preferredPortOf(`shared:${projectNameOf(target)}:${service}`), async (candidate) => Object.values(ports).includes(candidate) ||
|
|
69
|
+
(candidate !== previous && claimed.has(candidate)) ||
|
|
70
|
+
((await isPortInUse(candidate)) &&
|
|
71
|
+
running.get(service) !== candidate));
|
|
72
|
+
}
|
|
73
|
+
const path = statePathOf(target);
|
|
74
|
+
writeFileSync(`${path}.tmp`, JSON.stringify({ ports }, null, 2));
|
|
75
|
+
renameSync(`${path}.tmp`, path);
|
|
76
|
+
const project = sharedProjectOf(target, ports);
|
|
77
|
+
await assertServicesAreSafe(project, services);
|
|
78
|
+
await composeUp(project, services, timeoutMs);
|
|
79
|
+
return ports;
|
|
80
|
+
});
|
|
81
|
+
};
|
package/dist/slots.d.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export interface Holder {
|
|
2
|
+
readonly pid: number;
|
|
3
|
+
readonly startedAt: string;
|
|
4
|
+
/** What the holder runs, to say who a waiter waits for. */
|
|
5
|
+
readonly name: string;
|
|
6
|
+
}
|
|
7
|
+
export declare const slotDirectoryOf: () => string;
|
|
8
|
+
export declare const holderOf: (name: string, pid?: number) => Promise<Holder>;
|
|
9
|
+
export declare const slotCountOf: (override: string | undefined, dockerMemoryBytes: number, cores: number) => number;
|
|
10
|
+
export declare const slotCountOnMachine: () => Promise<number>;
|
|
11
|
+
export interface TakeSlotOptions {
|
|
12
|
+
readonly directory: string;
|
|
13
|
+
readonly size: number;
|
|
14
|
+
readonly holder: Holder;
|
|
15
|
+
/** Every slot on the machine, so nothing else runs beside a file that measures time. */
|
|
16
|
+
readonly exclusive?: boolean;
|
|
17
|
+
readonly onWait?: (reason: string) => void;
|
|
18
|
+
}
|
|
19
|
+
export declare const takeSlot: ({ directory, size, holder, exclusive, onWait, }: TakeSlotOptions) => Promise<void>;
|
|
20
|
+
/** Synchronous, so it can run as the process exits. */
|
|
21
|
+
export declare const releaseSlots: (directory: string, holder: Holder) => void;
|
package/dist/slots.js
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process';
|
|
2
|
+
import { linkSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync, } from 'node:fs';
|
|
3
|
+
import { availableParallelism } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { setTimeout as sleep } from 'node:timers/promises';
|
|
6
|
+
import { promisify } from 'node:util';
|
|
7
|
+
import { DevEnvError } from './errors.js';
|
|
8
|
+
import { machineDirectoryOf } from './machine.js';
|
|
9
|
+
import { startTimeOf } from './processes.js';
|
|
10
|
+
const run = promisify(execFile);
|
|
11
|
+
/*
|
|
12
|
+
* A lane's containers are Postgres and a proxy, about 150 MB together, which
|
|
13
|
+
* grows with the rows a file writes; the shared telemetry stack is about 1 GB.
|
|
14
|
+
*/
|
|
15
|
+
const LANE_MEMORY_BYTES = 384 * 1024 ** 2;
|
|
16
|
+
const SHARED_MEMORY_BYTES = 1024 ** 3;
|
|
17
|
+
/*
|
|
18
|
+
* Each lane also runs its Workers on the host, up to nine wrangler processes,
|
|
19
|
+
* and lanes beyond the host's cores slowed each other until tests timed out.
|
|
20
|
+
*/
|
|
21
|
+
const CORES_PER_LANE = 3;
|
|
22
|
+
const POLL_MS = 250;
|
|
23
|
+
const EXCLUSIVE = 'exclusive.json';
|
|
24
|
+
export const slotDirectoryOf = () => {
|
|
25
|
+
const directory = join(machineDirectoryOf(), 'lanes');
|
|
26
|
+
mkdirSync(directory, { recursive: true });
|
|
27
|
+
return directory;
|
|
28
|
+
};
|
|
29
|
+
export const holderOf = async (name, pid = process.pid) => {
|
|
30
|
+
const startedAt = await startTimeOf(pid);
|
|
31
|
+
if (startedAt === undefined)
|
|
32
|
+
throw new DevEnvError(`Process ${pid} is not running.`);
|
|
33
|
+
return { pid, startedAt, name };
|
|
34
|
+
};
|
|
35
|
+
/*
|
|
36
|
+
* The memory Docker has, less the shared stack, in lanes, and no more lanes
|
|
37
|
+
* than the host's cores carry. DEV_ENV_LANE_SLOTS overrides both.
|
|
38
|
+
*/
|
|
39
|
+
export const slotCountOf = (override, dockerMemoryBytes, cores) => {
|
|
40
|
+
if (override !== undefined && override !== '') {
|
|
41
|
+
const count = Number(override);
|
|
42
|
+
if (!Number.isInteger(count) || count < 1)
|
|
43
|
+
throw new DevEnvError(`DEV_ENV_LANE_SLOTS must be a whole number above 0, not "${override}".`);
|
|
44
|
+
return count;
|
|
45
|
+
}
|
|
46
|
+
const byMemory = Math.floor((dockerMemoryBytes - SHARED_MEMORY_BYTES) / LANE_MEMORY_BYTES);
|
|
47
|
+
return Math.max(1, Math.min(byMemory, Math.floor(cores / CORES_PER_LANE)));
|
|
48
|
+
};
|
|
49
|
+
let slotCount;
|
|
50
|
+
export const slotCountOnMachine = () => {
|
|
51
|
+
slotCount ??= (async () => {
|
|
52
|
+
const { stdout } = await run('docker', [
|
|
53
|
+
'info',
|
|
54
|
+
'--format',
|
|
55
|
+
'{{.MemTotal}}',
|
|
56
|
+
]);
|
|
57
|
+
return slotCountOf(process.env.DEV_ENV_LANE_SLOTS, Number(stdout.trim()), availableParallelism());
|
|
58
|
+
})();
|
|
59
|
+
return slotCount;
|
|
60
|
+
};
|
|
61
|
+
const read = (path) => {
|
|
62
|
+
try {
|
|
63
|
+
return JSON.parse(readFileSync(path, 'utf8'));
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return undefined;
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
const isSame = (a, b) => a.pid === b.pid && a.startedAt === b.startedAt;
|
|
70
|
+
/*
|
|
71
|
+
* Written to a private file and linked into place, so a reader never finds a
|
|
72
|
+
* half-written holder and only one creator wins the name.
|
|
73
|
+
*/
|
|
74
|
+
const tryCreate = (path, holder) => {
|
|
75
|
+
const draft = `${path}.${holder.pid}.tmp`;
|
|
76
|
+
writeFileSync(draft, JSON.stringify(holder));
|
|
77
|
+
try {
|
|
78
|
+
linkSync(draft, path);
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
81
|
+
catch (error) {
|
|
82
|
+
if (error.code === 'EEXIST')
|
|
83
|
+
return false;
|
|
84
|
+
throw error;
|
|
85
|
+
}
|
|
86
|
+
finally {
|
|
87
|
+
rmSync(draft, { force: true });
|
|
88
|
+
}
|
|
89
|
+
};
|
|
90
|
+
/* Every record, with those of a holder that died, or whose pid was reused, removed. */
|
|
91
|
+
const liveRecords = async (directory) => {
|
|
92
|
+
const held = [];
|
|
93
|
+
for (const file of readdirSync(directory)) {
|
|
94
|
+
if (!file.endsWith('.json'))
|
|
95
|
+
continue;
|
|
96
|
+
const holder = read(join(directory, file));
|
|
97
|
+
if (holder === undefined)
|
|
98
|
+
continue;
|
|
99
|
+
if ((await startTimeOf(holder.pid)) === holder.startedAt)
|
|
100
|
+
held.push({ file, holder });
|
|
101
|
+
else
|
|
102
|
+
rmSync(join(directory, file), { force: true });
|
|
103
|
+
}
|
|
104
|
+
return held;
|
|
105
|
+
};
|
|
106
|
+
const describeHolders = (held) => [...new Map(held.map(({ holder }) => [holder.pid, holder])).values()]
|
|
107
|
+
.map(({ name, pid }) => `${name} (pid ${pid})`)
|
|
108
|
+
.join(', ');
|
|
109
|
+
/*
|
|
110
|
+
* Returns once the holder has a slot, or every slot when exclusive. While one
|
|
111
|
+
* holder waits to run alone, no one else takes a slot, so it waits only for
|
|
112
|
+
* the files already running. A holder waiting holds nothing, so two that both
|
|
113
|
+
* want every slot never wait on each other.
|
|
114
|
+
*/
|
|
115
|
+
export const takeSlot = async ({ directory, size, holder, exclusive = false, onWait = () => undefined, }) => {
|
|
116
|
+
let lastReason = '';
|
|
117
|
+
const wait = async (reason) => {
|
|
118
|
+
if (reason !== lastReason)
|
|
119
|
+
onWait(reason);
|
|
120
|
+
lastReason = reason;
|
|
121
|
+
await sleep(POLL_MS);
|
|
122
|
+
};
|
|
123
|
+
for (;;) {
|
|
124
|
+
const held = await liveRecords(directory);
|
|
125
|
+
const marker = held.find(({ file }) => file === EXCLUSIVE);
|
|
126
|
+
if (marker !== undefined && !isSame(marker.holder, holder)) {
|
|
127
|
+
await wait(`${describeHolders([marker])} is running alone`);
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
const slots = held.filter(({ file }) => file !== EXCLUSIVE);
|
|
131
|
+
const ours = slots.filter((slot) => isSame(slot.holder, holder));
|
|
132
|
+
if (!exclusive) {
|
|
133
|
+
if (ours.length > 0)
|
|
134
|
+
return;
|
|
135
|
+
const free = Array.from({ length: size }, (_, i) => `slot-${i}.json`)
|
|
136
|
+
.filter((file) => !slots.some((slot) => slot.file === file))
|
|
137
|
+
.find((file) => tryCreate(join(directory, file), holder));
|
|
138
|
+
if (free !== undefined)
|
|
139
|
+
return;
|
|
140
|
+
await wait(`all ${size} lane slots are held by ${describeHolders(slots)}`);
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
if (marker === undefined) {
|
|
144
|
+
tryCreate(join(directory, EXCLUSIVE), holder);
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
147
|
+
for (let i = 0; i < size; i++)
|
|
148
|
+
if (!slots.some((slot) => slot.file === `slot-${i}.json`))
|
|
149
|
+
tryCreate(join(directory, `slot-${i}.json`), holder);
|
|
150
|
+
const others = (await liveRecords(directory)).filter((record) => !isSame(record.holder, holder));
|
|
151
|
+
if (others.length === 0)
|
|
152
|
+
return;
|
|
153
|
+
await wait(`waiting to run alone until ${describeHolders(others)} finish`);
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
/** Synchronous, so it can run as the process exits. */
|
|
157
|
+
export const releaseSlots = (directory, holder) => {
|
|
158
|
+
for (const file of readdirSync(directory)) {
|
|
159
|
+
if (!file.endsWith('.json'))
|
|
160
|
+
continue;
|
|
161
|
+
const recorded = read(join(directory, file));
|
|
162
|
+
if (recorded !== undefined && isSame(recorded, holder))
|
|
163
|
+
rmSync(join(directory, file), { force: true });
|
|
164
|
+
}
|
|
165
|
+
};
|
package/dist/state.d.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { Mode } from './checkout.ts';
|
|
2
|
+
export interface RecordedProcess {
|
|
3
|
+
readonly pid: number;
|
|
4
|
+
readonly startedAt: string;
|
|
5
|
+
readonly logPath: string;
|
|
6
|
+
}
|
|
7
|
+
export interface EnvironmentState {
|
|
8
|
+
readonly environmentId: string;
|
|
9
|
+
readonly target: string;
|
|
10
|
+
readonly mode: Mode;
|
|
11
|
+
readonly ports: Readonly<Record<string, number>>;
|
|
12
|
+
readonly processes: Readonly<Record<string, RecordedProcess>>;
|
|
13
|
+
/** What the test that started it asked for, so a process it restarts runs with the same; absent for a dev environment. */
|
|
14
|
+
readonly settings?: Readonly<Record<string, string>>;
|
|
15
|
+
}
|
|
16
|
+
export declare const environmentDirectoryOf: (root: string, target: string, mode: Mode) => string;
|
|
17
|
+
export declare const readState: (root: string, target: string, mode: Mode) => EnvironmentState | undefined;
|
|
18
|
+
export declare const writeState: (root: string, state: EnvironmentState) => void;
|
|
19
|
+
export declare const dataDirectoryOf: (root: string, target: string, mode: Mode) => string;
|
|
20
|
+
export declare const removeState: (root: string, target: string, mode: Mode) => void;
|
|
21
|
+
export declare const readAllStates: (root: string) => readonly EnvironmentState[];
|
package/dist/state.js
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync, } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { environmentIdOf, environmentKeyOf } from './checkout.js';
|
|
4
|
+
export const environmentDirectoryOf = (root, target, mode) => join(root, '.dev-env', environmentKeyOf(target, mode));
|
|
5
|
+
const statePathOf = (root, target, mode) => join(environmentDirectoryOf(root, target, mode), 'state.json');
|
|
6
|
+
const parseState = (path, root) => {
|
|
7
|
+
if (!existsSync(path))
|
|
8
|
+
return undefined;
|
|
9
|
+
const state = JSON.parse(readFileSync(path, 'utf8'));
|
|
10
|
+
// A checkout that was moved or copied carries ports and pids it never owned.
|
|
11
|
+
return state.environmentId === environmentIdOf(root) ? state : undefined;
|
|
12
|
+
};
|
|
13
|
+
export const readState = (root, target, mode) => parseState(statePathOf(root, target, mode), root);
|
|
14
|
+
export const writeState = (root, state) => {
|
|
15
|
+
const path = statePathOf(root, state.target, state.mode);
|
|
16
|
+
mkdirSync(environmentDirectoryOf(root, state.target, state.mode), {
|
|
17
|
+
recursive: true,
|
|
18
|
+
});
|
|
19
|
+
writeFileSync(`${path}.tmp`, JSON.stringify(state, null, 2));
|
|
20
|
+
renameSync(`${path}.tmp`, path);
|
|
21
|
+
};
|
|
22
|
+
export const dataDirectoryOf = (root, target, mode) => join(environmentDirectoryOf(root, target, mode), 'data');
|
|
23
|
+
/* Gives up the environment's ports and its processes' data. Logs stay, to explain a run that just failed. */
|
|
24
|
+
export const removeState = (root, target, mode) => {
|
|
25
|
+
rmSync(dataDirectoryOf(root, target, mode), { recursive: true, force: true });
|
|
26
|
+
rmSync(statePathOf(root, target, mode), { force: true });
|
|
27
|
+
};
|
|
28
|
+
export const readAllStates = (root) => {
|
|
29
|
+
const directory = join(root, '.dev-env');
|
|
30
|
+
if (!existsSync(directory))
|
|
31
|
+
return [];
|
|
32
|
+
return readdirSync(directory).flatMap((key) => {
|
|
33
|
+
const state = parseState(join(directory, key, 'state.json'), root);
|
|
34
|
+
return state === undefined ? [] : [state];
|
|
35
|
+
});
|
|
36
|
+
};
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { EnvironmentInfo } from './environment.ts';
|
|
2
|
+
export interface TestEnvironment {
|
|
3
|
+
/** Unique on the machine; a process's telemetry carries it, to read a shared service by. */
|
|
4
|
+
readonly name: string;
|
|
5
|
+
readonly services: EnvironmentInfo['services'];
|
|
6
|
+
/** Removes the environment's containers and volumes as well as stopping it. */
|
|
7
|
+
readonly stop: () => Promise<void>;
|
|
8
|
+
/** Takes one service down, keeping its port and data, to trigger a failure. */
|
|
9
|
+
readonly stopService: (service: string) => Promise<void>;
|
|
10
|
+
readonly startService: (service: string) => Promise<void>;
|
|
11
|
+
}
|
|
12
|
+
export interface TestEnvironmentOptions {
|
|
13
|
+
readonly cwd?: string;
|
|
14
|
+
/**
|
|
15
|
+
* Which lane to use, so files that run side by side never share data. It
|
|
16
|
+
* defaults to Vitest's worker number, which stays the same for every file a
|
|
17
|
+
* worker runs, so each worker reuses its own lane.
|
|
18
|
+
*/
|
|
19
|
+
readonly lane?: number;
|
|
20
|
+
/** Process services this test does not need, left stopped. */
|
|
21
|
+
readonly without?: readonly string[];
|
|
22
|
+
/** Settings the target's manifest reads when it starts each process, such as raised limits. */
|
|
23
|
+
readonly settings?: Readonly<Record<string, string>>;
|
|
24
|
+
/**
|
|
25
|
+
* Takes every lane slot on the machine until the file ends, for a file
|
|
26
|
+
* whose results depend on timing. DEV_ENV_EXCLUSIVE=1 sets it, so a Vitest
|
|
27
|
+
* project can give it to its files.
|
|
28
|
+
*/
|
|
29
|
+
readonly exclusive?: boolean;
|
|
30
|
+
}
|
|
31
|
+
export declare const startTestEnvironment: (target: string, options?: TestEnvironmentOptions) => Promise<TestEnvironment>;
|
|
32
|
+
/** Removes every test lane of the checkout, or of one target, for the end of a run. */
|
|
33
|
+
export declare const resetTestEnvironments: (options?: {
|
|
34
|
+
readonly cwd?: string;
|
|
35
|
+
readonly target?: string;
|
|
36
|
+
}) => Promise<void>;
|