@elench/testkit 0.1.41 → 0.1.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/bin/testkit.mjs +6 -1
- package/lib/cli/index.mjs +4 -2
- package/lib/config/index.mjs +11 -0
- package/lib/runner/default-runtime-runner.mjs +24 -5
- package/lib/runner/execution-config.mjs +17 -0
- package/lib/runner/execution-config.test.mjs +8 -0
- package/lib/runner/lifecycle.mjs +99 -1
- package/lib/runner/orchestrator.mjs +7 -0
- package/lib/runner/planning.mjs +28 -6
- package/lib/runner/planning.test.mjs +38 -0
- package/lib/runner/playwright-config.mjs +5 -0
- package/lib/runner/playwright-config.test.mjs +6 -1
- package/lib/runner/playwright-runner.mjs +20 -4
- package/lib/runner/reporting.mjs +2 -0
- package/lib/runner/runtime-manager.mjs +90 -43
- package/lib/runner/runtime-manager.test.mjs +36 -11
- package/lib/runner/services.mjs +4 -2
- package/lib/runner/worker-loop.mjs +8 -2
- package/lib/setup/index.d.ts +1 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -83,6 +83,7 @@ export default defineTestkitSetup({
|
|
|
83
83
|
}),
|
|
84
84
|
runtime: {
|
|
85
85
|
instances: 1,
|
|
86
|
+
maxConcurrentTasks: 4,
|
|
86
87
|
},
|
|
87
88
|
requirements: {
|
|
88
89
|
files: [
|
|
@@ -134,6 +135,7 @@ for:
|
|
|
134
135
|
- per-file wall clock timeout budget
|
|
135
136
|
- multi-service graphs
|
|
136
137
|
- local runtime instance counts
|
|
138
|
+
- per-runtime concurrent task caps
|
|
137
139
|
- local DB binding configuration
|
|
138
140
|
- explicit per-file or per-suite locks
|
|
139
141
|
- migrate / seed commands
|
package/bin/testkit.mjs
CHANGED
package/lib/cli/index.mjs
CHANGED
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
} from "./args.mjs";
|
|
12
12
|
import * as runner from "../runner/index.mjs";
|
|
13
13
|
|
|
14
|
-
export function run() {
|
|
14
|
+
export async function run(argv = process.argv) {
|
|
15
15
|
const cli = cac("testkit");
|
|
16
16
|
|
|
17
17
|
cli
|
|
@@ -91,5 +91,7 @@ export function run() {
|
|
|
91
91
|
});
|
|
92
92
|
|
|
93
93
|
cli.help();
|
|
94
|
-
cli.parse();
|
|
94
|
+
const parsed = cli.parse(argv, { run: false });
|
|
95
|
+
await cli.runMatchedCommand();
|
|
96
|
+
return parsed;
|
|
95
97
|
}
|
package/lib/config/index.mjs
CHANGED
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
DEFAULT_FILE_TIMEOUT_SECONDS,
|
|
13
13
|
normalizeDatabaseBinding,
|
|
14
14
|
normalizeExecutionConfig,
|
|
15
|
+
normalizeRuntimeMaxConcurrentTasks,
|
|
15
16
|
normalizeRuntimeInstances,
|
|
16
17
|
} from "../runner/execution-config.mjs";
|
|
17
18
|
|
|
@@ -265,6 +266,7 @@ function normalizeRuntimeConfig(value, serviceName) {
|
|
|
265
266
|
if (!value) {
|
|
266
267
|
return {
|
|
267
268
|
instances: 1,
|
|
269
|
+
maxConcurrentTasks: Number.POSITIVE_INFINITY,
|
|
268
270
|
};
|
|
269
271
|
}
|
|
270
272
|
|
|
@@ -273,6 +275,10 @@ function normalizeRuntimeConfig(value, serviceName) {
|
|
|
273
275
|
value.instances ?? 1,
|
|
274
276
|
`Service "${serviceName}" runtime.instances`
|
|
275
277
|
),
|
|
278
|
+
maxConcurrentTasks: normalizeRuntimeMaxConcurrentTasks(
|
|
279
|
+
value.maxConcurrentTasks,
|
|
280
|
+
`Service "${serviceName}" runtime.maxConcurrentTasks`
|
|
281
|
+
),
|
|
276
282
|
};
|
|
277
283
|
}
|
|
278
284
|
|
|
@@ -578,6 +584,11 @@ function validateServiceConfig({
|
|
|
578
584
|
if (runtime.instances < 1) {
|
|
579
585
|
throw new Error(`Service "${name}" runtime.instances must be a positive integer`);
|
|
580
586
|
}
|
|
587
|
+
if (runtime.maxConcurrentTasks <= 0) {
|
|
588
|
+
throw new Error(
|
|
589
|
+
`Service "${name}" runtime.maxConcurrentTasks must be a positive integer when provided`
|
|
590
|
+
);
|
|
591
|
+
}
|
|
581
592
|
|
|
582
593
|
for (const depName of dependsOn || []) {
|
|
583
594
|
if (depName === name) {
|
|
@@ -12,6 +12,7 @@ import { determineDefaultRuntimeFailure } from "./default-runtime-errors.mjs";
|
|
|
12
12
|
import { RUNTIME_ARTIFACT_MARKER } from "../runtime-src/k6/artifacts.js";
|
|
13
13
|
import { readDatabaseUrl } from "./state-io.mjs";
|
|
14
14
|
import { buildTaskExecutionEnv } from "./template.mjs";
|
|
15
|
+
import { killChildProcess } from "./processes.mjs";
|
|
15
16
|
|
|
16
17
|
export async function runHttpK6Task(targetConfig, task, lifecycle, lease) {
|
|
17
18
|
const baseUrl = targetConfig.testkit.local?.baseUrl;
|
|
@@ -24,7 +25,9 @@ export async function runHttpK6Task(targetConfig, task, lifecycle, lease) {
|
|
|
24
25
|
serviceName: targetConfig.name,
|
|
25
26
|
sourceFile: path.join(targetConfig.productDir, task.file),
|
|
26
27
|
});
|
|
27
|
-
|
|
28
|
+
if (lifecycle.isStopRequested()) {
|
|
29
|
+
throw new Error(`testkit run interrupted before starting ${task.file}`);
|
|
30
|
+
}
|
|
28
31
|
return runDefaultRuntimeTask(
|
|
29
32
|
targetConfig,
|
|
30
33
|
task,
|
|
@@ -45,7 +48,9 @@ export async function runDalTask(targetConfig, task, lifecycle, lease) {
|
|
|
45
48
|
serviceName: targetConfig.name,
|
|
46
49
|
sourceFile: path.join(targetConfig.productDir, task.file),
|
|
47
50
|
});
|
|
48
|
-
|
|
51
|
+
if (lifecycle.isStopRequested()) {
|
|
52
|
+
throw new Error(`testkit run interrupted before starting ${task.file}`);
|
|
53
|
+
}
|
|
49
54
|
return runDefaultRuntimeTask(
|
|
50
55
|
targetConfig,
|
|
51
56
|
task,
|
|
@@ -74,11 +79,25 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
|
|
|
74
79
|
process.env
|
|
75
80
|
),
|
|
76
81
|
reject: false,
|
|
77
|
-
cancelSignal: lifecycle.signal,
|
|
78
82
|
forceKillAfterDelay: 5_000,
|
|
79
83
|
}
|
|
80
84
|
);
|
|
81
|
-
|
|
85
|
+
lifecycle.registerProcess(subprocess, () => {
|
|
86
|
+
killChildProcess(subprocess, "SIGINT");
|
|
87
|
+
});
|
|
88
|
+
if (lifecycle.isStopRequested()) {
|
|
89
|
+
const interruptSubprocess = () => killChildProcess(subprocess, "SIGINT");
|
|
90
|
+
if (subprocess.pid) interruptSubprocess();
|
|
91
|
+
else subprocess.once?.("spawn", interruptSubprocess);
|
|
92
|
+
}
|
|
93
|
+
console.log(`·· ${targetConfig.runtimeLabel}:${task.suiteName} → ${task.file}`);
|
|
94
|
+
let result;
|
|
95
|
+
let timedOut;
|
|
96
|
+
try {
|
|
97
|
+
({ result, timedOut } = await settleSubprocess(subprocess, fileTimeoutSeconds));
|
|
98
|
+
} finally {
|
|
99
|
+
lifecycle.unregisterProcess(subprocess.pid);
|
|
100
|
+
}
|
|
82
101
|
|
|
83
102
|
const stdout = parseDefaultRuntimeOutput(result.stdout || "");
|
|
84
103
|
const stderr = parseDefaultRuntimeOutput(result.stderr || "");
|
|
@@ -118,7 +137,7 @@ export async function settleSubprocess(subprocess, fileTimeoutSeconds) {
|
|
|
118
137
|
new Promise((resolve) => {
|
|
119
138
|
timeoutHandle = setTimeout(async () => {
|
|
120
139
|
timedOut = true;
|
|
121
|
-
subprocess
|
|
140
|
+
killChildProcess(subprocess, "SIGTERM");
|
|
122
141
|
const result = await subprocess.catch((error) => error);
|
|
123
142
|
resolve({ result, timedOut: true });
|
|
124
143
|
}, timeoutMs);
|
|
@@ -20,6 +20,23 @@ export function normalizeRuntimeInstances(value, label = "runtime.instances") {
|
|
|
20
20
|
return normalizePositiveInteger(value, label);
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
+
export function parseRuntimeMaxConcurrentTasksOption(
|
|
24
|
+
value,
|
|
25
|
+
label = "runtime.maxConcurrentTasks"
|
|
26
|
+
) {
|
|
27
|
+
return parsePositiveInteger(value, label);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function normalizeRuntimeMaxConcurrentTasks(
|
|
31
|
+
value,
|
|
32
|
+
label = "runtime.maxConcurrentTasks"
|
|
33
|
+
) {
|
|
34
|
+
if (value === undefined || value === null) {
|
|
35
|
+
return Number.POSITIVE_INFINITY;
|
|
36
|
+
}
|
|
37
|
+
return normalizePositiveInteger(value, label);
|
|
38
|
+
}
|
|
39
|
+
|
|
23
40
|
export function normalizeDatabaseBinding(value, label = "database.binding") {
|
|
24
41
|
const normalized = String(value || "").trim();
|
|
25
42
|
if (!DATABASE_BINDINGS.has(normalized)) {
|
|
@@ -5,8 +5,10 @@ import {
|
|
|
5
5
|
DEFAULT_FILE_TIMEOUT_SECONDS,
|
|
6
6
|
normalizeDatabaseBinding,
|
|
7
7
|
normalizeExecutionConfig,
|
|
8
|
+
normalizeRuntimeMaxConcurrentTasks,
|
|
8
9
|
normalizeRuntimeInstances,
|
|
9
10
|
parseFileTimeoutOption,
|
|
11
|
+
parseRuntimeMaxConcurrentTasksOption,
|
|
10
12
|
parseRuntimeInstancesOption,
|
|
11
13
|
parseWorkersOption,
|
|
12
14
|
resolveExecutionConfig,
|
|
@@ -16,9 +18,13 @@ describe("execution-config", () => {
|
|
|
16
18
|
it("parses worker and runtime-instance options", () => {
|
|
17
19
|
expect(parseWorkersOption("8")).toBe(8);
|
|
18
20
|
expect(parseRuntimeInstancesOption("2")).toBe(2);
|
|
21
|
+
expect(parseRuntimeMaxConcurrentTasksOption("4")).toBe(4);
|
|
19
22
|
expect(parseFileTimeoutOption("45")).toBe(45);
|
|
20
23
|
expect(() => parseWorkersOption("0")).toThrow('Invalid --workers value "0"');
|
|
21
24
|
expect(() => parseRuntimeInstancesOption("0")).toThrow('Invalid runtime.instances value "0"');
|
|
25
|
+
expect(() => parseRuntimeMaxConcurrentTasksOption("0")).toThrow(
|
|
26
|
+
'Invalid runtime.maxConcurrentTasks value "0"'
|
|
27
|
+
);
|
|
22
28
|
expect(() => parseFileTimeoutOption("0")).toThrow(
|
|
23
29
|
'Invalid --file-timeout-seconds value "0"'
|
|
24
30
|
);
|
|
@@ -49,6 +55,8 @@ describe("execution-config", () => {
|
|
|
49
55
|
|
|
50
56
|
it("normalizes runtime instances and database bindings", () => {
|
|
51
57
|
expect(normalizeRuntimeInstances(2)).toBe(2);
|
|
58
|
+
expect(normalizeRuntimeMaxConcurrentTasks(undefined)).toBe(Number.POSITIVE_INFINITY);
|
|
59
|
+
expect(normalizeRuntimeMaxConcurrentTasks(3)).toBe(3);
|
|
52
60
|
for (const binding of DATABASE_BINDINGS) {
|
|
53
61
|
expect(normalizeDatabaseBinding(binding)).toBe(binding);
|
|
54
62
|
}
|
package/lib/runner/lifecycle.mjs
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import fs from "fs";
|
|
2
2
|
import path from "path";
|
|
3
3
|
import crypto from "crypto";
|
|
4
|
+
import { execFileSync } from "child_process";
|
|
4
5
|
import { destroyRuntimeDatabase, cleanupOrphanedLocalInfrastructure } from "../database/index.mjs";
|
|
5
6
|
|
|
6
7
|
const RUN_SCHEMA_VERSION = 1;
|
|
7
8
|
const RUNS_DIRNAME = path.join(".testkit", "_runs");
|
|
8
9
|
const TERMINATION_TIMEOUT_MS = 5_000;
|
|
10
|
+
const SHUTDOWN_HOLD_TIMEOUT_MS = 10_000;
|
|
9
11
|
|
|
10
12
|
export function createRunLifecycle(productDir) {
|
|
11
13
|
const runId = buildRunId();
|
|
@@ -25,6 +27,9 @@ export function createRunLifecycle(productDir) {
|
|
|
25
27
|
runtimeStateDirs: [],
|
|
26
28
|
};
|
|
27
29
|
const signalListeners = [];
|
|
30
|
+
const managedProcesses = new Set();
|
|
31
|
+
let shutdownHold = null;
|
|
32
|
+
let shutdownHoldTimeout = null;
|
|
28
33
|
|
|
29
34
|
function persist() {
|
|
30
35
|
fs.mkdirSync(getRunsDir(productDir), { recursive: true });
|
|
@@ -36,6 +41,24 @@ export function createRunLifecycle(productDir) {
|
|
|
36
41
|
persist();
|
|
37
42
|
}
|
|
38
43
|
|
|
44
|
+
function ensureShutdownHold() {
|
|
45
|
+
if (shutdownHold) return;
|
|
46
|
+
shutdownHold = setInterval(() => {}, 1_000);
|
|
47
|
+
shutdownHoldTimeout = setTimeout(() => {
|
|
48
|
+
releaseShutdownHold();
|
|
49
|
+
}, SHUTDOWN_HOLD_TIMEOUT_MS);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function releaseShutdownHold() {
|
|
53
|
+
if (!shutdownHold) return;
|
|
54
|
+
clearInterval(shutdownHold);
|
|
55
|
+
shutdownHold = null;
|
|
56
|
+
if (shutdownHoldTimeout) {
|
|
57
|
+
clearTimeout(shutdownHoldTimeout);
|
|
58
|
+
shutdownHoldTimeout = null;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
39
62
|
const api = {
|
|
40
63
|
runId,
|
|
41
64
|
manifestPath,
|
|
@@ -55,9 +78,18 @@ export function createRunLifecycle(productDir) {
|
|
|
55
78
|
});
|
|
56
79
|
},
|
|
57
80
|
requestStop(reason = "interrupted") {
|
|
81
|
+
ensureShutdownHold();
|
|
58
82
|
if (!abortController.signal.aborted) {
|
|
59
83
|
abortController.abort(new Error(`testkit run interrupted (${reason})`));
|
|
60
84
|
}
|
|
85
|
+
for (const entry of managedProcesses) {
|
|
86
|
+
try {
|
|
87
|
+
entry.terminate?.();
|
|
88
|
+
} catch {
|
|
89
|
+
// Best-effort interruption only.
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
terminateDescendantProcesses(process.pid, "SIGINT");
|
|
61
93
|
mutate((draft) => {
|
|
62
94
|
draft.status = "interrupting";
|
|
63
95
|
draft.interruptReason = reason;
|
|
@@ -72,7 +104,22 @@ export function createRunLifecycle(productDir) {
|
|
|
72
104
|
}
|
|
73
105
|
});
|
|
74
106
|
},
|
|
75
|
-
|
|
107
|
+
registerProcess(child, terminate) {
|
|
108
|
+
if (!child) return;
|
|
109
|
+
managedProcesses.add({
|
|
110
|
+
child,
|
|
111
|
+
terminate,
|
|
112
|
+
});
|
|
113
|
+
},
|
|
114
|
+
unregisterProcess(childPid) {
|
|
115
|
+
for (const entry of managedProcesses) {
|
|
116
|
+
if (entry.child?.pid === childPid) {
|
|
117
|
+
managedProcesses.delete(entry);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
},
|
|
121
|
+
registerService(config, child, cwd, terminate) {
|
|
122
|
+
api.registerProcess(child, terminate);
|
|
76
123
|
const ports = collectConfigPorts(config);
|
|
77
124
|
mutate((draft) => {
|
|
78
125
|
draft.services = draft.services.filter((service) => service.pid !== child.pid);
|
|
@@ -89,6 +136,7 @@ export function createRunLifecycle(productDir) {
|
|
|
89
136
|
});
|
|
90
137
|
},
|
|
91
138
|
unregisterService(childPid) {
|
|
139
|
+
api.unregisterProcess(childPid);
|
|
92
140
|
mutate((draft) => {
|
|
93
141
|
draft.services = draft.services.filter((service) => service.pid !== childPid);
|
|
94
142
|
});
|
|
@@ -116,6 +164,9 @@ export function createRunLifecycle(productDir) {
|
|
|
116
164
|
removeManifest() {
|
|
117
165
|
removeManifestFile(productDir, runId);
|
|
118
166
|
},
|
|
167
|
+
dispose() {
|
|
168
|
+
releaseShutdownHold();
|
|
169
|
+
},
|
|
119
170
|
};
|
|
120
171
|
|
|
121
172
|
return api;
|
|
@@ -264,6 +315,53 @@ function killProcessGroup(pid, signal) {
|
|
|
264
315
|
}
|
|
265
316
|
}
|
|
266
317
|
|
|
318
|
+
function terminateDescendantProcesses(rootPid, signal) {
|
|
319
|
+
const descendants = listDescendantPids(rootPid);
|
|
320
|
+
for (const pid of descendants) {
|
|
321
|
+
try {
|
|
322
|
+
process.kill(pid, signal);
|
|
323
|
+
} catch (error) {
|
|
324
|
+
if (error?.code !== "ESRCH") {
|
|
325
|
+
// Best-effort interruption only.
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
function listDescendantPids(rootPid) {
|
|
332
|
+
try {
|
|
333
|
+
const output = execFileSync("ps", ["-eo", "pid=,ppid="], {
|
|
334
|
+
encoding: "utf8",
|
|
335
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
336
|
+
});
|
|
337
|
+
const childrenByParent = new Map();
|
|
338
|
+
for (const line of output.split(/\r?\n/)) {
|
|
339
|
+
const trimmed = line.trim();
|
|
340
|
+
if (!trimmed) continue;
|
|
341
|
+
const [pidRaw, parentRaw] = trimmed.split(/\s+/);
|
|
342
|
+
const pid = Number(pidRaw);
|
|
343
|
+
const parentPid = Number(parentRaw);
|
|
344
|
+
if (!Number.isInteger(pid) || !Number.isInteger(parentPid)) continue;
|
|
345
|
+
const siblings = childrenByParent.get(parentPid) || [];
|
|
346
|
+
siblings.push(pid);
|
|
347
|
+
childrenByParent.set(parentPid, siblings);
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
const descendants = [];
|
|
351
|
+
const stack = [...(childrenByParent.get(rootPid) || [])];
|
|
352
|
+
while (stack.length > 0) {
|
|
353
|
+
const pid = stack.pop();
|
|
354
|
+
descendants.push(pid);
|
|
355
|
+
for (const childPid of childrenByParent.get(pid) || []) {
|
|
356
|
+
stack.push(childPid);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
return descendants;
|
|
360
|
+
} catch {
|
|
361
|
+
return [];
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
|
|
267
365
|
async function waitForPidExit(pid, timeoutMs) {
|
|
268
366
|
const startedAt = Date.now();
|
|
269
367
|
while (Date.now() - startedAt < timeoutMs) {
|
|
@@ -106,6 +106,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
|
|
|
106
106
|
const executedPlans = servicePlans.filter((plan) => !plan.skipped);
|
|
107
107
|
let workerCount = 0;
|
|
108
108
|
let runtimeInstanceCount = 0;
|
|
109
|
+
let runtimeStats = [];
|
|
109
110
|
let exitCode = 0;
|
|
110
111
|
const lifecycle = createRunLifecycle(productDir);
|
|
111
112
|
lifecycle.markRunning();
|
|
@@ -155,6 +156,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
|
|
|
155
156
|
}
|
|
156
157
|
}
|
|
157
158
|
}
|
|
159
|
+
runtimeStats = runtimeManager.getStats();
|
|
158
160
|
} finally {
|
|
159
161
|
await runtimeManager.cleanupAll();
|
|
160
162
|
}
|
|
@@ -174,6 +176,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
|
|
|
174
176
|
execution,
|
|
175
177
|
workerCount,
|
|
176
178
|
runtimeInstanceCount,
|
|
179
|
+
runtimeStats,
|
|
177
180
|
typeValues,
|
|
178
181
|
suiteSelectors,
|
|
179
182
|
fileNames: requestedFiles,
|
|
@@ -205,6 +208,9 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
|
|
|
205
208
|
if (results.some((result) => result.failed)) exitCode = 1;
|
|
206
209
|
if (lifecycle.isStopRequested()) exitCode = Math.max(exitCode, 130);
|
|
207
210
|
} finally {
|
|
211
|
+
if (lifecycle.isStopRequested()) {
|
|
212
|
+
exitCode = Math.max(exitCode, 130);
|
|
213
|
+
}
|
|
208
214
|
lifecycle.removeSignalHandlers();
|
|
209
215
|
lifecycle.markFinished(
|
|
210
216
|
exitCode === 0 ? "finished" : lifecycle.isStopRequested() ? "interrupted" : "failed"
|
|
@@ -212,6 +218,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
|
|
|
212
218
|
await cleanupRunById(productDir, lifecycle.runId);
|
|
213
219
|
await cleanupRuns(productDir, { includeActive: false });
|
|
214
220
|
lifecycle.removeManifest();
|
|
221
|
+
lifecycle.dispose();
|
|
215
222
|
process.exitCode = exitCode;
|
|
216
223
|
}
|
|
217
224
|
}
|
package/lib/runner/planning.mjs
CHANGED
|
@@ -113,6 +113,10 @@ export function buildRuntimeGraphs(servicePlans) {
|
|
|
113
113
|
if (existing) {
|
|
114
114
|
existing.targetNames.push(plan.config.name);
|
|
115
115
|
existing.instanceCount = Math.max(existing.instanceCount, plan.config.testkit.runtime.instances);
|
|
116
|
+
existing.maxConcurrentTasks = Math.min(
|
|
117
|
+
existing.maxConcurrentTasks,
|
|
118
|
+
resolveGraphMaxConcurrentTasks(plan.runtimeConfigs)
|
|
119
|
+
);
|
|
116
120
|
continue;
|
|
117
121
|
}
|
|
118
122
|
|
|
@@ -123,6 +127,7 @@ export function buildRuntimeGraphs(servicePlans) {
|
|
|
123
127
|
targetNames: [plan.config.name],
|
|
124
128
|
dirName: buildGraphDirName(plan.runtimeNames),
|
|
125
129
|
instanceCount: plan.config.testkit.runtime.instances,
|
|
130
|
+
maxConcurrentTasks: resolveGraphMaxConcurrentTasks(plan.runtimeConfigs),
|
|
126
131
|
};
|
|
127
132
|
graphs.push(graph);
|
|
128
133
|
graphByRuntimeKey.set(plan.runtimeKey, graph);
|
|
@@ -166,6 +171,7 @@ export function buildTaskQueue(servicePlans, graphs, timings) {
|
|
|
166
171
|
orderIndex: suite.orderIndex,
|
|
167
172
|
file,
|
|
168
173
|
locks: resolveTaskLocks(plan.config, suite, file),
|
|
174
|
+
resourceCost: 1,
|
|
169
175
|
timingKey,
|
|
170
176
|
estimatedDurationMs: estimateTaskDuration(timings, timingKey, suite),
|
|
171
177
|
});
|
|
@@ -183,16 +189,25 @@ export function buildTaskQueue(servicePlans, graphs, timings) {
|
|
|
183
189
|
);
|
|
184
190
|
}
|
|
185
191
|
|
|
186
|
-
export function claimNextTask(queue, preferredGraphKey) {
|
|
192
|
+
export function claimNextTask(queue, preferredGraphKey, isRunnable = () => true) {
|
|
187
193
|
if (queue.length === 0) return null;
|
|
188
194
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
195
|
+
const preferredIndexes = [];
|
|
196
|
+
const fallbackIndexes = [];
|
|
197
|
+
for (let index = 0; index < queue.length; index += 1) {
|
|
198
|
+
if (preferredGraphKey && queue[index].graphKey === preferredGraphKey) {
|
|
199
|
+
preferredIndexes.push(index);
|
|
200
|
+
} else {
|
|
201
|
+
fallbackIndexes.push(index);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
for (const index of [...preferredIndexes, ...fallbackIndexes]) {
|
|
206
|
+
if (!isRunnable(queue[index])) continue;
|
|
207
|
+
return queue.splice(index, 1)[0];
|
|
192
208
|
}
|
|
193
|
-
if (index === -1) index = 0;
|
|
194
209
|
|
|
195
|
-
return
|
|
210
|
+
return null;
|
|
196
211
|
}
|
|
197
212
|
|
|
198
213
|
export function buildGraphDirName(runtimeNames) {
|
|
@@ -267,3 +282,10 @@ function normalizePathSeparators(filePath) {
|
|
|
267
282
|
function slugSegment(value) {
|
|
268
283
|
return value.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "") || "svc";
|
|
269
284
|
}
|
|
285
|
+
|
|
286
|
+
function resolveGraphMaxConcurrentTasks(runtimeConfigs) {
|
|
287
|
+
return runtimeConfigs.reduce(
|
|
288
|
+
(currentMin, config) => Math.min(currentMin, config.testkit.runtime.maxConcurrentTasks),
|
|
289
|
+
Number.POSITIVE_INFINITY
|
|
290
|
+
);
|
|
291
|
+
}
|
|
@@ -22,6 +22,7 @@ function makeConfig(name, extras = {}) {
|
|
|
22
22
|
},
|
|
23
23
|
runtime: providedTestkit.runtime || {
|
|
24
24
|
instances: 1,
|
|
25
|
+
maxConcurrentTasks: Number.POSITIVE_INFINITY,
|
|
25
26
|
},
|
|
26
27
|
requirements: providedTestkit.requirements || {
|
|
27
28
|
suites: [],
|
|
@@ -156,6 +157,7 @@ describe("runner-planning", () => {
|
|
|
156
157
|
testkit: {
|
|
157
158
|
runtime: {
|
|
158
159
|
instances: 3,
|
|
160
|
+
maxConcurrentTasks: 2,
|
|
159
161
|
},
|
|
160
162
|
requirements: {
|
|
161
163
|
suites: [
|
|
@@ -226,6 +228,7 @@ describe("runner-planning", () => {
|
|
|
226
228
|
testkit: {
|
|
227
229
|
runtime: {
|
|
228
230
|
instances: 2,
|
|
231
|
+
maxConcurrentTasks: 4,
|
|
229
232
|
},
|
|
230
233
|
},
|
|
231
234
|
});
|
|
@@ -234,6 +237,7 @@ describe("runner-planning", () => {
|
|
|
234
237
|
testkit: {
|
|
235
238
|
runtime: {
|
|
236
239
|
instances: 1,
|
|
240
|
+
maxConcurrentTasks: 2,
|
|
237
241
|
},
|
|
238
242
|
},
|
|
239
243
|
});
|
|
@@ -282,11 +286,13 @@ describe("runner-planning", () => {
|
|
|
282
286
|
expect.objectContaining({
|
|
283
287
|
key: "api",
|
|
284
288
|
instanceCount: 2,
|
|
289
|
+
maxConcurrentTasks: 4,
|
|
285
290
|
targetNames: ["api"],
|
|
286
291
|
}),
|
|
287
292
|
expect.objectContaining({
|
|
288
293
|
key: "api|frontend",
|
|
289
294
|
instanceCount: 1,
|
|
295
|
+
maxConcurrentTasks: 2,
|
|
290
296
|
targetNames: ["frontend"],
|
|
291
297
|
}),
|
|
292
298
|
]);
|
|
@@ -304,4 +310,36 @@ describe("runner-planning", () => {
|
|
|
304
310
|
graphKey: "api",
|
|
305
311
|
});
|
|
306
312
|
});
|
|
313
|
+
|
|
314
|
+
it("skips blocked preferred-graph work and claims the next runnable task", () => {
|
|
315
|
+
const queue = [
|
|
316
|
+
{
|
|
317
|
+
id: 1,
|
|
318
|
+
graphKey: "api",
|
|
319
|
+
file: "blocked.js",
|
|
320
|
+
},
|
|
321
|
+
{
|
|
322
|
+
id: 2,
|
|
323
|
+
graphKey: "api|frontend",
|
|
324
|
+
file: "runnable.js",
|
|
325
|
+
},
|
|
326
|
+
];
|
|
327
|
+
|
|
328
|
+
const claimed = claimNextTask(
|
|
329
|
+
queue,
|
|
330
|
+
"api",
|
|
331
|
+
(task) => task.file === "runnable.js"
|
|
332
|
+
);
|
|
333
|
+
|
|
334
|
+
expect(claimed).toMatchObject({
|
|
335
|
+
id: 2,
|
|
336
|
+
file: "runnable.js",
|
|
337
|
+
});
|
|
338
|
+
expect(queue).toEqual([
|
|
339
|
+
expect.objectContaining({
|
|
340
|
+
id: 1,
|
|
341
|
+
file: "blocked.js",
|
|
342
|
+
}),
|
|
343
|
+
]);
|
|
344
|
+
});
|
|
307
345
|
});
|
|
@@ -28,6 +28,9 @@ export function ensurePlaywrightTestConfig(targetConfig, cwd, requestedFiles, le
|
|
|
28
28
|
` testDir: ${JSON.stringify(cwd)},\n` +
|
|
29
29
|
` testMatch: ${JSON.stringify(normalizedFiles)},\n` +
|
|
30
30
|
` outputDir: ${JSON.stringify(outputDir)},\n` +
|
|
31
|
+
` workers: 1,\n` +
|
|
32
|
+
` fullyParallel: false,\n` +
|
|
33
|
+
` webServer: undefined,\n` +
|
|
31
34
|
`};\n`;
|
|
32
35
|
} else {
|
|
33
36
|
source =
|
|
@@ -35,6 +38,8 @@ export function ensurePlaywrightTestConfig(targetConfig, cwd, requestedFiles, le
|
|
|
35
38
|
` testDir: ${JSON.stringify(cwd)},\n` +
|
|
36
39
|
` testMatch: ${JSON.stringify(normalizedFiles)},\n` +
|
|
37
40
|
` outputDir: ${JSON.stringify(outputDir)},\n` +
|
|
41
|
+
` workers: 1,\n` +
|
|
42
|
+
` fullyParallel: false,\n` +
|
|
38
43
|
`};\n`;
|
|
39
44
|
}
|
|
40
45
|
|
|
@@ -31,7 +31,7 @@ describe("runner-playwright-config", () => {
|
|
|
31
31
|
fs.mkdirSync(cwd, { recursive: true });
|
|
32
32
|
fs.writeFileSync(
|
|
33
33
|
path.join(cwd, "playwright.config.mjs"),
|
|
34
|
-
"export default { outputDir: 'shared-test-results' };\n"
|
|
34
|
+
"export default { outputDir: 'shared-test-results', workers: 8, fullyParallel: true, webServer: { command: 'npm run dev', url: 'http://127.0.0.1:3000' } };\n"
|
|
35
35
|
);
|
|
36
36
|
|
|
37
37
|
const configPath = ensurePlaywrightTestConfig(
|
|
@@ -44,10 +44,15 @@ describe("runner-playwright-config", () => {
|
|
|
44
44
|
|
|
45
45
|
const expectedOutputDir = resolvePlaywrightOutputDir(leaseDir);
|
|
46
46
|
expect(generated.default.outputDir).toBe(expectedOutputDir);
|
|
47
|
+
expect(generated.default.workers).toBe(1);
|
|
48
|
+
expect(generated.default.fullyParallel).toBe(false);
|
|
49
|
+
expect(generated.default.webServer).toBeUndefined();
|
|
47
50
|
expect(fs.existsSync(expectedOutputDir)).toBe(true);
|
|
48
51
|
expect(fs.readFileSync(configPath, "utf8")).toContain(
|
|
49
52
|
`outputDir: ${JSON.stringify(expectedOutputDir)}`
|
|
50
53
|
);
|
|
54
|
+
expect(fs.readFileSync(configPath, "utf8")).toContain("workers: 1");
|
|
55
|
+
expect(fs.readFileSync(configPath, "utf8")).toContain("fullyParallel: false");
|
|
51
56
|
});
|
|
52
57
|
|
|
53
58
|
it("requires a lease-scoped directory", () => {
|
|
@@ -8,6 +8,7 @@ import { ensurePlaywrightTestConfig } from "./playwright-config.mjs";
|
|
|
8
8
|
import { printBufferedOutput } from "./processes.mjs";
|
|
9
9
|
import { normalizePathSeparators } from "./state.mjs";
|
|
10
10
|
import { buildPlaywrightEnv } from "./template.mjs";
|
|
11
|
+
import { killChildProcess } from "./processes.mjs";
|
|
11
12
|
|
|
12
13
|
export async function runPlaywrightTask(targetConfig, task, lifecycle, lease) {
|
|
13
14
|
const local = targetConfig.testkit.local;
|
|
@@ -17,11 +18,12 @@ export async function runPlaywrightTask(targetConfig, task, lifecycle, lease) {
|
|
|
17
18
|
);
|
|
18
19
|
}
|
|
19
20
|
|
|
20
|
-
console.log(`\n── ${targetConfig.runtimeLabel} playwright:${targetConfig.name} (${task.file}) ──`);
|
|
21
|
-
|
|
22
21
|
const cwd = resolveServiceCwd(targetConfig.productDir, local.cwd);
|
|
23
22
|
const requestedFile = path.relative(cwd, path.join(targetConfig.productDir, task.file));
|
|
24
23
|
const playwrightConfigPath = ensurePlaywrightTestConfig(targetConfig, cwd, [requestedFile], lease);
|
|
24
|
+
if (lifecycle.isStopRequested()) {
|
|
25
|
+
throw new Error(`testkit run interrupted before starting ${task.file}`);
|
|
26
|
+
}
|
|
25
27
|
const startedAt = Date.now();
|
|
26
28
|
const fileTimeoutSeconds = targetConfig.testkit.execution.fileTimeoutSeconds;
|
|
27
29
|
const subprocess = execa(
|
|
@@ -31,11 +33,25 @@ export async function runPlaywrightTask(targetConfig, task, lifecycle, lease) {
|
|
|
31
33
|
cwd,
|
|
32
34
|
env: buildPlaywrightEnv(targetConfig, local.baseUrl, lease, process.env),
|
|
33
35
|
reject: false,
|
|
34
|
-
cancelSignal: lifecycle.signal,
|
|
35
36
|
forceKillAfterDelay: 5_000,
|
|
36
37
|
}
|
|
37
38
|
);
|
|
38
|
-
|
|
39
|
+
lifecycle.registerProcess(subprocess, () => {
|
|
40
|
+
killChildProcess(subprocess, "SIGINT");
|
|
41
|
+
});
|
|
42
|
+
if (lifecycle.isStopRequested()) {
|
|
43
|
+
const interruptSubprocess = () => killChildProcess(subprocess, "SIGINT");
|
|
44
|
+
if (subprocess.pid) interruptSubprocess();
|
|
45
|
+
else subprocess.once?.("spawn", interruptSubprocess);
|
|
46
|
+
}
|
|
47
|
+
console.log(`\n── ${targetConfig.runtimeLabel} playwright:${targetConfig.name} (${task.file}) ──`);
|
|
48
|
+
let result;
|
|
49
|
+
let timedOut;
|
|
50
|
+
try {
|
|
51
|
+
({ result, timedOut } = await settleSubprocess(subprocess, fileTimeoutSeconds));
|
|
52
|
+
} finally {
|
|
53
|
+
lifecycle.unregisterProcess(subprocess.pid);
|
|
54
|
+
}
|
|
39
55
|
|
|
40
56
|
if (result.stderr) {
|
|
41
57
|
printBufferedOutput(result.stderr, `[${targetConfig.runtimeLabel}:${targetConfig.name}:playwright]`);
|
package/lib/runner/reporting.mjs
CHANGED
|
@@ -97,6 +97,7 @@ export function buildRunArtifact({
|
|
|
97
97
|
execution,
|
|
98
98
|
workerCount,
|
|
99
99
|
runtimeInstanceCount,
|
|
100
|
+
runtimeStats,
|
|
100
101
|
typeValues,
|
|
101
102
|
suiteSelectors,
|
|
102
103
|
fileNames,
|
|
@@ -138,6 +139,7 @@ export function buildRunArtifact({
|
|
|
138
139
|
fileTimeoutSeconds: execution.fileTimeoutSeconds,
|
|
139
140
|
workerCount,
|
|
140
141
|
runtimeInstanceCount,
|
|
142
|
+
runtimeStats: runtimeStats || [],
|
|
141
143
|
dbBackend,
|
|
142
144
|
types: typeValues,
|
|
143
145
|
suiteSelectors: suiteSelectors.map((selector) => selector.raw),
|
|
@@ -21,53 +21,56 @@ export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {}
|
|
|
21
21
|
};
|
|
22
22
|
|
|
23
23
|
return {
|
|
24
|
+
canAcquire(task) {
|
|
25
|
+
const pool = getPool(pools, graphByKey, task, productDir, lifecycle);
|
|
26
|
+
if (!locksAvailable(locks, task.locks || [])) {
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
return claimableRuntimeSlot(pool, task) !== null;
|
|
30
|
+
},
|
|
24
31
|
async acquire(task) {
|
|
25
32
|
const pool = getPool(pools, graphByKey, task, productDir, lifecycle);
|
|
33
|
+
if (lifecycle.isStopRequested()) {
|
|
34
|
+
throw lifecycle.signal.reason || new Error("testkit run interrupted");
|
|
35
|
+
}
|
|
26
36
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
throw lifecycle.signal.reason || new Error("testkit run interrupted");
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
const leaseId = `lease-${task.id}-${nextLeaseCounter}`;
|
|
33
|
-
nextLeaseCounter += 1;
|
|
37
|
+
const leaseId = `lease-${task.id}-${nextLeaseCounter}`;
|
|
38
|
+
nextLeaseCounter += 1;
|
|
34
39
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
}
|
|
40
|
+
if (!tryAcquireLocks(locks, task.locks || [], leaseId)) {
|
|
41
|
+
throw new Error(`Task ${task.id} was claimed before its locks were available`);
|
|
42
|
+
}
|
|
39
43
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
}
|
|
44
|
+
const slot = claimRuntimeSlot(pool, task);
|
|
45
|
+
if (!slot) {
|
|
46
|
+
releaseLocks(locks, task.locks || [], leaseId);
|
|
47
|
+
throw new Error(`Task ${task.id} was claimed before runtime capacity was available`);
|
|
48
|
+
}
|
|
46
49
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
50
|
+
try {
|
|
51
|
+
const context = await getReadyContext(slot, productDir, task, lifecycle, runtimeHooks);
|
|
52
|
+
const leaseDir = path.join(context.runtimeDir, "leases", leaseId);
|
|
53
|
+
fs.mkdirSync(leaseDir, { recursive: true });
|
|
54
|
+
return {
|
|
55
|
+
leaseId,
|
|
56
|
+
leaseDir,
|
|
57
|
+
lockNames: task.locks || [],
|
|
58
|
+
resourceCost: task.resourceCost || 1,
|
|
59
|
+
slot,
|
|
60
|
+
context,
|
|
61
|
+
};
|
|
62
|
+
} catch (error) {
|
|
63
|
+
releaseRuntimeSlot(slot, task);
|
|
64
|
+
releaseLocks(locks, task.locks || [], leaseId);
|
|
65
|
+
cleanupLeaseDir({ leaseDir: path.join(slot.context?.runtimeDir || productDir, "leases", leaseId) });
|
|
66
|
+
await drainRuntimeSlot(slot, lifecycle, runtimeHooks);
|
|
67
|
+
throw error;
|
|
65
68
|
}
|
|
66
69
|
},
|
|
67
70
|
async release(lease, options = {}) {
|
|
68
71
|
if (!lease?.slot) return;
|
|
69
72
|
releaseLocks(locks, lease.lockNames || [], lease.leaseId);
|
|
70
|
-
releaseRuntimeSlot(lease.slot);
|
|
73
|
+
releaseRuntimeSlot(lease.slot, { resourceCost: lease.resourceCost || 1 });
|
|
71
74
|
cleanupLeaseDir(lease);
|
|
72
75
|
if (options.invalidate) {
|
|
73
76
|
lease.slot.draining = true;
|
|
@@ -84,6 +87,23 @@ export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {}
|
|
|
84
87
|
}
|
|
85
88
|
}
|
|
86
89
|
},
|
|
90
|
+
getStats() {
|
|
91
|
+
return [...pools.values()]
|
|
92
|
+
.map((pool) => ({
|
|
93
|
+
graphKey: pool.slots[0]?.graph.key || null,
|
|
94
|
+
targetNames: [...(pool.slots[0]?.graph.targetNames || [])],
|
|
95
|
+
maxConcurrentTasks: Number.isFinite(pool.slots[0]?.graph.maxConcurrentTasks)
|
|
96
|
+
? pool.slots[0]?.graph.maxConcurrentTasks
|
|
97
|
+
: null,
|
|
98
|
+
runtimeCount: pool.slots.length,
|
|
99
|
+
runtimes: pool.slots.map((slot) => ({
|
|
100
|
+
runtimeId: slot.runtimeId,
|
|
101
|
+
peakLeaseCount: slot.peakLeaseCount,
|
|
102
|
+
peakResourceUnits: slot.peakResourceUnits,
|
|
103
|
+
})),
|
|
104
|
+
}))
|
|
105
|
+
.sort((left, right) => String(left.graphKey).localeCompare(String(right.graphKey)));
|
|
106
|
+
},
|
|
87
107
|
};
|
|
88
108
|
}
|
|
89
109
|
|
|
@@ -105,6 +125,9 @@ function getPool(pools, graphByKey, task, productDir, lifecycle) {
|
|
|
105
125
|
context: null,
|
|
106
126
|
contextPromise: null,
|
|
107
127
|
activeLeaseCount: 0,
|
|
128
|
+
activeResourceUnits: 0,
|
|
129
|
+
peakLeaseCount: 0,
|
|
130
|
+
peakResourceUnits: 0,
|
|
108
131
|
draining: false,
|
|
109
132
|
})),
|
|
110
133
|
};
|
|
@@ -112,21 +135,30 @@ function getPool(pools, graphByKey, task, productDir, lifecycle) {
|
|
|
112
135
|
return pool;
|
|
113
136
|
}
|
|
114
137
|
|
|
115
|
-
function claimRuntimeSlot(pool) {
|
|
138
|
+
function claimRuntimeSlot(pool, task) {
|
|
139
|
+
const resourceCost = task.resourceCost || 1;
|
|
116
140
|
const available = pool.slots.filter((slot) => !slot.draining);
|
|
117
141
|
if (available.length === 0) return null;
|
|
118
142
|
|
|
119
|
-
const slot = [...available]
|
|
120
|
-
(
|
|
121
|
-
|
|
122
|
-
left
|
|
123
|
-
|
|
143
|
+
const slot = [...available]
|
|
144
|
+
.filter((candidate) => slotHasCapacity(candidate, resourceCost))
|
|
145
|
+
.sort(
|
|
146
|
+
(left, right) =>
|
|
147
|
+
left.activeResourceUnits - right.activeResourceUnits ||
|
|
148
|
+
left.activeLeaseCount - right.activeLeaseCount ||
|
|
149
|
+
left.runtimeId.localeCompare(right.runtimeId)
|
|
150
|
+
)[0];
|
|
151
|
+
if (!slot) return null;
|
|
124
152
|
slot.activeLeaseCount += 1;
|
|
153
|
+
slot.activeResourceUnits += resourceCost;
|
|
154
|
+
slot.peakLeaseCount = Math.max(slot.peakLeaseCount, slot.activeLeaseCount);
|
|
155
|
+
slot.peakResourceUnits = Math.max(slot.peakResourceUnits, slot.activeResourceUnits);
|
|
125
156
|
return slot;
|
|
126
157
|
}
|
|
127
158
|
|
|
128
|
-
function releaseRuntimeSlot(slot) {
|
|
159
|
+
function releaseRuntimeSlot(slot, task = {}) {
|
|
129
160
|
slot.activeLeaseCount = Math.max(0, slot.activeLeaseCount - 1);
|
|
161
|
+
slot.activeResourceUnits = Math.max(0, slot.activeResourceUnits - (task.resourceCost || 1));
|
|
130
162
|
}
|
|
131
163
|
|
|
132
164
|
async function getReadyContext(slot, productDir, task, lifecycle, runtimeHooks) {
|
|
@@ -163,6 +195,21 @@ function tryAcquireLocks(lockMap, lockNames, leaseId) {
|
|
|
163
195
|
return true;
|
|
164
196
|
}
|
|
165
197
|
|
|
198
|
+
function locksAvailable(lockMap, lockNames) {
|
|
199
|
+
return [...new Set(lockNames)].every((lockName) => !lockMap.has(lockName));
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function claimableRuntimeSlot(pool, task) {
|
|
203
|
+
const resourceCost = task.resourceCost || 1;
|
|
204
|
+
return (
|
|
205
|
+
pool.slots.find((slot) => !slot.draining && slotHasCapacity(slot, resourceCost)) || null
|
|
206
|
+
);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
function slotHasCapacity(slot, resourceCost) {
|
|
210
|
+
return slot.activeResourceUnits + resourceCost <= slot.graph.maxConcurrentTasks;
|
|
211
|
+
}
|
|
212
|
+
|
|
166
213
|
function releaseLocks(lockMap, lockNames, leaseId) {
|
|
167
214
|
for (const lockName of [...new Set(lockNames)].sort()) {
|
|
168
215
|
if (lockMap.get(lockName) === leaseId) {
|
|
@@ -76,6 +76,7 @@ describe("runtime-manager", () => {
|
|
|
76
76
|
dirName: "api",
|
|
77
77
|
targetNames: ["api"],
|
|
78
78
|
instanceCount: 2,
|
|
79
|
+
maxConcurrentTasks: 1,
|
|
79
80
|
},
|
|
80
81
|
],
|
|
81
82
|
hooks: makeHooks(events),
|
|
@@ -93,7 +94,7 @@ describe("runtime-manager", () => {
|
|
|
93
94
|
await manager.cleanupAll();
|
|
94
95
|
});
|
|
95
96
|
|
|
96
|
-
it("
|
|
97
|
+
it("marks conflicting locks unavailable until the first lease releases", async () => {
|
|
97
98
|
const productDir = makeTempDir("testkit-runtime-manager-");
|
|
98
99
|
const manager = createRuntimeManager({
|
|
99
100
|
productDir,
|
|
@@ -104,24 +105,17 @@ describe("runtime-manager", () => {
|
|
|
104
105
|
dirName: "api",
|
|
105
106
|
targetNames: ["api"],
|
|
106
107
|
instanceCount: 1,
|
|
108
|
+
maxConcurrentTasks: 1,
|
|
107
109
|
},
|
|
108
110
|
],
|
|
109
111
|
hooks: makeHooks({ created: [], ready: [], cleaned: [] }),
|
|
110
112
|
});
|
|
111
113
|
|
|
112
114
|
const firstLease = await manager.acquire(makeTask(1, { locks: ["shared-lock"] }));
|
|
113
|
-
|
|
114
|
-
const secondPromise = manager.acquire(makeTask(2, { locks: ["shared-lock"] })).then((lease) => {
|
|
115
|
-
secondSettled = true;
|
|
116
|
-
return lease;
|
|
117
|
-
});
|
|
118
|
-
|
|
119
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
120
|
-
expect(secondSettled).toBe(false);
|
|
115
|
+
expect(manager.canAcquire(makeTask(2, { locks: ["shared-lock"] }))).toBe(false);
|
|
121
116
|
|
|
122
117
|
await manager.release(firstLease);
|
|
123
|
-
const secondLease = await
|
|
124
|
-
expect(secondSettled).toBe(true);
|
|
118
|
+
const secondLease = await manager.acquire(makeTask(2, { locks: ["shared-lock"] }));
|
|
125
119
|
|
|
126
120
|
await manager.release(secondLease);
|
|
127
121
|
await manager.cleanupAll();
|
|
@@ -139,6 +133,7 @@ describe("runtime-manager", () => {
|
|
|
139
133
|
dirName: "api",
|
|
140
134
|
targetNames: ["api"],
|
|
141
135
|
instanceCount: 1,
|
|
136
|
+
maxConcurrentTasks: 2,
|
|
142
137
|
},
|
|
143
138
|
],
|
|
144
139
|
hooks: makeHooks(events),
|
|
@@ -165,6 +160,7 @@ describe("runtime-manager", () => {
|
|
|
165
160
|
dirName: "api",
|
|
166
161
|
targetNames: ["api"],
|
|
167
162
|
instanceCount: 1,
|
|
163
|
+
maxConcurrentTasks: 1,
|
|
168
164
|
},
|
|
169
165
|
],
|
|
170
166
|
hooks: makeHooks({ created: [], ready: [], cleaned: [] }),
|
|
@@ -178,4 +174,33 @@ describe("runtime-manager", () => {
|
|
|
178
174
|
|
|
179
175
|
await manager.cleanupAll();
|
|
180
176
|
});
|
|
177
|
+
|
|
178
|
+
it("exposes runtime capacity through canAcquire", async () => {
|
|
179
|
+
const productDir = makeTempDir("testkit-runtime-manager-");
|
|
180
|
+
const manager = createRuntimeManager({
|
|
181
|
+
productDir,
|
|
182
|
+
lifecycle: makeLifecycle(),
|
|
183
|
+
graphs: [
|
|
184
|
+
{
|
|
185
|
+
key: "api",
|
|
186
|
+
dirName: "api",
|
|
187
|
+
targetNames: ["api"],
|
|
188
|
+
instanceCount: 1,
|
|
189
|
+
maxConcurrentTasks: 2,
|
|
190
|
+
},
|
|
191
|
+
],
|
|
192
|
+
hooks: makeHooks({ created: [], ready: [], cleaned: [] }),
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
expect(manager.canAcquire(makeTask(1))).toBe(true);
|
|
196
|
+
const leaseOne = await manager.acquire(makeTask(1));
|
|
197
|
+
expect(manager.canAcquire(makeTask(2))).toBe(true);
|
|
198
|
+
const leaseTwo = await manager.acquire(makeTask(2));
|
|
199
|
+
expect(manager.canAcquire(makeTask(3))).toBe(false);
|
|
200
|
+
|
|
201
|
+
await manager.release(leaseOne);
|
|
202
|
+
expect(manager.canAcquire(makeTask(3))).toBe(true);
|
|
203
|
+
await manager.release(leaseTwo);
|
|
204
|
+
await manager.cleanupAll();
|
|
205
|
+
});
|
|
181
206
|
});
|
package/lib/runner/services.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { resolveServiceCwd } from "../config/index.mjs";
|
|
2
2
|
import { buildExecutionEnv, numericPortFromUrl } from "./template.mjs";
|
|
3
3
|
import { DEFAULT_READY_TIMEOUT_MS, assertLocalServicePortsAvailable, isPortInUse, waitForReady } from "./readiness.mjs";
|
|
4
|
-
import { pipeOutput, startDetachedCommand, stopChildProcess, sleep } from "./processes.mjs";
|
|
4
|
+
import { killChildProcess, pipeOutput, startDetachedCommand, stopChildProcess, sleep } from "./processes.mjs";
|
|
5
5
|
import { readDatabaseUrl } from "./state-io.mjs";
|
|
6
6
|
|
|
7
7
|
export async function startLocalServices(runtimeConfigs, lifecycle) {
|
|
@@ -43,7 +43,9 @@ export async function startLocalService(config, lifecycle) {
|
|
|
43
43
|
pipeOutput(child.stdout, `[${config.runtimeLabel}:${config.name}]`),
|
|
44
44
|
pipeOutput(child.stderr, `[${config.runtimeLabel}:${config.name}]`),
|
|
45
45
|
];
|
|
46
|
-
lifecycle.registerService(config, child, cwd)
|
|
46
|
+
lifecycle.registerService(config, child, cwd, () => {
|
|
47
|
+
killChildProcess(child, "SIGTERM");
|
|
48
|
+
});
|
|
47
49
|
|
|
48
50
|
const readyTimeoutMs = config.testkit.local.readyTimeoutMs || DEFAULT_READY_TIMEOUT_MS;
|
|
49
51
|
|
|
@@ -32,8 +32,14 @@ export async function runWorker(
|
|
|
32
32
|
try {
|
|
33
33
|
while (true) {
|
|
34
34
|
if (lifecycle.isStopRequested()) break;
|
|
35
|
-
const task = claimNextTask(queue, worker.currentGraphKey)
|
|
36
|
-
|
|
35
|
+
const task = claimNextTask(queue, worker.currentGraphKey, (candidate) =>
|
|
36
|
+
runtimeManager.canAcquire(candidate)
|
|
37
|
+
);
|
|
38
|
+
if (!task) {
|
|
39
|
+
if (queue.length === 0) break;
|
|
40
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
37
43
|
|
|
38
44
|
let lease = null;
|
|
39
45
|
try {
|
package/lib/setup/index.d.ts
CHANGED