@elench/testkit 0.1.49 → 0.1.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -1
- package/lib/config/index.mjs +21 -5
- package/lib/database/index.mjs +105 -12
- package/lib/database/index.test.mjs +95 -0
- package/lib/runner/processes.mjs +16 -2
- package/lib/runner/processes.test.mjs +21 -0
- package/lib/runner/results.mjs +2 -1
- package/lib/runner/results.test.mjs +61 -0
- package/lib/runner/runtime-manager.mjs +34 -0
- package/lib/runner/runtime-manager.test.mjs +46 -0
- package/lib/runner/runtime-preparation.mjs +14 -0
- package/lib/runner/services.mjs +11 -1
- package/lib/runner/template-step-module-runner.mjs +25 -0
- package/lib/runner/template-steps.mjs +54 -45
- package/lib/runner/worker-loop.mjs +21 -0
- package/lib/setup/index.d.ts +14 -0
- package/lib/setup/index.mjs +12 -5
- package/lib/setup/index.test.mjs +34 -0
- package/lib/toolchains/index.mjs +565 -0
- package/lib/toolchains/index.test.mjs +168 -0
- package/lib/toolchains/semver.mjs +222 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -67,6 +67,7 @@ import {
|
|
|
67
67
|
localDatabase,
|
|
68
68
|
moduleStep,
|
|
69
69
|
nextService,
|
|
70
|
+
nodeToolchain,
|
|
70
71
|
service,
|
|
71
72
|
sqlFileStep,
|
|
72
73
|
tsxService,
|
|
@@ -85,6 +86,13 @@ export default defineTestkitSetup({
|
|
|
85
86
|
cacheTtlSeconds: 900,
|
|
86
87
|
},
|
|
87
88
|
},
|
|
89
|
+
toolchains: {
|
|
90
|
+
frontendNode: nodeToolchain({
|
|
91
|
+
cwd: "frontend",
|
|
92
|
+
detect: "auto",
|
|
93
|
+
install: "download",
|
|
94
|
+
}),
|
|
95
|
+
},
|
|
88
96
|
services: {
|
|
89
97
|
api: service({
|
|
90
98
|
...tsxService({
|
|
@@ -119,7 +127,7 @@ export default defineTestkitSetup({
|
|
|
119
127
|
...nextService({
|
|
120
128
|
cwd: "frontend",
|
|
121
129
|
port: 3000,
|
|
122
|
-
start: "
|
|
130
|
+
start: "./node_modules/.bin/next start --port {port}",
|
|
123
131
|
env: {
|
|
124
132
|
NEXT_DIST_DIR: "{prepareDir}/dist",
|
|
125
133
|
NEXT_PUBLIC_API_URL: "{baseUrl:api}",
|
|
@@ -130,6 +138,7 @@ export default defineTestkitSetup({
|
|
|
130
138
|
runtime: {
|
|
131
139
|
instances: 1,
|
|
132
140
|
maxConcurrentTasks: 2,
|
|
141
|
+
toolchain: "frontendNode",
|
|
133
142
|
prepare: {
|
|
134
143
|
inputs: ["frontend/src", "frontend/public", "frontend/package.json"],
|
|
135
144
|
steps: [commandStep("npm run build", { cwd: "frontend" })],
|
|
@@ -164,6 +173,7 @@ for:
|
|
|
164
173
|
- multi-service graphs
|
|
165
174
|
- local runtime instance counts
|
|
166
175
|
- per-runtime concurrent task caps
|
|
176
|
+
- repo-managed Node toolchains for prepare/start commands
|
|
167
177
|
- one-time runtime preparation steps for stable shared servers
|
|
168
178
|
- local DB binding configuration
|
|
169
179
|
- template database migrate / seed / verify stages
|
|
@@ -181,6 +191,41 @@ inputs, and writes cache state under the service runtime directory. This is the
|
|
|
181
191
|
right way to move expensive browser targets from `next dev` / watch mode to
|
|
182
192
|
stable build-and-start flows.
|
|
183
193
|
|
|
194
|
+
`runtime.toolchain` is the first-class way to make those prepare/start commands
|
|
195
|
+
run under the correct Node toolchain instead of whatever `node`/`npm` happened
|
|
196
|
+
to launch `testkit`. Node toolchains support:
|
|
197
|
+
|
|
198
|
+
- host verification mode: `install: "require-host"`
|
|
199
|
+
- cached repo-local provisioning mode: `install: "download"`
|
|
200
|
+
- auto-detection from:
|
|
201
|
+
- `package.json#volta.node`
|
|
202
|
+
- `.nvmrc`
|
|
203
|
+
- `.node-version`
|
|
204
|
+
- `.tool-versions` (`nodejs`)
|
|
205
|
+
- `package.json#engines.node`
|
|
206
|
+
- `package.json#volta.npm`
|
|
207
|
+
- `package.json#packageManager`
|
|
208
|
+
- `package.json#engines.npm`
|
|
209
|
+
|
|
210
|
+
Example:
|
|
211
|
+
|
|
212
|
+
```ts
|
|
213
|
+
toolchains: {
|
|
214
|
+
frontendNode: nodeToolchain({
|
|
215
|
+
cwd: "frontend",
|
|
216
|
+
detect: "auto",
|
|
217
|
+
install: "download",
|
|
218
|
+
}),
|
|
219
|
+
},
|
|
220
|
+
services: {
|
|
221
|
+
frontend: service({
|
|
222
|
+
runtime: {
|
|
223
|
+
toolchain: "frontendNode",
|
|
224
|
+
},
|
|
225
|
+
}),
|
|
226
|
+
}
|
|
227
|
+
```
|
|
228
|
+
|
|
184
229
|
If `reporting.knownFailuresFile` is configured, `testkit` enriches
|
|
185
230
|
`.testkit/results/latest.json` and `testkit.status.json` with:
|
|
186
231
|
|
package/lib/config/index.mjs
CHANGED
|
@@ -16,6 +16,10 @@ import {
|
|
|
16
16
|
normalizeRuntimeInstances,
|
|
17
17
|
} from "../runner/execution-config.mjs";
|
|
18
18
|
import { normalizeKnownFailureIssueValidationConfig } from "../known-failures/github.mjs";
|
|
19
|
+
import {
|
|
20
|
+
normalizeRuntimeToolchain,
|
|
21
|
+
normalizeToolchainRegistry,
|
|
22
|
+
} from "../toolchains/index.mjs";
|
|
19
23
|
|
|
20
24
|
const TESTKIT_K6_BIN = "TESTKIT_K6_BIN";
|
|
21
25
|
const DEFAULT_LOCAL_IMAGE = "pgvector/pgvector:pg16";
|
|
@@ -32,6 +36,7 @@ export async function loadConfigs(opts = {}) {
|
|
|
32
36
|
const { setup, setupFile } = await loadTestkitSetup(productDir);
|
|
33
37
|
const execution = normalizeRepoExecution(setup.execution);
|
|
34
38
|
const reporting = normalizeReportingConfig(setup.reporting);
|
|
39
|
+
const toolchains = normalizeToolchainRegistry(setup.toolchains);
|
|
35
40
|
const explicitServices = setup.services || {};
|
|
36
41
|
const discovery = discoverProject(productDir, explicitServices);
|
|
37
42
|
const serviceNames = new Set([
|
|
@@ -49,6 +54,7 @@ export async function loadConfigs(opts = {}) {
|
|
|
49
54
|
setupFile,
|
|
50
55
|
execution,
|
|
51
56
|
reporting,
|
|
57
|
+
toolchains,
|
|
52
58
|
explicitService: explicitServices[name] || {},
|
|
53
59
|
discoveredService: discovery.services[name] || null,
|
|
54
60
|
suites: discovery.suitesByService[name] || {},
|
|
@@ -109,6 +115,7 @@ function normalizeServiceConfig({
|
|
|
109
115
|
setupFile,
|
|
110
116
|
execution,
|
|
111
117
|
reporting,
|
|
118
|
+
toolchains,
|
|
112
119
|
explicitService,
|
|
113
120
|
discoveredService,
|
|
114
121
|
suites,
|
|
@@ -125,7 +132,7 @@ function normalizeServiceConfig({
|
|
|
125
132
|
);
|
|
126
133
|
}
|
|
127
134
|
const database = normalizeDatabaseConfig(explicitService, name);
|
|
128
|
-
const runtime = normalizeRuntimeConfig(explicitService.runtime, name);
|
|
135
|
+
const runtime = normalizeRuntimeConfig(explicitService.runtime, name, toolchains);
|
|
129
136
|
const skip = normalizeSkipConfig(explicitService.skip, {
|
|
130
137
|
name,
|
|
131
138
|
productDir,
|
|
@@ -212,7 +219,7 @@ function inferLocalRuntime(productDir, cwd) {
|
|
|
212
219
|
if (detectNextApp(absoluteCwd)) {
|
|
213
220
|
return {
|
|
214
221
|
cwd,
|
|
215
|
-
start: "
|
|
222
|
+
start: "./node_modules/.bin/next dev -p {port}",
|
|
216
223
|
port: 3000,
|
|
217
224
|
baseUrl: "http://127.0.0.1:{port}",
|
|
218
225
|
readyUrl: "http://127.0.0.1:{port}",
|
|
@@ -223,7 +230,7 @@ function inferLocalRuntime(productDir, cwd) {
|
|
|
223
230
|
if (fs.existsSync(path.join(absoluteCwd, "cmd", "server"))) {
|
|
224
231
|
return {
|
|
225
232
|
cwd,
|
|
226
|
-
start: "
|
|
233
|
+
start: "go run ./cmd/server",
|
|
227
234
|
port: 3000,
|
|
228
235
|
baseUrl: "http://127.0.0.1:{port}",
|
|
229
236
|
readyUrl: "http://127.0.0.1:{port}/health",
|
|
@@ -234,7 +241,7 @@ function inferLocalRuntime(productDir, cwd) {
|
|
|
234
241
|
if (fs.existsSync(path.join(absoluteCwd, "package.json")) && fs.existsSync(path.join(absoluteCwd, "src"))) {
|
|
235
242
|
return {
|
|
236
243
|
cwd,
|
|
237
|
-
start: "
|
|
244
|
+
start: "./node_modules/.bin/tsx watch src/index.ts",
|
|
238
245
|
port: 3000,
|
|
239
246
|
baseUrl: "http://127.0.0.1:{port}",
|
|
240
247
|
readyUrl: "http://127.0.0.1:{port}/health",
|
|
@@ -274,7 +281,7 @@ function normalizeDatabaseConfig(explicitService, serviceName) {
|
|
|
274
281
|
};
|
|
275
282
|
}
|
|
276
283
|
|
|
277
|
-
function normalizeRuntimeConfig(value, serviceName) {
|
|
284
|
+
function normalizeRuntimeConfig(value, serviceName, toolchains) {
|
|
278
285
|
if (!value) {
|
|
279
286
|
return {
|
|
280
287
|
instances: 1,
|
|
@@ -283,6 +290,7 @@ function normalizeRuntimeConfig(value, serviceName) {
|
|
|
283
290
|
inputs: [],
|
|
284
291
|
steps: [],
|
|
285
292
|
},
|
|
293
|
+
toolchain: null,
|
|
286
294
|
};
|
|
287
295
|
}
|
|
288
296
|
|
|
@@ -296,6 +304,11 @@ function normalizeRuntimeConfig(value, serviceName) {
|
|
|
296
304
|
`Service "${serviceName}" runtime.maxConcurrentTasks`
|
|
297
305
|
),
|
|
298
306
|
prepare: normalizeRuntimePrepareConfig(value.prepare, serviceName),
|
|
307
|
+
toolchain: normalizeRuntimeToolchain(
|
|
308
|
+
value.toolchain,
|
|
309
|
+
`Service "${serviceName}" runtime.toolchain`,
|
|
310
|
+
toolchains || {}
|
|
311
|
+
),
|
|
299
312
|
};
|
|
300
313
|
}
|
|
301
314
|
|
|
@@ -739,6 +752,9 @@ function validateServiceConfig({
|
|
|
739
752
|
if (local?.cwd) {
|
|
740
753
|
ensureExistingPath(productDir, local.cwd, `Service "${name}" local.cwd`);
|
|
741
754
|
}
|
|
755
|
+
if (runtime.toolchain?.cwd) {
|
|
756
|
+
ensureExistingPath(productDir, runtime.toolchain.cwd, `Service "${name}" runtime.toolchain.cwd`);
|
|
757
|
+
}
|
|
742
758
|
for (const [stageName, steps] of Object.entries(database?.template || {})) {
|
|
743
759
|
if (stageName === "inputs") continue;
|
|
744
760
|
for (const step of steps || []) {
|
package/lib/database/index.mjs
CHANGED
|
@@ -33,6 +33,8 @@ const LOCAL_PASSWORD = "testkit";
|
|
|
33
33
|
const LOCAL_ADMIN_DB = "postgres";
|
|
34
34
|
const LOCAL_READY_TIMEOUT_MS = 60_000;
|
|
35
35
|
const LOCAL_POLL_INTERVAL_MS = 1_000;
|
|
36
|
+
const LOCAL_DROP_DATABASE_TIMEOUT_MS = 15_000;
|
|
37
|
+
const LOCAL_DROP_DATABASE_POLL_INTERVAL_MS = 250;
|
|
36
38
|
|
|
37
39
|
export async function prepareDatabaseRuntime(config) {
|
|
38
40
|
const db = config.testkit.database;
|
|
@@ -398,18 +400,78 @@ async function cloneDatabaseFromTemplate(infra, dbName, templateDbName) {
|
|
|
398
400
|
}
|
|
399
401
|
|
|
400
402
|
async function dropDatabaseIfExists(infra, dbName) {
|
|
401
|
-
await
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
403
|
+
await dropDatabaseWithForceOrDrain(infra, dbName);
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
export async function dropDatabaseWithForceOrDrain(infra, dbName, hooks = {}) {
|
|
407
|
+
const runAdminQueryFn = hooks.runAdminQuery || runAdminQuery;
|
|
408
|
+
const databaseExistsFn = hooks.databaseExists || databaseExists;
|
|
409
|
+
const sleepFn = hooks.sleep || sleep;
|
|
410
|
+
|
|
411
|
+
try {
|
|
412
|
+
await runAdminQueryFn(infra, [
|
|
413
|
+
"-c",
|
|
414
|
+
`DROP DATABASE IF EXISTS "${escapeIdentifier(dbName)}" WITH (FORCE)`,
|
|
415
|
+
]);
|
|
416
|
+
return;
|
|
417
|
+
} catch (error) {
|
|
418
|
+
if (!isUnsupportedForceDropError(error)) {
|
|
419
|
+
throw error;
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
if (!(await databaseExistsFn(infra, dbName))) {
|
|
424
|
+
return;
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
let restoreConnections = false;
|
|
428
|
+
try {
|
|
429
|
+
await runAdminQueryFn(infra, [
|
|
430
|
+
"-c",
|
|
431
|
+
`ALTER DATABASE "${escapeIdentifier(dbName)}" WITH ALLOW_CONNECTIONS false`,
|
|
432
|
+
]);
|
|
433
|
+
restoreConnections = true;
|
|
434
|
+
await waitForDatabaseConnectionsToDrain(infra, dbName, {
|
|
435
|
+
runAdminQuery: runAdminQueryFn,
|
|
436
|
+
sleep: sleepFn,
|
|
437
|
+
});
|
|
438
|
+
await runAdminQueryFn(infra, [
|
|
439
|
+
"-c",
|
|
440
|
+
`DROP DATABASE IF EXISTS "${escapeIdentifier(dbName)}"`,
|
|
441
|
+
]);
|
|
442
|
+
restoreConnections = false;
|
|
443
|
+
} finally {
|
|
444
|
+
if (restoreConnections && (await databaseExistsFn(infra, dbName))) {
|
|
445
|
+
await runAdminQueryFn(infra, [
|
|
446
|
+
"-c",
|
|
447
|
+
`ALTER DATABASE "${escapeIdentifier(dbName)}" WITH ALLOW_CONNECTIONS true`,
|
|
448
|
+
]).catch(() => {
|
|
449
|
+
// Best-effort restoration for failed fallback drops.
|
|
450
|
+
});
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
export async function waitForDatabaseConnectionsToDrain(infra, dbName, hooks = {}) {
|
|
456
|
+
const runAdminQueryFn = hooks.runAdminQuery || runAdminQuery;
|
|
457
|
+
const sleepFn = hooks.sleep || sleep;
|
|
458
|
+
const now = hooks.now || Date.now;
|
|
459
|
+
const timeoutMs = hooks.timeoutMs ?? LOCAL_DROP_DATABASE_TIMEOUT_MS;
|
|
460
|
+
const deadline = now() + timeoutMs;
|
|
461
|
+
|
|
462
|
+
while (true) {
|
|
463
|
+
await terminateDatabaseConnections(infra, dbName, runAdminQueryFn);
|
|
464
|
+
const remainingConnections = await countDatabaseConnections(infra, dbName, runAdminQueryFn);
|
|
465
|
+
if (remainingConnections === 0) {
|
|
466
|
+
return;
|
|
467
|
+
}
|
|
468
|
+
if (now() >= deadline) {
|
|
469
|
+
throw new Error(
|
|
470
|
+
`Timed out waiting for database "${dbName}" connections to close (${remainingConnections} remaining)`
|
|
471
|
+
);
|
|
472
|
+
}
|
|
473
|
+
await sleepFn(LOCAL_DROP_DATABASE_POLL_INTERVAL_MS);
|
|
474
|
+
}
|
|
413
475
|
}
|
|
414
476
|
|
|
415
477
|
async function runAdminQuery(infra, args) {
|
|
@@ -431,6 +493,37 @@ async function runAdminQuery(infra, args) {
|
|
|
431
493
|
return stdout;
|
|
432
494
|
}
|
|
433
495
|
|
|
496
|
+
async function terminateDatabaseConnections(infra, dbName, runAdminQueryFn) {
|
|
497
|
+
await runAdminQueryFn(infra, [
|
|
498
|
+
"-c",
|
|
499
|
+
[
|
|
500
|
+
"SELECT pg_terminate_backend(pid)",
|
|
501
|
+
"FROM pg_stat_activity",
|
|
502
|
+
`WHERE datname = '${escapeSqlLiteral(dbName)}' AND pid <> pg_backend_pid();`,
|
|
503
|
+
].join(" "),
|
|
504
|
+
]);
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
async function countDatabaseConnections(infra, dbName, runAdminQueryFn) {
|
|
508
|
+
const result = await runAdminQueryFn(infra, [
|
|
509
|
+
"-tAc",
|
|
510
|
+
[
|
|
511
|
+
"SELECT COUNT(*)",
|
|
512
|
+
"FROM pg_stat_activity",
|
|
513
|
+
`WHERE datname = '${escapeSqlLiteral(dbName)}' AND pid <> pg_backend_pid();`,
|
|
514
|
+
].join(" "),
|
|
515
|
+
]);
|
|
516
|
+
return Number.parseInt(result.trim(), 10) || 0;
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
function isUnsupportedForceDropError(error) {
|
|
520
|
+
const text = `${error?.shortMessage || ""}\n${error?.stderr || ""}\n${error?.stdout || ""}\n${error?.message || ""}`;
|
|
521
|
+
return (
|
|
522
|
+
text.includes('syntax error at or near "WITH"') ||
|
|
523
|
+
text.includes('option "force"')
|
|
524
|
+
);
|
|
525
|
+
}
|
|
526
|
+
|
|
434
527
|
async function computeTemplateFingerprint(config) {
|
|
435
528
|
return computeTemplateFingerprintModel(config);
|
|
436
529
|
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import {
|
|
3
|
+
dropDatabaseWithForceOrDrain,
|
|
4
|
+
waitForDatabaseConnectionsToDrain,
|
|
5
|
+
} from "./index.mjs";
|
|
6
|
+
|
|
7
|
+
describe("database lifecycle helpers", () => {
|
|
8
|
+
it("uses DROP DATABASE ... WITH (FORCE) when supported", async () => {
|
|
9
|
+
const calls = [];
|
|
10
|
+
|
|
11
|
+
await dropDatabaseWithForceOrDrain(
|
|
12
|
+
{ containerName: "pg", password: "pw", user: "user" },
|
|
13
|
+
"demo",
|
|
14
|
+
{
|
|
15
|
+
async runAdminQuery(_infra, args) {
|
|
16
|
+
calls.push(args);
|
|
17
|
+
return "";
|
|
18
|
+
},
|
|
19
|
+
}
|
|
20
|
+
);
|
|
21
|
+
|
|
22
|
+
expect(calls).toEqual([["-c", 'DROP DATABASE IF EXISTS "demo" WITH (FORCE)']]);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it("falls back to draining connections when forced drop is unsupported", async () => {
|
|
26
|
+
const calls = [];
|
|
27
|
+
const counts = ["2", "0"];
|
|
28
|
+
|
|
29
|
+
await dropDatabaseWithForceOrDrain(
|
|
30
|
+
{ containerName: "pg", password: "pw", user: "user" },
|
|
31
|
+
"demo",
|
|
32
|
+
{
|
|
33
|
+
async runAdminQuery(_infra, args) {
|
|
34
|
+
calls.push(args);
|
|
35
|
+
if (args[1] === 'DROP DATABASE IF EXISTS "demo" WITH (FORCE)') {
|
|
36
|
+
const error = new Error('syntax error at or near "WITH"');
|
|
37
|
+
error.stderr = 'ERROR: syntax error at or near "WITH"';
|
|
38
|
+
throw error;
|
|
39
|
+
}
|
|
40
|
+
if (args[0] === "-tAc") {
|
|
41
|
+
return counts.shift() || "0";
|
|
42
|
+
}
|
|
43
|
+
return "";
|
|
44
|
+
},
|
|
45
|
+
async databaseExists() {
|
|
46
|
+
return true;
|
|
47
|
+
},
|
|
48
|
+
async sleep() {},
|
|
49
|
+
}
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
expect(calls).toEqual([
|
|
53
|
+
["-c", 'DROP DATABASE IF EXISTS "demo" WITH (FORCE)'],
|
|
54
|
+
["-c", 'ALTER DATABASE "demo" WITH ALLOW_CONNECTIONS false'],
|
|
55
|
+
[
|
|
56
|
+
"-c",
|
|
57
|
+
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'demo' AND pid <> pg_backend_pid();",
|
|
58
|
+
],
|
|
59
|
+
[
|
|
60
|
+
"-tAc",
|
|
61
|
+
"SELECT COUNT(*) FROM pg_stat_activity WHERE datname = 'demo' AND pid <> pg_backend_pid();",
|
|
62
|
+
],
|
|
63
|
+
[
|
|
64
|
+
"-c",
|
|
65
|
+
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'demo' AND pid <> pg_backend_pid();",
|
|
66
|
+
],
|
|
67
|
+
[
|
|
68
|
+
"-tAc",
|
|
69
|
+
"SELECT COUNT(*) FROM pg_stat_activity WHERE datname = 'demo' AND pid <> pg_backend_pid();",
|
|
70
|
+
],
|
|
71
|
+
["-c", 'DROP DATABASE IF EXISTS "demo"'],
|
|
72
|
+
]);
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
it("times out clearly while waiting for lingering connections to drain", async () => {
|
|
76
|
+
let now = 0;
|
|
77
|
+
await expect(() =>
|
|
78
|
+
waitForDatabaseConnectionsToDrain(
|
|
79
|
+
{ containerName: "pg", password: "pw", user: "user" },
|
|
80
|
+
"demo",
|
|
81
|
+
{
|
|
82
|
+
async runAdminQuery(_infra, args) {
|
|
83
|
+
if (args[0] === "-tAc") return "1";
|
|
84
|
+
return "";
|
|
85
|
+
},
|
|
86
|
+
async sleep() {
|
|
87
|
+
now += 10;
|
|
88
|
+
},
|
|
89
|
+
now: () => now,
|
|
90
|
+
timeoutMs: 20,
|
|
91
|
+
}
|
|
92
|
+
)
|
|
93
|
+
).rejects.toThrow(/Timed out waiting for database "demo" connections to close/);
|
|
94
|
+
});
|
|
95
|
+
});
|
package/lib/runner/processes.mjs
CHANGED
|
@@ -1,8 +1,22 @@
|
|
|
1
1
|
import { spawn } from "child_process";
|
|
2
2
|
|
|
3
|
+
export function normalizeServiceStartCommand(command) {
|
|
4
|
+
if (typeof command !== "string") {
|
|
5
|
+
throw new Error("Service start command must be a string");
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
const trimmed = command.trim();
|
|
9
|
+
if (trimmed.length === 0) {
|
|
10
|
+
throw new Error("Service start command must not be empty");
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
return trimmed.replace(/^exec\s+/u, "");
|
|
14
|
+
}
|
|
15
|
+
|
|
3
16
|
export function startDetachedCommand(command, cwd, env) {
|
|
17
|
+
const normalizedCommand = normalizeServiceStartCommand(command);
|
|
4
18
|
if (process.platform === "win32") {
|
|
5
|
-
return spawn(
|
|
19
|
+
return spawn(normalizedCommand, {
|
|
6
20
|
cwd,
|
|
7
21
|
env,
|
|
8
22
|
detached: true,
|
|
@@ -12,7 +26,7 @@ export function startDetachedCommand(command, cwd, env) {
|
|
|
12
26
|
}
|
|
13
27
|
|
|
14
28
|
const shell = process.env.SHELL || "/bin/sh";
|
|
15
|
-
return spawn(shell, ["-lc", `exec ${
|
|
29
|
+
return spawn(shell, ["-lc", `exec ${normalizedCommand}`], {
|
|
16
30
|
cwd,
|
|
17
31
|
env,
|
|
18
32
|
detached: true,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { normalizeServiceStartCommand } from "./processes.mjs";
|
|
3
|
+
|
|
4
|
+
describe("runner processes", () => {
|
|
5
|
+
it("strips a single leading exec prefix for backward compatibility", () => {
|
|
6
|
+
expect(normalizeServiceStartCommand("exec node server.mjs")).toBe("node server.mjs");
|
|
7
|
+
expect(normalizeServiceStartCommand(" exec ./node_modules/.bin/next dev -p {port} ")).toBe(
|
|
8
|
+
"./node_modules/.bin/next dev -p {port}"
|
|
9
|
+
);
|
|
10
|
+
});
|
|
11
|
+
|
|
12
|
+
it("leaves plain commands unchanged", () => {
|
|
13
|
+
expect(normalizeServiceStartCommand("node server.mjs")).toBe("node server.mjs");
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
it("rejects empty commands", () => {
|
|
17
|
+
expect(() => normalizeServiceStartCommand(" ")).toThrow(
|
|
18
|
+
/Service start command must not be empty/
|
|
19
|
+
);
|
|
20
|
+
});
|
|
21
|
+
});
|
package/lib/runner/results.mjs
CHANGED
|
@@ -114,7 +114,7 @@ export function recordTaskOutcome(trackers, task, outcome, finishedAt = Date.now
|
|
|
114
114
|
const normalizedPath = normalizePathSeparators(task.file);
|
|
115
115
|
const existingFileResult = suite.fileResultsByPath.get(normalizedPath);
|
|
116
116
|
const status = normalizeOutcomeStatus(outcome);
|
|
117
|
-
if (status !== "skipped") {
|
|
117
|
+
if (status !== "skipped" && status !== "not_run") {
|
|
118
118
|
suite.completedFileCount += 1;
|
|
119
119
|
}
|
|
120
120
|
if (existingFileResult) {
|
|
@@ -291,6 +291,7 @@ function formatFrameworkForArtifact(framework) {
|
|
|
291
291
|
}
|
|
292
292
|
|
|
293
293
|
function normalizeOutcomeStatus(outcome) {
|
|
294
|
+
if (outcome?.status === "not_run") return "not_run";
|
|
294
295
|
if (outcome?.status === "skipped") return "skipped";
|
|
295
296
|
return outcome?.failed ? "failed" : "passed";
|
|
296
297
|
}
|
|
@@ -307,6 +307,67 @@ describe("runner results", () => {
|
|
|
307
307
|
]);
|
|
308
308
|
});
|
|
309
309
|
|
|
310
|
+
it("preserves not-run task outcomes in suite and service summaries", () => {
|
|
311
|
+
const trackers = buildServiceTrackers(
|
|
312
|
+
[
|
|
313
|
+
{
|
|
314
|
+
skipped: false,
|
|
315
|
+
config: {
|
|
316
|
+
name: "api",
|
|
317
|
+
testkit: {
|
|
318
|
+
database: {
|
|
319
|
+
selectedBackend: null,
|
|
320
|
+
},
|
|
321
|
+
},
|
|
322
|
+
},
|
|
323
|
+
suites: [
|
|
324
|
+
{
|
|
325
|
+
name: "health",
|
|
326
|
+
type: "integration",
|
|
327
|
+
framework: "k6",
|
|
328
|
+
files: ["tests/a.int.testkit.ts", "tests/b.int.testkit.ts"],
|
|
329
|
+
orderIndex: 0,
|
|
330
|
+
},
|
|
331
|
+
],
|
|
332
|
+
},
|
|
333
|
+
],
|
|
334
|
+
1000
|
|
335
|
+
);
|
|
336
|
+
|
|
337
|
+
recordTaskOutcome(
|
|
338
|
+
trackers,
|
|
339
|
+
{
|
|
340
|
+
serviceName: "api",
|
|
341
|
+
suiteKey: "integration:health",
|
|
342
|
+
file: "tests/a.int.testkit.ts",
|
|
343
|
+
},
|
|
344
|
+
{
|
|
345
|
+
failed: false,
|
|
346
|
+
status: "not_run",
|
|
347
|
+
reason: "Graph initialization failed",
|
|
348
|
+
durationMs: 0,
|
|
349
|
+
error: null,
|
|
350
|
+
},
|
|
351
|
+
1000
|
|
352
|
+
);
|
|
353
|
+
|
|
354
|
+
const result = finalizeServiceResult(trackers.get("api"), 1000, 1200);
|
|
355
|
+
|
|
356
|
+
expect(result.completedFileCount).toBe(0);
|
|
357
|
+
expect(result.passedFileCount).toBe(0);
|
|
358
|
+
expect(result.failedFileCount).toBe(0);
|
|
359
|
+
expect(result.skippedFileCount).toBe(0);
|
|
360
|
+
expect(result.notRunFileCount).toBe(2);
|
|
361
|
+
expect(result.suites[0].files).toContainEqual({
|
|
362
|
+
path: "tests/a.int.testkit.ts",
|
|
363
|
+
failed: false,
|
|
364
|
+
status: "not_run",
|
|
365
|
+
durationMs: 0,
|
|
366
|
+
error: null,
|
|
367
|
+
reason: "Graph initialization failed",
|
|
368
|
+
});
|
|
369
|
+
});
|
|
370
|
+
|
|
310
371
|
it("summarizes mixed db backends", () => {
|
|
311
372
|
expect(
|
|
312
373
|
summarizeDbBackend([{ dbBackend: "local" }, { dbBackend: "neon" }])
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import fs from "fs";
|
|
2
2
|
import path from "path";
|
|
3
3
|
import { buildRuntimeIds } from "./execution-config.mjs";
|
|
4
|
+
import { formatError } from "./formatting.mjs";
|
|
4
5
|
import {
|
|
5
6
|
cleanupRuntimeInstanceContext,
|
|
6
7
|
createRuntimeInstanceContext,
|
|
@@ -11,6 +12,7 @@ export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {}
|
|
|
11
12
|
const graphByKey = new Map(graphs.map((graph) => [graph.key, graph]));
|
|
12
13
|
const pools = new Map();
|
|
13
14
|
const locks = new Map();
|
|
15
|
+
const failedGraphs = new Map();
|
|
14
16
|
let nextLeaseCounter = 1;
|
|
15
17
|
const runtimeHooks = {
|
|
16
18
|
cleanupRuntimeInstanceContext,
|
|
@@ -22,6 +24,9 @@ export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {}
|
|
|
22
24
|
|
|
23
25
|
return {
|
|
24
26
|
canAcquire(task) {
|
|
27
|
+
if (failedGraphs.has(task.graphKey)) {
|
|
28
|
+
return false;
|
|
29
|
+
}
|
|
25
30
|
const pool = getPool(pools, graphByKey, task, productDir, lifecycle);
|
|
26
31
|
if (!locksAvailable(locks, task.locks || [])) {
|
|
27
32
|
return false;
|
|
@@ -64,6 +69,9 @@ export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {}
|
|
|
64
69
|
releaseLocks(locks, task.locks || [], leaseId);
|
|
65
70
|
cleanupLeaseDir({ leaseDir: path.join(slot.context?.runtimeDir || productDir, "leases", leaseId) });
|
|
66
71
|
await drainRuntimeSlot(slot, lifecycle, runtimeHooks);
|
|
72
|
+
if (!lifecycle.isStopRequested()) {
|
|
73
|
+
markGraphFailed(failedGraphs, task.graphKey, formatError(error));
|
|
74
|
+
}
|
|
67
75
|
throw error;
|
|
68
76
|
}
|
|
69
77
|
},
|
|
@@ -104,6 +112,27 @@ export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {}
|
|
|
104
112
|
}))
|
|
105
113
|
.sort((left, right) => String(left.graphKey).localeCompare(String(right.graphKey)));
|
|
106
114
|
},
|
|
115
|
+
drainFailedTasks(queue) {
|
|
116
|
+
const drained = [];
|
|
117
|
+
for (let index = queue.length - 1; index >= 0; index -= 1) {
|
|
118
|
+
const task = queue[index];
|
|
119
|
+
const message = failedGraphs.get(task.graphKey);
|
|
120
|
+
if (!message) continue;
|
|
121
|
+
|
|
122
|
+
queue.splice(index, 1);
|
|
123
|
+
drained.push({
|
|
124
|
+
task,
|
|
125
|
+
reason: `Not run because runtime graph failed to initialize: ${message}`,
|
|
126
|
+
message,
|
|
127
|
+
graph: graphByKey.get(task.graphKey) || {
|
|
128
|
+
key: task.graphKey,
|
|
129
|
+
targetNames: [task.targetName],
|
|
130
|
+
},
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
return drained.reverse();
|
|
135
|
+
},
|
|
107
136
|
};
|
|
108
137
|
}
|
|
109
138
|
|
|
@@ -218,6 +247,11 @@ function releaseLocks(lockMap, lockNames, leaseId) {
|
|
|
218
247
|
}
|
|
219
248
|
}
|
|
220
249
|
|
|
250
|
+
function markGraphFailed(failedGraphs, graphKey, message) {
|
|
251
|
+
if (!graphKey || failedGraphs.has(graphKey)) return;
|
|
252
|
+
failedGraphs.set(graphKey, message);
|
|
253
|
+
}
|
|
254
|
+
|
|
221
255
|
function sleep(ms) {
|
|
222
256
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
223
257
|
}
|
|
@@ -203,4 +203,50 @@ describe("runtime-manager", () => {
|
|
|
203
203
|
await manager.release(leaseTwo);
|
|
204
204
|
await manager.cleanupAll();
|
|
205
205
|
});
|
|
206
|
+
|
|
207
|
+
it("blocks a graph after fatal runtime initialization failure and drains remaining tasks", async () => {
|
|
208
|
+
const productDir = makeTempDir("testkit-runtime-manager-");
|
|
209
|
+
const manager = createRuntimeManager({
|
|
210
|
+
productDir,
|
|
211
|
+
lifecycle: makeLifecycle(),
|
|
212
|
+
graphs: [
|
|
213
|
+
{
|
|
214
|
+
key: "api",
|
|
215
|
+
dirName: "api",
|
|
216
|
+
targetNames: ["api"],
|
|
217
|
+
instanceCount: 1,
|
|
218
|
+
maxConcurrentTasks: 1,
|
|
219
|
+
},
|
|
220
|
+
],
|
|
221
|
+
hooks: {
|
|
222
|
+
...makeHooks({ created: [], ready: [], cleaned: [] }),
|
|
223
|
+
async ensureRuntimeInstanceReady() {
|
|
224
|
+
throw new Error("prepare failed");
|
|
225
|
+
},
|
|
226
|
+
},
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
const firstTask = makeTask(1, { file: "tests/a.int.testkit.ts", targetName: "api" });
|
|
230
|
+
const secondTask = makeTask(2, { file: "tests/b.int.testkit.ts", targetName: "api" });
|
|
231
|
+
|
|
232
|
+
await expect(manager.acquire(firstTask)).rejects.toThrow(/prepare failed/);
|
|
233
|
+
expect(manager.canAcquire(secondTask)).toBe(false);
|
|
234
|
+
|
|
235
|
+
const queue = [secondTask];
|
|
236
|
+
expect(manager.drainFailedTasks(queue)).toEqual([
|
|
237
|
+
{
|
|
238
|
+
task: secondTask,
|
|
239
|
+
reason: "Not run because runtime graph failed to initialize: prepare failed",
|
|
240
|
+
message: "prepare failed",
|
|
241
|
+
graph: {
|
|
242
|
+
key: "api",
|
|
243
|
+
dirName: "api",
|
|
244
|
+
targetNames: ["api"],
|
|
245
|
+
instanceCount: 1,
|
|
246
|
+
maxConcurrentTasks: 1,
|
|
247
|
+
},
|
|
248
|
+
},
|
|
249
|
+
]);
|
|
250
|
+
expect(queue).toEqual([]);
|
|
251
|
+
});
|
|
206
252
|
});
|