@elench/testkit 0.1.11 → 0.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -28,6 +28,13 @@ npx @elench/testkit e2e
28
28
  npx @elench/testkit --framework playwright
29
29
  npx @elench/testkit --framework k6
30
30
 
31
+ # Parallelize with isolated worker stacks
32
+ npx @elench/testkit --jobs 3
33
+
34
+ # Run a deterministic shard
35
+ npx @elench/testkit --shard 1/3
36
+ npx @elench/testkit --jobs 2 --shard 2/3
37
+
31
38
  # Specific service / suite
32
39
  npx @elench/testkit frontend e2e -s auth
33
40
  npx @elench/testkit bourne int -s health
@@ -41,10 +48,11 @@ npx @elench/testkit destroy
41
48
 
42
49
  1. **Discovery** — reads `runner.manifest.json` for services, suites, files, and frameworks
43
50
  2. **Config** — reads `testkit.config.json` for local runtime, migration, dependency, and database settings
51
+ Per-service `.env` files declared in config are loaded when present.
44
52
  3. **Database** — provisions a Neon branch when a service declares one
45
53
  4. **Seed** — runs optional product seed commands against the provisioned database
46
54
  5. **Runtime** — starts required local services, waits for readiness, and injects test env
47
- 6. **Execution** — runs `k6` suites file-by-file and Playwright suites suite-by-suite
55
+ 6. **Execution** — distributes suites across isolated worker stacks, runs `k6` suites file-by-file, and runs Playwright suites suite-by-suite
48
56
  7. **Cleanup** — stops local processes and preserves `.testkit/` state for inspection or later destroy
49
57
 
50
58
  ## File roles
@@ -52,6 +60,42 @@ npx @elench/testkit destroy
52
60
  - `runner.manifest.json`: canonical test inventory
53
61
  - `testkit.config.json`: local execution and provisioning config
54
62
 
63
+ `testkit.config.json` can also declare:
64
+
65
+ - `envFile` / `envFiles` for service-specific environment loading
66
+ - `databaseFrom` for dependency services that must share another service's `DATABASE_URL`
67
+
68
+ ## Parallel execution
69
+
70
+ `@elench/testkit` can run suites in parallel with `--jobs <n>`.
71
+
72
+ Each worker gets its own:
73
+ - Neon branch
74
+ - `.testkit` state subtree
75
+ - local service ports
76
+
77
+ This keeps suites isolated while still reusing one stack per worker across multiple assigned suites.
78
+
79
+ Use `--shard <i/n>` to split the selected suite set deterministically before worker scheduling.
80
+
81
+ ## Suite metadata
82
+
83
+ `runner.manifest.json` remains the source of truth for suites. Optional per-suite `testkit` metadata can tune execution:
84
+
85
+ ```json
86
+ {
87
+ "name": "health",
88
+ "files": ["tests/example.js"],
89
+ "testkit": {
90
+ "maxFileConcurrency": 2,
91
+ "weight": 3
92
+ }
93
+ }
94
+ ```
95
+
96
+ - `maxFileConcurrency`: k6-only opt-in for running files within the suite concurrently
97
+ - `weight`: optional scheduling weight when distributing suites across workers
98
+
55
99
  ## Schema
56
100
 
57
101
  See [testkit-config-schema.md](testkit-config-schema.md).
package/lib/cli.mjs CHANGED
@@ -13,6 +13,10 @@ export function run() {
13
13
  .command("[first] [second] [third]", "Run test suites (int, e2e, dal, all)")
14
14
  .option("-s, --suite <name>", "Run specific suite(s)", { default: [] })
15
15
  .option("--dir <path>", "Explicit product directory")
16
+ .option("--jobs <n>", "Number of isolated worker stacks per service", {
17
+ default: "1",
18
+ })
19
+ .option("--shard <i/n>", "Run only shard i of n at suite granularity")
16
20
  .option("--framework <name>", "Filter by framework (k6, playwright, all)", {
17
21
  default: "all",
18
22
  })
@@ -84,9 +88,42 @@ export function run() {
84
88
  );
85
89
  }
86
90
 
91
+ const jobs = Number.parseInt(String(options.jobs), 10);
92
+ if (!Number.isInteger(jobs) || jobs <= 0) {
93
+ throw new Error(`Invalid --jobs value "${options.jobs}". Expected a positive integer.`);
94
+ }
95
+
96
+ let shard = null;
97
+ if (options.shard) {
98
+ const match = String(options.shard).match(/^(\d+)\/(\d+)$/);
99
+ if (!match) {
100
+ throw new Error(
101
+ `Invalid --shard value "${options.shard}". Expected the form "i/n", e.g. 1/3.`
102
+ );
103
+ }
104
+ const index = Number.parseInt(match[1], 10);
105
+ const total = Number.parseInt(match[2], 10);
106
+ if (index <= 0 || total <= 0 || index > total) {
107
+ throw new Error(
108
+ `Invalid --shard value "${options.shard}". Expected 1 <= i <= n.`
109
+ );
110
+ }
111
+ shard = { index, total };
112
+ }
113
+
87
114
  const suiteType = type || "all";
88
115
  const suiteNames = Array.isArray(options.suite) ? options.suite : [options.suite].filter(Boolean);
89
- await runner.runAll(configs, suiteType, suiteNames, options, allConfigs);
116
+ await runner.runAll(
117
+ configs,
118
+ suiteType,
119
+ suiteNames,
120
+ {
121
+ ...options,
122
+ jobs,
123
+ shard,
124
+ },
125
+ allConfigs
126
+ );
90
127
  });
91
128
 
92
129
  cli.help();
package/lib/config.mjs CHANGED
@@ -68,13 +68,17 @@ export function loadConfigs(opts = {}) {
68
68
  }
69
69
 
70
70
  validateMergedService(name, runnerService, serviceConfig, productDir);
71
+ const serviceEnv = loadServiceEnv(productDir, serviceConfig);
71
72
 
72
73
  return {
73
74
  name,
74
75
  productDir,
75
76
  stateDir: path.join(productDir, ".testkit", name),
76
77
  suites: runnerService.suites,
77
- testkit: serviceConfig,
78
+ testkit: {
79
+ ...serviceConfig,
80
+ serviceEnv,
81
+ },
78
82
  };
79
83
  });
80
84
  }
@@ -161,6 +165,31 @@ function loadRunnerManifest(productDir) {
161
165
  `Service "${serviceName}" suite "${suite.name}" uses unsupported framework "${framework}"`
162
166
  );
163
167
  }
168
+
169
+ if (suite.testkit !== undefined) {
170
+ if (!isObject(suite.testkit)) {
171
+ throw new Error(
172
+ `Service "${serviceName}" suite "${suite.name}" testkit config must be an object`
173
+ );
174
+ }
175
+ if (
176
+ suite.testkit.maxFileConcurrency !== undefined &&
177
+ (!Number.isInteger(suite.testkit.maxFileConcurrency) ||
178
+ suite.testkit.maxFileConcurrency <= 0)
179
+ ) {
180
+ throw new Error(
181
+ `Service "${serviceName}" suite "${suite.name}" testkit.maxFileConcurrency must be a positive integer`
182
+ );
183
+ }
184
+ if (
185
+ suite.testkit.weight !== undefined &&
186
+ (!Number.isInteger(suite.testkit.weight) || suite.testkit.weight <= 0)
187
+ ) {
188
+ throw new Error(
189
+ `Service "${serviceName}" suite "${suite.name}" testkit.weight must be a positive integer`
190
+ );
191
+ }
192
+ }
164
193
  }
165
194
  }
166
195
  }
@@ -203,6 +232,20 @@ function validateConfigCoverage(runner, config) {
203
232
  );
204
233
  }
205
234
  }
235
+
236
+ const databaseFrom = config.services[serviceName].databaseFrom;
237
+ if (databaseFrom) {
238
+ if (!config.services[databaseFrom]) {
239
+ throw new Error(
240
+ `Service "${serviceName}" databaseFrom "${databaseFrom}" is missing from ${TESTKIT_CONFIG}`
241
+ );
242
+ }
243
+ if (!runner.services[databaseFrom]) {
244
+ throw new Error(
245
+ `Service "${serviceName}" databaseFrom "${databaseFrom}" is missing from ${RUNNER_MANIFEST}`
246
+ );
247
+ }
248
+ }
206
249
  }
207
250
  }
208
251
 
@@ -238,7 +281,11 @@ export function isSiblingProduct(name) {
238
281
 
239
282
  function validateMergedService(name, runnerService, serviceConfig, productDir) {
240
283
  const usesLocalExecution = Object.values(runnerService.suites).some((suites) =>
241
- suites.some((suite) => suite.framework !== "k6" || !isDalSuiteType(suite, runnerService, suites))
284
+ suites.some(
285
+ (suite) =>
286
+ (suite.framework && suite.framework !== "k6") ||
287
+ !isDalSuiteType(suite, runnerService, suites)
288
+ )
242
289
  );
243
290
 
244
291
  if (usesLocalExecution && !isObject(serviceConfig.local)) {
@@ -281,6 +328,7 @@ function validateMergedService(name, runnerService, serviceConfig, productDir) {
281
328
  );
282
329
  }
283
330
  }
331
+
284
332
  }
285
333
 
286
334
  function validateServiceConfig(name, service, configPath) {
@@ -294,6 +342,27 @@ function validateServiceConfig(name, service, configPath) {
294
342
  }
295
343
  }
296
344
 
345
+ if (service.databaseFrom !== undefined && typeof service.databaseFrom !== "string") {
346
+ throw new Error(`Service "${name}" databaseFrom must be a string`);
347
+ }
348
+
349
+ if (service.database !== undefined && service.databaseFrom !== undefined) {
350
+ throw new Error(
351
+ `Service "${name}" cannot define both database and databaseFrom`
352
+ );
353
+ }
354
+
355
+ if (service.envFile !== undefined && typeof service.envFile !== "string") {
356
+ throw new Error(`Service "${name}" envFile must be a string`);
357
+ }
358
+
359
+ if (
360
+ service.envFiles !== undefined &&
361
+ (!Array.isArray(service.envFiles) || service.envFiles.some((v) => typeof v !== "string"))
362
+ ) {
363
+ throw new Error(`Service "${name}" envFiles must be an array of strings`);
364
+ }
365
+
297
366
  if (service.database !== undefined) {
298
367
  if (!isObject(service.database)) {
299
368
  throw new Error(`Service "${name}" database must be an object`);
@@ -339,6 +408,12 @@ function validateServiceConfig(name, service, configPath) {
339
408
  requireString(service.local, "start", `Service "${name}" local.start`);
340
409
  requireString(service.local, "baseUrl", `Service "${name}" local.baseUrl`);
341
410
  requireString(service.local, "readyUrl", `Service "${name}" local.readyUrl`);
411
+ if (
412
+ service.local.port !== undefined &&
413
+ (!Number.isInteger(service.local.port) || service.local.port <= 0)
414
+ ) {
415
+ throw new Error(`Service "${name}" local.port must be a positive integer`);
416
+ }
342
417
  if (service.local.cwd !== undefined && typeof service.local.cwd !== "string") {
343
418
  throw new Error(`Service "${name}" local.cwd must be a string`);
344
419
  }
@@ -354,6 +429,23 @@ function validateServiceConfig(name, service, configPath) {
354
429
  }
355
430
  }
356
431
 
432
+ function loadServiceEnv(productDir, serviceConfig) {
433
+ const env = {};
434
+ for (const envFile of getServiceEnvFiles(serviceConfig)) {
435
+ Object.assign(env, parseDotenv(resolveServiceCwd(productDir, envFile)));
436
+ }
437
+ return env;
438
+ }
439
+
440
+ function getServiceEnvFiles(serviceConfig) {
441
+ const files = [];
442
+ if (serviceConfig.envFile) files.push(serviceConfig.envFile);
443
+ if (Array.isArray(serviceConfig.envFiles)) {
444
+ files.push(...serviceConfig.envFiles);
445
+ }
446
+ return files;
447
+ }
448
+
357
449
  function requireString(obj, key, label) {
358
450
  if (typeof obj[key] !== "string" || obj[key].length === 0) {
359
451
  throw new Error(`${label} must be a non-empty string`);
package/lib/runner.mjs CHANGED
@@ -13,54 +13,36 @@ import {
13
13
  const TYPE_ORDER = ["dal", "integration", "e2e", "load"];
14
14
  const HTTP_K6_TYPES = new Set(["integration", "e2e", "load"]);
15
15
  const DEFAULT_READY_TIMEOUT_MS = 120_000;
16
+ const PORT_STRIDE = 100;
16
17
 
17
18
  export async function runAll(configs, suiteType, suiteNames, opts, allConfigs = configs) {
18
19
  const configMap = new Map(allConfigs.map((config) => [config.name, config]));
19
- let failed = false;
20
-
21
- for (const config of configs) {
22
- console.log(`\n══ ${config.name} ══`);
23
- const serviceFailed = await runService(config, configMap, suiteType, suiteNames, opts);
24
- if (serviceFailed) failed = true;
25
- }
20
+ const targetSpan = Math.max(1, opts.jobs || 1);
21
+ const results = await Promise.all(
22
+ configs.map(async (config, targetSlot) => {
23
+ console.log(`\n══ ${config.name} ══`);
24
+ return runService(config, configMap, suiteType, suiteNames, opts, {
25
+ targetSlot,
26
+ targetSpan,
27
+ });
28
+ })
29
+ );
26
30
 
27
- if (failed) process.exit(1);
31
+ if (results.some(Boolean)) process.exit(1);
28
32
  }
29
33
 
30
34
  export async function destroy(config) {
31
- const runtimeServices = [];
32
- if (config.testkit.dependsOn) {
33
- for (const depName of config.testkit.dependsOn) {
34
- runtimeServices.push({
35
- name: depName,
36
- stateDir: path.join(config.stateDir, "deps", depName),
37
- testkit: null,
38
- });
39
- }
40
- }
41
- runtimeServices.push({
42
- name: config.name,
43
- stateDir: config.stateDir,
44
- testkit: config.testkit,
45
- });
35
+ if (!fs.existsSync(config.stateDir)) return;
46
36
 
47
- for (const runtime of runtimeServices) {
48
- if (!fs.existsSync(runtime.stateDir)) continue;
49
- const dbFile = path.join(runtime.stateDir, "neon_branch_id");
50
- if (fs.existsSync(dbFile)) {
51
- const serviceConfig =
52
- runtime.name === config.name
53
- ? config.testkit
54
- : { database: { projectId: readStateValue(path.join(runtime.stateDir, "neon_project_id")) } };
55
-
56
- const projectId = serviceConfig.database?.projectId;
57
- if (projectId) {
58
- await runScript("neon-down.sh", {
59
- NEON_PROJECT_ID: projectId,
60
- STATE_DIR: runtime.stateDir,
61
- });
62
- }
63
- }
37
+ const runtimeStateDirs = findRuntimeStateDirs(config.stateDir);
38
+ for (const stateDir of runtimeStateDirs) {
39
+ const projectId = readStateValue(path.join(stateDir, "neon_project_id"));
40
+ if (!projectId) continue;
41
+
42
+ await runScript("neon-down.sh", {
43
+ NEON_PROJECT_ID: projectId,
44
+ STATE_DIR: stateDir,
45
+ });
64
46
  }
65
47
 
66
48
  fs.rmSync(config.stateDir, { recursive: true, force: true });
@@ -75,8 +57,11 @@ export function showStatus(config) {
75
57
  printStateDir(config.stateDir, " ");
76
58
  }
77
59
 
78
- async function runService(targetConfig, configMap, suiteType, suiteNames, opts) {
79
- const suites = collectSuites(targetConfig, suiteType, suiteNames, opts.framework);
60
+ async function runService(targetConfig, configMap, suiteType, suiteNames, opts, runtimeSlot) {
61
+ const suites = applyShard(
62
+ collectSuites(targetConfig, suiteType, suiteNames, opts.framework),
63
+ opts.shard
64
+ );
80
65
  if (suites.length === 0) {
81
66
  console.log(
82
67
  `No test files for ${targetConfig.name} type="${suiteType}" suites=${suiteNames.join(",") || "all"} — skipping`
@@ -87,25 +72,25 @@ async function runService(targetConfig, configMap, suiteType, suiteNames, opts)
87
72
  const runtimeConfigs = resolveRuntimeConfigs(targetConfig, configMap);
88
73
  fs.mkdirSync(targetConfig.stateDir, { recursive: true });
89
74
 
90
- let startedServices = [];
91
- let failed = false;
92
-
93
- try {
94
- await prepareDatabases(runtimeConfigs, targetConfig);
95
- await runMigrations(runtimeConfigs, targetConfig);
96
- await runSeeds(runtimeConfigs, targetConfig);
75
+ const jobs = Math.max(1, Math.min(opts.jobs || 1, suites.length));
76
+ const workerPlans = buildWorkerPlans(
77
+ targetConfig,
78
+ runtimeConfigs,
79
+ suites,
80
+ jobs,
81
+ runtimeSlot
82
+ );
97
83
 
98
- if (needsLocalRuntime(suites)) {
99
- startedServices = await startLocalServices(runtimeConfigs, targetConfig);
100
- }
84
+ const results = await Promise.allSettled(workerPlans.map((plan) => runWorkerPlan(plan)));
85
+ let failed = false;
101
86
 
102
- for (const suite of suites) {
103
- console.log(`\n── ${suite.type}:${suite.name} (${suite.framework}) ──`);
104
- const result = await runSuite(targetConfig, suite, targetConfig.stateDir);
105
- if (result.failed) failed = true;
87
+ for (const result of results) {
88
+ if (result.status === "rejected") {
89
+ failed = true;
90
+ console.error(result.reason);
91
+ continue;
106
92
  }
107
- } finally {
108
- await stopLocalServices(startedServices);
93
+ if (result.value) failed = true;
109
94
  }
110
95
 
111
96
  return failed;
@@ -119,23 +104,40 @@ function collectSuites(config, suiteType, suiteNames, frameworkFilter) {
119
104
 
120
105
  const selectedNames = new Set(suiteNames);
121
106
  const suites = [];
107
+ let orderIndex = 0;
122
108
 
123
109
  for (const type of types) {
124
110
  for (const suite of config.suites[type] || []) {
125
111
  const framework = suite.framework || "k6";
126
112
  if (selectedNames.size > 0 && !selectedNames.has(suite.name)) continue;
127
113
  if (frameworkFilter && frameworkFilter !== "all" && framework !== frameworkFilter) continue;
114
+
128
115
  suites.push({
129
116
  ...suite,
130
117
  framework,
131
118
  type,
119
+ orderIndex,
120
+ sortKey: `${type}:${suite.name}`,
121
+ weight:
122
+ suite.testkit?.weight ||
123
+ (framework === "playwright"
124
+ ? Math.max(2, suite.files.length)
125
+ : Math.max(1, suite.files.length)),
126
+ maxFileConcurrency:
127
+ framework === "k6" ? suite.testkit?.maxFileConcurrency || 1 : 1,
132
128
  });
129
+ orderIndex += 1;
133
130
  }
134
131
  }
135
132
 
136
133
  return suites;
137
134
  }
138
135
 
136
+ function applyShard(suites, shard) {
137
+ if (!shard) return suites;
138
+ return suites.filter((_, index) => index % shard.total === shard.index - 1);
139
+ }
140
+
139
141
  function orderedTypes(types) {
140
142
  const ordered = [];
141
143
  for (const known of TYPE_ORDER) {
@@ -175,39 +177,310 @@ function resolveRuntimeConfigs(targetConfig, configMap) {
175
177
  return ordered;
176
178
  }
177
179
 
178
- async function prepareDatabases(runtimeConfigs, targetConfig) {
180
+ function buildWorkerPlans(targetConfig, runtimeConfigs, suites, jobs, runtimeSlot) {
181
+ const buckets = distributeSuites(suites, jobs);
182
+ return buckets
183
+ .map((bucket, index) =>
184
+ createWorkerPlan(targetConfig, runtimeConfigs, bucket.suites, index + 1, runtimeSlot)
185
+ )
186
+ .filter(Boolean);
187
+ }
188
+
189
+ function distributeSuites(suites, jobs) {
190
+ const buckets = Array.from({ length: jobs }, () => ({
191
+ suites: [],
192
+ totalWeight: 0,
193
+ }));
194
+ const ordered = [...suites].sort(
195
+ (a, b) => b.weight - a.weight || a.sortKey.localeCompare(b.sortKey)
196
+ );
197
+
198
+ for (const suite of ordered) {
199
+ let bestBucket = buckets[0];
200
+ for (const bucket of buckets.slice(1)) {
201
+ if (bucket.totalWeight < bestBucket.totalWeight) {
202
+ bestBucket = bucket;
203
+ continue;
204
+ }
205
+ if (
206
+ bucket.totalWeight === bestBucket.totalWeight &&
207
+ bucket.suites.length < bestBucket.suites.length
208
+ ) {
209
+ bestBucket = bucket;
210
+ }
211
+ }
212
+
213
+ bestBucket.suites.push(suite);
214
+ bestBucket.totalWeight += suite.weight;
215
+ }
216
+
217
+ for (const bucket of buckets) {
218
+ bucket.suites.sort((a, b) => a.orderIndex - b.orderIndex);
219
+ }
220
+
221
+ return buckets.filter((bucket) => bucket.suites.length > 0);
222
+ }
223
+
224
+ function createWorkerPlan(targetConfig, runtimeConfigs, suites, workerId, runtimeSlot) {
225
+ if (suites.length === 0) return null;
226
+
227
+ const workerStateDir = path.join(targetConfig.stateDir, "workers", `worker-${workerId}`);
228
+ const workerRuntimeConfigs = resolveWorkerRuntimeConfigs(
229
+ targetConfig,
230
+ runtimeConfigs,
231
+ workerId,
232
+ workerStateDir,
233
+ runtimeSlot
234
+ );
235
+ const workerTargetConfig = workerRuntimeConfigs.find(
236
+ (config) => config.name === targetConfig.name
237
+ );
238
+
239
+ return {
240
+ workerId,
241
+ suites,
242
+ runtimeConfigs: workerRuntimeConfigs,
243
+ targetConfig: workerTargetConfig,
244
+ };
245
+ }
246
+
247
+ function resolveWorkerRuntimeConfigs(
248
+ targetConfig,
249
+ runtimeConfigs,
250
+ workerId,
251
+ workerStateDir,
252
+ runtimeSlot
253
+ ) {
254
+ const portMap = buildPortMap(runtimeConfigs, workerId, runtimeSlot);
255
+ const baseUrlByService = new Map();
256
+ const readyUrlByService = new Map();
257
+
258
+ for (const config of runtimeConfigs) {
259
+ if (!config.testkit.local) continue;
260
+ baseUrlByService.set(
261
+ config.name,
262
+ resolveRuntimeUrl(config.testkit.local.baseUrl, config.name, targetConfig, workerId, {
263
+ workerStateDir,
264
+ portMap,
265
+ baseUrlByService,
266
+ readyUrlByService,
267
+ })
268
+ );
269
+ readyUrlByService.set(
270
+ config.name,
271
+ resolveRuntimeUrl(config.testkit.local.readyUrl, config.name, targetConfig, workerId, {
272
+ workerStateDir,
273
+ portMap,
274
+ baseUrlByService,
275
+ readyUrlByService,
276
+ })
277
+ );
278
+ }
279
+
280
+ const urlMappings = [];
281
+ for (const config of runtimeConfigs) {
282
+ if (!config.testkit.local) continue;
283
+ const resolvedBaseUrl = baseUrlByService.get(config.name);
284
+ const resolvedReadyUrl = readyUrlByService.get(config.name);
285
+ if (resolvedBaseUrl) {
286
+ urlMappings.push([config.testkit.local.baseUrl, resolvedBaseUrl]);
287
+ }
288
+ if (resolvedReadyUrl) {
289
+ urlMappings.push([config.testkit.local.readyUrl, resolvedReadyUrl]);
290
+ }
291
+ }
292
+
293
+ return runtimeConfigs.map((config) =>
294
+ resolveWorkerConfig(
295
+ config,
296
+ targetConfig,
297
+ workerId,
298
+ workerStateDir,
299
+ portMap,
300
+ baseUrlByService,
301
+ readyUrlByService,
302
+ urlMappings
303
+ )
304
+ );
305
+ }
306
+
307
+ function buildPortMap(runtimeConfigs, workerId, runtimeSlot) {
308
+ const portMap = new Map();
309
+ const seen = new Map();
310
+ const offset = PORT_STRIDE * ((workerId - 1) + runtimeSlot.targetSlot * runtimeSlot.targetSpan);
311
+
312
+ for (const config of runtimeConfigs) {
313
+ if (!config.testkit.local) continue;
314
+
315
+ const basePort = config.testkit.local.port || numericPortFromUrl(config.testkit.local.baseUrl);
316
+ if (!basePort) continue;
317
+
318
+ const actualPort = basePort + offset;
319
+ const existing = seen.get(actualPort);
320
+ if (existing) {
321
+ throw new Error(
322
+ `Worker port collision: services "${existing}" and "${config.name}" both resolve to ${actualPort}. ` +
323
+ `Assign distinct local.port/baseUrl ports in testkit.config.json.`
324
+ );
325
+ }
326
+ seen.set(actualPort, config.name);
327
+ portMap.set(config.name, actualPort);
328
+ }
329
+
330
+ return portMap;
331
+ }
332
+
333
+ function resolveWorkerConfig(
334
+ config,
335
+ targetConfig,
336
+ workerId,
337
+ workerStateDir,
338
+ portMap,
339
+ baseUrlByService,
340
+ readyUrlByService,
341
+ urlMappings
342
+ ) {
343
+ const stateDir = resolveServiceStateDir(workerStateDir, targetConfig.name, config);
344
+ const context = {
345
+ workerId,
346
+ serviceName: config.name,
347
+ targetName: targetConfig.name,
348
+ serviceStateDir: stateDir,
349
+ portMap,
350
+ baseUrlByService,
351
+ readyUrlByService,
352
+ urlMappings,
353
+ };
354
+
355
+ const database = config.testkit.database
356
+ ? {
357
+ ...config.testkit.database,
358
+ branchName:
359
+ config.testkit.database.branchName !== undefined
360
+ ? finalizeString(config.testkit.database.branchName, context)
361
+ : `${targetConfig.name}-${config.name}-w${workerId}-testkit`,
362
+ }
363
+ : undefined;
364
+
365
+ const migrate = config.testkit.migrate
366
+ ? {
367
+ ...config.testkit.migrate,
368
+ cmd: finalizeString(config.testkit.migrate.cmd, context),
369
+ cwd:
370
+ config.testkit.migrate.cwd !== undefined
371
+ ? finalizeString(config.testkit.migrate.cwd, context)
372
+ : config.testkit.migrate.cwd,
373
+ }
374
+ : undefined;
375
+
376
+ const seed = config.testkit.seed
377
+ ? {
378
+ ...config.testkit.seed,
379
+ cmd: finalizeString(config.testkit.seed.cmd, context),
380
+ cwd:
381
+ config.testkit.seed.cwd !== undefined
382
+ ? finalizeString(config.testkit.seed.cwd, context)
383
+ : config.testkit.seed.cwd,
384
+ }
385
+ : undefined;
386
+
387
+ const local = config.testkit.local
388
+ ? {
389
+ ...config.testkit.local,
390
+ start: finalizeString(config.testkit.local.start, context),
391
+ cwd:
392
+ config.testkit.local.cwd !== undefined
393
+ ? finalizeString(config.testkit.local.cwd, context)
394
+ : config.testkit.local.cwd,
395
+ port: portMap.get(config.name) || config.testkit.local.port,
396
+ baseUrl: baseUrlByService.get(config.name),
397
+ readyUrl: readyUrlByService.get(config.name),
398
+ env: Object.fromEntries(
399
+ Object.entries(config.testkit.local.env || {}).map(([key, value]) => [
400
+ key,
401
+ finalizeString(String(value), context),
402
+ ])
403
+ ),
404
+ }
405
+ : undefined;
406
+
407
+ return {
408
+ ...config,
409
+ stateDir,
410
+ workerId,
411
+ workerLabel: `w${workerId}`,
412
+ targetName: targetConfig.name,
413
+ testkit: {
414
+ ...config.testkit,
415
+ database,
416
+ migrate,
417
+ seed,
418
+ local,
419
+ },
420
+ };
421
+ }
422
+
423
+ async function runWorkerPlan(plan) {
424
+ console.log(
425
+ `\n══ ${plan.targetConfig.name} worker ${plan.workerId} (${plan.suites.length} suite${plan.suites.length === 1 ? "" : "s"}) ══`
426
+ );
427
+
428
+ let startedServices = [];
429
+ let failed = false;
430
+
431
+ try {
432
+ await prepareDatabases(plan.runtimeConfigs);
433
+ await runMigrations(plan.runtimeConfigs);
434
+ await runSeeds(plan.runtimeConfigs);
435
+
436
+ if (needsLocalRuntime(plan.suites)) {
437
+ startedServices = await startLocalServices(plan.runtimeConfigs);
438
+ }
439
+
440
+ for (const suite of plan.suites) {
441
+ console.log(
442
+ `\n── ${plan.targetConfig.workerLabel} ${suite.type}:${suite.name} (${suite.framework}) ──`
443
+ );
444
+ const result = await runSuite(plan.targetConfig, suite);
445
+ if (result.failed) failed = true;
446
+ }
447
+ } finally {
448
+ await stopLocalServices(startedServices);
449
+ }
450
+
451
+ return failed;
452
+ }
453
+
454
+ async function prepareDatabases(runtimeConfigs) {
179
455
  for (const config of runtimeConfigs) {
180
456
  const db = config.testkit.database;
181
457
  if (!db) continue;
182
458
 
183
459
  requireNeonApiKey();
184
-
185
- const stateDir = getServiceStateDir(targetConfig, config.name);
186
- fs.mkdirSync(stateDir, { recursive: true });
460
+ fs.mkdirSync(config.stateDir, { recursive: true });
187
461
 
188
462
  await runScript("neon-up.sh", {
189
463
  NEON_PROJECT_ID: db.projectId,
190
464
  NEON_DB_NAME: db.dbName,
191
- NEON_BRANCH_NAME: db.branchName || `${targetConfig.name}-${config.name}-testkit`,
465
+ NEON_BRANCH_NAME: db.branchName,
192
466
  NEON_RESET: db.reset === false ? "false" : "true",
193
- STATE_DIR: stateDir,
467
+ STATE_DIR: config.stateDir,
194
468
  });
195
469
 
196
- fs.writeFileSync(path.join(stateDir, "neon_project_id"), db.projectId);
470
+ fs.writeFileSync(path.join(config.stateDir, "neon_project_id"), db.projectId);
197
471
  }
198
472
  }
199
473
 
200
- async function runMigrations(runtimeConfigs, targetConfig) {
474
+ async function runMigrations(runtimeConfigs) {
201
475
  for (const config of runtimeConfigs) {
202
476
  const migrate = config.testkit.migrate;
203
477
  if (!migrate) continue;
204
478
 
205
- const stateDir = getServiceStateDir(targetConfig, config.name);
206
- const env = { ...process.env };
207
- const dbUrl = readDatabaseUrl(stateDir);
479
+ const env = buildExecutionEnv(config);
480
+ const dbUrl = readDatabaseUrl(config.stateDir);
208
481
  if (dbUrl) env.DATABASE_URL = dbUrl;
209
482
 
210
- console.log(`\n── migrate:${config.name} ──`);
483
+ console.log(`\n── migrate:${config.workerLabel}:${config.name} ──`);
211
484
  await execaCommand(migrate.cmd, {
212
485
  cwd: resolveServiceCwd(config.productDir, migrate.cwd),
213
486
  env,
@@ -217,17 +490,16 @@ async function runMigrations(runtimeConfigs, targetConfig) {
217
490
  }
218
491
  }
219
492
 
220
- async function runSeeds(runtimeConfigs, targetConfig) {
493
+ async function runSeeds(runtimeConfigs) {
221
494
  for (const config of runtimeConfigs) {
222
495
  const seed = config.testkit.seed;
223
496
  if (!seed) continue;
224
497
 
225
- const stateDir = getServiceStateDir(targetConfig, config.name);
226
- const env = { ...process.env };
227
- const dbUrl = readDatabaseUrl(stateDir);
498
+ const env = buildExecutionEnv(config);
499
+ const dbUrl = readDatabaseUrl(config.stateDir);
228
500
  if (dbUrl) env.DATABASE_URL = dbUrl;
229
501
 
230
- console.log(`\n── seed:${config.name} ──`);
502
+ console.log(`\n── seed:${config.workerLabel}:${config.name} ──`);
231
503
  await execaCommand(seed.cmd, {
232
504
  cwd: resolveServiceCwd(config.productDir, seed.cwd),
233
505
  env,
@@ -237,14 +509,13 @@ async function runSeeds(runtimeConfigs, targetConfig) {
237
509
  }
238
510
  }
239
511
 
240
- async function startLocalServices(runtimeConfigs, targetConfig) {
512
+ async function startLocalServices(runtimeConfigs) {
241
513
  const started = [];
242
514
 
243
515
  try {
244
516
  for (const config of runtimeConfigs) {
245
517
  if (!config.testkit.local) continue;
246
- const stateDir = getServiceStateDir(targetConfig, config.name);
247
- const proc = await startLocalService(config, stateDir);
518
+ const proc = await startLocalService(config);
248
519
  started.push(proc);
249
520
  }
250
521
  } catch (error) {
@@ -255,25 +526,22 @@ async function startLocalServices(runtimeConfigs, targetConfig) {
255
526
  return started;
256
527
  }
257
528
 
258
- async function startLocalService(config, stateDir) {
529
+ async function startLocalService(config) {
259
530
  const cwd = resolveServiceCwd(config.productDir, config.testkit.local.cwd);
260
- const env = {
261
- ...process.env,
262
- ...config.testkit.local.env,
263
- };
264
- const port = portFromUrl(config.testkit.local.baseUrl);
531
+ const env = buildExecutionEnv(config, config.testkit.local.env);
532
+ const port = config.testkit.local.port || numericPortFromUrl(config.testkit.local.baseUrl);
265
533
  if (port) {
266
- env.PORT = port;
534
+ env.PORT = String(port);
267
535
  }
268
536
 
269
- const dbUrl = readDatabaseUrl(stateDir);
537
+ const dbUrl = readDatabaseUrl(config.stateDir);
270
538
  if (dbUrl) {
271
539
  env.DATABASE_URL = dbUrl;
272
540
  }
273
541
 
274
542
  await assertLocalServicePortsAvailable(config);
275
543
 
276
- console.log(`Starting ${config.name}: ${config.testkit.local.start}`);
544
+ console.log(`Starting ${config.workerLabel}:${config.name}: ${config.testkit.local.start}`);
277
545
  const child = spawn(config.testkit.local.start, {
278
546
  cwd,
279
547
  env,
@@ -281,15 +549,14 @@ async function startLocalService(config, stateDir) {
281
549
  stdio: ["ignore", "pipe", "pipe"],
282
550
  });
283
551
 
284
- pipeOutput(child.stdout, `[${config.name}]`);
285
- pipeOutput(child.stderr, `[${config.name}]`);
552
+ pipeOutput(child.stdout, `[${config.workerLabel}:${config.name}]`);
553
+ pipeOutput(child.stderr, `[${config.workerLabel}:${config.name}]`);
286
554
 
287
- const readyTimeoutMs =
288
- config.testkit.local.readyTimeoutMs || DEFAULT_READY_TIMEOUT_MS;
555
+ const readyTimeoutMs = config.testkit.local.readyTimeoutMs || DEFAULT_READY_TIMEOUT_MS;
289
556
 
290
557
  try {
291
558
  await waitForReady({
292
- name: config.name,
559
+ name: `${config.workerLabel}:${config.name}`,
293
560
  url: config.testkit.local.readyUrl,
294
561
  timeoutMs: readyTimeoutMs,
295
562
  process: child,
@@ -302,9 +569,9 @@ async function startLocalService(config, stateDir) {
302
569
  return { name: config.name, child };
303
570
  }
304
571
 
305
- async function runSuite(targetConfig, suite, stateDir) {
572
+ async function runSuite(targetConfig, suite) {
306
573
  if (suite.type === "dal") {
307
- return runDalSuite(targetConfig, suite, stateDir);
574
+ return runDalSuite(targetConfig, suite);
308
575
  }
309
576
 
310
577
  if (suite.framework === "playwright") {
@@ -327,43 +594,44 @@ async function runHttpK6Suite(targetConfig, suite) {
327
594
  }
328
595
 
329
596
  let failed = false;
330
- for (const file of suite.files) {
597
+ await runWithConcurrency(suite.files, suite.maxFileConcurrency, async (file) => {
331
598
  const absFile = path.join(targetConfig.productDir, file);
599
+ console.log(`·· ${targetConfig.workerLabel}:${suite.name} → ${file}`);
332
600
  try {
333
601
  await execa("k6", ["run", "-e", `BASE_URL=${baseUrl}`, absFile], {
334
602
  cwd: targetConfig.productDir,
335
- env: process.env,
603
+ env: buildExecutionEnv(targetConfig),
336
604
  stdio: "inherit",
337
605
  });
338
606
  } catch {
339
607
  failed = true;
340
608
  }
341
- }
609
+ });
342
610
 
343
611
  return { failed };
344
612
  }
345
613
 
346
- async function runDalSuite(targetConfig, suite, stateDir) {
347
- const databaseUrl = readDatabaseUrl(stateDir);
614
+ async function runDalSuite(targetConfig, suite) {
615
+ const databaseUrl = readDatabaseUrl(targetConfig.stateDir);
348
616
  if (!databaseUrl) {
349
617
  throw new Error(`Service "${targetConfig.name}" requires a database for DAL suites`);
350
618
  }
351
619
 
352
620
  const k6Binary = resolveDalBinary();
353
621
  let failed = false;
354
-
355
- for (const file of suite.files) {
622
+ await runWithConcurrency(suite.files, suite.maxFileConcurrency, async (file) => {
356
623
  const absFile = path.join(targetConfig.productDir, file);
624
+ console.log(`·· ${targetConfig.workerLabel}:${suite.name} → ${file}`);
357
625
  try {
358
626
  await execa(k6Binary, ["run", "-e", `DATABASE_URL=${databaseUrl}`, absFile], {
359
627
  cwd: targetConfig.productDir,
360
- env: process.env,
628
+ env: buildExecutionEnv(targetConfig),
361
629
  stdio: "inherit",
362
630
  });
363
631
  } catch {
364
632
  failed = true;
365
633
  }
366
- }
634
+ });
367
635
 
368
636
  return { failed };
369
637
  }
@@ -377,17 +645,14 @@ async function runPlaywrightSuite(targetConfig, suite) {
377
645
  }
378
646
 
379
647
  const cwd = resolveServiceCwd(targetConfig.productDir, local.cwd);
380
- const files = suite.files.map((file) => path.relative(cwd, path.join(targetConfig.productDir, file)));
648
+ const files = suite.files.map((file) =>
649
+ path.relative(cwd, path.join(targetConfig.productDir, file))
650
+ );
381
651
 
382
652
  try {
383
653
  await execa("npx", ["playwright", "test", ...files], {
384
654
  cwd,
385
- env: {
386
- ...process.env,
387
- BASE_URL: local.baseUrl,
388
- PLAYWRIGHT_HTML_OPEN: "never",
389
- TESTKIT_MANAGED_SERVERS: "1",
390
- },
655
+ env: buildPlaywrightEnv(targetConfig, local.baseUrl),
391
656
  stdio: "inherit",
392
657
  });
393
658
  return { failed: false };
@@ -442,11 +707,54 @@ function needsLocalRuntime(suites) {
442
707
  return suites.some((suite) => suite.type !== "dal");
443
708
  }
444
709
 
445
- function getServiceStateDir(targetConfig, serviceName) {
446
- if (targetConfig.name === serviceName) {
447
- return targetConfig.stateDir;
710
+ function resolveServiceStateDir(workerStateDir, targetName, config) {
711
+ const dbSource = config.testkit.databaseFrom || config.name;
712
+ return getWorkerServiceStateDir(workerStateDir, targetName, dbSource);
713
+ }
714
+
715
+ function getWorkerServiceStateDir(workerStateDir, targetName, serviceName) {
716
+ if (targetName === serviceName) {
717
+ return workerStateDir;
448
718
  }
449
- return path.join(targetConfig.stateDir, "deps", serviceName);
719
+ return path.join(workerStateDir, "deps", serviceName);
720
+ }
721
+
722
+ function buildExecutionEnv(config, extraEnv = {}) {
723
+ return {
724
+ ...process.env,
725
+ ...(config.testkit.serviceEnv || {}),
726
+ ...extraEnv,
727
+ };
728
+ }
729
+
730
+ function buildPlaywrightEnv(config, baseUrl) {
731
+ const env = buildExecutionEnv(config, {
732
+ BASE_URL: baseUrl,
733
+ PLAYWRIGHT_HTML_OPEN: "never",
734
+ PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS:
735
+ process.env.PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS || "1",
736
+ TESTKIT_MANAGED_SERVERS: "1",
737
+ TESTKIT_WORKER_ID: String(config.workerId),
738
+ });
739
+
740
+ const browsersPath = resolvePlaywrightBrowsersPath(env.PLAYWRIGHT_BROWSERS_PATH);
741
+ if (browsersPath) {
742
+ env.PLAYWRIGHT_BROWSERS_PATH = browsersPath;
743
+ }
744
+
745
+ return env;
746
+ }
747
+
748
+ function resolvePlaywrightBrowsersPath(configuredPath) {
749
+ const home = process.env.HOME;
750
+ if (home) {
751
+ const fallback = path.join(home, ".cache", "ms-playwright");
752
+ if (fs.existsSync(fallback)) {
753
+ return fallback;
754
+ }
755
+ }
756
+
757
+ return configuredPath;
450
758
  }
451
759
 
452
760
  function readDatabaseUrl(stateDir) {
@@ -490,14 +798,112 @@ function pipeOutput(stream, prefix) {
490
798
  });
491
799
  }
492
800
 
801
+ async function runWithConcurrency(items, limit, handler) {
802
+ const concurrency = Math.max(1, Math.min(limit || 1, items.length || 1));
803
+ let nextIndex = 0;
804
+
805
+ const workers = Array.from({ length: concurrency }, async () => {
806
+ while (nextIndex < items.length) {
807
+ const current = nextIndex;
808
+ nextIndex += 1;
809
+ await handler(items[current], current);
810
+ }
811
+ });
812
+
813
+ await Promise.all(workers);
814
+ }
815
+
493
816
  function sleep(ms) {
494
817
  return new Promise((resolve) => setTimeout(resolve, ms));
495
818
  }
496
819
 
497
- function portFromUrl(rawUrl) {
820
+ function resolveRuntimeUrl(rawUrl, serviceName, targetConfig, workerId, context) {
821
+ const resolved = resolveTemplateString(rawUrl, {
822
+ ...context,
823
+ targetName: targetConfig.name,
824
+ workerId,
825
+ serviceName,
826
+ });
827
+ const actualPort = context.portMap.get(serviceName);
828
+ return actualPort ? rewriteUrlPort(resolved, actualPort) : resolved;
829
+ }
830
+
831
+ function finalizeString(value, context) {
832
+ let resolved = resolveTemplateString(value, context);
833
+ for (const [source, destination] of context.urlMappings || []) {
834
+ if (source && destination && source !== destination) {
835
+ resolved = resolved.split(source).join(destination);
836
+ }
837
+ }
838
+ return resolved;
839
+ }
840
+
841
+ function resolveTemplateString(value, context) {
842
+ if (typeof value !== "string") return value;
843
+
844
+ return value.replace(/\{([a-zA-Z]+)(?::([a-zA-Z0-9_-]+))?\}/g, (_match, token, arg) => {
845
+ switch (token) {
846
+ case "worker":
847
+ return String(context.workerId);
848
+ case "target":
849
+ return context.targetName;
850
+ case "service":
851
+ return context.serviceName;
852
+ case "stateDir":
853
+ return context.serviceStateDir;
854
+ case "port": {
855
+ const serviceName = arg || context.serviceName;
856
+ const port = context.portMap.get(serviceName);
857
+ if (!port) {
858
+ throw new Error(`Unknown port placeholder for service "${serviceName}"`);
859
+ }
860
+ return String(port);
861
+ }
862
+ case "baseUrl": {
863
+ const serviceName = arg || context.serviceName;
864
+ const baseUrl = context.baseUrlByService.get(serviceName);
865
+ if (!baseUrl) {
866
+ throw new Error(`Unknown baseUrl placeholder for service "${serviceName}"`);
867
+ }
868
+ return baseUrl;
869
+ }
870
+ case "readyUrl": {
871
+ const serviceName = arg || context.serviceName;
872
+ const readyUrl = context.readyUrlByService.get(serviceName);
873
+ if (!readyUrl) {
874
+ throw new Error(`Unknown readyUrl placeholder for service "${serviceName}"`);
875
+ }
876
+ return readyUrl;
877
+ }
878
+ default:
879
+ throw new Error(`Unsupported template token "{${token}${arg ? `:${arg}` : ""}}"`);
880
+ }
881
+ });
882
+ }
883
+
884
+ function rewriteUrlPort(rawUrl, port) {
885
+ try {
886
+ const original = new URL(rawUrl);
887
+ if (!original.port) return rawUrl;
888
+
889
+ const rewritten = new URL(rawUrl);
890
+ rewritten.port = String(port);
891
+
892
+ let next = rewritten.toString();
893
+ if (!rawUrl.endsWith("/") && original.pathname === "/" && next.endsWith("/")) {
894
+ next = next.slice(0, -1);
895
+ }
896
+ return next;
897
+ } catch {
898
+ return rawUrl;
899
+ }
900
+ }
901
+
902
+ function numericPortFromUrl(rawUrl) {
498
903
  try {
499
904
  const url = new URL(rawUrl);
500
- return url.port || null;
905
+ const port = Number(url.port);
906
+ return Number.isInteger(port) && port > 0 ? port : null;
501
907
  } catch {
502
908
  return null;
503
909
  }
@@ -517,7 +923,8 @@ async function assertLocalServicePortsAvailable(config) {
517
923
 
518
924
  if (await isPortInUse(socket)) {
519
925
  throw new Error(
520
- `Cannot start "${config.name}" because ${key} is already in use. Stop the existing process and rerun testkit.`
926
+ `Cannot start "${config.workerLabel}:${config.name}" because ${key} is already in use. ` +
927
+ `Stop the existing process and rerun testkit.`
521
928
  );
522
929
  }
523
930
  }
@@ -572,3 +979,29 @@ async function isPortInUse({ host, port }) {
572
979
  socket.connect(port, host);
573
980
  });
574
981
  }
982
+
983
+ function findRuntimeStateDirs(rootDir) {
984
+ const found = [];
985
+
986
+ const visit = (dir) => {
987
+ if (!fs.existsSync(dir)) return;
988
+ const entries = fs.readdirSync(dir, { withFileTypes: true });
989
+ const hasRuntimeFiles = entries.some(
990
+ (entry) =>
991
+ entry.isFile() &&
992
+ (entry.name === "neon_project_id" || entry.name === "neon_branch_id")
993
+ );
994
+ if (hasRuntimeFiles) {
995
+ found.push(dir);
996
+ }
997
+
998
+ for (const entry of entries) {
999
+ if (entry.isDirectory()) {
1000
+ visit(path.join(dir, entry.name));
1001
+ }
1002
+ }
1003
+ };
1004
+
1005
+ visit(rootDir);
1006
+ return found.sort((a, b) => b.length - a.length);
1007
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@elench/testkit",
3
- "version": "0.1.11",
3
+ "version": "0.1.13",
4
4
  "description": "CLI for running manifest-defined local test suites across k6 and Playwright",
5
5
  "type": "module",
6
6
  "bin": {