@elench/testkit 0.1.38 → 0.1.40

package/README.md

@@ -21,8 +21,14 @@ npx @elench/testkit --type e2e
 npx @elench/testkit --type int,e2e,dal
 npx @elench/testkit --type pw
 
-# Parallelize with isolated worker stacks
-npx @elench/testkit --jobs 3
+# Shared local stack with parallel workers
+npx @elench/testkit --workers 8 --stack-mode shared
+
+# One file-level wall clock budget for every suite file
+npx @elench/testkit --file-timeout-seconds 60
+
+# Two reusable local stacks for browser-heavy suites
+npx @elench/testkit --workers 6 --stack-mode pooled --stack-count 2
 
 # Run a deterministic shard
 npx @elench/testkit --shard 1/3
@@ -61,6 +67,12 @@ import {
 } from "@elench/testkit/setup";
 
 export default defineTestkitSetup({
+  execution: {
+    workers: 8,
+    fileTimeoutSeconds: 60,
+    stackMode: "shared",
+    stackCount: 1,
+  },
   services: {
     api: service({
       ...tsxService({
@@ -109,6 +121,8 @@ export default defineTestkitSetup({
 `testkit.setup.ts` is optional for simple repos, but it is the primary escape hatch
 for:
 
+- worker and stack topology
+- per-file wall clock timeout budget
 - multi-service graphs
 - local DB configuration
 - migrate / seed commands
@@ -159,7 +173,20 @@ export default suite;
 Low-level runtime primitives remain available:
 
 ```ts
-import { check, group, http } from "@elench/testkit/runtime";
+import { check, group, http, waitFor } from "@elench/testkit/runtime";
+```
+
+`waitFor()` consumes the file budget configured by `execution.fileTimeoutSeconds`.
+Consumers should not set local timeout values in test files.
+
+```ts
+import { waitFor } from "@elench/testkit/runtime";
+
+const response = waitFor(
+  () => req("GET", "/api/v1/jobs/123", setupData),
+  (res) => JSON.parse(res.body).data?.status === "completed",
+  { description: "job 123 to complete" }
+);
 ```
 
 ## Discovery
@@ -197,7 +224,7 @@ Suite names are inferred from the colocated path:
 services that define `database: localDatabase(...)`.
 
 - template databases are cached
-- worker databases are cloned from templates
+- stack databases are cloned from templates
 - template fingerprints are derived automatically from env files, migrate/seed
   config, and repo contents
 

package/lib/cli/args.mjs

@@ -1,5 +1,11 @@
 import path from "path";
 import { normalizeTypeValues, parseSuiteSelectors } from "../runner/suite-selection.mjs";
+import {
+  parseFileTimeoutOption,
+  parseStackCountOption,
+  parseStackModeOption,
+  parseWorkersOption,
+} from "../runner/execution-config.mjs";
 
 export const POSITIONAL_TYPES = new Set(["int", "e2e", "dal", "load", "pw", "all"]);
 export const LIFECYCLE = new Set(["status", "destroy", "cleanup"]);
@@ -40,13 +46,12 @@ export function parseSuiteOption(values) {
   return parseSuiteSelectors(input);
 }
 
-export function parseJobsOption(value) {
-  const jobs = Number.parseInt(String(value), 10);
-  if (!Number.isInteger(jobs) || jobs <= 0) {
-    throw new Error(`Invalid --jobs value "${value}". Expected a positive integer.`);
-  }
-  return jobs;
-}
+export {
+  parseFileTimeoutOption,
+  parseWorkersOption,
+  parseStackModeOption,
+  parseStackCountOption,
+};
 
 export function parseShardOption(value) {
   if (!value) return null;

package/lib/cli/args.test.mjs

@@ -1,9 +1,12 @@
 import { describe, expect, it } from "vitest";
 import {
-  parseJobsOption,
+  parseFileTimeoutOption,
   parseShardOption,
+  parseStackCountOption,
+  parseStackModeOption,
   parseSuiteOption,
   parseTypeOption,
+  parseWorkersOption,
   resolveRequestedFiles,
   resolveCliSelection,
 } from "./args.mjs";
@@ -55,9 +58,16 @@ describe("cli-args", () => {
     ]);
   });
 
-  it("parses and validates jobs", () => {
-    expect(parseJobsOption("3")).toBe(3);
-    expect(() => parseJobsOption("0")).toThrow("Invalid --jobs value");
+  it("parses and validates execution options", () => {
+    expect(parseWorkersOption("3")).toBe(3);
+    expect(parseFileTimeoutOption("45")).toBe(45);
+    expect(parseStackCountOption("2")).toBe(2);
+    expect(parseStackModeOption("pooled")).toBe("pooled");
+    expect(() => parseWorkersOption("0")).toThrow("Invalid --workers value");
+    expect(() => parseFileTimeoutOption("0")).toThrow(
+      "Invalid --file-timeout-seconds value"
+    );
+    expect(() => parseStackModeOption("legacy")).toThrow("Invalid --stack-mode value");
   });
 
   it("parses and validates shards", () => {

package/lib/cli/index.mjs

@@ -1,10 +1,13 @@
 import { cac } from "cac";
 import { loadConfigs } from "../config/index.mjs";
 import {
-  parseJobsOption,
+  parseFileTimeoutOption,
   parseShardOption,
+  parseStackCountOption,
+  parseStackModeOption,
   parseSuiteOption,
   parseTypeOption,
+  parseWorkersOption,
   resolveRequestedFiles,
   resolveCliSelection,
 } from "./args.mjs";
@@ -22,9 +25,10 @@ export function run() {
     .option("-s, --suite <name>", "Run specific suite(s)", { default: [] })
     .option("-f, --file <path>", "Run specific file(s)", { default: [] })
     .option("--dir <path>", "Explicit product directory")
-    .option("--jobs <n>", "Number of isolated worker stacks for the whole run", {
-      default: "1",
-    })
+    .option("--workers <n>", "Number of test executors for the whole run")
+    .option("--file-timeout-seconds <n>", "Per-file wall-clock timeout in seconds")
+    .option("--stack-mode <mode>", "Stack topology: shared, pooled, or isolated")
+    .option("--stack-count <n>", "Number of prepared stacks when stack-mode=pooled")
     .option("--shard <i/n>", "Run only shard i of n at suite granularity")
     .option("--write-status", "Write a deterministic testkit.status.json snapshot")
     .option("--allow-partial-status", "Allow --write-status for filtered runs")
@@ -62,7 +66,14 @@ export function run() {
         return;
       }
 
-      const jobs = parseJobsOption(options.jobs);
+      const workers = options.workers == null ? null : parseWorkersOption(options.workers);
+      const fileTimeoutSeconds =
+        options.fileTimeoutSeconds == null
+          ? null
+          : parseFileTimeoutOption(options.fileTimeoutSeconds);
+      const stackMode = options.stackMode == null ? null : parseStackModeOption(options.stackMode);
+      const stackCount =
+        options.stackCount == null ? null : parseStackCountOption(options.stackCount);
       const shard = parseShardOption(options.shard);
       const typeValues = parseTypeOption(options.type, positionalType);
       const suiteSelectors = parseSuiteOption(options.suite);
@@ -77,7 +88,10 @@ export function run() {
           ...options,
           typeValues,
           fileNames,
-          jobs,
+          workers,
+          fileTimeoutSeconds,
+          stackMode,
+          stackCount,
           shard,
           serviceFilter: options.service || null,
         },

package/lib/config/index.mjs

@@ -8,6 +8,11 @@ import {
   parseSuiteSelectors,
   suiteSelectionType,
 } from "../runner/suite-selection.mjs";
+import {
+  DEFAULT_FILE_TIMEOUT_SECONDS,
+  normalizeExecutionConfig,
+  normalizeStackModeValue,
+} from "../runner/execution-config.mjs";
 
 const TESTKIT_K6_BIN = "TESTKIT_K6_BIN";
 const DEFAULT_LOCAL_IMAGE = "pgvector/pgvector:pg16";
@@ -22,6 +27,7 @@ export function parseDotenv(filePath) {
 export async function loadConfigs(opts = {}) {
   const productDir = resolveProductDir(process.cwd(), opts.dir);
   const { setup, setupFile } = await loadTestkitSetup(productDir);
+  const execution = normalizeRepoExecution(setup.execution);
   const explicitServices = setup.services || {};
   const discovery = discoverProject(productDir, explicitServices);
   const serviceNames = new Set([
@@ -37,6 +43,7 @@ export async function loadConfigs(opts = {}) {
         productDir,
         setup,
         setupFile,
+        execution,
         explicitService: explicitServices[name] || {},
         discoveredService: discovery.services[name] || null,
         suites: discovery.suitesByService[name] || {},
@@ -95,6 +102,7 @@ function normalizeServiceConfig({
   productDir,
   setup,
   setupFile,
+  execution,
   explicitService,
   discoveredService,
   suites,
@@ -113,6 +121,10 @@ function normalizeServiceConfig({
     productDir,
     suites,
   });
+  const serviceExecution = normalizeServiceExecution(explicitService.execution, {
+    name,
+    suites,
+  });
 
   if (!explicitService.databaseFrom && !database && (migrate || seed)) {
     throw new Error(
@@ -140,10 +152,12 @@ function normalizeServiceConfig({
     telemetry: normalizeTelemetryConfig(setup.telemetry),
     suites,
     testkit: {
+      execution,
       dependsOn: explicitService.dependsOn || [],
       database,
       databaseFrom: explicitService.databaseFrom,
       envFiles,
+      serviceExecution,
       serviceEnv,
       migrate,
       seed,
@@ -341,6 +355,103 @@ function normalizeSkipConfig(value, { name, productDir, suites }) {
   };
 }
 
+function normalizeServiceExecution(value, { name, suites }) {
+  if (!value) return { suites: [], files: [], fileStackModeByPath: new Map() };
+
+  const discoveredSuites = [];
+  const discoveredFiles = new Set();
+  for (const [type, typedSuites] of Object.entries(suites || {})) {
+    for (const suite of typedSuites || []) {
+      discoveredSuites.push({
+        displayType: suiteSelectionType(type, suite.framework || "k6"),
+        name: suite.name,
+      });
+      for (const file of suite.files || []) {
+        discoveredFiles.add(file);
+      }
+    }
+  }
+
+  const suiteRules = [];
+  const seenSelectors = new Set();
+  for (const rule of value.suites || []) {
+    if (!rule || typeof rule !== "object") {
+      throw new Error(`Service "${name}" execution.suites entries must be objects`);
+    }
+    const selector = String(rule.selector || "").trim();
+    if (!selector) {
+      throw new Error(`Service "${name}" execution.suites entries require a selector`);
+    }
+    const parsed = parseSuiteSelectors([selector]);
+    if (parsed.length !== 1) {
+      throw new Error(`Service "${name}" execution.suites["${selector}"] is invalid`);
+    }
+    const parsedSelector = parsed[0];
+    if (parsedSelector.kind !== "typed") {
+      throw new Error(
+        `Service "${name}" execution.suites["${selector}"] must use a typed selector like int:health`
+      );
+    }
+    if (seenSelectors.has(parsedSelector.raw)) {
+      throw new Error(
+        `Service "${name}" defines duplicate execution.suites selector "${parsedSelector.raw}"`
+      );
+    }
+    const matched = discoveredSuites.some((suite) =>
+      matchesSuiteSelectors(suite.displayType, suite.name, [parsedSelector])
+    );
+    if (!matched) {
+      throw new Error(
+        `Service "${name}" execution.suites selector "${parsedSelector.raw}" did not match any discovered suite`
+      );
+    }
+    seenSelectors.add(parsedSelector.raw);
+    suiteRules.push({
+      selector: parsedSelector,
+      stackMode: normalizeStackModeValue(
+        rule.stackMode,
+        `Service "${name}" execution.suites["${parsedSelector.raw}"].stackMode`
+      ),
+    });
+  }
+
+  const fileRules = [];
+  const seenFiles = new Set();
+  for (const [index, rule] of (value.files || []).entries()) {
+    if (!rule || typeof rule !== "object") {
+      throw new Error(`Service "${name}" execution.files entries must be objects`);
+    }
+
+    const filePath = String(rule.path || "").trim();
+    if (!filePath) {
+      throw new Error(`Service "${name}" execution.files[${index}] requires a path`);
+    }
+    if (!discoveredFiles.has(filePath)) {
+      throw new Error(
+        `Service "${name}" execution.files["${filePath}"] did not match any discovered test file`
+      );
+    }
+    if (seenFiles.has(filePath)) {
+      throw new Error(`Service "${name}" defines duplicate execution.files path "${filePath}"`);
+    }
+
+    seenFiles.add(filePath);
+    fileRules.push({
+      path: filePath,
+      stackMode: normalizeStackModeValue(
+        rule.stackMode,
+        `Service "${name}" execution.files["${filePath}"].stackMode`
+      ),
+    });
+  }
+
+  return {
+    suites: suiteRules,
+    files: fileRules,
+    fileStackModeByPath: new Map(fileRules.map((rule) => [rule.path, rule.stackMode])),
+  };
+}
+
 function inferEnvFiles(productDir, explicitService, local) {
   if (explicitService.envFile || explicitService.envFiles) {
     const files = [];
@@ -476,6 +587,18 @@ function normalizeTelemetryConfig(telemetry) {
   };
 }
 
+function normalizeRepoExecution(execution) {
+  if (!execution) {
+    return normalizeExecutionConfig({
+      workers: 1,
+      fileTimeoutSeconds: DEFAULT_FILE_TIMEOUT_SECONDS,
+      stackMode: "isolated",
+      stackCount: 1,
+    });
+  }
+  return normalizeExecutionConfig(execution);
+}
+
 function normalizePath(value) {
   return String(value || "")
     .split(path.sep)

package/lib/runner/default-runtime-runner.mjs

@@ -9,6 +9,10 @@ import { determineDefaultRuntimeFailure } from "./default-runtime-errors.mjs";
 import { formatBatchDescriptor } from "./formatting.mjs";
 import { buildExecutionEnv } from "./template.mjs";
 import { readDatabaseUrl } from "./state-io.mjs";
+import {
+  buildFileTimeoutEnv,
+  formatFileTimeoutBudgetError,
+} from "../shared/file-timeout.mjs";
 
 export async function runHttpK6Batch(targetConfig, batch, lifecycle) {
   const baseUrl = targetConfig.testkit.local?.baseUrl;
@@ -17,7 +21,7 @@ export async function runHttpK6Batch(targetConfig, batch, lifecycle) {
   }
 
   console.log(
-    `\n── ${targetConfig.workerLabel} ${batch.type}:${batch.tasks[0].suiteName}${formatBatchDescriptor(batch)} ──`
+    `\n── ${targetConfig.stackLabel} ${batch.type}:${batch.tasks[0].suiteName}${formatBatchDescriptor(batch)} ──`
   );
 
   return Promise.all(
@@ -32,7 +36,7 @@ export async function runDalBatch(targetConfig, batch, lifecycle) {
   }
 
   console.log(
-    `\n── ${targetConfig.workerLabel} ${batch.type}:${batch.tasks[0].suiteName}${formatBatchDescriptor(batch)} ──`
+    `\n── ${targetConfig.stackLabel} ${batch.type}:${batch.tasks[0].suiteName}${formatBatchDescriptor(batch)} ──`
   );
 
   return Promise.all(
@@ -46,7 +50,7 @@ async function runHttpK6Task(targetConfig, task, baseUrl, lifecycle) {
     serviceName: targetConfig.name,
     sourceFile: path.join(targetConfig.productDir, task.file),
   });
-  console.log(`·· ${targetConfig.workerLabel}:${task.suiteName} → ${task.file}`);
+  console.log(`·· ${targetConfig.stackLabel}:${task.suiteName} → ${task.file}`);
   return runDefaultRuntimeTask(
     targetConfig,
     task,
@@ -61,7 +65,7 @@ async function runDalTask(targetConfig, task, databaseUrl, lifecycle) {
     serviceName: targetConfig.name,
     sourceFile: path.join(targetConfig.productDir, task.file),
   });
-  console.log(`·· ${targetConfig.workerLabel}:${task.suiteName} → ${task.file}`);
+  console.log(`·· ${targetConfig.stackLabel}:${task.suiteName} → ${task.file}`);
   return runDefaultRuntimeTask(
     targetConfig,
     task,
@@ -72,20 +76,27 @@ async function runDalTask(targetConfig, task, databaseUrl, lifecycle) {
 
 export async function runDefaultRuntimeTask(targetConfig, task, args, lifecycle, firstLine) {
   const k6Binary = resolveK6Binary();
+  const getFirstLine = firstLine || defaultFirstLine;
   const summaryFile = buildDefaultRuntimeSummaryPath(targetConfig, task);
   fs.mkdirSync(path.dirname(summaryFile), { recursive: true });
   const startedAt = Date.now();
-  const result = await execa(
+  const fileTimeoutSeconds = targetConfig.testkit.execution.fileTimeoutSeconds;
+  const subprocess = execa(
     k6Binary,
     [...args.slice(0, 1), "--summary-export", summaryFile, ...args.slice(1)],
     {
       cwd: targetConfig.productDir,
-      env: buildExecutionEnv(targetConfig, {}, process.env),
+      env: buildExecutionEnv(
+        targetConfig,
+        buildFileTimeoutEnv(fileTimeoutSeconds, startedAt),
+        process.env
+      ),
       reject: false,
       cancelSignal: lifecycle.signal,
       forceKillAfterDelay: 5_000,
     }
   );
+  const { result, timedOut } = await settleDefaultRuntimeProcess(subprocess, fileTimeoutSeconds);
 
   const stdout = parseDefaultRuntimeOutput(result.stdout || "");
   const stderr = parseDefaultRuntimeOutput(result.stderr || "");
@@ -98,7 +109,9 @@ export async function runDefaultRuntimeTask(targetConfig, task, args, lifecycle,
     task,
     [...stdout.artifacts, ...stderr.artifacts]
   );
-  const runtimeError = determineDefaultRuntimeFailure(result, summary, firstLine);
+  const runtimeError = timedOut
+    ? formatFileTimeoutBudgetError(fileTimeoutSeconds)
+    : determineDefaultRuntimeFailure(result, summary, getFirstLine);
   const finishedAt = Date.now();
 
   return {
@@ -112,6 +125,35 @@ export async function runDefaultRuntimeTask(targetConfig, task, args, lifecycle,
   };
 }
 
+function defaultFirstLine(output) {
+  return String(output || "")
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .find(Boolean) || null;
+}
+
+async function settleDefaultRuntimeProcess(subprocess, fileTimeoutSeconds) {
+  const timeoutMs = fileTimeoutSeconds * 1000 + 1_000;
+  let timeoutHandle = null;
+  let timedOut = false;
+
+  try {
+    return await Promise.race([
+      subprocess.then((result) => ({ result, timedOut })),
+      new Promise((resolve) => {
+        timeoutHandle = setTimeout(async () => {
+          timedOut = true;
+          subprocess.kill("SIGTERM");
+          const result = await subprocess.catch((error) => error);
+          resolve({ result, timedOut: true });
+        }, timeoutMs);
+      }),
+    ]);
+  } finally {
+    if (timeoutHandle) clearTimeout(timeoutHandle);
+  }
+}
+
 export function buildDefaultRuntimeSummaryPath(targetConfig, task) {
   return path.join(
     targetConfig.stateDir || path.join(targetConfig.productDir, ".testkit"),

package/lib/runner/execution-config.mjs

@@ -0,0 +1,124 @@
+import {
+  DEFAULT_FILE_TIMEOUT_SECONDS,
+  normalizeFileTimeoutSeconds,
+  parseFileTimeoutOption,
+} from "../shared/file-timeout.mjs";
+
+export const STACK_MODES = new Set(["shared", "pooled", "isolated"]);
+
+export { DEFAULT_FILE_TIMEOUT_SECONDS, parseFileTimeoutOption };
+
+export function parseWorkersOption(value) {
+  return parsePositiveInteger(value, "--workers");
+}
+
+export function parseStackCountOption(value) {
+  return parsePositiveInteger(value, "--stack-count");
+}
+
+export function parseStackModeOption(value) {
+  return normalizeStackModeValue(value, "--stack-mode");
+}
+
+export function normalizeStackModeValue(value, label = "execution.stackMode") {
+  const normalized = String(value || "").trim();
+  if (!STACK_MODES.has(normalized)) {
+    throw new Error(
+      `Invalid ${label} value "${value}". Expected one of: shared, pooled, isolated.`
+    );
+  }
+  return normalized;
+}
+
+export function resolveExecutionConfig({ cli = {}, repo = {} } = {}) {
+  return normalizeExecutionConfig({
+    workers: cli.workers ?? repo.workers ?? 1,
+    stackMode: cli.stackMode ?? repo.stackMode ?? "isolated",
+    stackCount: cli.stackCount ?? repo.stackCount ?? null,
+    fileTimeoutSeconds:
+      cli.fileTimeoutSeconds ?? repo.fileTimeoutSeconds ?? DEFAULT_FILE_TIMEOUT_SECONDS,
+  });
+}
+
+export function normalizeExecutionConfig(input = {}) {
+  const workers = normalizePositiveInteger(input.workers, "execution.workers");
+  const stackMode = normalizeStackMode(input.stackMode);
+  const explicitStackCount =
+    input.stackCount == null ? null : normalizePositiveInteger(input.stackCount, "execution.stackCount");
+  const fileTimeoutSeconds = normalizeFileTimeoutSeconds(
+    input.fileTimeoutSeconds ?? DEFAULT_FILE_TIMEOUT_SECONDS
+  );
+
+  if (stackMode === "shared") {
+    if (explicitStackCount !== null && explicitStackCount !== 1) {
+      throw new Error(`execution.stackCount must be 1 when stackMode is "shared".`);
+    }
+    return {
+      workers,
+      stackMode,
+      stackCount: 1,
+      fileTimeoutSeconds,
+    };
+  }
+
+  if (stackMode === "pooled") {
+    return {
+      workers,
+      stackMode,
+      stackCount: explicitStackCount ?? 1,
+      fileTimeoutSeconds,
+    };
+  }
+
+  if (explicitStackCount !== null && explicitStackCount !== workers) {
+    throw new Error(
+      `execution.stackCount must equal execution.workers when stackMode is "isolated".`
+    );
+  }
+  return {
+    workers,
+    stackMode,
+    stackCount: workers,
+    fileTimeoutSeconds,
+  };
+}
+
+export function buildStackIds(execution) {
+  if (execution.stackMode === "shared") {
+    return ["shared"];
+  }
+
+  return Array.from({ length: execution.stackCount }, (_unused, index) => `stack-${index + 1}`);
+}
+
+export function resolveBatchStackMode(defaultMode, suiteMode = null) {
+  if (suiteMode == null) return defaultMode;
+  return normalizeStackMode(suiteMode);
+}
+
+export function resolveBatchAccessMode({ framework, type, stackMode }) {
+  if (stackMode === "isolated") return "exclusive";
+  if ((framework || "k6") === "playwright") return "exclusive";
+  if (type === "dal") return "exclusive";
+  return "shared";
+}
+
+function normalizeStackMode(value) {
+  return normalizeStackModeValue(value, "execution.stackMode");
+}
+
+function parsePositiveInteger(value, label) {
+  const parsed = Number.parseInt(String(value), 10);
+  if (!Number.isInteger(parsed) || parsed <= 0) {
+    throw new Error(`Invalid ${label} value "${value}". Expected a positive integer.`);
+  }
+  return parsed;
+}
+
+function normalizePositiveInteger(value, label) {
+  const parsed = Number(value);
+  if (!Number.isInteger(parsed) || parsed <= 0) {
+    throw new Error(`${label} must be a positive integer.`);
+  }
+  return parsed;
+}