@elench/testkit 0.1.41 → 0.1.43

package/lib/runner/lifecycle.mjs

@@ -1,11 +1,13 @@
 import fs from "fs";
 import path from "path";
 import crypto from "crypto";
+import { execFileSync } from "child_process";
 import { destroyRuntimeDatabase, cleanupOrphanedLocalInfrastructure } from "../database/index.mjs";
 
 const RUN_SCHEMA_VERSION = 1;
 const RUNS_DIRNAME = path.join(".testkit", "_runs");
 const TERMINATION_TIMEOUT_MS = 5_000;
+const SHUTDOWN_HOLD_TIMEOUT_MS = 10_000;
 
 export function createRunLifecycle(productDir) {
   const runId = buildRunId();
@@ -25,6 +27,9 @@ export function createRunLifecycle(productDir) {
     runtimeStateDirs: [],
   };
   const signalListeners = [];
+  const managedProcesses = new Set();
+  let shutdownHold = null;
+  let shutdownHoldTimeout = null;
 
   function persist() {
     fs.mkdirSync(getRunsDir(productDir), { recursive: true });
@@ -36,6 +41,24 @@ export function createRunLifecycle(productDir) {
     persist();
   }
 
+  function ensureShutdownHold() {
+    if (shutdownHold) return;
+    shutdownHold = setInterval(() => {}, 1_000);
+    shutdownHoldTimeout = setTimeout(() => {
+      releaseShutdownHold();
+    }, SHUTDOWN_HOLD_TIMEOUT_MS);
+  }
+
+  function releaseShutdownHold() {
+    if (!shutdownHold) return;
+    clearInterval(shutdownHold);
+    shutdownHold = null;
+    if (shutdownHoldTimeout) {
+      clearTimeout(shutdownHoldTimeout);
+      shutdownHoldTimeout = null;
+    }
+  }
+
   const api = {
     runId,
     manifestPath,
@@ -55,9 +78,18 @@ export function createRunLifecycle(productDir) {
       });
     },
     requestStop(reason = "interrupted") {
+      ensureShutdownHold();
       if (!abortController.signal.aborted) {
         abortController.abort(new Error(`testkit run interrupted (${reason})`));
       }
+      for (const entry of managedProcesses) {
+        try {
+          entry.terminate?.();
+        } catch {
+          // Best-effort interruption only.
+        }
+      }
+      terminateDescendantProcesses(process.pid, "SIGINT");
       mutate((draft) => {
         draft.status = "interrupting";
         draft.interruptReason = reason;
@@ -72,7 +104,22 @@ export function createRunLifecycle(productDir) {
         }
       });
     },
-    registerService(config, child, cwd) {
+    registerProcess(child, terminate) {
+      if (!child) return;
+      managedProcesses.add({
+        child,
+        terminate,
+      });
+    },
+    unregisterProcess(childPid) {
+      for (const entry of managedProcesses) {
+        if (entry.child?.pid === childPid) {
+          managedProcesses.delete(entry);
+        }
+      }
+    },
+    registerService(config, child, cwd, terminate) {
+      api.registerProcess(child, terminate);
       const ports = collectConfigPorts(config);
       mutate((draft) => {
         draft.services = draft.services.filter((service) => service.pid !== child.pid);
@@ -89,6 +136,7 @@ export function createRunLifecycle(productDir) {
       });
     },
     unregisterService(childPid) {
+      api.unregisterProcess(childPid);
       mutate((draft) => {
         draft.services = draft.services.filter((service) => service.pid !== childPid);
       });
@@ -116,6 +164,9 @@ export function createRunLifecycle(productDir) {
     removeManifest() {
       removeManifestFile(productDir, runId);
     },
+    dispose() {
+      releaseShutdownHold();
+    },
   };
 
   return api;
@@ -264,6 +315,53 @@ function killProcessGroup(pid, signal) {
   }
 }
 
+function terminateDescendantProcesses(rootPid, signal) {
+  const descendants = listDescendantPids(rootPid);
+  for (const pid of descendants) {
+    try {
+      process.kill(pid, signal);
+    } catch (error) {
+      if (error?.code !== "ESRCH") {
+        // Best-effort interruption only.
+      }
+    }
+  }
+}
+
+function listDescendantPids(rootPid) {
+  try {
+    const output = execFileSync("ps", ["-eo", "pid=,ppid="], {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "ignore"],
+    });
+    const childrenByParent = new Map();
+    for (const line of output.split(/\r?\n/)) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      const [pidRaw, parentRaw] = trimmed.split(/\s+/);
+      const pid = Number(pidRaw);
+      const parentPid = Number(parentRaw);
+      if (!Number.isInteger(pid) || !Number.isInteger(parentPid)) continue;
+      const siblings = childrenByParent.get(parentPid) || [];
+      siblings.push(pid);
+      childrenByParent.set(parentPid, siblings);
+    }
+
+    const descendants = [];
+    const stack = [...(childrenByParent.get(rootPid) || [])];
+    while (stack.length > 0) {
+      const pid = stack.pop();
+      descendants.push(pid);
+      for (const childPid of childrenByParent.get(pid) || []) {
+        stack.push(childPid);
+      }
+    }
+    return descendants;
+  } catch {
+    return [];
+  }
+}
+
 async function waitForPidExit(pid, timeoutMs) {
   const startedAt = Date.now();
   while (Date.now() - startedAt < timeoutMs) {

package/lib/runner/orchestrator.mjs

@@ -15,6 +15,7 @@ import {
   summarizeDbBackend,
 } from "./results.mjs";
 import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
+import { applyKnownFailuresToArtifacts, loadKnownFailuresConfig } from "./triage.mjs";
 import { buildRunSummaryLines, formatError } from "./formatting.mjs";
 import {
   loadTimings,
@@ -56,6 +57,10 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     },
     testkitVersion: readPackageMetadata().version,
   };
+  const knownFailures = loadKnownFailuresConfig(
+    productDir,
+    configs[0]?.testkit?.reporting || null
+  );
   const requestedFiles = opts.fileNames || [];
   if (requestedFiles.length > 0) {
     const unmatchedFiles = findUnmatchedRequestedFiles(
@@ -106,6 +111,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
   const executedPlans = servicePlans.filter((plan) => !plan.skipped);
   let workerCount = 0;
   let runtimeInstanceCount = 0;
+  let runtimeStats = [];
   let exitCode = 0;
   const lifecycle = createRunLifecycle(productDir);
   lifecycle.markRunning();
@@ -155,6 +161,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
             }
           }
         }
+        runtimeStats = runtimeManager.getStats();
       } finally {
         await runtimeManager.cleanupAll();
       }
@@ -166,7 +173,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     results = configs.map((config) =>
       finalizeServiceResult(trackers.get(config.name), startedAt, finishedAt)
     );
-    const artifact = buildRunArtifact({
+    const runArtifact = buildRunArtifact({
       productDir,
       results,
       startedAt,
@@ -174,6 +181,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
       execution,
       workerCount,
       runtimeInstanceCount,
+      runtimeStats,
       typeValues,
       suiteSelectors,
       fileNames: requestedFiles,
@@ -182,12 +190,8 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
       metadata,
       summarizeDbBackend,
     });
-
-    writeRunArtifact(productDir, artifact);
-    if (opts.writeStatus) {
-      writeStatusArtifact(
-        productDir,
-        buildStatusArtifact({
+    const statusArtifact = opts.writeStatus
+      ? buildStatusArtifact({
           productDir,
           results,
           typeValues,
@@ -197,14 +201,26 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
           serviceFilter: opts.serviceFilter || null,
           metadata,
         })
-      );
+      : null;
+    const enrichedArtifacts = applyKnownFailuresToArtifacts(
+      runArtifact,
+      statusArtifact,
+      knownFailures
+    );
+
+    writeRunArtifact(productDir, enrichedArtifacts.runArtifact);
+    if (opts.writeStatus) {
+      writeStatusArtifact(productDir, enrichedArtifacts.statusArtifact);
     }
 
     printRunSummary(results, finishedAt - startedAt);
-    await reportTelemetry(telemetry, artifact);
+    await reportTelemetry(telemetry, enrichedArtifacts.runArtifact);
     if (results.some((result) => result.failed)) exitCode = 1;
     if (lifecycle.isStopRequested()) exitCode = Math.max(exitCode, 130);
   } finally {
+    if (lifecycle.isStopRequested()) {
+      exitCode = Math.max(exitCode, 130);
+    }
     lifecycle.removeSignalHandlers();
     lifecycle.markFinished(
       exitCode === 0 ? "finished" : lifecycle.isStopRequested() ? "interrupted" : "failed"
@@ -212,6 +228,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     await cleanupRunById(productDir, lifecycle.runId);
     await cleanupRuns(productDir, { includeActive: false });
     lifecycle.removeManifest();
+    lifecycle.dispose();
     process.exitCode = exitCode;
   }
 }

package/lib/runner/planning.mjs

@@ -113,6 +113,10 @@ export function buildRuntimeGraphs(servicePlans) {
     if (existing) {
       existing.targetNames.push(plan.config.name);
       existing.instanceCount = Math.max(existing.instanceCount, plan.config.testkit.runtime.instances);
+      existing.maxConcurrentTasks = Math.min(
+        existing.maxConcurrentTasks,
+        resolveGraphMaxConcurrentTasks(plan.runtimeConfigs)
+      );
       continue;
     }
 
@@ -123,6 +127,7 @@ export function buildRuntimeGraphs(servicePlans) {
       targetNames: [plan.config.name],
       dirName: buildGraphDirName(plan.runtimeNames),
       instanceCount: plan.config.testkit.runtime.instances,
+      maxConcurrentTasks: resolveGraphMaxConcurrentTasks(plan.runtimeConfigs),
     };
     graphs.push(graph);
     graphByRuntimeKey.set(plan.runtimeKey, graph);
@@ -166,6 +171,7 @@ export function buildTaskQueue(servicePlans, graphs, timings) {
           orderIndex: suite.orderIndex,
           file,
           locks: resolveTaskLocks(plan.config, suite, file),
+          resourceCost: 1,
           timingKey,
           estimatedDurationMs: estimateTaskDuration(timings, timingKey, suite),
         });
@@ -183,16 +189,25 @@ export function buildTaskQueue(servicePlans, graphs, timings) {
   );
 }
 
-export function claimNextTask(queue, preferredGraphKey) {
+export function claimNextTask(queue, preferredGraphKey, isRunnable = () => true) {
   if (queue.length === 0) return null;
 
-  let index = -1;
-  if (preferredGraphKey) {
-    index = queue.findIndex((task) => task.graphKey === preferredGraphKey);
+  const preferredIndexes = [];
+  const fallbackIndexes = [];
+  for (let index = 0; index < queue.length; index += 1) {
+    if (preferredGraphKey && queue[index].graphKey === preferredGraphKey) {
+      preferredIndexes.push(index);
+    } else {
+      fallbackIndexes.push(index);
+    }
+  }
+
+  for (const index of [...preferredIndexes, ...fallbackIndexes]) {
+    if (!isRunnable(queue[index])) continue;
+    return queue.splice(index, 1)[0];
   }
-  if (index === -1) index = 0;
 
-  return queue.splice(index, 1)[0];
+  return null;
 }
 
 export function buildGraphDirName(runtimeNames) {
@@ -267,3 +282,10 @@ function normalizePathSeparators(filePath) {
 function slugSegment(value) {
   return value.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "") || "svc";
 }
+
+function resolveGraphMaxConcurrentTasks(runtimeConfigs) {
+  return runtimeConfigs.reduce(
+    (currentMin, config) => Math.min(currentMin, config.testkit.runtime.maxConcurrentTasks),
+    Number.POSITIVE_INFINITY
+  );
+}

package/lib/runner/planning.test.mjs

@@ -22,6 +22,7 @@ function makeConfig(name, extras = {}) {
       },
       runtime: providedTestkit.runtime || {
         instances: 1,
+        maxConcurrentTasks: Number.POSITIVE_INFINITY,
       },
       requirements: providedTestkit.requirements || {
         suites: [],
@@ -156,6 +157,7 @@ describe("runner-planning", () => {
       testkit: {
         runtime: {
           instances: 3,
+          maxConcurrentTasks: 2,
         },
         requirements: {
           suites: [
@@ -226,6 +228,7 @@ describe("runner-planning", () => {
       testkit: {
         runtime: {
           instances: 2,
+          maxConcurrentTasks: 4,
         },
       },
     });
@@ -234,6 +237,7 @@ describe("runner-planning", () => {
       testkit: {
         runtime: {
           instances: 1,
+          maxConcurrentTasks: 2,
         },
       },
     });
@@ -282,11 +286,13 @@ describe("runner-planning", () => {
       expect.objectContaining({
         key: "api",
         instanceCount: 2,
+        maxConcurrentTasks: 4,
         targetNames: ["api"],
       }),
       expect.objectContaining({
         key: "api|frontend",
         instanceCount: 1,
+        maxConcurrentTasks: 2,
         targetNames: ["frontend"],
       }),
     ]);
@@ -304,4 +310,36 @@ describe("runner-planning", () => {
       graphKey: "api",
     });
   });
+
+  it("skips blocked preferred-graph work and claims the next runnable task", () => {
+    const queue = [
+      {
+        id: 1,
+        graphKey: "api",
+        file: "blocked.js",
+      },
+      {
+        id: 2,
+        graphKey: "api|frontend",
+        file: "runnable.js",
+      },
+    ];
+
+    const claimed = claimNextTask(
+      queue,
+      "api",
+      (task) => task.file === "runnable.js"
+    );
+
+    expect(claimed).toMatchObject({
+      id: 2,
+      file: "runnable.js",
+    });
+    expect(queue).toEqual([
+      expect.objectContaining({
+        id: 1,
+        file: "blocked.js",
+      }),
+    ]);
+  });
 });

package/lib/runner/playwright-config.mjs

@@ -28,6 +28,9 @@ export function ensurePlaywrightTestConfig(targetConfig, cwd, requestedFiles, le
       `  testDir: ${JSON.stringify(cwd)},\n` +
       `  testMatch: ${JSON.stringify(normalizedFiles)},\n` +
       `  outputDir: ${JSON.stringify(outputDir)},\n` +
+      `  workers: 1,\n` +
+      `  fullyParallel: false,\n` +
+      `  webServer: undefined,\n` +
       `};\n`;
   } else {
     source =
@@ -35,6 +38,8 @@ export function ensurePlaywrightTestConfig(targetConfig, cwd, requestedFiles, le
       `  testDir: ${JSON.stringify(cwd)},\n` +
       `  testMatch: ${JSON.stringify(normalizedFiles)},\n` +
       `  outputDir: ${JSON.stringify(outputDir)},\n` +
+      `  workers: 1,\n` +
+      `  fullyParallel: false,\n` +
       `};\n`;
   }
 

package/lib/runner/playwright-config.test.mjs

@@ -31,7 +31,7 @@ describe("runner-playwright-config", () => {
     fs.mkdirSync(cwd, { recursive: true });
     fs.writeFileSync(
       path.join(cwd, "playwright.config.mjs"),
-      "export default { outputDir: 'shared-test-results' };\n"
+      "export default { outputDir: 'shared-test-results', workers: 8, fullyParallel: true, webServer: { command: 'npm run dev', url: 'http://127.0.0.1:3000' } };\n"
     );
 
     const configPath = ensurePlaywrightTestConfig(
@@ -44,10 +44,15 @@ describe("runner-playwright-config", () => {
 
     const expectedOutputDir = resolvePlaywrightOutputDir(leaseDir);
     expect(generated.default.outputDir).toBe(expectedOutputDir);
+    expect(generated.default.workers).toBe(1);
+    expect(generated.default.fullyParallel).toBe(false);
+    expect(generated.default.webServer).toBeUndefined();
     expect(fs.existsSync(expectedOutputDir)).toBe(true);
     expect(fs.readFileSync(configPath, "utf8")).toContain(
       `outputDir: ${JSON.stringify(expectedOutputDir)}`
     );
+    expect(fs.readFileSync(configPath, "utf8")).toContain("workers: 1");
+    expect(fs.readFileSync(configPath, "utf8")).toContain("fullyParallel: false");
   });
 
   it("requires a lease-scoped directory", () => {

package/lib/runner/playwright-runner.mjs

@@ -8,6 +8,7 @@ import { ensurePlaywrightTestConfig } from "./playwright-config.mjs";
 import { printBufferedOutput } from "./processes.mjs";
 import { normalizePathSeparators } from "./state.mjs";
 import { buildPlaywrightEnv } from "./template.mjs";
+import { killChildProcess } from "./processes.mjs";
 
 export async function runPlaywrightTask(targetConfig, task, lifecycle, lease) {
   const local = targetConfig.testkit.local;
@@ -17,11 +18,12 @@ export async function runPlaywrightTask(targetConfig, task, lifecycle, lease) {
     );
   }
 
-  console.log(`\n── ${targetConfig.runtimeLabel} playwright:${targetConfig.name} (${task.file}) ──`);
-
   const cwd = resolveServiceCwd(targetConfig.productDir, local.cwd);
   const requestedFile = path.relative(cwd, path.join(targetConfig.productDir, task.file));
   const playwrightConfigPath = ensurePlaywrightTestConfig(targetConfig, cwd, [requestedFile], lease);
+  if (lifecycle.isStopRequested()) {
+    throw new Error(`testkit run interrupted before starting ${task.file}`);
+  }
   const startedAt = Date.now();
   const fileTimeoutSeconds = targetConfig.testkit.execution.fileTimeoutSeconds;
   const subprocess = execa(
@@ -31,11 +33,25 @@ export async function runPlaywrightTask(targetConfig, task, lifecycle, lease) {
       cwd,
       env: buildPlaywrightEnv(targetConfig, local.baseUrl, lease, process.env),
       reject: false,
-      cancelSignal: lifecycle.signal,
       forceKillAfterDelay: 5_000,
     }
   );
-  const { result, timedOut } = await settleSubprocess(subprocess, fileTimeoutSeconds);
+  lifecycle.registerProcess(subprocess, () => {
+    killChildProcess(subprocess, "SIGINT");
+  });
+  if (lifecycle.isStopRequested()) {
+    const interruptSubprocess = () => killChildProcess(subprocess, "SIGINT");
+    if (subprocess.pid) interruptSubprocess();
+    else subprocess.once?.("spawn", interruptSubprocess);
+  }
+  console.log(`\n── ${targetConfig.runtimeLabel} playwright:${targetConfig.name} (${task.file}) ──`);
+  let result;
+  let timedOut;
+  try {
+    ({ result, timedOut } = await settleSubprocess(subprocess, fileTimeoutSeconds));
+  } finally {
+    lifecycle.unregisterProcess(subprocess.pid);
+  }
 
   if (result.stderr) {
     printBufferedOutput(result.stderr, `[${targetConfig.runtimeLabel}:${targetConfig.name}:playwright]`);
@@ -60,5 +76,6 @@ export async function runPlaywrightTask(targetConfig, task, lifecycle, lease) {
     durationMs: fileResult?.durationMs > 0 ? fileResult.durationMs : durationMs,
     startedAt,
     finishedAt,
+    failureDetails: timedOut ? [] : fileResult?.failureDetails || [],
   };
 }

package/lib/runner/reporting.mjs

@@ -23,9 +23,15 @@ export function buildStatusArtifact({
           path: file.path,
           status: file.status,
         };
+        if (file.error) {
+          test.error = file.error;
+        }
         if (file.reason) {
           test.reason = file.reason;
         }
+        if (Array.isArray(file.failureDetails) && file.failureDetails.length > 0) {
+          test.failureDetails = file.failureDetails;
+        }
         tests.push(test);
       }
     }
@@ -72,7 +78,7 @@ export function buildStatusArtifact({
     scope.serviceFilter === null;
 
   return {
-    schemaVersion: 3,
+    schemaVersion: 4,
     source: "testkit",
     notice: "Generated file. Do not edit manually.",
     product: {
@@ -97,6 +103,7 @@ export function buildRunArtifact({
   execution,
   workerCount,
   runtimeInstanceCount,
+  runtimeStats,
   typeValues,
   suiteSelectors,
   fileNames,
@@ -120,7 +127,7 @@ export function buildRunArtifact({
   const dbBackend = summarizeDbBackend(results);
 
   return {
-    schemaVersion: 3,
+    schemaVersion: 4,
     source: "testkit",
     generatedAt: new Date(finishedAt).toISOString(),
     product: {
@@ -138,6 +145,7 @@ export function buildRunArtifact({
       fileTimeoutSeconds: execution.fileTimeoutSeconds,
       workerCount,
       runtimeInstanceCount,
+      runtimeStats: runtimeStats || [],
       dbBackend,
       types: typeValues,
       suiteSelectors: suiteSelectors.map((selector) => selector.raw),

package/lib/runner/reporting.test.mjs

@@ -78,7 +78,7 @@ describe("runner reporting", () => {
     });
 
     expect(artifact.product.name).toBe("my-product");
-    expect(artifact.schemaVersion).toBe(3);
+    expect(artifact.schemaVersion).toBe(4);
     expect(artifact.run).toMatchObject({
       workers: 2,
       fileTimeoutSeconds: 60,
@@ -149,7 +149,7 @@ describe("runner reporting", () => {
     });
 
     expect(status).toEqual({
-      schemaVersion: 3,
+      schemaVersion: 4,
       source: "testkit",
       notice: "Generated file. Do not edit manually.",
       product: {

package/lib/runner/results.mjs

@@ -1,4 +1,5 @@
 import path from "path";
+import { mergeFailureDetails } from "./failure-details.mjs";
 
 export function buildServiceTrackers(servicePlans, startedAt) {
   const trackers = new Map();
@@ -46,6 +47,7 @@ export function buildServiceTrackers(servicePlans, startedAt) {
               reason: null,
               status: "not_run",
               artifacts: [],
+              failureDetails: [],
             },
           ];
         }),
@@ -59,6 +61,7 @@ export function buildServiceTrackers(servicePlans, startedAt) {
             reason: file.reason,
             status: "skipped",
             artifacts: [],
+            failureDetails: [],
           },
         ]),
       ]),
@@ -121,6 +124,7 @@ export function recordTaskOutcome(trackers, task, outcome, finishedAt = Date.now
     existingFileResult.reason = outcome.reason || null;
     existingFileResult.status = status;
     existingFileResult.artifacts = Array.isArray(outcome.artifacts) ? outcome.artifacts : [];
+    existingFileResult.failureDetails = mergeFailureDetails(outcome.failureDetails);
   } else {
     suite.fileResultsByPath.set(normalizedPath, {
       path: normalizedPath,
@@ -130,6 +134,7 @@ export function recordTaskOutcome(trackers, task, outcome, finishedAt = Date.now
       reason: outcome.reason || null,
       status,
       artifacts: Array.isArray(outcome.artifacts) ? outcome.artifacts : [],
+      failureDetails: mergeFailureDetails(outcome.failureDetails),
     });
   }
   if (status === "failed" && !suite.failedFileSet.has(task.file)) {
@@ -247,6 +252,9 @@ function finalizeSuite(suite) {
       durationMs: file.durationMs,
       error: file.error,
       reason: file.reason,
+      ...(Array.isArray(file.failureDetails) && file.failureDetails.length > 0
+        ? { failureDetails: file.failureDetails }
+        : {}),
       ...(Array.isArray(file.artifacts) && file.artifacts.length > 0
         ? { artifacts: file.artifacts }
         : {}),