@elench/testkit 0.1.40 → 0.1.42

package/lib/runner/lifecycle.mjs

@@ -1,11 +1,13 @@
 import fs from "fs";
 import path from "path";
 import crypto from "crypto";
+import { execFileSync } from "child_process";
 import { destroyRuntimeDatabase, cleanupOrphanedLocalInfrastructure } from "../database/index.mjs";
 
 const RUN_SCHEMA_VERSION = 1;
 const RUNS_DIRNAME = path.join(".testkit", "_runs");
 const TERMINATION_TIMEOUT_MS = 5_000;
+const SHUTDOWN_HOLD_TIMEOUT_MS = 10_000;
 
 export function createRunLifecycle(productDir) {
   const runId = buildRunId();
@@ -21,10 +23,13 @@ export function createRunLifecycle(productDir) {
     interruptReason: null,
     services: [],
     graphDirs: [],
-    stackStateDirs: [],
+    runtimeDirs: [],
     runtimeStateDirs: [],
   };
   const signalListeners = [];
+  const managedProcesses = new Set();
+  let shutdownHold = null;
+  let shutdownHoldTimeout = null;
 
   function persist() {
     fs.mkdirSync(getRunsDir(productDir), { recursive: true });
@@ -36,6 +41,24 @@ export function createRunLifecycle(productDir) {
     persist();
   }
 
+  function ensureShutdownHold() {
+    if (shutdownHold) return;
+    shutdownHold = setInterval(() => {}, 1_000);
+    shutdownHoldTimeout = setTimeout(() => {
+      releaseShutdownHold();
+    }, SHUTDOWN_HOLD_TIMEOUT_MS);
+  }
+
+  function releaseShutdownHold() {
+    if (!shutdownHold) return;
+    clearInterval(shutdownHold);
+    shutdownHold = null;
+    if (shutdownHoldTimeout) {
+      clearTimeout(shutdownHoldTimeout);
+      shutdownHoldTimeout = null;
+    }
+  }
+
   const api = {
     runId,
     manifestPath,
@@ -55,9 +78,18 @@ export function createRunLifecycle(productDir) {
       });
     },
     requestStop(reason = "interrupted") {
+      ensureShutdownHold();
       if (!abortController.signal.aborted) {
         abortController.abort(new Error(`testkit run interrupted (${reason})`));
       }
+      for (const entry of managedProcesses) {
+        try {
+          entry.terminate?.();
+        } catch {
+          // Best-effort interruption only.
+        }
+      }
+      terminateDescendantProcesses(process.pid, "SIGINT");
       mutate((draft) => {
         draft.status = "interrupting";
         draft.interruptReason = reason;
@@ -66,19 +98,34 @@ export function createRunLifecycle(productDir) {
     trackGraphContext(context) {
       mutate((draft) => {
         pushUnique(draft.graphDirs, context.graphDir);
-        pushUnique(draft.stackStateDirs, context.stackStateDir);
+        pushUnique(draft.runtimeDirs, context.runtimeDir);
         for (const runtimeConfig of context.runtimeConfigs || []) {
           if (runtimeConfig.stateDir) pushUnique(draft.runtimeStateDirs, runtimeConfig.stateDir);
         }
       });
     },
-    registerService(config, child, cwd) {
+    registerProcess(child, terminate) {
+      if (!child) return;
+      managedProcesses.add({
+        child,
+        terminate,
+      });
+    },
+    unregisterProcess(childPid) {
+      for (const entry of managedProcesses) {
+        if (entry.child?.pid === childPid) {
+          managedProcesses.delete(entry);
+        }
+      }
+    },
+    registerService(config, child, cwd, terminate) {
+      api.registerProcess(child, terminate);
       const ports = collectConfigPorts(config);
       mutate((draft) => {
         draft.services = draft.services.filter((service) => service.pid !== child.pid);
         draft.services.push({
           serviceName: config.name,
-          stackLabel: config.stackLabel,
+          runtimeLabel: config.runtimeLabel,
           command: config.testkit.local?.start || null,
           cwd,
           pid: child.pid,
@@ -89,6 +136,7 @@ export function createRunLifecycle(productDir) {
       });
     },
     unregisterService(childPid) {
+      api.unregisterProcess(childPid);
       mutate((draft) => {
         draft.services = draft.services.filter((service) => service.pid !== childPid);
       });
@@ -116,6 +164,9 @@ export function createRunLifecycle(productDir) {
     removeManifest() {
       removeManifestFile(productDir, runId);
     },
+    dispose() {
+      releaseShutdownHold();
+    },
   };
 
   return api;
@@ -195,8 +246,8 @@ export function findPortOwner(productDir, { host, port }) {
 }
 
 export function formatRunSummary(manifest) {
-  const stackLabels = [...new Set((manifest.services || []).map((service) => service.stackLabel).filter(Boolean))];
-  return `${manifest.runId} pid=${manifest.pid}${stackLabels.length > 0 ? ` stacks=${stackLabels.join(",")}` : ""}`;
+  const runtimeLabels = [...new Set((manifest.services || []).map((service) => service.runtimeLabel).filter(Boolean))];
+  return `${manifest.runId} pid=${manifest.pid}${runtimeLabels.length > 0 ? ` runtimes=${runtimeLabels.join(",")}` : ""}`;
 }
 
 export function isPidRunning(pid) {
@@ -220,8 +271,8 @@ async function cleanupRunManifest(productDir, manifest, { removeRuntimeState = f
       await destroyRuntimeDatabase({ productDir, stateDir });
     }
 
-    for (const stackStateDir of [...new Set(manifest.stackStateDirs || [])].sort((a, b) => b.length - a.length)) {
-      fs.rmSync(stackStateDir, { recursive: true, force: true });
+    for (const runtimeDir of [...new Set(manifest.runtimeDirs || [])].sort((a, b) => b.length - a.length)) {
+      fs.rmSync(runtimeDir, { recursive: true, force: true });
     }
 
     for (const graphDir of [...new Set(manifest.graphDirs || [])].sort((a, b) => b.length - a.length)) {
@@ -264,6 +315,53 @@ function killProcessGroup(pid, signal) {
   }
 }
 
+function terminateDescendantProcesses(rootPid, signal) {
+  const descendants = listDescendantPids(rootPid);
+  for (const pid of descendants) {
+    try {
+      process.kill(pid, signal);
+    } catch (error) {
+      if (error?.code !== "ESRCH") {
+        // Best-effort interruption only.
+      }
+    }
+  }
+}
+
+function listDescendantPids(rootPid) {
+  try {
+    const output = execFileSync("ps", ["-eo", "pid=,ppid="], {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "ignore"],
+    });
+    const childrenByParent = new Map();
+    for (const line of output.split(/\r?\n/)) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      const [pidRaw, parentRaw] = trimmed.split(/\s+/);
+      const pid = Number(pidRaw);
+      const parentPid = Number(parentRaw);
+      if (!Number.isInteger(pid) || !Number.isInteger(parentPid)) continue;
+      const siblings = childrenByParent.get(parentPid) || [];
+      siblings.push(pid);
+      childrenByParent.set(parentPid, siblings);
+    }
+
+    const descendants = [];
+    const stack = [...(childrenByParent.get(rootPid) || [])];
+    while (stack.length > 0) {
+      const pid = stack.pop();
+      descendants.push(pid);
+      for (const childPid of childrenByParent.get(pid) || []) {
+        stack.push(childPid);
+      }
+    }
+    return descendants;
+  } catch {
+    return [];
+  }
+}
+
 async function waitForPidExit(pid, timeoutMs) {
   const startedAt = Date.now();
   while (Date.now() - startedAt < timeoutMs) {

package/lib/runner/orchestrator.mjs

@@ -2,7 +2,7 @@ import {
   applyShard,
   buildRuntimeGraphs,
   buildTaskQueue,
-  claimNextBatch,
+  claimNextTask,
   collectSuites,
   resolveRuntimeConfigs,
 } from "./planning.mjs";
@@ -36,7 +36,7 @@ import {
   safeUsername,
 } from "./metadata.mjs";
 import { resolveExecutionConfig } from "./execution-config.mjs";
-import { createStackManager } from "./stack-manager.mjs";
+import { createRuntimeManager } from "./runtime-manager.mjs";
 import { createWorker, runWorker } from "./worker-loop.mjs";
 import { findUnmatchedRequestedFiles, isFullRunSelection } from "./selection.mjs";
 import { uploadTelemetryArtifact } from "../telemetry/index.mjs";
@@ -105,7 +105,8 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
   const trackers = buildServiceTrackers(servicePlans, startedAt);
   const executedPlans = servicePlans.filter((plan) => !plan.skipped);
   let workerCount = 0;
-  let stackCount = 0;
+  let runtimeInstanceCount = 0;
+  let runtimeStats = [];
   let exitCode = 0;
   const lifecycle = createRunLifecycle(productDir);
   lifecycle.markRunning();
@@ -119,14 +120,13 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
       const graphs = buildRuntimeGraphs(executedPlans);
       const queue = buildTaskQueue(executedPlans, graphs, timings);
       workerCount = Math.max(1, Math.min(execution.workers, queue.length));
-      stackCount = execution.stackMode === "shared" ? 1 : execution.stackCount;
+      runtimeInstanceCount = graphs.reduce((sum, graph) => sum + graph.instanceCount, 0);
       const workers = Array.from({ length: workerCount }, (_unused, index) =>
         createWorker(index + 1, productDir)
       );
-      const stackManager = createStackManager({
+      const runtimeManager = createRuntimeManager({
         productDir,
         graphs,
-        execution,
         lifecycle,
       });
       const timingUpdates = [];
@@ -137,11 +137,11 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
             runWorker(
               worker,
               queue,
-              stackManager,
+              runtimeManager,
               trackers,
               timingUpdates,
               lifecycle,
-              claimNextBatch,
+              claimNextTask,
               recordTaskOutcome,
               recordGraphError
             )
@@ -156,8 +156,9 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
             }
           }
         }
+        runtimeStats = runtimeManager.getStats();
       } finally {
-        await stackManager.cleanupAll();
+        await runtimeManager.cleanupAll();
       }
 
       saveTimings(productDir, timings, timingUpdates);
@@ -174,7 +175,8 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
       finishedAt,
       execution,
       workerCount,
-      stackCount,
+      runtimeInstanceCount,
+      runtimeStats,
       typeValues,
       suiteSelectors,
       fileNames: requestedFiles,
@@ -206,6 +208,9 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     if (results.some((result) => result.failed)) exitCode = 1;
     if (lifecycle.isStopRequested()) exitCode = Math.max(exitCode, 130);
   } finally {
+    if (lifecycle.isStopRequested()) {
+      exitCode = Math.max(exitCode, 130);
+    }
     lifecycle.removeSignalHandlers();
     lifecycle.markFinished(
       exitCode === 0 ? "finished" : lifecycle.isStopRequested() ? "interrupted" : "failed"
@@ -213,6 +218,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     await cleanupRunById(productDir, lifecycle.runId);
     await cleanupRuns(productDir, { includeActive: false });
     lifecycle.removeManifest();
+    lifecycle.dispose();
     process.exitCode = exitCode;
   }
 }

package/lib/runner/planning.mjs

@@ -4,12 +4,11 @@ import {
   matchesSuiteSelectors,
   suiteSelectionType,
 } from "./suite-selection.mjs";
-import { resolveBatchAccessMode, resolveBatchStackMode } from "./execution-config.mjs";
 
 const TYPE_ORDER = ["dal", "integration", "e2e", "load"];
 
-export function batchNeedsLocalRuntime(batch) {
-  return batch.tasks.some((task) => task.type !== "dal");
+export function taskNeedsLocalRuntime(task) {
+  return task.type !== "dal";
 }
 
 export function resolveRuntimeConfigs(targetConfig, configMap) {
@@ -78,10 +77,6 @@ export function collectSuites(config, typeValues, suiteSelectors, fileNames = []
           (framework === "playwright"
             ? Math.max(2, Math.max(1, files.length))
             : Math.max(1, files.length)),
-        maxFileConcurrency:
-          framework === "k6" || framework === "playwright"
-            ? suite.testkit?.maxFileConcurrency || 1
-            : 1,
         totalFileCount: selectedSuiteFiles.length,
       });
       orderIndex += 1;
@@ -93,7 +88,7 @@ export function collectSuites(config, typeValues, suiteSelectors, fileNames = []
 
 export function applyShard(suites, shard) {
   if (!shard) return suites;
-  return suites.filter((unused, index) => index % shard.total === shard.index - 1);
+  return suites.filter((_unused, index) => index % shard.total === shard.index - 1);
 }
 
 export function orderedTypes(types) {
@@ -109,12 +104,19 @@ export function orderedTypes(types) {
 
 export function buildRuntimeGraphs(servicePlans) {
   const executed = servicePlans.filter((plan) => !plan.skipped);
-  const uniqueGraphs = [];
+  const graphs = [];
   const graphByRuntimeKey = new Map();
 
   for (const plan of executed) {
-    if (graphByRuntimeKey.has(plan.runtimeKey)) {
-      graphByRuntimeKey.get(plan.runtimeKey).exactTargets.push(plan.config.name);
+    plan.assignedGraphKey = plan.runtimeKey;
+    const existing = graphByRuntimeKey.get(plan.runtimeKey);
+    if (existing) {
+      existing.targetNames.push(plan.config.name);
+      existing.instanceCount = Math.max(existing.instanceCount, plan.config.testkit.runtime.instances);
+      existing.maxConcurrentTasks = Math.min(
+        existing.maxConcurrentTasks,
+        resolveGraphMaxConcurrentTasks(plan.runtimeConfigs)
+      );
       continue;
     }
 
@@ -122,48 +124,23 @@ export function buildRuntimeGraphs(servicePlans) {
       key: plan.runtimeKey,
       runtimeNames: plan.runtimeNames,
       runtimeConfigs: plan.runtimeConfigs,
-      exactTargets: [plan.config.name],
-      assignedTargets: [],
-      dirName: null,
-      rootConfig: null,
+      targetNames: [plan.config.name],
+      dirName: buildGraphDirName(plan.runtimeNames),
+      instanceCount: plan.config.testkit.runtime.instances,
+      maxConcurrentTasks: resolveGraphMaxConcurrentTasks(plan.runtimeConfigs),
     };
-    uniqueGraphs.push(graph);
+    graphs.push(graph);
     graphByRuntimeKey.set(plan.runtimeKey, graph);
   }
 
-  const maximalGraphs = uniqueGraphs.filter(
-    (graph) =>
-      !uniqueGraphs.some(
-        (other) =>
-          other.key !== graph.key &&
-          isRuntimeSuperset(other.runtimeNames, graph.runtimeNames)
-      )
-  );
-
-  for (const plan of executed) {
-    const compatible = maximalGraphs.filter((graph) =>
-      isRuntimeSuperset(graph.runtimeNames, plan.runtimeNames)
-    );
-    if (compatible.length === 0) {
-      throw new Error(`No runtime graph found for service "${plan.config.name}"`);
-    }
+  const sortedGraphs = graphs.sort((left, right) => left.dirName.localeCompare(right.dirName));
+  const maxInstanceCount = Math.max(1, ...sortedGraphs.map((graph) => graph.instanceCount));
 
-    const assigned = compatible.sort(compareGraphsForAssignment)[0];
-    plan.assignedGraphKey = assigned.key;
-    assigned.assignedTargets.push(plan.config.name);
-  }
-
-  for (const graph of maximalGraphs) {
-    const rootName = [...graph.exactTargets].sort()[0];
-    const rootPlan = executed.find((plan) => plan.config.name === rootName);
-    if (!rootPlan) {
-      throw new Error(`Missing root plan for graph "${graph.key}"`);
-    }
-    graph.rootConfig = rootPlan.config;
-    graph.dirName = buildGraphDirName(graph.runtimeNames);
-  }
-
-  return maximalGraphs.sort((a, b) => a.dirName.localeCompare(b.dirName));
+  return sortedGraphs.map((graph, index) => ({
+    ...graph,
+    portNamespaceIndex: index,
+    portNamespaceStride: maxInstanceCount,
+  }));
 }
 
 export function buildTaskQueue(servicePlans, graphs, timings) {
@@ -181,7 +158,6 @@ export function buildTaskQueue(servicePlans, graphs, timings) {
 
     for (const suite of plan.suites) {
       for (const file of suite.files) {
-        const stackMode = resolveTaskStackMode(plan.config, plan.execution, suite, file);
         const timingKey = buildTimingKey(plan.config.name, suite, file);
         tasks.push({
           id: nextId,
@@ -192,18 +168,12 @@ export function buildTaskQueue(servicePlans, graphs, timings) {
           suiteName: suite.name,
           type: suite.type,
           framework: suite.framework,
-          stackMode,
-          accessMode: resolveBatchAccessMode({
-            framework: suite.framework,
-            type: suite.type,
-            stackMode,
-          }),
           orderIndex: suite.orderIndex,
           file,
+          locks: resolveTaskLocks(plan.config, suite, file),
+          resourceCost: 1,
           timingKey,
           estimatedDurationMs: estimateTaskDuration(timings, timingKey, suite),
-          maxBatchSize:
-            suite.maxFileConcurrency || 1,
         });
         nextId += 1;
       }
@@ -219,68 +189,30 @@ export function buildTaskQueue(servicePlans, graphs, timings) {
   );
 }
 
-export function claimNextBatch(queue, preferredGraphKey) {
+export function claimNextTask(queue, preferredGraphKey, isRunnable = () => true) {
   if (queue.length === 0) return null;
 
-  let index = -1;
-  if (preferredGraphKey) {
-    index = queue.findIndex((task) => task.graphKey === preferredGraphKey);
-  }
-  if (index === -1) index = 0;
-
-  const seed = queue.splice(index, 1)[0];
-  const tasks = [seed];
-
-  if (seed.maxBatchSize > 1) {
-    for (let cursor = 0; cursor < queue.length; cursor += 1) {
-      if (tasks.length >= seed.maxBatchSize) break;
-      const candidate = queue[cursor];
-      if (
-        candidate.framework === seed.framework &&
-        candidate.type === seed.type &&
-        candidate.graphKey === seed.graphKey &&
-        candidate.targetName === seed.targetName &&
-        candidate.suiteKey === seed.suiteKey &&
-        candidate.stackMode === seed.stackMode &&
-        candidate.accessMode === seed.accessMode
-      ) {
-        tasks.push(candidate);
-        queue.splice(cursor, 1);
-        cursor -= 1;
-      }
+  const preferredIndexes = [];
+  const fallbackIndexes = [];
+  for (let index = 0; index < queue.length; index += 1) {
+    if (preferredGraphKey && queue[index].graphKey === preferredGraphKey) {
+      preferredIndexes.push(index);
+    } else {
+      fallbackIndexes.push(index);
     }
   }
 
-  tasks.sort(
-    (a, b) =>
-      a.orderIndex - b.orderIndex ||
-      a.file.localeCompare(b.file)
-  );
-
-  return {
-    graphKey: seed.graphKey,
-    targetName: seed.targetName,
-    framework: seed.framework,
-    type: seed.type,
-    stackMode: seed.stackMode,
-    accessMode: seed.accessMode,
-    tasks,
-  };
-}
-
-export function isRuntimeSuperset(candidate, target) {
-  return target.every((name) => candidate.includes(name));
-}
-
-export function compareGraphsForAssignment(left, right) {
-  if (left.runtimeNames.length !== right.runtimeNames.length) {
-    return left.runtimeNames.length - right.runtimeNames.length;
+  for (const index of [...preferredIndexes, ...fallbackIndexes]) {
+    if (!isRunnable(queue[index])) continue;
+    return queue.splice(index, 1)[0];
   }
-  return left.key.localeCompare(right.key);
+
+  return null;
 }
 
-function normalizePathSeparators(filePath) {
-  return String(filePath).split("\\").join("/");
+export function buildGraphDirName(runtimeNames) {
+  const slug = runtimeNames.map(slugSegment).join("__");
+  return slug.length > 0 ? slug : "graph";
 }
 
 function applySkipRules(config, displayType, suiteName, files, opts = {}) {
@@ -324,40 +256,36 @@ function applySkipRules(config, displayType, suiteName, files, opts = {}) {
   };
 }
 
-function resolveSuiteStackMode(config, displayType, suiteName) {
-  const defaultStackMode = config.testkit.execution.stackMode;
-  const rules = config.testkit.serviceExecution?.suites || [];
-  const matchedRule = rules.find((rule) =>
-    matchesSuiteSelectors(displayType, suiteName, [rule.selector])
-  );
-  return resolveBatchStackMode(defaultStackMode, matchedRule?.stackMode || null);
-}
+function resolveTaskLocks(config, suite, file) {
+  const locks = new Set();
+  const matchedSuiteRules = config.testkit.requirements?.suites || [];
+  for (const rule of matchedSuiteRules) {
+    if (matchesSuiteSelectors(suite.displayType, suite.name, [rule.selector])) {
+      for (const lockName of rule.locks || []) {
+        locks.add(lockName);
+      }
+    }
+  }
 
-function resolveTaskStackMode(config, execution, suite, file) {
-  const effectiveExecution = execution || config.testkit.execution;
   const normalizedFile = normalizePathSeparators(file);
-  const fileOverride = config.testkit.serviceExecution?.fileStackModeByPath?.get(normalizedFile);
-  if (fileOverride) {
-    return resolveBatchStackMode(effectiveExecution.stackMode, fileOverride);
+  for (const lockName of config.testkit.requirements?.fileLocksByPath?.get(normalizedFile) || []) {
+    locks.add(lockName);
   }
-  return resolveSuiteStackMode(
-    {
-      ...config,
-      testkit: {
-        ...config.testkit,
-        execution: effectiveExecution,
-      },
-    },
-    suite.displayType,
-    suite.name
-  );
+
+  return [...locks].sort();
 }
 
-export function buildGraphDirName(runtimeNames) {
-  const slug = runtimeNames.map(slugSegment).join("__");
-  return slug.length > 0 ? slug : "graph";
+function normalizePathSeparators(filePath) {
+  return String(filePath).split("\\").join("/");
 }
 
 function slugSegment(value) {
   return value.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "") || "svc";
 }
+
+function resolveGraphMaxConcurrentTasks(runtimeConfigs) {
+  return runtimeConfigs.reduce(
+    (currentMin, config) => Math.min(currentMin, config.testkit.runtime.maxConcurrentTasks),
+    Number.POSITIVE_INFINITY
+  );
+}