@elench/testkit 0.1.31 → 0.1.32

package/lib/cli/args.mjs

@@ -1,7 +1,7 @@
 import path from "path";
 
 export const SUITE_TYPES = new Set(["int", "integration", "e2e", "dal", "load", "all"]);
-export const LIFECYCLE = new Set(["status", "destroy"]);
+export const LIFECYCLE = new Set(["status", "destroy", "cleanup"]);
 export const RESERVED = new Set([...SUITE_TYPES, ...LIFECYCLE]);
 
 export function resolveCliSelection({ first, second, serviceNames }) {

package/lib/cli/index.mjs

@@ -56,6 +56,11 @@ export function run() {
       }
 
       // Lifecycle commands
+      if (type === "cleanup") {
+        await runner.cleanup(allConfigs[0]?.productDir || process.cwd());
+        return;
+      }
+
       if (type === "status" || type === "destroy") {
         for (const config of configs) {
           if (configs.length > 1) console.log(`\n── ${config.name} ──`);

package/lib/runner/index.mjs

@@ -90,6 +90,16 @@ import {
   writeGraphMetadata as writeGraphMetadataModel,
 } from "./state.mjs";
 import { uploadTelemetryArtifact } from "../telemetry/index.mjs";
+import {
+  cleanupRunById,
+  cleanupRuns,
+  cleanupStaleRuns,
+  createRunLifecycle,
+  findPortOwner,
+  formatRunSummary,
+  isPidRunning,
+  listRunManifests,
+} from "./lifecycle.mjs";
 
 const HTTP_K6_TYPES = new Set(["integration", "e2e", "load"]);
 const DEFAULT_READY_TIMEOUT_MS = 120_000;
@@ -100,6 +110,7 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
   const startedAt = Date.now();
   const telemetry = configs[0]?.telemetry || null;
   const productDir = configs[0]?.productDir || process.cwd();
+  await cleanupStaleRuns(productDir);
   const metadata = {
     git: collectGitMetadata(productDir),
     host: {
@@ -134,80 +145,97 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
   const trackers = buildServiceTrackers(servicePlans, startedAt);
   const executedPlans = servicePlans.filter((plan) => !plan.skipped);
   let workerCount = 0;
+  let exitCode = 0;
+  const lifecycle = createRunLifecycle(productDir);
+  lifecycle.markRunning();
+  lifecycle.installSignalHandlers();
+  let results = [];
+  let finishedAt = Date.now();
+  try {
+    if (executedPlans.length > 0) {
+      const timings = loadTimings(productDir);
+      const graphs = buildRuntimeGraphs(executedPlans);
+      const queue = buildTaskQueue(executedPlans, graphs, timings);
+      workerCount = Math.max(1, Math.min(opts.jobs || 1, queue.length));
+      const graphByKey = new Map(graphs.map((graph) => [graph.key, graph]));
+      const workers = Array.from({ length: workerCount }, (_unused, index) =>
+        createWorker(index + 1, productDir)
+      );
+      const timingUpdates = [];
 
-  if (executedPlans.length > 0) {
-    const timings = loadTimings(productDir);
-    const graphs = buildRuntimeGraphs(executedPlans);
-    const queue = buildTaskQueue(executedPlans, graphs, timings);
-    workerCount = Math.max(1, Math.min(opts.jobs || 1, queue.length));
-    const graphByKey = new Map(graphs.map((graph) => [graph.key, graph]));
-    const workers = Array.from({ length: workerCount }, (_unused, index) =>
-      createWorker(index + 1, productDir)
-    );
-    const timingUpdates = [];
-
-    const workerResults = await Promise.allSettled(
-      workers.map((worker) =>
-        runWorker(worker, queue, graphByKey, trackers, timingUpdates)
-      )
-    );
+      const workerResults = await Promise.allSettled(
+        workers.map((worker) =>
+          runWorker(worker, queue, graphByKey, trackers, timingUpdates, lifecycle)
+        )
+      );
 
-    for (const result of workerResults) {
-      if (result.status === "rejected") {
-        const message = formatError(result.reason);
-        for (const tracker of trackers.values()) {
-          if (!tracker.skipped) addTrackerError(tracker, message);
+      for (const result of workerResults) {
+        if (result.status === "rejected") {
+          const message = formatError(result.reason);
+          for (const tracker of trackers.values()) {
+            if (!tracker.skipped) addTrackerError(tracker, message);
+          }
         }
       }
-    }
-
-    saveTimings(productDir, timings, timingUpdates);
-  }
-
-  const finishedAt = Date.now();
-  const results = configs.map((config) =>
-    finalizeServiceResult(trackers.get(config.name), startedAt, finishedAt)
-  );
-  const artifact = buildRunArtifact({
-    productDir,
-    results,
-    startedAt,
-    finishedAt,
-    requestedJobs: opts.jobs || 1,
-    workerCount,
-    suiteType,
-    suiteNames,
-    fileNames: requestedFiles,
-    framework: opts.framework || "all",
-    shard: opts.shard || null,
-    serviceFilter: opts.serviceFilter || null,
-    metadata,
-  });
 
-  writeRunArtifact(productDir, artifact);
-  if (opts.writeStatus) {
-    writeStatusArtifact(
+      saveTimings(productDir, timings, timingUpdates);
+    }
+    finishedAt = Date.now();
+    results = configs.map((config) =>
+      finalizeServiceResult(trackers.get(config.name), startedAt, finishedAt)
+    );
+    const artifact = buildRunArtifact({
       productDir,
-      buildStatusArtifact({
+      results,
+      startedAt,
+      finishedAt,
+      requestedJobs: opts.jobs || 1,
+      workerCount,
+      suiteType,
+      suiteNames,
+      fileNames: requestedFiles,
+      framework: opts.framework || "all",
+      shard: opts.shard || null,
+      serviceFilter: opts.serviceFilter || null,
+      metadata,
+    });
+
+    writeRunArtifact(productDir, artifact);
+    if (opts.writeStatus) {
+      writeStatusArtifact(
         productDir,
-        results,
-        suiteType,
-        suiteNames,
-        fileNames: requestedFiles,
-        framework: opts.framework || "all",
-        shard: opts.shard || null,
-        serviceFilter: opts.serviceFilter || null,
-        metadata,
-      })
+        buildStatusArtifact({
+          productDir,
+          results,
+          suiteType,
+          suiteNames,
+          fileNames: requestedFiles,
+          framework: opts.framework || "all",
+          shard: opts.shard || null,
+          serviceFilter: opts.serviceFilter || null,
+          metadata,
+        })
+      );
+    }
+
+    printRunSummary(results, finishedAt - startedAt);
+    await reportTelemetry(telemetry, artifact);
+    if (results.some((result) => result.failed)) exitCode = 1;
+    if (lifecycle.isStopRequested()) exitCode = Math.max(exitCode, 130);
+  } finally {
+    lifecycle.removeSignalHandlers();
+    lifecycle.markFinished(
+      exitCode === 0 ? "finished" : lifecycle.isStopRequested() ? "interrupted" : "failed"
     );
+    await cleanupRunById(productDir, lifecycle.runId);
+    await cleanupRuns(productDir, { includeActive: false });
+    lifecycle.removeManifest();
+    process.exitCode = exitCode;
   }
-
-  printRunSummary(results, finishedAt - startedAt);
-  await reportTelemetry(telemetry, artifact);
-  if (results.some((result) => result.failed)) process.exit(1);
 }
 
 export async function destroy(config) {
+  await cleanupRuns(config.productDir, { includeActive: true });
   const roots = new Set([config.stateDir, ...findGraphDirsForService(config.productDir, config.name)]);
 
   for (const rootDir of roots) {
@@ -227,6 +255,7 @@ export async function destroy(config) {
 }
 
 export function showStatus(config) {
+  printRunStatus(config.productDir);
   const graphDirs = findGraphDirsForService(config.productDir, config.name);
   const hasDirectState = fs.existsSync(config.stateDir);
   const hasGraphState = graphDirs.length > 0;
@@ -247,6 +276,21 @@ export function showStatus(config) {
   showServiceDatabaseStatus(config.productDir, config.name);
 }
 
+export async function cleanup(productDir) {
+  const summary = await cleanupRuns(productDir, { includeActive: false });
+  if (summary.cleaned.length === 0 && summary.skippedActive.length === 0) {
+    console.log("No stale runs to clean.");
+    return;
+  }
+
+  for (const manifest of summary.cleaned) {
+    console.log(`Cleaned stale run ${formatRunSummary(manifest)}`);
+  }
+  for (const manifest of summary.skippedActive) {
+    console.log(`Active run still present: ${formatRunSummary(manifest)}`);
+  }
+}
+
 function collectServicePlans(configs, configMap, suiteType, suiteNames, opts) {
   return configs.map((config) => {
     console.log(`\n══ ${config.name} ══`);
@@ -304,19 +348,20 @@ function createWorker(workerId, productDir) {
   };
 }
 
-async function runWorker(worker, queue, graphByKey, trackers, timingUpdates) {
+async function runWorker(worker, queue, graphByKey, trackers, timingUpdates, lifecycle) {
   const startedAt = Date.now();
   console.log(`\n══ global worker ${worker.workerId} ══`);
   const errors = [];
 
   try {
     while (true) {
+      if (lifecycle.isStopRequested()) break;
       const batch = claimNextBatch(queue, worker.currentGraphKey);
       if (!batch) break;
 
       try {
-        const context = await ensureWorkerGraph(worker, batch, graphByKey);
-        const outcomes = await runBatch(context, batch);
+        const context = await ensureWorkerGraph(worker, batch, graphByKey, lifecycle);
+        const outcomes = await runBatch(context, batch, lifecycle);
         for (const outcome of outcomes) {
           recordTaskOutcome(trackers, outcome.task, outcome);
           timingUpdates.push({
@@ -329,11 +374,11 @@ async function runWorker(worker, queue, graphByKey, trackers, timingUpdates) {
         const message = formatError(error);
         errors.push(message);
         recordGraphError(trackers, graphByKey.get(batch.graphKey), message);
-        await resetCurrentGraph(worker);
+        await resetCurrentGraph(worker, lifecycle);
       }
     }
   } finally {
-    await cleanupWorker(worker);
+    await cleanupWorker(worker, lifecycle);
   }
 
   return {
@@ -350,14 +395,14 @@ function claimNextBatch(queue, preferredGraphKey) {
   return claimNextBatchModel(queue, preferredGraphKey);
 }
 
-async function ensureWorkerGraph(worker, batch, graphByKey) {
+async function ensureWorkerGraph(worker, batch, graphByKey, lifecycle) {
   const graph = graphByKey.get(batch.graphKey);
   if (!graph) {
     throw new Error(`Unknown graph "${batch.graphKey}"`);
   }
 
   if (worker.currentGraphKey && worker.currentGraphKey !== batch.graphKey) {
-    await deactivateGraphContext(worker.graphContexts.get(worker.currentGraphKey));
+    await deactivateGraphContext(worker.graphContexts.get(worker.currentGraphKey), lifecycle);
     worker.graphSwitches += 1;
     worker.currentGraphKey = null;
   }
@@ -366,6 +411,7 @@ async function ensureWorkerGraph(worker, batch, graphByKey) {
   if (!context) {
     context = createGraphContext(worker, graph);
     worker.graphContexts.set(batch.graphKey, context);
+    lifecycle.trackGraphContext(context);
   }
 
   if (!context.prepared) {
@@ -374,7 +420,7 @@ async function ensureWorkerGraph(worker, batch, graphByKey) {
   }
 
   if (batchNeedsLocalRuntime(batch) && !context.started) {
-    context.startedServices = await startLocalServices(context.runtimeConfigs);
+    context.startedServices = await startLocalServices(context.runtimeConfigs, lifecycle);
     context.started = true;
   }
 
@@ -407,40 +453,40 @@ function createGraphContext(worker, graph) {
   };
 }
 
-async function deactivateGraphContext(context) {
+async function deactivateGraphContext(context, lifecycle) {
   if (!context?.started) return;
-  await stopLocalServices(context.startedServices);
+  await stopLocalServices(context.startedServices, lifecycle);
   context.started = false;
   context.startedServices = [];
 }
 
-async function resetCurrentGraph(worker) {
+async function resetCurrentGraph(worker, lifecycle) {
   if (!worker.currentGraphKey) return;
-  await deactivateGraphContext(worker.graphContexts.get(worker.currentGraphKey));
+  await deactivateGraphContext(worker.graphContexts.get(worker.currentGraphKey), lifecycle);
   worker.currentGraphKey = null;
 }
 
-async function cleanupWorker(worker) {
+async function cleanupWorker(worker, lifecycle) {
   for (const context of worker.graphContexts.values()) {
-    await deactivateGraphContext(context);
+    await deactivateGraphContext(context, lifecycle);
   }
   worker.currentGraphKey = null;
 }
 
-async function runBatch(context, batch) {
+async function runBatch(context, batch, lifecycle) {
   const targetConfig = context.configByName.get(batch.targetName);
   if (!targetConfig) {
     throw new Error(`Worker graph missing target config "${batch.targetName}"`);
   }
 
   if (batch.framework === "playwright") {
-    return runPlaywrightBatch(targetConfig, batch);
+    return runPlaywrightBatch(targetConfig, batch, lifecycle);
   }
   if (batch.type === "dal") {
-    return runDalBatch(targetConfig, batch);
+    return runDalBatch(targetConfig, batch, lifecycle);
   }
   if (batch.framework === "k6" && HTTP_K6_TYPES.has(batch.type)) {
-    return runHttpK6Batch(targetConfig, batch);
+    return runHttpK6Batch(targetConfig, batch, lifecycle);
   }
 
   throw new Error(
@@ -491,13 +537,13 @@ async function runSeed(config, databaseUrl) {
   });
 }
 
-async function startLocalServices(runtimeConfigs) {
+async function startLocalServices(runtimeConfigs, lifecycle) {
   const started = [];
 
   try {
     for (const config of runtimeConfigs) {
       if (!config.testkit.local) continue;
-      const proc = await startLocalService(config);
+      const proc = await startLocalService(config, lifecycle);
       started.push(proc);
     }
   } catch (error) {
@@ -508,7 +554,7 @@ async function startLocalServices(runtimeConfigs) {
   return started;
 }
 
-async function startLocalService(config) {
+async function startLocalService(config, lifecycle) {
   const cwd = resolveServiceCwd(config.productDir, config.testkit.local.cwd);
   const env = buildExecutionEnv(config, config.testkit.local.env);
   const port = config.testkit.local.port || numericPortFromUrl(config.testkit.local.baseUrl);
@@ -524,18 +570,13 @@ async function startLocalService(config) {
   await assertLocalServicePortsAvailable(config);
 
   console.log(`Starting ${config.workerLabel}:${config.name}: ${config.testkit.local.start}`);
-  const child = spawn(config.testkit.local.start, {
-    cwd,
-    env,
-    detached: true,
-    shell: true,
-    stdio: ["ignore", "pipe", "pipe"],
-  });
+  const child = startDetachedCommand(config.testkit.local.start, cwd, env);
 
   const outputDrains = [
     pipeOutput(child.stdout, `[${config.workerLabel}:${config.name}]`),
     pipeOutput(child.stderr, `[${config.workerLabel}:${config.name}]`),
   ];
+  lifecycle.registerService(config, child, cwd);
 
   const readyTimeoutMs = config.testkit.local.readyTimeoutMs || DEFAULT_READY_TIMEOUT_MS;
 
@@ -545,16 +586,18 @@ async function startLocalService(config) {
       url: config.testkit.local.readyUrl,
       timeoutMs: readyTimeoutMs,
       process: child,
+      signal: lifecycle.signal,
     });
   } catch (error) {
     await stopChildProcess(child, outputDrains);
+    lifecycle.unregisterService(child.pid);
     throw error;
   }
 
   return { name: config.name, child, outputDrains };
 }
 
-async function runHttpK6Batch(targetConfig, batch) {
+async function runHttpK6Batch(targetConfig, batch, lifecycle) {
   const baseUrl = targetConfig.testkit.local?.baseUrl;
   if (!baseUrl) {
     throw new Error(`Service "${targetConfig.name}" requires local.baseUrl for HTTP suites`);
@@ -565,11 +608,11 @@ async function runHttpK6Batch(targetConfig, batch) {
   );
 
   return Promise.all(
-    batch.tasks.map((task) => runHttpK6Task(targetConfig, task, baseUrl))
+    batch.tasks.map((task) => runHttpK6Task(targetConfig, task, baseUrl, lifecycle))
   );
 }
 
-async function runHttpK6Task(targetConfig, task, baseUrl) {
+async function runHttpK6Task(targetConfig, task, baseUrl, lifecycle) {
   const absFile = path.join(targetConfig.productDir, task.file);
   const k6Binary = resolveK6Binary();
   const bundledFile = await bundleK6File({
@@ -585,10 +628,10 @@ async function runHttpK6Task(targetConfig, task, baseUrl) {
     "-e",
     `BASE_URL=${baseUrl}`,
     bundledFile,
-  ]);
+  ], lifecycle);
 }
 
-async function runDalBatch(targetConfig, batch) {
+async function runDalBatch(targetConfig, batch, lifecycle) {
   const databaseUrl = readDatabaseUrl(targetConfig.stateDir);
   if (!databaseUrl) {
     throw new Error(`Service "${targetConfig.name}" requires a database for DAL suites`);
@@ -599,11 +642,11 @@ async function runDalBatch(targetConfig, batch) {
   );
 
   return Promise.all(
-    batch.tasks.map((task) => runDalTask(targetConfig, task, databaseUrl))
+    batch.tasks.map((task) => runDalTask(targetConfig, task, databaseUrl, lifecycle))
   );
 }
 
-async function runDalTask(targetConfig, task, databaseUrl) {
+async function runDalTask(targetConfig, task, databaseUrl, lifecycle) {
   const absFile = path.join(targetConfig.productDir, task.file);
   const k6Binary = resolveK6Binary();
   const bundledFile = await bundleK6File({
@@ -619,10 +662,10 @@ async function runDalTask(targetConfig, task, databaseUrl) {
     "-e",
     `DATABASE_URL=${databaseUrl}`,
     bundledFile,
-  ]);
+  ], lifecycle);
 }
 
-async function runPlaywrightBatch(targetConfig, batch) {
+async function runPlaywrightBatch(targetConfig, batch, lifecycle) {
   const local = targetConfig.testkit.local;
   if (!local?.baseUrl) {
     throw new Error(
@@ -647,6 +690,8 @@ async function runPlaywrightBatch(targetConfig, batch) {
       cwd,
       env: buildPlaywrightEnv(targetConfig, local.baseUrl),
       reject: false,
+      cancelSignal: lifecycle.signal,
+      forceKillAfterDelay: 5_000,
     }
   );
 
@@ -694,9 +739,10 @@ async function runPlaywrightBatch(targetConfig, batch) {
   });
 }
 
-async function stopLocalServices(started) {
+async function stopLocalServices(started, lifecycle) {
   for (const service of [...started].reverse()) {
     await stopChildProcess(service.child, service.outputDrains);
+    lifecycle?.unregisterService(service.child.pid);
   }
 }
 
@@ -721,10 +767,13 @@ async function stopChildProcess(child, outputDrains = []) {
   await Promise.all(outputDrains);
 }
 
-async function waitForReady({ name, url, timeoutMs, process }) {
+async function waitForReady({ name, url, timeoutMs, process, signal }) {
   const start = Date.now();
 
   while (Date.now() - start < timeoutMs) {
+    if (signal?.aborted) {
+      throw signal.reason || new Error(`Service "${name}" startup aborted`);
+    }
     if (process.exitCode !== null) {
       throw new Error(`Service "${name}" exited before becoming ready`);
     }
@@ -1085,7 +1134,7 @@ function formatError(error) {
   return formatErrorModel(error);
 }
 
-async function runDefaultRuntimeTask(targetConfig, task, args) {
+async function runDefaultRuntimeTask(targetConfig, task, args, lifecycle) {
   const k6Binary = resolveK6Binary();
   const summaryFile = buildDefaultRuntimeSummaryPath(targetConfig, task);
   fs.mkdirSync(path.dirname(summaryFile), { recursive: true });
@@ -1094,6 +1143,8 @@ async function runDefaultRuntimeTask(targetConfig, task, args) {
     cwd: targetConfig.productDir,
     env: buildExecutionEnv(targetConfig),
     reject: false,
+    cancelSignal: lifecycle.signal,
+    forceKillAfterDelay: 5_000,
   });
 
   if (result.stdout) process.stdout.write(result.stdout);
@@ -1317,14 +1368,38 @@ async function assertLocalServicePortsAvailable(config) {
     seen.add(key);
 
     if (await isPortInUse(socket)) {
+      await cleanupStaleRuns(config.productDir);
+    }
+
+    if (await isPortInUse(socket)) {
+      const owner = findPortOwner(config.productDir, socket);
+      const ownerDetail = owner
+        ? owner.active
+          ? ` Active testkit run ${formatRunSummary(owner.manifest)} owns ${key} via ${owner.service.workerLabel}:${owner.service.serviceName}.`
+          : ` Stale testkit run ${formatRunSummary(owner.manifest)} owns ${key}.`
+        : "";
       throw new Error(
         `Cannot start "${config.workerLabel}:${config.name}" because ${key} is already in use. ` +
-          `Stop the existing process and rerun testkit.`
+          `Stop the existing process and rerun testkit.${ownerDetail}`
       );
     }
   }
 }
 
+function printRunStatus(productDir) {
+  const manifests = listRunManifests(productDir);
+  if (manifests.length === 0) return;
+
+  console.log("  runs/");
+  for (const manifest of manifests) {
+    const state = isPidRunning(manifest.pid) ? "active" : "stale";
+    const ports = [...new Set((manifest.services || []).flatMap((service) => (service.ports || []).map((socket) => `${socket.host}:${socket.port}`)))];
+    console.log(
+      `    ${manifest.runId}: ${state} pid=${manifest.pid}${ports.length > 0 ? ` ports=${ports.join(",")}` : ""}`
+    );
+  }
+}
+
 function socketFromUrl(rawUrl) {
   return socketFromUrlModel(rawUrl);
 }
@@ -1385,6 +1460,26 @@ function killChildProcess(child, signal) {
   }
 }
 
+function startDetachedCommand(command, cwd, env) {
+  if (process.platform === "win32") {
+    return spawn(command, {
+      cwd,
+      env,
+      detached: true,
+      shell: true,
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+  }
+
+  const shell = process.env.SHELL || "/bin/sh";
+  return spawn(shell, ["-lc", `exec ${command}`], {
+    cwd,
+    env,
+    detached: true,
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+}
+
 function readDatabaseUrl(stateDir) {
   return readStateValue(path.join(stateDir, "database_url"));
 }

package/lib/runner/lifecycle.mjs

@@ -0,0 +1,349 @@
+import fs from "fs";
+import path from "path";
+import crypto from "crypto";
+import { destroyRuntimeDatabase, cleanupOrphanedLocalInfrastructure } from "../database/index.mjs";
+
+const RUN_SCHEMA_VERSION = 1;
+const RUNS_DIRNAME = path.join(".testkit", "_runs");
+const TERMINATION_TIMEOUT_MS = 5_000;
+
+export function createRunLifecycle(productDir) {
+  const runId = buildRunId();
+  const manifestPath = path.join(getRunsDir(productDir), `${runId}.json`);
+  const abortController = new AbortController();
+  const state = {
+    schemaVersion: RUN_SCHEMA_VERSION,
+    runId,
+    productDir,
+    pid: process.pid,
+    status: "starting",
+    startedAt: new Date().toISOString(),
+    interruptReason: null,
+    services: [],
+    graphDirs: [],
+    workerStateDirs: [],
+    runtimeStateDirs: [],
+  };
+  const signalListeners = [];
+
+  function persist() {
+    fs.mkdirSync(getRunsDir(productDir), { recursive: true });
+    fs.writeFileSync(manifestPath, `${JSON.stringify(state, null, 2)}\n`);
+  }
+
+  function mutate(mutator) {
+    mutator(state);
+    persist();
+  }
+
+  const api = {
+    runId,
+    manifestPath,
+    signal: abortController.signal,
+    isStopRequested() {
+      return abortController.signal.aborted;
+    },
+    markRunning() {
+      mutate((draft) => {
+        draft.status = "running";
+      });
+    },
+    markFinished(status = "finished") {
+      mutate((draft) => {
+        draft.status = status;
+        draft.finishedAt = new Date().toISOString();
+      });
+    },
+    requestStop(reason = "interrupted") {
+      if (!abortController.signal.aborted) {
+        abortController.abort(new Error(`testkit run interrupted (${reason})`));
+      }
+      mutate((draft) => {
+        draft.status = "interrupting";
+        draft.interruptReason = reason;
+      });
+    },
+    trackGraphContext(context) {
+      mutate((draft) => {
+        pushUnique(draft.graphDirs, context.graphDir);
+        pushUnique(draft.workerStateDirs, context.workerStateDir);
+        for (const runtimeConfig of context.runtimeConfigs || []) {
+          if (runtimeConfig.stateDir) pushUnique(draft.runtimeStateDirs, runtimeConfig.stateDir);
+        }
+      });
+    },
+    registerService(config, child, cwd) {
+      const ports = collectConfigPorts(config);
+      mutate((draft) => {
+        draft.services = draft.services.filter((service) => service.pid !== child.pid);
+        draft.services.push({
+          serviceName: config.name,
+          workerLabel: config.workerLabel,
+          command: config.testkit.local?.start || null,
+          cwd,
+          pid: child.pid,
+          processGroupId: child.pid,
+          ports,
+          startedAt: new Date().toISOString(),
+        });
+      });
+    },
+    unregisterService(childPid) {
+      mutate((draft) => {
+        draft.services = draft.services.filter((service) => service.pid !== childPid);
+      });
+    },
+    installSignalHandlers() {
+      const install = (eventName, handler) => {
+        process.on(eventName, handler);
+        signalListeners.push([eventName, handler]);
+      };
+
+      const handleSignal = (signalName) => {
+        api.requestStop(signalName);
+      };
+
+      install("SIGINT", () => handleSignal("SIGINT"));
+      install("SIGTERM", () => handleSignal("SIGTERM"));
+      install("SIGHUP", () => handleSignal("SIGHUP"));
+    },
+    removeSignalHandlers() {
+      while (signalListeners.length > 0) {
+        const [eventName, handler] = signalListeners.pop();
+        process.off(eventName, handler);
+      }
+    },
+    removeManifest() {
+      removeManifestFile(productDir, runId);
+    },
+  };
+
+  return api;
+}
+
+export async function cleanupRuns(productDir, { includeActive = false } = {}) {
+  const manifests = listRunManifests(productDir);
+  const summary = {
+    cleaned: [],
+    skippedActive: [],
+  };
+
+  for (const manifest of manifests) {
+    const isActive = isPidRunning(manifest.pid);
+    if (isActive && !includeActive) {
+      summary.skippedActive.push(manifest);
+      continue;
+    }
+    await cleanupRunManifest(productDir, manifest, { removeRuntimeState: false });
+    summary.cleaned.push(manifest);
+  }
+
+  await cleanupOrphanedLocalInfrastructure(productDir);
+  return summary;
+}
+
+export async function cleanupStaleRuns(productDir) {
+  const manifests = listRunManifests(productDir);
+  const cleaned = [];
+
+  for (const manifest of manifests) {
+    if (isPidRunning(manifest.pid)) continue;
+    await cleanupRunManifest(productDir, manifest, { removeRuntimeState: false });
+    cleaned.push(manifest);
+  }
+
+  await cleanupOrphanedLocalInfrastructure(productDir);
+  return cleaned;
+}
+
+export async function cleanupRunById(productDir, runId) {
+  const manifest = listRunManifests(productDir).find((entry) => entry.runId === runId);
+  if (!manifest) return false;
+  await cleanupRunManifest(productDir, manifest, { removeRuntimeState: false });
+  await cleanupOrphanedLocalInfrastructure(productDir);
+  return true;
+}
+
+export function listRunManifests(productDir) {
+  const runsDir = getRunsDir(productDir);
+  if (!fs.existsSync(runsDir)) return [];
+
+  return fs
+    .readdirSync(runsDir)
+    .filter((entry) => entry.endsWith(".json"))
+    .map((entry) => readRunManifest(path.join(runsDir, entry)))
+    .filter(Boolean)
+    .sort((left, right) => String(left.startedAt || "").localeCompare(String(right.startedAt || "")));
+}
+
+export function findPortOwner(productDir, { host, port }) {
+  const manifests = listRunManifests(productDir);
+  for (const manifest of manifests) {
+    for (const service of manifest.services || []) {
+      for (const socket of service.ports || []) {
+        if (normalizeHost(socket.host) === normalizeHost(host) && Number(socket.port) === Number(port)) {
+          return {
+            manifest,
+            service,
+            active: isPidRunning(manifest.pid),
+          };
+        }
+      }
+    }
+  }
+  return null;
+}
+
+export function formatRunSummary(manifest) {
+  const workerLabels = [...new Set((manifest.services || []).map((service) => service.workerLabel).filter(Boolean))];
+  return `${manifest.runId} pid=${manifest.pid}${workerLabels.length > 0 ? ` workers=${workerLabels.join(",")}` : ""}`;
+}
+
+export function isPidRunning(pid) {
+  if (!Number.isInteger(Number(pid)) || Number(pid) <= 0) return false;
+  try {
+    process.kill(Number(pid), 0);
+    return true;
+  } catch (error) {
+    return error?.code === "EPERM";
+  }
+}
+
+async function cleanupRunManifest(productDir, manifest, { removeRuntimeState = false } = {}) {
+  for (const service of [...(manifest.services || [])].reverse()) {
+    await terminateOwnedProcess(service);
+  }
+
+  if (removeRuntimeState) {
+    const runtimeStateDirs = [...new Set(manifest.runtimeStateDirs || [])].sort((a, b) => b.length - a.length);
+    for (const stateDir of runtimeStateDirs) {
+      await destroyRuntimeDatabase({ productDir, stateDir });
+    }
+
+    for (const workerStateDir of [...new Set(manifest.workerStateDirs || [])].sort((a, b) => b.length - a.length)) {
+      fs.rmSync(workerStateDir, { recursive: true, force: true });
+    }
+
+    for (const graphDir of [...new Set(manifest.graphDirs || [])].sort((a, b) => b.length - a.length)) {
+      pruneEmptyParents(graphDir, path.join(productDir, ".testkit", "_graphs"));
+    }
+  }
+
+  removeManifestFile(productDir, manifest.runId);
+}
+
+async function terminateOwnedProcess(service) {
+  const pid = Number(service.processGroupId || service.pid);
+  if (!Number.isInteger(pid) || pid <= 0) return;
+  if (!isPidRunning(pid)) return;
+
+  killProcessGroup(pid, "SIGTERM");
+  const exited = await waitForPidExit(pid, TERMINATION_TIMEOUT_MS);
+  if (!exited) {
+    killProcessGroup(pid, "SIGKILL");
+    await waitForPidExit(pid, TERMINATION_TIMEOUT_MS);
+  }
+}
+
+function killProcessGroup(pid, signal) {
+  try {
+    process.kill(-pid, signal);
+    return;
+  } catch (error) {
+    if (error?.code !== "ESRCH") {
+      // Fall through and try the direct pid.
+    } else {
+      return;
+    }
+  }
+
+  try {
+    process.kill(pid, signal);
+  } catch (error) {
+    if (error?.code !== "ESRCH") throw error;
+  }
+}
+
+async function waitForPidExit(pid, timeoutMs) {
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < timeoutMs) {
+    if (!isPidRunning(pid)) return true;
+    await sleep(100);
+  }
+  return !isPidRunning(pid);
+}
+
+function readRunManifest(filePath) {
+  try {
+    const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
+    if (!parsed?.runId) return null;
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+function collectConfigPorts(config) {
+  const seen = new Set();
+  const ports = [];
+  for (const rawUrl of [config.testkit.local?.baseUrl, config.testkit.local?.readyUrl]) {
+    if (!rawUrl) continue;
+    try {
+      const parsed = new URL(rawUrl);
+      const host = normalizeHost(parsed.hostname);
+      const port = Number(parsed.port || (parsed.protocol === "https:" ? 443 : 80));
+      const key = `${host}:${port}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      ports.push({ host, port });
+    } catch {
+      // Ignore malformed URLs here; startup validation handles them elsewhere.
+    }
+  }
+  return ports;
+}
+
+function pushUnique(list, value) {
+  if (!value) return;
+  if (!list.includes(value)) list.push(value);
+}
+
+function getRunsDir(productDir) {
+  return path.join(productDir, RUNS_DIRNAME);
+}
+
+function buildRunId() {
+  return `${Date.now()}-${process.pid}-${crypto.randomBytes(4).toString("hex")}`;
+}
+
+function normalizeHost(host) {
+  if (!host) return "127.0.0.1";
+  return host === "::1" ? "127.0.0.1" : host;
+}
+
+function pruneEmptyParents(startDir, stopDir) {
+  let current = startDir;
+  while (current && current.startsWith(stopDir)) {
+    if (!fs.existsSync(current)) {
+      current = path.dirname(current);
+      continue;
+    }
+    const entries = fs.readdirSync(current);
+    if (entries.length > 0) break;
+    fs.rmSync(current, { recursive: true, force: true });
+    if (current === stopDir) break;
+    current = path.dirname(current);
+  }
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function removeManifestFile(productDir, runId) {
+  const runsDir = getRunsDir(productDir);
+  fs.rmSync(path.join(runsDir, `${runId}.json`), { force: true });
+  if (fs.existsSync(runsDir) && fs.readdirSync(runsDir).length === 0) {
+    fs.rmSync(runsDir, { recursive: true, force: true });
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/testkit",
-  "version": "0.1.31",
+  "version": "0.1.32",
   "description": "CLI for discovering and running local HTTP, DAL, and Playwright test suites",
   "type": "module",
   "types": "./lib/index.d.ts",