@elench/testkit 0.1.16 → 0.1.18

package/lib/runner/index.mjs

@@ -0,0 +1,1221 @@
+import fs from "fs";
+import path from "path";
+import { spawn } from "child_process";
+import net from "net";
+import { execa, execaCommand } from "execa";
+import { resolveDalBinary, resolveServiceCwd } from "../config/index.mjs";
+import {
+  cleanupOrphanedLocalInfrastructure,
+  destroyRuntimeDatabase,
+  destroyServiceDatabaseCache,
+  isDatabaseStateDir,
+  prepareDatabaseRuntime,
+  showServiceDatabaseStatus,
+} from "../database/index.mjs";
+import {
+  batchNeedsLocalRuntime as batchNeedsLocalRuntimeModel,
+  buildGraphDirName as buildGraphDirNameModel,
+  buildRuntimeGraphs as buildRuntimeGraphsModel,
+  buildTaskQueue as buildTaskQueueModel,
+  claimNextBatch as claimNextBatchModel,
+  collectSuites as collectSuitesModel,
+  compareGraphsForAssignment as compareGraphsForAssignmentModel,
+  isRuntimeSuperset as isRuntimeSupersetModel,
+  orderedTypes as orderedTypesModel,
+  resolveRuntimeConfigs as resolveRuntimeConfigsModel,
+  applyShard as applyShardModel,
+} from "./planning.mjs";
+import {
+  buildExecutionEnv as buildExecutionEnvModel,
+  buildPlaywrightEnv as buildPlaywrightEnvModel,
+  buildPortMap as buildPortMapModel,
+  finalizeString as finalizeStringModel,
+  getWorkerServiceStateDir as getWorkerServiceStateDirModel,
+  normalizeSocketHost as normalizeSocketHostModel,
+  numericPortFromUrl as numericPortFromUrlModel,
+  resolveRuntimeUrl as resolveRuntimeUrlModel,
+  resolveServiceStateDir as resolveServiceStateDirModel,
+  resolveTemplateString as resolveTemplateStringModel,
+  resolveWorkerConfig as resolveWorkerConfigModel,
+  resolveWorkerRuntimeConfigs as resolveWorkerRuntimeConfigsModel,
+  rewriteUrlPort as rewriteUrlPortModel,
+  socketFromUrl as socketFromUrlModel,
+} from "./template.mjs";
+import {
+  addTrackerError as addTrackerErrorModel,
+  buildRunArtifact as buildRunArtifactModel,
+  buildServiceTrackers as buildServiceTrackersModel,
+  finalizeServiceResult as finalizeServiceResultModel,
+  formatDuration as formatDurationModel,
+  formatError as formatErrorModel,
+  formatServiceSummary as formatServiceSummaryModel,
+  longestServiceName as longestServiceNameModel,
+  recordGraphError as recordGraphErrorModel,
+  recordTaskOutcome as recordTaskOutcomeModel,
+  summarizeDbBackend as summarizeDbBackendModel,
+} from "./results.mjs";
+import {
+  applyTimingUpdates,
+  buildTimingKey as buildTimingKeyModel,
+  createEmptyTimings,
+  estimateTaskDuration as estimateTaskDurationModel,
+  normalizeTimings,
+} from "../timing/index.mjs";
+import {
+  choosePlaywrightFinalResult as choosePlaywrightFinalResultModel,
+  collectPlaywrightSpec as collectPlaywrightSpecModel,
+  extractPlaywrightFailure as extractPlaywrightFailureModel,
+  extractReporterFile as extractReporterFileModel,
+  firstLine as firstLineModel,
+  formatPlaywrightReporterError as formatPlaywrightReporterErrorModel,
+  isPlaywrightPassingStatus as isPlaywrightPassingStatusModel,
+  normalizeReportedFile as normalizeReportedFileModel,
+  parsePlaywrightJsonResults as parsePlaywrightJsonResultsModel,
+  visitPlaywrightSuites as visitPlaywrightSuitesModel,
+} from "../reporters/playwright.mjs";
+import {
+  collectGitMetadata as collectGitMetadataModel,
+  readPackageMetadata as readPackageMetadataModel,
+  safeHostname as safeHostnameModel,
+  safeUsername as safeUsernameModel,
+} from "./metadata.mjs";
+import {
+  findGraphDirsForService as findGraphDirsForServiceModel,
+  findRuntimeStateDirs as findRuntimeStateDirsModel,
+  normalizePathSeparators as normalizePathSeparatorsModel,
+  readGraphMetadata as readGraphMetadataModel,
+  writeGraphMetadata as writeGraphMetadataModel,
+} from "./state.mjs";
+import { uploadTelemetryArtifact } from "../telemetry/index.mjs";
+
+const HTTP_K6_TYPES = new Set(["integration", "e2e", "load"]);
+const DEFAULT_READY_TIMEOUT_MS = 120_000;
+const TIMINGS_FILENAME = "timings.json";
+
+export async function runAll(configs, suiteType, suiteNames, opts, allConfigs = configs) {
+  const configMap = new Map(allConfigs.map((config) => [config.name, config]));
+  const startedAt = Date.now();
+  const telemetry = configs[0]?.telemetry || null;
+  const servicePlans = collectServicePlans(configs, configMap, suiteType, suiteNames, opts);
+  const trackers = buildServiceTrackers(servicePlans, startedAt);
+  const executedPlans = servicePlans.filter((plan) => !plan.skipped);
+  let workerCount = 0;
+
+  if (executedPlans.length > 0) {
+    const productDir = executedPlans[0].config.productDir;
+    const timings = loadTimings(productDir);
+    const graphs = buildRuntimeGraphs(executedPlans);
+    const queue = buildTaskQueue(executedPlans, graphs, timings);
+    workerCount = Math.max(1, Math.min(opts.jobs || 1, queue.length));
+    const graphByKey = new Map(graphs.map((graph) => [graph.key, graph]));
+    const workers = Array.from({ length: workerCount }, (_unused, index) =>
+      createWorker(index + 1, productDir)
+    );
+    const timingUpdates = [];
+
+    const workerResults = await Promise.allSettled(
+      workers.map((worker) =>
+        runWorker(worker, queue, graphByKey, trackers, timingUpdates)
+      )
+    );
+
+    for (const result of workerResults) {
+      if (result.status === "rejected") {
+        const message = formatError(result.reason);
+        for (const tracker of trackers.values()) {
+          if (!tracker.skipped) addTrackerError(tracker, message);
+        }
+      }
+    }
+
+    saveTimings(productDir, timings, timingUpdates);
+  }
+
+  const finishedAt = Date.now();
+  const results = configs.map((config) =>
+    finalizeServiceResult(trackers.get(config.name), startedAt, finishedAt)
+  );
+  const artifact = buildRunArtifact({
+    productDir: configs[0]?.productDir || process.cwd(),
+    results,
+    startedAt,
+    finishedAt,
+    requestedJobs: opts.jobs || 1,
+    workerCount,
+    suiteType,
+    suiteNames,
+    framework: opts.framework || "all",
+    shard: opts.shard || null,
+    serviceFilter: configs.length === 1 ? configs[0].name : null,
+  });
+
+  writeRunArtifact(configs[0]?.productDir || process.cwd(), artifact);
+
+  printRunSummary(results, finishedAt - startedAt);
+  await reportTelemetry(telemetry, artifact);
+  if (results.some((result) => result.failed)) process.exit(1);
+}
+
+export async function destroy(config) {
+  const roots = new Set([config.stateDir, ...findGraphDirsForService(config.productDir, config.name)]);
+
+  for (const rootDir of roots) {
+    if (!fs.existsSync(rootDir)) continue;
+    const runtimeStateDirs = findRuntimeStateDirs(rootDir);
+    for (const stateDir of runtimeStateDirs) {
+      await destroyRuntimeDatabase({
+        productDir: config.productDir,
+        stateDir,
+      });
+    }
+    fs.rmSync(rootDir, { recursive: true, force: true });
+  }
+
+  await destroyServiceDatabaseCache(config.productDir, config.name);
+  await cleanupOrphanedLocalInfrastructure(config.productDir);
+}
+
+export function showStatus(config) {
+  const graphDirs = findGraphDirsForService(config.productDir, config.name);
+  const hasDirectState = fs.existsSync(config.stateDir);
+  const hasGraphState = graphDirs.length > 0;
+
+  if (!hasDirectState && !hasGraphState) {
+    console.log("No state — run tests first.");
+  } else {
+    if (hasDirectState) {
+      console.log("  service-state/");
+      printStateDir(config.stateDir, "    ");
+    }
+    for (const graphDir of graphDirs) {
+      console.log(`  graph-state/${path.basename(graphDir)}/`);
+      printStateDir(graphDir, "    ");
+    }
+  }
+
+  showServiceDatabaseStatus(config.productDir, config.name);
+}
+
+function collectServicePlans(configs, configMap, suiteType, suiteNames, opts) {
+  return configs.map((config) => {
+    console.log(`\n══ ${config.name} ══`);
+    const suites = applyShard(
+      collectSuites(config, suiteType, suiteNames, opts.framework),
+      opts.shard
+    );
+
+    if (suites.length === 0) {
+      console.log(
+        `No test files for ${config.name} type="${suiteType}" suites=${suiteNames.join(",") || "all"} — skipping`
+      );
+      return {
+        config,
+        skipped: true,
+        suites: [],
+        runtimeConfigs: [],
+        runtimeNames: [],
+        runtimeKey: null,
+      };
+    }
+
+    const runtimeConfigs = resolveRuntimeConfigs(config, configMap);
+    return {
+      config,
+      skipped: false,
+      suites,
+      runtimeConfigs,
+      runtimeNames: runtimeConfigs.map((runtimeConfig) => runtimeConfig.name).sort(),
+      runtimeKey: runtimeConfigs.map((runtimeConfig) => runtimeConfig.name).sort().join("|"),
+    };
+  });
+}
+
+function buildServiceTrackers(servicePlans, startedAt) {
+  return buildServiceTrackersModel(servicePlans, startedAt);
+}
+
+function buildRuntimeGraphs(servicePlans) {
+  return buildRuntimeGraphsModel(servicePlans);
+}
+
+function buildTaskQueue(servicePlans, graphs, timings) {
+  return buildTaskQueueModel(servicePlans, graphs, timings);
+}
+
+function createWorker(workerId, productDir) {
+  return {
+    workerId,
+    productDir,
+    currentGraphKey: null,
+    graphContexts: new Map(),
+    graphSwitches: 0,
+    taskCount: 0,
+  };
+}
+
+async function runWorker(worker, queue, graphByKey, trackers, timingUpdates) {
+  const startedAt = Date.now();
+  console.log(`\n══ global worker ${worker.workerId} ══`);
+  const errors = [];
+
+  try {
+    while (true) {
+      const batch = claimNextBatch(queue, worker.currentGraphKey);
+      if (!batch) break;
+
+      try {
+        const context = await ensureWorkerGraph(worker, batch, graphByKey);
+        const outcomes = await runBatch(context, batch);
+        for (const outcome of outcomes) {
+          recordTaskOutcome(trackers, outcome.task, outcome);
+          timingUpdates.push({
+            key: outcome.task.timingKey,
+            durationMs: outcome.durationMs,
+          });
+          worker.taskCount += 1;
+        }
+      } catch (error) {
+        const message = formatError(error);
+        errors.push(message);
+        recordGraphError(trackers, graphByKey.get(batch.graphKey), message);
+        await resetCurrentGraph(worker);
+      }
+    }
+  } finally {
+    await cleanupWorker(worker);
+  }
+
+  return {
+    workerId: worker.workerId,
+    failed: errors.length > 0,
+    durationMs: Date.now() - startedAt,
+    taskCount: worker.taskCount,
+    graphSwitches: worker.graphSwitches,
+    errors,
+  };
+}
+
+function claimNextBatch(queue, preferredGraphKey) {
+  return claimNextBatchModel(queue, preferredGraphKey);
+}
+
+async function ensureWorkerGraph(worker, batch, graphByKey) {
+  const graph = graphByKey.get(batch.graphKey);
+  if (!graph) {
+    throw new Error(`Unknown graph "${batch.graphKey}"`);
+  }
+
+  if (worker.currentGraphKey && worker.currentGraphKey !== batch.graphKey) {
+    await deactivateGraphContext(worker.graphContexts.get(worker.currentGraphKey));
+    worker.graphSwitches += 1;
+    worker.currentGraphKey = null;
+  }
+
+  let context = worker.graphContexts.get(batch.graphKey);
+  if (!context) {
+    context = createGraphContext(worker, graph);
+    worker.graphContexts.set(batch.graphKey, context);
+  }
+
+  if (!context.prepared) {
+    await prepareDatabases(context.runtimeConfigs);
+    context.prepared = true;
+  }
+
+  if (batchNeedsLocalRuntime(batch) && !context.started) {
+    context.startedServices = await startLocalServices(context.runtimeConfigs);
+    context.started = true;
+  }
+
+  worker.currentGraphKey = batch.graphKey;
+  return context;
+}
+
+function createGraphContext(worker, graph) {
+  const graphDir = path.join(worker.productDir, ".testkit", "_graphs", graph.dirName);
+  const workerStateDir = path.join(graphDir, "workers", `worker-${worker.workerId}`);
+  fs.mkdirSync(workerStateDir, { recursive: true });
+  writeGraphMetadata(graphDir, graph);
+
+  const runtimeConfigs = resolveWorkerRuntimeConfigs(
+    graph.rootConfig,
+    graph.runtimeConfigs,
+    worker.workerId,
+    workerStateDir
+  );
+
+  return {
+    graphKey: graph.key,
+    graphDir,
+    workerStateDir,
+    runtimeConfigs,
+    configByName: new Map(runtimeConfigs.map((config) => [config.name, config])),
+    prepared: false,
+    started: false,
+    startedServices: [],
+  };
+}
+
+async function deactivateGraphContext(context) {
+  if (!context?.started) return;
+  await stopLocalServices(context.startedServices);
+  context.started = false;
+  context.startedServices = [];
+}
+
+async function resetCurrentGraph(worker) {
+  if (!worker.currentGraphKey) return;
+  await deactivateGraphContext(worker.graphContexts.get(worker.currentGraphKey));
+  worker.currentGraphKey = null;
+}
+
+async function cleanupWorker(worker) {
+  for (const context of worker.graphContexts.values()) {
+    await deactivateGraphContext(context);
+  }
+  worker.currentGraphKey = null;
+}
+
+async function runBatch(context, batch) {
+  const targetConfig = context.configByName.get(batch.targetName);
+  if (!targetConfig) {
+    throw new Error(`Worker graph missing target config "${batch.targetName}"`);
+  }
+
+  if (batch.framework === "playwright") {
+    return runPlaywrightBatch(targetConfig, batch);
+  }
+  if (batch.type === "dal") {
+    return runDalBatch(targetConfig, batch);
+  }
+  if (batch.framework === "k6" && HTTP_K6_TYPES.has(batch.type)) {
+    return runHttpK6Batch(targetConfig, batch);
+  }
+
+  throw new Error(
+    `Unsupported task combination for ${batch.targetName}: type=${batch.type} framework=${batch.framework}`
+  );
+}
+
+async function prepareDatabases(runtimeConfigs) {
+  for (const config of runtimeConfigs) {
+    await prepareDatabaseRuntime(config, {
+      runMigrate: config.testkit.migrate
+        ? (databaseUrl) => runMigrate(config, databaseUrl)
+        : null,
+      runSeed: config.testkit.seed ? (databaseUrl) => runSeed(config, databaseUrl) : null,
+    });
+  }
+}
+
+async function runMigrate(config, databaseUrl) {
+  const migrate = config.testkit.migrate;
+  if (!migrate) return;
+
+  const env = buildExecutionEnv(config);
+  if (databaseUrl) env.DATABASE_URL = databaseUrl;
+
+  console.log(`\n── migrate:${config.workerLabel}:${config.name} ──`);
+  await execaCommand(migrate.cmd, {
+    cwd: resolveServiceCwd(config.productDir, migrate.cwd),
+    env,
+    stdio: "inherit",
+    shell: true,
+  });
+}
+
+async function runSeed(config, databaseUrl) {
+  const seed = config.testkit.seed;
+  if (!seed) return;
+
+  const env = buildExecutionEnv(config);
+  if (databaseUrl) env.DATABASE_URL = databaseUrl;
+
+  console.log(`\n── seed:${config.workerLabel}:${config.name} ──`);
+  await execaCommand(seed.cmd, {
+    cwd: resolveServiceCwd(config.productDir, seed.cwd),
+    env,
+    stdio: "inherit",
+    shell: true,
+  });
+}
+
+async function startLocalServices(runtimeConfigs) {
+  const started = [];
+
+  try {
+    for (const config of runtimeConfigs) {
+      if (!config.testkit.local) continue;
+      const proc = await startLocalService(config);
+      started.push(proc);
+    }
+  } catch (error) {
+    await stopLocalServices(started);
+    throw error;
+  }
+
+  return started;
+}
+
+async function startLocalService(config) {
+  const cwd = resolveServiceCwd(config.productDir, config.testkit.local.cwd);
+  const env = buildExecutionEnv(config, config.testkit.local.env);
+  const port = config.testkit.local.port || numericPortFromUrl(config.testkit.local.baseUrl);
+  if (port) {
+    env.PORT = String(port);
+  }
+
+  const dbUrl = readDatabaseUrl(config.stateDir);
+  if (dbUrl) {
+    env.DATABASE_URL = dbUrl;
+  }
+
+  await assertLocalServicePortsAvailable(config);
+
+  console.log(`Starting ${config.workerLabel}:${config.name}: ${config.testkit.local.start}`);
+  const child = spawn(config.testkit.local.start, {
+    cwd,
+    env,
+    detached: true,
+    shell: true,
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+
+  const outputDrains = [
+    pipeOutput(child.stdout, `[${config.workerLabel}:${config.name}]`),
+    pipeOutput(child.stderr, `[${config.workerLabel}:${config.name}]`),
+  ];
+
+  const readyTimeoutMs = config.testkit.local.readyTimeoutMs || DEFAULT_READY_TIMEOUT_MS;
+
+  try {
+    await waitForReady({
+      name: `${config.workerLabel}:${config.name}`,
+      url: config.testkit.local.readyUrl,
+      timeoutMs: readyTimeoutMs,
+      process: child,
+    });
+  } catch (error) {
+    await stopChildProcess(child, outputDrains);
+    throw error;
+  }
+
+  return { name: config.name, child, outputDrains };
+}
+
+async function runHttpK6Batch(targetConfig, batch) {
+  const baseUrl = targetConfig.testkit.local?.baseUrl;
+  if (!baseUrl) {
+    throw new Error(`Service "${targetConfig.name}" requires local.baseUrl for HTTP k6 suites`);
+  }
+
+  console.log(
+    `\n── ${targetConfig.workerLabel} ${batch.type}:${batch.tasks[0].suiteName} (${batch.framework}, ${batch.tasks.length} file${batch.tasks.length === 1 ? "" : "s"}) ──`
+  );
+
+  return Promise.all(
+    batch.tasks.map((task) => runHttpK6Task(targetConfig, task, baseUrl))
+  );
+}
+
+async function runHttpK6Task(targetConfig, task, baseUrl) {
+  const absFile = path.join(targetConfig.productDir, task.file);
+  console.log(`·· ${targetConfig.workerLabel}:${task.suiteName} → ${task.file}`);
+  const startedAt = Date.now();
+  try {
+    await execa("k6", ["run", "--address", "127.0.0.1:0", "-e", `BASE_URL=${baseUrl}`, absFile], {
+      cwd: targetConfig.productDir,
+      env: buildExecutionEnv(targetConfig),
+      stdio: "inherit",
+    });
+    return {
+      task,
+      failed: false,
+      error: null,
+      durationMs: Date.now() - startedAt,
+    };
+  } catch (error) {
+    return {
+      task,
+      failed: true,
+      error: formatError(error),
+      durationMs: Date.now() - startedAt,
+    };
+  }
+}
+
+async function runDalBatch(targetConfig, batch) {
+  const databaseUrl = readDatabaseUrl(targetConfig.stateDir);
+  if (!databaseUrl) {
+    throw new Error(`Service "${targetConfig.name}" requires a database for DAL suites`);
+  }
+
+  console.log(
+    `\n── ${targetConfig.workerLabel} ${batch.type}:${batch.tasks[0].suiteName} (${batch.framework}, ${batch.tasks.length} file${batch.tasks.length === 1 ? "" : "s"}) ──`
+  );
+
+  return Promise.all(
+    batch.tasks.map((task) => runDalTask(targetConfig, task, databaseUrl))
+  );
+}
+
+async function runDalTask(targetConfig, task, databaseUrl) {
+  const absFile = path.join(targetConfig.productDir, task.file);
+  const k6Binary = resolveDalBinary();
+  console.log(`·· ${targetConfig.workerLabel}:${task.suiteName} → ${task.file}`);
+  const startedAt = Date.now();
+  try {
+    await execa(
+      k6Binary,
+      ["run", "--address", "127.0.0.1:0", "-e", `DATABASE_URL=${databaseUrl}`, absFile],
+      {
+      cwd: targetConfig.productDir,
+      env: buildExecutionEnv(targetConfig),
+      stdio: "inherit",
+      }
+    );
+    return {
+      task,
+      failed: false,
+      error: null,
+      durationMs: Date.now() - startedAt,
+    };
+  } catch (error) {
+    return {
+      task,
+      failed: true,
+      error: formatError(error),
+      durationMs: Date.now() - startedAt,
+    };
+  }
+}
+
+async function runPlaywrightBatch(targetConfig, batch) {
+  const local = targetConfig.testkit.local;
+  if (!local?.baseUrl) {
+    throw new Error(
+      `Service "${targetConfig.name}" requires local.baseUrl for Playwright suites`
+    );
+  }
+
+  console.log(
+    `\n── ${targetConfig.workerLabel} playwright:${targetConfig.name} (${batch.tasks.length} file${batch.tasks.length === 1 ? "" : "s"}) ──`
+  );
+
+  const cwd = resolveServiceCwd(targetConfig.productDir, local.cwd);
+  const requestedFiles = batch.tasks.map((task) =>
+    path.relative(cwd, path.join(targetConfig.productDir, task.file))
+  );
+  const startedAt = Date.now();
+  const result = await execa(
+    "npx",
+    ["playwright", "test", "--reporter=json", ...requestedFiles],
+    {
+      cwd,
+      env: buildPlaywrightEnv(targetConfig, local.baseUrl),
+      reject: false,
+    }
+  );
+
+  if (result.stderr) {
+    printBufferedOutput(result.stderr, `[${targetConfig.workerLabel}:${targetConfig.name}:playwright]`);
+  }
+
+  const parsed = parsePlaywrightJsonResults(result.stdout, cwd);
+  const batchDurationMs = Date.now() - startedAt;
+  const genericError =
+    result.exitCode === 0
+      ? parsed.errors[0] || null
+      : parsed.errors[0] ||
+        result.stderr.trim() ||
+        `Playwright exited with code ${result.exitCode}`;
+
+  return batch.tasks.map((task) => {
+    const relativeFile = normalizePathSeparators(
+      path.relative(cwd, path.join(targetConfig.productDir, task.file))
+    );
+    const fileResult = parsed.fileResults.get(relativeFile);
+    if (fileResult) {
+      return {
+        task,
+        failed: fileResult.failed,
+        error: fileResult.error,
+        durationMs:
+          fileResult.durationMs > 0
+            ? fileResult.durationMs
+            : Math.round(batchDurationMs / Math.max(1, batch.tasks.length)),
+      };
+    }
+
+    return {
+      task,
+      failed: result.exitCode !== 0,
+      error: result.exitCode !== 0 ? genericError : null,
+      durationMs: Math.round(batchDurationMs / Math.max(1, batch.tasks.length)),
+    };
+  });
+}
+
+async function stopLocalServices(started) {
+  for (const service of [...started].reverse()) {
+    await stopChildProcess(service.child, service.outputDrains);
+  }
+}
+
+async function stopChildProcess(child, outputDrains = []) {
+  if (!child) return;
+  if (child.exitCode !== null) {
+    await Promise.all(outputDrains);
+    return;
+  }
+
+  killChildProcess(child, "SIGTERM");
+  const exited = await Promise.race([
+    new Promise((resolve) => child.once("exit", () => resolve(true))),
+    sleep(5_000).then(() => false),
+  ]);
+
+  if (!exited && child.exitCode === null) {
+    killChildProcess(child, "SIGKILL");
+    await new Promise((resolve) => child.once("exit", resolve));
+  }
+
+  await Promise.all(outputDrains);
+}
+
+async function waitForReady({ name, url, timeoutMs, process }) {
+  const start = Date.now();
+
+  while (Date.now() - start < timeoutMs) {
+    if (process.exitCode !== null) {
+      throw new Error(`Service "${name}" exited before becoming ready`);
+    }
+
+    try {
+      const response = await fetch(url);
+      if (response.ok) return;
+    } catch {
+      // Service still warming up.
+    }
+
+    await sleep(1_000);
+  }
+
+  throw new Error(`Timed out waiting for "${name}" to become ready at ${url}`);
+}
+
+function batchNeedsLocalRuntime(batch) {
+  return batchNeedsLocalRuntimeModel(batch);
+}
+
+function resolveRuntimeConfigs(targetConfig, configMap) {
+  return resolveRuntimeConfigsModel(targetConfig, configMap);
+}
+
+function collectSuites(config, suiteType, suiteNames, frameworkFilter) {
+  return collectSuitesModel(config, suiteType, suiteNames, frameworkFilter);
+}
+
+function applyShard(suites, shard) {
+  return applyShardModel(suites, shard);
+}
+
+function orderedTypes(types) {
+  return orderedTypesModel(types);
+}
+
+function resolveWorkerRuntimeConfigs(targetConfig, runtimeConfigs, workerId, workerStateDir) {
+  return resolveWorkerRuntimeConfigsModel(targetConfig, runtimeConfigs, workerId, workerStateDir);
+}
+
+function buildPortMap(runtimeConfigs, workerId) {
+  return buildPortMapModel(runtimeConfigs, workerId);
+}
+
+function resolveWorkerConfig(
+  config,
+  targetConfig,
+  workerId,
+  workerStateDir,
+  portMap,
+  baseUrlByService,
+  readyUrlByService,
+  urlMappings
+) {
+  return resolveWorkerConfigModel(
+    config,
+    targetConfig,
+    workerId,
+    workerStateDir,
+    portMap,
+    baseUrlByService,
+    readyUrlByService,
+    urlMappings
+  );
+}
+
+function resolveServiceStateDir(workerStateDir, targetName, config) {
+  return resolveServiceStateDirModel(workerStateDir, targetName, config);
+}
+
+function getWorkerServiceStateDir(workerStateDir, targetName, serviceName) {
+  return getWorkerServiceStateDirModel(workerStateDir, targetName, serviceName);
+}
+
+function buildExecutionEnv(config, extraEnv = {}) {
+  return buildExecutionEnvModel(config, extraEnv, process.env);
+}
+
+function buildPlaywrightEnv(config, baseUrl) {
+  return buildPlaywrightEnvModel(config, baseUrl, process.env);
+}
+
+function recordTaskOutcome(trackers, task, outcome) {
+  return recordTaskOutcomeModel(trackers, task, outcome);
+}
+
+function recordGraphError(trackers, graph, message) {
+  return recordGraphErrorModel(trackers, graph, message);
+}
+
+function addTrackerError(tracker, message) {
+  return addTrackerErrorModel(tracker, message);
+}
+
+function finalizeServiceResult(tracker, startedAt, finishedAt) {
+  return finalizeServiceResultModel(tracker, startedAt, finishedAt);
+}
+
+function printRunSummary(results, durationMs) {
+  const totalServices = results.length;
+  const executedServices = results.filter((result) => !result.skipped);
+  const skippedServices = results.filter((result) => result.skipped);
+  const failedServices = executedServices.filter((result) => result.failed);
+  const passedServices = executedServices.filter((result) => !result.failed);
+  const totalSuites = executedServices.reduce((sum, result) => sum + result.suiteCount, 0);
+  const completedSuites = executedServices.reduce(
+    (sum, result) => sum + result.completedSuiteCount,
+    0
+  );
+  const failedSuites = executedServices.reduce((sum, result) => sum + result.failedSuiteCount, 0);
+  const passedSuites = completedSuites - failedSuites;
+
+  console.log("\n══ Summary ══");
+  console.log(
+    [
+      `services ${passedServices.length}/${executedServices.length} passed`,
+      `suites ${passedSuites}/${totalSuites} passed`,
+      skippedServices.length > 0 ? `${skippedServices.length} skipped` : null,
+      `duration ${formatDuration(durationMs)}`,
+    ]
+      .filter(Boolean)
+      .join("  ·  ")
+  );
+
+  for (const result of results) {
+    const status = result.skipped ? "SKIP" : result.failed ? "FAIL" : "PASS";
+    const detail = result.skipped ? "no matching suites" : formatServiceSummary(result);
+    console.log(
+      `${status.padEnd(4)} ${result.name.padEnd(longestServiceName(results))}  ${detail}  ·  ${formatDuration(result.durationMs)}`
+    );
+
+    if (result.failed) {
+      const failedSuitesForService = result.suites.filter((suite) => suite.failed);
+      for (const suite of failedSuitesForService) {
+        const fileDetail =
+          suite.failedFiles.length > 0 ? `  (${suite.failedFiles.join(", ")})` : "";
+        console.log(
+          `  - ${suite.type}:${suite.name} [${suite.framework}]${fileDetail} · ${formatDuration(suite.durationMs)}`
+        );
+        if (suite.error) {
+          console.log(`    ${suite.error}`);
+        }
+      }
+      for (const error of result.errors) {
+        console.log(`  - worker error: ${error}`);
+      }
+    }
+  }
+
+  if (failedServices.length > 0) {
+    console.log(`\nResult: FAILED (${failedServices.length}/${totalServices} services failed)`);
+    return;
+  }
+
+  console.log("\nResult: PASSED");
+}
+
+async function reportTelemetry(telemetry, artifact) {
+  if (!telemetry?.enabled) return;
+
+  try {
+    const outcome = await uploadTelemetryArtifact(telemetry, artifact);
+    if (outcome?.ok) {
+      console.log("Telemetry: uploaded run artifact");
+      return;
+    }
+    if (outcome?.reason === "missing-token") {
+      console.log(
+        `Telemetry: skipped upload because ${telemetry.tokenEnv || "configured token env"} is not set`
+      );
+      return;
+    }
+    if (outcome?.reason && !outcome.skipped) return;
+  } catch (error) {
+    console.log(`Telemetry: upload failed (${formatError(error)})`);
+  }
+}
+
+function longestServiceName(results) {
+  return longestServiceNameModel(results);
+}
+
+function buildRunArtifact({
+  productDir,
+  results,
+  startedAt,
+  finishedAt,
+  requestedJobs,
+  workerCount,
+  suiteType,
+  suiteNames,
+  framework,
+  shard,
+  serviceFilter,
+}) {
+  return buildRunArtifactModel({
+    productDir,
+    results,
+    startedAt,
+    finishedAt,
+    requestedJobs,
+    workerCount,
+    suiteType,
+    suiteNames,
+    framework,
+    shard,
+    serviceFilter,
+    metadata: {
+      git: collectGitMetadata(productDir),
+      host: {
+        hostname: safeHostname(),
+        username: safeUsername(),
+      },
+      testkitVersion: readPackageMetadata().version,
+    },
+  });
+}
+
+function writeRunArtifact(productDir, artifact) {
+  const resultsDir = path.join(productDir, ".testkit", "results");
+  fs.mkdirSync(resultsDir, { recursive: true });
+  fs.writeFileSync(path.join(resultsDir, "latest.json"), JSON.stringify(artifact, null, 2));
+}
+
+function summarizeDbBackend(results) {
+  return summarizeDbBackendModel(results);
+}
+
+function collectGitMetadata(productDir) {
+  return collectGitMetadataModel(productDir);
+}
+
+function readPackageMetadata() {
+  return readPackageMetadataModel();
+}
+
+function safeHostname() {
+  return safeHostnameModel();
+}
+
+function safeUsername() {
+  return safeUsernameModel();
+}
+
+function formatDuration(durationMs) {
+  return formatDurationModel(durationMs);
+}
+
+function formatServiceSummary(result) {
+  return formatServiceSummaryModel(result);
+}
+
+function formatError(error) {
+  return formatErrorModel(error);
+}
+
+function loadTimings(productDir) {
+  const filePath = path.join(productDir, ".testkit", TIMINGS_FILENAME);
+  if (!fs.existsSync(filePath)) {
+    return createEmptyTimings();
+  }
+
+  try {
+    return normalizeTimings(JSON.parse(fs.readFileSync(filePath, "utf8")));
+  } catch {
+    return createEmptyTimings();
+  }
+}
+
+function saveTimings(productDir, timings, updates) {
+  if (updates.length === 0) return;
+  const next = applyTimingUpdates(timings, updates);
+
+  const rootDir = path.join(productDir, ".testkit");
+  fs.mkdirSync(rootDir, { recursive: true });
+  fs.writeFileSync(
+    path.join(rootDir, TIMINGS_FILENAME),
+    JSON.stringify(next, null, 2)
+  );
+}
+
+function estimateTaskDuration(timings, timingKey, suite) {
+  return estimateTaskDurationModel(timings, timingKey, suite);
+}
+
+function buildTimingKey(serviceName, suite, file) {
+  return buildTimingKeyModel(serviceName, suite, file);
+}
+
+function parsePlaywrightJsonResults(stdout, cwd) {
+  return parsePlaywrightJsonResultsModel(stdout, cwd);
+}
+
+function visitPlaywrightSuites(suites, inheritedFile, fileResults, cwd) {
+  return visitPlaywrightSuitesModel(suites, inheritedFile, fileResults, cwd);
+}
+
+function collectPlaywrightSpec(spec, inheritedFile, fileResults, cwd) {
+  return collectPlaywrightSpecModel(spec, inheritedFile, fileResults, cwd);
+}
+
+function choosePlaywrightFinalResult(results) {
+  return choosePlaywrightFinalResultModel(results);
+}
+
+function isPlaywrightPassingStatus(status) {
+  return isPlaywrightPassingStatusModel(status);
+}
+
+function extractPlaywrightFailure(finalResult, spec, test) {
+  return extractPlaywrightFailureModel(finalResult, spec, test);
+}
+
+function formatPlaywrightReporterError(error) {
+  return formatPlaywrightReporterErrorModel(error);
+}
+
+function extractReporterFile(node) {
+  return extractReporterFileModel(node);
+}
+
+function normalizeReportedFile(filePath, cwd) {
+  return normalizeReportedFileModel(filePath, cwd);
+}
+
+function firstLine(value) {
+  return firstLineModel(value);
+}
+
+function printBufferedOutput(output, prefix) {
+  for (const line of output.split(/\r?\n/)) {
+    if (line.trim().length > 0) {
+      console.log(`${prefix} ${line}`);
+    }
+  }
+}
+
+function resolveRuntimeUrl(rawUrl, serviceName, targetConfig, workerId, context) {
+  return resolveRuntimeUrlModel(rawUrl, serviceName, targetConfig, workerId, context);
+}
+
+function finalizeString(value, context) {
+  return finalizeStringModel(value, context);
+}
+
+function resolveTemplateString(value, context) {
+  return resolveTemplateStringModel(value, context);
+}
+
+function rewriteUrlPort(rawUrl, port) {
+  return rewriteUrlPortModel(rawUrl, port);
+}
+
+function numericPortFromUrl(rawUrl) {
+  return numericPortFromUrlModel(rawUrl);
+}
+
+async function assertLocalServicePortsAvailable(config) {
+  const endpoints = [config.testkit.local.baseUrl, config.testkit.local.readyUrl];
+  const seen = new Set();
+
+  for (const endpoint of endpoints) {
+    const socket = socketFromUrl(endpoint);
+    if (!socket) continue;
+
+    const key = `${socket.host}:${socket.port}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+
+    if (await isPortInUse(socket)) {
+      throw new Error(
+        `Cannot start "${config.workerLabel}:${config.name}" because ${key} is already in use. ` +
+          `Stop the existing process and rerun testkit.`
+      );
+    }
+  }
+}
+
+function socketFromUrl(rawUrl) {
+  return socketFromUrlModel(rawUrl);
+}
+
+function normalizeSocketHost(hostname) {
+  return normalizeSocketHostModel(hostname);
+}
+
+async function isPortInUse({ host, port }) {
+  return new Promise((resolve, reject) => {
+    const socket = new net.Socket();
+    let settled = false;
+
+    const finish = (value, error = null) => {
+      if (settled) return;
+      settled = true;
+      socket.destroy();
+      if (error) {
+        reject(error);
+        return;
+      }
+      resolve(value);
+    };
+
+    socket.setTimeout(1_000);
+    socket.once("connect", () => finish(true));
+    socket.once("timeout", () => finish(false));
+    socket.once("error", (error) => {
+      if (["ECONNREFUSED", "EHOSTUNREACH", "ENOTFOUND"].includes(error.code)) {
+        finish(false);
+        return;
+      }
+      finish(false, error);
+    });
+
+    socket.connect(port, host);
+  });
+}
+
+function killChildProcess(child, signal) {
+  if (!child?.pid) return;
+
+  try {
+    process.kill(-child.pid, signal);
+    return;
+  } catch (error) {
+    if (error?.code !== "ESRCH") {
+      // Fall back to the direct child if process-group signalling is unavailable.
+    } else {
+      return;
+    }
+  }
+
+  try {
+    child.kill(signal);
+  } catch (error) {
+    if (error?.code !== "ESRCH") throw error;
+  }
+}
+
+function readDatabaseUrl(stateDir) {
+  return readStateValue(path.join(stateDir, "database_url"));
+}
+
+function readStateValue(filePath) {
+  if (!fs.existsSync(filePath)) return null;
+  return fs.readFileSync(filePath, "utf8").trim();
+}
+
+function printStateDir(dir, indent) {
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const filePath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      console.log(`${indent}${entry.name}/`);
+      printStateDir(filePath, `${indent}  `);
+      continue;
+    }
+    const value =
+      entry.name === "password" ? "********" : fs.readFileSync(filePath, "utf8").trim();
+    console.log(`${indent}${entry.name}: ${value}`);
+  }
+}
+
+function pipeOutput(stream, prefix) {
+  if (!stream) return Promise.resolve();
+
+  let pending = "";
+  return new Promise((resolve) => {
+    let settled = false;
+    const settle = () => {
+      if (settled) return;
+      settled = true;
+      resolve();
+    };
+
+    stream.on("data", (chunk) => {
+      pending += chunk.toString();
+      const lines = pending.split(/\r?\n/);
+      pending = lines.pop() || "";
+      for (const line of lines) {
+        if (line.length > 0) console.log(`${prefix} ${line}`);
+      }
+    });
+    stream.on("end", () => {
+      if (pending.length > 0) {
+        console.log(`${prefix} ${pending}`);
+      }
+      settle();
+    });
+    stream.on("close", settle);
+    stream.on("error", settle);
+  });
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function findRuntimeStateDirs(rootDir) {
+  return findRuntimeStateDirsModel(rootDir, isDatabaseStateDir);
+}
+
+function findGraphDirsForService(productDir, serviceName) {
+  return findGraphDirsForServiceModel(productDir, serviceName);
+}
+
+function writeGraphMetadata(graphDir, graph) {
+  return writeGraphMetadataModel(graphDir, graph);
+}
+
+function readGraphMetadata(graphDir) {
+  return readGraphMetadataModel(graphDir);
+}
+
+function isRuntimeSuperset(candidate, target) {
+  return isRuntimeSupersetModel(candidate, target);
+}
+
+function compareGraphsForAssignment(left, right) {
+  return compareGraphsForAssignmentModel(left, right);
+}
+
+function buildGraphDirName(runtimeNames) {
+  return buildGraphDirNameModel(runtimeNames);
+}
+
+function slugSegment(value) {
+  return value.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "") || "svc";
+}
+
+function normalizePathSeparators(filePath) {
+  return normalizePathSeparatorsModel(filePath);
+}