@elench/testkit 0.1.32 → 0.1.34

package/lib/runner/maintenance.mjs

@@ -0,0 +1,72 @@
+import fs from "fs";
+import {
+  cleanupOrphanedLocalInfrastructure,
+  destroyRuntimeDatabase,
+  destroyServiceDatabaseCache,
+  isDatabaseStateDir,
+  showServiceDatabaseStatus,
+} from "../database/index.mjs";
+import { cleanupRuns, formatRunSummary } from "./lifecycle.mjs";
+import { printRunStatus } from "./readiness.mjs";
+import { findGraphDirsForService, findRuntimeStateDirs } from "./state.mjs";
+import { printStateDir } from "./state-io.mjs";
+
+export async function destroy(config) {
+  await cleanupRuns(config.productDir, { includeActive: true });
+  const roots = new Set([
+    config.stateDir,
+    ...findGraphDirsForService(config.productDir, config.name),
+  ]);
+
+  for (const rootDir of roots) {
+    if (!fs.existsSync(rootDir)) continue;
+    const runtimeStateDirs = findRuntimeStateDirs(rootDir, isDatabaseStateDir);
+    for (const stateDir of runtimeStateDirs) {
+      await destroyRuntimeDatabase({
+        productDir: config.productDir,
+        stateDir,
+      });
+    }
+    fs.rmSync(rootDir, { recursive: true, force: true });
+  }
+
+  await destroyServiceDatabaseCache(config.productDir, config.name);
+  await cleanupOrphanedLocalInfrastructure(config.productDir);
+}
+
+export function showStatus(config) {
+  printRunStatus(config.productDir);
+  const graphDirs = findGraphDirsForService(config.productDir, config.name);
+  const hasDirectState = fs.existsSync(config.stateDir);
+  const hasGraphState = graphDirs.length > 0;
+
+  if (!hasDirectState && !hasGraphState) {
+    console.log("No state — run tests first.");
+  } else {
+    if (hasDirectState) {
+      console.log("  service-state/");
+      printStateDir(config.stateDir, "    ");
+    }
+    for (const graphDir of graphDirs) {
+      console.log(`  graph-state/${graphDir.split("/").at(-1)}/`);
+      printStateDir(graphDir, "    ");
+    }
+  }
+
+  showServiceDatabaseStatus(config.productDir, config.name);
+}
+
+export async function cleanup(productDir) {
+  const summary = await cleanupRuns(productDir, { includeActive: false });
+  if (summary.cleaned.length === 0 && summary.skippedActive.length === 0) {
+    console.log("No stale runs to clean.");
+    return;
+  }
+
+  for (const manifest of summary.cleaned) {
+    console.log(`Cleaned stale run ${formatRunSummary(manifest)}`);
+  }
+  for (const manifest of summary.skippedActive) {
+    console.log(`Active run still present: ${formatRunSummary(manifest)}`);
+  }
+}

package/lib/runner/orchestrator.mjs

@@ -0,0 +1,254 @@
+import {
+  applyShard,
+  buildRuntimeGraphs,
+  buildTaskQueue,
+  claimNextBatch,
+  collectSuites,
+  resolveRuntimeConfigs,
+} from "./planning.mjs";
+import {
+  addTrackerError,
+  buildServiceTrackers,
+  finalizeServiceResult,
+  recordGraphError,
+  recordTaskOutcome,
+  summarizeDbBackend,
+} from "./results.mjs";
+import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
+import { buildRunSummaryLines, formatError } from "./formatting.mjs";
+import { loadTimings, saveTimings, writeRunArtifact, writeStatusArtifact } from "./artifacts.mjs";
+import {
+  cleanupRunById,
+  cleanupRuns,
+  cleanupStaleRuns,
+  createRunLifecycle,
+} from "./lifecycle.mjs";
+import {
+  collectGitMetadata,
+  readPackageMetadata,
+  safeHostname,
+  safeUsername,
+} from "./metadata.mjs";
+import { createWorker, runWorker } from "./worker-loop.mjs";
+import { findUnmatchedRequestedFiles, isFullRunSelection } from "./selection.mjs";
+import { uploadTelemetryArtifact } from "../telemetry/index.mjs";
+
+export async function runAll(configs, suiteType, suiteNames, opts, allConfigs = configs) {
+  const configMap = new Map(allConfigs.map((config) => [config.name, config]));
+  const startedAt = Date.now();
+  const telemetry = configs[0]?.telemetry || null;
+  const productDir = configs[0]?.productDir || process.cwd();
+  await cleanupStaleRuns(productDir);
+  const metadata = {
+    git: collectGitMetadata(productDir),
+    host: {
+      hostname: safeHostname(),
+      username: safeUsername(),
+    },
+    testkitVersion: readPackageMetadata().version,
+  };
+  const requestedFiles = opts.fileNames || [];
+  if (requestedFiles.length > 0) {
+    const unmatchedFiles = findUnmatchedRequestedFiles(
+      configs,
+      suiteType,
+      suiteNames,
+      opts.framework || "all",
+      requestedFiles,
+      collectSuites,
+      normalizePathSeparators
+    );
+    if (unmatchedFiles.length > 0) {
+      throw new Error(
+        `Requested file${unmatchedFiles.length === 1 ? "" : "s"} did not match any selected suites:\n` +
+          unmatchedFiles.map((file) => `- ${file}`).join("\n")
+      );
+    }
+  }
+  if (
+    opts.writeStatus &&
+    !opts.allowPartialStatus &&
+    !isFullRunSelection(
+      suiteNames,
+      requestedFiles,
+      opts.framework || "all",
+      opts.shard || null,
+      opts.serviceFilter || null
+    )
+  ) {
+    throw new Error(
+      "Refusing to overwrite testkit.status.json from a filtered run. " +
+        "Run the full suite with --write-status, or pass --allow-partial-status to opt in."
+    );
+  }
+
+  const servicePlans = collectServicePlans(configs, configMap, suiteType, suiteNames, opts);
+  const trackers = buildServiceTrackers(servicePlans, startedAt);
+  const executedPlans = servicePlans.filter((plan) => !plan.skipped);
+  let workerCount = 0;
+  let exitCode = 0;
+  const lifecycle = createRunLifecycle(productDir);
+  lifecycle.markRunning();
+  lifecycle.installSignalHandlers();
+  let results = [];
+  let finishedAt = Date.now();
+
+  try {
+    if (executedPlans.length > 0) {
+      const timings = loadTimings(productDir);
+      const graphs = buildRuntimeGraphs(executedPlans);
+      const queue = buildTaskQueue(executedPlans, graphs, timings);
+      workerCount = Math.max(1, Math.min(opts.jobs || 1, queue.length));
+      const graphByKey = new Map(graphs.map((graph) => [graph.key, graph]));
+      const workers = Array.from({ length: workerCount }, (_unused, index) =>
+        createWorker(index + 1, productDir)
+      );
+      const timingUpdates = [];
+
+      const workerResults = await Promise.allSettled(
+        workers.map((worker) =>
+          runWorker(
+            worker,
+            queue,
+            graphByKey,
+            trackers,
+            timingUpdates,
+            lifecycle,
+            claimNextBatch,
+            recordTaskOutcome,
+            recordGraphError
+          )
+        )
+      );
+
+      for (const result of workerResults) {
+        if (result.status === "rejected") {
+          const message = formatError(result.reason);
+          for (const tracker of trackers.values()) {
+            if (!tracker.skipped) addTrackerError(tracker, message);
+          }
+        }
+      }
+
+      saveTimings(productDir, timings, timingUpdates);
+    }
+
+    finishedAt = Date.now();
+    results = configs.map((config) =>
+      finalizeServiceResult(trackers.get(config.name), startedAt, finishedAt)
+    );
+    const artifact = buildRunArtifact({
+      productDir,
+      results,
+      startedAt,
+      finishedAt,
+      requestedJobs: opts.jobs || 1,
+      workerCount,
+      suiteType,
+      suiteNames,
+      fileNames: requestedFiles,
+      framework: opts.framework || "all",
+      shard: opts.shard || null,
+      serviceFilter: opts.serviceFilter || null,
+      metadata,
+      summarizeDbBackend,
+    });
+
+    writeRunArtifact(productDir, artifact);
+    if (opts.writeStatus) {
+      writeStatusArtifact(
+        productDir,
+        buildStatusArtifact({
+          productDir,
+          results,
+          suiteType,
+          suiteNames,
+          fileNames: requestedFiles,
+          framework: opts.framework || "all",
+          shard: opts.shard || null,
+          serviceFilter: opts.serviceFilter || null,
+          metadata,
+        })
+      );
+    }
+
+    printRunSummary(results, finishedAt - startedAt);
+    await reportTelemetry(telemetry, artifact);
+    if (results.some((result) => result.failed)) exitCode = 1;
+    if (lifecycle.isStopRequested()) exitCode = Math.max(exitCode, 130);
+  } finally {
+    lifecycle.removeSignalHandlers();
+    lifecycle.markFinished(
+      exitCode === 0 ? "finished" : lifecycle.isStopRequested() ? "interrupted" : "failed"
+    );
+    await cleanupRunById(productDir, lifecycle.runId);
+    await cleanupRuns(productDir, { includeActive: false });
+    lifecycle.removeManifest();
+    process.exitCode = exitCode;
+  }
+}
+
+function collectServicePlans(configs, configMap, suiteType, suiteNames, opts) {
+  return configs.map((config) => {
+    console.log(`\n══ ${config.name} ══`);
+    const suites = applyShard(
+      collectSuites(config, suiteType, suiteNames, opts.framework, opts.fileNames || []),
+      opts.shard
+    );
+
+    if (suites.length === 0) {
+      console.log(
+        `No test files for ${config.name} type="${suiteType}" suites=${suiteNames.join(",") || "all"} files=${(opts.fileNames || []).join(",") || "all"} — skipping`
+      );
+      return {
+        config,
+        skipped: true,
+        suites: [],
+        runtimeConfigs: [],
+        runtimeNames: [],
+        runtimeKey: null,
+      };
+    }
+
+    const runtimeConfigs = resolveRuntimeConfigs(config, configMap);
+    return {
+      config,
+      skipped: false,
+      suites,
+      runtimeConfigs,
+      runtimeNames: runtimeConfigs.map((runtimeConfig) => runtimeConfig.name).sort(),
+      runtimeKey: runtimeConfigs.map((runtimeConfig) => runtimeConfig.name).sort().join("|"),
+    };
+  });
+}
+
+function printRunSummary(results, durationMs) {
+  for (const line of buildRunSummaryLines(results, durationMs)) {
+    console.log(line);
+  }
+}
+
+async function reportTelemetry(telemetry, artifact) {
+  if (!telemetry?.enabled) return;
+
+  try {
+    const outcome = await uploadTelemetryArtifact(telemetry, artifact);
+    if (outcome?.ok) {
+      console.log("Telemetry: uploaded run artifact");
+      return;
+    }
+    if (outcome?.reason === "missing-token") {
+      console.log(
+        `Telemetry: skipped upload because ${telemetry.tokenEnv || "configured token env"} is not set`
+      );
+      return;
+    }
+    if (outcome?.reason && !outcome.skipped) return;
+  } catch (error) {
+    console.log(`Telemetry: upload failed (${formatError(error)})`);
+  }
+}
+
+function normalizePathSeparators(filePath) {
+  return filePath.split("\\").join("/");
+}

package/lib/runner/playwright-config.mjs

@@ -0,0 +1,61 @@
+import fs from "fs";
+import path from "path";
+import { pathToFileURL } from "url";
+import { normalizePathSeparators } from "./state.mjs";
+
+export function ensurePlaywrightTestConfig(targetConfig, cwd, requestedFiles) {
+  const stateDir = targetConfig.stateDir || path.join(targetConfig.productDir, ".testkit");
+  const outputDir = resolvePlaywrightOutputDir(stateDir);
+  fs.mkdirSync(stateDir, { recursive: true });
+  fs.mkdirSync(outputDir, { recursive: true });
+  const configPath = path.join(stateDir, "playwright.testkit.config.mjs");
+  const baseConfigPath = findPlaywrightConfig(cwd);
+  const normalizedFiles = requestedFiles.map(normalizePathSeparators);
+
+  let source = "";
+  if (baseConfigPath) {
+    source =
+      `import baseConfig from ${JSON.stringify(pathToFileURL(baseConfigPath).href)};\n` +
+      `const resolvedBase = typeof baseConfig === "function" ? await baseConfig() : baseConfig;\n` +
+      `export default {\n` +
+      `  ...(resolvedBase || {}),\n` +
+      `  testDir: ${JSON.stringify(cwd)},\n` +
+      `  testMatch: ${JSON.stringify(normalizedFiles)},\n` +
+      `  outputDir: ${JSON.stringify(outputDir)},\n` +
+      `};\n`;
+  } else {
+    source =
+      `export default {\n` +
+      `  testDir: ${JSON.stringify(cwd)},\n` +
+      `  testMatch: ${JSON.stringify(normalizedFiles)},\n` +
+      `  outputDir: ${JSON.stringify(outputDir)},\n` +
+      `};\n`;
+  }
+
+  fs.writeFileSync(configPath, source);
+  return configPath;
+}
+
+export function resolvePlaywrightOutputDir(stateDir) {
+  return path.join(stateDir, "playwright-output");
+}
+
+export function findPlaywrightConfig(cwd) {
+  const candidates = [
+    "playwright.config.ts",
+    "playwright.config.mts",
+    "playwright.config.js",
+    "playwright.config.mjs",
+    "playwright.config.cjs",
+    "playwright.config.cts",
+  ];
+
+  for (const candidate of candidates) {
+    const candidatePath = path.join(cwd, candidate);
+    if (fs.existsSync(candidatePath)) {
+      return candidatePath;
+    }
+  }
+
+  return null;
+}

package/lib/runner/playwright-config.test.mjs

@@ -0,0 +1,58 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { pathToFileURL } from "url";
+import { afterEach, describe, expect, it } from "vitest";
+import {
+  ensurePlaywrightTestConfig,
+  findPlaywrightConfig,
+  resolvePlaywrightOutputDir,
+} from "./playwright-config.mjs";
+
+const cleanups = [];
+
+afterEach(() => {
+  while (cleanups.length > 0) {
+    cleanups.pop()();
+  }
+});
+
+function makeTempDir(prefix) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+  cleanups.push(() => fs.rmSync(dir, { recursive: true, force: true }));
+  return dir;
+}
+
+describe("runner-playwright-config", () => {
+  it("uses a shard-local output directory under the state dir", async () => {
+    const productDir = makeTempDir("testkit-playwright-product-");
+    const stateDir = path.join(productDir, ".testkit", "worker-3");
+    const cwd = path.join(productDir, "frontend");
+    fs.mkdirSync(cwd, { recursive: true });
+    fs.writeFileSync(
+      path.join(cwd, "playwright.config.mjs"),
+      "export default { outputDir: 'shared-test-results' };\n"
+    );
+
+    const configPath = ensurePlaywrightTestConfig(
+      { productDir, stateDir },
+      cwd,
+      ["frontend/__testkit__/homepage/homepage.pw.testkit.ts"]
+    );
+    const generated = await import(pathToFileURL(configPath).href + `?t=${Date.now()}`);
+
+    const expectedOutputDir = resolvePlaywrightOutputDir(stateDir);
+    expect(generated.default.outputDir).toBe(expectedOutputDir);
+    expect(fs.existsSync(expectedOutputDir)).toBe(true);
+    expect(fs.readFileSync(configPath, "utf8")).toContain(
+      `outputDir: ${JSON.stringify(expectedOutputDir)}`
+    );
+  });
+
+  it("finds a supported playwright config file", () => {
+    const cwd = makeTempDir("testkit-playwright-cwd-");
+    fs.writeFileSync(path.join(cwd, "playwright.config.ts"), "export default {};\n");
+
+    expect(findPlaywrightConfig(cwd)).toBe(path.join(cwd, "playwright.config.ts"));
+  });
+});

package/lib/runner/playwright-runner.mjs

@@ -0,0 +1,85 @@
+import path from "path";
+import { execa } from "execa";
+import {
+  parsePlaywrightJsonResults,
+} from "../reporters/playwright.mjs";
+import { resolveServiceCwd, } from "../config/index.mjs";
+import { formatPlaywrightBatchFiles } from "./formatting.mjs";
+import { printBufferedOutput } from "./processes.mjs";
+import { ensurePlaywrightTestConfig } from "./playwright-config.mjs";
+import { buildPlaywrightEnv } from "./template.mjs";
+import { normalizePathSeparators } from "./state.mjs";
+
+export async function runPlaywrightBatch(targetConfig, batch, lifecycle) {
+  const local = targetConfig.testkit.local;
+  if (!local?.baseUrl) {
+    throw new Error(
+      `Service "${targetConfig.name}" requires local.baseUrl for Playwright suites`
+    );
+  }
+
+  console.log(
+    `\n── ${targetConfig.workerLabel} playwright:${targetConfig.name} (${batch.tasks.length} file${batch.tasks.length === 1 ? "" : "s"})${formatPlaywrightBatchFiles(batch)} ──`
+  );
+
+  const cwd = resolveServiceCwd(targetConfig.productDir, local.cwd);
+  const requestedFiles = batch.tasks.map((task) =>
+    path.relative(cwd, path.join(targetConfig.productDir, task.file))
+  );
+  const playwrightConfigPath = ensurePlaywrightTestConfig(targetConfig, cwd, requestedFiles);
+  const startedAt = Date.now();
+  const result = await execa(
+    "npx",
+    ["playwright", "test", "--config", playwrightConfigPath, "--reporter=json"],
+    {
+      cwd,
+      env: buildPlaywrightEnv(targetConfig, local.baseUrl, process.env),
+      reject: false,
+      cancelSignal: lifecycle.signal,
+      forceKillAfterDelay: 5_000,
+    }
+  );
+
+  if (result.stderr) {
+    printBufferedOutput(result.stderr, `[${targetConfig.workerLabel}:${targetConfig.name}:playwright]`);
+  }
+
+  const parsed = parsePlaywrightJsonResults(result.stdout, cwd);
+  const finishedAt = Date.now();
+  const batchDurationMs = finishedAt - startedAt;
+  const genericError =
+    result.exitCode === 0
+      ? parsed.errors[0] || null
+      : parsed.errors[0] ||
+        result.stderr.trim() ||
+        `Playwright exited with code ${result.exitCode}`;
+
+  return batch.tasks.map((task) => {
+    const relativeFile = normalizePathSeparators(
+      path.relative(cwd, path.join(targetConfig.productDir, task.file))
+    );
+    const fileResult = parsed.fileResults.get(relativeFile);
+    if (fileResult) {
+      return {
+        task,
+        failed: fileResult.failed,
+        error: fileResult.error,
+        durationMs:
+          fileResult.durationMs > 0
+            ? fileResult.durationMs
+            : Math.round(batchDurationMs / Math.max(1, batch.tasks.length)),
+        startedAt,
+        finishedAt,
+      };
+    }
+
+    return {
+      task,
+      failed: result.exitCode !== 0,
+      error: result.exitCode !== 0 ? genericError : null,
+      durationMs: Math.round(batchDurationMs / Math.max(1, batch.tasks.length)),
+      startedAt,
+      finishedAt,
+    };
+  });
+}

package/lib/runner/processes.mjs

@@ -0,0 +1,106 @@
+import { spawn } from "child_process";
+
+export function startDetachedCommand(command, cwd, env) {
+  if (process.platform === "win32") {
+    return spawn(command, {
+      cwd,
+      env,
+      detached: true,
+      shell: true,
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+  }
+
+  const shell = process.env.SHELL || "/bin/sh";
+  return spawn(shell, ["-lc", `exec ${command}`], {
+    cwd,
+    env,
+    detached: true,
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+}
+
+export function killChildProcess(child, signal) {
+  if (!child?.pid) return;
+
+  try {
+    process.kill(-child.pid, signal);
+    return;
+  } catch (error) {
+    if (error?.code !== "ESRCH") {
+      // Fall back to the direct child if process-group signalling is unavailable.
+    } else {
+      return;
+    }
+  }
+
+  try {
+    child.kill(signal);
+  } catch (error) {
+    if (error?.code !== "ESRCH") throw error;
+  }
+}
+
+export function pipeOutput(stream, prefix) {
+  if (!stream) return Promise.resolve();
+
+  let pending = "";
+  return new Promise((resolve) => {
+    let settled = false;
+    const settle = () => {
+      if (settled) return;
+      settled = true;
+      resolve();
+    };
+
+    stream.on("data", (chunk) => {
+      pending += chunk.toString();
+      const lines = pending.split(/\r?\n/);
+      pending = lines.pop() || "";
+      for (const line of lines) {
+        if (line.length > 0) console.log(`${prefix} ${line}`);
+      }
+    });
+    stream.on("end", () => {
+      if (pending.length > 0) {
+        console.log(`${prefix} ${pending}`);
+      }
+      settle();
+    });
+    stream.on("close", settle);
+    stream.on("error", settle);
+  });
+}
+
+export function printBufferedOutput(output, prefix) {
+  for (const line of output.split(/\r?\n/)) {
+    if (line.trim().length > 0) {
+      console.log(`${prefix} ${line}`);
+    }
+  }
+}
+
+export async function stopChildProcess(child, outputDrains = []) {
+  if (!child) return;
+  if (child.exitCode !== null) {
+    await Promise.all(outputDrains);
+    return;
+  }
+
+  killChildProcess(child, "SIGTERM");
+  const exited = await Promise.race([
+    new Promise((resolve) => child.once("exit", () => resolve(true))),
+    sleep(5_000).then(() => false),
+  ]);
+
+  if (!exited && child.exitCode === null) {
+    killChildProcess(child, "SIGKILL");
+    await new Promise((resolve) => child.once("exit", resolve));
+  }
+
+  await Promise.all(outputDrains);
+}
+
+export function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}