@elench/testkit 0.1.31 → 0.1.33

package/lib/runner/lifecycle.mjs

@@ -0,0 +1,349 @@
+import fs from "fs";
+import path from "path";
+import crypto from "crypto";
+import { destroyRuntimeDatabase, cleanupOrphanedLocalInfrastructure } from "../database/index.mjs";
+
+const RUN_SCHEMA_VERSION = 1;
+const RUNS_DIRNAME = path.join(".testkit", "_runs");
+const TERMINATION_TIMEOUT_MS = 5_000;
+
+export function createRunLifecycle(productDir) {
+  const runId = buildRunId();
+  const manifestPath = path.join(getRunsDir(productDir), `${runId}.json`);
+  const abortController = new AbortController();
+  const state = {
+    schemaVersion: RUN_SCHEMA_VERSION,
+    runId,
+    productDir,
+    pid: process.pid,
+    status: "starting",
+    startedAt: new Date().toISOString(),
+    interruptReason: null,
+    services: [],
+    graphDirs: [],
+    workerStateDirs: [],
+    runtimeStateDirs: [],
+  };
+  const signalListeners = [];
+
+  function persist() {
+    fs.mkdirSync(getRunsDir(productDir), { recursive: true });
+    fs.writeFileSync(manifestPath, `${JSON.stringify(state, null, 2)}\n`);
+  }
+
+  function mutate(mutator) {
+    mutator(state);
+    persist();
+  }
+
+  const api = {
+    runId,
+    manifestPath,
+    signal: abortController.signal,
+    isStopRequested() {
+      return abortController.signal.aborted;
+    },
+    markRunning() {
+      mutate((draft) => {
+        draft.status = "running";
+      });
+    },
+    markFinished(status = "finished") {
+      mutate((draft) => {
+        draft.status = status;
+        draft.finishedAt = new Date().toISOString();
+      });
+    },
+    requestStop(reason = "interrupted") {
+      if (!abortController.signal.aborted) {
+        abortController.abort(new Error(`testkit run interrupted (${reason})`));
+      }
+      mutate((draft) => {
+        draft.status = "interrupting";
+        draft.interruptReason = reason;
+      });
+    },
+    trackGraphContext(context) {
+      mutate((draft) => {
+        pushUnique(draft.graphDirs, context.graphDir);
+        pushUnique(draft.workerStateDirs, context.workerStateDir);
+        for (const runtimeConfig of context.runtimeConfigs || []) {
+          if (runtimeConfig.stateDir) pushUnique(draft.runtimeStateDirs, runtimeConfig.stateDir);
+        }
+      });
+    },
+    registerService(config, child, cwd) {
+      const ports = collectConfigPorts(config);
+      mutate((draft) => {
+        draft.services = draft.services.filter((service) => service.pid !== child.pid);
+        draft.services.push({
+          serviceName: config.name,
+          workerLabel: config.workerLabel,
+          command: config.testkit.local?.start || null,
+          cwd,
+          pid: child.pid,
+          processGroupId: child.pid,
+          ports,
+          startedAt: new Date().toISOString(),
+        });
+      });
+    },
+    unregisterService(childPid) {
+      mutate((draft) => {
+        draft.services = draft.services.filter((service) => service.pid !== childPid);
+      });
+    },
+    installSignalHandlers() {
+      const install = (eventName, handler) => {
+        process.on(eventName, handler);
+        signalListeners.push([eventName, handler]);
+      };
+
+      const handleSignal = (signalName) => {
+        api.requestStop(signalName);
+      };
+
+      install("SIGINT", () => handleSignal("SIGINT"));
+      install("SIGTERM", () => handleSignal("SIGTERM"));
+      install("SIGHUP", () => handleSignal("SIGHUP"));
+    },
+    removeSignalHandlers() {
+      while (signalListeners.length > 0) {
+        const [eventName, handler] = signalListeners.pop();
+        process.off(eventName, handler);
+      }
+    },
+    removeManifest() {
+      removeManifestFile(productDir, runId);
+    },
+  };
+
+  return api;
+}
+
+export async function cleanupRuns(productDir, { includeActive = false } = {}) {
+  const manifests = listRunManifests(productDir);
+  const summary = {
+    cleaned: [],
+    skippedActive: [],
+  };
+
+  for (const manifest of manifests) {
+    const isActive = isPidRunning(manifest.pid);
+    if (isActive && !includeActive) {
+      summary.skippedActive.push(manifest);
+      continue;
+    }
+    await cleanupRunManifest(productDir, manifest, { removeRuntimeState: false });
+    summary.cleaned.push(manifest);
+  }
+
+  await cleanupOrphanedLocalInfrastructure(productDir);
+  return summary;
+}
+
+export async function cleanupStaleRuns(productDir) {
+  const manifests = listRunManifests(productDir);
+  const cleaned = [];
+
+  for (const manifest of manifests) {
+    if (isPidRunning(manifest.pid)) continue;
+    await cleanupRunManifest(productDir, manifest, { removeRuntimeState: false });
+    cleaned.push(manifest);
+  }
+
+  await cleanupOrphanedLocalInfrastructure(productDir);
+  return cleaned;
+}
+
+export async function cleanupRunById(productDir, runId) {
+  const manifest = listRunManifests(productDir).find((entry) => entry.runId === runId);
+  if (!manifest) return false;
+  await cleanupRunManifest(productDir, manifest, { removeRuntimeState: false });
+  await cleanupOrphanedLocalInfrastructure(productDir);
+  return true;
+}
+
+export function listRunManifests(productDir) {
+  const runsDir = getRunsDir(productDir);
+  if (!fs.existsSync(runsDir)) return [];
+
+  return fs
+    .readdirSync(runsDir)
+    .filter((entry) => entry.endsWith(".json"))
+    .map((entry) => readRunManifest(path.join(runsDir, entry)))
+    .filter(Boolean)
+    .sort((left, right) => String(left.startedAt || "").localeCompare(String(right.startedAt || "")));
+}
+
+export function findPortOwner(productDir, { host, port }) {
+  const manifests = listRunManifests(productDir);
+  for (const manifest of manifests) {
+    for (const service of manifest.services || []) {
+      for (const socket of service.ports || []) {
+        if (normalizeHost(socket.host) === normalizeHost(host) && Number(socket.port) === Number(port)) {
+          return {
+            manifest,
+            service,
+            active: isPidRunning(manifest.pid),
+          };
+        }
+      }
+    }
+  }
+  return null;
+}
+
+export function formatRunSummary(manifest) {
+  const workerLabels = [...new Set((manifest.services || []).map((service) => service.workerLabel).filter(Boolean))];
+  return `${manifest.runId} pid=${manifest.pid}${workerLabels.length > 0 ? ` workers=${workerLabels.join(",")}` : ""}`;
+}
+
+export function isPidRunning(pid) {
+  if (!Number.isInteger(Number(pid)) || Number(pid) <= 0) return false;
+  try {
+    process.kill(Number(pid), 0);
+    return true;
+  } catch (error) {
+    return error?.code === "EPERM";
+  }
+}
+
+async function cleanupRunManifest(productDir, manifest, { removeRuntimeState = false } = {}) {
+  for (const service of [...(manifest.services || [])].reverse()) {
+    await terminateOwnedProcess(service);
+  }
+
+  if (removeRuntimeState) {
+    const runtimeStateDirs = [...new Set(manifest.runtimeStateDirs || [])].sort((a, b) => b.length - a.length);
+    for (const stateDir of runtimeStateDirs) {
+      await destroyRuntimeDatabase({ productDir, stateDir });
+    }
+
+    for (const workerStateDir of [...new Set(manifest.workerStateDirs || [])].sort((a, b) => b.length - a.length)) {
+      fs.rmSync(workerStateDir, { recursive: true, force: true });
+    }
+
+    for (const graphDir of [...new Set(manifest.graphDirs || [])].sort((a, b) => b.length - a.length)) {
+      pruneEmptyParents(graphDir, path.join(productDir, ".testkit", "_graphs"));
+    }
+  }
+
+  removeManifestFile(productDir, manifest.runId);
+}
+
+async function terminateOwnedProcess(service) {
+  const pid = Number(service.processGroupId || service.pid);
+  if (!Number.isInteger(pid) || pid <= 0) return;
+  if (!isPidRunning(pid)) return;
+
+  killProcessGroup(pid, "SIGTERM");
+  const exited = await waitForPidExit(pid, TERMINATION_TIMEOUT_MS);
+  if (!exited) {
+    killProcessGroup(pid, "SIGKILL");
+    await waitForPidExit(pid, TERMINATION_TIMEOUT_MS);
+  }
+}
+
+function killProcessGroup(pid, signal) {
+  try {
+    process.kill(-pid, signal);
+    return;
+  } catch (error) {
+    if (error?.code !== "ESRCH") {
+      // Fall through and try the direct pid.
+    } else {
+      return;
+    }
+  }
+
+  try {
+    process.kill(pid, signal);
+  } catch (error) {
+    if (error?.code !== "ESRCH") throw error;
+  }
+}
+
+async function waitForPidExit(pid, timeoutMs) {
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < timeoutMs) {
+    if (!isPidRunning(pid)) return true;
+    await sleep(100);
+  }
+  return !isPidRunning(pid);
+}
+
+function readRunManifest(filePath) {
+  try {
+    const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
+    if (!parsed?.runId) return null;
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+function collectConfigPorts(config) {
+  const seen = new Set();
+  const ports = [];
+  for (const rawUrl of [config.testkit.local?.baseUrl, config.testkit.local?.readyUrl]) {
+    if (!rawUrl) continue;
+    try {
+      const parsed = new URL(rawUrl);
+      const host = normalizeHost(parsed.hostname);
+      const port = Number(parsed.port || (parsed.protocol === "https:" ? 443 : 80));
+      const key = `${host}:${port}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      ports.push({ host, port });
+    } catch {
+      // Ignore malformed URLs here; startup validation handles them elsewhere.
+    }
+  }
+  return ports;
+}
+
+function pushUnique(list, value) {
+  if (!value) return;
+  if (!list.includes(value)) list.push(value);
+}
+
+function getRunsDir(productDir) {
+  return path.join(productDir, RUNS_DIRNAME);
+}
+
+function buildRunId() {
+  return `${Date.now()}-${process.pid}-${crypto.randomBytes(4).toString("hex")}`;
+}
+
+function normalizeHost(host) {
+  if (!host) return "127.0.0.1";
+  return host === "::1" ? "127.0.0.1" : host;
+}
+
+function pruneEmptyParents(startDir, stopDir) {
+  let current = startDir;
+  while (current && current.startsWith(stopDir)) {
+    if (!fs.existsSync(current)) {
+      current = path.dirname(current);
+      continue;
+    }
+    const entries = fs.readdirSync(current);
+    if (entries.length > 0) break;
+    fs.rmSync(current, { recursive: true, force: true });
+    if (current === stopDir) break;
+    current = path.dirname(current);
+  }
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function removeManifestFile(productDir, runId) {
+  const runsDir = getRunsDir(productDir);
+  fs.rmSync(path.join(runsDir, `${runId}.json`), { force: true });
+  if (fs.existsSync(runsDir) && fs.readdirSync(runsDir).length === 0) {
+    fs.rmSync(runsDir, { recursive: true, force: true });
+  }
+}

package/lib/runner/maintenance.mjs

@@ -0,0 +1,72 @@
+import fs from "fs";
+import {
+  cleanupOrphanedLocalInfrastructure,
+  destroyRuntimeDatabase,
+  destroyServiceDatabaseCache,
+  isDatabaseStateDir,
+  showServiceDatabaseStatus,
+} from "../database/index.mjs";
+import { cleanupRuns, formatRunSummary } from "./lifecycle.mjs";
+import { printRunStatus } from "./readiness.mjs";
+import { findGraphDirsForService, findRuntimeStateDirs } from "./state.mjs";
+import { printStateDir } from "./state-io.mjs";
+
+export async function destroy(config) {
+  await cleanupRuns(config.productDir, { includeActive: true });
+  const roots = new Set([
+    config.stateDir,
+    ...findGraphDirsForService(config.productDir, config.name),
+  ]);
+
+  for (const rootDir of roots) {
+    if (!fs.existsSync(rootDir)) continue;
+    const runtimeStateDirs = findRuntimeStateDirs(rootDir, isDatabaseStateDir);
+    for (const stateDir of runtimeStateDirs) {
+      await destroyRuntimeDatabase({
+        productDir: config.productDir,
+        stateDir,
+      });
+    }
+    fs.rmSync(rootDir, { recursive: true, force: true });
+  }
+
+  await destroyServiceDatabaseCache(config.productDir, config.name);
+  await cleanupOrphanedLocalInfrastructure(config.productDir);
+}
+
+export function showStatus(config) {
+  printRunStatus(config.productDir);
+  const graphDirs = findGraphDirsForService(config.productDir, config.name);
+  const hasDirectState = fs.existsSync(config.stateDir);
+  const hasGraphState = graphDirs.length > 0;
+
+  if (!hasDirectState && !hasGraphState) {
+    console.log("No state — run tests first.");
+  } else {
+    if (hasDirectState) {
+      console.log("  service-state/");
+      printStateDir(config.stateDir, "    ");
+    }
+    for (const graphDir of graphDirs) {
+      console.log(`  graph-state/${graphDir.split("/").at(-1)}/`);
+      printStateDir(graphDir, "    ");
+    }
+  }
+
+  showServiceDatabaseStatus(config.productDir, config.name);
+}
+
+export async function cleanup(productDir) {
+  const summary = await cleanupRuns(productDir, { includeActive: false });
+  if (summary.cleaned.length === 0 && summary.skippedActive.length === 0) {
+    console.log("No stale runs to clean.");
+    return;
+  }
+
+  for (const manifest of summary.cleaned) {
+    console.log(`Cleaned stale run ${formatRunSummary(manifest)}`);
+  }
+  for (const manifest of summary.skippedActive) {
+    console.log(`Active run still present: ${formatRunSummary(manifest)}`);
+  }
+}

package/lib/runner/orchestrator.mjs

@@ -0,0 +1,254 @@
+import {
+  applyShard,
+  buildRuntimeGraphs,
+  buildTaskQueue,
+  claimNextBatch,
+  collectSuites,
+  resolveRuntimeConfigs,
+} from "./planning.mjs";
+import {
+  addTrackerError,
+  buildServiceTrackers,
+  finalizeServiceResult,
+  recordGraphError,
+  recordTaskOutcome,
+  summarizeDbBackend,
+} from "./results.mjs";
+import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
+import { buildRunSummaryLines, formatError } from "./formatting.mjs";
+import { loadTimings, saveTimings, writeRunArtifact, writeStatusArtifact } from "./artifacts.mjs";
+import {
+  cleanupRunById,
+  cleanupRuns,
+  cleanupStaleRuns,
+  createRunLifecycle,
+} from "./lifecycle.mjs";
+import {
+  collectGitMetadata,
+  readPackageMetadata,
+  safeHostname,
+  safeUsername,
+} from "./metadata.mjs";
+import { createWorker, runWorker } from "./worker-loop.mjs";
+import { findUnmatchedRequestedFiles, isFullRunSelection } from "./selection.mjs";
+import { uploadTelemetryArtifact } from "../telemetry/index.mjs";
+
+export async function runAll(configs, suiteType, suiteNames, opts, allConfigs = configs) {
+  const configMap = new Map(allConfigs.map((config) => [config.name, config]));
+  const startedAt = Date.now();
+  const telemetry = configs[0]?.telemetry || null;
+  const productDir = configs[0]?.productDir || process.cwd();
+  await cleanupStaleRuns(productDir);
+  const metadata = {
+    git: collectGitMetadata(productDir),
+    host: {
+      hostname: safeHostname(),
+      username: safeUsername(),
+    },
+    testkitVersion: readPackageMetadata().version,
+  };
+  const requestedFiles = opts.fileNames || [];
+  if (requestedFiles.length > 0) {
+    const unmatchedFiles = findUnmatchedRequestedFiles(
+      configs,
+      suiteType,
+      suiteNames,
+      opts.framework || "all",
+      requestedFiles,
+      collectSuites,
+      normalizePathSeparators
+    );
+    if (unmatchedFiles.length > 0) {
+      throw new Error(
+        `Requested file${unmatchedFiles.length === 1 ? "" : "s"} did not match any selected suites:\n` +
+          unmatchedFiles.map((file) => `- ${file}`).join("\n")
+      );
+    }
+  }
+  if (
+    opts.writeStatus &&
+    !opts.allowPartialStatus &&
+    !isFullRunSelection(
+      suiteNames,
+      requestedFiles,
+      opts.framework || "all",
+      opts.shard || null,
+      opts.serviceFilter || null
+    )
+  ) {
+    throw new Error(
+      "Refusing to overwrite testkit.status.json from a filtered run. " +
+        "Run the full suite with --write-status, or pass --allow-partial-status to opt in."
+    );
+  }
+
+  const servicePlans = collectServicePlans(configs, configMap, suiteType, suiteNames, opts);
+  const trackers = buildServiceTrackers(servicePlans, startedAt);
+  const executedPlans = servicePlans.filter((plan) => !plan.skipped);
+  let workerCount = 0;
+  let exitCode = 0;
+  const lifecycle = createRunLifecycle(productDir);
+  lifecycle.markRunning();
+  lifecycle.installSignalHandlers();
+  let results = [];
+  let finishedAt = Date.now();
+
+  try {
+    if (executedPlans.length > 0) {
+      const timings = loadTimings(productDir);
+      const graphs = buildRuntimeGraphs(executedPlans);
+      const queue = buildTaskQueue(executedPlans, graphs, timings);
+      workerCount = Math.max(1, Math.min(opts.jobs || 1, queue.length));
+      const graphByKey = new Map(graphs.map((graph) => [graph.key, graph]));
+      const workers = Array.from({ length: workerCount }, (_unused, index) =>
+        createWorker(index + 1, productDir)
+      );
+      const timingUpdates = [];
+
+      const workerResults = await Promise.allSettled(
+        workers.map((worker) =>
+          runWorker(
+            worker,
+            queue,
+            graphByKey,
+            trackers,
+            timingUpdates,
+            lifecycle,
+            claimNextBatch,
+            recordTaskOutcome,
+            recordGraphError
+          )
+        )
+      );
+
+      for (const result of workerResults) {
+        if (result.status === "rejected") {
+          const message = formatError(result.reason);
+          for (const tracker of trackers.values()) {
+            if (!tracker.skipped) addTrackerError(tracker, message);
+          }
+        }
+      }
+
+      saveTimings(productDir, timings, timingUpdates);
+    }
+
+    finishedAt = Date.now();
+    results = configs.map((config) =>
+      finalizeServiceResult(trackers.get(config.name), startedAt, finishedAt)
+    );
+    const artifact = buildRunArtifact({
+      productDir,
+      results,
+      startedAt,
+      finishedAt,
+      requestedJobs: opts.jobs || 1,
+      workerCount,
+      suiteType,
+      suiteNames,
+      fileNames: requestedFiles,
+      framework: opts.framework || "all",
+      shard: opts.shard || null,
+      serviceFilter: opts.serviceFilter || null,
+      metadata,
+      summarizeDbBackend,
+    });
+
+    writeRunArtifact(productDir, artifact);
+    if (opts.writeStatus) {
+      writeStatusArtifact(
+        productDir,
+        buildStatusArtifact({
+          productDir,
+          results,
+          suiteType,
+          suiteNames,
+          fileNames: requestedFiles,
+          framework: opts.framework || "all",
+          shard: opts.shard || null,
+          serviceFilter: opts.serviceFilter || null,
+          metadata,
+        })
+      );
+    }
+
+    printRunSummary(results, finishedAt - startedAt);
+    await reportTelemetry(telemetry, artifact);
+    if (results.some((result) => result.failed)) exitCode = 1;
+    if (lifecycle.isStopRequested()) exitCode = Math.max(exitCode, 130);
+  } finally {
+    lifecycle.removeSignalHandlers();
+    lifecycle.markFinished(
+      exitCode === 0 ? "finished" : lifecycle.isStopRequested() ? "interrupted" : "failed"
+    );
+    await cleanupRunById(productDir, lifecycle.runId);
+    await cleanupRuns(productDir, { includeActive: false });
+    lifecycle.removeManifest();
+    process.exitCode = exitCode;
+  }
+}
+
+function collectServicePlans(configs, configMap, suiteType, suiteNames, opts) {
+  return configs.map((config) => {
+    console.log(`\n══ ${config.name} ══`);
+    const suites = applyShard(
+      collectSuites(config, suiteType, suiteNames, opts.framework, opts.fileNames || []),
+      opts.shard
+    );
+
+    if (suites.length === 0) {
+      console.log(
+        `No test files for ${config.name} type="${suiteType}" suites=${suiteNames.join(",") || "all"} files=${(opts.fileNames || []).join(",") || "all"} — skipping`
+      );
+      return {
+        config,
+        skipped: true,
+        suites: [],
+        runtimeConfigs: [],
+        runtimeNames: [],
+        runtimeKey: null,
+      };
+    }
+
+    const runtimeConfigs = resolveRuntimeConfigs(config, configMap);
+    return {
+      config,
+      skipped: false,
+      suites,
+      runtimeConfigs,
+      runtimeNames: runtimeConfigs.map((runtimeConfig) => runtimeConfig.name).sort(),
+      runtimeKey: runtimeConfigs.map((runtimeConfig) => runtimeConfig.name).sort().join("|"),
+    };
+  });
+}
+
+function printRunSummary(results, durationMs) {
+  for (const line of buildRunSummaryLines(results, durationMs)) {
+    console.log(line);
+  }
+}
+
+async function reportTelemetry(telemetry, artifact) {
+  if (!telemetry?.enabled) return;
+
+  try {
+    const outcome = await uploadTelemetryArtifact(telemetry, artifact);
+    if (outcome?.ok) {
+      console.log("Telemetry: uploaded run artifact");
+      return;
+    }
+    if (outcome?.reason === "missing-token") {
+      console.log(
+        `Telemetry: skipped upload because ${telemetry.tokenEnv || "configured token env"} is not set`
+      );
+      return;
+    }
+    if (outcome?.reason && !outcome.skipped) return;
+  } catch (error) {
+    console.log(`Telemetry: upload failed (${formatError(error)})`);
+  }
+}
+
+function normalizePathSeparators(filePath) {
+  return filePath.split("\\").join("/");
+}