@elench/testkit 0.1.13 → 0.1.15

package/lib/runner.mjs

@@ -3,12 +3,15 @@ import path from "path";
 import { spawn } from "child_process";
 import net from "net";
 import { execa, execaCommand } from "execa";
-import { runScript } from "./exec.mjs";
+import { resolveDalBinary, resolveServiceCwd } from "./config.mjs";
 import {
-  requireNeonApiKey,
-  resolveDalBinary,
-  resolveServiceCwd,
-} from "./config.mjs";
+  cleanupOrphanedLocalInfrastructure,
+  destroyRuntimeDatabase,
+  destroyServiceDatabaseCache,
+  isDatabaseStateDir,
+  prepareDatabaseRuntime,
+  showServiceDatabaseStatus,
+} from "./database.mjs";
 
 const TYPE_ORDER = ["dal", "integration", "e2e", "load"];
 const HTTP_K6_TYPES = new Set(["integration", "e2e", "load"]);
@@ -18,6 +21,7 @@ const PORT_STRIDE = 100;
 export async function runAll(configs, suiteType, suiteNames, opts, allConfigs = configs) {
   const configMap = new Map(allConfigs.map((config) => [config.name, config]));
   const targetSpan = Math.max(1, opts.jobs || 1);
+  const startedAt = Date.now();
   const results = await Promise.all(
     configs.map(async (config, targetSlot) => {
       console.log(`\n══ ${config.name} ══`);
@@ -28,7 +32,8 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
     })
   );
 
-  if (results.some(Boolean)) process.exit(1);
+  printRunSummary(results, Date.now() - startedAt);
+  if (results.some((result) => result.failed)) process.exit(1);
 }
 
 export async function destroy(config) {
@@ -36,28 +41,28 @@ export async function destroy(config) {
 
   const runtimeStateDirs = findRuntimeStateDirs(config.stateDir);
   for (const stateDir of runtimeStateDirs) {
-    const projectId = readStateValue(path.join(stateDir, "neon_project_id"));
-    if (!projectId) continue;
-
-    await runScript("neon-down.sh", {
-      NEON_PROJECT_ID: projectId,
-      STATE_DIR: stateDir,
+    await destroyRuntimeDatabase({
+      productDir: config.productDir,
+      stateDir,
     });
   }
 
+  await destroyServiceDatabaseCache(config.productDir, config.name);
   fs.rmSync(config.stateDir, { recursive: true, force: true });
+  await cleanupOrphanedLocalInfrastructure(config.productDir);
 }
 
 export function showStatus(config) {
   if (!fs.existsSync(config.stateDir)) {
     console.log("No state — run tests first.");
-    return;
+  } else {
+    printStateDir(config.stateDir, "  ");
   }
-
-  printStateDir(config.stateDir, "  ");
+  showServiceDatabaseStatus(config.productDir, config.name);
 }
 
 async function runService(targetConfig, configMap, suiteType, suiteNames, opts, runtimeSlot) {
+  const startedAt = Date.now();
   const suites = applyShard(
     collectSuites(targetConfig, suiteType, suiteNames, opts.framework),
     opts.shard
@@ -66,7 +71,17 @@ async function runService(targetConfig, configMap, suiteType, suiteNames, opts,
     console.log(
       `No test files for ${targetConfig.name} type="${suiteType}" suites=${suiteNames.join(",") || "all"} — skipping`
     );
-    return false;
+    return {
+      name: targetConfig.name,
+      failed: false,
+      skipped: true,
+      suiteCount: 0,
+      completedSuiteCount: 0,
+      failedSuiteCount: 0,
+      durationMs: Date.now() - startedAt,
+      workers: [],
+      errors: [],
+    };
   }
 
   const runtimeConfigs = resolveRuntimeConfigs(targetConfig, configMap);
@@ -83,17 +98,35 @@ async function runService(targetConfig, configMap, suiteType, suiteNames, opts,
 
   const results = await Promise.allSettled(workerPlans.map((plan) => runWorkerPlan(plan)));
   let failed = false;
+  const workers = [];
+  const errors = [];
+  let failedSuiteCount = 0;
+  let completedSuiteCount = 0;
 
   for (const result of results) {
     if (result.status === "rejected") {
       failed = true;
       console.error(result.reason);
+      errors.push(formatError(result.reason));
       continue;
     }
-    if (result.value) failed = true;
+    workers.push(result.value);
+    completedSuiteCount += result.value.completedSuiteCount;
+    failedSuiteCount += result.value.failedSuiteCount;
+    if (result.value.failed) failed = true;
   }
 
-  return failed;
+  return {
+    name: targetConfig.name,
+    failed,
+    skipped: false,
+    suiteCount: suites.length,
+    completedSuiteCount,
+    failedSuiteCount,
+    durationMs: Date.now() - startedAt,
+    workers,
+    errors,
+  };
 }
 
 function collectSuites(config, suiteType, suiteNames, frameworkFilter) {
@@ -356,9 +389,12 @@ function resolveWorkerConfig(
     ? {
         ...config.testkit.database,
         branchName:
+          config.testkit.database.provider === "neon" &&
           config.testkit.database.branchName !== undefined
             ? finalizeString(config.testkit.database.branchName, context)
-            : `${targetConfig.name}-${config.name}-w${workerId}-testkit`,
+            : config.testkit.database.provider === "neon"
+              ? `${targetConfig.name}-${config.name}-w${workerId}-testkit`
+              : undefined,
       }
     : undefined;
 
@@ -421,17 +457,18 @@ function resolveWorkerConfig(
 }
 
 async function runWorkerPlan(plan) {
+  const startedAt = Date.now();
   console.log(
     `\n══ ${plan.targetConfig.name} worker ${plan.workerId} (${plan.suites.length} suite${plan.suites.length === 1 ? "" : "s"}) ══`
   );
 
   let startedServices = [];
   let failed = false;
+  const suiteResults = [];
+  let fatalError = null;
 
   try {
     await prepareDatabases(plan.runtimeConfigs);
-    await runMigrations(plan.runtimeConfigs);
-    await runSeeds(plan.runtimeConfigs);
 
     if (needsLocalRuntime(plan.suites)) {
       startedServices = await startLocalServices(plan.runtimeConfigs);
@@ -442,71 +479,70 @@ async function runWorkerPlan(plan) {
         `\n── ${plan.targetConfig.workerLabel} ${suite.type}:${suite.name} (${suite.framework}) ──`
       );
       const result = await runSuite(plan.targetConfig, suite);
+      suiteResults.push(result);
       if (result.failed) failed = true;
     }
+  } catch (error) {
+    fatalError = error;
+    failed = true;
+    throw error;
   } finally {
     await stopLocalServices(startedServices);
   }
 
-  return failed;
+  return {
+    workerId: plan.workerId,
+    failed,
+    fatalError: fatalError ? formatError(fatalError) : null,
+    durationMs: Date.now() - startedAt,
+    suiteCount: plan.suites.length,
+    completedSuiteCount: suiteResults.length,
+    failedSuiteCount: suiteResults.filter((result) => result.failed).length,
+    suites: suiteResults,
+  };
 }
 
 async function prepareDatabases(runtimeConfigs) {
   for (const config of runtimeConfigs) {
-    const db = config.testkit.database;
-    if (!db) continue;
-
-    requireNeonApiKey();
-    fs.mkdirSync(config.stateDir, { recursive: true });
-
-    await runScript("neon-up.sh", {
-      NEON_PROJECT_ID: db.projectId,
-      NEON_DB_NAME: db.dbName,
-      NEON_BRANCH_NAME: db.branchName,
-      NEON_RESET: db.reset === false ? "false" : "true",
-      STATE_DIR: config.stateDir,
+    await prepareDatabaseRuntime(config, {
+      runMigrate: config.testkit.migrate
+        ? (databaseUrl) => runMigrate(config, databaseUrl)
+        : null,
+      runSeed: config.testkit.seed ? (databaseUrl) => runSeed(config, databaseUrl) : null,
     });
-
-    fs.writeFileSync(path.join(config.stateDir, "neon_project_id"), db.projectId);
   }
 }
 
-async function runMigrations(runtimeConfigs) {
-  for (const config of runtimeConfigs) {
-    const migrate = config.testkit.migrate;
-    if (!migrate) continue;
+async function runMigrate(config, databaseUrl) {
+  const migrate = config.testkit.migrate;
+  if (!migrate) return;
 
-    const env = buildExecutionEnv(config);
-    const dbUrl = readDatabaseUrl(config.stateDir);
-    if (dbUrl) env.DATABASE_URL = dbUrl;
+  const env = buildExecutionEnv(config);
+  if (databaseUrl) env.DATABASE_URL = databaseUrl;
 
-    console.log(`\n── migrate:${config.workerLabel}:${config.name} ──`);
-    await execaCommand(migrate.cmd, {
-      cwd: resolveServiceCwd(config.productDir, migrate.cwd),
-      env,
-      stdio: "inherit",
-      shell: true,
-    });
-  }
+  console.log(`\n── migrate:${config.workerLabel}:${config.name} ──`);
+  await execaCommand(migrate.cmd, {
+    cwd: resolveServiceCwd(config.productDir, migrate.cwd),
+    env,
+    stdio: "inherit",
+    shell: true,
+  });
 }
 
-async function runSeeds(runtimeConfigs) {
-  for (const config of runtimeConfigs) {
-    const seed = config.testkit.seed;
-    if (!seed) continue;
+async function runSeed(config, databaseUrl) {
+  const seed = config.testkit.seed;
+  if (!seed) return;
 
-    const env = buildExecutionEnv(config);
-    const dbUrl = readDatabaseUrl(config.stateDir);
-    if (dbUrl) env.DATABASE_URL = dbUrl;
+  const env = buildExecutionEnv(config);
+  if (databaseUrl) env.DATABASE_URL = databaseUrl;
 
-    console.log(`\n── seed:${config.workerLabel}:${config.name} ──`);
-    await execaCommand(seed.cmd, {
-      cwd: resolveServiceCwd(config.productDir, seed.cwd),
-      env,
-      stdio: "inherit",
-      shell: true,
-    });
-  }
+  console.log(`\n── seed:${config.workerLabel}:${config.name} ──`);
+  await execaCommand(seed.cmd, {
+    cwd: resolveServiceCwd(config.productDir, seed.cwd),
+    env,
+    stdio: "inherit",
+    shell: true,
+  });
 }
 
 async function startLocalServices(runtimeConfigs) {
@@ -545,12 +581,15 @@ async function startLocalService(config) {
   const child = spawn(config.testkit.local.start, {
     cwd,
     env,
+    detached: true,
     shell: true,
     stdio: ["ignore", "pipe", "pipe"],
   });
 
-  pipeOutput(child.stdout, `[${config.workerLabel}:${config.name}]`);
-  pipeOutput(child.stderr, `[${config.workerLabel}:${config.name}]`);
+  const outputDrains = [
+    pipeOutput(child.stdout, `[${config.workerLabel}:${config.name}]`),
+    pipeOutput(child.stderr, `[${config.workerLabel}:${config.name}]`),
+  ];
 
   const readyTimeoutMs = config.testkit.local.readyTimeoutMs || DEFAULT_READY_TIMEOUT_MS;
 
@@ -562,11 +601,11 @@ async function startLocalService(config) {
       process: child,
     });
   } catch (error) {
-    await stopChildProcess(child);
+    await stopChildProcess(child, outputDrains);
     throw error;
   }
 
-  return { name: config.name, child };
+  return { name: config.name, child, outputDrains };
 }
 
 async function runSuite(targetConfig, suite) {
@@ -593,7 +632,9 @@ async function runHttpK6Suite(targetConfig, suite) {
     throw new Error(`Service "${targetConfig.name}" requires local.baseUrl for HTTP k6 suites`);
   }
 
+  const startedAt = Date.now();
   let failed = false;
+  const failedFiles = [];
   await runWithConcurrency(suite.files, suite.maxFileConcurrency, async (file) => {
     const absFile = path.join(targetConfig.productDir, file);
     console.log(`·· ${targetConfig.workerLabel}:${suite.name} → ${file}`);
@@ -605,10 +646,11 @@ async function runHttpK6Suite(targetConfig, suite) {
       });
     } catch {
       failed = true;
+      failedFiles.push(file);
     }
   });
 
-  return { failed };
+  return buildSuiteResult(suite, failed, startedAt, failedFiles);
 }
 
 async function runDalSuite(targetConfig, suite) {
@@ -618,7 +660,9 @@ async function runDalSuite(targetConfig, suite) {
   }
 
   const k6Binary = resolveDalBinary();
+  const startedAt = Date.now();
   let failed = false;
+  const failedFiles = [];
   await runWithConcurrency(suite.files, suite.maxFileConcurrency, async (file) => {
     const absFile = path.join(targetConfig.productDir, file);
     console.log(`·· ${targetConfig.workerLabel}:${suite.name} → ${file}`);
@@ -630,10 +674,11 @@ async function runDalSuite(targetConfig, suite) {
       });
     } catch {
       failed = true;
+      failedFiles.push(file);
     }
   });
 
-  return { failed };
+  return buildSuiteResult(suite, failed, startedAt, failedFiles);
 }
 
 async function runPlaywrightSuite(targetConfig, suite) {
@@ -648,6 +693,7 @@ async function runPlaywrightSuite(targetConfig, suite) {
   const files = suite.files.map((file) =>
     path.relative(cwd, path.join(targetConfig.productDir, file))
   );
+  const startedAt = Date.now();
 
   try {
     await execa("npx", ["playwright", "test", ...files], {
@@ -655,31 +701,37 @@ async function runPlaywrightSuite(targetConfig, suite) {
       env: buildPlaywrightEnv(targetConfig, local.baseUrl),
       stdio: "inherit",
     });
-    return { failed: false };
+    return buildSuiteResult(suite, false, startedAt);
   } catch {
-    return { failed: true };
+    return buildSuiteResult(suite, true, startedAt);
   }
 }
 
 async function stopLocalServices(started) {
   for (const service of [...started].reverse()) {
-    await stopChildProcess(service.child);
+    await stopChildProcess(service.child, service.outputDrains);
   }
 }
 
-async function stopChildProcess(child) {
-  if (!child || child.exitCode !== null) return;
+async function stopChildProcess(child, outputDrains = []) {
+  if (!child) return;
+  if (child.exitCode !== null) {
+    await Promise.all(outputDrains);
+    return;
+  }
 
-  child.kill("SIGTERM");
+  killChildProcess(child, "SIGTERM");
   const exited = await Promise.race([
     new Promise((resolve) => child.once("exit", () => resolve(true))),
     sleep(5_000).then(() => false),
   ]);
 
   if (!exited && child.exitCode === null) {
-    child.kill("SIGKILL");
+    killChildProcess(child, "SIGKILL");
     await new Promise((resolve) => child.once("exit", resolve));
   }
+
+  await Promise.all(outputDrains);
 }
 
 async function waitForReady({ name, url, timeoutMs, process }) {
@@ -757,6 +809,124 @@ function resolvePlaywrightBrowsersPath(configuredPath) {
   return configuredPath;
 }
 
+function buildSuiteResult(suite, failed, startedAt, failedFiles = []) {
+  return {
+    name: suite.name,
+    type: suite.type,
+    framework: suite.framework,
+    failed,
+    fileCount: suite.files.length,
+    failedFiles,
+    durationMs: Date.now() - startedAt,
+  };
+}
+
+function printRunSummary(results, durationMs) {
+  const totalServices = results.length;
+  const executedServices = results.filter((result) => !result.skipped);
+  const skippedServices = results.filter((result) => result.skipped);
+  const failedServices = executedServices.filter((result) => result.failed);
+  const passedServices = executedServices.filter((result) => !result.failed);
+  const totalSuites = executedServices.reduce((sum, result) => sum + result.suiteCount, 0);
+  const completedSuites = executedServices.reduce(
+    (sum, result) => sum + result.completedSuiteCount,
+    0
+  );
+  const failedSuites = executedServices.reduce((sum, result) => sum + result.failedSuiteCount, 0);
+  const passedSuites = completedSuites - failedSuites;
+
+  console.log("\n══ Summary ══");
+  console.log(
+    [
+      `services ${passedServices.length}/${executedServices.length} passed`,
+      `suites ${passedSuites}/${totalSuites} passed`,
+      skippedServices.length > 0 ? `${skippedServices.length} skipped` : null,
+      `duration ${formatDuration(durationMs)}`,
+    ]
+      .filter(Boolean)
+      .join("  ·  ")
+  );
+
+  for (const result of results) {
+    const status = result.skipped ? "SKIP" : result.failed ? "FAIL" : "PASS";
+    const detail = result.skipped ? "no matching suites" : formatServiceSummary(result);
+    console.log(
+      `${status.padEnd(4)} ${result.name.padEnd(longestServiceName(results))}  ${detail}  ·  ${formatDuration(result.durationMs)}`
+    );
+
+    if (result.failed) {
+      const failedSuiteResults = result.workers.flatMap((worker) =>
+        worker.suites.filter((suite) => suite.failed)
+      );
+      for (const suite of failedSuiteResults) {
+        const fileDetail =
+          suite.failedFiles.length > 0 ? `  (${suite.failedFiles.join(", ")})` : "";
+        console.log(
+          `  - ${suite.type}:${suite.name} [${suite.framework}]${fileDetail} · ${formatDuration(suite.durationMs)}`
+        );
+      }
+      for (const error of result.errors) {
+        console.log(`  - worker error: ${error}`);
+      }
+    }
+  }
+
+  if (failedServices.length > 0) {
+    console.log(`\nResult: FAILED (${failedServices.length}/${totalServices} services failed)`);
+    return;
+  }
+
+  console.log("\nResult: PASSED");
+}
+
+function longestServiceName(results) {
+  return results.reduce((max, result) => Math.max(max, result.name.length), 4);
+}
+
+function formatDuration(durationMs) {
+  const totalSeconds = Math.max(0, Math.round(durationMs / 1000));
+  const minutes = Math.floor(totalSeconds / 60);
+  const seconds = totalSeconds % 60;
+  if (minutes === 0) return `${seconds}s`;
+  return `${minutes}m ${String(seconds).padStart(2, "0")}s`;
+}
+
+function formatError(error) {
+  if (error instanceof Error) return error.message;
+  return String(error);
+}
+
+function formatServiceSummary(result) {
+  const passedSuites = result.completedSuiteCount - result.failedSuiteCount;
+  const notRun = result.suiteCount - result.completedSuiteCount;
+  let detail = `${passedSuites}/${result.suiteCount} suites passed`;
+  if (notRun > 0) {
+    detail += `, ${notRun} not run`;
+  }
+  return detail;
+}
+
+function killChildProcess(child, signal) {
+  if (!child?.pid) return;
+
+  try {
+    process.kill(-child.pid, signal);
+    return;
+  } catch (error) {
+    if (error?.code !== "ESRCH") {
+      // Fall back to the direct child if process-group signalling is unavailable.
+    } else {
+      return;
+    }
+  }
+
+  try {
+    child.kill(signal);
+  } catch (error) {
+    if (error?.code !== "ESRCH") throw error;
+  }
+}
+
 function readDatabaseUrl(stateDir) {
   return readStateValue(path.join(stateDir, "database_url"));
 }
@@ -780,21 +950,33 @@ function printStateDir(dir, indent) {
 }
 
 function pipeOutput(stream, prefix) {
-  if (!stream) return;
+  if (!stream) return Promise.resolve();
 
   let pending = "";
-  stream.on("data", (chunk) => {
-    pending += chunk.toString();
-    const lines = pending.split(/\r?\n/);
-    pending = lines.pop() || "";
-    for (const line of lines) {
-      if (line.length > 0) console.log(`${prefix} ${line}`);
-    }
-  });
-  stream.on("end", () => {
-    if (pending.length > 0) {
-      console.log(`${prefix} ${pending}`);
-    }
+  return new Promise((resolve) => {
+    let settled = false;
+    const settle = () => {
+      if (settled) return;
+      settled = true;
+      resolve();
+    };
+
+    stream.on("data", (chunk) => {
+      pending += chunk.toString();
+      const lines = pending.split(/\r?\n/);
+      pending = lines.pop() || "";
+      for (const line of lines) {
+        if (line.length > 0) console.log(`${prefix} ${line}`);
+      }
+    });
+    stream.on("end", () => {
+      if (pending.length > 0) {
+        console.log(`${prefix} ${pending}`);
+      }
+      settle();
+    });
+    stream.on("close", settle);
+    stream.on("error", settle);
   });
 }
 
@@ -986,12 +1168,7 @@ function findRuntimeStateDirs(rootDir) {
   const visit = (dir) => {
     if (!fs.existsSync(dir)) return;
     const entries = fs.readdirSync(dir, { withFileTypes: true });
-    const hasRuntimeFiles = entries.some(
-      (entry) =>
-        entry.isFile() &&
-        (entry.name === "neon_project_id" || entry.name === "neon_branch_id")
-    );
-    if (hasRuntimeFiles) {
+    if (isDatabaseStateDir(dir)) {
       found.push(dir);
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/testkit",
-  "version": "0.1.13",
+  "version": "0.1.15",
   "description": "CLI for running manifest-defined local test suites across k6 and Playwright",
   "type": "module",
   "bin": {