@elench/testkit 0.1.112 → 0.1.114

package/lib/runner/default-runtime-runner.mjs

@@ -1,6 +1,5 @@
 import fs from "fs";
 import path from "path";
-import { execa } from "execa";
 import { bundleK6File } from "../bundler/index.mjs";
 import { resolveK6Binary } from "../config/binaries.mjs";
 import {
@@ -12,12 +11,12 @@ import {
   determineDefaultRuntimeFailure,
   extractDefaultRuntimeFatalDetail,
 } from "./default-runtime-errors.mjs";
-import { collectFailureDetailsFromRuntimeArtifacts } from "./failure-details.mjs";
+import { collectFailureDetailsFromRuntimeArtifacts, collectCheckDetailsFromRuntimeArtifacts } from "./failure-details.mjs";
 import { RUNTIME_ARTIFACT_MARKER } from "../runtime-src/k6/artifacts.js";
 import { readDatabaseUrl } from "./state-io.mjs";
 import { buildTaskExecutionEnv } from "./template.mjs";
 import { registerManagedProcess, unregisterManagedProcess } from "./managed-processes.mjs";
-import { killChildProcess } from "./processes.mjs";
+import { settleManagedSubprocess, startManagedSubprocess } from "./subprocess.mjs";
 
 export async function runHttpK6Task(targetConfig, task, lifecycle, lease, reporter = null) {
   const baseUrl = targetConfig.testkit.local?.baseUrl;
@@ -88,7 +87,7 @@ export async function runDefaultRuntimeTask(
   fs.mkdirSync(path.dirname(summaryFile), { recursive: true });
   const startedAt = Date.now();
   const fileTimeoutSeconds = targetConfig.testkit.execution.fileTimeoutSeconds;
-  const subprocess = execa(
+  const subprocess = startManagedSubprocess(
     k6Binary,
     [...args.slice(0, 1), "--summary-export", summaryFile, ...args.slice(1)],
     {
@@ -102,8 +101,6 @@ export async function runDefaultRuntimeTask(
         },
         process.env
       ),
-      reject: false,
-      forceKillAfterDelay: 5_000,
     }
   );
   registerManagedProcess(lifecycle, subprocess, "SIGINT");
@@ -111,7 +108,9 @@ export async function runDefaultRuntimeTask(
   let result;
   let timedOut;
   try {
-    ({ result, timedOut } = await settleSubprocess(subprocess, fileTimeoutSeconds));
+    ({ result, timedOut } = await settleManagedSubprocess(subprocess, {
+      timeoutMs: fileTimeoutSeconds * 1000 + 1_000,
+    }));
   } finally {
     unregisterManagedProcess(lifecycle, subprocess);
   }
@@ -147,6 +146,7 @@ export async function runDefaultRuntimeTask(
       : null,
   ]);
   const failureDetails = collectFailureDetailsFromRuntimeArtifacts(rawRuntimeArtifacts);
+  const checkDetails = collectCheckDetailsFromRuntimeArtifacts(rawRuntimeArtifacts);
   const fatalRuntimeDetail = extractDefaultRuntimeFatalDetail(result.stderr || "", getFirstLine);
   if (fatalRuntimeDetail && !failureDetails.some((detail) => detail?.kind === "runtime-exception")) {
     failureDetails.unshift(fatalRuntimeDetail);
@@ -165,31 +165,10 @@ export async function runDefaultRuntimeTask(
     finishedAt,
     artifacts: [...runtimeArtifacts, ...outputArtifacts],
     failureDetails,
+    checkDetails,
   };
 }
 
-export async function settleSubprocess(subprocess, fileTimeoutSeconds) {
-  const timeoutMs = fileTimeoutSeconds * 1000 + 1_000;
-  let timeoutHandle = null;
-  let timedOut = false;
-
-  try {
-    return await Promise.race([
-      subprocess.then((result) => ({ result, timedOut })),
-      new Promise((resolve) => {
-        timeoutHandle = setTimeout(async () => {
-          timedOut = true;
-          killChildProcess(subprocess, "SIGTERM");
-          const result = await subprocess.catch((error) => error);
-          resolve({ result, timedOut: true });
-        }, timeoutMs);
-      }),
-    ]);
-  } finally {
-    if (timeoutHandle) clearTimeout(timeoutHandle);
-  }
-}
-
 export function buildDefaultRuntimeSummaryPath(lease, task) {
   return path.join(lease.leaseDir, "default-runtime", `task-${task.id}.summary.json`);
 }

package/lib/runner/failure-details.mjs

@@ -90,6 +90,28 @@ export function collectFailureDetailsFromRuntimeArtifacts(artifacts) {
   return mergeFailureDetails(details);
 }
 
+export function collectCheckDetailsFromRuntimeArtifacts(artifacts) {
+  const checks = [];
+
+  for (const artifact of Array.isArray(artifacts) ? artifacts : []) {
+    if (artifact?.kind !== "testkit.checks") continue;
+    for (const check of Array.isArray(artifact?.data?.checks) ? artifact.data.checks : []) {
+      const name = normalizeNonEmptyString(check?.name) || "unnamed";
+      const passes = Math.max(0, Number(check?.passes) || 0);
+      const fails = Math.max(0, Number(check?.fails) || 0);
+      checks.push({
+        name,
+        path: normalizeStringArray(check?.path),
+        passes,
+        fails,
+        passed: passes > 0 && fails === 0,
+      });
+    }
+  }
+
+  return checks;
+}
+
 function normalizeStringArray(value) {
   if (!Array.isArray(value)) return [];
   return value

package/lib/runner/lifecycle.mjs

@@ -1,8 +1,8 @@
 import fs from "fs";
 import path from "path";
 import crypto from "crypto";
-import { execFileSync } from "child_process";
 import { destroyRuntimeDatabase, cleanupOrphanedLocalInfrastructure } from "../database/index.mjs";
+import { killProcessTree, listDescendantPids } from "./processes.mjs";
 
 const RUN_SCHEMA_VERSION = 1;
 const RUNS_DIRNAME = path.join(".testkit", "_runs");
@@ -297,22 +297,7 @@ async function terminateOwnedProcess(service) {
 }
 
 function killProcessGroup(pid, signal) {
-  try {
-    process.kill(-pid, signal);
-    return;
-  } catch (error) {
-    if (error?.code !== "ESRCH") {
-      // Fall through and try the direct pid.
-    } else {
-      return;
-    }
-  }
-
-  try {
-    process.kill(pid, signal);
-  } catch (error) {
-    if (error?.code !== "ESRCH") throw error;
-  }
+  killProcessTree(pid, signal);
 }
 
 function terminateDescendantProcesses(rootPid, signal) {
@@ -328,40 +313,6 @@ function terminateDescendantProcesses(rootPid, signal) {
   }
 }
 
-function listDescendantPids(rootPid) {
-  try {
-    const output = execFileSync("ps", ["-eo", "pid=,ppid="], {
-      encoding: "utf8",
-      stdio: ["ignore", "pipe", "ignore"],
-    });
-    const childrenByParent = new Map();
-    for (const line of output.split(/\r?\n/)) {
-      const trimmed = line.trim();
-      if (!trimmed) continue;
-      const [pidRaw, parentRaw] = trimmed.split(/\s+/);
-      const pid = Number(pidRaw);
-      const parentPid = Number(parentRaw);
-      if (!Number.isInteger(pid) || !Number.isInteger(parentPid)) continue;
-      const siblings = childrenByParent.get(parentPid) || [];
-      siblings.push(pid);
-      childrenByParent.set(parentPid, siblings);
-    }
-
-    const descendants = [];
-    const stack = [...(childrenByParent.get(rootPid) || [])];
-    while (stack.length > 0) {
-      const pid = stack.pop();
-      descendants.push(pid);
-      for (const childPid of childrenByParent.get(pid) || []) {
-        stack.push(childPid);
-      }
-    }
-    return descendants;
-  } catch {
-    return [];
-  }
-}
-
 async function waitForPidExit(pid, timeoutMs) {
   const startedAt = Date.now();
   while (Date.now() - startedAt < timeoutMs) {

package/lib/runner/managed-processes.mjs

@@ -1,8 +1,9 @@
 import { killChildProcess } from "./processes.mjs";
+import { terminateSubprocess } from "./subprocess.mjs";
 
 export function registerManagedProcess(lifecycle, subprocess, signal = "SIGINT") {
   lifecycle.registerProcess(subprocess, () => {
-    killChildProcess(subprocess, signal);
+    void terminateSubprocess(subprocess, { gracefulSignal: signal });
   });
   if (lifecycle.isStopRequested()) {
     interruptManagedProcess(subprocess, signal);

package/lib/runner/playwright-config.mjs

@@ -3,7 +3,7 @@ import path from "path";
 import { pathToFileURL } from "url";
 import { normalizePathSeparators } from "./state.mjs";
 
-export function ensurePlaywrightTestConfig(targetConfig, cwd, requestedFiles, lease) {
+export function ensurePlaywrightTestConfig(targetConfig, cwd, requestedFiles, lease, options = {}) {
   if (!lease?.leaseDir) {
     throw new Error(
       `Playwright task for service "${targetConfig.name}" requires a lease-scoped directory`
@@ -17,6 +17,7 @@ export function ensurePlaywrightTestConfig(targetConfig, cwd, requestedFiles, le
   const configPath = path.join(stateDir, "playwright.testkit.config.mjs");
   const baseConfigPath = findPlaywrightConfig(cwd);
   const normalizedFiles = requestedFiles.map(normalizePathSeparators);
+  const globalTimeoutMs = normalizeOptionalPositiveInteger(options.globalTimeoutMs, "globalTimeoutMs");
 
   let source = "";
   if (baseConfigPath) {
@@ -28,6 +29,7 @@ export function ensurePlaywrightTestConfig(targetConfig, cwd, requestedFiles, le
       `  testDir: ${JSON.stringify(cwd)},\n` +
       `  testMatch: ${JSON.stringify(normalizedFiles)},\n` +
       `  outputDir: ${JSON.stringify(outputDir)},\n` +
+      (globalTimeoutMs ? `  globalTimeout: ${globalTimeoutMs},\n` : "") +
       `  workers: 1,\n` +
       `  fullyParallel: false,\n` +
       `  webServer: undefined,\n` +
@@ -38,6 +40,7 @@ export function ensurePlaywrightTestConfig(targetConfig, cwd, requestedFiles, le
       `  testDir: ${JSON.stringify(cwd)},\n` +
       `  testMatch: ${JSON.stringify(normalizedFiles)},\n` +
       `  outputDir: ${JSON.stringify(outputDir)},\n` +
+      (globalTimeoutMs ? `  globalTimeout: ${globalTimeoutMs},\n` : "") +
       `  workers: 1,\n` +
       `  fullyParallel: false,\n` +
       `};\n`;
@@ -47,6 +50,15 @@ export function ensurePlaywrightTestConfig(targetConfig, cwd, requestedFiles, le
   return configPath;
 }
 
+function normalizeOptionalPositiveInteger(value, label) {
+  if (value == null) return null;
+  const normalized = Number(value);
+  if (!Number.isInteger(normalized) || normalized <= 0) {
+    throw new Error(`${label} must be a positive integer.`);
+  }
+  return normalized;
+}
+
 export function resolvePlaywrightOutputDir(stateDir) {
   return path.join(stateDir, "playwright-output");
 }

package/lib/runner/playwright-runner.mjs

@@ -1,14 +1,14 @@
 import path from "path";
-import { execa } from "execa";
+import fs from "fs";
 import { parsePlaywrightJsonResults } from "../reporters/playwright.mjs";
 import { resolveServiceCwd } from "../config/paths.mjs";
-import { formatFileTimeoutBudgetError } from "../shared/file-timeout.mjs";
+import { buildFileTimeoutEnv, formatFileTimeoutBudgetError } from "../shared/file-timeout.mjs";
 import { persistTaskOutputArtifacts } from "./artifacts.mjs";
-import { settleSubprocess } from "./default-runtime-runner.mjs";
 import { ensurePlaywrightTestConfig } from "./playwright-config.mjs";
 import { registerManagedProcess, unregisterManagedProcess } from "./managed-processes.mjs";
 import { normalizePathSeparators } from "./state.mjs";
 import { buildPlaywrightEnv } from "./template.mjs";
+import { settleManagedSubprocess, startManagedSubprocess } from "./subprocess.mjs";
 
 export async function runPlaywrightTask(targetConfig, task, lifecycle, lease, reporter = null) {
   const local = targetConfig.testkit.local;
@@ -20,20 +20,36 @@ export async function runPlaywrightTask(targetConfig, task, lifecycle, lease, re
 
   const cwd = resolveServiceCwd(targetConfig.productDir, local.cwd);
   const requestedFile = path.relative(cwd, path.join(targetConfig.productDir, task.file));
-  const playwrightConfigPath = ensurePlaywrightTestConfig(targetConfig, cwd, [requestedFile], lease);
   if (lifecycle.isStopRequested()) {
     throw new Error(`testkit run interrupted before starting ${task.file}`);
   }
   const startedAt = Date.now();
   const fileTimeoutSeconds = targetConfig.testkit.execution.fileTimeoutSeconds;
-  const subprocess = execa(
+  const playwrightConfigPath = ensurePlaywrightTestConfig(
+    targetConfig,
+    cwd,
+    [requestedFile],
+    lease,
+    { globalTimeoutMs: fileTimeoutSeconds * 1000 }
+  );
+  const jsonReportPath = buildPlaywrightJsonReportPath(lease, task);
+  const subprocess = startManagedSubprocess(
     "npx",
     ["playwright", "test", "--config", playwrightConfigPath, "--reporter=json"],
     {
       cwd,
-      env: buildPlaywrightEnv(targetConfig, local.baseUrl, lease, process.env),
-      reject: false,
-      forceKillAfterDelay: 5_000,
+      env: {
+        ...buildPlaywrightEnv(
+          targetConfig,
+          local.baseUrl,
+          lease,
+          {
+            ...process.env,
+            ...buildFileTimeoutEnv(fileTimeoutSeconds, startedAt),
+          }
+        ),
+        PLAYWRIGHT_JSON_OUTPUT_FILE: jsonReportPath,
+      },
     }
   );
   registerManagedProcess(lifecycle, subprocess, "SIGINT");
@@ -41,17 +57,30 @@ export async function runPlaywrightTask(targetConfig, task, lifecycle, lease, re
   let result;
   let timedOut;
   try {
-    ({ result, timedOut } = await settleSubprocess(subprocess, fileTimeoutSeconds));
+    ({ result, timedOut } = await settleManagedSubprocess(subprocess, {
+      timeoutMs: fileTimeoutSeconds * 1000 + 1_000,
+    }));
   } finally {
     unregisterManagedProcess(lifecycle, subprocess);
   }
 
-  const parsed = parsePlaywrightJsonResults(result.stdout || "", cwd);
+  const jsonReport = readOptionalText(jsonReportPath);
+  const stdoutReport = jsonReport ? "" : extractJsonReportFromStdout(result.stdout || "");
+  const parsed = parsePlaywrightJsonResults(jsonReport || stdoutReport, cwd);
   const finishedAt = Date.now();
   const durationMs = finishedAt - startedAt;
   const relativeFile = normalizePathSeparators(requestedFile);
   const fileResult = parsed.fileResults.get(relativeFile);
   const outputArtifacts = persistTaskOutputArtifacts(targetConfig.productDir, task, [
+    result.stdout
+      ? {
+          name: "playwright-stdout",
+          kind: "runtime.output",
+          summary: result.stdout.split(/\r?\n/).map((line) => line.trim()).find(Boolean) || "captured stdout",
+          stream: "stdout",
+          text: result.stdout,
+        }
+      : null,
     result.stderr
       ? {
           name: "playwright-stderr",
@@ -62,16 +91,19 @@ export async function runPlaywrightTask(targetConfig, task, lifecycle, lease, re
         }
       : null,
   ]);
-  const genericError = timedOut
+  const parsedError = stripAnsi(parsed.errors[0] || "");
+  const globalTimeoutError = isPlaywrightGlobalTimeoutError(parsedError);
+  const genericError = timedOut || globalTimeoutError
     ? formatFileTimeoutBudgetError(fileTimeoutSeconds)
     : result.exitCode === 0
-      ? parsed.errors[0] || null
-      : parsed.errors[0] || result.stderr.trim() || `Playwright exited with code ${result.exitCode}`;
+      ? parsedError || null
+      : parsedError || firstNonEmptyLine(result.stderr) || firstNonEmptyLine(result.stdout) || `Playwright exited with code ${result.exitCode}`;
+  const failed = timedOut || globalTimeoutError || (fileResult ? fileResult.status === "failed" : genericError !== null);
 
   return {
     task,
-    failed: timedOut ? true : fileResult ? fileResult.status === "failed" : result.exitCode !== 0,
-    status: timedOut ? "failed" : fileResult?.status || (result.exitCode === 0 ? "passed" : "failed"),
+    failed,
+    status: timedOut ? "failed" : fileResult?.status || (failed ? "failed" : "passed"),
     error: timedOut ? genericError : fileResult?.error || genericError,
     durationMs: fileResult?.durationMs > 0 ? fileResult.durationMs : durationMs,
     startedAt,
@@ -80,3 +112,41 @@ export async function runPlaywrightTask(targetConfig, task, lifecycle, lease, re
     failureDetails: timedOut ? [] : fileResult?.failureDetails || [],
   };
 }
+
+export function buildPlaywrightJsonReportPath(lease, task) {
+  if (!lease?.leaseDir) {
+    throw new Error(`Playwright task ${task?.file || ""} requires a lease-scoped directory`);
+  }
+  const reportDir = path.join(lease.leaseDir, "playwright-reports");
+  fs.mkdirSync(reportDir, { recursive: true });
+  return path.join(reportDir, `task-${task.id}.json`);
+}
+
+function readOptionalText(filePath) {
+  try {
+    return fs.readFileSync(filePath, "utf8");
+  } catch {
+    return "";
+  }
+}
+
+function extractJsonReportFromStdout(stdout) {
+  const trimmed = String(stdout || "").trim();
+  if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) return "";
+  return trimmed;
+}
+
+function firstNonEmptyLine(value) {
+  return stripAnsi(value)
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .find(Boolean) || null;
+}
+
+function isPlaywrightGlobalTimeoutError(value) {
+  return /Timed out waiting \d+(?:\.\d+)?s for the test suite to run/i.test(value || "");
+}
+
+function stripAnsi(value) {
+  return String(value || "").replace(/\u001b\[[0-9;]*m/g, "");
+}

package/lib/runner/processes.mjs

@@ -1,4 +1,4 @@
-import { spawn } from "child_process";
+import { execFileSync, spawn } from "child_process";
 
 export function normalizeServiceStartCommand(command) {
   if (typeof command !== "string") {
@@ -36,9 +36,16 @@ export function startDetachedCommand(command, cwd, env) {
 
 export function killChildProcess(child, signal) {
   if (!child?.pid) return;
+  killProcessTree(child.pid, signal);
+}
+
+export function killProcessTree(pid, signal) {
+  const normalizedPid = Number(pid);
+  if (!Number.isInteger(normalizedPid) || normalizedPid <= 0) return;
+  const descendants = listDescendantPids(normalizedPid);
 
   try {
-    process.kill(-child.pid, signal);
+    process.kill(-normalizedPid, signal);
     return;
   } catch (error) {
     if (error?.code !== "ESRCH") {
@@ -49,10 +56,18 @@ export function killChildProcess(child, signal) {
   }
 
   try {
-    child.kill(signal);
+    process.kill(normalizedPid, signal);
   } catch (error) {
     if (error?.code !== "ESRCH") throw error;
   }
+
+  for (const childPid of descendants) {
+    try {
+      process.kill(childPid, signal);
+    } catch (error) {
+      if (error?.code !== "ESRCH") throw error;
+    }
+  }
 }
 
 export function captureOutput(stream, options = {}) {
@@ -116,3 +131,44 @@ export async function stopChildProcess(child, outputDrains = []) {
 export function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
+
+export function listDescendantPids(rootPid) {
+  const normalizedRootPid = Number(rootPid);
+  if (!Number.isInteger(normalizedRootPid) || normalizedRootPid <= 0) return [];
+
+  if (process.platform === "win32") {
+    return [];
+  }
+
+  try {
+    const output = execFileSync("ps", ["-eo", "pid=,ppid="], {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "ignore"],
+    });
+    const childrenByParent = new Map();
+    for (const line of output.split(/\r?\n/)) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      const [pidRaw, parentRaw] = trimmed.split(/\s+/);
+      const pid = Number(pidRaw);
+      const parentPid = Number(parentRaw);
+      if (!Number.isInteger(pid) || !Number.isInteger(parentPid)) continue;
+      const siblings = childrenByParent.get(parentPid) || [];
+      siblings.push(pid);
+      childrenByParent.set(parentPid, siblings);
+    }
+
+    const descendants = [];
+    const stack = [...(childrenByParent.get(normalizedRootPid) || [])];
+    while (stack.length > 0) {
+      const pid = stack.pop();
+      descendants.push(pid);
+      for (const childPid of childrenByParent.get(pid) || []) {
+        stack.push(childPid);
+      }
+    }
+    return descendants;
+  } catch {
+    return [];
+  }
+}

package/lib/runner/results.mjs

@@ -48,6 +48,7 @@ export function buildServiceTrackers(servicePlans, startedAt) {
               status: "not_run",
               artifacts: [],
               failureDetails: [],
+              checkDetails: [],
             },
           ];
         }),
@@ -62,6 +63,7 @@ export function buildServiceTrackers(servicePlans, startedAt) {
             status: "skipped",
             artifacts: [],
             failureDetails: [],
+            checkDetails: [],
           },
         ]),
       ]),
@@ -125,6 +127,7 @@ export function recordTaskOutcome(trackers, task, outcome, finishedAt = Date.now
     existingFileResult.status = status;
     existingFileResult.artifacts = Array.isArray(outcome.artifacts) ? outcome.artifacts : [];
     existingFileResult.failureDetails = mergeFailureDetails(outcome.failureDetails);
+    existingFileResult.checkDetails = normalizeCheckDetails(outcome.checkDetails);
   } else {
     suite.fileResultsByPath.set(normalizedPath, {
       path: normalizedPath,
@@ -135,6 +138,7 @@ export function recordTaskOutcome(trackers, task, outcome, finishedAt = Date.now
       status,
       artifacts: Array.isArray(outcome.artifacts) ? outcome.artifacts : [],
       failureDetails: mergeFailureDetails(outcome.failureDetails),
+      checkDetails: normalizeCheckDetails(outcome.checkDetails),
     });
   }
   if (status === "failed" && !suite.failedFileSet.has(task.file)) {
@@ -258,6 +262,9 @@ function finalizeSuite(suite) {
       ...(Array.isArray(file.artifacts) && file.artifacts.length > 0
         ? { artifacts: file.artifacts }
         : {}),
+      ...(Array.isArray(file.checkDetails) && file.checkDetails.length > 0
+        ? { checkDetails: file.checkDetails }
+        : {}),
     }));
 
   return {
@@ -289,3 +296,27 @@ function normalizeOutcomeStatus(outcome) {
   if (outcome?.status === "skipped") return "skipped";
   return outcome?.failed ? "failed" : "passed";
 }
+
+function normalizeCheckDetails(value) {
+  if (!Array.isArray(value)) return [];
+  return value.map((check) => {
+    const passes = normalizeCount(check?.passes);
+    const fails = normalizeCount(check?.fails);
+    return {
+      name: typeof check?.name === "string" && check.name.trim().length > 0 ? check.name.trim() : "unnamed",
+      path: Array.isArray(check?.path)
+        ? check.path
+          .map((entry) => String(entry || "").trim())
+          .filter(Boolean)
+        : [],
+      passes,
+      fails,
+      passed: passes > 0 && fails === 0,
+    };
+  });
+}
+
+function normalizeCount(value) {
+  const count = Number(value);
+  return Number.isFinite(count) && count > 0 ? count : 0;
+}