@elench/testkit 0.1.32 → 0.1.34

package/lib/runner/artifacts.mjs

@@ -0,0 +1,43 @@
+import fs from "fs";
+import path from "path";
+import {
+  applyTimingUpdates,
+  createEmptyTimings,
+  normalizeTimings,
+} from "../timing/index.mjs";
+
+const TIMINGS_FILENAME = "timings.json";
+
+export function writeRunArtifact(productDir, artifact) {
+  const resultsDir = path.join(productDir, ".testkit", "results");
+  fs.mkdirSync(resultsDir, { recursive: true });
+  fs.writeFileSync(path.join(resultsDir, "latest.json"), JSON.stringify(artifact, null, 2));
+}
+
+export function writeStatusArtifact(productDir, artifact) {
+  fs.writeFileSync(
+    path.join(productDir, "testkit.status.json"),
+    `${JSON.stringify(artifact, null, 2)}\n`
+  );
+}
+
+export function loadTimings(productDir) {
+  const filePath = path.join(productDir, ".testkit", TIMINGS_FILENAME);
+  if (!fs.existsSync(filePath)) {
+    return createEmptyTimings();
+  }
+
+  try {
+    return normalizeTimings(JSON.parse(fs.readFileSync(filePath, "utf8")));
+  } catch {
+    return createEmptyTimings();
+  }
+}
+
+export function saveTimings(productDir, timings, updates) {
+  if (updates.length === 0) return;
+  const next = applyTimingUpdates(timings, updates);
+  const rootDir = path.join(productDir, ".testkit");
+  fs.mkdirSync(rootDir, { recursive: true });
+  fs.writeFileSync(path.join(rootDir, TIMINGS_FILENAME), JSON.stringify(next, null, 2));
+}

package/lib/runner/default-runtime-errors.mjs

@@ -0,0 +1,53 @@
+export function determineDefaultRuntimeFailure(result, summary, firstLine) {
+  const fatalRuntimeError = extractDefaultRuntimeFatalError(result.stderr || "", firstLine);
+  if (fatalRuntimeError) {
+    return `Default runtime uncaught error: ${fatalRuntimeError}`;
+  }
+
+  const failedThresholds = extractDefaultRuntimeThresholdFailures(summary);
+  if (failedThresholds.length > 0) {
+    return `Default runtime thresholds failed: ${failedThresholds.join(", ")}`;
+  }
+
+  if (result.exitCode !== 0) {
+    return sanitizeDefaultRuntimeExitError(
+      result.exitCode,
+      result.stderr || result.stdout || "",
+      firstLine
+    );
+  }
+
+  return null;
+}
+
+export function extractDefaultRuntimeFatalError(stderr, firstLine) {
+  if (!stderr || !/source=stacktrace/.test(stderr)) return null;
+  const matched = stderr.match(/Error:\s([^\n]+)/);
+  return matched?.[1]?.trim() || firstLine(stderr);
+}
+
+export function extractDefaultRuntimeThresholdFailures(summary) {
+  const metrics = summary?.metrics;
+  if (!metrics || typeof metrics !== "object") return [];
+
+  const failures = [];
+  for (const [metricName, metricSummary] of Object.entries(metrics)) {
+    const thresholds = metricSummary?.thresholds;
+    if (!thresholds || typeof thresholds !== "object") continue;
+    for (const [threshold, crossed] of Object.entries(thresholds)) {
+      if (crossed === true) {
+        failures.push(`${metricName}(${threshold})`);
+      }
+    }
+  }
+
+  return failures.sort();
+}
+
+export function sanitizeDefaultRuntimeExitError(exitCode, output, firstLine) {
+  const message = firstLine(output);
+  if (message) {
+    return `Default runtime failed with exit code ${exitCode}: ${message}`;
+  }
+  return `Default runtime failed with exit code ${exitCode}`;
+}

package/lib/runner/default-runtime-errors.test.mjs

@@ -0,0 +1,49 @@
+import { describe, expect, it } from "vitest";
+import {
+  determineDefaultRuntimeFailure,
+  extractDefaultRuntimeFatalError,
+  extractDefaultRuntimeThresholdFailures,
+  sanitizeDefaultRuntimeExitError,
+} from "./default-runtime-errors.mjs";
+
+const firstLine = (value) => value.split(/\r?\n/)[0];
+
+describe("default runtime errors", () => {
+  it("extracts fatal stacktrace errors", () => {
+    expect(
+      extractDefaultRuntimeFatalError("source=stacktrace\nError: boom\nmore", firstLine)
+    ).toBe("boom");
+  });
+
+  it("extracts failed thresholds", () => {
+    expect(
+      extractDefaultRuntimeThresholdFailures({
+        metrics: {
+          checks: {
+            thresholds: {
+              "rate==1": true,
+              "rate>0.9": false,
+            },
+          },
+          other: {},
+        },
+      })
+    ).toEqual(["checks(rate==1)"]);
+  });
+
+  it("sanitizes exit errors", () => {
+    expect(sanitizeDefaultRuntimeExitError(99, "bad\nstack", firstLine)).toBe(
+      "Default runtime failed with exit code 99: bad"
+    );
+  });
+
+  it("chooses the highest-priority runtime failure", () => {
+    expect(
+      determineDefaultRuntimeFailure(
+        { exitCode: 0, stderr: "source=stacktrace\nError: boom", stdout: "" },
+        null,
+        firstLine
+      )
+    ).toBe("Default runtime uncaught error: boom");
+  });
+});

package/lib/runner/default-runtime-runner.mjs

@@ -0,0 +1,119 @@
+import fs from "fs";
+import path from "path";
+import { execa } from "execa";
+import { bundleK6File } from "../bundler/index.mjs";
+import { resolveK6Binary } from "../config/index.mjs";
+import { determineDefaultRuntimeFailure } from "./default-runtime-errors.mjs";
+import { formatBatchDescriptor } from "./formatting.mjs";
+import { buildExecutionEnv } from "./template.mjs";
+import { readDatabaseUrl } from "./state-io.mjs";
+
+export async function runHttpK6Batch(targetConfig, batch, lifecycle) {
+  const baseUrl = targetConfig.testkit.local?.baseUrl;
+  if (!baseUrl) {
+    throw new Error(`Service "${targetConfig.name}" requires local.baseUrl for HTTP suites`);
+  }
+
+  console.log(
+    `\n── ${targetConfig.workerLabel} ${batch.type}:${batch.tasks[0].suiteName}${formatBatchDescriptor(batch)} ──`
+  );
+
+  return Promise.all(
+    batch.tasks.map((task) => runHttpK6Task(targetConfig, task, baseUrl, lifecycle))
+  );
+}
+
+export async function runDalBatch(targetConfig, batch, lifecycle) {
+  const databaseUrl = readDatabaseUrl(targetConfig.stateDir);
+  if (!databaseUrl) {
+    throw new Error(`Service "${targetConfig.name}" requires a database for DAL suites`);
+  }
+
+  console.log(
+    `\n── ${targetConfig.workerLabel} ${batch.type}:${batch.tasks[0].suiteName}${formatBatchDescriptor(batch)} ──`
+  );
+
+  return Promise.all(
+    batch.tasks.map((task) => runDalTask(targetConfig, task, databaseUrl, lifecycle))
+  );
+}
+
+async function runHttpK6Task(targetConfig, task, baseUrl, lifecycle) {
+  const bundledFile = await bundleK6File({
+    productDir: targetConfig.productDir,
+    serviceName: targetConfig.name,
+    sourceFile: path.join(targetConfig.productDir, task.file),
+  });
+  console.log(`·· ${targetConfig.workerLabel}:${task.suiteName} → ${task.file}`);
+  return runDefaultRuntimeTask(
+    targetConfig,
+    task,
+    ["run", "--address", "127.0.0.1:0", "-e", `BASE_URL=${baseUrl}`, bundledFile],
+    lifecycle
+  );
+}
+
+async function runDalTask(targetConfig, task, databaseUrl, lifecycle) {
+  const bundledFile = await bundleK6File({
+    productDir: targetConfig.productDir,
+    serviceName: targetConfig.name,
+    sourceFile: path.join(targetConfig.productDir, task.file),
+  });
+  console.log(`·· ${targetConfig.workerLabel}:${task.suiteName} → ${task.file}`);
+  return runDefaultRuntimeTask(
+    targetConfig,
+    task,
+    ["run", "--address", "127.0.0.1:0", "-e", `DATABASE_URL=${databaseUrl}`, bundledFile],
+    lifecycle
+  );
+}
+
+export async function runDefaultRuntimeTask(targetConfig, task, args, lifecycle, firstLine) {
+  const k6Binary = resolveK6Binary();
+  const summaryFile = buildDefaultRuntimeSummaryPath(targetConfig, task);
+  fs.mkdirSync(path.dirname(summaryFile), { recursive: true });
+  const startedAt = Date.now();
+  const result = await execa(
+    k6Binary,
+    [...args.slice(0, 1), "--summary-export", summaryFile, ...args.slice(1)],
+    {
+      cwd: targetConfig.productDir,
+      env: buildExecutionEnv(targetConfig, {}, process.env),
+      reject: false,
+      cancelSignal: lifecycle.signal,
+      forceKillAfterDelay: 5_000,
+    }
+  );
+
+  if (result.stdout) process.stdout.write(result.stdout);
+  if (result.stderr) process.stderr.write(result.stderr);
+
+  const summary = readDefaultRuntimeSummary(summaryFile);
+  const runtimeError = determineDefaultRuntimeFailure(result, summary, firstLine);
+  const finishedAt = Date.now();
+
+  return {
+    task,
+    failed: runtimeError !== null,
+    error: runtimeError,
+    durationMs: finishedAt - startedAt,
+    startedAt,
+    finishedAt,
+  };
+}
+
+export function buildDefaultRuntimeSummaryPath(targetConfig, task) {
+  return path.join(
+    targetConfig.stateDir || path.join(targetConfig.productDir, ".testkit"),
+    "_runtime",
+    `task-${task.id}.summary.json`
+  );
+}
+
+export function readDefaultRuntimeSummary(filePath) {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf8"));
+  } catch {
+    return null;
+  }
+}

package/lib/runner/formatting.mjs

@@ -0,0 +1,129 @@
+export function formatDuration(durationMs) {
+  const totalSeconds = Math.max(0, Math.round(durationMs / 1000));
+  const minutes = Math.floor(totalSeconds / 60);
+  const seconds = totalSeconds % 60;
+  if (minutes === 0) return `${seconds}s`;
+  return `${minutes}m ${String(seconds).padStart(2, "0")}s`;
+}
+
+export function formatServiceSummary(result) {
+  const passedSuites = result.completedSuiteCount - result.failedSuiteCount;
+  const notRunSuites = result.suiteCount - result.completedSuiteCount;
+  let detail = `${passedSuites}/${result.suiteCount} suites passed`;
+  if ((result.totalFileCount || 0) > 0) {
+    detail += `, ${result.passedFileCount}/${result.totalFileCount} files passed`;
+  }
+  if (notRunSuites > 0) {
+    detail += `, ${notRunSuites} ${pluralize(notRunSuites, "suite", "suites")} not run`;
+  } else if ((result.notRunFileCount || 0) > 0) {
+    detail += `, ${result.notRunFileCount} ${pluralize(result.notRunFileCount, "file", "files")} not run`;
+  }
+  return detail;
+}
+
+export function formatError(error) {
+  if (error instanceof Error) return sanitizeErrorMessage(error.message);
+  return sanitizeErrorMessage(String(error));
+}
+
+export function longestServiceName(results) {
+  return results.reduce((max, result) => Math.max(max, result.name.length), 4);
+}
+
+export function formatBatchDescriptor(batch) {
+  const fileLabel = `${batch.tasks.length} file${batch.tasks.length === 1 ? "" : "s"}`;
+  const frameworkLabel = formatFrameworkLabel(batch.framework);
+  return frameworkLabel ? ` (${frameworkLabel}, ${fileLabel})` : ` (${fileLabel})`;
+}
+
+export function formatPlaywrightBatchFiles(batch) {
+  if (!batch?.tasks?.length) return "";
+  const files = batch.tasks.map((task) => task.file);
+  if (files.length === 1) return ` · ${files[0]}`;
+  const preview = files.slice(0, 3).join(", ");
+  const suffix = files.length > 3 ? `, +${files.length - 3} more` : "";
+  return ` · ${preview}${suffix}`;
+}
+
+export function formatFrameworkLabel(framework) {
+  if (!framework || framework === "k6") return "";
+  return framework;
+}
+
+export function formatSuiteFramework(framework) {
+  const label = formatFrameworkLabel(framework);
+  return label ? ` [${label}]` : "";
+}
+
+export function buildRunSummaryLines(results, durationMs) {
+  const totalServices = results.length;
+  const executedServices = results.filter((result) => !result.skipped);
+  const skippedServices = results.filter((result) => result.skipped);
+  const failedServices = executedServices.filter((result) => result.failed);
+  const passedServices = executedServices.filter((result) => !result.failed);
+  const totalSuites = executedServices.reduce((sum, result) => sum + result.suiteCount, 0);
+  const completedSuites = executedServices.reduce(
+    (sum, result) => sum + result.completedSuiteCount,
+    0
+  );
+  const failedSuites = executedServices.reduce((sum, result) => sum + result.failedSuiteCount, 0);
+  const passedSuites = completedSuites - failedSuites;
+  const totalFiles = executedServices.reduce((sum, result) => sum + (result.totalFileCount || 0), 0);
+  const passedFiles = executedServices.reduce((sum, result) => sum + (result.passedFileCount || 0), 0);
+  const lines = [
+    "",
+    "══ Summary ══",
+    [
+      `services ${passedServices.length}/${executedServices.length} passed`,
+      `suites ${passedSuites}/${totalSuites} passed`,
+      totalFiles > 0 ? `files ${passedFiles}/${totalFiles} passed` : null,
+      skippedServices.length > 0 ? `${skippedServices.length} skipped` : null,
+      `duration ${formatDuration(durationMs)}`,
+    ]
+      .filter(Boolean)
+      .join("  ·  "),
+  ];
+
+  for (const result of results) {
+    const status = result.skipped ? "SKIP" : result.failed ? "FAIL" : "PASS";
+    const detail = result.skipped ? "no matching suites" : formatServiceSummary(result);
+    lines.push(
+      `${status.padEnd(4)} ${result.name.padEnd(longestServiceName(results))}  ${detail}  ·  ${formatDuration(result.durationMs)}`
+    );
+
+    if (!result.failed) continue;
+
+    for (const suite of result.suites.filter((entry) => entry.failed)) {
+      const fileDetail =
+        suite.failedFiles.length > 0 ? `  (${suite.failedFiles.join(", ")})` : "";
+      lines.push(
+        `  - ${suite.type}:${suite.name}${formatSuiteFramework(suite.framework)}${fileDetail} · ${formatDuration(suite.durationMs)}`
+      );
+      if (suite.error) {
+        lines.push(`    ${suite.error}`);
+      }
+    }
+    for (const error of result.errors) {
+      lines.push(`  - worker error: ${error}`);
+    }
+  }
+
+  if (failedServices.length > 0) {
+    lines.push("", `Result: FAILED (${failedServices.length}/${totalServices} services failed)`);
+    return lines;
+  }
+
+  lines.push("", "Result: PASSED");
+  return lines;
+}
+
+function sanitizeErrorMessage(message) {
+  return message
+    .replace(/Command failed with exit code (\d+): .*?[\\/]vendor[\\/]k6 run\b/g, "Default runtime failed with exit code $1:")
+    .replace(/Command failed with exit code (\d+): k6 run\b/g, "Default runtime failed with exit code $1:")
+    .replace(/[\\/]vendor[\\/]k6\b/g, "default-runtime");
+}
+
+function pluralize(value, singular, plural) {
+  return value === 1 ? singular : plural;
+}

package/lib/runner/formatting.test.mjs

@@ -0,0 +1,100 @@
+import { describe, expect, it } from "vitest";
+import {
+  buildRunSummaryLines,
+  formatBatchDescriptor,
+  formatDuration,
+  formatError,
+  formatFrameworkLabel,
+  formatPlaywrightBatchFiles,
+  formatServiceSummary,
+  formatSuiteFramework,
+  longestServiceName,
+} from "./formatting.mjs";
+
+describe("runner formatting", () => {
+  it("formats durations compactly", () => {
+    expect(formatDuration(900)).toBe("1s");
+    expect(formatDuration(61_000)).toBe("1m 01s");
+  });
+
+  it("sanitizes default-runtime error messages", () => {
+    expect(
+      formatError(new Error("Command failed with exit code 99: /tmp/vendor/k6 run test.js"))
+    ).toBe("Default runtime failed with exit code 99: test.js");
+  });
+
+  it("formats service summaries with not-run files", () => {
+    expect(
+      formatServiceSummary({
+        completedSuiteCount: 2,
+        failedSuiteCount: 1,
+        suiteCount: 3,
+        totalFileCount: 5,
+        passedFileCount: 3,
+        notRunFileCount: 1,
+      })
+    ).toBe("1/3 suites passed, 3/5 files passed, 1 suite not run");
+  });
+
+  it("formats batch descriptors", () => {
+    expect(formatBatchDescriptor({ framework: "k6", tasks: [{}, {}] })).toBe(" (2 files)");
+    expect(formatBatchDescriptor({ framework: "playwright", tasks: [{}] })).toBe(
+      " (playwright, 1 file)"
+    );
+  });
+
+  it("formats Playwright file previews", () => {
+    expect(formatPlaywrightBatchFiles({ tasks: [{ file: "a" }] })).toBe(" · a");
+    expect(
+      formatPlaywrightBatchFiles({
+        tasks: [{ file: "a" }, { file: "b" }, { file: "c" }, { file: "d" }],
+      })
+    ).toBe(" · a, b, c, +1 more");
+  });
+
+  it("formats framework labels", () => {
+    expect(formatFrameworkLabel("k6")).toBe("");
+    expect(formatFrameworkLabel("playwright")).toBe("playwright");
+    expect(formatSuiteFramework("playwright")).toBe(" [playwright]");
+  });
+
+  it("computes longest service name", () => {
+    expect(longestServiceName([{ name: "api" }, { name: "frontend" }])).toBe(8);
+  });
+
+  it("builds printable summary lines", () => {
+    const lines = buildRunSummaryLines(
+      [
+        {
+          name: "frontend",
+          skipped: false,
+          failed: true,
+          suiteCount: 2,
+          completedSuiteCount: 2,
+          failedSuiteCount: 1,
+          totalFileCount: 3,
+          passedFileCount: 2,
+          durationMs: 20_000,
+          suites: [
+            {
+              failed: true,
+              type: "e2e",
+              name: "auth",
+              framework: "playwright",
+              failedFiles: ["a.pw.testkit.ts"],
+              durationMs: 12_000,
+              error: "boom",
+            },
+          ],
+          errors: ["worker broke"],
+        },
+      ],
+      20_000
+    );
+
+    expect(lines.join("\n")).toContain("services 0/1 passed");
+    expect(lines.join("\n")).toContain("FAIL frontend");
+    expect(lines.join("\n")).toContain("worker error: worker broke");
+    expect(lines.at(-1)).toBe("Result: FAILED (1/1 services failed)");
+  });
+});