@elench/testkit 0.1.51 → 0.1.53

package/lib/runner/formatting.mjs

@@ -49,6 +49,56 @@ export function formatSuiteFramework(framework) {
 }
 
 export function buildRunSummaryLines(results, durationMs, knownFailureIssueValidation = null) {
+  return buildCompactRunSummaryLines(results, durationMs, knownFailureIssueValidation);
+}
+
+export function buildCompactRunSummaryLines(
+  results,
+  durationMs,
+  knownFailureIssueValidation = null
+) {
+  const totals = summarizeResults(results);
+  const lines = [
+    "",
+    `Summary: ${totals.passedFiles} passed, ${totals.failedFiles} failed, ${totals.skippedFiles} skipped, ${totals.notRunFiles} not run across ${totals.totalFiles} ${pluralize(totals.totalFiles, "file", "files")} in ${formatDuration(durationMs)}`,
+  ];
+
+  const failures = collectFailedFiles(results);
+  if (failures.length > 0) {
+    lines.push("", "Failures:");
+    for (const failure of failures) {
+      lines.push(`  ${failure.file.path}`);
+      lines.push(`    ${failure.primaryMessage}`);
+      for (const detail of failure.extraDetails.slice(0, 2)) {
+        lines.push(`    ${detail}`);
+      }
+    }
+  }
+
+  const serviceErrors = collectServiceErrors(results);
+  if (serviceErrors.length > 0) {
+    lines.push("", "Runtime Errors:");
+    for (const item of serviceErrors) {
+      lines.push(`  ${item.service}`);
+      lines.push(`    ${item.message}`);
+    }
+  }
+
+  const knownFailureIssueLines = buildKnownFailureIssueValidationSummaryLines(
+    knownFailureIssueValidation
+  );
+  if (knownFailureIssueLines.length > 0) {
+    lines.push(...knownFailureIssueLines);
+  }
+
+  lines.push("");
+  lines.push(
+    totals.failedServices > 0 ? `Result: FAILED (${totals.failedServices}/${totals.totalServices} services failed)` : "Result: PASSED"
+  );
+  return lines;
+}
+
+export function buildDebugRunSummaryLines(results, durationMs, knownFailureIssueValidation = null) {
   const totalServices = results.length;
   const executedServices = results.filter((result) => !result.skipped);
   const skippedServices = results.filter((result) => result.skipped);
@@ -140,6 +190,53 @@ function sanitizeErrorMessage(message) {
     .replace(/[\\/]vendor[\\/]k6\b/g, "default-runtime");
 }
 
+function summarizeResults(results) {
+  const executedServices = results.filter((result) => !result.skipped);
+  return {
+    totalServices: results.length,
+    failedServices: executedServices.filter((result) => result.failed).length,
+    totalFiles: executedServices.reduce((sum, result) => sum + (result.totalFileCount || 0), 0),
+    passedFiles: executedServices.reduce((sum, result) => sum + (result.passedFileCount || 0), 0),
+    failedFiles: executedServices.reduce((sum, result) => sum + (result.failedFileCount || 0), 0),
+    skippedFiles: executedServices.reduce((sum, result) => sum + (result.skippedFileCount || 0), 0),
+    notRunFiles: executedServices.reduce((sum, result) => sum + (result.notRunFileCount || 0), 0),
+  };
+}
+
+function collectFailedFiles(results) {
+  const failures = [];
+  for (const result of results) {
+    for (const suite of result.suites || []) {
+      for (const file of suite.files || []) {
+        if (file.status !== "failed") continue;
+        const detailMessages = (file.failureDetails || [])
+          .map((detail) => detail.message || detail.title)
+          .filter(Boolean)
+          .map((message) => sanitizeErrorMessage(String(message).trim()));
+        failures.push({
+          file,
+          primaryMessage: sanitizeErrorMessage(file.error || detailMessages[0] || suite.error || "Failed"),
+          extraDetails: detailMessages.slice(file.error ? 0 : 1),
+        });
+      }
+    }
+  }
+  return failures.sort((left, right) => left.file.path.localeCompare(right.file.path));
+}
+
+function collectServiceErrors(results) {
+  const items = [];
+  for (const result of results) {
+    for (const error of result.errors || []) {
+      items.push({
+        service: result.name,
+        message: sanitizeErrorMessage(error),
+      });
+    }
+  }
+  return items;
+}
+
 function pluralize(value, singular, plural) {
   return value === 1 ? singular : plural;
 }

package/lib/runner/formatting.test.mjs

@@ -94,9 +94,9 @@ describe("runner formatting", () => {
       20_000
     );
 
-    expect(lines.join("\n")).toContain("services 0/1 passed");
-    expect(lines.join("\n")).toContain("FAIL frontend");
-    expect(lines.join("\n")).toContain("worker error: worker broke");
+    expect(lines.join("\n")).toContain("Summary: 2 passed, 0 failed, 0 skipped, 0 not run across 3 files");
+    expect(lines.join("\n")).toContain("Runtime Errors:");
+    expect(lines.join("\n")).toContain("worker broke");
     expect(lines.at(-1)).toBe("Result: FAILED (1/1 services failed)");
   });
 
@@ -123,9 +123,7 @@ describe("runner formatting", () => {
       0
     );
 
-    expect(lines.join("\n")).toContain("suites 1 skipped");
-    expect(lines.join("\n")).toContain("files 1 skipped");
-    expect(lines.join("\n")).toContain("SKIP api");
+    expect(lines.join("\n")).toContain("Summary: 0 passed, 0 failed, 1 skipped, 0 not run across 1 file");
     expect(lines.at(-1)).toBe("Result: PASSED");
   });
 

package/lib/runner/logs.mjs

@@ -0,0 +1,72 @@
+import fs from "fs";
+import path from "path";
+
+const RESULT_LOGS_DIRNAME = "logs";
+
+export function createRunLogRegistry(productDir) {
+  const records = new Map();
+
+  return {
+    ensureServiceLogRecord(config) {
+      const key = `${config.runtimeLabel || config.name}:${config.name}`;
+      const existing = records.get(key);
+      if (existing) return existing;
+
+      const fileName = `${sanitizePathSegment(config.runtimeLabel || config.name)}__${sanitizePathSegment(config.name)}.log`;
+      const relativePath = path.join(".testkit", "results", RESULT_LOGS_DIRNAME, fileName);
+      const absolutePath = path.join(productDir, relativePath);
+      fs.mkdirSync(path.dirname(absolutePath), { recursive: true });
+      const stream = fs.createWriteStream(absolutePath, { flags: "a" });
+      const record = {
+        key,
+        serviceName: config.name,
+        runtimeLabel: config.runtimeLabel || config.name,
+        path: normalizePath(relativePath),
+        absolutePath,
+        stream,
+      };
+      records.set(key, record);
+      return record;
+    },
+    append(record, streamName, line) {
+      if (!record || typeof line !== "string") return;
+      record.stream.write(`${new Date().toISOString()} [${streamName}] ${line}\n`);
+    },
+    listServiceLogs() {
+      return [...records.values()]
+        .map((record) => ({
+          serviceName: record.serviceName,
+          runtimeLabel: record.runtimeLabel,
+          path: record.path,
+        }))
+        .sort(
+          (left, right) =>
+            left.serviceName.localeCompare(right.serviceName) ||
+            left.runtimeLabel.localeCompare(right.runtimeLabel)
+        );
+    },
+    closeAll() {
+      for (const record of records.values()) {
+        record.stream.end();
+      }
+    },
+  };
+}
+
+export function readLogTail(absolutePath, lineCount = 80) {
+  if (!absolutePath || !fs.existsSync(absolutePath)) return [];
+  const lines = fs.readFileSync(absolutePath, "utf8").split(/\r?\n/).filter(Boolean);
+  return lines.slice(Math.max(0, lines.length - lineCount));
+}
+
+function sanitizePathSegment(value) {
+  return String(value)
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9._-]+/g, "-")
+    .replace(/^-+|-+$/g, "") || "log";
+}
+
+function normalizePath(filePath) {
+  return filePath.split(path.sep).join("/");
+}

package/lib/runner/orchestrator.mjs

@@ -16,7 +16,7 @@ import {
 } from "./results.mjs";
 import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
 import { applyKnownFailuresToArtifacts, loadKnownFailuresConfig } from "./triage.mjs";
-import { buildRunSummaryLines, formatError } from "./formatting.mjs";
+import { formatError } from "./formatting.mjs";
 import {
   shouldFailKnownFailureIssueValidation,
   validateKnownFailureIssues,
@@ -28,6 +28,7 @@ import {
   writeRunArtifact,
   writeStatusArtifact,
 } from "./artifacts.mjs";
+import { createRunLogRegistry } from "./logs.mjs";
 import {
   cleanupRunById,
   cleanupRuns,
@@ -65,6 +66,8 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     productDir,
     configs[0]?.testkit?.reporting || null
   );
+  const reporter = opts.reporter || null;
+  const logRegistry = createRunLogRegistry(productDir);
   const requestedFiles = opts.fileNames || [];
   if (requestedFiles.length > 0) {
     const unmatchedFiles = findUnmatchedRequestedFiles(
@@ -109,7 +112,8 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     typeValues,
     suiteSelectors,
     opts,
-    execution
+    execution,
+    reporter
   );
   const trackers = buildServiceTrackers(servicePlans, startedAt);
   const executedPlans = servicePlans.filter((plan) => !plan.skipped);
@@ -138,6 +142,10 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
         productDir,
         graphs,
         lifecycle,
+        runtimeOptions: {
+          reporter,
+          logRegistry,
+        },
       });
       const timingUpdates = [];
 
@@ -153,7 +161,8 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
               lifecycle,
               claimNextTask,
               recordTaskOutcome,
-              recordGraphError
+              recordGraphError,
+              reporter
             )
           )
         );
@@ -194,6 +203,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
       serviceFilter: opts.serviceFilter || null,
       metadata,
       summarizeDbBackend,
+      serviceLogs: logRegistry.listServiceLogs(),
     });
     const statusArtifact = opts.writeStatus
       ? buildStatusArtifact({
@@ -231,13 +241,19 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
       writeStatusArtifact(productDir, enrichedArtifacts.statusArtifact);
     }
 
-    printRunSummary(results, finishedAt - startedAt, knownFailureIssueValidation);
-    await reportTelemetry(telemetry, enrichedArtifacts.runArtifact);
+    reporter?.runSummary?.(results, finishedAt - startedAt, knownFailureIssueValidation);
+    await reportTelemetry(telemetry, enrichedArtifacts.runArtifact, reporter);
     if (results.some((result) => result.failed)) exitCode = 1;
     if (shouldFailKnownFailureIssueValidation(knownFailureIssueValidation)) {
       exitCode = 1;
     }
     if (lifecycle.isStopRequested()) exitCode = Math.max(exitCode, 130);
+    return {
+      runArtifact: enrichedArtifacts.runArtifact,
+      statusArtifact: enrichedArtifacts.statusArtifact,
+      results,
+      exitCode,
+    };
   } finally {
     if (lifecycle.isStopRequested()) {
       exitCode = Math.max(exitCode, 130);
@@ -250,21 +266,22 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     await cleanupRuns(productDir, { includeActive: false });
     lifecycle.removeManifest();
     lifecycle.dispose();
+    logRegistry.closeAll();
     process.exitCode = exitCode;
   }
 }
 
-function collectServicePlans(configs, configMap, typeValues, suiteSelectors, opts, execution) {
+function collectServicePlans(configs, configMap, typeValues, suiteSelectors, opts, execution, reporter) {
   return configs.map((config) => {
-    console.log(`\n══ ${config.name} ══`);
     const suites = applyShard(
       collectSuites(config, typeValues, suiteSelectors, opts.fileNames || [], opts),
       opts.shard
     );
 
     if (suites.length === 0) {
-      console.log(
-        `No test files for ${config.name} types=${typeValues.join(",") || "all"} suites=${suiteSelectors.map((selector) => selector.raw).join(",") || "all"} files=${(opts.fileNames || []).join(",") || "all"} — skipping`
+      reporter?.serviceSkipped?.(
+        config,
+        `no matching files (types=${typeValues.join(",") || "all"} suites=${suiteSelectors.map((selector) => selector.raw).join(",") || "all"} files=${(opts.fileNames || []).join(",") || "all"})`
       );
       return {
         config,
@@ -277,6 +294,17 @@ function collectServicePlans(configs, configMap, typeValues, suiteSelectors, opt
       };
     }
 
+    for (const suite of suites) {
+      for (const skippedFile of suite.skippedFiles || []) {
+        reporter?.plannedSkip?.({
+          serviceName: config.name,
+          type: suite.displayType || suite.type,
+          file: skippedFile.path,
+          reason: skippedFile.reason,
+        });
+      }
+    }
+
     const runtimeConfigs = resolveRuntimeConfigs(config, configMap);
     return {
       config,
@@ -290,30 +318,24 @@ function collectServicePlans(configs, configMap, typeValues, suiteSelectors, opt
   });
 }
 
-function printRunSummary(results, durationMs, knownFailureIssueValidation = null) {
-  for (const line of buildRunSummaryLines(results, durationMs, knownFailureIssueValidation)) {
-    console.log(line);
-  }
-}
-
-async function reportTelemetry(telemetry, artifact) {
+async function reportTelemetry(telemetry, artifact, reporter = null) {
   if (!telemetry?.enabled) return;
 
   try {
     const outcome = await uploadTelemetryArtifact(telemetry, artifact);
     if (outcome?.ok) {
-      console.log("Telemetry: uploaded run artifact");
+      reporter?.telemetry?.("Telemetry: uploaded run artifact");
       return;
     }
     if (outcome?.reason === "missing-token") {
-      console.log(
+      reporter?.telemetry?.(
         `Telemetry: skipped upload because ${telemetry.tokenEnv || "configured token env"} is not set`
       );
       return;
     }
     if (outcome?.reason && !outcome.skipped) return;
   } catch (error) {
-    console.log(`Telemetry: upload failed (${formatError(error)})`);
+    reporter?.telemetry?.(`Telemetry: upload failed (${formatError(error)})`);
   }
 }
 

package/lib/runner/playwright-runner.mjs

@@ -3,14 +3,14 @@ import { execa } from "execa";
 import { parsePlaywrightJsonResults } from "../reporters/playwright.mjs";
 import { resolveServiceCwd } from "../config/index.mjs";
 import { formatFileTimeoutBudgetError } from "../shared/file-timeout.mjs";
+import { persistTaskOutputArtifacts } from "./artifacts.mjs";
 import { settleSubprocess } from "./default-runtime-runner.mjs";
 import { ensurePlaywrightTestConfig } from "./playwright-config.mjs";
-import { printBufferedOutput } from "./processes.mjs";
 import { normalizePathSeparators } from "./state.mjs";
 import { buildPlaywrightEnv } from "./template.mjs";
 import { killChildProcess } from "./processes.mjs";
 
-export async function runPlaywrightTask(targetConfig, task, lifecycle, lease) {
+export async function runPlaywrightTask(targetConfig, task, lifecycle, lease, reporter = null) {
   const local = targetConfig.testkit.local;
   if (!local?.baseUrl) {
     throw new Error(
@@ -44,7 +44,7 @@ export async function runPlaywrightTask(targetConfig, task, lifecycle, lease) {
     if (subprocess.pid) interruptSubprocess();
     else subprocess.once?.("spawn", interruptSubprocess);
   }
-  console.log(`\n── ${targetConfig.runtimeLabel} playwright:${targetConfig.name} (${task.file}) ──`);
+  reporter?.taskStarted?.(task, targetConfig);
   let result;
   let timedOut;
   try {
@@ -53,15 +53,22 @@ export async function runPlaywrightTask(targetConfig, task, lifecycle, lease) {
     lifecycle.unregisterProcess(subprocess.pid);
   }
 
-  if (result.stderr) {
-    printBufferedOutput(result.stderr, `[${targetConfig.runtimeLabel}:${targetConfig.name}:playwright]`);
-  }
-
   const parsed = parsePlaywrightJsonResults(result.stdout || "", cwd);
   const finishedAt = Date.now();
   const durationMs = finishedAt - startedAt;
   const relativeFile = normalizePathSeparators(requestedFile);
   const fileResult = parsed.fileResults.get(relativeFile);
+  const outputArtifacts = persistTaskOutputArtifacts(targetConfig.productDir, task, [
+    result.stderr
+      ? {
+          name: "playwright-stderr",
+          kind: "runtime.output",
+          summary: result.stderr.split(/\r?\n/).map((line) => line.trim()).find(Boolean) || "captured stderr",
+          stream: "stderr",
+          text: result.stderr,
+        }
+      : null,
+  ]);
   const genericError = timedOut
     ? formatFileTimeoutBudgetError(fileTimeoutSeconds)
     : result.exitCode === 0
@@ -76,6 +83,7 @@ export async function runPlaywrightTask(targetConfig, task, lifecycle, lease) {
     durationMs: fileResult?.durationMs > 0 ? fileResult.durationMs : durationMs,
     startedAt,
     finishedAt,
+    artifacts: outputArtifacts,
     failureDetails: timedOut ? [] : fileResult?.failureDetails || [],
   };
 }

package/lib/runner/processes.mjs

@@ -55,9 +55,12 @@ export function killChildProcess(child, signal) {
   }
 }
 
-export function pipeOutput(stream, prefix) {
+export function captureOutput(stream, options = {}) {
   if (!stream) return Promise.resolve();
 
+  const onLine = typeof options.onLine === "function" ? options.onLine : null;
+  const liveWriter = typeof options.liveWriter === "function" ? options.liveWriter : null;
+  const livePrefix = options.livePrefix || "";
   let pending = "";
   return new Promise((resolve) => {
     let settled = false;
@@ -72,12 +75,15 @@ export function pipeOutput(stream, prefix) {
       const lines = pending.split(/\r?\n/);
       pending = lines.pop() || "";
       for (const line of lines) {
-        if (line.length > 0) console.log(`${prefix} ${line}`);
+        if (line.length === 0) continue;
+        onLine?.(line);
+        if (liveWriter) liveWriter(livePrefix ? `${livePrefix} ${line}` : line);
       }
     });
     stream.on("end", () => {
       if (pending.length > 0) {
-        console.log(`${prefix} ${pending}`);
+        onLine?.(pending);
+        if (liveWriter) liveWriter(livePrefix ? `${livePrefix} ${pending}` : pending);
       }
       settle();
     });
@@ -86,14 +92,6 @@ export function pipeOutput(stream, prefix) {
   });
 }
 
-export function printBufferedOutput(output, prefix) {
-  for (const line of output.split(/\r?\n/)) {
-    if (line.trim().length > 0) {
-      console.log(`${prefix} ${line}`);
-    }
-  }
-}
-
 export async function stopChildProcess(child, outputDrains = []) {
   if (!child) return;
   if (child.exitCode !== null) {

package/lib/runner/reporting.mjs

@@ -111,6 +111,7 @@ export function buildRunArtifact({
   serviceFilter,
   metadata,
   summarizeDbBackend,
+  serviceLogs = [],
 }) {
   const executed = results.filter((result) => !result.skipped);
   const effectivelySkippedServices = executed.filter(isEffectivelySkippedService);
@@ -127,7 +128,7 @@ export function buildRunArtifact({
   const dbBackend = summarizeDbBackend(results);
 
   return {
-    schemaVersion: 6,
+    schemaVersion: 7,
     source: "testkit",
     generatedAt: new Date(finishedAt).toISOString(),
     product: {
@@ -176,6 +177,9 @@ export function buildRunArtifact({
         notRun: notRunFiles,
       },
     },
+    logs: {
+      services: serviceLogs,
+    },
     services: results.map((result) => ({
       name: result.name,
       failed: result.failed,

package/lib/runner/reporting.test.mjs

@@ -78,7 +78,7 @@ describe("runner reporting", () => {
     });
 
     expect(artifact.product.name).toBe("my-product");
-    expect(artifact.schemaVersion).toBe(6);
+    expect(artifact.schemaVersion).toBe(7);
     expect(artifact.run).toMatchObject({
       workers: 2,
       fileTimeoutSeconds: 60,
@@ -107,6 +107,9 @@ describe("runner reporting", () => {
     });
     expect(artifact.services[0].durationMs).toBe(1200);
     expect(artifact.services[0].totalTaskDurationMs).toBe(2400);
+    expect(artifact.logs).toEqual({
+      services: [],
+    });
   });
 
   it("builds deterministic status artifacts", () => {

package/lib/runner/runtime-contexts.mjs

@@ -35,12 +35,12 @@ export function createRuntimeInstanceContext(runtimeId, graph, productDir) {
   };
 }
 
-export async function ensureRuntimeInstanceReady(context, task, lifecycle) {
+export async function ensureRuntimeInstanceReady(context, task, lifecycle, options = {}) {
   if (!context.prepared) {
     if (!context.preparationPromise) {
       context.preparationPromise = (async () => {
         await prepareDatabases(context.runtimeConfigs);
-        await prepareRuntimeServices(context.runtimeConfigs);
+        await prepareRuntimeServices(context.runtimeConfigs, options);
         context.prepared = true;
       })().finally(() => {
         context.preparationPromise = null;
@@ -52,7 +52,11 @@ export async function ensureRuntimeInstanceReady(context, task, lifecycle) {
   if (taskNeedsLocalRuntime(task) && !context.started) {
     if (!context.startupPromise) {
       context.startupPromise = (async () => {
-        context.startedServices = await startLocalServices(context.runtimeConfigs, lifecycle);
+        context.startedServices = await startLocalServices(
+          context.runtimeConfigs,
+          lifecycle,
+          options
+        );
         context.started = true;
       })().finally(() => {
         context.startupPromise = null;

package/lib/runner/runtime-manager.mjs

@@ -8,7 +8,7 @@ import {
   ensureRuntimeInstanceReady,
 } from "./runtime-contexts.mjs";
 
-export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {} }) {
+export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {}, runtimeOptions = {} }) {
   const graphByKey = new Map(graphs.map((graph) => [graph.key, graph]));
   const pools = new Map();
   const locks = new Map();
@@ -19,6 +19,7 @@ export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {}
     createRuntimeInstanceContext,
     ensureRuntimeInstanceReady,
     sleep,
+    options: runtimeOptions,
     ...hooks,
   };
 
@@ -199,7 +200,12 @@ async function getReadyContext(slot, productDir, task, lifecycle, runtimeHooks)
     slot.contextPromise = Promise.resolve(slot.context);
   }
   await slot.contextPromise;
-  await runtimeHooks.ensureRuntimeInstanceReady(slot.context, task, lifecycle);
+  await runtimeHooks.ensureRuntimeInstanceReady(
+    slot.context,
+    task,
+    lifecycle,
+    runtimeHooks.options || {}
+  );
   return slot.context;
 }
 

package/lib/runner/runtime-preparation.mjs

@@ -13,13 +13,13 @@ import {
 
 const MANIFEST_FILE = "prepare-manifest.json";
 
-export async function prepareRuntimeServices(runtimeConfigs) {
+export async function prepareRuntimeServices(runtimeConfigs, options = {}) {
   for (const config of runtimeConfigs) {
-    await prepareRuntimeService(config);
+    await prepareRuntimeService(config, options);
   }
 }
 
-export async function prepareRuntimeService(config) {
+export async function prepareRuntimeService(config, options = {}) {
   const prepare = config.testkit.runtime.prepare;
   if (!prepare || prepare.steps.length === 0) return;
 
@@ -43,12 +43,17 @@ export async function prepareRuntimeService(config) {
   }
 
   try {
-    await announceResolvedToolchain(config, await resolveConfiguredToolchain(config));
+    await announceResolvedToolchain(
+      config,
+      await resolveConfiguredToolchain(config),
+      options.reporter
+    );
     await runConfiguredSteps({
       config,
       steps: prepare.steps,
       env,
       labelPrefix: "runtime:prepare",
+      reporter: options.reporter,
     });
     writePrepareManifest(manifestPath, {
       fingerprint,