@elench/testkit 0.1.53 → 0.1.55

package/lib/runner/default-runtime-runner.mjs

@@ -8,7 +8,10 @@ import {
   formatFileTimeoutBudgetError,
 } from "../shared/file-timeout.mjs";
 import { persistTaskArtifacts, persistTaskOutputArtifacts } from "./artifacts.mjs";
-import { determineDefaultRuntimeFailure } from "./default-runtime-errors.mjs";
+import {
+  determineDefaultRuntimeFailure,
+  extractDefaultRuntimeFatalDetail,
+} from "./default-runtime-errors.mjs";
 import { collectFailureDetailsFromRuntimeArtifacts } from "./failure-details.mjs";
 import { RUNTIME_ARTIFACT_MARKER } from "../runtime-src/k6/artifacts.js";
 import { readDatabaseUrl } from "./state-io.mjs";
@@ -147,6 +150,10 @@ export async function runDefaultRuntimeTask(
       : null,
   ]);
   const failureDetails = collectFailureDetailsFromRuntimeArtifacts(rawRuntimeArtifacts);
+  const fatalRuntimeDetail = extractDefaultRuntimeFatalDetail(result.stderr || "", getFirstLine);
+  if (fatalRuntimeDetail && !failureDetails.some((detail) => detail?.kind === "runtime-exception")) {
+    failureDetails.unshift(fatalRuntimeDetail);
+  }
   const runtimeError = timedOut
     ? formatFileTimeoutBudgetError(fileTimeoutSeconds)
     : determineDefaultRuntimeFailure(result, summary, getFirstLine);

package/lib/runner/failure-details.mjs

@@ -28,6 +28,24 @@ export function normalizeFailureDetail(detail) {
   const message = normalizeNonEmptyString(detail.message);
   if (message) normalized.message = message;
 
+  if (detail.expected !== undefined) normalized.expected = cloneJsonValue(detail.expected);
+  if (detail.actual !== undefined) normalized.actual = cloneJsonValue(detail.actual);
+
+  const traceId = normalizeNonEmptyString(detail.traceId);
+  if (traceId) normalized.traceId = traceId;
+
+  const request = normalizeRecord(detail.request);
+  if (request) normalized.request = request;
+
+  const response = normalizeRecord(detail.response);
+  if (response) normalized.response = response;
+
+  const location = normalizeRecord(detail.location);
+  if (location) normalized.location = location;
+
+  const stack = normalizeNonEmptyString(detail.stack);
+  if (stack) normalized.stack = stack;
+
   return normalized;
 }
 
@@ -89,3 +107,16 @@ function normalizePositiveInteger(value) {
   if (!Number.isInteger(value) || value <= 0) return null;
   return value;
 }
+
+function normalizeRecord(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+  return cloneJsonValue(value);
+}
+
+function cloneJsonValue(value) {
+  try {
+    return JSON.parse(JSON.stringify(value));
+  } catch {
+    return String(value);
+  }
+}

package/lib/runner/failure-details.test.mjs

@@ -60,4 +60,55 @@ describe("runner failure details", () => {
       },
     ]);
   });
+
+  it("preserves rich assertion metadata", () => {
+    expect(
+      mergeFailureDetails([
+        {
+          kind: "http-assertion",
+          key: "GET /health > status is 200",
+          title: "status is 200",
+          expected: 200,
+          actual: 404,
+          request: {
+            method: "GET",
+            path: "/health",
+            requestId: "req-1",
+          },
+          response: {
+            status: 404,
+            bodyPreview: '{"error":"nope"}',
+          },
+          location: {
+            path: "/tmp/example.ts",
+            line: 12,
+            column: 4,
+          },
+        },
+      ])
+    ).toEqual([
+      {
+        kind: "http-assertion",
+        key: "GET /health > status is 200",
+        title: "status is 200",
+        count: 1,
+        expected: 200,
+        actual: 404,
+        request: {
+          method: "GET",
+          path: "/health",
+          requestId: "req-1",
+        },
+        response: {
+          status: 404,
+          bodyPreview: '{"error":"nope"}',
+        },
+        location: {
+          path: "/tmp/example.ts",
+          line: 12,
+          column: 4,
+        },
+      },
+    ]);
+  });
 });

package/lib/runner/formatting.mjs

@@ -69,7 +69,7 @@ export function buildCompactRunSummaryLines(
     for (const failure of failures) {
       lines.push(`  ${failure.file.path}`);
       lines.push(`    ${failure.primaryMessage}`);
-      for (const detail of failure.extraDetails.slice(0, 2)) {
+      for (const detail of failure.extraLines.slice(0, 3)) {
         lines.push(`    ${detail}`);
       }
     }
@@ -209,14 +209,33 @@ function collectFailedFiles(results) {
     for (const suite of result.suites || []) {
       for (const file of suite.files || []) {
         if (file.status !== "failed") continue;
-        const detailMessages = (file.failureDetails || [])
+        const rankedDetails = rankFailureDetails(file.failureDetails || []);
+        const primaryDetail = rankedDetails[0] || null;
+        const fallbackMessages = rankedDetails
           .map((detail) => detail.message || detail.title)
           .filter(Boolean)
           .map((message) => sanitizeErrorMessage(String(message).trim()));
+        const extraLines = [];
+        if (primaryDetail) {
+          const responseLine = formatFailureResponsePreview(primaryDetail);
+          if (responseLine) extraLines.push(responseLine);
+          const triageLine = formatTriageLine(file.triage || null);
+          if (triageLine) extraLines.push(triageLine);
+          const requestLine = formatFailureRequestHint(primaryDetail);
+          if (requestLine) extraLines.push(requestLine);
+        } else {
+          const triageLine = formatTriageLine(file.triage || null);
+          if (triageLine) extraLines.push(triageLine);
+        }
+
+        for (const detail of rankedDetails.slice(primaryDetail ? 1 : 0)) {
+          const line = sanitizeErrorMessage(String(detail.message || detail.title || "").trim());
+          if (line && !extraLines.includes(line)) extraLines.push(line);
+        }
         failures.push({
           file,
-          primaryMessage: sanitizeErrorMessage(file.error || detailMessages[0] || suite.error || "Failed"),
-          extraDetails: detailMessages.slice(file.error ? 0 : 1),
+          primaryMessage: resolvePrimaryFailureMessage(file, suite, primaryDetail, fallbackMessages),
+          extraLines,
         });
       }
     }
@@ -224,6 +243,97 @@ function collectFailedFiles(results) {
   return failures.sort((left, right) => left.file.path.localeCompare(right.file.path));
 }
 
+function resolvePrimaryFailureMessage(file, suite, primaryDetail, fallbackMessages) {
+  if (primaryDetail?.message) {
+    return sanitizeErrorMessage(String(primaryDetail.message).trim());
+  }
+
+  const fileError = sanitizeErrorMessage(file.error || "");
+  if (fileError && !isThresholdWrapperMessage(fileError)) {
+    return fileError;
+  }
+
+  if (fallbackMessages.length > 0) {
+    return fallbackMessages[0];
+  }
+
+  if (fileError) {
+    return fileError;
+  }
+
+  return sanitizeErrorMessage(suite.error || "Failed");
+}
+
+function rankFailureDetails(details) {
+  return [...(Array.isArray(details) ? details : [])].sort((left, right) => {
+    return failureDetailRank(left) - failureDetailRank(right) || String(left?.key || "").localeCompare(String(right?.key || ""));
+  });
+}
+
+function failureDetailRank(detail) {
+  if (detail?.kind === "http-assertion") return 1;
+  if (detail?.request && detail?.response) return 2;
+  if (detail?.location || detail?.stack) return 3;
+  if (detail?.message) return 4;
+  return 5;
+}
+
+function formatFailureResponsePreview(detail) {
+  const bodyPreview = detail?.response?.bodyPreview;
+  if (!bodyPreview) return null;
+  return `response: ${sanitizeInline(bodyPreview, 220)}`;
+}
+
+function formatFailureRequestHint(detail) {
+  const method = detail?.request?.method;
+  const path = detail?.request?.path;
+  if (!method || !path) return null;
+  const requestId = detail?.request?.requestId;
+  if (requestId) {
+    return `logs: requestId=${requestId}`;
+  }
+  return `request: ${method} ${path}`;
+}
+
+function formatTriageLine(triage) {
+  if (!triage) return null;
+  if (triage.status === "untriaged") return "triage: untriaged";
+
+  const entry = triage.entries?.[0];
+  if (!entry?.issue) {
+    if (triage.status === "validation_unavailable") {
+      const reason = triage.availability?.reason ? ` (${triage.availability.reason})` : "";
+      return `triage: validation unavailable${reason}`;
+    }
+    return null;
+  }
+
+  const issueLabel = `#${entry.issue.number}`;
+  const validationStatus = entry.validationStatus || null;
+  if (validationStatus === "closed_but_failing") {
+    return `triage: known issue ${issueLabel} closed but still failing`;
+  }
+  if (validationStatus === "validation_unavailable") {
+    const reason = triage.availability?.mode === "cache" ? "cache" : "validation unavailable";
+    return `triage: known issue ${issueLabel} (${reason})`;
+  }
+  if (entry.github?.state === "open" || entry.state === "open") {
+    return `triage: known issue ${issueLabel} open`;
+  }
+  if (entry.github?.state === "closed" || entry.state === "closed") {
+    return `triage: known issue ${issueLabel} closed`;
+  }
+  return `triage: known issue ${issueLabel}`;
+}
+
+function isThresholdWrapperMessage(message) {
+  return /Default runtime thresholds failed:/.test(String(message || ""));
+}
+
+function sanitizeInline(message, maxLength = 180) {
+  return String(message || "").replace(/\s+/g, " ").trim().slice(0, maxLength);
+}
+
 function collectServiceErrors(results) {
   const items = [];
   for (const result of results) {

package/lib/runner/formatting.test.mjs

@@ -162,4 +162,81 @@ describe("runner formatting", () => {
     expect(lines.join("\n")).toContain("2 closed issues still failing");
     expect(lines.join("\n")).toContain("1 title mismatch");
   });
+
+  it("prefers structured HTTP assertion details over threshold wrapper text", () => {
+    const lines = buildRunSummaryLines(
+      [
+        {
+          name: "api",
+          skipped: false,
+          failed: true,
+          suiteCount: 1,
+          completedSuiteCount: 1,
+          skippedSuiteCount: 0,
+          failedSuiteCount: 1,
+          totalFileCount: 1,
+          passedFileCount: 0,
+          failedFileCount: 1,
+          skippedFileCount: 0,
+          notRunFileCount: 0,
+          durationMs: 500,
+          suites: [
+            {
+              failed: true,
+              type: "int",
+              name: "default",
+              framework: "k6",
+              failedFiles: ["__testkit__/health/health.int.testkit.ts"],
+              durationMs: 500,
+              files: [
+                {
+                  path: "__testkit__/health/health.int.testkit.ts",
+                  status: "failed",
+                  error: "Default runtime thresholds failed: checks(rate==1.0)",
+                  failureDetails: [
+                    {
+                      kind: "http-assertion",
+                      key: "GET /health > status is 200",
+                      title: "status is 200",
+                      message: "GET /health expected 200, got 404",
+                      request: {
+                        method: "GET",
+                        path: "/health",
+                        requestId: "req-1",
+                      },
+                      response: {
+                        status: 404,
+                        bodyPreview: '{"error":"nope"}',
+                      },
+                    },
+                  ],
+                  triage: {
+                    status: "known_failure",
+                    entries: [
+                      {
+                        id: "health-is-bad",
+                        state: "open",
+                        issue: {
+                          repo: "acme/example",
+                          number: 42,
+                        },
+                      },
+                    ],
+                  },
+                },
+              ],
+              error: null,
+            },
+          ],
+          errors: [],
+        },
+      ],
+      500
+    );
+
+    expect(lines.join("\n")).toContain("GET /health expected 200, got 404");
+    expect(lines.join("\n")).toContain('response: {"error":"nope"}');
+    expect(lines.join("\n")).toContain("triage: known issue #42 open");
+    expect(lines.join("\n")).not.toContain("Default runtime thresholds failed: checks(rate==1.0)\n    response:");
+  });
 });

package/lib/runner/logs.mjs

@@ -2,21 +2,26 @@ import fs from "fs";
 import path from "path";
 
 const RESULT_LOGS_DIRNAME = "logs";
+const RESULT_SETUP_DIRNAME = "setup";
 
 export function createRunLogRegistry(productDir) {
-  const records = new Map();
+  const serviceRecords = new Map();
+  const setupRecords = new Map();
 
   return {
     ensureServiceLogRecord(config) {
       const key = `${config.runtimeLabel || config.name}:${config.name}`;
-      const existing = records.get(key);
+      const existing = serviceRecords.get(key);
       if (existing) return existing;
 
       const fileName = `${sanitizePathSegment(config.runtimeLabel || config.name)}__${sanitizePathSegment(config.name)}.log`;
       const relativePath = path.join(".testkit", "results", RESULT_LOGS_DIRNAME, fileName);
       const absolutePath = path.join(productDir, relativePath);
       fs.mkdirSync(path.dirname(absolutePath), { recursive: true });
-      const stream = fs.createWriteStream(absolutePath, { flags: "a" });
+      const stream = fs.createWriteStream(absolutePath, {
+        fd: fs.openSync(absolutePath, "a"),
+        flags: "a",
+      });
       const record = {
         key,
         serviceName: config.name,
@@ -25,7 +30,32 @@ export function createRunLogRegistry(productDir) {
         absolutePath,
         stream,
       };
-      records.set(key, record);
+      serviceRecords.set(key, record);
+      return record;
+    },
+    ensureSetupLogRecord(config, stage) {
+      const key = `${config.runtimeLabel || config.name}:${config.name}:${stage}`;
+      const existing = setupRecords.get(key);
+      if (existing) return existing;
+
+      const fileName = `${sanitizePathSegment(config.runtimeLabel || config.name)}__${sanitizePathSegment(config.name)}__${sanitizePathSegment(stage)}.log`;
+      const relativePath = path.join(".testkit", "results", RESULT_SETUP_DIRNAME, fileName);
+      const absolutePath = path.join(productDir, relativePath);
+      fs.mkdirSync(path.dirname(absolutePath), { recursive: true });
+      const stream = fs.createWriteStream(absolutePath, {
+        fd: fs.openSync(absolutePath, "a"),
+        flags: "a",
+      });
+      const record = {
+        key,
+        serviceName: config.name,
+        runtimeLabel: config.runtimeLabel || config.name,
+        stage,
+        path: normalizePath(relativePath),
+        absolutePath,
+        stream,
+      };
+      setupRecords.set(key, record);
       return record;
     },
     append(record, streamName, line) {
@@ -33,7 +63,7 @@ export function createRunLogRegistry(productDir) {
       record.stream.write(`${new Date().toISOString()} [${streamName}] ${line}\n`);
     },
     listServiceLogs() {
-      return [...records.values()]
+      return [...serviceRecords.values()]
         .map((record) => ({
           serviceName: record.serviceName,
           runtimeLabel: record.runtimeLabel,
@@ -45,8 +75,26 @@ export function createRunLogRegistry(productDir) {
             left.runtimeLabel.localeCompare(right.runtimeLabel)
         );
     },
+    listSetupLogs() {
+      return [...setupRecords.values()]
+        .map((record) => ({
+          serviceName: record.serviceName,
+          runtimeLabel: record.runtimeLabel,
+          stage: record.stage,
+          path: record.path,
+        }))
+        .sort(
+          (left, right) =>
+            left.serviceName.localeCompare(right.serviceName) ||
+            left.runtimeLabel.localeCompare(right.runtimeLabel) ||
+            left.stage.localeCompare(right.stage)
+        );
+    },
     closeAll() {
-      for (const record of records.values()) {
+      for (const record of serviceRecords.values()) {
+        record.stream.end();
+      }
+      for (const record of setupRecords.values()) {
         record.stream.end();
       }
     },
@@ -59,6 +107,23 @@ export function readLogTail(absolutePath, lineCount = 80) {
   return lines.slice(Math.max(0, lines.length - lineCount));
 }
 
+export function findLogSliceByRequestId(absolutePath, requestId, contextLines = 2) {
+  if (!absolutePath || !requestId || !fs.existsSync(absolutePath)) return [];
+  const lines = fs.readFileSync(absolutePath, "utf8").split(/\r?\n/).filter(Boolean);
+  const matches = [];
+
+  for (let index = 0; index < lines.length; index += 1) {
+    if (!lines[index].includes(requestId)) continue;
+    const start = Math.max(0, index - contextLines);
+    const end = Math.min(lines.length, index + contextLines + 1);
+    for (let cursor = start; cursor < end; cursor += 1) {
+      if (!matches.includes(lines[cursor])) matches.push(lines[cursor]);
+    }
+  }
+
+  return matches;
+}
+
 function sanitizePathSegment(value) {
   return String(value)
     .trim()

package/lib/runner/orchestrator.mjs

@@ -14,8 +14,12 @@ import {
   recordTaskOutcome,
   summarizeDbBackend,
 } from "./results.mjs";
-import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
-import { applyKnownFailuresToArtifacts, loadKnownFailuresConfig } from "./triage.mjs";
+import { buildLiveRunArtifact, buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
+import {
+  applyKnownFailureIssueValidationToArtifacts,
+  applyKnownFailuresToArtifacts,
+  loadKnownFailuresConfig,
+} from "./triage.mjs";
 import { formatError } from "./formatting.mjs";
 import {
   shouldFailKnownFailureIssueValidation,
@@ -25,10 +29,12 @@ import {
   loadTimings,
   resetResultArtifacts,
   saveTimings,
+  writeLiveRunArtifact,
   writeRunArtifact,
   writeStatusArtifact,
 } from "./artifacts.mjs";
 import { createRunLogRegistry } from "./logs.mjs";
+import { createSetupOperationRegistry } from "./setup-operations.mjs";
 import {
   cleanupRunById,
   cleanupRuns,
@@ -68,6 +74,9 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
   );
   const reporter = opts.reporter || null;
   const logRegistry = createRunLogRegistry(productDir);
+  let workerCount = 0;
+  let runtimeInstanceCount = 0;
+  let runtimeStats = [];
   const requestedFiles = opts.fileNames || [];
   if (requestedFiles.length > 0) {
     const unmatchedFiles = findUnmatchedRequestedFiles(
@@ -116,10 +125,40 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     reporter
   );
   const trackers = buildServiceTrackers(servicePlans, startedAt);
+  const writeLiveSnapshot = () => {
+    const now = Date.now();
+    const partialResults = configs.map((config) =>
+      finalizeServiceResult(trackers.get(config.name), startedAt, now)
+    );
+    writeLiveRunArtifact(
+      productDir,
+      buildLiveRunArtifact({
+        productDir,
+        results: partialResults,
+        startedAt,
+        updatedAt: now,
+        execution,
+        workerCount,
+        runtimeInstanceCount,
+        runtimeStats,
+        typeValues,
+        suiteSelectors,
+        fileNames: requestedFiles,
+        shard: opts.shard || null,
+        serviceFilter: opts.serviceFilter || null,
+        metadata,
+        summarizeDbBackend,
+        serviceLogs: logRegistry.listServiceLogs(),
+        setupLogs: logRegistry.listSetupLogs(),
+        setupOperations: setupRegistry.listOperations(),
+      })
+    );
+  };
+  const setupRegistry = createSetupOperationRegistry({
+    logRegistry,
+    onChange: writeLiveSnapshot,
+  });
   const executedPlans = servicePlans.filter((plan) => !plan.skipped);
-  let workerCount = 0;
-  let runtimeInstanceCount = 0;
-  let runtimeStats = [];
   let exitCode = 0;
   const lifecycle = createRunLifecycle(productDir);
   lifecycle.markRunning();
@@ -127,6 +166,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
   let results = [];
   let finishedAt = Date.now();
   let knownFailureIssueValidation = null;
+  writeLiveSnapshot();
 
   try {
     if (executedPlans.length > 0) {
@@ -145,6 +185,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
         runtimeOptions: {
           reporter,
           logRegistry,
+          setupRegistry,
         },
       });
       const timingUpdates = [];
@@ -160,8 +201,14 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
               timingUpdates,
               lifecycle,
               claimNextTask,
-              recordTaskOutcome,
-              recordGraphError,
+              (allTrackers, task, outcome, now) => {
+                recordTaskOutcome(allTrackers, task, outcome, now);
+                writeLiveSnapshot();
+              },
+              (allTrackers, graph, message, now) => {
+                recordGraphError(allTrackers, graph, message, now);
+                writeLiveSnapshot();
+              },
               reporter
             )
           )
@@ -173,9 +220,11 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
             for (const tracker of trackers.values()) {
               if (!tracker.skipped) addTrackerError(tracker, message);
             }
+            writeLiveSnapshot();
           }
         }
         runtimeStats = runtimeManager.getStats();
+        writeLiveSnapshot();
       } finally {
         await runtimeManager.cleanupAll();
       }
@@ -204,6 +253,8 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
       metadata,
       summarizeDbBackend,
       serviceLogs: logRegistry.listServiceLogs(),
+      setupLogs: logRegistry.listSetupLogs(),
+      setupOperations: setupRegistry.listOperations(),
     });
     const statusArtifact = opts.writeStatus
       ? buildStatusArtifact({
@@ -230,6 +281,11 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
       config: configs[0]?.testkit?.reporting?.issueValidation || null,
       gitMetadata: metadata.git,
     });
+    applyKnownFailureIssueValidationToArtifacts(
+      enrichedArtifacts.runArtifact,
+      enrichedArtifacts.statusArtifact,
+      knownFailureIssueValidation
+    );
     attachKnownFailureIssueValidation(
       enrichedArtifacts.runArtifact,
       enrichedArtifacts.statusArtifact,