@elench/testkit 0.1.79 → 0.1.81

package/lib/{known-failures → regressions}/index.test.mjs

@@ -3,10 +3,10 @@ import os from "os";
 import path from "path";
 import { afterEach, describe, expect, it } from "vitest";
 import {
-  findMatchingKnownFailureEntries,
-  normalizeKnownFailuresDocument,
-  renderKnownFailuresMarkdown,
-  validateKnownFailuresDocument,
+  findMatchingRegressionEntries,
+  normalizeRegressionCatalogDocument,
+  renderRegressionCatalogMarkdown,
+  validateRegressionCatalogDocument,
 } from "./index.mjs";
 
 const tempDirs = [];
@@ -17,25 +17,22 @@ afterEach(() => {
   }
 });
 
-describe("known failures core", () => {
+describe("regression catalog", () => {
   it("matches entries by service, type, path, and failure key", () => {
-    const document = normalizeKnownFailuresDocument({
+    const document = normalizeRegressionCatalogDocument({
       schemaVersion: 1,
       entries: [
         {
           id: "bad-message",
-          title: "Bad message bug",
           classification: "product_bug",
-          state: "open",
           issue: {
             repo: "acme/repo",
             number: 12,
-            url: "https://github.com/acme/repo/issues/12",
           },
-          description: "Wrong message",
-          whyFailing: "Payload is wrong",
-          lastReviewedAt: "2026-04-27",
-          matches: [
+          summary: "API returns the wrong message",
+          cause: "The payload is wrong.",
+          lastReviewedAt: "2026-05-04",
+          fingerprints: [
             {
               service: "api",
               type: "int",
@@ -47,7 +44,7 @@ describe("known failures core", () => {
       ],
     });
 
-    const matches = findMatchingKnownFailureEntries(document, {
+    const matches = findMatchingRegressionEntries(document, {
       service: "api",
       type: "int",
       path: "src/api/routes/__testkit__/failing.int.testkit.ts",
@@ -59,24 +56,21 @@ describe("known failures core", () => {
     expect(matches[0].id).toBe("bad-message");
   });
 
-  it("matches failureKey against a detail title when the key becomes richer", () => {
-    const document = normalizeKnownFailuresDocument({
+  it("matches fingerprint failureKey against detail title", () => {
+    const document = normalizeRegressionCatalogDocument({
       schemaVersion: 1,
       entries: [
         {
           id: "missing-route",
-          title: "Missing route bug",
           classification: "product_bug",
-          state: "open",
           issue: {
             repo: "acme/repo",
             number: 13,
-            url: "https://github.com/acme/repo/issues/13",
           },
-          description: "Wrong status code",
-          whyFailing: "The route returns 404",
-          lastReviewedAt: "2026-04-28",
-          matches: [
+          summary: "Missing route returns the wrong status",
+          cause: "The route returns 404.",
+          lastReviewedAt: "2026-05-04",
+          fingerprints: [
             {
               service: "api",
               type: "int",
@@ -88,7 +82,7 @@ describe("known failures core", () => {
       ],
     });
 
-    const matches = findMatchingKnownFailureEntries(document, {
+    const matches = findMatchingRegressionEntries(document, {
       service: "api",
       type: "int",
       path: "__testkit__/health/http-failure.int.testkit.ts",
@@ -105,31 +99,28 @@ describe("known failures core", () => {
     expect(matches[0].id).toBe("missing-route");
   });
 
-  it("validates status coverage and filesystem matches", () => {
-    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-known-failures-"));
+  it("validates status coverage and filesystem fingerprints", () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-regressions-"));
     tempDirs.push(tempDir);
     const testPath = path.join(tempDir, "src/api/routes/__testkit__/failing.int.testkit.ts");
     fs.mkdirSync(path.dirname(testPath), { recursive: true });
     fs.writeFileSync(testPath, "export default {};\n");
 
-    const document = normalizeKnownFailuresDocument({
+    const document = normalizeRegressionCatalogDocument({
       schemaVersion: 1,
       issueRepo: "acme/repo",
       entries: [
         {
           id: "bad-message",
-          title: "Bad message bug",
           classification: "product_bug",
-          state: "open",
           issue: {
             repo: "acme/repo",
             number: 12,
-            url: "https://github.com/acme/repo/issues/12",
           },
-          description: "Wrong message",
-          whyFailing: "Payload is wrong",
-          lastReviewedAt: "2026-04-27",
-          matches: [
+          summary: "API returns the wrong message",
+          cause: "The payload is wrong.",
+          lastReviewedAt: "2026-05-04",
+          fingerprints: [
             {
               service: "api",
               type: "int",
@@ -140,7 +131,7 @@ describe("known failures core", () => {
       ],
     });
 
-    const result = validateKnownFailuresDocument(document, {
+    const result = validateRegressionCatalogDocument(document, {
       productDir: tempDir,
       statusArtifact: {
         tests: [
@@ -162,37 +153,35 @@ describe("known failures core", () => {
 
     expect(result.errors).toEqual([]);
     expect(result.stats.failedTests).toBe(2);
-    expect(result.stats.triagedFailedTests).toBe(1);
+    expect(result.stats.diagnosedFailedTests).toBe(1);
     expect(result.warnings).toContain(
-      "Untriaged failed test: src/api/routes/__testkit__/other.int.testkit.ts"
+      "New failing test not yet in regression catalog: src/api/routes/__testkit__/other.int.testkit.ts"
     );
   });
 
   it("renders markdown from the canonical document", () => {
-    const markdown = renderKnownFailuresMarkdown(
-      normalizeKnownFailuresDocument({
+    const markdown = renderRegressionCatalogMarkdown(
+      normalizeRegressionCatalogDocument({
         schemaVersion: 1,
         entries: [
           {
             id: "bad-message",
-            title: "Bad message bug",
             classification: "product_bug",
-            state: "open",
             issue: {
               repo: "acme/repo",
               number: 12,
-              url: "https://github.com/acme/repo/issues/12",
             },
-            description: "Wrong message",
-            whyFailing: "Payload is wrong",
-            lastReviewedAt: "2026-04-27",
-            matches: [{ path: "src/api/routes/__testkit__/failing.int.testkit.ts" }],
+            summary: "API returns the wrong message",
+            cause: "The payload is wrong.",
+            lastReviewedAt: "2026-05-04",
+            fingerprints: [{ path: "src/api/routes/__testkit__/failing.int.testkit.ts" }],
           },
         ],
       })
     );
 
-    expect(markdown).toContain("# Bourne Bugs");
-    expect(markdown).toContain("[#12](https://github.com/acme/repo/issues/12)");
+    expect(markdown).toContain("# Regression Catalog");
+    expect(markdown).toContain("API returns the wrong message");
+    expect(markdown).toContain("#12");
   });
 });

package/lib/runner/formatting.mjs

@@ -1,4 +1,5 @@
-import { buildKnownFailureIssueValidationSummaryLines } from "../known-failures/github.mjs";
+import { findMatchingRegressionEntries } from "../regressions/index.mjs";
+import { buildRegressionSyncSummaryLines } from "../regressions/github.mjs";
 
 export function formatDuration(durationMs) {
   const totalSeconds = Math.max(0, Math.round(durationMs / 1000));
@@ -48,57 +49,75 @@ export function formatSuiteFramework(framework) {
   return label ? ` [${label}]` : "";
 }
 
-export function buildRunSummaryLines(results, durationMs, knownFailureIssueValidation = null) {
-  return buildCompactRunSummaryLines(results, durationMs, knownFailureIssueValidation);
+export function buildRunSummaryLines(results, durationMs, regressionReport = null) {
+  return buildCompactRunSummaryLines(results, durationMs, regressionReport);
+}
+
+export function buildRunSummaryData(results, durationMs, regressionReport = null) {
+  const totals = summarizeResults(results);
+  const serviceErrors = collectServiceErrors(results);
+  const regressionSummary = regressionReport?.summary || null;
+  return {
+    result: totals.failedServices > 0 ? "FAILED" : "PASSED",
+    totalServices: totals.totalServices,
+    failedServices: totals.failedServices,
+    passed: totals.passedFiles,
+    failed: totals.failedFiles,
+    skipped: totals.skippedFiles,
+    notRun: totals.notRunFiles,
+    files: totals.totalFiles,
+    duration: formatDuration(durationMs),
+    serviceErrors: serviceErrors.length,
+    newRegressions: regressionSummary?.newRegressions || 0,
+    knownRegressions: regressionSummary?.knownRegressions || 0,
+    fixedKnownRegressions: regressionSummary?.fixedKnownRegressions || 0,
+    catalogStale: regressionSummary?.catalogStale || 0,
+    catalogSyncUnavailable: Boolean(regressionSummary?.catalogSyncUnavailable),
+    usedStaleCache: Boolean(regressionSummary?.usedStaleCache),
+  };
 }
 
 export function buildCompactRunSummaryLines(
   results,
   durationMs,
-  knownFailureIssueValidation = null
+  regressionReport = null
 ) {
-  const totals = summarizeResults(results);
+  const summary = buildRunSummaryData(results, durationMs, regressionReport);
   const lines = [
     "",
-    `Summary: ${totals.passedFiles} passed, ${totals.failedFiles} failed, ${totals.skippedFiles} skipped, ${totals.notRunFiles} not run across ${totals.totalFiles} ${pluralize(totals.totalFiles, "file", "files")} in ${formatDuration(durationMs)}`,
+    `Summary: ${summary.passed} passed, ${summary.failed} failed, ${summary.skipped} skipped, ${summary.notRun} not run across ${summary.files} ${pluralize(summary.files, "file", "files")} in ${summary.duration}`,
   ];
 
-  const failures = collectFailedFiles(results);
-  if (failures.length > 0) {
-    lines.push("", "Failures:");
-    for (const failure of failures) {
-      lines.push(`  ${failure.file.path}`);
-      lines.push(`    ${failure.primaryMessage}`);
-      for (const detail of failure.extraLines.slice(0, 3)) {
-        lines.push(`    ${detail}`);
-      }
-    }
+  if (summary.serviceErrors > 0) {
+    lines.push(`Runtime errors: ${summary.serviceErrors}`);
   }
 
-  const serviceErrors = collectServiceErrors(results);
-  if (serviceErrors.length > 0) {
-    lines.push("", "Runtime Errors:");
-    for (const item of serviceErrors) {
-      lines.push(`  ${item.service}`);
-      lines.push(`    ${item.message}`);
-    }
+  if (summary.newRegressions > 0) {
+    lines.push(`New regressions: ${summary.newRegressions}`);
   }
-
-  const knownFailureIssueLines = buildKnownFailureIssueValidationSummaryLines(
-    knownFailureIssueValidation
-  );
-  if (knownFailureIssueLines.length > 0) {
-    lines.push(...knownFailureIssueLines);
+  if (summary.knownRegressions > 0) {
+    lines.push(`Known regressions: ${summary.knownRegressions}`);
+  }
+  if (summary.fixedKnownRegressions > 0) {
+    lines.push(`Fixed known regressions: ${summary.fixedKnownRegressions}`);
+  }
+  if (summary.catalogStale > 0) {
+    lines.push(`Catalog stale: ${summary.catalogStale}`);
+  }
+  if (summary.catalogSyncUnavailable) {
+    lines.push("Catalog sync unavailable");
   }
 
   lines.push("");
   lines.push(
-    totals.failedServices > 0 ? `Result: FAILED (${totals.failedServices}/${totals.totalServices} services failed)` : "Result: PASSED"
+    summary.result === "FAILED"
+      ? `Result: FAILED (${summary.failedServices}/${summary.totalServices} services failed)`
+      : "Result: PASSED"
   );
   return lines;
 }
 
-export function buildDebugRunSummaryLines(results, durationMs, knownFailureIssueValidation = null) {
+export function buildDebugRunSummaryLines(results, durationMs, regressionReport = null) {
   const totalServices = results.length;
   const executedServices = results.filter((result) => !result.skipped);
   const skippedServices = results.filter((result) => result.skipped);
@@ -167,11 +186,19 @@ export function buildDebugRunSummaryLines(results, durationMs, knownFailureIssue
     }
   }
 
-  const knownFailureIssueLines = buildKnownFailureIssueValidationSummaryLines(
-    knownFailureIssueValidation
+  const regressionSyncLines = buildRegressionSyncSummaryLines(
+    regressionReport?.catalog?.configured ? {
+      summary: {
+        byCode: {
+          closed_but_failing: regressionReport.summary.catalogStale,
+          validation_unavailable: regressionReport.summary.catalogSyncUnavailable ? 1 : 0,
+          used_stale_cache: regressionReport.summary.usedStaleCache ? 1 : 0,
+        },
+      },
+    } : null
   );
-  if (knownFailureIssueLines.length > 0) {
-    lines.push(...knownFailureIssueLines);
+  if (regressionSyncLines.length > 0) {
+    lines.push(...regressionSyncLines);
   }
 
   if (failedServices.length > 0) {
@@ -183,6 +210,43 @@ export function buildDebugRunSummaryLines(results, durationMs, knownFailureIssue
   return lines;
 }
 
+export function buildFailurePresentation(fileSummary, regressionCatalog = null) {
+  const rankedDetails = rankFailureDetails(fileSummary.failureDetails || []);
+  const primaryDetail = rankedDetails[0] || null;
+  const fallbackMessages = rankedDetails
+    .map((detail) => detail.message || detail.title)
+    .filter(Boolean)
+    .map((message) => sanitizeErrorMessage(String(message).trim()));
+  const details = [];
+
+  const responseLine = formatFailureResponsePreview(primaryDetail);
+  if (responseLine) details.push(responseLine);
+
+  const regressionLine = formatInlineRegressionLine(fileSummary, regressionCatalog);
+  if (regressionLine) details.push(regressionLine);
+
+  const requestLine = formatFailureRequestHint(primaryDetail);
+  if (requestLine) details.push(requestLine);
+
+  for (const detail of rankedDetails.slice(primaryDetail ? 1 : 0)) {
+    const line = sanitizeInline(String(detail.message || detail.title || ""), 220);
+    if (line && !details.includes(line)) details.push(line);
+  }
+
+  return {
+    primary: sanitizeInline(
+      resolvePrimaryFailureMessage(
+        fileSummary,
+        { error: fileSummary.suiteError || null },
+        primaryDetail,
+        fallbackMessages
+      ),
+      240
+    ),
+    details,
+  };
+}
+
 function sanitizeErrorMessage(message) {
   return message
     .replace(/Command failed with exit code (\d+): .*?[\\/]vendor[\\/]k6 run\b/g, "Default runtime failed with exit code $1:")
@@ -203,46 +267,6 @@ function summarizeResults(results) {
   };
 }
 
-function collectFailedFiles(results) {
-  const failures = [];
-  for (const result of results) {
-    for (const suite of result.suites || []) {
-      for (const file of suite.files || []) {
-        if (file.status !== "failed") continue;
-        const rankedDetails = rankFailureDetails(file.failureDetails || []);
-        const primaryDetail = rankedDetails[0] || null;
-        const fallbackMessages = rankedDetails
-          .map((detail) => detail.message || detail.title)
-          .filter(Boolean)
-          .map((message) => sanitizeErrorMessage(String(message).trim()));
-        const extraLines = [];
-        if (primaryDetail) {
-          const responseLine = formatFailureResponsePreview(primaryDetail);
-          if (responseLine) extraLines.push(responseLine);
-          const triageLine = formatTriageLine(file.triage || null);
-          if (triageLine) extraLines.push(triageLine);
-          const requestLine = formatFailureRequestHint(primaryDetail);
-          if (requestLine) extraLines.push(requestLine);
-        } else {
-          const triageLine = formatTriageLine(file.triage || null);
-          if (triageLine) extraLines.push(triageLine);
-        }
-
-        for (const detail of rankedDetails.slice(primaryDetail ? 1 : 0)) {
-          const line = sanitizeErrorMessage(String(detail.message || detail.title || "").trim());
-          if (line && !extraLines.includes(line)) extraLines.push(line);
-        }
-        failures.push({
-          file,
-          primaryMessage: resolvePrimaryFailureMessage(file, suite, primaryDetail, fallbackMessages),
-          extraLines,
-        });
-      }
-    }
-  }
-  return failures.sort((left, right) => left.file.path.localeCompare(right.file.path));
-}
-
 function resolvePrimaryFailureMessage(file, suite, primaryDetail, fallbackMessages) {
   if (primaryDetail?.message) {
     return sanitizeErrorMessage(String(primaryDetail.message).trim());
@@ -295,35 +319,13 @@ function formatFailureRequestHint(detail) {
   return `request: ${method} ${path}`;
 }
 
-function formatTriageLine(triage) {
-  if (!triage) return null;
-  if (triage.status === "untriaged") return "triage: untriaged";
-
-  const entry = triage.entries?.[0];
-  if (!entry?.issue) {
-    if (triage.status === "validation_unavailable") {
-      const reason = triage.availability?.reason ? ` (${triage.availability.reason})` : "";
-      return `triage: validation unavailable${reason}`;
-    }
-    return null;
-  }
+function formatInlineRegressionLine(fileSummary, regressionCatalog) {
+  if (!regressionCatalog) return "regression: new";
+  const matches = findMatchingRegressionEntries(regressionCatalog, fileSummary);
+  if (matches.length === 0) return "regression: new";
 
-  const issueLabel = `#${entry.issue.number}`;
-  const validationStatus = entry.validationStatus || null;
-  if (validationStatus === "closed_but_failing") {
-    return `triage: known issue ${issueLabel} closed but still failing`;
-  }
-  if (validationStatus === "validation_unavailable") {
-    const reason = triage.availability?.mode === "cache" ? "cache" : "validation unavailable";
-    return `triage: known issue ${issueLabel} (${reason})`;
-  }
-  if (entry.github?.state === "open" || entry.state === "open") {
-    return `triage: known issue ${issueLabel} open`;
-  }
-  if (entry.github?.state === "closed" || entry.state === "closed") {
-    return `triage: known issue ${issueLabel} closed`;
-  }
-  return `triage: known issue ${issueLabel}`;
+  const entry = matches[0];
+  return `regression: known #${entry.issue.number} ${entry.classification}`;
 }
 
 function isThresholdWrapperMessage(message) {