@elench/testkit 0.1.80 → 0.1.82

package/lib/{known-failures → regressions}/index.test.mjs

@@ -3,10 +3,10 @@ import os from "os";
 import path from "path";
 import { afterEach, describe, expect, it } from "vitest";
 import {
-  findMatchingKnownFailureEntries,
-  normalizeKnownFailuresDocument,
-  renderKnownFailuresMarkdown,
-  validateKnownFailuresDocument,
+  findMatchingRegressionEntries,
+  normalizeRegressionCatalogDocument,
+  renderRegressionCatalogMarkdown,
+  validateRegressionCatalogDocument,
 } from "./index.mjs";
 
 const tempDirs = [];
@@ -17,25 +17,22 @@ afterEach(() => {
   }
 });
 
-describe("known failures core", () => {
+describe("regression catalog", () => {
   it("matches entries by service, type, path, and failure key", () => {
-    const document = normalizeKnownFailuresDocument({
+    const document = normalizeRegressionCatalogDocument({
       schemaVersion: 1,
       entries: [
         {
           id: "bad-message",
-          title: "Bad message bug",
           classification: "product_bug",
-          state: "open",
           issue: {
             repo: "acme/repo",
             number: 12,
-            url: "https://github.com/acme/repo/issues/12",
           },
-          description: "Wrong message",
-          whyFailing: "Payload is wrong",
-          lastReviewedAt: "2026-04-27",
-          matches: [
+          summary: "API returns the wrong message",
+          cause: "The payload is wrong.",
+          lastReviewedAt: "2026-05-04",
+          fingerprints: [
             {
               service: "api",
               type: "int",
@@ -47,7 +44,7 @@ describe("known failures core", () => {
       ],
     });
 
-    const matches = findMatchingKnownFailureEntries(document, {
+    const matches = findMatchingRegressionEntries(document, {
       service: "api",
       type: "int",
       path: "src/api/routes/__testkit__/failing.int.testkit.ts",
@@ -59,24 +56,21 @@ describe("known failures core", () => {
     expect(matches[0].id).toBe("bad-message");
   });
 
-  it("matches failureKey against a detail title when the key becomes richer", () => {
-    const document = normalizeKnownFailuresDocument({
+  it("matches fingerprint failureKey against detail title", () => {
+    const document = normalizeRegressionCatalogDocument({
       schemaVersion: 1,
       entries: [
         {
           id: "missing-route",
-          title: "Missing route bug",
           classification: "product_bug",
-          state: "open",
           issue: {
             repo: "acme/repo",
             number: 13,
-            url: "https://github.com/acme/repo/issues/13",
           },
-          description: "Wrong status code",
-          whyFailing: "The route returns 404",
-          lastReviewedAt: "2026-04-28",
-          matches: [
+          summary: "Missing route returns the wrong status",
+          cause: "The route returns 404.",
+          lastReviewedAt: "2026-05-04",
+          fingerprints: [
             {
               service: "api",
               type: "int",
@@ -88,7 +82,7 @@ describe("known failures core", () => {
       ],
     });
 
-    const matches = findMatchingKnownFailureEntries(document, {
+    const matches = findMatchingRegressionEntries(document, {
       service: "api",
       type: "int",
       path: "__testkit__/health/http-failure.int.testkit.ts",
@@ -105,31 +99,28 @@ describe("known failures core", () => {
     expect(matches[0].id).toBe("missing-route");
   });
 
-  it("validates status coverage and filesystem matches", () => {
-    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-known-failures-"));
+  it("validates status coverage and filesystem fingerprints", () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-regressions-"));
     tempDirs.push(tempDir);
     const testPath = path.join(tempDir, "src/api/routes/__testkit__/failing.int.testkit.ts");
     fs.mkdirSync(path.dirname(testPath), { recursive: true });
     fs.writeFileSync(testPath, "export default {};\n");
 
-    const document = normalizeKnownFailuresDocument({
+    const document = normalizeRegressionCatalogDocument({
       schemaVersion: 1,
       issueRepo: "acme/repo",
       entries: [
         {
           id: "bad-message",
-          title: "Bad message bug",
           classification: "product_bug",
-          state: "open",
           issue: {
             repo: "acme/repo",
             number: 12,
-            url: "https://github.com/acme/repo/issues/12",
           },
-          description: "Wrong message",
-          whyFailing: "Payload is wrong",
-          lastReviewedAt: "2026-04-27",
-          matches: [
+          summary: "API returns the wrong message",
+          cause: "The payload is wrong.",
+          lastReviewedAt: "2026-05-04",
+          fingerprints: [
             {
               service: "api",
               type: "int",
@@ -140,7 +131,7 @@ describe("known failures core", () => {
       ],
     });
 
-    const result = validateKnownFailuresDocument(document, {
+    const result = validateRegressionCatalogDocument(document, {
       productDir: tempDir,
       statusArtifact: {
         tests: [
@@ -162,37 +153,35 @@ describe("known failures core", () => {
 
     expect(result.errors).toEqual([]);
     expect(result.stats.failedTests).toBe(2);
-    expect(result.stats.triagedFailedTests).toBe(1);
+    expect(result.stats.diagnosedFailedTests).toBe(1);
     expect(result.warnings).toContain(
-      "Untriaged failed test: src/api/routes/__testkit__/other.int.testkit.ts"
+      "New failing test not yet in regression catalog: src/api/routes/__testkit__/other.int.testkit.ts"
     );
   });
 
   it("renders markdown from the canonical document", () => {
-    const markdown = renderKnownFailuresMarkdown(
-      normalizeKnownFailuresDocument({
+    const markdown = renderRegressionCatalogMarkdown(
+      normalizeRegressionCatalogDocument({
         schemaVersion: 1,
         entries: [
           {
             id: "bad-message",
-            title: "Bad message bug",
             classification: "product_bug",
-            state: "open",
             issue: {
               repo: "acme/repo",
               number: 12,
-              url: "https://github.com/acme/repo/issues/12",
             },
-            description: "Wrong message",
-            whyFailing: "Payload is wrong",
-            lastReviewedAt: "2026-04-27",
-            matches: [{ path: "src/api/routes/__testkit__/failing.int.testkit.ts" }],
+            summary: "API returns the wrong message",
+            cause: "The payload is wrong.",
+            lastReviewedAt: "2026-05-04",
+            fingerprints: [{ path: "src/api/routes/__testkit__/failing.int.testkit.ts" }],
           },
         ],
       })
     );
 
-    expect(markdown).toContain("# Bourne Bugs");
-    expect(markdown).toContain("[#12](https://github.com/acme/repo/issues/12)");
+    expect(markdown).toContain("# Regression Catalog");
+    expect(markdown).toContain("API returns the wrong message");
+    expect(markdown).toContain("#12");
   });
 });

package/lib/runner/formatting.mjs

@@ -1,8 +1,5 @@
-import { findMatchingKnownFailureEntries } from "../known-failures/index.mjs";
-import {
-  buildKnownFailureIssueValidationSummaryLines,
-  buildKnownFailureIssueValidationSummaryParts,
-} from "../known-failures/github.mjs";
+import { findMatchingRegressionEntries } from "../regressions/index.mjs";
+import { buildRegressionSyncSummaryLines } from "../regressions/github.mjs";
 
 export function formatDuration(durationMs) {
   const totalSeconds = Math.max(0, Math.round(durationMs / 1000));
@@ -52,14 +49,14 @@ export function formatSuiteFramework(framework) {
   return label ? ` [${label}]` : "";
 }
 
-export function buildRunSummaryLines(results, durationMs, knownFailureIssueValidation = null) {
-  return buildCompactRunSummaryLines(results, durationMs, knownFailureIssueValidation);
+export function buildRunSummaryLines(results, durationMs, regressionReport = null) {
+  return buildCompactRunSummaryLines(results, durationMs, regressionReport);
 }
 
-export function buildRunSummaryData(results, durationMs, knownFailureIssueValidation = null) {
+export function buildRunSummaryData(results, durationMs, regressionReport = null) {
   const totals = summarizeResults(results);
   const serviceErrors = collectServiceErrors(results);
-  const knownFailureParts = buildKnownFailureIssueValidationSummaryParts(knownFailureIssueValidation);
+  const regressionSummary = regressionReport?.summary || null;
   return {
     result: totals.failedServices > 0 ? "FAILED" : "PASSED",
     totalServices: totals.totalServices,
@@ -71,16 +68,21 @@ export function buildRunSummaryData(results, durationMs, knownFailureIssueValida
     files: totals.totalFiles,
     duration: formatDuration(durationMs),
     serviceErrors: serviceErrors.length,
-    knownFailureIssues: knownFailureParts,
+    newRegressions: regressionSummary?.newRegressions || 0,
+    knownRegressions: regressionSummary?.knownRegressions || 0,
+    fixedKnownRegressions: regressionSummary?.fixedKnownRegressions || 0,
+    catalogStale: regressionSummary?.catalogStale || 0,
+    catalogSyncUnavailable: Boolean(regressionSummary?.catalogSyncUnavailable),
+    usedStaleCache: Boolean(regressionSummary?.usedStaleCache),
   };
 }
 
 export function buildCompactRunSummaryLines(
   results,
   durationMs,
-  knownFailureIssueValidation = null
+  regressionReport = null
 ) {
-  const summary = buildRunSummaryData(results, durationMs, knownFailureIssueValidation);
+  const summary = buildRunSummaryData(results, durationMs, regressionReport);
   const lines = [
     "",
     `Summary: ${summary.passed} passed, ${summary.failed} failed, ${summary.skipped} skipped, ${summary.notRun} not run across ${summary.files} ${pluralize(summary.files, "file", "files")} in ${summary.duration}`,
@@ -90,8 +92,20 @@ export function buildCompactRunSummaryLines(
     lines.push(`Runtime errors: ${summary.serviceErrors}`);
   }
 
-  if (summary.knownFailureIssues.length > 0) {
-    lines.push(`Known-failure issues: ${summary.knownFailureIssues.join(", ")}`);
+  if (summary.newRegressions > 0) {
+    lines.push(`New regressions: ${summary.newRegressions}`);
+  }
+  if (summary.knownRegressions > 0) {
+    lines.push(`Known regressions: ${summary.knownRegressions}`);
+  }
+  if (summary.fixedKnownRegressions > 0) {
+    lines.push(`Fixed known regressions: ${summary.fixedKnownRegressions}`);
+  }
+  if (summary.catalogStale > 0) {
+    lines.push(`Catalog stale: ${summary.catalogStale}`);
+  }
+  if (summary.catalogSyncUnavailable) {
+    lines.push("Catalog sync unavailable");
   }
 
   lines.push("");
@@ -103,7 +117,7 @@ export function buildCompactRunSummaryLines(
   return lines;
 }
 
-export function buildDebugRunSummaryLines(results, durationMs, knownFailureIssueValidation = null) {
+export function buildDebugRunSummaryLines(results, durationMs, regressionReport = null) {
   const totalServices = results.length;
   const executedServices = results.filter((result) => !result.skipped);
   const skippedServices = results.filter((result) => result.skipped);
@@ -172,11 +186,19 @@ export function buildDebugRunSummaryLines(results, durationMs, knownFailureIssue
     }
   }
 
-  const knownFailureIssueLines = buildKnownFailureIssueValidationSummaryLines(
-    knownFailureIssueValidation
+  const regressionSyncLines = buildRegressionSyncSummaryLines(
+    regressionReport?.catalog?.configured ? {
+      summary: {
+        byCode: {
+          closed_but_failing: regressionReport.summary.catalogStale,
+          validation_unavailable: regressionReport.summary.catalogSyncUnavailable ? 1 : 0,
+          used_stale_cache: regressionReport.summary.usedStaleCache ? 1 : 0,
+        },
+      },
+    } : null
   );
-  if (knownFailureIssueLines.length > 0) {
-    lines.push(...knownFailureIssueLines);
+  if (regressionSyncLines.length > 0) {
+    lines.push(...regressionSyncLines);
   }
 
   if (failedServices.length > 0) {
@@ -188,7 +210,7 @@ export function buildDebugRunSummaryLines(results, durationMs, knownFailureIssue
   return lines;
 }
 
-export function buildFailurePresentation(fileSummary, knownFailures = null) {
+export function buildFailurePresentation(fileSummary, regressionCatalog = null) {
   const rankedDetails = rankFailureDetails(fileSummary.failureDetails || []);
   const primaryDetail = rankedDetails[0] || null;
   const fallbackMessages = rankedDetails
@@ -200,8 +222,8 @@ export function buildFailurePresentation(fileSummary, knownFailures = null) {
   const responseLine = formatFailureResponsePreview(primaryDetail);
   if (responseLine) details.push(responseLine);
 
-  const triageLine = formatInlineKnownFailureLine(fileSummary, knownFailures);
-  if (triageLine) details.push(triageLine);
+  const regressionLine = formatInlineRegressionLine(fileSummary, regressionCatalog);
+  if (regressionLine) details.push(regressionLine);
 
   const requestLine = formatFailureRequestHint(primaryDetail);
   if (requestLine) details.push(requestLine);
@@ -297,20 +319,13 @@ function formatFailureRequestHint(detail) {
   return `request: ${method} ${path}`;
 }
 
-function formatInlineKnownFailureLine(fileSummary, knownFailures) {
-  if (!knownFailures) return "triage: untriaged";
-  const matches = findMatchingKnownFailureEntries(knownFailures, fileSummary);
-  if (matches.length === 0) return "triage: untriaged";
+function formatInlineRegressionLine(fileSummary, regressionCatalog) {
+  if (!regressionCatalog) return "regression: new";
+  const matches = findMatchingRegressionEntries(regressionCatalog, fileSummary);
+  if (matches.length === 0) return "regression: new";
 
   const entry = matches[0];
-  const issueLabel = `#${entry.issue.number}`;
-  if (entry.state === "open") {
-    return `triage: known issue ${issueLabel} open`;
-  }
-  if (entry.state === "closed") {
-    return `triage: known issue ${issueLabel} closed`;
-  }
-  return `triage: known issue ${issueLabel}`;
+  return `regression: known #${entry.issue.number} ${entry.classification}`;
 }
 
 function isThresholdWrapperMessage(message) {

package/lib/runner/formatting.test.mjs

@@ -87,10 +87,12 @@ describe("runner formatting", () => {
       20_000,
       {
         summary: {
-          byCode: {
-            closed_but_failing: 2,
-            state_mismatch: 1,
-          },
+          newRegressions: 1,
+          knownRegressions: 2,
+          fixedKnownRegressions: 3,
+          catalogStale: 4,
+          catalogSyncUnavailable: true,
+          usedStaleCache: false,
         },
       }
     );
@@ -105,10 +107,11 @@ describe("runner formatting", () => {
       duration: "20s",
       serviceErrors: 1,
     });
-    expect(summary.knownFailureIssues).toEqual([
-      "2 closed issues still failing",
-      "1 state mismatch",
-    ]);
+    expect(summary.newRegressions).toBe(1);
+    expect(summary.knownRegressions).toBe(2);
+    expect(summary.fixedKnownRegressions).toBe(3);
+    expect(summary.catalogStale).toBe(4);
+    expect(summary.catalogSyncUnavailable).toBe(true);
   });
 
   it("builds compact summary lines without failure and runtime dump sections", () => {
@@ -137,6 +140,7 @@ describe("runner formatting", () => {
 
     expect(lines.join("\n")).toContain("Summary: 0 passed, 1 failed, 0 skipped, 0 not run across 1 file");
     expect(lines.join("\n")).toContain("Runtime errors: 1");
+    expect(lines.join("\n")).not.toContain("Catalog issues:");
     expect(lines.join("\n")).not.toContain("Failures:");
     expect(lines.join("\n")).not.toContain("Runtime Errors:");
   });
@@ -172,18 +176,15 @@ describe("runner formatting", () => {
         entries: [
           {
             id: "health-is-bad",
-            title: "Health is bad",
             classification: "product_bug",
-            state: "open",
             issue: {
               repo: "acme/example",
               number: 42,
-              url: "https://github.com/acme/example/issues/42",
             },
-            description: "desc",
-            whyFailing: "because",
+            summary: "Health is bad",
+            cause: "because",
             lastReviewedAt: "2026-01-01",
-            matches: [
+            fingerprints: [
               {
                 service: "api",
                 type: "int",
@@ -198,7 +199,7 @@ describe("runner formatting", () => {
 
     expect(presentation.primary).toBe("GET /health expected 200, got 404");
     expect(presentation.details).toContain('response: {"error":"nope"}');
-    expect(presentation.details).toContain("triage: known issue #42 open");
+    expect(presentation.details).toContain("regression: known #42 product_bug");
     expect(presentation.details).toContain("logs: requestId=req-1");
   });
 });

package/lib/runner/metadata.mjs

@@ -3,7 +3,7 @@ import os from "os";
 import path from "path";
 import { execFileSync } from "child_process";
 import { fileURLToPath } from "url";
-import { parseGitHubRepoSlug } from "../known-failures/github.mjs";
+import { parseGitHubRepoSlug } from "../regressions/github.mjs";
 
 export function collectGitMetadata(productDir) {
   const read = (args) => {

package/lib/runner/orchestrator.mjs

@@ -13,7 +13,7 @@ import {
   recordGraphError,
   recordTaskOutcome,
 } from "./results.mjs";
-import { loadKnownFailuresConfig } from "./triage.mjs";
+import { loadRegressionCatalogConfig } from "./regressions.mjs";
 import { formatError } from "./formatting.mjs";
 import {
   loadTimings,
@@ -56,12 +56,12 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     },
     testkitVersion: readPackageMetadata().version,
   };
-  const knownFailures = loadKnownFailuresConfig(
+  const regressionCatalog = loadRegressionCatalogConfig(
     productDir,
-    configs[0]?.testkit?.reporting || null
+    configs[0]?.testkit?.regressions || null
   );
   const reporter = opts.reporter || null;
-  reporter?.setKnownFailures?.(knownFailures);
+  reporter?.setRegressionCatalog?.(regressionCatalog);
   const logRegistry = createRunLogRegistry(productDir);
   const workerState = {
     workerCount: 0,
@@ -114,7 +114,6 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
   lifecycle.installSignalHandlers();
   let results = [];
   let finishedAt = Date.now();
-  let knownFailureIssueValidation = null;
   writeLiveSnapshot();
 
   try {
@@ -209,15 +208,14 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
       metadata,
       logRegistry,
       setupRegistry,
-      knownFailures,
-      issueValidationConfig: configs[0]?.testkit?.reporting?.issueValidation || null,
+      regressionCatalog,
+      regressionSyncConfig: configs[0]?.testkit?.regressions?.sync || null,
       telemetry,
       reporter,
       writeStatus: opts.writeStatus,
     });
-    knownFailureIssueValidation = finalized.knownFailureIssueValidation;
     if (results.some((result) => result.failed)) exitCode = 1;
-    if (finalized.shouldFailIssueValidation) {
+    if (finalized.shouldFailRegressionSync) {
       exitCode = 1;
     }
     if (lifecycle.isStopRequested()) exitCode = Math.max(exitCode, 130);