@elench/testkit 0.1.80 → 0.1.82

package/lib/runner/regressions.mjs

@@ -0,0 +1,304 @@
+import {
+  buildRegressionFileIdentity,
+  findMatchingRegressionEntries,
+  loadRegressionCatalogConfig,
+} from "../regressions/index.mjs";
+
+export { loadRegressionCatalogConfig };
+
+export function applyRegressionAnalysisToArtifacts(
+  runArtifact,
+  statusArtifact,
+  regressionCatalog,
+  regressionSync
+) {
+  const runEntries = extractRunFileEntries(runArtifact);
+  const statusEntries = extractStatusFileEntries(statusArtifact);
+  const fileSummaries = new Map();
+  const syncById = new Map((regressionSync?.entries || []).map((entry) => [entry.id, entry]));
+  const staleEntryIds = new Set();
+  const newRegressionDrafts = [];
+  const fixedRegressionDrafts = [];
+
+  for (const entry of [...runEntries, ...statusEntries]) {
+    const key = buildRegressionFileIdentity(entry.service, entry.type, entry.path);
+    if (!fileSummaries.has(key)) {
+      fileSummaries.set(key, {
+        service: entry.service,
+        type: entry.type,
+        path: entry.path,
+        status: entry.status,
+        error: entry.error || null,
+        failureDetails: Array.isArray(entry.failureDetails) ? entry.failureDetails : [],
+      });
+    }
+  }
+
+  const diagnosesByFileKey = new Map();
+
+  for (const fileSummary of fileSummaries.values()) {
+    const diagnosis = buildFileDiagnosis(fileSummary, regressionCatalog, syncById);
+    const fileKey = buildRegressionFileIdentity(
+      fileSummary.service,
+      fileSummary.type,
+      fileSummary.path
+    );
+    diagnosesByFileKey.set(fileKey, diagnosis);
+    for (const entry of diagnosis.entries) {
+      if (entry.catalogFindings.length > 0) {
+        staleEntryIds.add(entry.id);
+      }
+    }
+    if (diagnosis.status === "new_regression") {
+      newRegressionDrafts.push(buildNewRegressionDraft(fileSummary, regressionCatalog));
+    }
+    if (diagnosis.status === "fixed_known_regression") {
+      for (const entry of diagnosis.entries) {
+        fixedRegressionDrafts.push({
+          id: entry.id,
+          issue: entry.issue,
+          summary: entry.summary,
+          suggestedAction: "review-for-removal",
+        });
+      }
+    }
+  }
+
+  for (const entry of [...runEntries, ...statusEntries]) {
+    const fileKey = buildRegressionFileIdentity(entry.service, entry.type, entry.path);
+    const diagnosis = diagnosesByFileKey.get(fileKey);
+    if (!diagnosis) continue;
+    setEntryDiagnosis(entry, diagnosis);
+  }
+
+  const summaryTests = statusArtifact?.tests || runEntries;
+  const report = buildRegressionReport(
+    summaryTests,
+    regressionCatalog,
+    regressionSync,
+    diagnosesByFileKey,
+    staleEntryIds,
+    newRegressionDrafts,
+    fixedRegressionDrafts
+  );
+
+  runArtifact.regressions = report;
+  if (statusArtifact) {
+    statusArtifact.regressions = report;
+  }
+
+  return { runArtifact, statusArtifact, regressionReport: report };
+}
+
+function buildFileDiagnosis(fileSummary, regressionCatalog, syncById) {
+  const matchedEntries = regressionCatalog
+    ? findMatchingRegressionEntries(regressionCatalog, fileSummary)
+    : [];
+  const status = resolveDiagnosisStatus(fileSummary.status, matchedEntries.length);
+
+  return {
+    status,
+    classifications: [...new Set(matchedEntries.map((entry) => entry.classification))].sort(),
+    entries: matchedEntries.map((entry) => toDiagnosisEntry(entry, syncById.get(entry.id) || null)),
+  };
+}
+
+function resolveDiagnosisStatus(fileStatus, matchCount) {
+  if (fileStatus === "failed") {
+    return matchCount > 0 ? "known_regression" : "new_regression";
+  }
+  if (fileStatus === "passed" && matchCount > 0) {
+    return "fixed_known_regression";
+  }
+  if (matchCount > 0) {
+    return "tracked_regression_not_executed";
+  }
+  return "not_applicable";
+}
+
+function toDiagnosisEntry(entry, syncEntry) {
+  return {
+    id: entry.id,
+    classification: entry.classification,
+    issue: entry.issue,
+    summary: entry.summary,
+    cause: entry.cause,
+    lastReviewedAt: entry.lastReviewedAt,
+    github: syncEntry?.github || null,
+    syncStatus: syncEntry?.status || null,
+    catalogFindings: Array.isArray(syncEntry?.findings) ? syncEntry.findings : [],
+  };
+}
+
+function buildRegressionReport(
+  tests,
+  regressionCatalog,
+  regressionSync,
+  diagnosesByFileKey,
+  staleEntryIds,
+  newRegressionDrafts,
+  fixedRegressionDrafts
+) {
+  const summary = {
+    newRegressions: 0,
+    knownRegressions: 0,
+    fixedKnownRegressions: 0,
+    catalogStale: staleEntryIds.size,
+    catalogSyncUnavailable: (regressionSync?.summary?.byCode?.validation_unavailable || 0) > 0,
+    usedStaleCache: (regressionSync?.summary?.byCode?.used_stale_cache || 0) > 0,
+  };
+
+  for (const test of tests) {
+    const diagnosis = test.diagnosis || diagnosesByFileKey.get(
+      buildRegressionFileIdentity(test.service, test.type, test.path)
+    );
+    switch (diagnosis?.status) {
+      case "new_regression":
+        summary.newRegressions += 1;
+        break;
+      case "known_regression":
+        summary.knownRegressions += 1;
+        break;
+      case "fixed_known_regression":
+        summary.fixedKnownRegressions += 1;
+        break;
+      default:
+        break;
+    }
+  }
+
+  return {
+    summary,
+    catalog: {
+      configured: Boolean(regressionCatalog),
+      entryCount: regressionCatalog?.entries?.length || 0,
+      staleEntries: (regressionSync?.entries || []).filter((entry) =>
+        Array.isArray(entry.findings) && entry.findings.length > 0
+      ),
+      findings: regressionSync?.findings || [],
+      sync: {
+        mode: regressionSync?.mode || null,
+        checkedAt: regressionSync?.checkedAt || null,
+        usedStaleCache: summary.usedStaleCache,
+        unavailable: summary.catalogSyncUnavailable,
+      },
+    },
+    drafts: {
+      newRegressions: newRegressionDrafts,
+      fixedRegressions: dedupeDraftsById(fixedRegressionDrafts),
+    },
+  };
+}
+
+function buildNewRegressionDraft(fileSummary, regressionCatalog) {
+  return {
+    id: suggestRegressionId(fileSummary),
+    classification: suggestRegressionClassification(fileSummary),
+    issue: {
+      repo: regressionCatalog?.issueRepo || null,
+      number: null,
+    },
+    summary: suggestRegressionSummary(fileSummary),
+    cause: suggestRegressionCause(fileSummary),
+    fingerprints: [
+      {
+        service: fileSummary.service,
+        type: fileSummary.type,
+        path: fileSummary.path,
+        ...(fileSummary.failureDetails?.[0]?.key ? { failureKey: fileSummary.failureDetails[0].key } : {}),
+      },
+    ],
+  };
+}
+
+function suggestRegressionId(fileSummary) {
+  const base = `${fileSummary.service}-${fileSummary.type}-${fileSummary.path}`
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+  const detail = fileSummary.failureDetails?.[0]?.key
+    ? `-${String(fileSummary.failureDetails[0].key).toLowerCase().replace(/[^a-z0-9]+/g, "-")}`
+    : "";
+  return `${base}${detail}`.replace(/-+/g, "-").slice(0, 96);
+}
+
+function suggestRegressionClassification(fileSummary) {
+  const message = [
+    fileSummary.error,
+    ...((fileSummary.failureDetails || []).map((detail) => detail?.message || detail?.title || "")),
+  ]
+    .filter(Boolean)
+    .join(" ")
+    .toLowerCase();
+
+  if (
+    message.includes("timed out") ||
+    message.includes("already in use") ||
+    message.includes("never becomes ready") ||
+    message.includes("runtime error")
+  ) {
+    return "infra";
+  }
+  return "product_bug";
+}
+
+function suggestRegressionSummary(fileSummary) {
+  const primaryDetail = fileSummary.failureDetails?.[0];
+  if (primaryDetail?.message) return String(primaryDetail.message).trim().slice(0, 160);
+  if (primaryDetail?.title) return String(primaryDetail.title).trim().slice(0, 160);
+  if (fileSummary.error) return String(fileSummary.error).trim().slice(0, 160);
+  return `${fileSummary.type} regression in ${fileSummary.path}`;
+}
+
+function suggestRegressionCause(fileSummary) {
+  const primaryDetail = fileSummary.failureDetails?.[0];
+  if (primaryDetail?.response?.bodyPreview) {
+    return `Observed response preview: ${String(primaryDetail.response.bodyPreview).slice(0, 200)}`;
+  }
+  if (fileSummary.error) {
+    return `Observed failure: ${String(fileSummary.error).slice(0, 200)}`;
+  }
+  return "Investigate the observed failure and replace this draft cause with the underlying technical root cause.";
+}
+
+function dedupeDraftsById(entries) {
+  const map = new Map();
+  for (const entry of entries) {
+    if (!map.has(entry.id)) map.set(entry.id, entry);
+  }
+  return [...map.values()];
+}
+
+function extractRunFileEntries(runArtifact) {
+  const entries = [];
+
+  for (const service of runArtifact.services || []) {
+    for (const suite of service.suites || []) {
+      for (const file of suite.files || []) {
+        entries.push({
+          target: file,
+          service: service.name,
+          type: suite.type,
+          path: file.path,
+          status: file.status,
+          error: file.error || null,
+          failureDetails: Array.isArray(file.failureDetails) ? file.failureDetails : [],
+        });
+      }
+    }
+  }
+
+  return entries;
+}
+
+function extractStatusFileEntries(statusArtifact) {
+  return statusArtifact?.tests || [];
+}
+
+function setEntryDiagnosis(entry, diagnosis) {
+  if (entry?.target) {
+    entry.target.diagnosis = diagnosis;
+    return;
+  }
+  entry.diagnosis = diagnosis;
+}

package/lib/runner/{triage.test.mjs → regressions.test.mjs}

@@ -1,27 +1,24 @@
 import { describe, expect, it } from "vitest";
-import { normalizeKnownFailuresDocument } from "../known-failures/index.mjs";
-import { applyKnownFailuresToArtifacts } from "./triage.mjs";
+import { normalizeRegressionCatalogDocument } from "../regressions/index.mjs";
+import { applyRegressionAnalysisToArtifacts } from "./regressions.mjs";
 
-describe("runner triage", () => {
-  it("matches exact failure keys and enriches both artifacts", () => {
-    const knownFailures = normalizeKnownFailuresDocument({
+describe("runner regressions", () => {
+  it("classifies known failed regressions and enriches both artifacts", () => {
+    const regressionCatalog = normalizeRegressionCatalogDocument({
       schemaVersion: 1,
       issueRepo: "acme/repo",
       entries: [
         {
           id: "bad-message",
-          title: "Bad message bug",
           classification: "product_bug",
-          state: "open",
           issue: {
             repo: "acme/repo",
             number: 12,
-            url: "https://github.com/acme/repo/issues/12",
           },
-          description: "The API returns the wrong message.",
-          whyFailing: "The endpoint payload is wrong.",
-          lastReviewedAt: "2026-04-27",
-          matches: [
+          summary: "API returns the wrong message",
+          cause: "The endpoint payload is wrong.",
+          lastReviewedAt: "2026-05-04",
+          fingerprints: [
             {
               service: "api",
               type: "int",
@@ -80,36 +77,49 @@ describe("runner triage", () => {
       ],
     };
 
-    const enriched = applyKnownFailuresToArtifacts(runArtifact, statusArtifact, knownFailures);
-    expect(enriched.statusArtifact.tests[0].triage).toMatchObject({
-      status: "known_failure",
+    const enriched = applyRegressionAnalysisToArtifacts(
+      runArtifact,
+      statusArtifact,
+      regressionCatalog,
+      {
+        entries: [
+          {
+            id: "bad-message",
+            status: "open_and_failing",
+            github: {
+              state: "OPEN",
+              url: "https://github.com/acme/repo/issues/12",
+              cached: false,
+            },
+            findings: [],
+          },
+        ],
+        findings: [],
+      }
+    );
+
+    expect(enriched.statusArtifact.tests[0].diagnosis).toMatchObject({
+      status: "known_regression",
       classifications: ["product_bug"],
     });
-    expect(enriched.runArtifact.services[0].suites[0].files[0].triage.entries[0]).toMatchObject({
+    expect(enriched.runArtifact.services[0].suites[0].files[0].diagnosis.entries[0]).toMatchObject({
       id: "bad-message",
       issue: {
         number: 12,
       },
     });
-    expect(enriched.statusArtifact.triageSummary).toEqual({
-      failed: {
-        total: 1,
-        known: 1,
-        untriaged: 0,
-        byClassification: {
-          product_bug: 1,
-        },
-      },
-      entries: {
-        total: 1,
-        matchedByFailedTests: 1,
-        unmatched: 0,
-      },
+    expect(enriched.statusArtifact.regressions.summary).toEqual({
+      newRegressions: 0,
+      knownRegressions: 1,
+      fixedKnownRegressions: 0,
+      catalogStale: 0,
+      catalogSyncUnavailable: false,
+      usedStaleCache: false,
     });
   });
 
-  it("marks unmatched failed tests as untriaged", () => {
-    const enriched = applyKnownFailuresToArtifacts(
+  it("marks unmatched failed tests as new regressions and emits drafts", () => {
+    const enriched = applyRegressionAnalysisToArtifacts(
       {
         services: [
           {
@@ -140,15 +150,19 @@ describe("runner triage", () => {
           },
         ],
       },
-      normalizeKnownFailuresDocument({
+      normalizeRegressionCatalogDocument({
         schemaVersion: 1,
         entries: [],
-      })
+      }),
+      null
     );
 
-    expect(enriched.statusArtifact.tests[0].triage).toEqual({
-      status: "untriaged",
+    expect(enriched.statusArtifact.tests[0].diagnosis).toEqual({
+      status: "new_regression",
+      classifications: [],
       entries: [],
     });
+    expect(enriched.statusArtifact.regressions.summary.newRegressions).toBe(1);
+    expect(enriched.statusArtifact.regressions.drafts.newRegressions).toHaveLength(1);
   });
 });

package/lib/runner/reporting.mjs

@@ -80,7 +80,7 @@ export function buildStatusArtifact({
     scope.serviceFilter === null;
 
   return {
-    schemaVersion: 6,
+    schemaVersion: 7,
     source: "testkit",
     notice: "Generated file. Do not edit manually.",
     product: {
@@ -134,7 +134,7 @@ export function buildRunArtifact({
   const dbBackend = summarizeDbBackend(results);
 
   return {
-    schemaVersion: 8,
+    schemaVersion: 9,
     source: "testkit",
     generatedAt: new Date(finishedAt).toISOString(),
     product: {

package/lib/runner/reporting.test.mjs

@@ -79,7 +79,7 @@ describe("runner reporting", () => {
     });
 
     expect(artifact.product.name).toBe("my-product");
-    expect(artifact.schemaVersion).toBe(8);
+    expect(artifact.schemaVersion).toBe(9);
     expect(artifact.run).toMatchObject({
       workers: 2,
       fileTimeoutSeconds: 60,
@@ -233,7 +233,7 @@ describe("runner reporting", () => {
     });
 
     expect(status).toEqual({
-      schemaVersion: 6,
+      schemaVersion: 7,
       source: "testkit",
       notice: "Generated file. Do not edit manually.",
       product: {

package/lib/runner/run-finalization.mjs

@@ -1,11 +1,11 @@
 import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
-import { applyKnownFailureIssueValidationToArtifacts, applyKnownFailuresToArtifacts } from "./triage.mjs";
+import { applyRegressionAnalysisToArtifacts } from "./regressions.mjs";
 import { writeRunArtifact, writeStatusArtifact } from "./artifacts.mjs";
 import { summarizeDbBackend } from "./results.mjs";
 import { formatError } from "./formatting.mjs";
 import { uploadTelemetryArtifact } from "../telemetry/index.mjs";
 import { loadHistory, saveHistory, updateHistoryFromRunArtifact } from "../history/index.mjs";
-import { shouldFailKnownFailureIssueValidation, validateKnownFailureIssues } from "../known-failures/github.mjs";
+import { shouldFailRegressionSync, validateRegressionIssues } from "../regressions/github.mjs";
 
 export async function finalizeRunArtifacts({
   productDir,
@@ -20,8 +20,8 @@ export async function finalizeRunArtifacts({
   metadata,
   logRegistry,
   setupRegistry,
-  knownFailures,
-  issueValidationConfig,
+  regressionCatalog,
+  regressionSyncConfig,
   telemetry,
   reporter,
   writeStatus,
@@ -60,24 +60,19 @@ export async function finalizeRunArtifacts({
         metadata,
       })
     : null;
-  const enrichedArtifacts = applyKnownFailuresToArtifacts(runArtifact, statusArtifact, knownFailures);
-  const knownFailureIssueValidation = await validateKnownFailureIssues({
+  const regressionSync = await validateRegressionIssues({
     productDir,
-    document: knownFailures,
-    runArtifact: enrichedArtifacts.runArtifact,
-    statusArtifact: enrichedArtifacts.statusArtifact,
-    config: issueValidationConfig,
+    document: regressionCatalog,
+    runArtifact,
+    statusArtifact,
+    config: regressionSyncConfig,
     gitMetadata: metadata.git,
   });
-  applyKnownFailureIssueValidationToArtifacts(
-    enrichedArtifacts.runArtifact,
-    enrichedArtifacts.statusArtifact,
-    knownFailureIssueValidation
-  );
-  attachKnownFailureIssueValidation(
-    enrichedArtifacts.runArtifact,
-    enrichedArtifacts.statusArtifact,
-    knownFailureIssueValidation
+  const enrichedArtifacts = applyRegressionAnalysisToArtifacts(
+    runArtifact,
+    statusArtifact,
+    regressionCatalog,
+    regressionSync
   );
 
   writeRunArtifact(productDir, enrichedArtifacts.runArtifact);
@@ -91,14 +86,15 @@ export async function finalizeRunArtifacts({
   );
   saveHistory(productDir, nextHistory);
 
-  reporter?.runSummary?.(results, finishedAt - startedAt, knownFailureIssueValidation);
+  reporter?.runSummary?.(results, finishedAt - startedAt, enrichedArtifacts.regressionReport);
   await reportTelemetry(telemetry, enrichedArtifacts.runArtifact, reporter);
 
   return {
     runArtifact: enrichedArtifacts.runArtifact,
     statusArtifact: enrichedArtifacts.statusArtifact,
-    knownFailureIssueValidation,
-    shouldFailIssueValidation: shouldFailKnownFailureIssueValidation(knownFailureIssueValidation),
+    regressionReport: enrichedArtifacts.regressionReport,
+    regressionSync,
+    shouldFailRegressionSync: shouldFailRegressionSync(regressionSync),
   };
 }
 
@@ -122,11 +118,3 @@ async function reportTelemetry(telemetry, artifact, reporter = null) {
     reporter?.telemetry?.(`Telemetry: upload failed (${formatError(error)})`);
   }
 }
-
-function attachKnownFailureIssueValidation(runArtifact, statusArtifact, validation) {
-  if (!validation) return;
-  runArtifact.knownFailuresIssueValidation = validation;
-  if (statusArtifact) {
-    statusArtifact.knownFailuresIssueValidation = validation;
-  }
-}

package/lib/runner/template-steps.mjs

@@ -35,7 +35,7 @@ const PLAYWRIGHT_ENTRY = path.join(PACKAGE_ROOT, "lib", "playwright", "index.mjs
 const RUNTIME_ENTRY = path.join(PACKAGE_ROOT, "lib", "runtime", "index.mjs");
 const VITEST_ENTRY = path.join(PACKAGE_ROOT, "lib", "vitest", "index.mjs");
 const DISCOVERY_ENTRY = path.join(PACKAGE_ROOT, "lib", "discovery", "index.mjs");
-const KNOWN_FAILURES_ENTRY = path.join(PACKAGE_ROOT, "lib", "known-failures", "index.mjs");
+const REGRESSIONS_ENTRY = path.join(PACKAGE_ROOT, "lib", "regressions", "index.mjs");
 const MODULE_RUNNER_ENTRY = path.join(
   PACKAGE_ROOT,
   "lib",
@@ -263,7 +263,7 @@ function resolvePackageSubpath(specifier) {
   if (subpath === "/runtime") return RUNTIME_ENTRY;
   if (subpath === "/vitest") return VITEST_ENTRY;
   if (subpath === "/discovery") return DISCOVERY_ENTRY;
-  if (subpath === "/known-failures") return KNOWN_FAILURES_ENTRY;
+  if (subpath === "/regressions") return REGRESSIONS_ENTRY;
 
   throw new Error(`Unsupported @elench/testkit import "${specifier}" while loading template step`);
 }