@elench/testkit 0.1.79 → 0.1.81

package/lib/runner/formatting.test.mjs

@@ -1,5 +1,7 @@
 import { describe, expect, it } from "vitest";
 import {
+  buildFailurePresentation,
+  buildRunSummaryData,
   buildRunSummaryLines,
   formatDuration,
   formatError,
@@ -61,8 +63,8 @@ describe("runner formatting", () => {
     expect(longestServiceName([{ name: "api" }, { name: "frontend" }])).toBe(8);
   });
 
-  it("builds printable summary lines", () => {
-    const lines = buildRunSummaryLines(
+  it("builds aggregate summary data without failure dumps", () => {
+    const summary = buildRunSummaryData(
       [
         {
           name: "frontend",
@@ -74,96 +76,45 @@ describe("runner formatting", () => {
           failedSuiteCount: 1,
           totalFileCount: 3,
           passedFileCount: 2,
+          failedFileCount: 1,
           skippedFileCount: 0,
           notRunFileCount: 0,
           durationMs: 20_000,
-          suites: [
-            {
-              failed: true,
-              type: "e2e",
-              name: "auth",
-              framework: "playwright",
-              failedFiles: ["a.pw.testkit.ts"],
-              durationMs: 12_000,
-              error: "boom",
-            },
-          ],
-          errors: ["worker broke"],
-        },
-      ],
-      20_000
-    );
-
-    expect(lines.join("\n")).toContain("Summary: 2 passed, 0 failed, 0 skipped, 0 not run across 3 files");
-    expect(lines.join("\n")).toContain("Runtime Errors:");
-    expect(lines.join("\n")).toContain("worker broke");
-    expect(lines.at(-1)).toBe("Result: FAILED (1/1 services failed)");
-  });
-
-  it("marks services with only skipped suites as SKIP", () => {
-    const lines = buildRunSummaryLines(
-      [
-        {
-          name: "api",
-          skipped: false,
-          failed: false,
-          suiteCount: 1,
-          completedSuiteCount: 1,
-          skippedSuiteCount: 1,
-          failedSuiteCount: 0,
-          totalFileCount: 1,
-          passedFileCount: 0,
-          skippedFileCount: 1,
-          notRunFileCount: 0,
-          durationMs: 0,
           suites: [],
-          errors: [],
-        },
-      ],
-      0
-    );
-
-    expect(lines.join("\n")).toContain("Summary: 0 passed, 0 failed, 1 skipped, 0 not run across 1 file");
-    expect(lines.at(-1)).toBe("Result: PASSED");
-  });
-
-  it("appends known-failure issue validation summary lines", () => {
-    const lines = buildRunSummaryLines(
-      [
-        {
-          name: "api",
-          skipped: false,
-          failed: false,
-          suiteCount: 1,
-          completedSuiteCount: 1,
-          skippedSuiteCount: 0,
-          failedSuiteCount: 0,
-          totalFileCount: 1,
-          passedFileCount: 1,
-          skippedFileCount: 0,
-          notRunFileCount: 0,
-          durationMs: 1_000,
-          suites: [],
-          errors: [],
+          errors: ["worker broke"],
         },
       ],
-      1_000,
+      20_000,
       {
         summary: {
-          byCode: {
-            closed_but_failing: 2,
-            title_mismatch: 1,
-          },
+          newRegressions: 1,
+          knownRegressions: 2,
+          fixedKnownRegressions: 3,
+          catalogStale: 4,
+          catalogSyncUnavailable: true,
+          usedStaleCache: false,
         },
       }
     );
 
-    expect(lines.join("\n")).toContain("Known-failure issues:");
-    expect(lines.join("\n")).toContain("2 closed issues still failing");
-    expect(lines.join("\n")).toContain("1 title mismatch");
+    expect(summary).toMatchObject({
+      result: "FAILED",
+      passed: 2,
+      failed: 1,
+      skipped: 0,
+      notRun: 0,
+      files: 3,
+      duration: "20s",
+      serviceErrors: 1,
+    });
+    expect(summary.newRegressions).toBe(1);
+    expect(summary.knownRegressions).toBe(2);
+    expect(summary.fixedKnownRegressions).toBe(3);
+    expect(summary.catalogStale).toBe(4);
+    expect(summary.catalogSyncUnavailable).toBe(true);
   });
 
-  it("prefers structured HTTP assertion details over threshold wrapper text", () => {
+  it("builds compact summary lines without failure and runtime dump sections", () => {
     const lines = buildRunSummaryLines(
       [
         {
@@ -180,63 +131,75 @@ describe("runner formatting", () => {
           skippedFileCount: 0,
           notRunFileCount: 0,
           durationMs: 500,
-          suites: [
-            {
-              failed: true,
-              type: "int",
-              name: "default",
-              framework: "k6",
-              failedFiles: ["__testkit__/health/health.int.testkit.ts"],
-              durationMs: 500,
-              files: [
-                {
-                  path: "__testkit__/health/health.int.testkit.ts",
-                  status: "failed",
-                  error: "Default runtime thresholds failed: checks(rate==1.0)",
-                  failureDetails: [
-                    {
-                      kind: "http-assertion",
-                      key: "GET /health > status is 200",
-                      title: "status is 200",
-                      message: "GET /health expected 200, got 404",
-                      request: {
-                        method: "GET",
-                        path: "/health",
-                        requestId: "req-1",
-                      },
-                      response: {
-                        status: 404,
-                        bodyPreview: '{"error":"nope"}',
-                      },
-                    },
-                  ],
-                  triage: {
-                    status: "known_failure",
-                    entries: [
-                      {
-                        id: "health-is-bad",
-                        state: "open",
-                        issue: {
-                          repo: "acme/example",
-                          number: 42,
-                        },
-                      },
-                    ],
-                  },
-                },
-              ],
-              error: null,
-            },
-          ],
-          errors: [],
+          suites: [],
+          errors: ["worker broke"],
         },
       ],
       500
     );
 
-    expect(lines.join("\n")).toContain("GET /health expected 200, got 404");
-    expect(lines.join("\n")).toContain('response: {"error":"nope"}');
-    expect(lines.join("\n")).toContain("triage: known issue #42 open");
-    expect(lines.join("\n")).not.toContain("Default runtime thresholds failed: checks(rate==1.0)\n    response:");
+    expect(lines.join("\n")).toContain("Summary: 0 passed, 1 failed, 0 skipped, 0 not run across 1 file");
+    expect(lines.join("\n")).toContain("Runtime errors: 1");
+    expect(lines.join("\n")).not.toContain("Catalog issues:");
+    expect(lines.join("\n")).not.toContain("Failures:");
+    expect(lines.join("\n")).not.toContain("Runtime Errors:");
+  });
+
+  it("builds inline failure presentations from structured HTTP assertion details", () => {
+    const presentation = buildFailurePresentation(
+      {
+        service: "api",
+        type: "int",
+        path: "__testkit__/health/health.int.testkit.ts",
+        error: "Default runtime thresholds failed: checks(rate==1.0)",
+        failureDetails: [
+          {
+            kind: "http-assertion",
+            key: "GET /health > status is 200",
+            title: "status is 200",
+            message: "GET /health expected 200, got 404",
+            request: {
+              method: "GET",
+              path: "/health",
+              requestId: "req-1",
+            },
+            response: {
+              status: 404,
+              bodyPreview: '{"error":"nope"}',
+            },
+          },
+        ],
+      },
+      {
+        schemaVersion: 1,
+        issueRepo: "acme/example",
+        entries: [
+          {
+            id: "health-is-bad",
+            classification: "product_bug",
+            issue: {
+              repo: "acme/example",
+              number: 42,
+            },
+            summary: "Health is bad",
+            cause: "because",
+            lastReviewedAt: "2026-01-01",
+            fingerprints: [
+              {
+                service: "api",
+                type: "int",
+                path: "__testkit__/health/health.int.testkit.ts",
+                failureKey: "GET /health > status is 200",
+              },
+            ],
+          },
+        ],
+      }
+    );
+
+    expect(presentation.primary).toBe("GET /health expected 200, got 404");
+    expect(presentation.details).toContain('response: {"error":"nope"}');
+    expect(presentation.details).toContain("regression: known #42 product_bug");
+    expect(presentation.details).toContain("logs: requestId=req-1");
   });
 });

package/lib/runner/metadata.mjs

@@ -3,7 +3,7 @@ import os from "os";
 import path from "path";
 import { execFileSync } from "child_process";
 import { fileURLToPath } from "url";
-import { parseGitHubRepoSlug } from "../known-failures/github.mjs";
+import { parseGitHubRepoSlug } from "../regressions/github.mjs";
 
 export function collectGitMetadata(productDir) {
   const read = (args) => {

package/lib/runner/orchestrator.mjs

@@ -13,7 +13,7 @@ import {
   recordGraphError,
   recordTaskOutcome,
 } from "./results.mjs";
-import { loadKnownFailuresConfig } from "./triage.mjs";
+import { loadRegressionCatalogConfig } from "./regressions.mjs";
 import { formatError } from "./formatting.mjs";
 import {
   loadTimings,
@@ -56,11 +56,12 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     },
     testkitVersion: readPackageMetadata().version,
   };
-  const knownFailures = loadKnownFailuresConfig(
+  const regressionCatalog = loadRegressionCatalogConfig(
     productDir,
-    configs[0]?.testkit?.reporting || null
+    configs[0]?.testkit?.regressions || null
   );
   const reporter = opts.reporter || null;
+  reporter?.setRegressionCatalog?.(regressionCatalog);
   const logRegistry = createRunLogRegistry(productDir);
   const workerState = {
     workerCount: 0,
@@ -113,7 +114,6 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
   lifecycle.installSignalHandlers();
   let results = [];
   let finishedAt = Date.now();
-  let knownFailureIssueValidation = null;
   writeLiveSnapshot();
 
   try {
@@ -208,15 +208,14 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
       metadata,
       logRegistry,
       setupRegistry,
-      knownFailures,
-      issueValidationConfig: configs[0]?.testkit?.reporting?.issueValidation || null,
+      regressionCatalog,
+      regressionSyncConfig: configs[0]?.testkit?.regressions?.sync || null,
       telemetry,
       reporter,
       writeStatus: opts.writeStatus,
     });
-    knownFailureIssueValidation = finalized.knownFailureIssueValidation;
     if (results.some((result) => result.failed)) exitCode = 1;
-    if (finalized.shouldFailIssueValidation) {
+    if (finalized.shouldFailRegressionSync) {
       exitCode = 1;
     }
     if (lifecycle.isStopRequested()) exitCode = Math.max(exitCode, 130);

package/lib/runner/regressions.mjs

@@ -0,0 +1,304 @@
+import {
+  buildRegressionFileIdentity,
+  findMatchingRegressionEntries,
+  loadRegressionCatalogConfig,
+} from "../regressions/index.mjs";
+
+export { loadRegressionCatalogConfig };
+
+export function applyRegressionAnalysisToArtifacts(
+  runArtifact,
+  statusArtifact,
+  regressionCatalog,
+  regressionSync
+) {
+  const runEntries = extractRunFileEntries(runArtifact);
+  const statusEntries = extractStatusFileEntries(statusArtifact);
+  const fileSummaries = new Map();
+  const syncById = new Map((regressionSync?.entries || []).map((entry) => [entry.id, entry]));
+  const staleEntryIds = new Set();
+  const newRegressionDrafts = [];
+  const fixedRegressionDrafts = [];
+
+  for (const entry of [...runEntries, ...statusEntries]) {
+    const key = buildRegressionFileIdentity(entry.service, entry.type, entry.path);
+    if (!fileSummaries.has(key)) {
+      fileSummaries.set(key, {
+        service: entry.service,
+        type: entry.type,
+        path: entry.path,
+        status: entry.status,
+        error: entry.error || null,
+        failureDetails: Array.isArray(entry.failureDetails) ? entry.failureDetails : [],
+      });
+    }
+  }
+
+  const diagnosesByFileKey = new Map();
+
+  for (const fileSummary of fileSummaries.values()) {
+    const diagnosis = buildFileDiagnosis(fileSummary, regressionCatalog, syncById);
+    const fileKey = buildRegressionFileIdentity(
+      fileSummary.service,
+      fileSummary.type,
+      fileSummary.path
+    );
+    diagnosesByFileKey.set(fileKey, diagnosis);
+    for (const entry of diagnosis.entries) {
+      if (entry.catalogFindings.length > 0) {
+        staleEntryIds.add(entry.id);
+      }
+    }
+    if (diagnosis.status === "new_regression") {
+      newRegressionDrafts.push(buildNewRegressionDraft(fileSummary, regressionCatalog));
+    }
+    if (diagnosis.status === "fixed_known_regression") {
+      for (const entry of diagnosis.entries) {
+        fixedRegressionDrafts.push({
+          id: entry.id,
+          issue: entry.issue,
+          summary: entry.summary,
+          suggestedAction: "review-for-removal",
+        });
+      }
+    }
+  }
+
+  for (const entry of [...runEntries, ...statusEntries]) {
+    const fileKey = buildRegressionFileIdentity(entry.service, entry.type, entry.path);
+    const diagnosis = diagnosesByFileKey.get(fileKey);
+    if (!diagnosis) continue;
+    setEntryDiagnosis(entry, diagnosis);
+  }
+
+  const summaryTests = statusArtifact?.tests || runEntries;
+  const report = buildRegressionReport(
+    summaryTests,
+    regressionCatalog,
+    regressionSync,
+    diagnosesByFileKey,
+    staleEntryIds,
+    newRegressionDrafts,
+    fixedRegressionDrafts
+  );
+
+  runArtifact.regressions = report;
+  if (statusArtifact) {
+    statusArtifact.regressions = report;
+  }
+
+  return { runArtifact, statusArtifact, regressionReport: report };
+}
+
+function buildFileDiagnosis(fileSummary, regressionCatalog, syncById) {
+  const matchedEntries = regressionCatalog
+    ? findMatchingRegressionEntries(regressionCatalog, fileSummary)
+    : [];
+  const status = resolveDiagnosisStatus(fileSummary.status, matchedEntries.length);
+
+  return {
+    status,
+    classifications: [...new Set(matchedEntries.map((entry) => entry.classification))].sort(),
+    entries: matchedEntries.map((entry) => toDiagnosisEntry(entry, syncById.get(entry.id) || null)),
+  };
+}
+
+function resolveDiagnosisStatus(fileStatus, matchCount) {
+  if (fileStatus === "failed") {
+    return matchCount > 0 ? "known_regression" : "new_regression";
+  }
+  if (fileStatus === "passed" && matchCount > 0) {
+    return "fixed_known_regression";
+  }
+  if (matchCount > 0) {
+    return "tracked_regression_not_executed";
+  }
+  return "not_applicable";
+}
+
+function toDiagnosisEntry(entry, syncEntry) {
+  return {
+    id: entry.id,
+    classification: entry.classification,
+    issue: entry.issue,
+    summary: entry.summary,
+    cause: entry.cause,
+    lastReviewedAt: entry.lastReviewedAt,
+    github: syncEntry?.github || null,
+    syncStatus: syncEntry?.status || null,
+    catalogFindings: Array.isArray(syncEntry?.findings) ? syncEntry.findings : [],
+  };
+}
+
+function buildRegressionReport(
+  tests,
+  regressionCatalog,
+  regressionSync,
+  diagnosesByFileKey,
+  staleEntryIds,
+  newRegressionDrafts,
+  fixedRegressionDrafts
+) {
+  const summary = {
+    newRegressions: 0,
+    knownRegressions: 0,
+    fixedKnownRegressions: 0,
+    catalogStale: staleEntryIds.size,
+    catalogSyncUnavailable: (regressionSync?.summary?.byCode?.validation_unavailable || 0) > 0,
+    usedStaleCache: (regressionSync?.summary?.byCode?.used_stale_cache || 0) > 0,
+  };
+
+  for (const test of tests) {
+    const diagnosis = test.diagnosis || diagnosesByFileKey.get(
+      buildRegressionFileIdentity(test.service, test.type, test.path)
+    );
+    switch (diagnosis?.status) {
+      case "new_regression":
+        summary.newRegressions += 1;
+        break;
+      case "known_regression":
+        summary.knownRegressions += 1;
+        break;
+      case "fixed_known_regression":
+        summary.fixedKnownRegressions += 1;
+        break;
+      default:
+        break;
+    }
+  }
+
+  return {
+    summary,
+    catalog: {
+      configured: Boolean(regressionCatalog),
+      entryCount: regressionCatalog?.entries?.length || 0,
+      staleEntries: (regressionSync?.entries || []).filter((entry) =>
+        Array.isArray(entry.findings) && entry.findings.length > 0
+      ),
+      findings: regressionSync?.findings || [],
+      sync: {
+        mode: regressionSync?.mode || null,
+        checkedAt: regressionSync?.checkedAt || null,
+        usedStaleCache: summary.usedStaleCache,
+        unavailable: summary.catalogSyncUnavailable,
+      },
+    },
+    drafts: {
+      newRegressions: newRegressionDrafts,
+      fixedRegressions: dedupeDraftsById(fixedRegressionDrafts),
+    },
+  };
+}
+
+function buildNewRegressionDraft(fileSummary, regressionCatalog) {
+  return {
+    id: suggestRegressionId(fileSummary),
+    classification: suggestRegressionClassification(fileSummary),
+    issue: {
+      repo: regressionCatalog?.issueRepo || null,
+      number: null,
+    },
+    summary: suggestRegressionSummary(fileSummary),
+    cause: suggestRegressionCause(fileSummary),
+    fingerprints: [
+      {
+        service: fileSummary.service,
+        type: fileSummary.type,
+        path: fileSummary.path,
+        ...(fileSummary.failureDetails?.[0]?.key ? { failureKey: fileSummary.failureDetails[0].key } : {}),
+      },
+    ],
+  };
+}
+
+function suggestRegressionId(fileSummary) {
+  const base = `${fileSummary.service}-${fileSummary.type}-${fileSummary.path}`
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+  const detail = fileSummary.failureDetails?.[0]?.key
+    ? `-${String(fileSummary.failureDetails[0].key).toLowerCase().replace(/[^a-z0-9]+/g, "-")}`
+    : "";
+  return `${base}${detail}`.replace(/-+/g, "-").slice(0, 96);
+}
+
+function suggestRegressionClassification(fileSummary) {
+  const message = [
+    fileSummary.error,
+    ...((fileSummary.failureDetails || []).map((detail) => detail?.message || detail?.title || "")),
+  ]
+    .filter(Boolean)
+    .join(" ")
+    .toLowerCase();
+
+  if (
+    message.includes("timed out") ||
+    message.includes("already in use") ||
+    message.includes("never becomes ready") ||
+    message.includes("runtime error")
+  ) {
+    return "infra";
+  }
+  return "product_bug";
+}
+
+function suggestRegressionSummary(fileSummary) {
+  const primaryDetail = fileSummary.failureDetails?.[0];
+  if (primaryDetail?.message) return String(primaryDetail.message).trim().slice(0, 160);
+  if (primaryDetail?.title) return String(primaryDetail.title).trim().slice(0, 160);
+  if (fileSummary.error) return String(fileSummary.error).trim().slice(0, 160);
+  return `${fileSummary.type} regression in ${fileSummary.path}`;
+}
+
+function suggestRegressionCause(fileSummary) {
+  const primaryDetail = fileSummary.failureDetails?.[0];
+  if (primaryDetail?.response?.bodyPreview) {
+    return `Observed response preview: ${String(primaryDetail.response.bodyPreview).slice(0, 200)}`;
+  }
+  if (fileSummary.error) {
+    return `Observed failure: ${String(fileSummary.error).slice(0, 200)}`;
+  }
+  return "Investigate the observed failure and replace this draft cause with the underlying technical root cause.";
+}
+
+function dedupeDraftsById(entries) {
+  const map = new Map();
+  for (const entry of entries) {
+    if (!map.has(entry.id)) map.set(entry.id, entry);
+  }
+  return [...map.values()];
+}
+
+function extractRunFileEntries(runArtifact) {
+  const entries = [];
+
+  for (const service of runArtifact.services || []) {
+    for (const suite of service.suites || []) {
+      for (const file of suite.files || []) {
+        entries.push({
+          target: file,
+          service: service.name,
+          type: suite.type,
+          path: file.path,
+          status: file.status,
+          error: file.error || null,
+          failureDetails: Array.isArray(file.failureDetails) ? file.failureDetails : [],
+        });
+      }
+    }
+  }
+
+  return entries;
+}
+
+function extractStatusFileEntries(statusArtifact) {
+  return statusArtifact?.tests || [];
+}
+
+function setEntryDiagnosis(entry, diagnosis) {
+  if (entry?.target) {
+    entry.target.diagnosis = diagnosis;
+    return;
+  }
+  entry.diagnosis = diagnosis;
+}