@elench/testkit 0.1.79 → 0.1.81

package/lib/runner/{triage.test.mjs → regressions.test.mjs}

@@ -1,27 +1,24 @@
 import { describe, expect, it } from "vitest";
-import { normalizeKnownFailuresDocument } from "../known-failures/index.mjs";
-import { applyKnownFailuresToArtifacts } from "./triage.mjs";
+import { normalizeRegressionCatalogDocument } from "../regressions/index.mjs";
+import { applyRegressionAnalysisToArtifacts } from "./regressions.mjs";
 
-describe("runner triage", () => {
-  it("matches exact failure keys and enriches both artifacts", () => {
-    const knownFailures = normalizeKnownFailuresDocument({
+describe("runner regressions", () => {
+  it("classifies known failed regressions and enriches both artifacts", () => {
+    const regressionCatalog = normalizeRegressionCatalogDocument({
       schemaVersion: 1,
       issueRepo: "acme/repo",
       entries: [
         {
           id: "bad-message",
-          title: "Bad message bug",
           classification: "product_bug",
-          state: "open",
           issue: {
             repo: "acme/repo",
             number: 12,
-            url: "https://github.com/acme/repo/issues/12",
           },
-          description: "The API returns the wrong message.",
-          whyFailing: "The endpoint payload is wrong.",
-          lastReviewedAt: "2026-04-27",
-          matches: [
+          summary: "API returns the wrong message",
+          cause: "The endpoint payload is wrong.",
+          lastReviewedAt: "2026-05-04",
+          fingerprints: [
             {
               service: "api",
               type: "int",
@@ -80,36 +77,49 @@ describe("runner triage", () => {
       ],
     };
 
-    const enriched = applyKnownFailuresToArtifacts(runArtifact, statusArtifact, knownFailures);
-    expect(enriched.statusArtifact.tests[0].triage).toMatchObject({
-      status: "known_failure",
+    const enriched = applyRegressionAnalysisToArtifacts(
+      runArtifact,
+      statusArtifact,
+      regressionCatalog,
+      {
+        entries: [
+          {
+            id: "bad-message",
+            status: "open_and_failing",
+            github: {
+              state: "OPEN",
+              url: "https://github.com/acme/repo/issues/12",
+              cached: false,
+            },
+            findings: [],
+          },
+        ],
+        findings: [],
+      }
+    );
+
+    expect(enriched.statusArtifact.tests[0].diagnosis).toMatchObject({
+      status: "known_regression",
       classifications: ["product_bug"],
     });
-    expect(enriched.runArtifact.services[0].suites[0].files[0].triage.entries[0]).toMatchObject({
+    expect(enriched.runArtifact.services[0].suites[0].files[0].diagnosis.entries[0]).toMatchObject({
       id: "bad-message",
       issue: {
         number: 12,
       },
     });
-    expect(enriched.statusArtifact.triageSummary).toEqual({
-      failed: {
-        total: 1,
-        known: 1,
-        untriaged: 0,
-        byClassification: {
-          product_bug: 1,
-        },
-      },
-      entries: {
-        total: 1,
-        matchedByFailedTests: 1,
-        unmatched: 0,
-      },
+    expect(enriched.statusArtifact.regressions.summary).toEqual({
+      newRegressions: 0,
+      knownRegressions: 1,
+      fixedKnownRegressions: 0,
+      catalogStale: 0,
+      catalogSyncUnavailable: false,
+      usedStaleCache: false,
     });
   });
 
-  it("marks unmatched failed tests as untriaged", () => {
-    const enriched = applyKnownFailuresToArtifacts(
+  it("marks unmatched failed tests as new regressions and emits drafts", () => {
+    const enriched = applyRegressionAnalysisToArtifacts(
       {
         services: [
           {
@@ -140,15 +150,19 @@ describe("runner triage", () => {
           },
         ],
       },
-      normalizeKnownFailuresDocument({
+      normalizeRegressionCatalogDocument({
         schemaVersion: 1,
         entries: [],
-      })
+      }),
+      null
     );
 
-    expect(enriched.statusArtifact.tests[0].triage).toEqual({
-      status: "untriaged",
+    expect(enriched.statusArtifact.tests[0].diagnosis).toEqual({
+      status: "new_regression",
+      classifications: [],
       entries: [],
     });
+    expect(enriched.statusArtifact.regressions.summary.newRegressions).toBe(1);
+    expect(enriched.statusArtifact.regressions.drafts.newRegressions).toHaveLength(1);
   });
 });

package/lib/runner/reporting.mjs

@@ -80,7 +80,7 @@ export function buildStatusArtifact({
     scope.serviceFilter === null;
 
   return {
-    schemaVersion: 6,
+    schemaVersion: 7,
     source: "testkit",
     notice: "Generated file. Do not edit manually.",
     product: {
@@ -134,7 +134,7 @@ export function buildRunArtifact({
   const dbBackend = summarizeDbBackend(results);
 
   return {
-    schemaVersion: 8,
+    schemaVersion: 9,
     source: "testkit",
     generatedAt: new Date(finishedAt).toISOString(),
     product: {

package/lib/runner/reporting.test.mjs

@@ -79,7 +79,7 @@ describe("runner reporting", () => {
     });
 
     expect(artifact.product.name).toBe("my-product");
-    expect(artifact.schemaVersion).toBe(8);
+    expect(artifact.schemaVersion).toBe(9);
     expect(artifact.run).toMatchObject({
       workers: 2,
       fileTimeoutSeconds: 60,
@@ -233,7 +233,7 @@ describe("runner reporting", () => {
     });
 
     expect(status).toEqual({
-      schemaVersion: 6,
+      schemaVersion: 7,
       source: "testkit",
       notice: "Generated file. Do not edit manually.",
       product: {

package/lib/runner/run-finalization.mjs

@@ -1,11 +1,11 @@
 import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
-import { applyKnownFailureIssueValidationToArtifacts, applyKnownFailuresToArtifacts } from "./triage.mjs";
+import { applyRegressionAnalysisToArtifacts } from "./regressions.mjs";
 import { writeRunArtifact, writeStatusArtifact } from "./artifacts.mjs";
 import { summarizeDbBackend } from "./results.mjs";
 import { formatError } from "./formatting.mjs";
 import { uploadTelemetryArtifact } from "../telemetry/index.mjs";
 import { loadHistory, saveHistory, updateHistoryFromRunArtifact } from "../history/index.mjs";
-import { shouldFailKnownFailureIssueValidation, validateKnownFailureIssues } from "../known-failures/github.mjs";
+import { shouldFailRegressionSync, validateRegressionIssues } from "../regressions/github.mjs";
 
 export async function finalizeRunArtifacts({
   productDir,
@@ -20,8 +20,8 @@ export async function finalizeRunArtifacts({
   metadata,
   logRegistry,
   setupRegistry,
-  knownFailures,
-  issueValidationConfig,
+  regressionCatalog,
+  regressionSyncConfig,
   telemetry,
   reporter,
   writeStatus,
@@ -60,24 +60,19 @@ export async function finalizeRunArtifacts({
         metadata,
       })
     : null;
-  const enrichedArtifacts = applyKnownFailuresToArtifacts(runArtifact, statusArtifact, knownFailures);
-  const knownFailureIssueValidation = await validateKnownFailureIssues({
+  const regressionSync = await validateRegressionIssues({
     productDir,
-    document: knownFailures,
-    runArtifact: enrichedArtifacts.runArtifact,
-    statusArtifact: enrichedArtifacts.statusArtifact,
-    config: issueValidationConfig,
+    document: regressionCatalog,
+    runArtifact,
+    statusArtifact,
+    config: regressionSyncConfig,
     gitMetadata: metadata.git,
   });
-  applyKnownFailureIssueValidationToArtifacts(
-    enrichedArtifacts.runArtifact,
-    enrichedArtifacts.statusArtifact,
-    knownFailureIssueValidation
-  );
-  attachKnownFailureIssueValidation(
-    enrichedArtifacts.runArtifact,
-    enrichedArtifacts.statusArtifact,
-    knownFailureIssueValidation
+  const enrichedArtifacts = applyRegressionAnalysisToArtifacts(
+    runArtifact,
+    statusArtifact,
+    regressionCatalog,
+    regressionSync
   );
 
   writeRunArtifact(productDir, enrichedArtifacts.runArtifact);
@@ -91,14 +86,15 @@ export async function finalizeRunArtifacts({
   );
   saveHistory(productDir, nextHistory);
 
-  reporter?.runSummary?.(results, finishedAt - startedAt, knownFailureIssueValidation);
+  reporter?.runSummary?.(results, finishedAt - startedAt, enrichedArtifacts.regressionReport);
   await reportTelemetry(telemetry, enrichedArtifacts.runArtifact, reporter);
 
   return {
     runArtifact: enrichedArtifacts.runArtifact,
     statusArtifact: enrichedArtifacts.statusArtifact,
-    knownFailureIssueValidation,
-    shouldFailIssueValidation: shouldFailKnownFailureIssueValidation(knownFailureIssueValidation),
+    regressionReport: enrichedArtifacts.regressionReport,
+    regressionSync,
+    shouldFailRegressionSync: shouldFailRegressionSync(regressionSync),
   };
 }
 
@@ -122,11 +118,3 @@ async function reportTelemetry(telemetry, artifact, reporter = null) {
     reporter?.telemetry?.(`Telemetry: upload failed (${formatError(error)})`);
   }
 }
-
-function attachKnownFailureIssueValidation(runArtifact, statusArtifact, validation) {
-  if (!validation) return;
-  runArtifact.knownFailuresIssueValidation = validation;
-  if (statusArtifact) {
-    statusArtifact.knownFailuresIssueValidation = validation;
-  }
-}

package/lib/runner/template-steps.mjs

@@ -35,7 +35,7 @@ const PLAYWRIGHT_ENTRY = path.join(PACKAGE_ROOT, "lib", "playwright", "index.mjs
 const RUNTIME_ENTRY = path.join(PACKAGE_ROOT, "lib", "runtime", "index.mjs");
 const VITEST_ENTRY = path.join(PACKAGE_ROOT, "lib", "vitest", "index.mjs");
 const DISCOVERY_ENTRY = path.join(PACKAGE_ROOT, "lib", "discovery", "index.mjs");
-const KNOWN_FAILURES_ENTRY = path.join(PACKAGE_ROOT, "lib", "known-failures", "index.mjs");
+const REGRESSIONS_ENTRY = path.join(PACKAGE_ROOT, "lib", "regressions", "index.mjs");
 const MODULE_RUNNER_ENTRY = path.join(
   PACKAGE_ROOT,
   "lib",
@@ -263,7 +263,7 @@ function resolvePackageSubpath(specifier) {
   if (subpath === "/runtime") return RUNTIME_ENTRY;
   if (subpath === "/vitest") return VITEST_ENTRY;
   if (subpath === "/discovery") return DISCOVERY_ENTRY;
-  if (subpath === "/known-failures") return KNOWN_FAILURES_ENTRY;
+  if (subpath === "/regressions") return REGRESSIONS_ENTRY;
 
   throw new Error(`Unsupported @elench/testkit import "${specifier}" while loading template step`);
 }

package/lib/runner/worker-loop.mjs

@@ -81,6 +81,7 @@ export async function runWorker(
       } catch (error) {
         const message = formatError(error);
         errors.push(message);
+        reporter?.runtimeError?.(task, message);
         recordGraphError(trackers, { targetNames: lease?.context?.targetNames || [task.targetName] }, message);
         await runtimeManager.release(lease, {
           invalidate: lease !== null,

package/node_modules/@elench/next-analysis/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/next-analysis",
-  "version": "0.1.79",
+  "version": "0.1.81",
   "description": "SWC-backed Next.js source analysis primitives for Erench tools",
   "type": "module",
   "exports": {

package/node_modules/@elench/testkit-bridge/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/testkit-bridge",
-  "version": "0.1.79",
+  "version": "0.1.81",
   "description": "Browser bridge helpers for testkit",
   "type": "module",
   "main": "./dist/index.js",
@@ -22,7 +22,7 @@
     "typecheck": "tsc -p tsconfig.json --noEmit"
   },
   "dependencies": {
-    "@elench/testkit-protocol": "0.1.79"
+    "@elench/testkit-protocol": "0.1.81"
   },
   "private": false
 }

package/node_modules/@elench/testkit-protocol/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/testkit-protocol",
-  "version": "0.1.79",
+  "version": "0.1.81",
   "description": "Shared browser protocol for testkit bridge and extension consumers",
   "type": "module",
   "main": "./dist/index.js",

package/node_modules/@elench/ts-analysis/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/ts-analysis",
-  "version": "0.1.79",
+  "version": "0.1.81",
   "description": "TypeScript compiler-backed source analysis primitives for Erench tools",
   "type": "module",
   "exports": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/testkit",
-  "version": "0.1.79",
+  "version": "0.1.81",
   "description": "CLI for discovering and running local HTTP, DAL, and Playwright test suites",
   "type": "module",
   "workspaces": [
@@ -40,9 +40,9 @@
       "types": "./lib/discovery/index.d.ts",
       "default": "./lib/discovery/index.mjs"
     },
-    "./known-failures": {
-      "types": "./lib/known-failures/index.d.ts",
-      "default": "./lib/known-failures/index.mjs"
+    "./regressions": {
+      "types": "./lib/regressions/index.d.ts",
+      "default": "./lib/regressions/index.mjs"
     },
     "./package.json": "./package.json"
   },
@@ -81,10 +81,10 @@
   },
   "dependencies": {
     "@babel/code-frame": "^7.29.0",
-    "@elench/next-analysis": "0.1.79",
-    "@elench/testkit-bridge": "0.1.79",
-    "@elench/testkit-protocol": "0.1.79",
-    "@elench/ts-analysis": "0.1.79",
+    "@elench/next-analysis": "0.1.81",
+    "@elench/testkit-bridge": "0.1.81",
+    "@elench/testkit-protocol": "0.1.81",
+    "@elench/ts-analysis": "0.1.81",
     "@oclif/core": "^4.10.6",
     "esbuild": "^0.25.11",
     "execa": "^9.5.0",
@@ -92,7 +92,10 @@
     "ink": "^7.0.1",
     "picocolors": "^1.1.1",
     "react": "^19.2.5",
-    "typescript": "^5.9.3"
+    "string-width": "^8.1.0",
+    "strip-ansi": "^7.1.2",
+    "typescript": "^5.9.3",
+    "wrap-ansi": "^10.0.0"
   },
   "engines": {
     "node": ">=18"

package/lib/cli/commands/known-failures/render.mjs

@@ -1,19 +0,0 @@
-import { Command } from "@oclif/core";
-import { makeKnownFailuresFlags } from "../../command-helpers.mjs";
-import { runKnownFailuresRenderCommand } from "../../known-failures.mjs";
-
-export default class KnownFailuresRenderCommand extends Command {
-  static summary = "Render known failures markdown";
-
-  static enableJsonFlag = true;
-
-  static flags = makeKnownFailuresFlags();
-
-  async run() {
-    const { flags } = await this.parse(KnownFailuresRenderCommand);
-    await runKnownFailuresRenderCommand({
-      ...flags,
-    });
-    return { ok: true };
-  }
-}

package/lib/cli/commands/known-failures/validate.mjs

@@ -1,20 +0,0 @@
-import { Command } from "@oclif/core";
-import { makeKnownFailuresFlags } from "../../command-helpers.mjs";
-import { runKnownFailuresValidateCommand } from "../../known-failures.mjs";
-
-export default class KnownFailuresValidateCommand extends Command {
-  static summary = "Validate known failures against the latest status artifact";
-
-  static enableJsonFlag = true;
-
-  static flags = makeKnownFailuresFlags();
-
-  async run() {
-    const { flags } = await this.parse(KnownFailuresValidateCommand);
-    await runKnownFailuresValidateCommand({
-      ...flags,
-      issueMode: flags["issue-mode"] || null,
-    });
-    return { ok: true };
-  }
-}

package/lib/cli/known-failures.mjs

@@ -1,164 +0,0 @@
-import fs from "fs";
-import path from "path";
-import { resolveProductDir } from "../config/index.mjs";
-import { loadTestkitConfig } from "../config/config-loader.mjs";
-import {
-  buildKnownFailureIssueValidationSummaryLines,
-  normalizeKnownFailureIssueValidationConfig,
-  shouldFailKnownFailureIssueValidation,
-  validateKnownFailureIssues,
-} from "../known-failures/github.mjs";
-import {
-  loadKnownFailuresDocument,
-  renderKnownFailuresMarkdown,
-  validateKnownFailuresDocument,
-} from "../known-failures/index.mjs";
-import { collectGitMetadata } from "../runner/metadata.mjs";
-
-const DEFAULT_ISSUE_VALIDATION = {
-  provider: "github",
-  mode: "error",
-  cacheTtlSeconds: 900,
-};
-
-export async function runKnownFailuresRenderCommand(options = {}) {
-  const context = await resolveKnownFailuresContext(options);
-  const document = loadKnownFailuresDocument(context.knownFailuresPath, context.knownFailuresRelativePath);
-  const markdown = renderKnownFailuresMarkdown(document);
-
-  if (options.output) {
-    const outputPath = path.resolve(context.productDir, options.output);
-    fs.mkdirSync(path.dirname(outputPath), { recursive: true });
-    fs.writeFileSync(outputPath, markdown);
-    console.log(`Wrote ${path.relative(context.productDir, outputPath)}`);
-    return;
-  }
-
-  process.stdout.write(markdown);
-}
-
-export async function runKnownFailuresValidateCommand(options = {}) {
-  const context = await resolveKnownFailuresContext(options);
-  const document = loadKnownFailuresDocument(context.knownFailuresPath, context.knownFailuresRelativePath);
-  const statusArtifact = context.statusArtifactPath
-    ? readJsonIfExists(context.statusArtifactPath)
-    : undefined;
-
-  const documentValidation = validateKnownFailuresDocument(document, {
-    productDir: context.productDir,
-    statusArtifact,
-  });
-
-  const issueValidation = await validateKnownFailureIssues({
-    productDir: context.productDir,
-    document,
-    statusArtifact,
-    config: context.issueValidationConfig,
-    gitMetadata: collectGitMetadata(context.productDir),
-  });
-
-  const issueErrors = collectIssueMessages(issueValidation, "error");
-  const issueWarnings = collectIssueMessages(issueValidation, "warning");
-  const hasErrors =
-    documentValidation.errors.length > 0 ||
-    shouldFailKnownFailureIssueValidation(issueValidation) ||
-    issueErrors.length > 0;
-
-  if (hasErrors) {
-    console.error("Known failures validation failed:");
-    for (const error of documentValidation.errors) {
-      console.error(`- ${error}`);
-    }
-    for (const error of issueErrors) {
-      console.error(`- ${error}`);
-    }
-    if (documentValidation.warnings.length > 0 || issueWarnings.length > 0) {
-      console.error("");
-      for (const warning of documentValidation.warnings) {
-        console.error(`! ${warning}`);
-      }
-      for (const warning of issueWarnings) {
-        console.error(`! ${warning}`);
-      }
-    }
-    for (const line of buildKnownFailureIssueValidationSummaryLines(issueValidation)) {
-      console.error(line);
-    }
-    process.exitCode = 1;
-    return;
-  }
-
-  console.log(
-    `Known failures valid: ${documentValidation.stats.entries} entries, ${documentValidation.stats.matches} matches`
-  );
-  if (documentValidation.stats.failedTests > 0) {
-    console.log(
-      `Status coverage: ${documentValidation.stats.triagedFailedTests}/${documentValidation.stats.failedTests} failed tests triaged`
-    );
-  }
-  for (const line of buildKnownFailureIssueValidationSummaryLines(issueValidation)) {
-    console.log(line);
-  }
-  for (const warning of [...documentValidation.warnings, ...issueWarnings]) {
-    console.warn(`! ${warning}`);
-  }
-}
-
-async function resolveKnownFailuresContext(options) {
-  const productDir = resolveProductDir(process.cwd(), options.dir);
-  const { config } = await loadTestkitConfig(productDir);
-  const reporting = config?.reporting || {};
-
-  const knownFailuresRelativePath = normalizeOptionalString(options.input)
-    || normalizeOptionalString(reporting.knownFailuresFile);
-  if (!knownFailuresRelativePath) {
-    throw new Error(
-      "Known failures file not configured. Set reporting.knownFailuresFile in testkit.config.ts or pass --input."
-    );
-  }
-
-  const statusRelativePath = normalizeOptionalString(options.status) || "testkit.status.json";
-  const configuredIssueValidation = normalizeKnownFailureIssueValidationConfig(reporting.issueValidation)
-    || DEFAULT_ISSUE_VALIDATION;
-  const issueMode = normalizeOptionalString(options.issueMode) || "error";
-
-  return {
-    productDir,
-    knownFailuresRelativePath,
-    knownFailuresPath: path.resolve(productDir, knownFailuresRelativePath),
-    statusArtifactPath: statusRelativePath ? path.resolve(productDir, statusRelativePath) : null,
-    issueValidationConfig: {
-      ...configuredIssueValidation,
-      mode: issueMode,
-    },
-  };
-}
-
-function collectIssueMessages(validation, severity) {
-  if (!validation) return [];
-
-  const messages = [
-    ...validation.findings
-      .filter((finding) => finding.severity === severity)
-      .map((finding) => finding.message),
-  ];
-  for (const entry of validation.entries) {
-    for (const finding of entry.findings) {
-      if (finding.severity === severity) {
-        messages.push(finding.message);
-      }
-    }
-  }
-  return [...new Set(messages)];
-}
-
-function normalizeOptionalString(value) {
-  if (typeof value !== "string") return null;
-  const normalized = value.trim();
-  return normalized.length > 0 ? normalized : null;
-}
-
-function readJsonIfExists(filePath) {
-  if (!fs.existsSync(filePath)) return undefined;
-  return JSON.parse(fs.readFileSync(filePath, "utf8"));
-}