@elench/testkit 0.1.79 → 0.1.81

package/README.md

@@ -62,9 +62,8 @@ npx @elench/testkit artifacts __testkit__/health/health.int.testkit.ts
 npx @elench/testkit logs __testkit__/health/health.int.testkit.ts
 npx @elench/testkit watch
 
-# Known-failures tooling
-npx @elench/testkit known-failures validate --issue-mode error
-npx @elench/testkit known-failures render --output KNOWN_FAILURES.md
+# Automatic regression intelligence
+# Configure testkit.regressions.json and testkit classifies new vs known regressions automatically during runs
 
 # Capture a template DB schema snapshot
 npx @elench/testkit db snapshot capture --service api --output scripts/testkit/schema-baseline.sql
@@ -81,6 +80,29 @@ persisted under `.testkit/results/` and inspected on demand with `show`,
 run counts, pass/fail/skip counts, average duration, and last observed status,
 and those summaries are exposed in compact, verbose, and JSON discovery output.
 
+## Automatic Regression Diagnosis
+
+If `regressions.file` is configured, every run automatically classifies observed
+results without any separate follow-up maintenance command.
+
+`testkit` distinguishes four user-facing outcomes:
+
+- `new regressions`
+- `known regressions`
+- `fixed known regressions`
+- `catalog stale`
+
+The default CLI keeps those signals lightweight:
+
+- failed files print inline diagnosis immediately under the file line
+- the final summary box reports aggregate regression counts only
+- machine-readable artifacts gain per-file `diagnosis` plus top-level
+  `regressions.summary`, `regressions.catalog`, and prepared `regressions.drafts`
+
+`catalog stale` is repo hygiene, not a test failure. It means the regression
+catalog or linked issue tracker metadata needs attention, for example because a
+linked issue is closed but the regression still reproduces.
+
 ## Tooling Adapters
 
 `testkit` also ships tool-specific config helpers so consumer repos do not need
@@ -159,9 +181,9 @@ export default defineConfig({
     workers: 8,
     fileTimeoutSeconds: 60,
   },
-  reporting: {
-    knownFailuresFile: "testkit.known-failures.json",
-    issueValidation: {
+  regressions: {
+    file: "testkit.regressions.json",
+    sync: {
       provider: "github",
       mode: "warn",
       cacheTtlSeconds: 900,
@@ -240,8 +262,8 @@ for:
 - template schema snapshot capture
 - explicit per-file or per-suite locks
 - named HTTP suite profiles
-- known-failure annotation merge for enriched status/run artifacts
-- optional GitHub-backed known-failure issue validation
+- automatic regression classification for new vs known failures
+- optional GitHub-backed regression issue sync
 - repo-declared suite/file skip policies with explicit reasons
 - telemetry upload configuration
 
@@ -303,47 +325,40 @@ services: {
 }
 ```
 
-If `reporting.knownFailuresFile` is configured, `testkit` enriches
+If `regressions.file` is configured, `testkit` enriches
 `.testkit/results/latest.json` and `testkit.status.json` with:
 
 - per-file `failureDetails`
-- per-file `triage` metadata (issue, classification, description)
-- top-level `triageSummary` counts for known vs untriaged failures
-
-Known-failure entry authoring uses this contract:
-
-- `title`
-  - exact issue-tracker title for the linked issue
-  - shared issues may reuse the same title across multiple local entries
-- `description`
-  - product-local bug slice or route-family summary
-  - this is where file-specific nuance belongs
-- `whyFailing`
+- per-file `diagnosis` metadata (new regression, known regression, fixed known regression)
+- top-level `regressions` summary and prepared draft updates
+
+Regression-catalog entry authoring uses this contract:
+
+- `summary`
+  - concise local statement of the regression slice
+- `cause`
   - underlying technical cause of the failure
+- `fingerprints`
+  - selectors that let testkit automatically recognize the regression in future runs
 
-If `reporting.issueValidation` is also configured, `testkit` validates known-failure
-issue references against GitHub and adds top-level `knownFailuresIssueValidation`
-data to the run/status artifacts. The most important stale-triage signal is:
+If `regressions.sync` is also configured, `testkit` syncs linked GitHub issues and
+adds top-level regression catalog health to the run/status artifacts. The most
+important catalog-staleness signal is:
 
-- a known-failure test still fails, but the linked GitHub issue is closed
+- a known regression still fails, but the linked GitHub issue is closed
 
-In `mode: "error"`, exact GitHub metadata drift is also treated as a validation
-failure:
+In `mode: "error"`, catalog health can also fail the run for problems such as:
 
-- title does not match the linked issue title
-- local open/closed state does not match the linked issue state
+- closed issues that still reproduce
+- missing issue refs
+- validation unavailability
 
 Reproduction warnings are execution-aware:
 
-- `failed` means the known failure reproduced
+- `failed` means the known regression reproduced
 - `passed` means the matched test executed and did not reproduce
 - `skipped` and `not_run` do not count as reproduction evidence
 
-Known-failure operations are also available as first-class CLI commands:
-
-- `testkit known-failures validate`
-- `testkit known-failures render`
-
 ## Authoring
 
 HTTP suites:

package/lib/cli/args.mjs

@@ -7,29 +7,17 @@ import {
 
 export const POSITIONAL_TYPES = new Set(["int", "e2e", "scenario", "dal", "load", "pw", "all"]);
 export const LIFECYCLE = new Set(["status", "destroy", "cleanup"]);
-export const KNOWN_FAILURES_ACTIONS = new Set(["render", "validate"]);
 export const RESERVED = new Set([...POSITIONAL_TYPES, ...LIFECYCLE]);
 
 export function resolveCliSelection({ first, second, third }) {
   let lifecycle = null;
   let positionalType = null;
-  let knownFailuresAction = null;
   let dbAction = null;
 
-  if (first === "known-failures") {
-    if (!second || !KNOWN_FAILURES_ACTIONS.has(second) || third) {
-      throw new Error(
-        'Unknown known-failures command. Expected "known-failures render" or "known-failures validate".'
-      );
-    }
-    knownFailuresAction = second;
-    return { lifecycle, positionalType, knownFailuresAction, dbAction };
-  }
-
   if (first === "db") {
     if (second === "snapshot" && third === "capture") {
       dbAction = "snapshot-capture";
-      return { lifecycle, positionalType, knownFailuresAction, dbAction };
+      return { lifecycle, positionalType, dbAction };
     }
     throw new Error('Unknown db command. Expected "db snapshot capture".');
   }
@@ -49,7 +37,7 @@ export function resolveCliSelection({ first, second, third }) {
     );
   }
 
-  return { lifecycle, positionalType, knownFailuresAction, dbAction };
+  return { lifecycle, positionalType, dbAction };
 }
 
 export function parseTypeOption(values, positionalType = null) {

package/lib/cli/args.test.mjs

@@ -19,7 +19,6 @@ describe("cli-args", () => {
       })
     ).toEqual({
       dbAction: null,
-      knownFailuresAction: null,
       lifecycle: null,
       positionalType: "int",
     });
@@ -34,26 +33,12 @@ describe("cli-args", () => {
       })
     ).toEqual({
       dbAction: null,
-      knownFailuresAction: null,
       lifecycle: "status",
       positionalType: null,
     });
   });
 
-  it("resolves known-failures and db subcommands", () => {
-    expect(
-      resolveCliSelection({
-        first: "known-failures",
-        second: "render",
-        third: null,
-      })
-    ).toEqual({
-      dbAction: null,
-      knownFailuresAction: "render",
-      lifecycle: null,
-      positionalType: null,
-    });
-
+  it("resolves db subcommands", () => {
     expect(
       resolveCliSelection({
         first: "db",
@@ -62,7 +47,6 @@ describe("cli-args", () => {
       })
     ).toEqual({
       dbAction: "snapshot-capture",
-      knownFailuresAction: null,
       lifecycle: null,
       positionalType: null,
     });

package/lib/cli/command-helpers.mjs

@@ -1,5 +1,5 @@
-import path from "path";
 import { Flags } from "@oclif/core";
+import path from "path";
 import { loadManagedConfigs } from "../app/configs.mjs";
 import {
   parseFileTimeoutOption,
@@ -142,25 +142,6 @@ export async function runStatusLike(commandName, flags) {
   return { ok: true, results: productResults };
 }
 
-export function makeKnownFailuresFlags() {
-  return {
-    ...sharedFlags,
-    input: Flags.string({
-      description: "Known failures JSON path (repo-relative)",
-    }),
-    output: Flags.string({
-      description: "Output path",
-    }),
-    status: Flags.string({
-      description: "Status artifact path",
-    }),
-    "issue-mode": Flags.string({
-      description: "Issue validation mode override: off, warn, error",
-      options: ["off", "warn", "error"],
-    }),
-  };
-}
-
 export function relativeToProduct(productDir, targetPath) {
   return path.relative(productDir, targetPath);
 }

package/lib/cli/entrypoint.mjs

@@ -12,7 +12,6 @@ export function normalizeCliArgs(argv) {
     "typecheck",
     "doctor",
     "browser",
-    "known-failures",
     "db",
     "help",
     "--help",
@@ -74,9 +73,6 @@ function findPositionals(args, flagsWithValues) {
 
 function reorderCommandArgs(args, positionals) {
   const commandTokens = [positionals[0]];
-  if (positionals[0]?.value === "known-failures" && positionals[1]) {
-    commandTokens.push(positionals[1]);
-  }
   if (positionals[0]?.value === "db" && positionals[1] && positionals[2]) {
     commandTokens.push(positionals[1], positionals[2]);
   }

package/lib/cli/presentation/colors.mjs

@@ -70,7 +70,7 @@ export function colorResultLine(line) {
 }
 
 export function colorSectionLine(line) {
-  if (line === "Failures:" || line === "Runtime Errors:" || line === "Known-failure issues:" || line === "Triage:") {
+  if (line === "Catalog issues:" || line === "Diagnosis:") {
     return pc.bold(line);
   }
   if (/^Summary: /.test(line)) return line.replace("Summary:", `${pc.bold("Summary:")}`);

package/lib/cli/presentation/failure-presentation.mjs

@@ -0,0 +1,31 @@
+import { buildFailurePresentation } from "../../runner/formatting.mjs";
+import { renderIndentedBlock } from "./terminal-layout.mjs";
+
+export function renderFailureBlock(task, outcome, { width, regressionCatalog } = {}) {
+  const presentation = buildFailurePresentation(
+    {
+      service: task.serviceName,
+      type: normalizeRegressionType(task),
+      path: task.file,
+      error: outcome.error || null,
+      failureDetails: Array.isArray(outcome.failureDetails) ? outcome.failureDetails : [],
+      suiteError: null,
+    },
+    regressionCatalog
+  );
+
+  const lines = [];
+  if (presentation.primary) {
+    lines.push(...renderIndentedBlock(presentation.primary, { width, indent: "    " }));
+  }
+  for (const detail of presentation.details) {
+    lines.push(...renderIndentedBlock(detail, { width, indent: "    " }));
+  }
+  return lines;
+}
+
+function normalizeRegressionType(task) {
+  if (task.framework === "playwright") return "pw";
+  if (task.type === "integration") return "int";
+  return task.type;
+}

package/lib/cli/presentation/run-reporter.mjs

@@ -1,89 +1,28 @@
 import path from "path";
-import figures from "figures";
 import {
-  buildCompactRunSummaryLines,
+  buildRunSummaryData,
   buildDebugRunSummaryLines,
   formatDuration,
 } from "../../runner/formatting.mjs";
-import { boldRed, colorSectionLine, dim, red, statusLabel } from "./colors.mjs";
-
-const ANSI_RE = /\x1b\[[0-9;]*m/g;
-const DURATION_RE = /\b(\d+m\s+\d+s|\d+s)\b/g;
-
-function stripAnsi(text) {
-  return text.replace(ANSI_RE, "");
-}
-
-function boxLines(lines) {
-  const nonEmpty = lines.filter((l) => stripAnsi(l).trim().length > 0);
-  if (nonEmpty.length === 0) return lines;
-  const maxWidth = lines.reduce((max, l) => Math.max(max, stripAnsi(l).length), 0);
-  const pad = 1;
-  const innerWidth = maxWidth + pad * 2;
-  const top = `${figures.lineDownRight}${figures.line.repeat(innerWidth)}${figures.lineDownLeft}`;
-  const bottom = `${figures.lineUpRight}${figures.line.repeat(innerWidth)}${figures.lineUpLeft}`;
-  const boxed = [top];
-  for (const line of lines) {
-    const visible = stripAnsi(line).length;
-    const right = innerWidth - pad - visible;
-    boxed.push(`${figures.lineVertical}${" ".repeat(pad)}${line}${" ".repeat(Math.max(0, right))}${figures.lineVertical}`);
-  }
-  boxed.push(bottom);
-  return boxed;
-}
-
-function colorFailureSummaryLines(lines) {
-  const result = [];
-  let inFailuresSection = false;
-  let lastWasFilePath = false;
-  for (const line of lines) {
-    const raw = stripAnsi(line);
-    if (raw === "Failures:" || raw === "Runtime Errors:") {
-      inFailuresSection = true;
-      lastWasFilePath = false;
-      result.push(line);
-      continue;
-    }
-    if (/^(Summary:|Result:|Known-failure|Triage:)/.test(raw) || raw === "") {
-      inFailuresSection = false;
-      lastWasFilePath = false;
-    }
-    if (inFailuresSection) {
-      if (/^ {2}\S/.test(raw)) {
-        lastWasFilePath = true;
-        result.push(boldRed(raw));
-        continue;
-      }
-      if (/^ {4}\S/.test(raw)) {
-        if (lastWasFilePath) {
-          lastWasFilePath = false;
-          result.push(boldRed(raw));
-        } else {
-          result.push(red(raw));
-        }
-        continue;
-      }
-    }
-    lastWasFilePath = false;
-    result.push(line);
-  }
-  return result;
-}
-
-function dimDurations(line) {
-  return line.replace(DURATION_RE, (match) => dim(match));
-}
+import { boldRed, colorSectionLine, dim, statusLabel } from "./colors.mjs";
+import { renderFailureBlock } from "./failure-presentation.mjs";
+import { renderSummaryBox } from "./summary-box.mjs";
+import { getTerminalWidth } from "./terminal-layout.mjs";
 
 export function createRunReporter({ outputMode = "compact", stdout = process.stdout, stderr = process.stderr } = {}) {
   const mode = outputMode || "compact";
   let completedCount = 0;
   let totalFileCount = 0;
+  let regressionCatalog = null;
 
   return {
     outputMode: mode,
     setTotalFileCount(count) {
       totalFileCount = count;
     },
+    setRegressionCatalog(document) {
+      regressionCatalog = document;
+    },
     writeLine(line = "") {
       stdout.write(`${line}\n`);
     },
@@ -142,32 +81,72 @@ export function createRunReporter({ outputMode = "compact", stdout = process.std
       completedCount += 1;
       const status = outcome.status === "skipped" ? "SKIP" : outcome.failed ? "FAIL" : "PASS";
       const duration = formatDuration(outcome.durationMs || 0);
-      const primaryFailure = firstFailureDetail(outcome);
-      const preferredFailure =
-        primaryFailure && isThresholdWrapperMessage(outcome.error) ? primaryFailure : outcome.error || primaryFailure;
       const detail =
-        status === "FAIL"
-          ? ` ${boldRed(shortenMessage(preferredFailure || "failed"))}`
-          : outcome.status === "not_run"
+        outcome.status === "not_run"
             ? ` ${dim(shortenMessage(outcome.reason || "not run"))}`
             : "";
       const progress = mode === "compact" && totalFileCount > 0 ? `${dim(`[${completedCount}/${totalFileCount}]`)} ` : "";
       stdout.write(
         `${progress}${statusLabel(status)} ${task.serviceName} ${displayTaskType(task)} ${normalizePath(task.file)} ${dim(duration)}${detail}\n`
       );
+      if (status === "FAIL") {
+        const detailLines = renderFailureBlock(task, outcome, {
+          width: getTerminalWidth(stdout, 100),
+          regressionCatalog,
+        });
+        for (const line of detailLines) {
+          stdout.write(`${line}\n`);
+        }
+      }
+    },
+    runtimeError(task, message) {
+      if (mode === "json") return;
+      stdout.write(`${statusLabel("FAIL")} ${task.serviceName} runtime error\n`);
+      stdout.write(`    ${boldRed(shortenMessage(message || "runtime error"))}\n`);
     },
     telemetry(message) {
       if (mode === "json") return;
       stdout.write(`${message}\n`);
     },
-    runSummary(results, durationMs, knownFailureIssueValidation = null) {
-      const lines =
-        mode === "debug"
-          ? buildDebugRunSummaryLines(results, durationMs, knownFailureIssueValidation)
-          : buildCompactRunSummaryLines(results, durationMs, knownFailureIssueValidation);
-      const colored = colorFailureSummaryLines(lines.map((line) => colorSectionLine(line)));
-      const dimmed = colored.map((line) => dimDurations(line));
-      const boxed = boxLines(dimmed);
+    runSummary(results, durationMs, regressionReport = null) {
+      if (mode === "debug") {
+        const lines = buildDebugRunSummaryLines(results, durationMs, regressionReport);
+        stdout.write("\n");
+        for (const line of lines) stdout.write(`${colorSectionLine(line)}\n`);
+        return;
+      }
+
+      const summary = buildRunSummaryData(results, durationMs, regressionReport);
+      const rows = [
+        ["Result", summary.result],
+        ["Passed", String(summary.passed)],
+        ["Failed", String(summary.failed)],
+        ["Skipped", String(summary.skipped)],
+        ["Not run", String(summary.notRun)],
+        ["Files", String(summary.files)],
+        ["Duration", summary.duration],
+      ];
+      if (summary.serviceErrors > 0) {
+        rows.push(["Runtime errors", String(summary.serviceErrors)]);
+      }
+      if (summary.newRegressions > 0) {
+        rows.push(["New regressions", String(summary.newRegressions)]);
+      }
+      if (summary.knownRegressions > 0) {
+        rows.push(["Known regressions", String(summary.knownRegressions)]);
+      }
+      if (summary.fixedKnownRegressions > 0) {
+        rows.push(["Fixed known", String(summary.fixedKnownRegressions)]);
+      }
+      if (summary.catalogStale > 0) {
+        rows.push(["Catalog stale", String(summary.catalogStale)]);
+      }
+      if (summary.catalogSyncUnavailable) {
+        rows.push(["Catalog sync", "Unavailable"]);
+      } else if (summary.usedStaleCache) {
+        rows.push(["Catalog sync", "Used stale cache"]);
+      }
+      const boxed = renderSummaryBox(rows, { stdout });
       stdout.write("\n");
       for (const line of boxed) stdout.write(`${line}\n`);
     },
@@ -187,15 +166,6 @@ function shortenMessage(message) {
   return String(message).replace(/\s+/g, " ").trim().slice(0, 180);
 }
 
-function firstFailureDetail(outcome) {
-  const detail = outcome.failureDetails?.[0];
-  return detail?.message || detail?.title || null;
-}
-
-function isThresholdWrapperMessage(message) {
-  return /Default runtime thresholds failed:/.test(String(message || ""));
-}
-
 function normalizePath(filePath) {
   return String(filePath).split(path.sep).join("/");
 }