@elench/testkit 0.1.80 → 0.1.82

package/README.md

@@ -62,9 +62,8 @@ npx @elench/testkit artifacts __testkit__/health/health.int.testkit.ts
 npx @elench/testkit logs __testkit__/health/health.int.testkit.ts
 npx @elench/testkit watch
 
-# Known-failures tooling
-npx @elench/testkit known-failures validate --issue-mode error
-npx @elench/testkit known-failures render --output KNOWN_FAILURES.md
+# Automatic regression intelligence
+# Configure testkit.regressions.json and testkit classifies new vs known regressions automatically during runs
 
 # Capture a template DB schema snapshot
 npx @elench/testkit db snapshot capture --service api --output scripts/testkit/schema-baseline.sql
@@ -81,6 +80,29 @@ persisted under `.testkit/results/` and inspected on demand with `show`,
 run counts, pass/fail/skip counts, average duration, and last observed status,
 and those summaries are exposed in compact, verbose, and JSON discovery output.
 
+## Automatic Regression Diagnosis
+
+If `regressions.file` is configured, every run automatically classifies observed
+results without any separate follow-up maintenance command.
+
+`testkit` distinguishes four user-facing outcomes:
+
+- `new regressions`
+- `known regressions`
+- `fixed known regressions`
+- `catalog stale`
+
+The default CLI keeps those signals lightweight:
+
+- failed files print inline diagnosis immediately under the file line
+- the final summary box reports aggregate regression counts only
+- machine-readable artifacts gain per-file `diagnosis` plus top-level
+  `regressions.summary`, `regressions.catalog`, and prepared `regressions.drafts`
+
+`catalog stale` is repo hygiene, not a test failure. It means the regression
+catalog or linked issue tracker metadata needs attention, for example because a
+linked issue is closed but the regression still reproduces.
+
 ## Tooling Adapters
 
 `testkit` also ships tool-specific config helpers so consumer repos do not need
@@ -159,9 +181,9 @@ export default defineConfig({
     workers: 8,
     fileTimeoutSeconds: 60,
   },
-  reporting: {
-    knownFailuresFile: "testkit.known-failures.json",
-    issueValidation: {
+  regressions: {
+    file: "testkit.regressions.json",
+    sync: {
       provider: "github",
       mode: "warn",
       cacheTtlSeconds: 900,
@@ -240,8 +262,8 @@ for:
 - template schema snapshot capture
 - explicit per-file or per-suite locks
 - named HTTP suite profiles
-- known-failure annotation merge for enriched status/run artifacts
-- optional GitHub-backed known-failure issue validation
+- automatic regression classification for new vs known failures
+- optional GitHub-backed regression issue sync
 - repo-declared suite/file skip policies with explicit reasons
 - telemetry upload configuration
 
@@ -303,47 +325,40 @@ services: {
 }
 ```
 
-If `reporting.knownFailuresFile` is configured, `testkit` enriches
+If `regressions.file` is configured, `testkit` enriches
 `.testkit/results/latest.json` and `testkit.status.json` with:
 
 - per-file `failureDetails`
-- per-file `triage` metadata (issue, classification, description)
-- top-level `triageSummary` counts for known vs untriaged failures
-
-Known-failure entry authoring uses this contract:
-
-- `title`
-  - exact issue-tracker title for the linked issue
-  - shared issues may reuse the same title across multiple local entries
-- `description`
-  - product-local bug slice or route-family summary
-  - this is where file-specific nuance belongs
-- `whyFailing`
+- per-file `diagnosis` metadata (new regression, known regression, fixed known regression)
+- top-level `regressions` summary and prepared draft updates
+
+Regression-catalog entry authoring uses this contract:
+
+- `summary`
+  - concise local statement of the regression slice
+- `cause`
   - underlying technical cause of the failure
+- `fingerprints`
+  - selectors that let testkit automatically recognize the regression in future runs
 
-If `reporting.issueValidation` is also configured, `testkit` validates known-failure
-issue references against GitHub and adds top-level `knownFailuresIssueValidation`
-data to the run/status artifacts. The most important stale-triage signal is:
+If `regressions.sync` is also configured, `testkit` syncs linked GitHub issues and
+adds top-level regression catalog health to the run/status artifacts. The most
+important catalog-staleness signal is:
 
-- a known-failure test still fails, but the linked GitHub issue is closed
+- a known regression still fails, but the linked GitHub issue is closed
 
-In `mode: "error"`, exact GitHub metadata drift is also treated as a validation
-failure:
+In `mode: "error"`, catalog health can also fail the run for problems such as:
 
-- title does not match the linked issue title
-- local open/closed state does not match the linked issue state
+- closed issues that still reproduce
+- missing issue refs
+- validation unavailability
 
 Reproduction warnings are execution-aware:
 
-- `failed` means the known failure reproduced
+- `failed` means the known regression reproduced
 - `passed` means the matched test executed and did not reproduce
 - `skipped` and `not_run` do not count as reproduction evidence
 
-Known-failure operations are also available as first-class CLI commands:
-
-- `testkit known-failures validate`
-- `testkit known-failures render`
-
 ## Authoring
 
 HTTP suites:
@@ -365,32 +380,39 @@ Named HTTP profiles live in `testkit.config.ts` and can be referenced by name:
 
 ```ts
 import { defineHttpSuite } from "@elench/testkit";
-import { defineConfig, profiles } from "@elench/testkit/config";
+import { auth, defineConfig } from "@elench/testkit/config";
+
+const appAuth = auth.fixture({
+  contract: auth.contracts.jsonSession({
+    authCookie: "session",
+    organizationIdPath: "data.organizations[0].id",
+  }),
+  topology: auth.topologies.crossOrg({
+    namespace: "example-app",
+    actors: {
+      primary: { org: "primary" },
+      reviewer: { org: "primary" },
+      outsider: { org: "secondary" },
+    },
+  }),
+});
 
 export default defineConfig({
   profiles: {
-    http: {
-      defaultAuth: profiles.localJson({
-        password: "password",
-        identities: {
-          primary: {
-            email: "test@example.com",
-          },
-        },
-        session: {
-          authCookie: "session",
-        },
-        headers: {
-          contentTypeJson: true,
-          forwardedFor: "deterministic",
-        },
-      }).session(),
-    },
+    http: appAuth.profiles({
+      defaultAuth: auth.profile.actor("primary"),
+      reviewers: auth.profile.actors({
+        actors: ["reviewer", "outsider"],
+        primaryActor: "reviewer",
+      }),
+      raw: auth.profile.raw(),
+    }),
   },
 });
 
-const suite = defineHttpSuite({ profile: "defaultAuth" }, ({ req, setupData }) => {
-  req("GET", "/api/auth/session", setupData);
+const suite = defineHttpSuite({ profile: "defaultAuth" }, ({ actor, req }) => {
+  req.get("/api/auth/session");
+  actor?.req.get("/api/auth/session");
 });
 ```
 
@@ -440,7 +462,7 @@ Consumers should not set local timeout values in test files.
 import { waitFor } from "@elench/testkit/runtime";
 
 const response = waitFor(
-  () => req("GET", "/api/v1/jobs/123", setupData),
+  () => req.get("/api/v1/jobs/123"),
   (res) => JSON.parse(res.body).data?.status === "completed",
   { description: "job 123 to complete" }
 );

package/lib/cli/args.mjs

@@ -7,29 +7,17 @@ import {
 
 export const POSITIONAL_TYPES = new Set(["int", "e2e", "scenario", "dal", "load", "pw", "all"]);
 export const LIFECYCLE = new Set(["status", "destroy", "cleanup"]);
-export const KNOWN_FAILURES_ACTIONS = new Set(["render", "validate"]);
 export const RESERVED = new Set([...POSITIONAL_TYPES, ...LIFECYCLE]);
 
 export function resolveCliSelection({ first, second, third }) {
   let lifecycle = null;
   let positionalType = null;
-  let knownFailuresAction = null;
   let dbAction = null;
 
-  if (first === "known-failures") {
-    if (!second || !KNOWN_FAILURES_ACTIONS.has(second) || third) {
-      throw new Error(
-        'Unknown known-failures command. Expected "known-failures render" or "known-failures validate".'
-      );
-    }
-    knownFailuresAction = second;
-    return { lifecycle, positionalType, knownFailuresAction, dbAction };
-  }
-
   if (first === "db") {
     if (second === "snapshot" && third === "capture") {
       dbAction = "snapshot-capture";
-      return { lifecycle, positionalType, knownFailuresAction, dbAction };
+      return { lifecycle, positionalType, dbAction };
     }
     throw new Error('Unknown db command. Expected "db snapshot capture".');
   }
@@ -49,7 +37,7 @@ export function resolveCliSelection({ first, second, third }) {
     );
   }
 
-  return { lifecycle, positionalType, knownFailuresAction, dbAction };
+  return { lifecycle, positionalType, dbAction };
 }
 
 export function parseTypeOption(values, positionalType = null) {

package/lib/cli/args.test.mjs

@@ -19,7 +19,6 @@ describe("cli-args", () => {
       })
     ).toEqual({
       dbAction: null,
-      knownFailuresAction: null,
       lifecycle: null,
       positionalType: "int",
     });
@@ -34,26 +33,12 @@ describe("cli-args", () => {
       })
     ).toEqual({
       dbAction: null,
-      knownFailuresAction: null,
       lifecycle: "status",
       positionalType: null,
     });
   });
 
-  it("resolves known-failures and db subcommands", () => {
-    expect(
-      resolveCliSelection({
-        first: "known-failures",
-        second: "render",
-        third: null,
-      })
-    ).toEqual({
-      dbAction: null,
-      knownFailuresAction: "render",
-      lifecycle: null,
-      positionalType: null,
-    });
-
+  it("resolves db subcommands", () => {
     expect(
       resolveCliSelection({
         first: "db",
@@ -62,7 +47,6 @@ describe("cli-args", () => {
       })
     ).toEqual({
       dbAction: "snapshot-capture",
-      knownFailuresAction: null,
       lifecycle: null,
       positionalType: null,
     });

package/lib/cli/command-helpers.mjs

@@ -1,5 +1,5 @@
-import path from "path";
 import { Flags } from "@oclif/core";
+import path from "path";
 import { loadManagedConfigs } from "../app/configs.mjs";
 import {
   parseFileTimeoutOption,
@@ -142,25 +142,6 @@ export async function runStatusLike(commandName, flags) {
   return { ok: true, results: productResults };
 }
 
-export function makeKnownFailuresFlags() {
-  return {
-    ...sharedFlags,
-    input: Flags.string({
-      description: "Known failures JSON path (repo-relative)",
-    }),
-    output: Flags.string({
-      description: "Output path",
-    }),
-    status: Flags.string({
-      description: "Status artifact path",
-    }),
-    "issue-mode": Flags.string({
-      description: "Issue validation mode override: off, warn, error",
-      options: ["off", "warn", "error"],
-    }),
-  };
-}
-
 export function relativeToProduct(productDir, targetPath) {
   return path.relative(productDir, targetPath);
 }

package/lib/cli/entrypoint.mjs

@@ -12,7 +12,6 @@ export function normalizeCliArgs(argv) {
     "typecheck",
     "doctor",
     "browser",
-    "known-failures",
     "db",
     "help",
     "--help",
@@ -74,9 +73,6 @@ function findPositionals(args, flagsWithValues) {
 
 function reorderCommandArgs(args, positionals) {
   const commandTokens = [positionals[0]];
-  if (positionals[0]?.value === "known-failures" && positionals[1]) {
-    commandTokens.push(positionals[1]);
-  }
   if (positionals[0]?.value === "db" && positionals[1] && positionals[2]) {
     commandTokens.push(positionals[1], positionals[2]);
   }

package/lib/cli/presentation/colors.mjs

@@ -70,7 +70,7 @@ export function colorResultLine(line) {
 }
 
 export function colorSectionLine(line) {
-  if (line === "Known-failure issues:" || line === "Triage:") {
+  if (line === "Catalog issues:" || line === "Diagnosis:") {
     return pc.bold(line);
   }
   if (/^Summary: /.test(line)) return line.replace("Summary:", `${pc.bold("Summary:")}`);

package/lib/cli/presentation/failure-presentation.mjs

@@ -1,17 +1,17 @@
 import { buildFailurePresentation } from "../../runner/formatting.mjs";
 import { renderIndentedBlock } from "./terminal-layout.mjs";
 
-export function renderFailureBlock(task, outcome, { width, knownFailures } = {}) {
+export function renderFailureBlock(task, outcome, { width, regressionCatalog } = {}) {
   const presentation = buildFailurePresentation(
     {
       service: task.serviceName,
-      type: normalizeKnownFailureType(task),
+      type: normalizeRegressionType(task),
       path: task.file,
       error: outcome.error || null,
       failureDetails: Array.isArray(outcome.failureDetails) ? outcome.failureDetails : [],
       suiteError: null,
     },
-    knownFailures
+    regressionCatalog
   );
 
   const lines = [];
@@ -24,7 +24,7 @@ export function renderFailureBlock(task, outcome, { width, knownFailures } = {})
   return lines;
 }
 
-function normalizeKnownFailureType(task) {
+function normalizeRegressionType(task) {
   if (task.framework === "playwright") return "pw";
   if (task.type === "integration") return "int";
   return task.type;

package/lib/cli/presentation/run-reporter.mjs

@@ -13,15 +13,15 @@ export function createRunReporter({ outputMode = "compact", stdout = process.std
   const mode = outputMode || "compact";
   let completedCount = 0;
   let totalFileCount = 0;
-  let knownFailures = null;
+  let regressionCatalog = null;
 
   return {
     outputMode: mode,
     setTotalFileCount(count) {
       totalFileCount = count;
     },
-    setKnownFailures(document) {
-      knownFailures = document;
+    setRegressionCatalog(document) {
+      regressionCatalog = document;
     },
     writeLine(line = "") {
       stdout.write(`${line}\n`);
@@ -92,7 +92,7 @@ export function createRunReporter({ outputMode = "compact", stdout = process.std
       if (status === "FAIL") {
         const detailLines = renderFailureBlock(task, outcome, {
           width: getTerminalWidth(stdout, 100),
-          knownFailures,
+          regressionCatalog,
         });
         for (const line of detailLines) {
           stdout.write(`${line}\n`);
@@ -108,15 +108,15 @@ export function createRunReporter({ outputMode = "compact", stdout = process.std
       if (mode === "json") return;
       stdout.write(`${message}\n`);
     },
-    runSummary(results, durationMs, knownFailureIssueValidation = null) {
+    runSummary(results, durationMs, regressionReport = null) {
       if (mode === "debug") {
-        const lines = buildDebugRunSummaryLines(results, durationMs, knownFailureIssueValidation);
+        const lines = buildDebugRunSummaryLines(results, durationMs, regressionReport);
         stdout.write("\n");
         for (const line of lines) stdout.write(`${colorSectionLine(line)}\n`);
         return;
       }
 
-      const summary = buildRunSummaryData(results, durationMs, knownFailureIssueValidation);
+      const summary = buildRunSummaryData(results, durationMs, regressionReport);
       const rows = [
         ["Result", summary.result],
         ["Passed", String(summary.passed)],
@@ -129,8 +129,22 @@ export function createRunReporter({ outputMode = "compact", stdout = process.std
       if (summary.serviceErrors > 0) {
         rows.push(["Runtime errors", String(summary.serviceErrors)]);
       }
-      if (summary.knownFailureIssues.length > 0) {
-        rows.push(["Known issues", summary.knownFailureIssues.join(", ")]);
+      if (summary.newRegressions > 0) {
+        rows.push(["New regressions", String(summary.newRegressions)]);
+      }
+      if (summary.knownRegressions > 0) {
+        rows.push(["Known regressions", String(summary.knownRegressions)]);
+      }
+      if (summary.fixedKnownRegressions > 0) {
+        rows.push(["Fixed known", String(summary.fixedKnownRegressions)]);
+      }
+      if (summary.catalogStale > 0) {
+        rows.push(["Catalog stale", String(summary.catalogStale)]);
+      }
+      if (summary.catalogSyncUnavailable) {
+        rows.push(["Catalog sync", "Unavailable"]);
+      } else if (summary.usedStaleCache) {
+        rows.push(["Catalog sync", "Used stale cache"]);
       }
       const boxed = renderSummaryBox(rows, { stdout });
       stdout.write("\n");

package/lib/cli/presentation/run-reporter.test.mjs

@@ -127,7 +127,7 @@ describe("run reporter task output", () => {
     expect(lines[0]).toContain(`${figures.cross} FAIL api int __testkit__/health/health.int.testkit.ts 4s`);
     expect(lines[1]).toBe("    GET /health expected 200, got 404");
     expect(lines[2]).toContain('    response: {"error":"route not found"}');
-    expect(lines[3]).toBe("    triage: untriaged");
+    expect(lines[3]).toBe("    regression: new");
     expect(lines[4]).toBe("    logs: requestId=req-1");
   });
 
@@ -161,10 +161,12 @@ describe("run reporter task output", () => {
       20_000,
       {
         summary: {
-          byCode: {
-            closed_but_failing: 2,
-            state_mismatch: 1,
-          },
+          newRegressions: 1,
+          knownRegressions: 2,
+          fixedKnownRegressions: 3,
+          catalogStale: 4,
+          catalogSyncUnavailable: false,
+          usedStaleCache: true,
         },
       }
     );
@@ -179,7 +181,11 @@ describe("run reporter task output", () => {
     expect(output).toContain("│ Not run");
     expect(output).toContain("│ Files");
     expect(output).toContain("│ Duration");
-    expect(output).toContain("│ Known issues");
+    expect(output).toContain("│ New regressions");
+    expect(output).toContain("│ Known regressions");
+    expect(output).toContain("│ Fixed known");
+    expect(output).toContain("│ Catalog stale");
+    expect(output).toContain("│ Catalog sync");
     expect(output).not.toContain("Failures:");
     expect(output).not.toContain("Runtime Errors:");
   });

package/lib/cli/presentation/summary-box.test.mjs

@@ -31,13 +31,13 @@ describe("summary box", () => {
 
   it("wraps long values instead of widening to content length", () => {
     const lines = renderSummaryBox(
-      [["Known issues", "2 closed issues still failing, 6 state mismatches, used stale GitHub cache"]],
+      [["Catalog sync", "Used stale GitHub cache while catalog validation was unavailable"]],
       { stdout: createStream(40) }
     );
 
     expect(lines.length).toBeGreaterThan(4);
-    expect(lines.join("\n")).toContain("Known issues");
-    expect(lines.join("\n")).toContain("state");
-    expect(lines.join("\n")).toContain("mismatches");
+    expect(lines.join("\n")).toContain("Catalog sync");
+    expect(lines.join("\n")).toContain("stale");
+    expect(lines.join("\n")).toContain("cache");
   });
 });

package/lib/cli/viewer.mjs

@@ -109,11 +109,11 @@ export function formatFileDetail(productDir, runArtifact, subject, options = {})
     for (const line of codeFrame) lines.push(`  ${line}`);
   }
 
-  if (subject.file.triage) {
+  if (subject.file.diagnosis) {
     lines.push("");
-    lines.push("Triage:");
-    const triageLines = formatTriage(subject.file.triage);
-    for (const line of triageLines) lines.push(`  ${line}`);
+    lines.push("Diagnosis:");
+    const diagnosisLines = formatDiagnosis(subject.file.diagnosis);
+    for (const line of diagnosisLines) lines.push(`  ${line}`);
   }
 
   const setupOperations = getSetupOperationsForService(runArtifact, subject.service.name);
@@ -290,30 +290,29 @@ function formatResponseLine(detail) {
   return parts.join(" ");
 }
 
-function formatTriage(triage) {
-  const lines = [`status: ${triage.status}`];
-  if (triage.classifications?.length) {
-    lines.push(`classification: ${triage.classifications.join(", ")}`);
+function formatDiagnosis(diagnosis) {
+  const lines = [`status: ${diagnosis.status}`];
+  if (diagnosis.classifications?.length) {
+    lines.push(`classification: ${diagnosis.classifications.join(", ")}`);
   }
-  if (triage.availability?.mode) {
-    lines.push(
-      `validation: ${triage.availability.mode}${triage.availability.reason ? ` (${triage.availability.reason})` : ""}`
-    );
-  }
-  for (const entry of triage.entries || []) {
+  for (const entry of diagnosis.entries || []) {
     lines.push(`issue: ${entry.issue.repo}#${entry.issue.number}`);
-    if (entry.issue.url) lines.push(`url: ${entry.issue.url}`);
+    lines.push(`summary: ${entry.summary}`);
+    if (entry.github?.url) lines.push(`url: ${entry.github.url}`);
     if (entry.github?.state) {
       lines.push(`github state: ${entry.github.state}${entry.github.cached ? " (cache)" : ""}`);
     }
-    if (entry.validationStatus) lines.push(`validation status: ${entry.validationStatus}`);
-    if (entry.findings?.length) {
-      for (const finding of entry.findings.slice(0, 3)) {
-        lines.push(`finding: ${finding.message}`);
+    if (entry.syncStatus) lines.push(`catalog status: ${entry.syncStatus}`);
+    if (entry.catalogFindings?.length) {
+      for (const finding of entry.catalogFindings.slice(0, 3)) {
+        lines.push(`catalog finding: ${finding.message}`);
       }
     }
     break;
   }
+  if (diagnosis.status === "new_regression") {
+    lines.push("next: review .testkit/results/latest.json drafts.newRegressions for a prepared catalog entry");
+  }
   return lines;
 }
 

package/lib/config/index.mjs

@@ -14,9 +14,9 @@ import {
   detectNextApp,
   inferLocalRuntime,
   normalizeBrowserServiceConfig,
+  normalizeRegressionsConfig,
   normalizeOptionalString,
   normalizeRepoExecution,
-  normalizeReportingConfig,
   normalizeRuntimeConfig,
 } from "./runtime.mjs";
 import { normalizeServiceRequirements, normalizeSkipConfig } from "./skip-config.mjs";
@@ -30,7 +30,7 @@ export async function loadConfigContext(opts = {}) {
   const configContext = opts.configContext || (await loadTestkitConfig(productDir));
   const { config, configFile } = configContext;
   const execution = normalizeRepoExecution(config.execution);
-  const reporting = normalizeReportingConfig(config.reporting);
+  const regressions = normalizeRegressionsConfig(config.regressions);
   const toolchains = normalizeToolchainRegistry(config.toolchains);
   const discoveryConfig = normalizeRepoDiscoveryConfig(config.discovery);
   const explicitServices = config.services || {};
@@ -54,7 +54,7 @@ export async function loadConfigContext(opts = {}) {
         configFile,
         execution,
         discovery: discoveryConfig,
-        reporting,
+        regressions,
         toolchains,
         explicitService: explicitServices[name] || {},
         discoveredService: discovery.services[name] || null,
@@ -71,7 +71,7 @@ export async function loadConfigContext(opts = {}) {
     configFile,
     execution,
     discovery: discoveryConfig,
-    reporting,
+    regressions,
     toolchains,
     explicitServices,
     discovery,
@@ -102,7 +102,7 @@ function normalizeServiceConfig({
   configFile,
   execution,
   discovery,
-  reporting,
+  regressions,
   toolchains,
   explicitService,
   discoveredService,
@@ -156,7 +156,7 @@ function normalizeServiceConfig({
     suites,
     testkit: {
       execution,
-      reporting,
+      regressions,
       dependsOn: explicitService.dependsOn || [],
       discovery: normalizedDiscovery,
       database,

package/lib/config/runtime.mjs

@@ -15,23 +15,23 @@ import {
   parseModuleSpecifier,
 } from "../shared/configured-steps.mjs";
 import { buildConfigToPrepare, normalizeBuildConfig } from "../shared/build-config.mjs";
-import { normalizeKnownFailureIssueValidationConfig } from "../known-failures/github.mjs";
+import { normalizeRegressionSyncConfig } from "../regressions/github.mjs";
 import { normalizeRuntimeToolchain } from "../toolchains/index.mjs";
 import { resolveServiceCwd } from "./paths.mjs";
 
-export function normalizeReportingConfig(value) {
+export function normalizeRegressionsConfig(value) {
   if (!value) return null;
 
-  const knownFailuresFile = normalizeOptionalString(value.knownFailuresFile);
-  if (!knownFailuresFile) {
-    throw new Error('testkit.config.ts reporting.knownFailuresFile must be a non-empty string');
+  const file = normalizeOptionalString(value.file);
+  if (!file) {
+    throw new Error('testkit.config.ts regressions.file must be a non-empty string');
   }
 
-  const issueValidation = normalizeKnownFailureIssueValidationConfig(value.issueValidation);
+  const sync = normalizeRegressionSyncConfig(value.sync);
 
   return {
-    knownFailuresFile,
-    issueValidation,
+    file,
+    sync,
   };
 }