@elench/testkit 0.1.80 → 0.1.81

package/lib/regressions/index.d.ts

@@ -0,0 +1,189 @@
+export type RegressionClassification =
+  | "expected_failure"
+  | "infra"
+  | "product_bug"
+  | "stale_test"
+  | "test_bug";
+
+export interface RegressionIssueRef {
+  repo: string;
+  number: number;
+}
+
+export interface RegressionFingerprint {
+  service?: string;
+  type?: string;
+  path: string;
+  failureKey?: string;
+  errorIncludes?: string;
+}
+
+export interface RegressionCatalogEntry {
+  id: string;
+  classification: RegressionClassification;
+  issue: RegressionIssueRef;
+  summary: string;
+  cause: string;
+  lastReviewedAt: string;
+  fingerprints: RegressionFingerprint[];
+}
+
+export interface RegressionCatalogDocument {
+  schemaVersion: 1;
+  issueRepo?: string | null;
+  entries: RegressionCatalogEntry[];
+}
+
+export interface RegressionCatalogValidationResult {
+  errors: string[];
+  warnings: string[];
+  stats: {
+    entries: number;
+    fingerprints: number;
+    failedTests: number;
+    diagnosedFailedTests: number;
+    newFailedTests: number;
+  };
+}
+
+export interface RegressionSyncConfig {
+  provider?: "github";
+  mode?: "off" | "warn" | "error";
+  cacheTtlSeconds?: number;
+}
+
+export interface RegressionSyncFinding {
+  code: string;
+  severity: "warning" | "error";
+  message: string;
+}
+
+export interface RegressionSyncEntry {
+  id: string;
+  summary: string;
+  issue: RegressionIssueRef;
+  observed: {
+    matchedTests: number;
+    executedTests: number;
+    passedTests: number;
+    failedTests: number;
+    skippedTests: number;
+    notRunTests: number;
+    reproduced: boolean;
+  };
+  github: {
+    exists: boolean | null;
+    title: string | null;
+    state: string | null;
+    url: string | null;
+    checkedAt: string | null;
+    cached: boolean;
+  };
+  findings: RegressionSyncFinding[];
+  status: string;
+}
+
+export interface RegressionSyncResult {
+  schemaVersion: 2;
+  provider: "github";
+  mode: "off" | "warn" | "error";
+  checkedAt: string;
+  repo: {
+    detected: string | null;
+    remoteUrl: string | null;
+    configured: string | null;
+  };
+  availability: {
+    hasFreshData: boolean;
+    usedCachedFallback: boolean;
+  };
+  findings: RegressionSyncFinding[];
+  summary: {
+    entries: number;
+    observedEntries: number;
+    errors: number;
+    warnings: number;
+    byCode: Record<string, number>;
+    byStatus: Record<string, number>;
+  };
+  entries: RegressionSyncEntry[];
+}
+
+export declare function loadRegressionCatalogConfig(
+  productDir: string,
+  config: { file?: string | null } | null
+): RegressionCatalogDocument | null;
+export declare function loadRegressionCatalogDocument(
+  filePath: string,
+  relativePath?: string
+): RegressionCatalogDocument;
+export declare function normalizeRegressionCatalogDocument(
+  document: unknown,
+  relativePath?: string
+): RegressionCatalogDocument;
+export declare function validateRegressionCatalogDocument(
+  document: RegressionCatalogDocument,
+  options?: {
+    productDir?: string;
+    statusArtifactPath?: string;
+    statusArtifact?: unknown;
+  }
+): RegressionCatalogValidationResult;
+export declare function renderRegressionCatalogMarkdown(
+  document: RegressionCatalogDocument
+): string;
+export declare function findMatchingRegressionEntries(
+  document: RegressionCatalogDocument,
+  fileSummary: {
+    service?: string;
+    type?: string;
+    path: string;
+    status?: string;
+    error?: string | null;
+    failureDetails?: Array<{ key?: string; title?: string }>;
+  }
+): RegressionCatalogEntry[];
+export declare function matchesRegressionEntry(
+  entry: RegressionCatalogEntry,
+  fileSummary: {
+    service?: string;
+    type?: string;
+    path: string;
+    error?: string | null;
+    failureDetails?: Array<{ key?: string; title?: string }>;
+  }
+): boolean;
+export declare function buildRegressionFileIdentity(
+  service: string,
+  type: string,
+  filePath: string
+): string;
+
+export declare function normalizeRegressionSyncConfig(
+  value: unknown
+): {
+  provider: "github";
+  mode: "off" | "warn" | "error";
+  cacheTtlSeconds: number;
+} | null;
+export declare function validateRegressionIssues(options: {
+  productDir: string;
+  document: RegressionCatalogDocument | null;
+  runArtifact?: unknown;
+  statusArtifact?: unknown;
+  config?: RegressionSyncConfig | null;
+  gitMetadata?: {
+    repoSlug?: string | null;
+    remoteUrl?: string | null;
+  } | null;
+}): Promise<RegressionSyncResult | null>;
+export declare function shouldFailRegressionSync(
+  result: RegressionSyncResult | null
+): boolean;
+export declare function buildRegressionSyncSummaryLines(
+  result: RegressionSyncResult | null
+): string[];
+export declare function buildRegressionSyncSummaryParts(
+  result: RegressionSyncResult | null
+): string[];
+export declare function parseGitHubRepoSlug(remoteUrl: string | null | undefined): string | null;

package/lib/{known-failures → regressions}/index.mjs

@@ -8,34 +8,33 @@ const CLASSIFICATIONS = new Set([
   "stale_test",
   "test_bug",
 ]);
-const STATES = new Set(["closed", "open"]);
 
-export function loadKnownFailuresConfig(productDir, config) {
-  const relativePath = config?.knownFailuresFile;
+export function loadRegressionCatalogConfig(productDir, config) {
+  const relativePath = config?.file;
   if (!relativePath) return null;
 
   const absolutePath = path.resolve(productDir, relativePath);
   if (!fs.existsSync(absolutePath)) {
-    throw new Error(`Known failures file not found: ${relativePath}`);
+    throw new Error(`Regression catalog not found: ${relativePath}`);
   }
 
-  return loadKnownFailuresDocument(absolutePath, relativePath);
+  return loadRegressionCatalogDocument(absolutePath, relativePath);
 }
 
-export function loadKnownFailuresDocument(filePath, relativePath = filePath) {
+export function loadRegressionCatalogDocument(filePath, relativePath = filePath) {
   let parsed;
   try {
     parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
   } catch (error) {
     throw new Error(
-      `Could not parse known failures file ${relativePath}: ${formatErrorMessage(error)}`
+      `Could not parse regression catalog ${relativePath}: ${formatErrorMessage(error)}`
     );
   }
 
-  return normalizeKnownFailuresDocument(parsed, relativePath);
+  return normalizeRegressionCatalogDocument(parsed, relativePath);
 }
 
-export function normalizeKnownFailuresDocument(document, relativePath = "known failures file") {
+export function normalizeRegressionCatalogDocument(document, relativePath = "regression catalog") {
   if (!document || typeof document !== "object") {
     throw new Error(`${relativePath} must contain a JSON object`);
   }
@@ -48,7 +47,7 @@ export function normalizeKnownFailuresDocument(document, relativePath = "known f
 
   const ids = new Set();
   const entries = document.entries.map((entry, index) => {
-    const normalized = normalizeKnownFailureEntry(entry, `${relativePath} entries[${index}]`);
+    const normalized = normalizeRegressionEntry(entry, `${relativePath} entries[${index}]`);
     if (ids.has(normalized.id)) {
       throw new Error(`${relativePath} has duplicate entry id "${normalized.id}"`);
     }
@@ -63,40 +62,45 @@ export function normalizeKnownFailuresDocument(document, relativePath = "known f
   };
 }
 
-export function validateKnownFailuresDocument(
+export function validateRegressionCatalogDocument(
   document,
   { productDir, statusArtifactPath, statusArtifact } = {}
 ) {
   const errors = [];
   const warnings = [];
-  const matchKeys = new Set();
-  let matchCount = 0;
+  const fingerprintKeys = new Set();
+  let fingerprintCount = 0;
 
   for (const entry of document.entries) {
-    for (const match of entry.matches) {
-      matchCount += 1;
+    for (const fingerprint of entry.fingerprints) {
+      fingerprintCount += 1;
       if (productDir) {
-        const absolutePath = path.join(productDir, match.path);
+        const absolutePath = path.join(productDir, fingerprint.path);
         if (!fs.existsSync(absolutePath)) {
-          errors.push(`Missing matched file: ${match.path} (${entry.id})`);
+          errors.push(`Missing matched file: ${fingerprint.path} (${entry.id})`);
         }
       }
 
-      const matchKey = [
+      const fingerprintKey = [
         entry.id,
-        match.service || "",
-        match.type || "",
-        match.path,
-        match.failureKey || "",
-        match.errorIncludes || "",
+        fingerprint.service || "",
+        fingerprint.type || "",
+        fingerprint.path,
+        fingerprint.failureKey || "",
+        fingerprint.errorIncludes || "",
       ].join("::");
-      if (matchKeys.has(matchKey)) {
-        errors.push(`Duplicate match selector in ${entry.id}: ${match.path}`);
+      if (fingerprintKeys.has(fingerprintKey)) {
+        errors.push(`Duplicate fingerprint selector in ${entry.id}: ${fingerprint.path}`);
       }
-      matchKeys.add(matchKey);
-
-      if (!match.path.includes("__testkit__") || !match.path.endsWith(".testkit.ts")) {
-        warnings.push(`Match path does not look like a testkit file: ${match.path} (${entry.id})`);
+      fingerprintKeys.add(fingerprintKey);
+
+      if (
+        !fingerprint.path.includes("__testkit__") ||
+        !fingerprint.path.endsWith(".testkit.ts")
+      ) {
+        warnings.push(
+          `Fingerprint path does not look like a testkit file: ${fingerprint.path} (${entry.id})`
+        );
       }
     }
 
@@ -105,14 +109,11 @@ export function validateKnownFailuresDocument(
         `Entry ${entry.id} uses issue repo ${entry.issue.repo} instead of document issueRepo ${document.issueRepo}`
       );
     }
-    if (!entry.issue.url.endsWith(`/issues/${entry.issue.number}`)) {
-      errors.push(`Issue URL does not match issue number for ${entry.id}`);
-    }
   }
 
   let failedTests = 0;
-  let triagedFailedTests = 0;
-  let untriagedFailedTests = 0;
+  let diagnosedFailedTests = 0;
+  let newFailedTests = 0;
   const parsedStatusArtifact = statusArtifact
     ? parseStatusArtifact(statusArtifact)
     : statusArtifactPath && fs.existsSync(statusArtifactPath)
@@ -122,11 +123,11 @@ export function validateKnownFailuresDocument(
   const failed = parsedStatusArtifact.tests.filter((test) => test.status === "failed");
   failedTests = failed.length;
   for (const test of failed) {
-    if (findMatchingKnownFailureEntries(document, test).length > 0) {
-      triagedFailedTests += 1;
+    if (findMatchingRegressionEntries(document, test).length > 0) {
+      diagnosedFailedTests += 1;
     } else {
-      untriagedFailedTests += 1;
-      warnings.push(`Untriaged failed test: ${test.path}`);
+      newFailedTests += 1;
+      warnings.push(`New failing test not yet in regression catalog: ${test.path}`);
     }
   }
 
@@ -135,41 +136,39 @@ export function validateKnownFailuresDocument(
     warnings,
     stats: {
       entries: document.entries.length,
-      matches: matchCount,
+      fingerprints: fingerprintCount,
       failedTests,
-      triagedFailedTests,
-      untriagedFailedTests,
+      diagnosedFailedTests,
+      newFailedTests,
     },
   };
 }
 
-export function renderKnownFailuresMarkdown(document) {
+export function renderRegressionCatalogMarkdown(document) {
   const lines = [
-    "# Bourne Bugs",
+    "# Regression Catalog",
     "",
-    "Canonical source: `testkit.known-failures.json`",
+    "Canonical source: `testkit.regressions.json`",
     "",
-    `Tracked bug classes: ${document.entries.length}`,
+    `Tracked regressions: ${document.entries.length}`,
     `Tracked issue references: ${new Set(document.entries.map((entry) => entry.issue.number)).size}`,
     "",
   ];
 
   document.entries.forEach((entry, index) => {
     lines.push(
-      `## ${index + 1}. ${entry.title} — [#${entry.issue.number}](${entry.issue.url})`,
+      `## ${index + 1}. ${entry.summary} — #${entry.issue.number}`,
       "",
       `- Classification: \`${entry.classification}\``,
-      `- State: \`${entry.state}\``,
-      `- Description: ${entry.description}`,
-      `- Why failing: ${entry.whyFailing}`,
+      `- Cause: ${entry.cause}`,
       `- Last reviewed: ${entry.lastReviewedAt}`,
-      "- Matches:"
+      "- Fingerprints:"
     );
 
-    for (const match of entry.matches) {
+    for (const fingerprint of entry.fingerprints) {
       lines.push(
-        `  - \`${match.path}\`${match.failureKey ? ` — \`${match.failureKey}\`` : ""}${
-          match.errorIncludes ? ` — error contains \`${match.errorIncludes}\`` : ""
+        `  - \`${fingerprint.path}\`${fingerprint.failureKey ? ` — \`${fingerprint.failureKey}\`` : ""}${
+          fingerprint.errorIncludes ? ` — error contains \`${fingerprint.errorIncludes}\`` : ""
         }`
       );
     }
@@ -180,70 +179,68 @@ export function renderKnownFailuresMarkdown(document) {
   return `${lines.join("\n").trimEnd()}\n`;
 }
 
-export function findMatchingKnownFailureEntries(document, fileSummary) {
-  return document.entries.filter((entry) => matchesKnownFailureEntry(entry, fileSummary));
+export function findMatchingRegressionEntries(document, fileSummary) {
+  return document.entries.filter((entry) => matchesRegressionEntry(entry, fileSummary));
 }
 
-export function matchesKnownFailureEntry(entry, fileSummary) {
-  return entry.matches.some((match) => matchesKnownFailureMatch(match, fileSummary));
+export function matchesRegressionEntry(entry, fileSummary) {
+  return entry.fingerprints.some((fingerprint) =>
+    matchesRegressionFingerprint(fingerprint, fileSummary)
+  );
 }
 
-export function matchesKnownFailureMatch(match, fileSummary) {
-  if (match.service && match.service !== fileSummary.service) return false;
-  if (match.type && match.type !== fileSummary.type) return false;
-  if (match.path !== fileSummary.path) return false;
-  if (match.failureKey) {
+export function matchesRegressionFingerprint(fingerprint, fileSummary) {
+  if (fingerprint.service && fingerprint.service !== fileSummary.service) return false;
+  if (fingerprint.type && fingerprint.type !== fileSummary.type) return false;
+  if (fingerprint.path !== fileSummary.path) return false;
+  if (fingerprint.failureKey) {
     const failureKeys = Array.isArray(fileSummary.failureDetails)
       ? fileSummary.failureDetails.flatMap((detail) => [detail?.key, detail?.title].filter(Boolean))
       : [];
-    if (!failureKeys.includes(match.failureKey)) return false;
+    if (!failureKeys.includes(fingerprint.failureKey)) return false;
   }
-  if (match.errorIncludes && !String(fileSummary.error || "").includes(match.errorIncludes)) {
+  if (fingerprint.errorIncludes && !String(fileSummary.error || "").includes(fingerprint.errorIncludes)) {
     return false;
   }
   return true;
 }
 
-export function buildKnownFailureFileIdentity(service, type, filePath) {
+export function buildRegressionFileIdentity(service, type, filePath) {
   return `${service}::${type}::${filePath}`;
 }
 
-function normalizeKnownFailureEntry(entry, label) {
+function normalizeRegressionEntry(entry, label) {
   if (!entry || typeof entry !== "object") {
     throw new Error(`${label} must be an object`);
   }
 
   const id = requireNonEmptyString(entry.id, `${label}.id`);
-  const title = requireNonEmptyString(entry.title, `${label}.title`);
   const classification = requireEnumValue(
     entry.classification,
     CLASSIFICATIONS,
     `${label}.classification`
   );
-  const state = requireEnumValue(entry.state, STATES, `${label}.state`);
-  const description = requireNonEmptyString(entry.description, `${label}.description`);
-  const whyFailing = requireNonEmptyString(entry.whyFailing, `${label}.whyFailing`);
+  const summary = requireNonEmptyString(entry.summary, `${label}.summary`);
+  const cause = requireNonEmptyString(entry.cause, `${label}.cause`);
   const lastReviewedAt = requireNonEmptyString(entry.lastReviewedAt, `${label}.lastReviewedAt`);
-  if (!Array.isArray(entry.matches) || entry.matches.length === 0) {
-    throw new Error(`${label}.matches must be a non-empty array`);
+  if (!Array.isArray(entry.fingerprints) || entry.fingerprints.length === 0) {
+    throw new Error(`${label}.fingerprints must be a non-empty array`);
   }
 
   return {
     id,
-    title,
     classification,
-    state,
-    issue: normalizeKnownFailureIssue(entry.issue, `${label}.issue`),
-    description,
-    whyFailing,
+    issue: normalizeRegressionIssue(entry.issue, `${label}.issue`),
+    summary,
+    cause,
     lastReviewedAt,
-    matches: entry.matches.map((match, index) =>
-      normalizeKnownFailureMatch(match, `${label}.matches[${index}]`)
+    fingerprints: entry.fingerprints.map((fingerprint, index) =>
+      normalizeRegressionFingerprint(fingerprint, `${label}.fingerprints[${index}]`)
     ),
   };
 }
 
-function normalizeKnownFailureIssue(issue, label) {
+function normalizeRegressionIssue(issue, label) {
   if (!issue || typeof issue !== "object") {
     throw new Error(`${label} must be an object`);
   }
@@ -253,31 +250,30 @@ function normalizeKnownFailureIssue(issue, label) {
   if (!Number.isInteger(number) || number <= 0) {
     throw new Error(`${label}.number must be a positive integer`);
   }
-  const url = requireNonEmptyString(issue.url, `${label}.url`);
 
-  return { repo, number, url };
+  return { repo, number };
 }
 
-function normalizeKnownFailureMatch(match, label) {
-  if (!match || typeof match !== "object") {
+function normalizeRegressionFingerprint(fingerprint, label) {
+  if (!fingerprint || typeof fingerprint !== "object") {
     throw new Error(`${label} must be an object`);
   }
 
-  const pathValue = requireNonEmptyString(match.path, `${label}.path`);
+  const pathValue = requireNonEmptyString(fingerprint.path, `${label}.path`);
   const normalized = {
     path: pathValue,
   };
 
-  const service = normalizeOptionalString(match.service);
+  const service = normalizeOptionalString(fingerprint.service);
   if (service) normalized.service = service;
 
-  const type = normalizeOptionalString(match.type);
+  const type = normalizeOptionalString(fingerprint.type);
   if (type) normalized.type = type;
 
-  const failureKey = normalizeOptionalString(match.failureKey);
+  const failureKey = normalizeOptionalString(fingerprint.failureKey);
   if (failureKey) normalized.failureKey = failureKey;
 
-  const errorIncludes = normalizeOptionalString(match.errorIncludes);
+  const errorIncludes = normalizeOptionalString(fingerprint.errorIncludes);
   if (errorIncludes) normalized.errorIncludes = errorIncludes;
 
   return normalized;
@@ -321,9 +317,10 @@ function formatErrorMessage(error) {
 }
 
 export {
-  buildKnownFailureIssueValidationSummaryLines,
-  normalizeKnownFailureIssueValidationConfig,
+  buildRegressionSyncSummaryLines,
+  buildRegressionSyncSummaryParts,
+  normalizeRegressionSyncConfig,
   parseGitHubRepoSlug,
-  shouldFailKnownFailureIssueValidation,
-  validateKnownFailureIssues,
+  shouldFailRegressionSync,
+  validateRegressionIssues,
 } from "./github.mjs";