@elench/testkit 0.1.80 → 0.1.81

package/lib/{known-failures → regressions}/github.mjs

@@ -1,4 +1,4 @@
-import { findMatchingKnownFailureEntries } from "./index.mjs";
+import { findMatchingRegressionEntries } from "./index.mjs";
 import {
   applyStaleCacheFallback,
   loadIssueCache,
@@ -12,37 +12,34 @@ import {
   fetchIssuesByRepo,
   parseGitHubRepoSlug,
 } from "./github-transport.mjs";
+
 export { parseGitHubRepoSlug } from "./github-transport.mjs";
+
 const DEFAULT_CACHE_TTL_SECONDS = 15 * 60;
 const CACHE_SCHEMA_VERSION = 1;
-const CACHE_PATH = [".testkit", "known-failures", "github-issues-cache.json"];
+const CACHE_PATH = [".testkit", "regressions", "github-issues-cache.json"];
 const MODES = new Set(["off", "warn", "error"]);
 
-export function normalizeKnownFailureIssueValidationConfig(value) {
+export function normalizeRegressionSyncConfig(value) {
   if (value == null) return null;
   if (!value || typeof value !== "object") {
-    throw new Error("testkit.config.ts reporting.issueValidation must be an object");
+    throw new Error("testkit.config.ts regressions.sync must be an object");
   }
 
   const provider = normalizeOptionalString(value.provider) || "github";
   if (provider !== "github") {
-    throw new Error('testkit.config.ts reporting.issueValidation.provider must be "github"');
+    throw new Error('testkit.config.ts regressions.sync.provider must be "github"');
   }
 
   const mode = normalizeOptionalString(value.mode) || "warn";
   if (!MODES.has(mode)) {
-    throw new Error(
-      'testkit.config.ts reporting.issueValidation.mode must be one of: off, warn, error'
-    );
+    throw new Error('testkit.config.ts regressions.sync.mode must be one of: off, warn, error');
   }
 
   const cacheTtlSeconds =
     value.cacheTtlSeconds == null
       ? DEFAULT_CACHE_TTL_SECONDS
-      : normalizePositiveInteger(
-          value.cacheTtlSeconds,
-          "testkit.config.ts reporting.issueValidation.cacheTtlSeconds"
-        );
+      : normalizePositiveInteger(value.cacheTtlSeconds, "testkit.config.ts regressions.sync.cacheTtlSeconds");
 
   return {
     provider,
@@ -51,7 +48,7 @@ export function normalizeKnownFailureIssueValidationConfig(value) {
   };
 }
 
-export async function validateKnownFailureIssues({
+export async function validateRegressionIssues({
   productDir,
   document,
   runArtifact,
@@ -61,10 +58,10 @@ export async function validateKnownFailureIssues({
   transport = null,
   now = Date.now(),
 }) {
-  const normalizedConfig = normalizeKnownFailureIssueValidationConfig(config);
+  const normalizedConfig = normalizeRegressionSyncConfig(config);
   if (!document || !normalizedConfig || normalizedConfig.mode === "off") return null;
 
-  const checks = collectObservedKnownFailureEntries(document, runArtifact, statusArtifact);
+  const checks = collectObservedRegressionEntries(document, runArtifact, statusArtifact);
   const issueNumbersByRepo = groupIssueNumbersByRepo(document.entries);
   const repoSlug = gitMetadata?.repoSlug || null;
   const remoteUrl = gitMetadata?.remoteUrl || null;
@@ -87,11 +84,10 @@ export async function validateKnownFailureIssues({
           cacheResolution.staleByRepo,
           availabilityFindings
         );
-        const severity = normalizedConfig.mode === "error" ? "error" : "warning";
         availabilityFindings.push({
           code: "validation_unavailable",
-          severity,
-          message: `GitHub issue validation unavailable: ${formatErrorMessage(error)}`,
+          severity: normalizedConfig.mode === "error" ? "error" : "warning",
+          message: `GitHub issue sync unavailable: ${formatErrorMessage(error)}`,
         });
       }
     } else {
@@ -100,12 +96,11 @@ export async function validateKnownFailureIssues({
         cacheResolution.staleByRepo,
         availabilityFindings
       );
-      const severity = normalizedConfig.mode === "error" ? "error" : "warning";
       availabilityFindings.push({
         code: "validation_unavailable",
-        severity,
+        severity: normalizedConfig.mode === "error" ? "error" : "warning",
         message:
-          "GitHub issue validation unavailable: set GH_TOKEN/GITHUB_TOKEN or install/authenticate gh",
+          "GitHub issue sync unavailable: set GH_TOKEN/GITHUB_TOKEN or install/authenticate gh",
       });
     }
   }
@@ -113,11 +108,7 @@ export async function validateKnownFailureIssues({
   const entries = document.entries.map((entry) => {
     const observed = checks.get(entry.id) || createObservedCheck(entry);
     const issueData = issuesByRepo.get(entry.issue.repo)?.get(entry.issue.number) || null;
-    return buildIssueValidationEntry({
-      entry,
-      observed,
-      issueData,
-    });
+    return buildIssueValidationEntry({ entry, observed, issueData });
   });
 
   const globalFindings = [...availabilityFindings];
@@ -126,7 +117,7 @@ export async function validateKnownFailureIssues({
       code: "detected_repo_mismatch",
       severity: normalizedConfig.mode === "error" ? "error" : "warning",
       message:
-        `Known failures issueRepo ${document.issueRepo} does not match detected repo ${repoSlug}` +
+        `Regression catalog issueRepo ${document.issueRepo} does not match detected repo ${repoSlug}` +
         (remoteUrl ? ` (${remoteUrl})` : ""),
     });
   }
@@ -152,12 +143,12 @@ export async function validateKnownFailureIssues({
   };
 }
 
-export function shouldFailKnownFailureIssueValidation(result) {
+export function shouldFailRegressionSync(result) {
   if (!result || result.mode !== "error") return false;
   return (result.summary?.errors || 0) > 0;
 }
 
-export function buildKnownFailureIssueValidationSummaryParts(result) {
+export function buildRegressionSyncSummaryParts(result) {
   if (!result) return [];
   const parts = [];
   const byCode = result.summary?.byCode || {};
@@ -167,23 +158,17 @@ export function buildKnownFailureIssueValidationSummaryParts(result) {
   if (byCode.issue_missing) {
     parts.push(`${byCode.issue_missing} missing issue reference${pluralSuffix(byCode.issue_missing)}`);
   }
-  if (byCode.open_not_reproduced) {
-    parts.push(`${byCode.open_not_reproduced} open issue${pluralSuffix(byCode.open_not_reproduced)} not reproduced`);
-  }
   if (byCode.closed_not_reproduced) {
-    parts.push(`${byCode.closed_not_reproduced} closed issue${pluralSuffix(byCode.closed_not_reproduced)} not reproduced`);
+    parts.push(`${byCode.closed_not_reproduced} fixed known regression${pluralSuffix(byCode.closed_not_reproduced)}`);
   }
-  if (byCode.title_mismatch) {
-    parts.push(`${byCode.title_mismatch} title mismatch${pluralSuffix(byCode.title_mismatch)}`);
-  }
-  if (byCode.state_mismatch) {
-    parts.push(`${byCode.state_mismatch} state mismatch${pluralSuffix(byCode.state_mismatch)}`);
+  if (byCode.open_not_reproduced) {
+    parts.push(`${byCode.open_not_reproduced} known regression${pluralSuffix(byCode.open_not_reproduced)} no longer reproducing`);
   }
   if (byCode.detected_repo_mismatch) {
     parts.push(`${byCode.detected_repo_mismatch} repo mismatch${pluralSuffix(byCode.detected_repo_mismatch)}`);
   }
   if (byCode.validation_unavailable) {
-    parts.push("GitHub validation unavailable");
+    parts.push("GitHub issue sync unavailable");
   }
   if (byCode.used_stale_cache) {
     parts.push("used stale GitHub cache");
@@ -192,24 +177,22 @@ export function buildKnownFailureIssueValidationSummaryParts(result) {
   return parts;
 }
 
-export function buildKnownFailureIssueValidationSummaryLines(result) {
-  const parts = buildKnownFailureIssueValidationSummaryParts(result);
+export function buildRegressionSyncSummaryLines(result) {
+  const parts = buildRegressionSyncSummaryParts(result);
   if (parts.length === 0) return [];
-
   return [
     "",
-    "Known-failure issues:",
+    "Catalog issues:",
     ...parts.map((part) => `  - ${part}`),
   ];
 }
 
-
-function collectObservedKnownFailureEntries(document, runArtifact, statusArtifact) {
+function collectObservedRegressionEntries(document, runArtifact, statusArtifact) {
   const observedTests = collectObservedTests(runArtifact, statusArtifact);
   const checks = new Map(document.entries.map((entry) => [entry.id, createObservedCheck(entry)]));
 
   for (const test of observedTests) {
-    const matchedEntries = findMatchingKnownFailureEntries(document, test);
+    const matchedEntries = findMatchingRegressionEntries(document, test);
     for (const entry of matchedEntries) {
       const observed = checks.get(entry.id) || createObservedCheck(entry);
       observed.matchedTests += 1;
@@ -292,23 +275,7 @@ function buildIssueValidationEntry({ entry, observed, issueData }) {
     findings.push({
       code: "issue_missing",
       severity: "error",
-      message: `Known failure ${entry.id} references missing issue #${entry.issue.number} in ${entry.issue.repo}`,
-    });
-  }
-
-  if (issueData?.exists && issueData.title && issueData.title !== entry.title) {
-    findings.push({
-      code: "title_mismatch",
-      severity: "error",
-      message: `Known failure ${entry.id} title does not match issue #${entry.issue.number}`,
-    });
-  }
-
-  if (issueData?.exists && normalizedIssueState && normalizedIssueState !== entry.state) {
-    findings.push({
-      code: "state_mismatch",
-      severity: "error",
-      message: `Known failure ${entry.id} state ${entry.state} does not match issue #${entry.issue.number} state ${normalizedIssueState}`,
+      message: `Regression catalog entry ${entry.id} references missing issue #${entry.issue.number} in ${entry.issue.repo}`,
     });
   }
 
@@ -316,7 +283,7 @@ function buildIssueValidationEntry({ entry, observed, issueData }) {
     findings.push({
       code: "closed_but_failing",
       severity: "error",
-      message: `Known failure ${entry.id} still fails but issue #${entry.issue.number} is closed`,
+      message: `Regression ${entry.id} still fails but issue #${entry.issue.number} is closed`,
     });
   }
 
@@ -330,20 +297,20 @@ function buildIssueValidationEntry({ entry, observed, issueData }) {
       findings.push({
         code: "open_not_reproduced",
         severity: "warning",
-        message: `Known failure ${entry.id} did not reproduce but issue #${entry.issue.number} is open`,
+        message: `Regression ${entry.id} did not reproduce but issue #${entry.issue.number} is open`,
       });
     } else if (normalizedIssueState === "closed") {
       findings.push({
         code: "closed_not_reproduced",
         severity: "warning",
-        message: `Known failure ${entry.id} did not reproduce and issue #${entry.issue.number} is closed`,
+        message: `Regression ${entry.id} did not reproduce and issue #${entry.issue.number} is closed`,
       });
     }
   }
 
   return {
     id: entry.id,
-    title: entry.title,
+    summary: entry.summary,
     issue: entry.issue,
     observed: {
       matchedTests: observed.matchedTests,
@@ -372,11 +339,11 @@ function buildIssueValidationEntry({ entry, observed, issueData }) {
           cached: false,
         },
     findings,
-    status: resolveIssueValidationStatus(issueData, observed, findings),
+    status: resolveIssueValidationStatus(issueData, observed),
   };
 }
 
-function resolveIssueValidationStatus(issueData, observed, findings) {
+function resolveIssueValidationStatus(issueData, observed) {
   if (!issueData) return observed.matchedTests > 0 ? "validation_unavailable" : "not_observed";
   if (issueData.exists === false) return "issue_missing";
   const normalizedState = normalizeIssueState(issueData.state);
@@ -399,7 +366,6 @@ function resolveIssueValidationStatus(issueData, observed, findings) {
   ) {
     return "closed_not_reproduced";
   }
-  if (findings.length > 0) return "metadata_mismatch";
   return "not_observed";
 }
 
@@ -453,14 +419,6 @@ function normalizePositiveInteger(value, label) {
   return value;
 }
 
-function chunkValues(values, size) {
-  const chunks = [];
-  for (let index = 0; index < values.length; index += size) {
-    chunks.push(values.slice(index, index + size));
-  }
-  return chunks;
-}
-
 function pluralSuffix(value) {
   return value === 1 ? "" : "s";
 }

package/lib/regressions/github.test.mjs

@@ -0,0 +1,324 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { afterEach, describe, expect, it } from "vitest";
+import { normalizeRegressionCatalogDocument } from "./index.mjs";
+import {
+  parseGitHubRepoSlug,
+  shouldFailRegressionSync,
+  validateRegressionIssues,
+} from "./github.mjs";
+
+const tempDirs = [];
+
+afterEach(() => {
+  for (const tempDir of tempDirs.splice(0)) {
+    fs.rmSync(tempDir, { recursive: true, force: true });
+  }
+});
+
+describe("regression GitHub sync", () => {
+  it("parses GitHub repo slugs from common origin URLs", () => {
+    expect(parseGitHubRepoSlug("https://github.com/acme/repo.git")).toBe("acme/repo");
+    expect(parseGitHubRepoSlug("git@github.com:acme/repo.git")).toBe("acme/repo");
+    expect(parseGitHubRepoSlug("ssh://git@github.com/acme/repo.git")).toBe("acme/repo");
+    expect(parseGitHubRepoSlug("https://gitlab.com/acme/repo.git")).toBe(null);
+  });
+
+  it("fails sync when a linked issue is closed but the regression still reproduces", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-regressions-gh-"));
+    tempDirs.push(tempDir);
+    const document = normalizeRegressionCatalogDocument({
+      schemaVersion: 1,
+      issueRepo: "acme/repo",
+      entries: [
+        {
+          id: "bad-message",
+          classification: "product_bug",
+          issue: {
+            repo: "acme/repo",
+            number: 12,
+          },
+          summary: "API returns the wrong message",
+          cause: "The payload is wrong.",
+          lastReviewedAt: "2026-05-04",
+          fingerprints: [
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            },
+          ],
+        },
+      ],
+    });
+
+    const result = await validateRegressionIssues({
+      productDir: tempDir,
+      document,
+      statusArtifact: {
+        tests: [
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            status: "failed",
+          },
+        ],
+      },
+      config: {
+        provider: "github",
+        mode: "error",
+        cacheTtlSeconds: 60,
+      },
+      gitMetadata: {
+        repoSlug: "acme/repo",
+        remoteUrl: "https://github.com/acme/repo.git",
+      },
+      transport: {
+        async fetchRepoIssues(repo, numbers) {
+          const map = new Map();
+          map.set(numbers[0], {
+            repo,
+            number: numbers[0],
+            exists: true,
+            title: "API returns the wrong message",
+            state: "CLOSED",
+            url: `https://github.com/${repo}/issues/${numbers[0]}`,
+            checkedAt: "2026-05-04T00:00:00.000Z",
+            source: "github",
+          });
+          return map;
+        },
+      },
+    });
+
+    expect(result.summary.byCode.closed_but_failing).toBe(1);
+    expect(result.summary.errors).toBeGreaterThan(0);
+    expect(result.entries[0].status).toBe("closed_but_failing");
+    expect(shouldFailRegressionSync(result)).toBe(true);
+  });
+
+  it("allows multiple local entries to share one GitHub issue", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-regressions-gh-shared-"));
+    tempDirs.push(tempDir);
+    const document = normalizeRegressionCatalogDocument({
+      schemaVersion: 1,
+      issueRepo: "acme/repo",
+      entries: [
+        {
+          id: "route-a-invalid-uuid",
+          classification: "product_bug",
+          issue: {
+            repo: "acme/repo",
+            number: 77,
+          },
+          summary: "Route A leaks UUID parse errors",
+          cause: "Route A is missing UUID validation.",
+          lastReviewedAt: "2026-05-04",
+          fingerprints: [
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/route-a.int.testkit.ts",
+            },
+          ],
+        },
+        {
+          id: "route-b-invalid-uuid",
+          classification: "product_bug",
+          issue: {
+            repo: "acme/repo",
+            number: 77,
+          },
+          summary: "Route B leaks UUID parse errors",
+          cause: "Route B is missing UUID validation.",
+          lastReviewedAt: "2026-05-04",
+          fingerprints: [
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/route-b.int.testkit.ts",
+            },
+          ],
+        },
+      ],
+    });
+
+    const result = await validateRegressionIssues({
+      productDir: tempDir,
+      document,
+      statusArtifact: {
+        tests: [
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/route-a.int.testkit.ts",
+            status: "failed",
+          },
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/route-b.int.testkit.ts",
+            status: "failed",
+          },
+        ],
+      },
+      config: {
+        provider: "github",
+        mode: "error",
+      },
+      transport: {
+        async fetchRepoIssues(repo, numbers) {
+          const map = new Map();
+          map.set(numbers[0], {
+            repo,
+            number: numbers[0],
+            exists: true,
+            title: "UUID path params reach Postgres and return 500 instead of 400",
+            state: "OPEN",
+            url: `https://github.com/${repo}/issues/${numbers[0]}`,
+            checkedAt: "2026-05-04T00:00:00.000Z",
+            source: "github",
+          });
+          return map;
+        },
+      },
+    });
+
+    expect(result.summary.errors).toBe(0);
+    expect(result.entries).toHaveLength(2);
+    expect(result.entries[0].findings).toEqual([]);
+    expect(result.entries[1].findings).toEqual([]);
+    expect(result.entries[0].status).toBe("open_and_failing");
+    expect(result.entries[1].status).toBe("open_and_failing");
+  });
+
+  it("warns when an open issue is not reproduced", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-regressions-gh-open-"));
+    tempDirs.push(tempDir);
+    const document = normalizeRegressionCatalogDocument({
+      schemaVersion: 1,
+      issueRepo: "acme/repo",
+      entries: [
+        {
+          id: "bad-message",
+          classification: "product_bug",
+          issue: {
+            repo: "acme/repo",
+            number: 12,
+          },
+          summary: "API returns the wrong message",
+          cause: "The payload is wrong.",
+          lastReviewedAt: "2026-05-04",
+          fingerprints: [
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            },
+          ],
+        },
+      ],
+    });
+
+    const result = await validateRegressionIssues({
+      productDir: tempDir,
+      document,
+      statusArtifact: {
+        tests: [
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            status: "passed",
+          },
+        ],
+      },
+      config: {
+        provider: "github",
+        mode: "warn",
+      },
+      gitMetadata: {
+        repoSlug: "acme/repo",
+      },
+      transport: {
+        async fetchRepoIssues(repo, numbers) {
+          const map = new Map();
+          map.set(numbers[0], {
+            repo,
+            number: numbers[0],
+            exists: true,
+            title: "API returns the wrong message",
+            state: "OPEN",
+            url: `https://github.com/${repo}/issues/${numbers[0]}`,
+            checkedAt: "2026-05-04T00:00:00.000Z",
+            source: "github",
+          });
+          return map;
+        },
+      },
+    });
+
+    expect(result.summary.byCode.open_not_reproduced).toBe(1);
+    expect(result.summary.errors).toBe(0);
+    expect(result.summary.warnings).toBe(1);
+    expect(result.entries[0].status).toBe("open_not_reproduced");
+  });
+
+  it("falls back to warning when sync is unavailable", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-regressions-gh-unavailable-"));
+    tempDirs.push(tempDir);
+    const document = normalizeRegressionCatalogDocument({
+      schemaVersion: 1,
+      entries: [
+        {
+          id: "bad-message",
+          classification: "product_bug",
+          issue: {
+            repo: "acme/repo",
+            number: 12,
+          },
+          summary: "API returns the wrong message",
+          cause: "The payload is wrong.",
+          lastReviewedAt: "2026-05-04",
+          fingerprints: [
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            },
+          ],
+        },
+      ],
+    });
+
+    const result = await validateRegressionIssues({
+      productDir: tempDir,
+      document,
+      statusArtifact: {
+        tests: [
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            status: "failed",
+          },
+        ],
+      },
+      config: {
+        provider: "github",
+        mode: "warn",
+      },
+      transport: {
+        async fetchRepoIssues() {
+          throw new Error("boom");
+        },
+      },
+    });
+
+    expect(result.summary.byCode.validation_unavailable).toBe(1);
+    expect(result.summary.errors).toBe(0);
+    expect(result.summary.warnings).toBe(1);
+  });
+});