@elench/testkit 0.1.44 → 0.1.46

package/README.md

@@ -161,12 +161,35 @@ If `reporting.knownFailuresFile` is configured, `testkit` enriches
 - per-file `triage` metadata (issue, classification, description)
 - top-level `triageSummary` counts for known vs untriaged failures
 
+Known-failure entry authoring uses this contract:
+
+- `title`
+  - exact issue-tracker title for the linked issue
+  - shared issues may reuse the same title across multiple local entries
+- `description`
+  - product-local bug slice or route-family summary
+  - this is where file-specific nuance belongs
+- `whyFailing`
+  - underlying technical cause of the failure
+
 If `reporting.issueValidation` is also configured, `testkit` validates known-failure
 issue references against GitHub and adds top-level `knownFailuresIssueValidation`
 data to the run/status artifacts. The most important stale-triage signal is:
 
 - a known-failure test still fails, but the linked GitHub issue is closed
 
+In `mode: "error"`, exact GitHub metadata drift is also treated as a validation
+failure:
+
+- title does not match the linked issue title
+- local open/closed state does not match the linked issue state
+
+Reproduction warnings are execution-aware:
+
+- `failed` means the known failure reproduced
+- `passed` means the matched test executed and did not reproduce
+- `skipped` and `not_run` do not count as reproduction evidence
+
 ## Authoring
 
 HTTP suites:

package/lib/known-failures/github.mjs

@@ -125,7 +125,7 @@ export async function validateKnownFailureIssues({
 
   const summary = buildIssueValidationSummary(entries, globalFindings);
   return {
-    schemaVersion: 1,
+    schemaVersion: 2,
     provider: "github",
     mode: normalizedConfig.mode,
     checkedAt: new Date(now).toISOString(),
@@ -363,7 +363,15 @@ function collectObservedKnownFailureEntries(document, runArtifact, statusArtifac
       const observed = checks.get(entry.id) || createObservedCheck(entry);
       observed.matchedTests += 1;
       if (test.status === "failed") {
+        observed.executedTests += 1;
         observed.failedTests += 1;
+      } else if (test.status === "passed") {
+        observed.executedTests += 1;
+        observed.passedTests += 1;
+      } else if (test.status === "skipped") {
+        observed.skippedTests += 1;
+      } else if (test.status === "not_run") {
+        observed.notRunTests += 1;
       }
       checks.set(entry.id, observed);
     }
@@ -399,7 +407,11 @@ function createObservedCheck(entry) {
   return {
     id: entry.id,
     matchedTests: 0,
+    executedTests: 0,
+    passedTests: 0,
     failedTests: 0,
+    skippedTests: 0,
+    notRunTests: 0,
   };
 }
 
@@ -436,7 +448,7 @@ function buildIssueValidationEntry({ entry, observed, issueData }) {
   if (issueData?.exists && issueData.title && issueData.title !== entry.title) {
     findings.push({
       code: "title_mismatch",
-      severity: "warning",
+      severity: "error",
       message: `Known failure ${entry.id} title does not match issue #${entry.issue.number}`,
     });
   }
@@ -444,7 +456,7 @@ function buildIssueValidationEntry({ entry, observed, issueData }) {
   if (issueData?.exists && normalizedIssueState && normalizedIssueState !== entry.state) {
     findings.push({
       code: "state_mismatch",
-      severity: "warning",
+      severity: "error",
       message: `Known failure ${entry.id} state ${entry.state} does not match issue #${entry.issue.number} state ${normalizedIssueState}`,
     });
   }
@@ -457,7 +469,12 @@ function buildIssueValidationEntry({ entry, observed, issueData }) {
     });
   }
 
-  if (issueData?.exists && observed.failedTests === 0 && observed.matchedTests > 0) {
+  if (
+    issueData?.exists &&
+    observed.executedTests > 0 &&
+    observed.failedTests === 0 &&
+    observed.passedTests > 0
+  ) {
     if (normalizedIssueState === "open") {
       findings.push({
         code: "open_not_reproduced",
@@ -479,7 +496,11 @@ function buildIssueValidationEntry({ entry, observed, issueData }) {
     issue: entry.issue,
     observed: {
       matchedTests: observed.matchedTests,
+      executedTests: observed.executedTests,
+      passedTests: observed.passedTests,
       failedTests: observed.failedTests,
+      skippedTests: observed.skippedTests,
+      notRunTests: observed.notRunTests,
       reproduced: observed.failedTests > 0,
     },
     github: issueData
@@ -510,8 +531,23 @@ function resolveIssueValidationStatus(issueData, observed, findings) {
   const normalizedState = normalizeIssueState(issueData.state);
   if (observed.failedTests > 0 && normalizedState === "closed") return "closed_but_failing";
   if (observed.failedTests > 0 && normalizedState === "open") return "open_and_failing";
-  if (observed.matchedTests > 0 && normalizedState === "open") return "open_not_reproduced";
-  if (observed.matchedTests > 0 && normalizedState === "closed") return "closed_not_reproduced";
+  if (observed.executedTests === 0 && observed.matchedTests > 0) return "not_executed";
+  if (
+    observed.executedTests > 0 &&
+    observed.failedTests === 0 &&
+    observed.passedTests > 0 &&
+    normalizedState === "open"
+  ) {
+    return "open_not_reproduced";
+  }
+  if (
+    observed.executedTests > 0 &&
+    observed.failedTests === 0 &&
+    observed.passedTests > 0 &&
+    normalizedState === "closed"
+  ) {
+    return "closed_not_reproduced";
+  }
   if (findings.length > 0) return "metadata_mismatch";
   return "not_observed";
 }

package/lib/known-failures/github.test.mjs

@@ -25,7 +25,7 @@ describe("known failures GitHub validation", () => {
     expect(parseGitHubRepoSlug("https://gitlab.com/acme/repo.git")).toBe(null);
   });
 
-  it("flags closed issues that still reproduce and title mismatches", async () => {
+  it("fails validation for title/state drift and closed issues that still reproduce", async () => {
     const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-issues-"));
     tempDirs.push(tempDir);
     const document = normalizeKnownFailuresDocument({
@@ -98,11 +98,114 @@ describe("known failures GitHub validation", () => {
 
     expect(result.summary.byCode.closed_but_failing).toBe(1);
     expect(result.summary.byCode.title_mismatch).toBe(1);
+    expect(result.summary.byCode.state_mismatch).toBe(1);
     expect(result.summary.errors).toBeGreaterThan(0);
     expect(result.entries[0].status).toBe("closed_but_failing");
     expect(shouldFailKnownFailureIssueValidation(result)).toBe(true);
   });
 
+  it("allows multiple local entries to share one issue title while keeping distinct descriptions", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-shared-title-"));
+    tempDirs.push(tempDir);
+    const document = normalizeKnownFailuresDocument({
+      schemaVersion: 1,
+      issueRepo: "acme/repo",
+      entries: [
+        {
+          id: "route-a-invalid-uuid",
+          title: "UUID path params reach Postgres and return 500 instead of 400",
+          classification: "product_bug",
+          state: "open",
+          issue: {
+            repo: "acme/repo",
+            number: 77,
+            url: "https://github.com/acme/repo/issues/77",
+          },
+          description: "Route A leaks Postgres UUID parse errors on malformed ids.",
+          whyFailing: "Route A is missing UUID param validation.",
+          lastReviewedAt: "2026-04-27",
+          matches: [
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/route-a.int.testkit.ts",
+            },
+          ],
+        },
+        {
+          id: "route-b-invalid-uuid",
+          title: "UUID path params reach Postgres and return 500 instead of 400",
+          classification: "product_bug",
+          state: "open",
+          issue: {
+            repo: "acme/repo",
+            number: 77,
+            url: "https://github.com/acme/repo/issues/77",
+          },
+          description: "Route B leaks Postgres UUID parse errors on malformed ids.",
+          whyFailing: "Route B is missing UUID param validation.",
+          lastReviewedAt: "2026-04-27",
+          matches: [
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/route-b.int.testkit.ts",
+            },
+          ],
+        },
+      ],
+    });
+
+    const result = await validateKnownFailureIssues({
+      productDir: tempDir,
+      document,
+      statusArtifact: {
+        tests: [
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/route-a.int.testkit.ts",
+            status: "failed",
+          },
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/route-b.int.testkit.ts",
+            status: "failed",
+          },
+        ],
+      },
+      config: {
+        provider: "github",
+        mode: "error",
+      },
+      transport: {
+        async fetchRepoIssues(repo, numbers) {
+          const map = new Map();
+          map.set(numbers[0], {
+            repo,
+            number: numbers[0],
+            exists: true,
+            title: "UUID path params reach Postgres and return 500 instead of 400",
+            state: "OPEN",
+            url: `https://github.com/${repo}/issues/${numbers[0]}`,
+            checkedAt: "2026-04-27T00:00:00.000Z",
+            source: "github",
+          });
+          return map;
+        },
+      },
+    });
+
+    expect(result.summary.byCode.title_mismatch).toBeUndefined();
+    expect(result.summary.errors).toBe(0);
+    expect(result.entries).toHaveLength(2);
+    expect(result.entries[0].findings).toEqual([]);
+    expect(result.entries[1].findings).toEqual([]);
+    expect(result.entries[0].status).toBe("open_and_failing");
+    expect(result.entries[1].status).toBe("open_and_failing");
+  });
+
   it("warns when an open issue is not reproduced", async () => {
     const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-open-"));
     tempDirs.push(tempDir);
@@ -176,6 +279,196 @@ describe("known failures GitHub validation", () => {
     expect(result.summary.errors).toBe(0);
     expect(result.summary.warnings).toBeGreaterThan(0);
     expect(shouldFailKnownFailureIssueValidation(result)).toBe(false);
+    expect(result.entries[0].observed).toMatchObject({
+      matchedTests: 1,
+      executedTests: 1,
+      passedTests: 1,
+      failedTests: 0,
+      skippedTests: 0,
+      notRunTests: 0,
+      reproduced: false,
+    });
+  });
+
+  it("does not treat skipped or not_run files as not reproduced", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-not-executed-"));
+    tempDirs.push(tempDir);
+    const document = normalizeKnownFailuresDocument({
+      schemaVersion: 1,
+      issueRepo: "acme/repo",
+      entries: [
+        {
+          id: "bad-message",
+          title: "Bad message bug",
+          classification: "product_bug",
+          state: "open",
+          issue: {
+            repo: "acme/repo",
+            number: 12,
+            url: "https://github.com/acme/repo/issues/12",
+          },
+          description: "Wrong message",
+          whyFailing: "Payload is wrong",
+          lastReviewedAt: "2026-04-27",
+          matches: [
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            },
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/blocked.int.testkit.ts",
+            },
+          ],
+        },
+      ],
+    });
+
+    const result = await validateKnownFailureIssues({
+      productDir: tempDir,
+      document,
+      statusArtifact: {
+        tests: [
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            status: "skipped",
+          },
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/blocked.int.testkit.ts",
+            status: "not_run",
+          },
+        ],
+      },
+      config: {
+        provider: "github",
+        mode: "warn",
+      },
+      transport: {
+        async fetchRepoIssues(repo, numbers) {
+          const map = new Map();
+          map.set(numbers[0], {
+            repo,
+            number: numbers[0],
+            exists: true,
+            title: "Bad message bug",
+            state: "OPEN",
+            url: `https://github.com/${repo}/issues/${numbers[0]}`,
+            checkedAt: "2026-04-27T00:00:00.000Z",
+            source: "github",
+          });
+          return map;
+        },
+      },
+    });
+
+    expect(result.summary.byCode.open_not_reproduced).toBeUndefined();
+    expect(result.entries[0].findings).toEqual([]);
+    expect(result.entries[0].status).toBe("not_executed");
+    expect(result.entries[0].observed).toMatchObject({
+      matchedTests: 2,
+      executedTests: 0,
+      passedTests: 0,
+      failedTests: 0,
+      skippedTests: 1,
+      notRunTests: 1,
+      reproduced: false,
+    });
+  });
+
+  it("still treats mixed passed and skipped execution as not reproduced", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-mixed-execution-"));
+    tempDirs.push(tempDir);
+    const document = normalizeKnownFailuresDocument({
+      schemaVersion: 1,
+      issueRepo: "acme/repo",
+      entries: [
+        {
+          id: "bad-message",
+          title: "Bad message bug",
+          classification: "product_bug",
+          state: "open",
+          issue: {
+            repo: "acme/repo",
+            number: 12,
+            url: "https://github.com/acme/repo/issues/12",
+          },
+          description: "Wrong message",
+          whyFailing: "Payload is wrong",
+          lastReviewedAt: "2026-04-27",
+          matches: [
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/passed.int.testkit.ts",
+            },
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/skipped.int.testkit.ts",
+            },
+          ],
+        },
+      ],
+    });
+
+    const result = await validateKnownFailureIssues({
+      productDir: tempDir,
+      document,
+      statusArtifact: {
+        tests: [
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/passed.int.testkit.ts",
+            status: "passed",
+          },
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/skipped.int.testkit.ts",
+            status: "skipped",
+          },
+        ],
+      },
+      config: {
+        provider: "github",
+        mode: "warn",
+      },
+      transport: {
+        async fetchRepoIssues(repo, numbers) {
+          const map = new Map();
+          map.set(numbers[0], {
+            repo,
+            number: numbers[0],
+            exists: true,
+            title: "Bad message bug",
+            state: "OPEN",
+            url: `https://github.com/${repo}/issues/${numbers[0]}`,
+            checkedAt: "2026-04-27T00:00:00.000Z",
+            source: "github",
+          });
+          return map;
+        },
+      },
+    });
+
+    expect(result.summary.byCode.open_not_reproduced).toBe(1);
+    expect(result.entries[0].status).toBe("open_not_reproduced");
+    expect(result.entries[0].observed).toMatchObject({
+      matchedTests: 2,
+      executedTests: 1,
+      passedTests: 1,
+      failedTests: 0,
+      skippedTests: 1,
+      notRunTests: 0,
+      reproduced: false,
+    });
   });
 
   it("falls back to warning when validation is unavailable", async () => {

package/lib/known-failures/index.d.ts

@@ -23,11 +23,14 @@ export interface KnownFailureMatch {
 
 export interface KnownFailureEntry {
   id: string;
+  /** Exact issue-tracker title for the linked issue. */
   title: string;
   classification: KnownFailureClassification;
   state: KnownFailureState;
   issue: KnownFailureIssueRef;
+  /** Product-local bug slice or route-family summary. */
   description: string;
+  /** Underlying technical cause of the failure. */
   whyFailing: string;
   lastReviewedAt: string;
   matches: KnownFailureMatch[];
@@ -69,7 +72,11 @@ export interface KnownFailureIssueValidationEntry {
   issue: KnownFailureIssueRef;
   observed: {
     matchedTests: number;
+    executedTests: number;
+    passedTests: number;
     failedTests: number;
+    skippedTests: number;
+    notRunTests: number;
     reproduced: boolean;
   };
   github: {
@@ -85,7 +92,7 @@ export interface KnownFailureIssueValidationEntry {
 }
 
 export interface KnownFailureIssueValidationResult {
-  schemaVersion: 1;
+  schemaVersion: 2;
   provider: "github";
   mode: "off" | "warn" | "error";
   checkedAt: string;

package/lib/runner/reporting.mjs

@@ -78,7 +78,7 @@ export function buildStatusArtifact({
     scope.serviceFilter === null;
 
   return {
-    schemaVersion: 5,
+    schemaVersion: 6,
     source: "testkit",
     notice: "Generated file. Do not edit manually.",
     product: {
@@ -127,7 +127,7 @@ export function buildRunArtifact({
   const dbBackend = summarizeDbBackend(results);
 
   return {
-    schemaVersion: 5,
+    schemaVersion: 6,
     source: "testkit",
     generatedAt: new Date(finishedAt).toISOString(),
     product: {

package/lib/runner/reporting.test.mjs

@@ -78,7 +78,7 @@ describe("runner reporting", () => {
     });
 
     expect(artifact.product.name).toBe("my-product");
-    expect(artifact.schemaVersion).toBe(5);
+    expect(artifact.schemaVersion).toBe(6);
     expect(artifact.run).toMatchObject({
       workers: 2,
       fileTimeoutSeconds: 60,
@@ -149,7 +149,7 @@ describe("runner reporting", () => {
     });
 
     expect(status).toEqual({
-      schemaVersion: 5,
+      schemaVersion: 6,
       source: "testkit",
       notice: "Generated file. Do not edit manually.",
       product: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/testkit",
-  "version": "0.1.44",
+  "version": "0.1.46",
   "description": "CLI for discovering and running local HTTP, DAL, and Playwright test suites",
   "type": "module",
   "types": "./lib/index.d.ts",