@elench/testkit 0.1.45 → 0.1.46

package/README.md

@@ -184,6 +184,12 @@ failure:
 - title does not match the linked issue title
 - local open/closed state does not match the linked issue state
 
+Reproduction warnings are execution-aware:
+
+- `failed` means the known failure reproduced
+- `passed` means the matched test executed and did not reproduce
+- `skipped` and `not_run` do not count as reproduction evidence
+
 ## Authoring
 
 HTTP suites:

package/lib/known-failures/github.mjs

@@ -125,7 +125,7 @@ export async function validateKnownFailureIssues({
 
   const summary = buildIssueValidationSummary(entries, globalFindings);
   return {
-    schemaVersion: 1,
+    schemaVersion: 2,
     provider: "github",
     mode: normalizedConfig.mode,
     checkedAt: new Date(now).toISOString(),
@@ -363,7 +363,15 @@ function collectObservedKnownFailureEntries(document, runArtifact, statusArtifac
       const observed = checks.get(entry.id) || createObservedCheck(entry);
       observed.matchedTests += 1;
       if (test.status === "failed") {
+        observed.executedTests += 1;
         observed.failedTests += 1;
+      } else if (test.status === "passed") {
+        observed.executedTests += 1;
+        observed.passedTests += 1;
+      } else if (test.status === "skipped") {
+        observed.skippedTests += 1;
+      } else if (test.status === "not_run") {
+        observed.notRunTests += 1;
       }
       checks.set(entry.id, observed);
     }
@@ -399,7 +407,11 @@ function createObservedCheck(entry) {
   return {
     id: entry.id,
     matchedTests: 0,
+    executedTests: 0,
+    passedTests: 0,
     failedTests: 0,
+    skippedTests: 0,
+    notRunTests: 0,
   };
 }
 
@@ -457,7 +469,12 @@ function buildIssueValidationEntry({ entry, observed, issueData }) {
     });
   }
 
-  if (issueData?.exists && observed.failedTests === 0 && observed.matchedTests > 0) {
+  if (
+    issueData?.exists &&
+    observed.executedTests > 0 &&
+    observed.failedTests === 0 &&
+    observed.passedTests > 0
+  ) {
     if (normalizedIssueState === "open") {
       findings.push({
         code: "open_not_reproduced",
@@ -479,7 +496,11 @@ function buildIssueValidationEntry({ entry, observed, issueData }) {
     issue: entry.issue,
     observed: {
       matchedTests: observed.matchedTests,
+      executedTests: observed.executedTests,
+      passedTests: observed.passedTests,
       failedTests: observed.failedTests,
+      skippedTests: observed.skippedTests,
+      notRunTests: observed.notRunTests,
       reproduced: observed.failedTests > 0,
     },
     github: issueData
@@ -510,8 +531,23 @@ function resolveIssueValidationStatus(issueData, observed, findings) {
   const normalizedState = normalizeIssueState(issueData.state);
   if (observed.failedTests > 0 && normalizedState === "closed") return "closed_but_failing";
   if (observed.failedTests > 0 && normalizedState === "open") return "open_and_failing";
-  if (observed.matchedTests > 0 && normalizedState === "open") return "open_not_reproduced";
-  if (observed.matchedTests > 0 && normalizedState === "closed") return "closed_not_reproduced";
+  if (observed.executedTests === 0 && observed.matchedTests > 0) return "not_executed";
+  if (
+    observed.executedTests > 0 &&
+    observed.failedTests === 0 &&
+    observed.passedTests > 0 &&
+    normalizedState === "open"
+  ) {
+    return "open_not_reproduced";
+  }
+  if (
+    observed.executedTests > 0 &&
+    observed.failedTests === 0 &&
+    observed.passedTests > 0 &&
+    normalizedState === "closed"
+  ) {
+    return "closed_not_reproduced";
+  }
   if (findings.length > 0) return "metadata_mismatch";
   return "not_observed";
 }

package/lib/known-failures/github.test.mjs

@@ -279,6 +279,196 @@ describe("known failures GitHub validation", () => {
     expect(result.summary.errors).toBe(0);
     expect(result.summary.warnings).toBeGreaterThan(0);
     expect(shouldFailKnownFailureIssueValidation(result)).toBe(false);
+    expect(result.entries[0].observed).toMatchObject({
+      matchedTests: 1,
+      executedTests: 1,
+      passedTests: 1,
+      failedTests: 0,
+      skippedTests: 0,
+      notRunTests: 0,
+      reproduced: false,
+    });
+  });
+
+  it("does not treat skipped or not_run files as not reproduced", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-not-executed-"));
+    tempDirs.push(tempDir);
+    const document = normalizeKnownFailuresDocument({
+      schemaVersion: 1,
+      issueRepo: "acme/repo",
+      entries: [
+        {
+          id: "bad-message",
+          title: "Bad message bug",
+          classification: "product_bug",
+          state: "open",
+          issue: {
+            repo: "acme/repo",
+            number: 12,
+            url: "https://github.com/acme/repo/issues/12",
+          },
+          description: "Wrong message",
+          whyFailing: "Payload is wrong",
+          lastReviewedAt: "2026-04-27",
+          matches: [
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            },
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/blocked.int.testkit.ts",
+            },
+          ],
+        },
+      ],
+    });
+
+    const result = await validateKnownFailureIssues({
+      productDir: tempDir,
+      document,
+      statusArtifact: {
+        tests: [
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            status: "skipped",
+          },
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/blocked.int.testkit.ts",
+            status: "not_run",
+          },
+        ],
+      },
+      config: {
+        provider: "github",
+        mode: "warn",
+      },
+      transport: {
+        async fetchRepoIssues(repo, numbers) {
+          const map = new Map();
+          map.set(numbers[0], {
+            repo,
+            number: numbers[0],
+            exists: true,
+            title: "Bad message bug",
+            state: "OPEN",
+            url: `https://github.com/${repo}/issues/${numbers[0]}`,
+            checkedAt: "2026-04-27T00:00:00.000Z",
+            source: "github",
+          });
+          return map;
+        },
+      },
+    });
+
+    expect(result.summary.byCode.open_not_reproduced).toBeUndefined();
+    expect(result.entries[0].findings).toEqual([]);
+    expect(result.entries[0].status).toBe("not_executed");
+    expect(result.entries[0].observed).toMatchObject({
+      matchedTests: 2,
+      executedTests: 0,
+      passedTests: 0,
+      failedTests: 0,
+      skippedTests: 1,
+      notRunTests: 1,
+      reproduced: false,
+    });
+  });
+
+  it("still treats mixed passed and skipped execution as not reproduced", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-mixed-execution-"));
+    tempDirs.push(tempDir);
+    const document = normalizeKnownFailuresDocument({
+      schemaVersion: 1,
+      issueRepo: "acme/repo",
+      entries: [
+        {
+          id: "bad-message",
+          title: "Bad message bug",
+          classification: "product_bug",
+          state: "open",
+          issue: {
+            repo: "acme/repo",
+            number: 12,
+            url: "https://github.com/acme/repo/issues/12",
+          },
+          description: "Wrong message",
+          whyFailing: "Payload is wrong",
+          lastReviewedAt: "2026-04-27",
+          matches: [
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/passed.int.testkit.ts",
+            },
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/skipped.int.testkit.ts",
+            },
+          ],
+        },
+      ],
+    });
+
+    const result = await validateKnownFailureIssues({
+      productDir: tempDir,
+      document,
+      statusArtifact: {
+        tests: [
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/passed.int.testkit.ts",
+            status: "passed",
+          },
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/skipped.int.testkit.ts",
+            status: "skipped",
+          },
+        ],
+      },
+      config: {
+        provider: "github",
+        mode: "warn",
+      },
+      transport: {
+        async fetchRepoIssues(repo, numbers) {
+          const map = new Map();
+          map.set(numbers[0], {
+            repo,
+            number: numbers[0],
+            exists: true,
+            title: "Bad message bug",
+            state: "OPEN",
+            url: `https://github.com/${repo}/issues/${numbers[0]}`,
+            checkedAt: "2026-04-27T00:00:00.000Z",
+            source: "github",
+          });
+          return map;
+        },
+      },
+    });
+
+    expect(result.summary.byCode.open_not_reproduced).toBe(1);
+    expect(result.entries[0].status).toBe("open_not_reproduced");
+    expect(result.entries[0].observed).toMatchObject({
+      matchedTests: 2,
+      executedTests: 1,
+      passedTests: 1,
+      failedTests: 0,
+      skippedTests: 1,
+      notRunTests: 0,
+      reproduced: false,
+    });
   });
 
   it("falls back to warning when validation is unavailable", async () => {

package/lib/known-failures/index.d.ts

@@ -72,7 +72,11 @@ export interface KnownFailureIssueValidationEntry {
   issue: KnownFailureIssueRef;
   observed: {
     matchedTests: number;
+    executedTests: number;
+    passedTests: number;
     failedTests: number;
+    skippedTests: number;
+    notRunTests: number;
     reproduced: boolean;
   };
   github: {
@@ -88,7 +92,7 @@ export interface KnownFailureIssueValidationEntry {
 }
 
 export interface KnownFailureIssueValidationResult {
-  schemaVersion: 1;
+  schemaVersion: 2;
   provider: "github";
   mode: "off" | "warn" | "error";
   checkedAt: string;

package/lib/runner/reporting.mjs

@@ -78,7 +78,7 @@ export function buildStatusArtifact({
     scope.serviceFilter === null;
 
   return {
-    schemaVersion: 5,
+    schemaVersion: 6,
     source: "testkit",
     notice: "Generated file. Do not edit manually.",
     product: {
@@ -127,7 +127,7 @@ export function buildRunArtifact({
   const dbBackend = summarizeDbBackend(results);
 
   return {
-    schemaVersion: 5,
+    schemaVersion: 6,
     source: "testkit",
     generatedAt: new Date(finishedAt).toISOString(),
     product: {

package/lib/runner/reporting.test.mjs

@@ -78,7 +78,7 @@ describe("runner reporting", () => {
     });
 
     expect(artifact.product.name).toBe("my-product");
-    expect(artifact.schemaVersion).toBe(5);
+    expect(artifact.schemaVersion).toBe(6);
     expect(artifact.run).toMatchObject({
       workers: 2,
       fileTimeoutSeconds: 60,
@@ -149,7 +149,7 @@ describe("runner reporting", () => {
     });
 
     expect(status).toEqual({
-      schemaVersion: 5,
+      schemaVersion: 6,
       source: "testkit",
       notice: "Generated file. Do not edit manually.",
       product: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/testkit",
-  "version": "0.1.45",
+  "version": "0.1.46",
   "description": "CLI for discovering and running local HTTP, DAL, and Playwright test suites",
   "type": "module",
   "types": "./lib/index.d.ts",