@elench/testkit 0.1.43 → 0.1.44

package/lib/known-failures/github.test.mjs

@@ -0,0 +1,219 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { afterEach, describe, expect, it } from "vitest";
+import { normalizeKnownFailuresDocument } from "./index.mjs";
+import {
+  parseGitHubRepoSlug,
+  shouldFailKnownFailureIssueValidation,
+  validateKnownFailureIssues,
+} from "./github.mjs";
+
+const tempDirs = [];
+
+afterEach(() => {
+  for (const tempDir of tempDirs.splice(0)) {
+    fs.rmSync(tempDir, { recursive: true, force: true });
+  }
+});
+
+describe("known failures GitHub validation", () => {
+  it("parses GitHub repo slugs from common origin URLs", () => {
+    expect(parseGitHubRepoSlug("https://github.com/acme/repo.git")).toBe("acme/repo");
+    expect(parseGitHubRepoSlug("git@github.com:acme/repo.git")).toBe("acme/repo");
+    expect(parseGitHubRepoSlug("ssh://git@github.com/acme/repo.git")).toBe("acme/repo");
+    expect(parseGitHubRepoSlug("https://gitlab.com/acme/repo.git")).toBe(null);
+  });
+
+  it("flags closed issues that still reproduce and title mismatches", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-issues-"));
+    tempDirs.push(tempDir);
+    const document = normalizeKnownFailuresDocument({
+      schemaVersion: 1,
+      issueRepo: "acme/repo",
+      entries: [
+        {
+          id: "bad-message",
+          title: "Bad message bug",
+          classification: "product_bug",
+          state: "open",
+          issue: {
+            repo: "acme/repo",
+            number: 12,
+            url: "https://github.com/acme/repo/issues/12",
+          },
+          description: "Wrong message",
+          whyFailing: "Payload is wrong",
+          lastReviewedAt: "2026-04-27",
+          matches: [
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            },
+          ],
+        },
+      ],
+    });
+
+    const result = await validateKnownFailureIssues({
+      productDir: tempDir,
+      document,
+      statusArtifact: {
+        tests: [
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            status: "failed",
+          },
+        ],
+      },
+      config: {
+        provider: "github",
+        mode: "error",
+        cacheTtlSeconds: 60,
+      },
+      gitMetadata: {
+        repoSlug: "acme/repo",
+        remoteUrl: "https://github.com/acme/repo.git",
+      },
+      transport: {
+        async fetchRepoIssues(repo, numbers) {
+          const map = new Map();
+          map.set(numbers[0], {
+            repo,
+            number: numbers[0],
+            exists: true,
+            title: "Different title",
+            state: "CLOSED",
+            url: `https://github.com/${repo}/issues/${numbers[0]}`,
+            checkedAt: "2026-04-27T00:00:00.000Z",
+            source: "github",
+          });
+          return map;
+        },
+      },
+    });
+
+    expect(result.summary.byCode.closed_but_failing).toBe(1);
+    expect(result.summary.byCode.title_mismatch).toBe(1);
+    expect(result.summary.errors).toBeGreaterThan(0);
+    expect(result.entries[0].status).toBe("closed_but_failing");
+    expect(shouldFailKnownFailureIssueValidation(result)).toBe(true);
+  });
+
+  it("warns when an open issue is not reproduced", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-open-"));
+    tempDirs.push(tempDir);
+    const document = normalizeKnownFailuresDocument({
+      schemaVersion: 1,
+      issueRepo: "acme/repo",
+      entries: [
+        {
+          id: "bad-message",
+          title: "Bad message bug",
+          classification: "product_bug",
+          state: "open",
+          issue: {
+            repo: "acme/repo",
+            number: 12,
+            url: "https://github.com/acme/repo/issues/12",
+          },
+          description: "Wrong message",
+          whyFailing: "Payload is wrong",
+          lastReviewedAt: "2026-04-27",
+          matches: [
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            },
+          ],
+        },
+      ],
+    });
+
+    const result = await validateKnownFailureIssues({
+      productDir: tempDir,
+      document,
+      statusArtifact: {
+        tests: [
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            status: "passed",
+          },
+        ],
+      },
+      config: {
+        provider: "github",
+        mode: "warn",
+      },
+      gitMetadata: {
+        repoSlug: "acme/repo",
+      },
+      transport: {
+        async fetchRepoIssues(repo, numbers) {
+          const map = new Map();
+          map.set(numbers[0], {
+            repo,
+            number: numbers[0],
+            exists: true,
+            title: "Bad message bug",
+            state: "OPEN",
+            url: `https://github.com/${repo}/issues/${numbers[0]}`,
+            checkedAt: "2026-04-27T00:00:00.000Z",
+            source: "github",
+          });
+          return map;
+        },
+      },
+    });
+
+    expect(result.summary.byCode.open_not_reproduced).toBe(1);
+    expect(result.summary.errors).toBe(0);
+    expect(result.summary.warnings).toBeGreaterThan(0);
+    expect(shouldFailKnownFailureIssueValidation(result)).toBe(false);
+  });
+
+  it("falls back to warning when validation is unavailable", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-unavailable-"));
+    tempDirs.push(tempDir);
+    const document = normalizeKnownFailuresDocument({
+      schemaVersion: 1,
+      entries: [
+        {
+          id: "bad-message",
+          title: "Bad message bug",
+          classification: "product_bug",
+          state: "open",
+          issue: {
+            repo: "acme/repo",
+            number: 12,
+            url: "https://github.com/acme/repo/issues/12",
+          },
+          description: "Wrong message",
+          whyFailing: "Payload is wrong",
+          lastReviewedAt: "2026-04-27",
+          matches: [{ path: "src/api/routes/__testkit__/failing.int.testkit.ts" }],
+        },
+      ],
+    });
+
+    const result = await validateKnownFailureIssues({
+      productDir: tempDir,
+      document,
+      config: {
+        provider: "github",
+        mode: "warn",
+      },
+      transport: null,
+    });
+
+    expect(result.summary.byCode.validation_unavailable).toBe(1);
+    expect(result.summary.errors).toBe(0);
+    expect(result.summary.warnings).toBeGreaterThan(0);
+  });
+});

package/lib/known-failures/index.d.ts

@@ -0,0 +1,185 @@
+export type KnownFailureClassification =
+  | "expected_failure"
+  | "infra"
+  | "product_bug"
+  | "stale_test"
+  | "test_bug";
+
+export type KnownFailureState = "closed" | "open";
+
+export interface KnownFailureIssueRef {
+  repo: string;
+  number: number;
+  url: string;
+}
+
+export interface KnownFailureMatch {
+  service?: string;
+  type?: string;
+  path: string;
+  failureKey?: string;
+  errorIncludes?: string;
+}
+
+export interface KnownFailureEntry {
+  id: string;
+  title: string;
+  classification: KnownFailureClassification;
+  state: KnownFailureState;
+  issue: KnownFailureIssueRef;
+  description: string;
+  whyFailing: string;
+  lastReviewedAt: string;
+  matches: KnownFailureMatch[];
+}
+
+export interface KnownFailuresDocument {
+  schemaVersion: 1;
+  issueRepo?: string | null;
+  entries: KnownFailureEntry[];
+}
+
+export interface KnownFailuresValidationResult {
+  errors: string[];
+  warnings: string[];
+  stats: {
+    entries: number;
+    matches: number;
+    failedTests: number;
+    triagedFailedTests: number;
+    untriagedFailedTests: number;
+  };
+}
+
+export interface KnownFailureIssueValidationConfig {
+  provider?: "github";
+  mode?: "off" | "warn" | "error";
+  cacheTtlSeconds?: number;
+}
+
+export interface KnownFailureIssueValidationFinding {
+  code: string;
+  severity: "warning" | "error";
+  message: string;
+}
+
+export interface KnownFailureIssueValidationEntry {
+  id: string;
+  title: string;
+  issue: KnownFailureIssueRef;
+  observed: {
+    matchedTests: number;
+    failedTests: number;
+    reproduced: boolean;
+  };
+  github: {
+    exists: boolean | null;
+    title: string | null;
+    state: string | null;
+    url: string | null;
+    checkedAt: string | null;
+    cached: boolean;
+  };
+  findings: KnownFailureIssueValidationFinding[];
+  status: string;
+}
+
+export interface KnownFailureIssueValidationResult {
+  schemaVersion: 1;
+  provider: "github";
+  mode: "off" | "warn" | "error";
+  checkedAt: string;
+  repo: {
+    detected: string | null;
+    remoteUrl: string | null;
+    configured: string | null;
+  };
+  availability: {
+    hasFreshData: boolean;
+    usedCachedFallback: boolean;
+  };
+  findings: KnownFailureIssueValidationFinding[];
+  summary: {
+    entries: number;
+    observedEntries: number;
+    errors: number;
+    warnings: number;
+    byCode: Record<string, number>;
+    byStatus: Record<string, number>;
+  };
+  entries: KnownFailureIssueValidationEntry[];
+}
+
+export declare function loadKnownFailuresConfig(
+  productDir: string,
+  config: { knownFailuresFile?: string | null } | null
+): KnownFailuresDocument | null;
+export declare function loadKnownFailuresDocument(
+  filePath: string,
+  relativePath?: string
+): KnownFailuresDocument;
+export declare function normalizeKnownFailuresDocument(
+  document: unknown,
+  relativePath?: string
+): KnownFailuresDocument;
+export declare function validateKnownFailuresDocument(
+  document: KnownFailuresDocument,
+  options?: {
+    productDir?: string;
+    statusArtifactPath?: string;
+    statusArtifact?: unknown;
+  }
+): KnownFailuresValidationResult;
+export declare function renderKnownFailuresMarkdown(document: KnownFailuresDocument): string;
+export declare function findMatchingKnownFailureEntries(
+  document: KnownFailuresDocument,
+  fileSummary: {
+    service?: string;
+    type?: string;
+    path: string;
+    status?: string;
+    error?: string | null;
+    failureDetails?: Array<{ key?: string }>;
+  }
+): KnownFailureEntry[];
+export declare function matchesKnownFailureEntry(
+  entry: KnownFailureEntry,
+  fileSummary: {
+    service?: string;
+    type?: string;
+    path: string;
+    error?: string | null;
+    failureDetails?: Array<{ key?: string }>;
+  }
+): boolean;
+export declare function buildKnownFailureFileIdentity(
+  service: string,
+  type: string,
+  filePath: string
+): string;
+
+export declare function normalizeKnownFailureIssueValidationConfig(
+  value: unknown
+): {
+  provider: "github";
+  mode: "off" | "warn" | "error";
+  cacheTtlSeconds: number;
+} | null;
+export declare function validateKnownFailureIssues(options: {
+  productDir: string;
+  document: KnownFailuresDocument | null;
+  runArtifact?: unknown;
+  statusArtifact?: unknown;
+  config?: KnownFailureIssueValidationConfig | null;
+  gitMetadata?: {
+    repoSlug?: string | null;
+    remoteUrl?: string | null;
+  } | null;
+}): Promise<KnownFailureIssueValidationResult | null>;
+export declare function shouldFailKnownFailureIssueValidation(
+  result: KnownFailureIssueValidationResult | null
+): boolean;
+export declare function buildKnownFailureIssueValidationSummaryLines(
+  result: KnownFailureIssueValidationResult | null
+): string[];
+export declare function parseGitHubRepoSlug(remoteUrl: string | null | undefined): string | null;