@elench/testkit 0.1.79 → 0.1.81

package/lib/known-failures/github.test.mjs

@@ -1,512 +0,0 @@
-import fs from "fs";
-import os from "os";
-import path from "path";
-import { afterEach, describe, expect, it } from "vitest";
-import { normalizeKnownFailuresDocument } from "./index.mjs";
-import {
-  parseGitHubRepoSlug,
-  shouldFailKnownFailureIssueValidation,
-  validateKnownFailureIssues,
-} from "./github.mjs";
-
-const tempDirs = [];
-
-afterEach(() => {
-  for (const tempDir of tempDirs.splice(0)) {
-    fs.rmSync(tempDir, { recursive: true, force: true });
-  }
-});
-
-describe("known failures GitHub validation", () => {
-  it("parses GitHub repo slugs from common origin URLs", () => {
-    expect(parseGitHubRepoSlug("https://github.com/acme/repo.git")).toBe("acme/repo");
-    expect(parseGitHubRepoSlug("git@github.com:acme/repo.git")).toBe("acme/repo");
-    expect(parseGitHubRepoSlug("ssh://git@github.com/acme/repo.git")).toBe("acme/repo");
-    expect(parseGitHubRepoSlug("https://gitlab.com/acme/repo.git")).toBe(null);
-  });
-
-  it("fails validation for title/state drift and closed issues that still reproduce", async () => {
-    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-issues-"));
-    tempDirs.push(tempDir);
-    const document = normalizeKnownFailuresDocument({
-      schemaVersion: 1,
-      issueRepo: "acme/repo",
-      entries: [
-        {
-          id: "bad-message",
-          title: "Bad message bug",
-          classification: "product_bug",
-          state: "open",
-          issue: {
-            repo: "acme/repo",
-            number: 12,
-            url: "https://github.com/acme/repo/issues/12",
-          },
-          description: "Wrong message",
-          whyFailing: "Payload is wrong",
-          lastReviewedAt: "2026-04-27",
-          matches: [
-            {
-              service: "api",
-              type: "int",
-              path: "src/api/routes/__testkit__/failing.int.testkit.ts",
-            },
-          ],
-        },
-      ],
-    });
-
-    const result = await validateKnownFailureIssues({
-      productDir: tempDir,
-      document,
-      statusArtifact: {
-        tests: [
-          {
-            service: "api",
-            type: "int",
-            path: "src/api/routes/__testkit__/failing.int.testkit.ts",
-            status: "failed",
-          },
-        ],
-      },
-      config: {
-        provider: "github",
-        mode: "error",
-        cacheTtlSeconds: 60,
-      },
-      gitMetadata: {
-        repoSlug: "acme/repo",
-        remoteUrl: "https://github.com/acme/repo.git",
-      },
-      transport: {
-        async fetchRepoIssues(repo, numbers) {
-          const map = new Map();
-          map.set(numbers[0], {
-            repo,
-            number: numbers[0],
-            exists: true,
-            title: "Different title",
-            state: "CLOSED",
-            url: `https://github.com/${repo}/issues/${numbers[0]}`,
-            checkedAt: "2026-04-27T00:00:00.000Z",
-            source: "github",
-          });
-          return map;
-        },
-      },
-    });
-
-    expect(result.summary.byCode.closed_but_failing).toBe(1);
-    expect(result.summary.byCode.title_mismatch).toBe(1);
-    expect(result.summary.byCode.state_mismatch).toBe(1);
-    expect(result.summary.errors).toBeGreaterThan(0);
-    expect(result.entries[0].status).toBe("closed_but_failing");
-    expect(shouldFailKnownFailureIssueValidation(result)).toBe(true);
-  });
-
-  it("allows multiple local entries to share one issue title while keeping distinct descriptions", async () => {
-    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-shared-title-"));
-    tempDirs.push(tempDir);
-    const document = normalizeKnownFailuresDocument({
-      schemaVersion: 1,
-      issueRepo: "acme/repo",
-      entries: [
-        {
-          id: "route-a-invalid-uuid",
-          title: "UUID path params reach Postgres and return 500 instead of 400",
-          classification: "product_bug",
-          state: "open",
-          issue: {
-            repo: "acme/repo",
-            number: 77,
-            url: "https://github.com/acme/repo/issues/77",
-          },
-          description: "Route A leaks Postgres UUID parse errors on malformed ids.",
-          whyFailing: "Route A is missing UUID param validation.",
-          lastReviewedAt: "2026-04-27",
-          matches: [
-            {
-              service: "api",
-              type: "int",
-              path: "src/api/routes/__testkit__/route-a.int.testkit.ts",
-            },
-          ],
-        },
-        {
-          id: "route-b-invalid-uuid",
-          title: "UUID path params reach Postgres and return 500 instead of 400",
-          classification: "product_bug",
-          state: "open",
-          issue: {
-            repo: "acme/repo",
-            number: 77,
-            url: "https://github.com/acme/repo/issues/77",
-          },
-          description: "Route B leaks Postgres UUID parse errors on malformed ids.",
-          whyFailing: "Route B is missing UUID param validation.",
-          lastReviewedAt: "2026-04-27",
-          matches: [
-            {
-              service: "api",
-              type: "int",
-              path: "src/api/routes/__testkit__/route-b.int.testkit.ts",
-            },
-          ],
-        },
-      ],
-    });
-
-    const result = await validateKnownFailureIssues({
-      productDir: tempDir,
-      document,
-      statusArtifact: {
-        tests: [
-          {
-            service: "api",
-            type: "int",
-            path: "src/api/routes/__testkit__/route-a.int.testkit.ts",
-            status: "failed",
-          },
-          {
-            service: "api",
-            type: "int",
-            path: "src/api/routes/__testkit__/route-b.int.testkit.ts",
-            status: "failed",
-          },
-        ],
-      },
-      config: {
-        provider: "github",
-        mode: "error",
-      },
-      transport: {
-        async fetchRepoIssues(repo, numbers) {
-          const map = new Map();
-          map.set(numbers[0], {
-            repo,
-            number: numbers[0],
-            exists: true,
-            title: "UUID path params reach Postgres and return 500 instead of 400",
-            state: "OPEN",
-            url: `https://github.com/${repo}/issues/${numbers[0]}`,
-            checkedAt: "2026-04-27T00:00:00.000Z",
-            source: "github",
-          });
-          return map;
-        },
-      },
-    });
-
-    expect(result.summary.byCode.title_mismatch).toBeUndefined();
-    expect(result.summary.errors).toBe(0);
-    expect(result.entries).toHaveLength(2);
-    expect(result.entries[0].findings).toEqual([]);
-    expect(result.entries[1].findings).toEqual([]);
-    expect(result.entries[0].status).toBe("open_and_failing");
-    expect(result.entries[1].status).toBe("open_and_failing");
-  });
-
-  it("warns when an open issue is not reproduced", async () => {
-    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-open-"));
-    tempDirs.push(tempDir);
-    const document = normalizeKnownFailuresDocument({
-      schemaVersion: 1,
-      issueRepo: "acme/repo",
-      entries: [
-        {
-          id: "bad-message",
-          title: "Bad message bug",
-          classification: "product_bug",
-          state: "open",
-          issue: {
-            repo: "acme/repo",
-            number: 12,
-            url: "https://github.com/acme/repo/issues/12",
-          },
-          description: "Wrong message",
-          whyFailing: "Payload is wrong",
-          lastReviewedAt: "2026-04-27",
-          matches: [
-            {
-              service: "api",
-              type: "int",
-              path: "src/api/routes/__testkit__/failing.int.testkit.ts",
-            },
-          ],
-        },
-      ],
-    });
-
-    const result = await validateKnownFailureIssues({
-      productDir: tempDir,
-      document,
-      statusArtifact: {
-        tests: [
-          {
-            service: "api",
-            type: "int",
-            path: "src/api/routes/__testkit__/failing.int.testkit.ts",
-            status: "passed",
-          },
-        ],
-      },
-      config: {
-        provider: "github",
-        mode: "warn",
-      },
-      gitMetadata: {
-        repoSlug: "acme/repo",
-      },
-      transport: {
-        async fetchRepoIssues(repo, numbers) {
-          const map = new Map();
-          map.set(numbers[0], {
-            repo,
-            number: numbers[0],
-            exists: true,
-            title: "Bad message bug",
-            state: "OPEN",
-            url: `https://github.com/${repo}/issues/${numbers[0]}`,
-            checkedAt: "2026-04-27T00:00:00.000Z",
-            source: "github",
-          });
-          return map;
-        },
-      },
-    });
-
-    expect(result.summary.byCode.open_not_reproduced).toBe(1);
-    expect(result.summary.errors).toBe(0);
-    expect(result.summary.warnings).toBeGreaterThan(0);
-    expect(shouldFailKnownFailureIssueValidation(result)).toBe(false);
-    expect(result.entries[0].observed).toMatchObject({
-      matchedTests: 1,
-      executedTests: 1,
-      passedTests: 1,
-      failedTests: 0,
-      skippedTests: 0,
-      notRunTests: 0,
-      reproduced: false,
-    });
-  });
-
-  it("does not treat skipped or not_run files as not reproduced", async () => {
-    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-not-executed-"));
-    tempDirs.push(tempDir);
-    const document = normalizeKnownFailuresDocument({
-      schemaVersion: 1,
-      issueRepo: "acme/repo",
-      entries: [
-        {
-          id: "bad-message",
-          title: "Bad message bug",
-          classification: "product_bug",
-          state: "open",
-          issue: {
-            repo: "acme/repo",
-            number: 12,
-            url: "https://github.com/acme/repo/issues/12",
-          },
-          description: "Wrong message",
-          whyFailing: "Payload is wrong",
-          lastReviewedAt: "2026-04-27",
-          matches: [
-            {
-              service: "api",
-              type: "int",
-              path: "src/api/routes/__testkit__/failing.int.testkit.ts",
-            },
-            {
-              service: "api",
-              type: "int",
-              path: "src/api/routes/__testkit__/blocked.int.testkit.ts",
-            },
-          ],
-        },
-      ],
-    });
-
-    const result = await validateKnownFailureIssues({
-      productDir: tempDir,
-      document,
-      statusArtifact: {
-        tests: [
-          {
-            service: "api",
-            type: "int",
-            path: "src/api/routes/__testkit__/failing.int.testkit.ts",
-            status: "skipped",
-          },
-          {
-            service: "api",
-            type: "int",
-            path: "src/api/routes/__testkit__/blocked.int.testkit.ts",
-            status: "not_run",
-          },
-        ],
-      },
-      config: {
-        provider: "github",
-        mode: "warn",
-      },
-      transport: {
-        async fetchRepoIssues(repo, numbers) {
-          const map = new Map();
-          map.set(numbers[0], {
-            repo,
-            number: numbers[0],
-            exists: true,
-            title: "Bad message bug",
-            state: "OPEN",
-            url: `https://github.com/${repo}/issues/${numbers[0]}`,
-            checkedAt: "2026-04-27T00:00:00.000Z",
-            source: "github",
-          });
-          return map;
-        },
-      },
-    });
-
-    expect(result.summary.byCode.open_not_reproduced).toBeUndefined();
-    expect(result.entries[0].findings).toEqual([]);
-    expect(result.entries[0].status).toBe("not_executed");
-    expect(result.entries[0].observed).toMatchObject({
-      matchedTests: 2,
-      executedTests: 0,
-      passedTests: 0,
-      failedTests: 0,
-      skippedTests: 1,
-      notRunTests: 1,
-      reproduced: false,
-    });
-  });
-
-  it("still treats mixed passed and skipped execution as not reproduced", async () => {
-    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-mixed-execution-"));
-    tempDirs.push(tempDir);
-    const document = normalizeKnownFailuresDocument({
-      schemaVersion: 1,
-      issueRepo: "acme/repo",
-      entries: [
-        {
-          id: "bad-message",
-          title: "Bad message bug",
-          classification: "product_bug",
-          state: "open",
-          issue: {
-            repo: "acme/repo",
-            number: 12,
-            url: "https://github.com/acme/repo/issues/12",
-          },
-          description: "Wrong message",
-          whyFailing: "Payload is wrong",
-          lastReviewedAt: "2026-04-27",
-          matches: [
-            {
-              service: "api",
-              type: "int",
-              path: "src/api/routes/__testkit__/passed.int.testkit.ts",
-            },
-            {
-              service: "api",
-              type: "int",
-              path: "src/api/routes/__testkit__/skipped.int.testkit.ts",
-            },
-          ],
-        },
-      ],
-    });
-
-    const result = await validateKnownFailureIssues({
-      productDir: tempDir,
-      document,
-      statusArtifact: {
-        tests: [
-          {
-            service: "api",
-            type: "int",
-            path: "src/api/routes/__testkit__/passed.int.testkit.ts",
-            status: "passed",
-          },
-          {
-            service: "api",
-            type: "int",
-            path: "src/api/routes/__testkit__/skipped.int.testkit.ts",
-            status: "skipped",
-          },
-        ],
-      },
-      config: {
-        provider: "github",
-        mode: "warn",
-      },
-      transport: {
-        async fetchRepoIssues(repo, numbers) {
-          const map = new Map();
-          map.set(numbers[0], {
-            repo,
-            number: numbers[0],
-            exists: true,
-            title: "Bad message bug",
-            state: "OPEN",
-            url: `https://github.com/${repo}/issues/${numbers[0]}`,
-            checkedAt: "2026-04-27T00:00:00.000Z",
-            source: "github",
-          });
-          return map;
-        },
-      },
-    });
-
-    expect(result.summary.byCode.open_not_reproduced).toBe(1);
-    expect(result.entries[0].status).toBe("open_not_reproduced");
-    expect(result.entries[0].observed).toMatchObject({
-      matchedTests: 2,
-      executedTests: 1,
-      passedTests: 1,
-      failedTests: 0,
-      skippedTests: 1,
-      notRunTests: 0,
-      reproduced: false,
-    });
-  });
-
-  it("falls back to warning when validation is unavailable", async () => {
-    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-unavailable-"));
-    tempDirs.push(tempDir);
-    const document = normalizeKnownFailuresDocument({
-      schemaVersion: 1,
-      entries: [
-        {
-          id: "bad-message",
-          title: "Bad message bug",
-          classification: "product_bug",
-          state: "open",
-          issue: {
-            repo: "acme/repo",
-            number: 12,
-            url: "https://github.com/acme/repo/issues/12",
-          },
-          description: "Wrong message",
-          whyFailing: "Payload is wrong",
-          lastReviewedAt: "2026-04-27",
-          matches: [{ path: "src/api/routes/__testkit__/failing.int.testkit.ts" }],
-        },
-      ],
-    });
-
-    const result = await validateKnownFailureIssues({
-      productDir: tempDir,
-      document,
-      config: {
-        provider: "github",
-        mode: "warn",
-      },
-      transport: null,
-    });
-
-    expect(result.summary.byCode.validation_unavailable).toBe(1);
-    expect(result.summary.errors).toBe(0);
-    expect(result.summary.warnings).toBeGreaterThan(0);
-  });
-});

package/lib/known-failures/index.d.ts

@@ -1,192 +0,0 @@
-export type KnownFailureClassification =
-  | "expected_failure"
-  | "infra"
-  | "product_bug"
-  | "stale_test"
-  | "test_bug";
-
-export type KnownFailureState = "closed" | "open";
-
-export interface KnownFailureIssueRef {
-  repo: string;
-  number: number;
-  url: string;
-}
-
-export interface KnownFailureMatch {
-  service?: string;
-  type?: string;
-  path: string;
-  failureKey?: string;
-  errorIncludes?: string;
-}
-
-export interface KnownFailureEntry {
-  id: string;
-  /** Exact issue-tracker title for the linked issue. */
-  title: string;
-  classification: KnownFailureClassification;
-  state: KnownFailureState;
-  issue: KnownFailureIssueRef;
-  /** Product-local bug slice or route-family summary. */
-  description: string;
-  /** Underlying technical cause of the failure. */
-  whyFailing: string;
-  lastReviewedAt: string;
-  matches: KnownFailureMatch[];
-}
-
-export interface KnownFailuresDocument {
-  schemaVersion: 1;
-  issueRepo?: string | null;
-  entries: KnownFailureEntry[];
-}
-
-export interface KnownFailuresValidationResult {
-  errors: string[];
-  warnings: string[];
-  stats: {
-    entries: number;
-    matches: number;
-    failedTests: number;
-    triagedFailedTests: number;
-    untriagedFailedTests: number;
-  };
-}
-
-export interface KnownFailureIssueValidationConfig {
-  provider?: "github";
-  mode?: "off" | "warn" | "error";
-  cacheTtlSeconds?: number;
-}
-
-export interface KnownFailureIssueValidationFinding {
-  code: string;
-  severity: "warning" | "error";
-  message: string;
-}
-
-export interface KnownFailureIssueValidationEntry {
-  id: string;
-  title: string;
-  issue: KnownFailureIssueRef;
-  observed: {
-    matchedTests: number;
-    executedTests: number;
-    passedTests: number;
-    failedTests: number;
-    skippedTests: number;
-    notRunTests: number;
-    reproduced: boolean;
-  };
-  github: {
-    exists: boolean | null;
-    title: string | null;
-    state: string | null;
-    url: string | null;
-    checkedAt: string | null;
-    cached: boolean;
-  };
-  findings: KnownFailureIssueValidationFinding[];
-  status: string;
-}
-
-export interface KnownFailureIssueValidationResult {
-  schemaVersion: 2;
-  provider: "github";
-  mode: "off" | "warn" | "error";
-  checkedAt: string;
-  repo: {
-    detected: string | null;
-    remoteUrl: string | null;
-    configured: string | null;
-  };
-  availability: {
-    hasFreshData: boolean;
-    usedCachedFallback: boolean;
-  };
-  findings: KnownFailureIssueValidationFinding[];
-  summary: {
-    entries: number;
-    observedEntries: number;
-    errors: number;
-    warnings: number;
-    byCode: Record<string, number>;
-    byStatus: Record<string, number>;
-  };
-  entries: KnownFailureIssueValidationEntry[];
-}
-
-export declare function loadKnownFailuresConfig(
-  productDir: string,
-  config: { knownFailuresFile?: string | null } | null
-): KnownFailuresDocument | null;
-export declare function loadKnownFailuresDocument(
-  filePath: string,
-  relativePath?: string
-): KnownFailuresDocument;
-export declare function normalizeKnownFailuresDocument(
-  document: unknown,
-  relativePath?: string
-): KnownFailuresDocument;
-export declare function validateKnownFailuresDocument(
-  document: KnownFailuresDocument,
-  options?: {
-    productDir?: string;
-    statusArtifactPath?: string;
-    statusArtifact?: unknown;
-  }
-): KnownFailuresValidationResult;
-export declare function renderKnownFailuresMarkdown(document: KnownFailuresDocument): string;
-export declare function findMatchingKnownFailureEntries(
-  document: KnownFailuresDocument,
-  fileSummary: {
-    service?: string;
-    type?: string;
-    path: string;
-    status?: string;
-    error?: string | null;
-    failureDetails?: Array<{ key?: string }>;
-  }
-): KnownFailureEntry[];
-export declare function matchesKnownFailureEntry(
-  entry: KnownFailureEntry,
-  fileSummary: {
-    service?: string;
-    type?: string;
-    path: string;
-    error?: string | null;
-    failureDetails?: Array<{ key?: string }>;
-  }
-): boolean;
-export declare function buildKnownFailureFileIdentity(
-  service: string,
-  type: string,
-  filePath: string
-): string;
-
-export declare function normalizeKnownFailureIssueValidationConfig(
-  value: unknown
-): {
-  provider: "github";
-  mode: "off" | "warn" | "error";
-  cacheTtlSeconds: number;
-} | null;
-export declare function validateKnownFailureIssues(options: {
-  productDir: string;
-  document: KnownFailuresDocument | null;
-  runArtifact?: unknown;
-  statusArtifact?: unknown;
-  config?: KnownFailureIssueValidationConfig | null;
-  gitMetadata?: {
-    repoSlug?: string | null;
-    remoteUrl?: string | null;
-  } | null;
-}): Promise<KnownFailureIssueValidationResult | null>;
-export declare function shouldFailKnownFailureIssueValidation(
-  result: KnownFailureIssueValidationResult | null
-): boolean;
-export declare function buildKnownFailureIssueValidationSummaryLines(
-  result: KnownFailureIssueValidationResult | null
-): string[];
-export declare function parseGitHubRepoSlug(remoteUrl: string | null | undefined): string | null;