@elench/testkit 0.1.80 → 0.1.81

package/lib/cli/commands/known-failures/validate.mjs

@@ -1,20 +0,0 @@
-import { Command } from "@oclif/core";
-import { makeKnownFailuresFlags } from "../../command-helpers.mjs";
-import { runKnownFailuresValidateCommand } from "../../known-failures.mjs";
-
-export default class KnownFailuresValidateCommand extends Command {
-  static summary = "Validate known failures against the latest status artifact";
-
-  static enableJsonFlag = true;
-
-  static flags = makeKnownFailuresFlags();
-
-  async run() {
-    const { flags } = await this.parse(KnownFailuresValidateCommand);
-    await runKnownFailuresValidateCommand({
-      ...flags,
-      issueMode: flags["issue-mode"] || null,
-    });
-    return { ok: true };
-  }
-}

package/lib/cli/known-failures.mjs

@@ -1,164 +0,0 @@
-import fs from "fs";
-import path from "path";
-import { resolveProductDir } from "../config/index.mjs";
-import { loadTestkitConfig } from "../config/config-loader.mjs";
-import {
-  buildKnownFailureIssueValidationSummaryLines,
-  normalizeKnownFailureIssueValidationConfig,
-  shouldFailKnownFailureIssueValidation,
-  validateKnownFailureIssues,
-} from "../known-failures/github.mjs";
-import {
-  loadKnownFailuresDocument,
-  renderKnownFailuresMarkdown,
-  validateKnownFailuresDocument,
-} from "../known-failures/index.mjs";
-import { collectGitMetadata } from "../runner/metadata.mjs";
-
-const DEFAULT_ISSUE_VALIDATION = {
-  provider: "github",
-  mode: "error",
-  cacheTtlSeconds: 900,
-};
-
-export async function runKnownFailuresRenderCommand(options = {}) {
-  const context = await resolveKnownFailuresContext(options);
-  const document = loadKnownFailuresDocument(context.knownFailuresPath, context.knownFailuresRelativePath);
-  const markdown = renderKnownFailuresMarkdown(document);
-
-  if (options.output) {
-    const outputPath = path.resolve(context.productDir, options.output);
-    fs.mkdirSync(path.dirname(outputPath), { recursive: true });
-    fs.writeFileSync(outputPath, markdown);
-    console.log(`Wrote ${path.relative(context.productDir, outputPath)}`);
-    return;
-  }
-
-  process.stdout.write(markdown);
-}
-
-export async function runKnownFailuresValidateCommand(options = {}) {
-  const context = await resolveKnownFailuresContext(options);
-  const document = loadKnownFailuresDocument(context.knownFailuresPath, context.knownFailuresRelativePath);
-  const statusArtifact = context.statusArtifactPath
-    ? readJsonIfExists(context.statusArtifactPath)
-    : undefined;
-
-  const documentValidation = validateKnownFailuresDocument(document, {
-    productDir: context.productDir,
-    statusArtifact,
-  });
-
-  const issueValidation = await validateKnownFailureIssues({
-    productDir: context.productDir,
-    document,
-    statusArtifact,
-    config: context.issueValidationConfig,
-    gitMetadata: collectGitMetadata(context.productDir),
-  });
-
-  const issueErrors = collectIssueMessages(issueValidation, "error");
-  const issueWarnings = collectIssueMessages(issueValidation, "warning");
-  const hasErrors =
-    documentValidation.errors.length > 0 ||
-    shouldFailKnownFailureIssueValidation(issueValidation) ||
-    issueErrors.length > 0;
-
-  if (hasErrors) {
-    console.error("Known failures validation failed:");
-    for (const error of documentValidation.errors) {
-      console.error(`- ${error}`);
-    }
-    for (const error of issueErrors) {
-      console.error(`- ${error}`);
-    }
-    if (documentValidation.warnings.length > 0 || issueWarnings.length > 0) {
-      console.error("");
-      for (const warning of documentValidation.warnings) {
-        console.error(`! ${warning}`);
-      }
-      for (const warning of issueWarnings) {
-        console.error(`! ${warning}`);
-      }
-    }
-    for (const line of buildKnownFailureIssueValidationSummaryLines(issueValidation)) {
-      console.error(line);
-    }
-    process.exitCode = 1;
-    return;
-  }
-
-  console.log(
-    `Known failures valid: ${documentValidation.stats.entries} entries, ${documentValidation.stats.matches} matches`
-  );
-  if (documentValidation.stats.failedTests > 0) {
-    console.log(
-      `Status coverage: ${documentValidation.stats.triagedFailedTests}/${documentValidation.stats.failedTests} failed tests triaged`
-    );
-  }
-  for (const line of buildKnownFailureIssueValidationSummaryLines(issueValidation)) {
-    console.log(line);
-  }
-  for (const warning of [...documentValidation.warnings, ...issueWarnings]) {
-    console.warn(`! ${warning}`);
-  }
-}
-
-async function resolveKnownFailuresContext(options) {
-  const productDir = resolveProductDir(process.cwd(), options.dir);
-  const { config } = await loadTestkitConfig(productDir);
-  const reporting = config?.reporting || {};
-
-  const knownFailuresRelativePath = normalizeOptionalString(options.input)
-    || normalizeOptionalString(reporting.knownFailuresFile);
-  if (!knownFailuresRelativePath) {
-    throw new Error(
-      "Known failures file not configured. Set reporting.knownFailuresFile in testkit.config.ts or pass --input."
-    );
-  }
-
-  const statusRelativePath = normalizeOptionalString(options.status) || "testkit.status.json";
-  const configuredIssueValidation = normalizeKnownFailureIssueValidationConfig(reporting.issueValidation)
-    || DEFAULT_ISSUE_VALIDATION;
-  const issueMode = normalizeOptionalString(options.issueMode) || "error";
-
-  return {
-    productDir,
-    knownFailuresRelativePath,
-    knownFailuresPath: path.resolve(productDir, knownFailuresRelativePath),
-    statusArtifactPath: statusRelativePath ? path.resolve(productDir, statusRelativePath) : null,
-    issueValidationConfig: {
-      ...configuredIssueValidation,
-      mode: issueMode,
-    },
-  };
-}
-
-function collectIssueMessages(validation, severity) {
-  if (!validation) return [];
-
-  const messages = [
-    ...validation.findings
-      .filter((finding) => finding.severity === severity)
-      .map((finding) => finding.message),
-  ];
-  for (const entry of validation.entries) {
-    for (const finding of entry.findings) {
-      if (finding.severity === severity) {
-        messages.push(finding.message);
-      }
-    }
-  }
-  return [...new Set(messages)];
-}
-
-function normalizeOptionalString(value) {
-  if (typeof value !== "string") return null;
-  const normalized = value.trim();
-  return normalized.length > 0 ? normalized : null;
-}
-
-function readJsonIfExists(filePath) {
-  if (!fs.existsSync(filePath)) return undefined;
-  return JSON.parse(fs.readFileSync(filePath, "utf8"));
-}

package/lib/known-failures/github.test.mjs

@@ -1,512 +0,0 @@
-import fs from "fs";
-import os from "os";
-import path from "path";
-import { afterEach, describe, expect, it } from "vitest";
-import { normalizeKnownFailuresDocument } from "./index.mjs";
-import {
-  parseGitHubRepoSlug,
-  shouldFailKnownFailureIssueValidation,
-  validateKnownFailureIssues,
-} from "./github.mjs";
-
-const tempDirs = [];
-
-afterEach(() => {
-  for (const tempDir of tempDirs.splice(0)) {
-    fs.rmSync(tempDir, { recursive: true, force: true });
-  }
-});
-
-describe("known failures GitHub validation", () => {
-  it("parses GitHub repo slugs from common origin URLs", () => {
-    expect(parseGitHubRepoSlug("https://github.com/acme/repo.git")).toBe("acme/repo");
-    expect(parseGitHubRepoSlug("git@github.com:acme/repo.git")).toBe("acme/repo");
-    expect(parseGitHubRepoSlug("ssh://git@github.com/acme/repo.git")).toBe("acme/repo");
-    expect(parseGitHubRepoSlug("https://gitlab.com/acme/repo.git")).toBe(null);
-  });
-
-  it("fails validation for title/state drift and closed issues that still reproduce", async () => {
-    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-issues-"));
-    tempDirs.push(tempDir);
-    const document = normalizeKnownFailuresDocument({
-      schemaVersion: 1,
-      issueRepo: "acme/repo",
-      entries: [
-        {
-          id: "bad-message",
-          title: "Bad message bug",
-          classification: "product_bug",
-          state: "open",
-          issue: {
-            repo: "acme/repo",
-            number: 12,
-            url: "https://github.com/acme/repo/issues/12",
-          },
-          description: "Wrong message",
-          whyFailing: "Payload is wrong",
-          lastReviewedAt: "2026-04-27",
-          matches: [
-            {
-              service: "api",
-              type: "int",
-              path: "src/api/routes/__testkit__/failing.int.testkit.ts",
-            },
-          ],
-        },
-      ],
-    });
-
-    const result = await validateKnownFailureIssues({
-      productDir: tempDir,
-      document,
-      statusArtifact: {
-        tests: [
-          {
-            service: "api",
-            type: "int",
-            path: "src/api/routes/__testkit__/failing.int.testkit.ts",
-            status: "failed",
-          },
-        ],
-      },
-      config: {
-        provider: "github",
-        mode: "error",
-        cacheTtlSeconds: 60,
-      },
-      gitMetadata: {
-        repoSlug: "acme/repo",
-        remoteUrl: "https://github.com/acme/repo.git",
-      },
-      transport: {
-        async fetchRepoIssues(repo, numbers) {
-          const map = new Map();
-          map.set(numbers[0], {
-            repo,
-            number: numbers[0],
-            exists: true,
-            title: "Different title",
-            state: "CLOSED",
-            url: `https://github.com/${repo}/issues/${numbers[0]}`,
-            checkedAt: "2026-04-27T00:00:00.000Z",
-            source: "github",
-          });
-          return map;
-        },
-      },
-    });
-
-    expect(result.summary.byCode.closed_but_failing).toBe(1);
-    expect(result.summary.byCode.title_mismatch).toBe(1);
-    expect(result.summary.byCode.state_mismatch).toBe(1);
-    expect(result.summary.errors).toBeGreaterThan(0);
-    expect(result.entries[0].status).toBe("closed_but_failing");
-    expect(shouldFailKnownFailureIssueValidation(result)).toBe(true);
-  });
-
-  it("allows multiple local entries to share one issue title while keeping distinct descriptions", async () => {
-    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-shared-title-"));
-    tempDirs.push(tempDir);
-    const document = normalizeKnownFailuresDocument({
-      schemaVersion: 1,
-      issueRepo: "acme/repo",
-      entries: [
-        {
-          id: "route-a-invalid-uuid",
-          title: "UUID path params reach Postgres and return 500 instead of 400",
-          classification: "product_bug",
-          state: "open",
-          issue: {
-            repo: "acme/repo",
-            number: 77,
-            url: "https://github.com/acme/repo/issues/77",
-          },
-          description: "Route A leaks Postgres UUID parse errors on malformed ids.",
-          whyFailing: "Route A is missing UUID param validation.",
-          lastReviewedAt: "2026-04-27",
-          matches: [
-            {
-              service: "api",
-              type: "int",
-              path: "src/api/routes/__testkit__/route-a.int.testkit.ts",
-            },
-          ],
-        },
-        {
-          id: "route-b-invalid-uuid",
-          title: "UUID path params reach Postgres and return 500 instead of 400",
-          classification: "product_bug",
-          state: "open",
-          issue: {
-            repo: "acme/repo",
-            number: 77,
-            url: "https://github.com/acme/repo/issues/77",
-          },
-          description: "Route B leaks Postgres UUID parse errors on malformed ids.",
-          whyFailing: "Route B is missing UUID param validation.",
-          lastReviewedAt: "2026-04-27",
-          matches: [
-            {
-              service: "api",
-              type: "int",
-              path: "src/api/routes/__testkit__/route-b.int.testkit.ts",
-            },
-          ],
-        },
-      ],
-    });
-
-    const result = await validateKnownFailureIssues({
-      productDir: tempDir,
-      document,
-      statusArtifact: {
-        tests: [
-          {
-            service: "api",
-            type: "int",
-            path: "src/api/routes/__testkit__/route-a.int.testkit.ts",
-            status: "failed",
-          },
-          {
-            service: "api",
-            type: "int",
-            path: "src/api/routes/__testkit__/route-b.int.testkit.ts",
-            status: "failed",
-          },
-        ],
-      },
-      config: {
-        provider: "github",
-        mode: "error",
-      },
-      transport: {
-        async fetchRepoIssues(repo, numbers) {
-          const map = new Map();
-          map.set(numbers[0], {
-            repo,
-            number: numbers[0],
-            exists: true,
-            title: "UUID path params reach Postgres and return 500 instead of 400",
-            state: "OPEN",
-            url: `https://github.com/${repo}/issues/${numbers[0]}`,
-            checkedAt: "2026-04-27T00:00:00.000Z",
-            source: "github",
-          });
-          return map;
-        },
-      },
-    });
-
-    expect(result.summary.byCode.title_mismatch).toBeUndefined();
-    expect(result.summary.errors).toBe(0);
-    expect(result.entries).toHaveLength(2);
-    expect(result.entries[0].findings).toEqual([]);
-    expect(result.entries[1].findings).toEqual([]);
-    expect(result.entries[0].status).toBe("open_and_failing");
-    expect(result.entries[1].status).toBe("open_and_failing");
-  });
-
-  it("warns when an open issue is not reproduced", async () => {
-    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-open-"));
-    tempDirs.push(tempDir);
-    const document = normalizeKnownFailuresDocument({
-      schemaVersion: 1,
-      issueRepo: "acme/repo",
-      entries: [
-        {
-          id: "bad-message",
-          title: "Bad message bug",
-          classification: "product_bug",
-          state: "open",
-          issue: {
-            repo: "acme/repo",
-            number: 12,
-            url: "https://github.com/acme/repo/issues/12",
-          },
-          description: "Wrong message",
-          whyFailing: "Payload is wrong",
-          lastReviewedAt: "2026-04-27",
-          matches: [
-            {
-              service: "api",
-              type: "int",
-              path: "src/api/routes/__testkit__/failing.int.testkit.ts",
-            },
-          ],
-        },
-      ],
-    });
-
-    const result = await validateKnownFailureIssues({
-      productDir: tempDir,
-      document,
-      statusArtifact: {
-        tests: [
-          {
-            service: "api",
-            type: "int",
-            path: "src/api/routes/__testkit__/failing.int.testkit.ts",
-            status: "passed",
-          },
-        ],
-      },
-      config: {
-        provider: "github",
-        mode: "warn",
-      },
-      gitMetadata: {
-        repoSlug: "acme/repo",
-      },
-      transport: {
-        async fetchRepoIssues(repo, numbers) {
-          const map = new Map();
-          map.set(numbers[0], {
-            repo,
-            number: numbers[0],
-            exists: true,
-            title: "Bad message bug",
-            state: "OPEN",
-            url: `https://github.com/${repo}/issues/${numbers[0]}`,
-            checkedAt: "2026-04-27T00:00:00.000Z",
-            source: "github",
-          });
-          return map;
-        },
-      },
-    });
-
-    expect(result.summary.byCode.open_not_reproduced).toBe(1);
-    expect(result.summary.errors).toBe(0);
-    expect(result.summary.warnings).toBeGreaterThan(0);
-    expect(shouldFailKnownFailureIssueValidation(result)).toBe(false);
-    expect(result.entries[0].observed).toMatchObject({
-      matchedTests: 1,
-      executedTests: 1,
-      passedTests: 1,
-      failedTests: 0,
-      skippedTests: 0,
-      notRunTests: 0,
-      reproduced: false,
-    });
-  });
-
-  it("does not treat skipped or not_run files as not reproduced", async () => {
-    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-not-executed-"));
-    tempDirs.push(tempDir);
-    const document = normalizeKnownFailuresDocument({
-      schemaVersion: 1,
-      issueRepo: "acme/repo",
-      entries: [
-        {
-          id: "bad-message",
-          title: "Bad message bug",
-          classification: "product_bug",
-          state: "open",
-          issue: {
-            repo: "acme/repo",
-            number: 12,
-            url: "https://github.com/acme/repo/issues/12",
-          },
-          description: "Wrong message",
-          whyFailing: "Payload is wrong",
-          lastReviewedAt: "2026-04-27",
-          matches: [
-            {
-              service: "api",
-              type: "int",
-              path: "src/api/routes/__testkit__/failing.int.testkit.ts",
-            },
-            {
-              service: "api",
-              type: "int",
-              path: "src/api/routes/__testkit__/blocked.int.testkit.ts",
-            },
-          ],
-        },
-      ],
-    });
-
-    const result = await validateKnownFailureIssues({
-      productDir: tempDir,
-      document,
-      statusArtifact: {
-        tests: [
-          {
-            service: "api",
-            type: "int",
-            path: "src/api/routes/__testkit__/failing.int.testkit.ts",
-            status: "skipped",
-          },
-          {
-            service: "api",
-            type: "int",
-            path: "src/api/routes/__testkit__/blocked.int.testkit.ts",
-            status: "not_run",
-          },
-        ],
-      },
-      config: {
-        provider: "github",
-        mode: "warn",
-      },
-      transport: {
-        async fetchRepoIssues(repo, numbers) {
-          const map = new Map();
-          map.set(numbers[0], {
-            repo,
-            number: numbers[0],
-            exists: true,
-            title: "Bad message bug",
-            state: "OPEN",
-            url: `https://github.com/${repo}/issues/${numbers[0]}`,
-            checkedAt: "2026-04-27T00:00:00.000Z",
-            source: "github",
-          });
-          return map;
-        },
-      },
-    });
-
-    expect(result.summary.byCode.open_not_reproduced).toBeUndefined();
-    expect(result.entries[0].findings).toEqual([]);
-    expect(result.entries[0].status).toBe("not_executed");
-    expect(result.entries[0].observed).toMatchObject({
-      matchedTests: 2,
-      executedTests: 0,
-      passedTests: 0,
-      failedTests: 0,
-      skippedTests: 1,
-      notRunTests: 1,
-      reproduced: false,
-    });
-  });
-
-  it("still treats mixed passed and skipped execution as not reproduced", async () => {
-    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-mixed-execution-"));
-    tempDirs.push(tempDir);
-    const document = normalizeKnownFailuresDocument({
-      schemaVersion: 1,
-      issueRepo: "acme/repo",
-      entries: [
-        {
-          id: "bad-message",
-          title: "Bad message bug",
-          classification: "product_bug",
-          state: "open",
-          issue: {
-            repo: "acme/repo",
-            number: 12,
-            url: "https://github.com/acme/repo/issues/12",
-          },
-          description: "Wrong message",
-          whyFailing: "Payload is wrong",
-          lastReviewedAt: "2026-04-27",
-          matches: [
-            {
-              service: "api",
-              type: "int",
-              path: "src/api/routes/__testkit__/passed.int.testkit.ts",
-            },
-            {
-              service: "api",
-              type: "int",
-              path: "src/api/routes/__testkit__/skipped.int.testkit.ts",
-            },
-          ],
-        },
-      ],
-    });
-
-    const result = await validateKnownFailureIssues({
-      productDir: tempDir,
-      document,
-      statusArtifact: {
-        tests: [
-          {
-            service: "api",
-            type: "int",
-            path: "src/api/routes/__testkit__/passed.int.testkit.ts",
-            status: "passed",
-          },
-          {
-            service: "api",
-            type: "int",
-            path: "src/api/routes/__testkit__/skipped.int.testkit.ts",
-            status: "skipped",
-          },
-        ],
-      },
-      config: {
-        provider: "github",
-        mode: "warn",
-      },
-      transport: {
-        async fetchRepoIssues(repo, numbers) {
-          const map = new Map();
-          map.set(numbers[0], {
-            repo,
-            number: numbers[0],
-            exists: true,
-            title: "Bad message bug",
-            state: "OPEN",
-            url: `https://github.com/${repo}/issues/${numbers[0]}`,
-            checkedAt: "2026-04-27T00:00:00.000Z",
-            source: "github",
-          });
-          return map;
-        },
-      },
-    });
-
-    expect(result.summary.byCode.open_not_reproduced).toBe(1);
-    expect(result.entries[0].status).toBe("open_not_reproduced");
-    expect(result.entries[0].observed).toMatchObject({
-      matchedTests: 2,
-      executedTests: 1,
-      passedTests: 1,
-      failedTests: 0,
-      skippedTests: 1,
-      notRunTests: 0,
-      reproduced: false,
-    });
-  });
-
-  it("falls back to warning when validation is unavailable", async () => {
-    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-unavailable-"));
-    tempDirs.push(tempDir);
-    const document = normalizeKnownFailuresDocument({
-      schemaVersion: 1,
-      entries: [
-        {
-          id: "bad-message",
-          title: "Bad message bug",
-          classification: "product_bug",
-          state: "open",
-          issue: {
-            repo: "acme/repo",
-            number: 12,
-            url: "https://github.com/acme/repo/issues/12",
-          },
-          description: "Wrong message",
-          whyFailing: "Payload is wrong",
-          lastReviewedAt: "2026-04-27",
-          matches: [{ path: "src/api/routes/__testkit__/failing.int.testkit.ts" }],
-        },
-      ],
-    });
-
-    const result = await validateKnownFailureIssues({
-      productDir: tempDir,
-      document,
-      config: {
-        provider: "github",
-        mode: "warn",
-      },
-      transport: null,
-    });
-
-    expect(result.summary.byCode.validation_unavailable).toBe(1);
-    expect(result.summary.errors).toBe(0);
-    expect(result.summary.warnings).toBeGreaterThan(0);
-  });
-});