@elench/testkit 0.1.43 → 0.1.45

package/lib/known-failures/index.mjs

@@ -0,0 +1,329 @@
+import fs from "fs";
+import path from "path";
+
+const CLASSIFICATIONS = new Set([
+  "expected_failure",
+  "infra",
+  "product_bug",
+  "stale_test",
+  "test_bug",
+]);
+const STATES = new Set(["closed", "open"]);
+
+export function loadKnownFailuresConfig(productDir, config) {
+  const relativePath = config?.knownFailuresFile;
+  if (!relativePath) return null;
+
+  const absolutePath = path.resolve(productDir, relativePath);
+  if (!fs.existsSync(absolutePath)) {
+    throw new Error(`Known failures file not found: ${relativePath}`);
+  }
+
+  return loadKnownFailuresDocument(absolutePath, relativePath);
+}
+
+export function loadKnownFailuresDocument(filePath, relativePath = filePath) {
+  let parsed;
+  try {
+    parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
+  } catch (error) {
+    throw new Error(
+      `Could not parse known failures file ${relativePath}: ${formatErrorMessage(error)}`
+    );
+  }
+
+  return normalizeKnownFailuresDocument(parsed, relativePath);
+}
+
+export function normalizeKnownFailuresDocument(document, relativePath = "known failures file") {
+  if (!document || typeof document !== "object") {
+    throw new Error(`${relativePath} must contain a JSON object`);
+  }
+  if (document.schemaVersion !== 1) {
+    throw new Error(`${relativePath} schemaVersion must be 1`);
+  }
+  if (!Array.isArray(document.entries)) {
+    throw new Error(`${relativePath} entries must be an array`);
+  }
+
+  const ids = new Set();
+  const entries = document.entries.map((entry, index) => {
+    const normalized = normalizeKnownFailureEntry(entry, `${relativePath} entries[${index}]`);
+    if (ids.has(normalized.id)) {
+      throw new Error(`${relativePath} has duplicate entry id "${normalized.id}"`);
+    }
+    ids.add(normalized.id);
+    return normalized;
+  });
+
+  return {
+    schemaVersion: 1,
+    issueRepo: normalizeOptionalString(document.issueRepo),
+    entries,
+  };
+}
+
+export function validateKnownFailuresDocument(
+  document,
+  { productDir, statusArtifactPath, statusArtifact } = {}
+) {
+  const errors = [];
+  const warnings = [];
+  const matchKeys = new Set();
+  let matchCount = 0;
+
+  for (const entry of document.entries) {
+    for (const match of entry.matches) {
+      matchCount += 1;
+      if (productDir) {
+        const absolutePath = path.join(productDir, match.path);
+        if (!fs.existsSync(absolutePath)) {
+          errors.push(`Missing matched file: ${match.path} (${entry.id})`);
+        }
+      }
+
+      const matchKey = [
+        entry.id,
+        match.service || "",
+        match.type || "",
+        match.path,
+        match.failureKey || "",
+        match.errorIncludes || "",
+      ].join("::");
+      if (matchKeys.has(matchKey)) {
+        errors.push(`Duplicate match selector in ${entry.id}: ${match.path}`);
+      }
+      matchKeys.add(matchKey);
+
+      if (!match.path.includes("__testkit__") || !match.path.endsWith(".testkit.ts")) {
+        warnings.push(`Match path does not look like a testkit file: ${match.path} (${entry.id})`);
+      }
+    }
+
+    if (document.issueRepo && entry.issue.repo !== document.issueRepo) {
+      warnings.push(
+        `Entry ${entry.id} uses issue repo ${entry.issue.repo} instead of document issueRepo ${document.issueRepo}`
+      );
+    }
+    if (!entry.issue.url.endsWith(`/issues/${entry.issue.number}`)) {
+      errors.push(`Issue URL does not match issue number for ${entry.id}`);
+    }
+  }
+
+  let failedTests = 0;
+  let triagedFailedTests = 0;
+  let untriagedFailedTests = 0;
+  const parsedStatusArtifact = statusArtifact
+    ? parseStatusArtifact(statusArtifact)
+    : statusArtifactPath && fs.existsSync(statusArtifactPath)
+      ? parseStatusArtifact(JSON.parse(fs.readFileSync(statusArtifactPath, "utf8")))
+      : { tests: [] };
+
+  const failed = parsedStatusArtifact.tests.filter((test) => test.status === "failed");
+  failedTests = failed.length;
+  for (const test of failed) {
+    if (findMatchingKnownFailureEntries(document, test).length > 0) {
+      triagedFailedTests += 1;
+    } else {
+      untriagedFailedTests += 1;
+      warnings.push(`Untriaged failed test: ${test.path}`);
+    }
+  }
+
+  return {
+    errors,
+    warnings,
+    stats: {
+      entries: document.entries.length,
+      matches: matchCount,
+      failedTests,
+      triagedFailedTests,
+      untriagedFailedTests,
+    },
+  };
+}
+
+export function renderKnownFailuresMarkdown(document) {
+  const lines = [
+    "# Bourne Bugs",
+    "",
+    "Canonical source: `testkit.known-failures.json`",
+    "",
+    `Tracked bug classes: ${document.entries.length}`,
+    `Tracked issue references: ${new Set(document.entries.map((entry) => entry.issue.number)).size}`,
+    "",
+  ];
+
+  document.entries.forEach((entry, index) => {
+    lines.push(
+      `## ${index + 1}. ${entry.title} — [#${entry.issue.number}](${entry.issue.url})`,
+      "",
+      `- Classification: \`${entry.classification}\``,
+      `- State: \`${entry.state}\``,
+      `- Description: ${entry.description}`,
+      `- Why failing: ${entry.whyFailing}`,
+      `- Last reviewed: ${entry.lastReviewedAt}`,
+      "- Matches:"
+    );
+
+    for (const match of entry.matches) {
+      lines.push(
+        `  - \`${match.path}\`${match.failureKey ? ` — \`${match.failureKey}\`` : ""}${
+          match.errorIncludes ? ` — error contains \`${match.errorIncludes}\`` : ""
+        }`
+      );
+    }
+
+    lines.push("");
+  });
+
+  return `${lines.join("\n").trimEnd()}\n`;
+}
+
+export function findMatchingKnownFailureEntries(document, fileSummary) {
+  return document.entries.filter((entry) => matchesKnownFailureEntry(entry, fileSummary));
+}
+
+export function matchesKnownFailureEntry(entry, fileSummary) {
+  return entry.matches.some((match) => matchesKnownFailureMatch(match, fileSummary));
+}
+
+export function matchesKnownFailureMatch(match, fileSummary) {
+  if (match.service && match.service !== fileSummary.service) return false;
+  if (match.type && match.type !== fileSummary.type) return false;
+  if (match.path !== fileSummary.path) return false;
+  if (match.failureKey) {
+    const failureKeys = Array.isArray(fileSummary.failureDetails)
+      ? fileSummary.failureDetails.map((detail) => detail?.key)
+      : [];
+    if (!failureKeys.includes(match.failureKey)) return false;
+  }
+  if (match.errorIncludes && !String(fileSummary.error || "").includes(match.errorIncludes)) {
+    return false;
+  }
+  return true;
+}
+
+export function buildKnownFailureFileIdentity(service, type, filePath) {
+  return `${service}::${type}::${filePath}`;
+}
+
+function normalizeKnownFailureEntry(entry, label) {
+  if (!entry || typeof entry !== "object") {
+    throw new Error(`${label} must be an object`);
+  }
+
+  const id = requireNonEmptyString(entry.id, `${label}.id`);
+  const title = requireNonEmptyString(entry.title, `${label}.title`);
+  const classification = requireEnumValue(
+    entry.classification,
+    CLASSIFICATIONS,
+    `${label}.classification`
+  );
+  const state = requireEnumValue(entry.state, STATES, `${label}.state`);
+  const description = requireNonEmptyString(entry.description, `${label}.description`);
+  const whyFailing = requireNonEmptyString(entry.whyFailing, `${label}.whyFailing`);
+  const lastReviewedAt = requireNonEmptyString(entry.lastReviewedAt, `${label}.lastReviewedAt`);
+  if (!Array.isArray(entry.matches) || entry.matches.length === 0) {
+    throw new Error(`${label}.matches must be a non-empty array`);
+  }
+
+  return {
+    id,
+    title,
+    classification,
+    state,
+    issue: normalizeKnownFailureIssue(entry.issue, `${label}.issue`),
+    description,
+    whyFailing,
+    lastReviewedAt,
+    matches: entry.matches.map((match, index) =>
+      normalizeKnownFailureMatch(match, `${label}.matches[${index}]`)
+    ),
+  };
+}
+
+function normalizeKnownFailureIssue(issue, label) {
+  if (!issue || typeof issue !== "object") {
+    throw new Error(`${label} must be an object`);
+  }
+
+  const repo = requireNonEmptyString(issue.repo, `${label}.repo`);
+  const number = issue.number;
+  if (!Number.isInteger(number) || number <= 0) {
+    throw new Error(`${label}.number must be a positive integer`);
+  }
+  const url = requireNonEmptyString(issue.url, `${label}.url`);
+
+  return { repo, number, url };
+}
+
+function normalizeKnownFailureMatch(match, label) {
+  if (!match || typeof match !== "object") {
+    throw new Error(`${label} must be an object`);
+  }
+
+  const pathValue = requireNonEmptyString(match.path, `${label}.path`);
+  const normalized = {
+    path: pathValue,
+  };
+
+  const service = normalizeOptionalString(match.service);
+  if (service) normalized.service = service;
+
+  const type = normalizeOptionalString(match.type);
+  if (type) normalized.type = type;
+
+  const failureKey = normalizeOptionalString(match.failureKey);
+  if (failureKey) normalized.failureKey = failureKey;
+
+  const errorIncludes = normalizeOptionalString(match.errorIncludes);
+  if (errorIncludes) normalized.errorIncludes = errorIncludes;
+
+  return normalized;
+}
+
+function parseStatusArtifact(value) {
+  if (!value || typeof value !== "object") {
+    return { tests: [] };
+  }
+
+  return {
+    tests: Array.isArray(value.tests) ? value.tests : [],
+  };
+}
+
+function requireNonEmptyString(value, label) {
+  const normalized = normalizeOptionalString(value);
+  if (!normalized) {
+    throw new Error(`${label} must be a non-empty string`);
+  }
+  return normalized;
+}
+
+function requireEnumValue(value, allowed, label) {
+  const normalized = requireNonEmptyString(value, label);
+  if (!allowed.has(normalized)) {
+    throw new Error(`${label} must be one of: ${[...allowed].sort().join(", ")}`);
+  }
+  return normalized;
+}
+
+function normalizeOptionalString(value) {
+  if (typeof value !== "string") return null;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : null;
+}
+
+function formatErrorMessage(error) {
+  if (error instanceof Error) return error.message;
+  return String(error);
+}
+
+export {
+  buildKnownFailureIssueValidationSummaryLines,
+  normalizeKnownFailureIssueValidationConfig,
+  parseGitHubRepoSlug,
+  shouldFailKnownFailureIssueValidation,
+  validateKnownFailureIssues,
+} from "./github.mjs";

package/lib/known-failures/index.test.mjs

@@ -0,0 +1,152 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { afterEach, describe, expect, it } from "vitest";
+import {
+  findMatchingKnownFailureEntries,
+  normalizeKnownFailuresDocument,
+  renderKnownFailuresMarkdown,
+  validateKnownFailuresDocument,
+} from "./index.mjs";
+
+const tempDirs = [];
+
+afterEach(() => {
+  for (const tempDir of tempDirs.splice(0)) {
+    fs.rmSync(tempDir, { recursive: true, force: true });
+  }
+});
+
+describe("known failures core", () => {
+  it("matches entries by service, type, path, and failure key", () => {
+    const document = normalizeKnownFailuresDocument({
+      schemaVersion: 1,
+      entries: [
+        {
+          id: "bad-message",
+          title: "Bad message bug",
+          classification: "product_bug",
+          state: "open",
+          issue: {
+            repo: "acme/repo",
+            number: 12,
+            url: "https://github.com/acme/repo/issues/12",
+          },
+          description: "Wrong message",
+          whyFailing: "Payload is wrong",
+          lastReviewedAt: "2026-04-27",
+          matches: [
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+              failureKey: "returns the wrong message",
+            },
+          ],
+        },
+      ],
+    });
+
+    const matches = findMatchingKnownFailureEntries(document, {
+      service: "api",
+      type: "int",
+      path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+      error: "boom",
+      failureDetails: [{ key: "returns the wrong message" }],
+    });
+
+    expect(matches).toHaveLength(1);
+    expect(matches[0].id).toBe("bad-message");
+  });
+
+  it("validates status coverage and filesystem matches", () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-known-failures-"));
+    tempDirs.push(tempDir);
+    const testPath = path.join(tempDir, "src/api/routes/__testkit__/failing.int.testkit.ts");
+    fs.mkdirSync(path.dirname(testPath), { recursive: true });
+    fs.writeFileSync(testPath, "export default {};\n");
+
+    const document = normalizeKnownFailuresDocument({
+      schemaVersion: 1,
+      issueRepo: "acme/repo",
+      entries: [
+        {
+          id: "bad-message",
+          title: "Bad message bug",
+          classification: "product_bug",
+          state: "open",
+          issue: {
+            repo: "acme/repo",
+            number: 12,
+            url: "https://github.com/acme/repo/issues/12",
+          },
+          description: "Wrong message",
+          whyFailing: "Payload is wrong",
+          lastReviewedAt: "2026-04-27",
+          matches: [
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            },
+          ],
+        },
+      ],
+    });
+
+    const result = validateKnownFailuresDocument(document, {
+      productDir: tempDir,
+      statusArtifact: {
+        tests: [
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            status: "failed",
+          },
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/other.int.testkit.ts",
+            status: "failed",
+          },
+        ],
+      },
+    });
+
+    expect(result.errors).toEqual([]);
+    expect(result.stats.failedTests).toBe(2);
+    expect(result.stats.triagedFailedTests).toBe(1);
+    expect(result.warnings).toContain(
+      "Untriaged failed test: src/api/routes/__testkit__/other.int.testkit.ts"
+    );
+  });
+
+  it("renders markdown from the canonical document", () => {
+    const markdown = renderKnownFailuresMarkdown(
+      normalizeKnownFailuresDocument({
+        schemaVersion: 1,
+        entries: [
+          {
+            id: "bad-message",
+            title: "Bad message bug",
+            classification: "product_bug",
+            state: "open",
+            issue: {
+              repo: "acme/repo",
+              number: 12,
+              url: "https://github.com/acme/repo/issues/12",
+            },
+            description: "Wrong message",
+            whyFailing: "Payload is wrong",
+            lastReviewedAt: "2026-04-27",
+            matches: [{ path: "src/api/routes/__testkit__/failing.int.testkit.ts" }],
+          },
+        ],
+      })
+    );
+
+    expect(markdown).toContain("# Bourne Bugs");
+    expect(markdown).toContain("[#12](https://github.com/acme/repo/issues/12)");
+  });
+});

package/lib/package.test.mjs

@@ -22,8 +22,13 @@ describe("package metadata", () => {
       types: "./lib/runtime/index.d.ts",
       default: "./lib/runtime/index.mjs",
     });
+    expect(packageJson.exports["./known-failures"]).toEqual({
+      types: "./lib/known-failures/index.d.ts",
+      default: "./lib/known-failures/index.mjs",
+    });
     expect(fs.existsSync(path.join(rootDir, "lib", "index.d.ts"))).toBe(true);
     expect(fs.existsSync(path.join(rootDir, "lib", "setup", "index.d.ts"))).toBe(true);
     expect(fs.existsSync(path.join(rootDir, "lib", "runtime", "index.d.ts"))).toBe(true);
+    expect(fs.existsSync(path.join(rootDir, "lib", "known-failures", "index.d.ts"))).toBe(true);
   });
 });

package/lib/runner/formatting.mjs

@@ -1,3 +1,5 @@
+import { buildKnownFailureIssueValidationSummaryLines } from "../known-failures/github.mjs";
+
 export function formatDuration(durationMs) {
   const totalSeconds = Math.max(0, Math.round(durationMs / 1000));
   const minutes = Math.floor(totalSeconds / 60);
@@ -46,7 +48,7 @@ export function formatSuiteFramework(framework) {
   return label ? ` [${label}]` : "";
 }
 
-export function buildRunSummaryLines(results, durationMs) {
+export function buildRunSummaryLines(results, durationMs, knownFailureIssueValidation = null) {
   const totalServices = results.length;
   const executedServices = results.filter((result) => !result.skipped);
   const skippedServices = results.filter((result) => result.skipped);
@@ -115,6 +117,13 @@ export function buildRunSummaryLines(results, durationMs) {
     }
   }
 
+  const knownFailureIssueLines = buildKnownFailureIssueValidationSummaryLines(
+    knownFailureIssueValidation
+  );
+  if (knownFailureIssueLines.length > 0) {
+    lines.push(...knownFailureIssueLines);
+  }
+
   if (failedServices.length > 0) {
     lines.push("", `Result: FAILED (${failedServices.length}/${totalServices} services failed)`);
     return lines;

package/lib/runner/formatting.test.mjs

@@ -128,4 +128,40 @@ describe("runner formatting", () => {
     expect(lines.join("\n")).toContain("SKIP api");
     expect(lines.at(-1)).toBe("Result: PASSED");
   });
+
+  it("appends known-failure issue validation summary lines", () => {
+    const lines = buildRunSummaryLines(
+      [
+        {
+          name: "api",
+          skipped: false,
+          failed: false,
+          suiteCount: 1,
+          completedSuiteCount: 1,
+          skippedSuiteCount: 0,
+          failedSuiteCount: 0,
+          totalFileCount: 1,
+          passedFileCount: 1,
+          skippedFileCount: 0,
+          notRunFileCount: 0,
+          durationMs: 1_000,
+          suites: [],
+          errors: [],
+        },
+      ],
+      1_000,
+      {
+        summary: {
+          byCode: {
+            closed_but_failing: 2,
+            title_mismatch: 1,
+          },
+        },
+      }
+    );
+
+    expect(lines.join("\n")).toContain("Known-failure issues:");
+    expect(lines.join("\n")).toContain("2 closed issues still failing");
+    expect(lines.join("\n")).toContain("1 title mismatch");
+  });
 });

package/lib/runner/metadata.mjs

@@ -3,6 +3,7 @@ import os from "os";
 import path from "path";
 import { execFileSync } from "child_process";
 import { fileURLToPath } from "url";
+import { parseGitHubRepoSlug } from "../known-failures/github.mjs";
 
 export function collectGitMetadata(productDir) {
   const read = (args) => {
@@ -13,10 +14,14 @@ export function collectGitMetadata(productDir) {
     }
   };
 
+  const remoteUrl = read(["remote", "get-url", "origin"]);
+
   return {
     branch: read(["rev-parse", "--abbrev-ref", "HEAD"]),
     commitSha: read(["rev-parse", "HEAD"]),
     repoRoot: read(["rev-parse", "--show-toplevel"]),
+    remoteUrl,
+    repoSlug: parseGitHubRepoSlug(remoteUrl),
   };
 }
 

package/lib/runner/orchestrator.mjs

@@ -17,6 +17,10 @@ import {
 import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
 import { applyKnownFailuresToArtifacts, loadKnownFailuresConfig } from "./triage.mjs";
 import { buildRunSummaryLines, formatError } from "./formatting.mjs";
+import {
+  shouldFailKnownFailureIssueValidation,
+  validateKnownFailureIssues,
+} from "../known-failures/github.mjs";
 import {
   loadTimings,
   resetResultArtifacts,
@@ -118,6 +122,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
   lifecycle.installSignalHandlers();
   let results = [];
   let finishedAt = Date.now();
+  let knownFailureIssueValidation = null;
 
   try {
     if (executedPlans.length > 0) {
@@ -207,15 +212,31 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
       statusArtifact,
       knownFailures
     );
+    knownFailureIssueValidation = await validateKnownFailureIssues({
+      productDir,
+      document: knownFailures,
+      runArtifact: enrichedArtifacts.runArtifact,
+      statusArtifact: enrichedArtifacts.statusArtifact,
+      config: configs[0]?.testkit?.reporting?.issueValidation || null,
+      gitMetadata: metadata.git,
+    });
+    attachKnownFailureIssueValidation(
+      enrichedArtifacts.runArtifact,
+      enrichedArtifacts.statusArtifact,
+      knownFailureIssueValidation
+    );
 
     writeRunArtifact(productDir, enrichedArtifacts.runArtifact);
     if (opts.writeStatus) {
       writeStatusArtifact(productDir, enrichedArtifacts.statusArtifact);
     }
 
-    printRunSummary(results, finishedAt - startedAt);
+    printRunSummary(results, finishedAt - startedAt, knownFailureIssueValidation);
     await reportTelemetry(telemetry, enrichedArtifacts.runArtifact);
     if (results.some((result) => result.failed)) exitCode = 1;
+    if (shouldFailKnownFailureIssueValidation(knownFailureIssueValidation)) {
+      exitCode = 1;
+    }
     if (lifecycle.isStopRequested()) exitCode = Math.max(exitCode, 130);
   } finally {
     if (lifecycle.isStopRequested()) {
@@ -269,8 +290,8 @@ function collectServicePlans(configs, configMap, typeValues, suiteSelectors, opt
   });
 }
 
-function printRunSummary(results, durationMs) {
-  for (const line of buildRunSummaryLines(results, durationMs)) {
+function printRunSummary(results, durationMs, knownFailureIssueValidation = null) {
+  for (const line of buildRunSummaryLines(results, durationMs, knownFailureIssueValidation)) {
     console.log(line);
   }
 }
@@ -299,3 +320,11 @@ async function reportTelemetry(telemetry, artifact) {
 function normalizePathSeparators(filePath) {
   return filePath.split("\\").join("/");
 }
+
+function attachKnownFailureIssueValidation(runArtifact, statusArtifact, validation) {
+  if (!validation) return;
+  runArtifact.knownFailuresIssueValidation = validation;
+  if (statusArtifact) {
+    statusArtifact.knownFailuresIssueValidation = validation;
+  }
+}

package/lib/runner/reporting.mjs

@@ -78,7 +78,7 @@ export function buildStatusArtifact({
     scope.serviceFilter === null;
 
   return {
-    schemaVersion: 4,
+    schemaVersion: 5,
     source: "testkit",
     notice: "Generated file. Do not edit manually.",
     product: {
@@ -127,7 +127,7 @@ export function buildRunArtifact({
   const dbBackend = summarizeDbBackend(results);
 
   return {
-    schemaVersion: 4,
+    schemaVersion: 5,
     source: "testkit",
     generatedAt: new Date(finishedAt).toISOString(),
     product: {

package/lib/runner/reporting.test.mjs

@@ -78,7 +78,7 @@ describe("runner reporting", () => {
     });
 
     expect(artifact.product.name).toBe("my-product");
-    expect(artifact.schemaVersion).toBe(4);
+    expect(artifact.schemaVersion).toBe(5);
     expect(artifact.run).toMatchObject({
       workers: 2,
       fileTimeoutSeconds: 60,
@@ -149,7 +149,7 @@ describe("runner reporting", () => {
     });
 
     expect(status).toEqual({
-      schemaVersion: 4,
+      schemaVersion: 5,
       source: "testkit",
       notice: "Generated file. Do not edit manually.",
       product: {