@elench/testkit 0.1.42 → 0.1.44

package/lib/known-failures/index.mjs

@@ -0,0 +1,329 @@
+import fs from "fs";
+import path from "path";
+
+const CLASSIFICATIONS = new Set([
+  "expected_failure",
+  "infra",
+  "product_bug",
+  "stale_test",
+  "test_bug",
+]);
+const STATES = new Set(["closed", "open"]);
+
+export function loadKnownFailuresConfig(productDir, config) {
+  const relativePath = config?.knownFailuresFile;
+  if (!relativePath) return null;
+
+  const absolutePath = path.resolve(productDir, relativePath);
+  if (!fs.existsSync(absolutePath)) {
+    throw new Error(`Known failures file not found: ${relativePath}`);
+  }
+
+  return loadKnownFailuresDocument(absolutePath, relativePath);
+}
+
+export function loadKnownFailuresDocument(filePath, relativePath = filePath) {
+  let parsed;
+  try {
+    parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
+  } catch (error) {
+    throw new Error(
+      `Could not parse known failures file ${relativePath}: ${formatErrorMessage(error)}`
+    );
+  }
+
+  return normalizeKnownFailuresDocument(parsed, relativePath);
+}
+
+export function normalizeKnownFailuresDocument(document, relativePath = "known failures file") {
+  if (!document || typeof document !== "object") {
+    throw new Error(`${relativePath} must contain a JSON object`);
+  }
+  if (document.schemaVersion !== 1) {
+    throw new Error(`${relativePath} schemaVersion must be 1`);
+  }
+  if (!Array.isArray(document.entries)) {
+    throw new Error(`${relativePath} entries must be an array`);
+  }
+
+  const ids = new Set();
+  const entries = document.entries.map((entry, index) => {
+    const normalized = normalizeKnownFailureEntry(entry, `${relativePath} entries[${index}]`);
+    if (ids.has(normalized.id)) {
+      throw new Error(`${relativePath} has duplicate entry id "${normalized.id}"`);
+    }
+    ids.add(normalized.id);
+    return normalized;
+  });
+
+  return {
+    schemaVersion: 1,
+    issueRepo: normalizeOptionalString(document.issueRepo),
+    entries,
+  };
+}
+
+export function validateKnownFailuresDocument(
+  document,
+  { productDir, statusArtifactPath, statusArtifact } = {}
+) {
+  const errors = [];
+  const warnings = [];
+  const matchKeys = new Set();
+  let matchCount = 0;
+
+  for (const entry of document.entries) {
+    for (const match of entry.matches) {
+      matchCount += 1;
+      if (productDir) {
+        const absolutePath = path.join(productDir, match.path);
+        if (!fs.existsSync(absolutePath)) {
+          errors.push(`Missing matched file: ${match.path} (${entry.id})`);
+        }
+      }
+
+      const matchKey = [
+        entry.id,
+        match.service || "",
+        match.type || "",
+        match.path,
+        match.failureKey || "",
+        match.errorIncludes || "",
+      ].join("::");
+      if (matchKeys.has(matchKey)) {
+        errors.push(`Duplicate match selector in ${entry.id}: ${match.path}`);
+      }
+      matchKeys.add(matchKey);
+
+      if (!match.path.includes("__testkit__") || !match.path.endsWith(".testkit.ts")) {
+        warnings.push(`Match path does not look like a testkit file: ${match.path} (${entry.id})`);
+      }
+    }
+
+    if (document.issueRepo && entry.issue.repo !== document.issueRepo) {
+      warnings.push(
+        `Entry ${entry.id} uses issue repo ${entry.issue.repo} instead of document issueRepo ${document.issueRepo}`
+      );
+    }
+    if (!entry.issue.url.endsWith(`/issues/${entry.issue.number}`)) {
+      errors.push(`Issue URL does not match issue number for ${entry.id}`);
+    }
+  }
+
+  let failedTests = 0;
+  let triagedFailedTests = 0;
+  let untriagedFailedTests = 0;
+  const parsedStatusArtifact = statusArtifact
+    ? parseStatusArtifact(statusArtifact)
+    : statusArtifactPath && fs.existsSync(statusArtifactPath)
+      ? parseStatusArtifact(JSON.parse(fs.readFileSync(statusArtifactPath, "utf8")))
+      : { tests: [] };
+
+  const failed = parsedStatusArtifact.tests.filter((test) => test.status === "failed");
+  failedTests = failed.length;
+  for (const test of failed) {
+    if (findMatchingKnownFailureEntries(document, test).length > 0) {
+      triagedFailedTests += 1;
+    } else {
+      untriagedFailedTests += 1;
+      warnings.push(`Untriaged failed test: ${test.path}`);
+    }
+  }
+
+  return {
+    errors,
+    warnings,
+    stats: {
+      entries: document.entries.length,
+      matches: matchCount,
+      failedTests,
+      triagedFailedTests,
+      untriagedFailedTests,
+    },
+  };
+}
+
+export function renderKnownFailuresMarkdown(document) {
+  const lines = [
+    "# Bourne Bugs",
+    "",
+    "Canonical source: `testkit.known-failures.json`",
+    "",
+    `Tracked bug classes: ${document.entries.length}`,
+    `Tracked issue references: ${new Set(document.entries.map((entry) => entry.issue.number)).size}`,
+    "",
+  ];
+
+  document.entries.forEach((entry, index) => {
+    lines.push(
+      `## ${index + 1}. ${entry.title} — [#${entry.issue.number}](${entry.issue.url})`,
+      "",
+      `- Classification: \`${entry.classification}\``,
+      `- State: \`${entry.state}\``,
+      `- Description: ${entry.description}`,
+      `- Why failing: ${entry.whyFailing}`,
+      `- Last reviewed: ${entry.lastReviewedAt}`,
+      "- Matches:"
+    );
+
+    for (const match of entry.matches) {
+      lines.push(
+        `  - \`${match.path}\`${match.failureKey ? ` — \`${match.failureKey}\`` : ""}${
+          match.errorIncludes ? ` — error contains \`${match.errorIncludes}\`` : ""
+        }`
+      );
+    }
+
+    lines.push("");
+  });
+
+  return `${lines.join("\n").trimEnd()}\n`;
+}
+
+export function findMatchingKnownFailureEntries(document, fileSummary) {
+  return document.entries.filter((entry) => matchesKnownFailureEntry(entry, fileSummary));
+}
+
+export function matchesKnownFailureEntry(entry, fileSummary) {
+  return entry.matches.some((match) => matchesKnownFailureMatch(match, fileSummary));
+}
+
+export function matchesKnownFailureMatch(match, fileSummary) {
+  if (match.service && match.service !== fileSummary.service) return false;
+  if (match.type && match.type !== fileSummary.type) return false;
+  if (match.path !== fileSummary.path) return false;
+  if (match.failureKey) {
+    const failureKeys = Array.isArray(fileSummary.failureDetails)
+      ? fileSummary.failureDetails.map((detail) => detail?.key)
+      : [];
+    if (!failureKeys.includes(match.failureKey)) return false;
+  }
+  if (match.errorIncludes && !String(fileSummary.error || "").includes(match.errorIncludes)) {
+    return false;
+  }
+  return true;
+}
+
+export function buildKnownFailureFileIdentity(service, type, filePath) {
+  return `${service}::${type}::${filePath}`;
+}
+
+function normalizeKnownFailureEntry(entry, label) {
+  if (!entry || typeof entry !== "object") {
+    throw new Error(`${label} must be an object`);
+  }
+
+  const id = requireNonEmptyString(entry.id, `${label}.id`);
+  const title = requireNonEmptyString(entry.title, `${label}.title`);
+  const classification = requireEnumValue(
+    entry.classification,
+    CLASSIFICATIONS,
+    `${label}.classification`
+  );
+  const state = requireEnumValue(entry.state, STATES, `${label}.state`);
+  const description = requireNonEmptyString(entry.description, `${label}.description`);
+  const whyFailing = requireNonEmptyString(entry.whyFailing, `${label}.whyFailing`);
+  const lastReviewedAt = requireNonEmptyString(entry.lastReviewedAt, `${label}.lastReviewedAt`);
+  if (!Array.isArray(entry.matches) || entry.matches.length === 0) {
+    throw new Error(`${label}.matches must be a non-empty array`);
+  }
+
+  return {
+    id,
+    title,
+    classification,
+    state,
+    issue: normalizeKnownFailureIssue(entry.issue, `${label}.issue`),
+    description,
+    whyFailing,
+    lastReviewedAt,
+    matches: entry.matches.map((match, index) =>
+      normalizeKnownFailureMatch(match, `${label}.matches[${index}]`)
+    ),
+  };
+}
+
+function normalizeKnownFailureIssue(issue, label) {
+  if (!issue || typeof issue !== "object") {
+    throw new Error(`${label} must be an object`);
+  }
+
+  const repo = requireNonEmptyString(issue.repo, `${label}.repo`);
+  const number = issue.number;
+  if (!Number.isInteger(number) || number <= 0) {
+    throw new Error(`${label}.number must be a positive integer`);
+  }
+  const url = requireNonEmptyString(issue.url, `${label}.url`);
+
+  return { repo, number, url };
+}
+
+function normalizeKnownFailureMatch(match, label) {
+  if (!match || typeof match !== "object") {
+    throw new Error(`${label} must be an object`);
+  }
+
+  const pathValue = requireNonEmptyString(match.path, `${label}.path`);
+  const normalized = {
+    path: pathValue,
+  };
+
+  const service = normalizeOptionalString(match.service);
+  if (service) normalized.service = service;
+
+  const type = normalizeOptionalString(match.type);
+  if (type) normalized.type = type;
+
+  const failureKey = normalizeOptionalString(match.failureKey);
+  if (failureKey) normalized.failureKey = failureKey;
+
+  const errorIncludes = normalizeOptionalString(match.errorIncludes);
+  if (errorIncludes) normalized.errorIncludes = errorIncludes;
+
+  return normalized;
+}
+
+function parseStatusArtifact(value) {
+  if (!value || typeof value !== "object") {
+    return { tests: [] };
+  }
+
+  return {
+    tests: Array.isArray(value.tests) ? value.tests : [],
+  };
+}
+
+function requireNonEmptyString(value, label) {
+  const normalized = normalizeOptionalString(value);
+  if (!normalized) {
+    throw new Error(`${label} must be a non-empty string`);
+  }
+  return normalized;
+}
+
+function requireEnumValue(value, allowed, label) {
+  const normalized = requireNonEmptyString(value, label);
+  if (!allowed.has(normalized)) {
+    throw new Error(`${label} must be one of: ${[...allowed].sort().join(", ")}`);
+  }
+  return normalized;
+}
+
+function normalizeOptionalString(value) {
+  if (typeof value !== "string") return null;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : null;
+}
+
+function formatErrorMessage(error) {
+  if (error instanceof Error) return error.message;
+  return String(error);
+}
+
+export {
+  buildKnownFailureIssueValidationSummaryLines,
+  normalizeKnownFailureIssueValidationConfig,
+  parseGitHubRepoSlug,
+  shouldFailKnownFailureIssueValidation,
+  validateKnownFailureIssues,
+} from "./github.mjs";

package/lib/known-failures/index.test.mjs

@@ -0,0 +1,152 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { afterEach, describe, expect, it } from "vitest";
+import {
+  findMatchingKnownFailureEntries,
+  normalizeKnownFailuresDocument,
+  renderKnownFailuresMarkdown,
+  validateKnownFailuresDocument,
+} from "./index.mjs";
+
+const tempDirs = [];
+
+afterEach(() => {
+  for (const tempDir of tempDirs.splice(0)) {
+    fs.rmSync(tempDir, { recursive: true, force: true });
+  }
+});
+
+describe("known failures core", () => {
+  it("matches entries by service, type, path, and failure key", () => {
+    const document = normalizeKnownFailuresDocument({
+      schemaVersion: 1,
+      entries: [
+        {
+          id: "bad-message",
+          title: "Bad message bug",
+          classification: "product_bug",
+          state: "open",
+          issue: {
+            repo: "acme/repo",
+            number: 12,
+            url: "https://github.com/acme/repo/issues/12",
+          },
+          description: "Wrong message",
+          whyFailing: "Payload is wrong",
+          lastReviewedAt: "2026-04-27",
+          matches: [
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+              failureKey: "returns the wrong message",
+            },
+          ],
+        },
+      ],
+    });
+
+    const matches = findMatchingKnownFailureEntries(document, {
+      service: "api",
+      type: "int",
+      path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+      error: "boom",
+      failureDetails: [{ key: "returns the wrong message" }],
+    });
+
+    expect(matches).toHaveLength(1);
+    expect(matches[0].id).toBe("bad-message");
+  });
+
+  it("validates status coverage and filesystem matches", () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-known-failures-"));
+    tempDirs.push(tempDir);
+    const testPath = path.join(tempDir, "src/api/routes/__testkit__/failing.int.testkit.ts");
+    fs.mkdirSync(path.dirname(testPath), { recursive: true });
+    fs.writeFileSync(testPath, "export default {};\n");
+
+    const document = normalizeKnownFailuresDocument({
+      schemaVersion: 1,
+      issueRepo: "acme/repo",
+      entries: [
+        {
+          id: "bad-message",
+          title: "Bad message bug",
+          classification: "product_bug",
+          state: "open",
+          issue: {
+            repo: "acme/repo",
+            number: 12,
+            url: "https://github.com/acme/repo/issues/12",
+          },
+          description: "Wrong message",
+          whyFailing: "Payload is wrong",
+          lastReviewedAt: "2026-04-27",
+          matches: [
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            },
+          ],
+        },
+      ],
+    });
+
+    const result = validateKnownFailuresDocument(document, {
+      productDir: tempDir,
+      statusArtifact: {
+        tests: [
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            status: "failed",
+          },
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/other.int.testkit.ts",
+            status: "failed",
+          },
+        ],
+      },
+    });
+
+    expect(result.errors).toEqual([]);
+    expect(result.stats.failedTests).toBe(2);
+    expect(result.stats.triagedFailedTests).toBe(1);
+    expect(result.warnings).toContain(
+      "Untriaged failed test: src/api/routes/__testkit__/other.int.testkit.ts"
+    );
+  });
+
+  it("renders markdown from the canonical document", () => {
+    const markdown = renderKnownFailuresMarkdown(
+      normalizeKnownFailuresDocument({
+        schemaVersion: 1,
+        entries: [
+          {
+            id: "bad-message",
+            title: "Bad message bug",
+            classification: "product_bug",
+            state: "open",
+            issue: {
+              repo: "acme/repo",
+              number: 12,
+              url: "https://github.com/acme/repo/issues/12",
+            },
+            description: "Wrong message",
+            whyFailing: "Payload is wrong",
+            lastReviewedAt: "2026-04-27",
+            matches: [{ path: "src/api/routes/__testkit__/failing.int.testkit.ts" }],
+          },
+        ],
+      })
+    );
+
+    expect(markdown).toContain("# Bourne Bugs");
+    expect(markdown).toContain("[#12](https://github.com/acme/repo/issues/12)");
+  });
+});

package/lib/package.test.mjs

@@ -22,8 +22,13 @@ describe("package metadata", () => {
       types: "./lib/runtime/index.d.ts",
       default: "./lib/runtime/index.mjs",
     });
+    expect(packageJson.exports["./known-failures"]).toEqual({
+      types: "./lib/known-failures/index.d.ts",
+      default: "./lib/known-failures/index.mjs",
+    });
     expect(fs.existsSync(path.join(rootDir, "lib", "index.d.ts"))).toBe(true);
     expect(fs.existsSync(path.join(rootDir, "lib", "setup", "index.d.ts"))).toBe(true);
     expect(fs.existsSync(path.join(rootDir, "lib", "runtime", "index.d.ts"))).toBe(true);
+    expect(fs.existsSync(path.join(rootDir, "lib", "known-failures", "index.d.ts"))).toBe(true);
   });
 });

package/lib/reporters/playwright.mjs

@@ -1,4 +1,5 @@
 import path from "path";
+import { mergeFailureDetails } from "../runner/failure-details.mjs";
 
 export function parsePlaywrightJsonResults(stdout, cwd) {
   if (!stdout.trim()) {
@@ -16,34 +17,42 @@ export function parsePlaywrightJsonResults(stdout, cwd) {
   }
 
   const fileResults = new Map();
-  visitPlaywrightSuites(parsed.suites || [], null, fileResults, cwd);
+  visitPlaywrightSuites(parsed.suites || [], null, [], fileResults, cwd);
   return {
     fileResults: sanitizePlaywrightFileResults(fileResults),
     errors: (parsed.errors || []).map(formatPlaywrightReporterError).filter(Boolean),
   };
 }
 
-export function visitPlaywrightSuites(suites, inheritedFile, fileResults, cwd) {
+export function visitPlaywrightSuites(suites, inheritedFile, inheritedTitlePath, fileResults, cwd) {
   for (const suite of suites || []) {
     const suiteFile = normalizeReportedFile(extractReporterFile(suite) || inheritedFile, cwd);
+    const suiteTitle = normalizeSuiteTitle(suite?.title, suiteFile);
+    const suiteTitlePath = suiteTitle
+      ? [...inheritedTitlePath, suiteTitle].filter(Boolean)
+      : inheritedTitlePath;
     for (const child of suite.suites || []) {
-      visitPlaywrightSuites([child], suiteFile, fileResults, cwd);
+      visitPlaywrightSuites([child], suiteFile, suiteTitlePath, fileResults, cwd);
     }
     for (const spec of suite.specs || []) {
-      collectPlaywrightSpec(spec, suiteFile, fileResults, cwd);
+      collectPlaywrightSpec(spec, suiteFile, suiteTitlePath, fileResults, cwd);
     }
   }
 }
 
-export function collectPlaywrightSpec(spec, inheritedFile, fileResults, cwd) {
+export function collectPlaywrightSpec(spec, inheritedFile, suiteTitlePath, fileResults, cwd) {
   const file = normalizeReportedFile(extractReporterFile(spec) || inheritedFile, cwd);
   if (!file) return;
+  const specTitle = firstLine(spec?.title || "Playwright spec failed") || "Playwright spec failed";
+  const specPath = [...suiteTitlePath, specTitle].filter(Boolean);
+  const failureKey = specPath.join(" > ");
 
   const current = fileResults.get(file) || {
     failed: false,
     status: "passed",
     error: null,
     durationMs: 0,
+    failureDetails: [],
     passedCount: 0,
     skippedCount: 0,
   };
@@ -62,6 +71,13 @@ export function collectPlaywrightSpec(spec, inheritedFile, fileResults, cwd) {
       current.failed = true;
       current.status = "failed";
       current.error ||= extractPlaywrightFailure(final, spec, test);
+      current.failureDetails.push({
+        kind: "playwright-spec",
+        key: failureKey,
+        title: specTitle,
+        suitePath: suiteTitlePath,
+        message: extractPlaywrightFailure(final, spec, test),
+      });
       continue;
     }
 
@@ -76,6 +92,7 @@ export function collectPlaywrightSpec(spec, inheritedFile, fileResults, cwd) {
   if (!current.failed) {
     current.status = current.passedCount === 0 && current.skippedCount > 0 ? "skipped" : "passed";
   }
+  current.failureDetails = mergeFailureDetails(current.failureDetails);
   fileResults.set(file, current);
 }
 
@@ -155,7 +172,19 @@ function sanitizePlaywrightFileResults(fileResults) {
       status: result.status,
       error: result.error,
       durationMs: result.durationMs,
+      failureDetails: mergeFailureDetails(result.failureDetails),
     });
   }
   return sanitized;
 }
+
+function normalizeSuiteTitle(title, reportedFile) {
+  const normalizedTitle = firstLine(title || "");
+  if (!normalizedTitle) return null;
+  if (!reportedFile) return normalizedTitle;
+  const fileName = reportedFile.split("/").pop();
+  if (normalizedTitle === reportedFile || normalizedTitle === fileName) {
+    return null;
+  }
+  return normalizedTitle;
+}

package/lib/reporters/playwright.test.mjs

@@ -52,6 +52,15 @@ describe("playwright-report", () => {
       status: "failed",
       error: "boom",
       durationMs: 15,
+      failureDetails: [
+        {
+          kind: "playwright-spec",
+          key: "auth works",
+          title: "auth works",
+          count: 1,
+          message: "boom",
+        },
+      ],
     });
   });
 
@@ -86,6 +95,7 @@ describe("playwright-report", () => {
       status: "skipped",
       error: null,
       durationMs: 7,
+      failureDetails: [],
     });
   });
 
@@ -134,6 +144,7 @@ describe("playwright-report", () => {
       status: "passed",
       error: null,
       durationMs: 8,
+      failureDetails: [],
     });
   });
 

package/lib/runner/default-runtime-runner.mjs

@@ -9,6 +9,7 @@ import {
 } from "../shared/file-timeout.mjs";
 import { persistTaskArtifacts } from "./artifacts.mjs";
 import { determineDefaultRuntimeFailure } from "./default-runtime-errors.mjs";
+import { collectFailureDetailsFromRuntimeArtifacts } from "./failure-details.mjs";
 import { RUNTIME_ARTIFACT_MARKER } from "../runtime-src/k6/artifacts.js";
 import { readDatabaseUrl } from "./state-io.mjs";
 import { buildTaskExecutionEnv } from "./template.mjs";
@@ -105,11 +106,13 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
   if (stderr.visibleOutput) process.stderr.write(stderr.visibleOutput);
 
   const summary = readDefaultRuntimeSummary(summaryFile);
+  const rawRuntimeArtifacts = [...stdout.artifacts, ...stderr.artifacts];
   const runtimeArtifacts = persistTaskArtifacts(
     targetConfig.productDir,
     task,
-    [...stdout.artifacts, ...stderr.artifacts]
+    rawRuntimeArtifacts
   );
+  const failureDetails = collectFailureDetailsFromRuntimeArtifacts(rawRuntimeArtifacts);
   const runtimeError = timedOut
     ? formatFileTimeoutBudgetError(fileTimeoutSeconds)
     : determineDefaultRuntimeFailure(result, summary, getFirstLine);
@@ -123,6 +126,7 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
     startedAt,
     finishedAt,
     artifacts: runtimeArtifacts,
+    failureDetails,
   };
 }