@elench/testkit 0.1.42 → 0.1.43

package/README.md

@@ -68,6 +68,9 @@ export default defineTestkitSetup({
     workers: 8,
     fileTimeoutSeconds: 60,
   },
+  reporting: {
+    knownFailuresFile: "testkit.known-failures.json",
+  },
   services: {
     api: service({
       ...tsxService({
@@ -141,9 +144,17 @@ for:
 - migrate / seed commands
 - test-local migrate / seed overrides
 - named HTTP suite profiles
+- known-failure annotation merge for enriched status/run artifacts
 - repo-declared suite/file skip policies with explicit reasons
 - telemetry upload configuration
 
+If `reporting.knownFailuresFile` is configured, `testkit` enriches
+`.testkit/results/latest.json` and `testkit.status.json` with:
+
+- per-file `failureDetails`
+- per-file `triage` metadata (issue, classification, description)
+- top-level `triageSummary` counts for known vs untriaged failures
+
 ## Authoring
 
 HTTP suites:

package/lib/config/index.mjs

@@ -30,6 +30,7 @@ export async function loadConfigs(opts = {}) {
   const productDir = resolveProductDir(process.cwd(), opts.dir);
   const { setup, setupFile } = await loadTestkitSetup(productDir);
   const execution = normalizeRepoExecution(setup.execution);
+  const reporting = normalizeReportingConfig(setup.reporting);
   const explicitServices = setup.services || {};
   const discovery = discoverProject(productDir, explicitServices);
   const serviceNames = new Set([
@@ -46,6 +47,7 @@ export async function loadConfigs(opts = {}) {
         setup,
         setupFile,
         execution,
+        reporting,
         explicitService: explicitServices[name] || {},
         discoveredService: discovery.services[name] || null,
         suites: discovery.suitesByService[name] || {},
@@ -105,6 +107,7 @@ function normalizeServiceConfig({
   setup,
   setupFile,
   execution,
+  reporting,
   explicitService,
   discoveredService,
   suites,
@@ -157,6 +160,7 @@ function normalizeServiceConfig({
     suites,
     testkit: {
       execution,
+      reporting,
       dependsOn: explicitService.dependsOn || [],
       database,
       databaseFrom: explicitService.databaseFrom,
@@ -172,6 +176,19 @@ function normalizeServiceConfig({
   };
 }
 
+function normalizeReportingConfig(value) {
+  if (!value) return null;
+
+  const knownFailuresFile = normalizeOptionalString(value.knownFailuresFile);
+  if (!knownFailuresFile) {
+    throw new Error('testkit.setup.ts reporting.knownFailuresFile must be a non-empty string');
+  }
+
+  return {
+    knownFailuresFile,
+  };
+}
+
 function normalizeLocalConfig(name, explicitService, discoveredService, productDir) {
   if (Object.prototype.hasOwnProperty.call(explicitService, "local")) {
     if (explicitService.local === false) {
@@ -282,6 +299,12 @@ function normalizeRuntimeConfig(value, serviceName) {
   };
 }
 
+function normalizeOptionalString(value) {
+  if (typeof value !== "string") return null;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : null;
+}
+
 function normalizeLifecycle(value) {
   if (!value) return undefined;
   if (!value.cmd && !value.testkitCmd) {

package/lib/reporters/playwright.mjs

@@ -1,4 +1,5 @@
 import path from "path";
+import { mergeFailureDetails } from "../runner/failure-details.mjs";
 
 export function parsePlaywrightJsonResults(stdout, cwd) {
   if (!stdout.trim()) {
@@ -16,34 +17,42 @@ export function parsePlaywrightJsonResults(stdout, cwd) {
   }
 
   const fileResults = new Map();
-  visitPlaywrightSuites(parsed.suites || [], null, fileResults, cwd);
+  visitPlaywrightSuites(parsed.suites || [], null, [], fileResults, cwd);
   return {
     fileResults: sanitizePlaywrightFileResults(fileResults),
     errors: (parsed.errors || []).map(formatPlaywrightReporterError).filter(Boolean),
   };
 }
 
-export function visitPlaywrightSuites(suites, inheritedFile, fileResults, cwd) {
+export function visitPlaywrightSuites(suites, inheritedFile, inheritedTitlePath, fileResults, cwd) {
   for (const suite of suites || []) {
     const suiteFile = normalizeReportedFile(extractReporterFile(suite) || inheritedFile, cwd);
+    const suiteTitle = normalizeSuiteTitle(suite?.title, suiteFile);
+    const suiteTitlePath = suiteTitle
+      ? [...inheritedTitlePath, suiteTitle].filter(Boolean)
+      : inheritedTitlePath;
     for (const child of suite.suites || []) {
-      visitPlaywrightSuites([child], suiteFile, fileResults, cwd);
+      visitPlaywrightSuites([child], suiteFile, suiteTitlePath, fileResults, cwd);
     }
     for (const spec of suite.specs || []) {
-      collectPlaywrightSpec(spec, suiteFile, fileResults, cwd);
+      collectPlaywrightSpec(spec, suiteFile, suiteTitlePath, fileResults, cwd);
     }
   }
 }
 
-export function collectPlaywrightSpec(spec, inheritedFile, fileResults, cwd) {
+export function collectPlaywrightSpec(spec, inheritedFile, suiteTitlePath, fileResults, cwd) {
   const file = normalizeReportedFile(extractReporterFile(spec) || inheritedFile, cwd);
   if (!file) return;
+  const specTitle = firstLine(spec?.title || "Playwright spec failed") || "Playwright spec failed";
+  const specPath = [...suiteTitlePath, specTitle].filter(Boolean);
+  const failureKey = specPath.join(" > ");
 
   const current = fileResults.get(file) || {
     failed: false,
     status: "passed",
     error: null,
     durationMs: 0,
+    failureDetails: [],
     passedCount: 0,
     skippedCount: 0,
   };
@@ -62,6 +71,13 @@ export function collectPlaywrightSpec(spec, inheritedFile, fileResults, cwd) {
       current.failed = true;
       current.status = "failed";
       current.error ||= extractPlaywrightFailure(final, spec, test);
+      current.failureDetails.push({
+        kind: "playwright-spec",
+        key: failureKey,
+        title: specTitle,
+        suitePath: suiteTitlePath,
+        message: extractPlaywrightFailure(final, spec, test),
+      });
       continue;
     }
 
@@ -76,6 +92,7 @@ export function collectPlaywrightSpec(spec, inheritedFile, fileResults, cwd) {
   if (!current.failed) {
     current.status = current.passedCount === 0 && current.skippedCount > 0 ? "skipped" : "passed";
   }
+  current.failureDetails = mergeFailureDetails(current.failureDetails);
   fileResults.set(file, current);
 }
 
@@ -155,7 +172,19 @@ function sanitizePlaywrightFileResults(fileResults) {
       status: result.status,
       error: result.error,
       durationMs: result.durationMs,
+      failureDetails: mergeFailureDetails(result.failureDetails),
     });
   }
   return sanitized;
 }
+
+function normalizeSuiteTitle(title, reportedFile) {
+  const normalizedTitle = firstLine(title || "");
+  if (!normalizedTitle) return null;
+  if (!reportedFile) return normalizedTitle;
+  const fileName = reportedFile.split("/").pop();
+  if (normalizedTitle === reportedFile || normalizedTitle === fileName) {
+    return null;
+  }
+  return normalizedTitle;
+}

package/lib/reporters/playwright.test.mjs

@@ -52,6 +52,15 @@ describe("playwright-report", () => {
       status: "failed",
       error: "boom",
       durationMs: 15,
+      failureDetails: [
+        {
+          kind: "playwright-spec",
+          key: "auth works",
+          title: "auth works",
+          count: 1,
+          message: "boom",
+        },
+      ],
     });
   });
 
@@ -86,6 +95,7 @@ describe("playwright-report", () => {
       status: "skipped",
       error: null,
       durationMs: 7,
+      failureDetails: [],
     });
   });
 
@@ -134,6 +144,7 @@ describe("playwright-report", () => {
       status: "passed",
       error: null,
       durationMs: 8,
+      failureDetails: [],
     });
   });
 

package/lib/runner/default-runtime-runner.mjs

@@ -9,6 +9,7 @@ import {
 } from "../shared/file-timeout.mjs";
 import { persistTaskArtifacts } from "./artifacts.mjs";
 import { determineDefaultRuntimeFailure } from "./default-runtime-errors.mjs";
+import { collectFailureDetailsFromRuntimeArtifacts } from "./failure-details.mjs";
 import { RUNTIME_ARTIFACT_MARKER } from "../runtime-src/k6/artifacts.js";
 import { readDatabaseUrl } from "./state-io.mjs";
 import { buildTaskExecutionEnv } from "./template.mjs";
@@ -105,11 +106,13 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
   if (stderr.visibleOutput) process.stderr.write(stderr.visibleOutput);
 
   const summary = readDefaultRuntimeSummary(summaryFile);
+  const rawRuntimeArtifacts = [...stdout.artifacts, ...stderr.artifacts];
   const runtimeArtifacts = persistTaskArtifacts(
     targetConfig.productDir,
     task,
-    [...stdout.artifacts, ...stderr.artifacts]
+    rawRuntimeArtifacts
   );
+  const failureDetails = collectFailureDetailsFromRuntimeArtifacts(rawRuntimeArtifacts);
   const runtimeError = timedOut
     ? formatFileTimeoutBudgetError(fileTimeoutSeconds)
     : determineDefaultRuntimeFailure(result, summary, getFirstLine);
@@ -123,6 +126,7 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
     startedAt,
     finishedAt,
     artifacts: runtimeArtifacts,
+    failureDetails,
   };
 }
 

package/lib/runner/failure-details.mjs

@@ -0,0 +1,91 @@
+export function normalizeFailureDetail(detail) {
+  if (!detail || typeof detail !== "object") return null;
+
+  const kind = normalizeNonEmptyString(detail.kind);
+  const key = normalizeNonEmptyString(detail.key);
+  const title = normalizeNonEmptyString(detail.title);
+  if (!kind || !key || !title) return null;
+
+  const normalized = {
+    kind,
+    key,
+    title,
+    count: normalizePositiveInteger(detail.count) || 1,
+  };
+
+  const groupPath = normalizeStringArray(detail.groupPath);
+  if (groupPath.length > 0) normalized.groupPath = groupPath;
+
+  const checkName = normalizeNonEmptyString(detail.checkName);
+  if (checkName) normalized.checkName = checkName;
+
+  const suitePath = normalizeStringArray(detail.suitePath);
+  if (suitePath.length > 0) normalized.suitePath = suitePath;
+
+  const phase = normalizeNonEmptyString(detail.phase);
+  if (phase) normalized.phase = phase;
+
+  const message = normalizeNonEmptyString(detail.message);
+  if (message) normalized.message = message;
+
+  return normalized;
+}
+
+export function mergeFailureDetails(details) {
+  const mergedByKey = new Map();
+
+  for (const detail of Array.isArray(details) ? details : []) {
+    const normalized = normalizeFailureDetail(detail);
+    if (!normalized) continue;
+    const dedupeKey = `${normalized.kind}::${normalized.key}`;
+    const existing = mergedByKey.get(dedupeKey);
+    if (!existing) {
+      mergedByKey.set(dedupeKey, { ...normalized });
+      continue;
+    }
+    existing.count += normalized.count;
+    if (!existing.message && normalized.message) {
+      existing.message = normalized.message;
+    }
+  }
+
+  return [...mergedByKey.values()].sort(
+    (left, right) =>
+      left.kind.localeCompare(right.kind) || left.key.localeCompare(right.key)
+  );
+}
+
+export function collectFailureDetailsFromRuntimeArtifacts(artifacts) {
+  const details = [];
+
+  for (const artifact of Array.isArray(artifacts) ? artifacts : []) {
+    if (artifact?.kind !== "testkit.failure-details") continue;
+    const phase = normalizeNonEmptyString(artifact?.data?.phase);
+    for (const detail of Array.isArray(artifact?.data?.failures) ? artifact.data.failures : []) {
+      details.push({
+        ...detail,
+        phase: detail?.phase || phase || null,
+      });
+    }
+  }
+
+  return mergeFailureDetails(details);
+}
+
+function normalizeStringArray(value) {
+  if (!Array.isArray(value)) return [];
+  return value
+    .map((entry) => normalizeNonEmptyString(entry))
+    .filter(Boolean);
+}
+
+function normalizeNonEmptyString(value) {
+  if (typeof value !== "string") return null;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : null;
+}
+
+function normalizePositiveInteger(value) {
+  if (!Number.isInteger(value) || value <= 0) return null;
+  return value;
+}

package/lib/runner/failure-details.test.mjs

@@ -0,0 +1,63 @@
+import { describe, expect, it } from "vitest";
+import {
+  collectFailureDetailsFromRuntimeArtifacts,
+  mergeFailureDetails,
+} from "./failure-details.mjs";
+
+describe("runner failure details", () => {
+  it("merges duplicate failure details by kind and key", () => {
+    expect(
+      mergeFailureDetails([
+        { kind: "k6-check", key: "group > check", title: "check" },
+        { kind: "k6-check", key: "group > check", title: "check" },
+        { kind: "playwright-spec", key: "spec title", title: "spec title" },
+      ])
+    ).toEqual([
+      {
+        kind: "k6-check",
+        key: "group > check",
+        title: "check",
+        count: 2,
+      },
+      {
+        kind: "playwright-spec",
+        key: "spec title",
+        title: "spec title",
+        count: 1,
+      },
+    ]);
+  });
+
+  it("extracts and normalizes runtime failure-detail artifacts", () => {
+    expect(
+      collectFailureDetailsFromRuntimeArtifacts([
+        {
+          kind: "testkit.failure-details",
+          data: {
+            phase: "exec",
+            failures: [
+              {
+                kind: "k6-check",
+                key: "status is 200",
+                title: "status is 200",
+              },
+              {
+                kind: "k6-check",
+                key: "status is 200",
+                title: "status is 200",
+              },
+            ],
+          },
+        },
+      ])
+    ).toEqual([
+      {
+        kind: "k6-check",
+        key: "status is 200",
+        title: "status is 200",
+        count: 2,
+        phase: "exec",
+      },
+    ]);
+  });
+});

package/lib/runner/orchestrator.mjs

@@ -15,6 +15,7 @@ import {
   summarizeDbBackend,
 } from "./results.mjs";
 import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
+import { applyKnownFailuresToArtifacts, loadKnownFailuresConfig } from "./triage.mjs";
 import { buildRunSummaryLines, formatError } from "./formatting.mjs";
 import {
   loadTimings,
@@ -56,6 +57,10 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     },
     testkitVersion: readPackageMetadata().version,
   };
+  const knownFailures = loadKnownFailuresConfig(
+    productDir,
+    configs[0]?.testkit?.reporting || null
+  );
   const requestedFiles = opts.fileNames || [];
   if (requestedFiles.length > 0) {
     const unmatchedFiles = findUnmatchedRequestedFiles(
@@ -168,7 +173,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     results = configs.map((config) =>
       finalizeServiceResult(trackers.get(config.name), startedAt, finishedAt)
     );
-    const artifact = buildRunArtifact({
+    const runArtifact = buildRunArtifact({
       productDir,
       results,
       startedAt,
@@ -185,12 +190,8 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
       metadata,
       summarizeDbBackend,
     });
-
-    writeRunArtifact(productDir, artifact);
-    if (opts.writeStatus) {
-      writeStatusArtifact(
-        productDir,
-        buildStatusArtifact({
+    const statusArtifact = opts.writeStatus
+      ? buildStatusArtifact({
           productDir,
           results,
           typeValues,
@@ -200,11 +201,20 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
           serviceFilter: opts.serviceFilter || null,
           metadata,
         })
-      );
+      : null;
+    const enrichedArtifacts = applyKnownFailuresToArtifacts(
+      runArtifact,
+      statusArtifact,
+      knownFailures
+    );
+
+    writeRunArtifact(productDir, enrichedArtifacts.runArtifact);
+    if (opts.writeStatus) {
+      writeStatusArtifact(productDir, enrichedArtifacts.statusArtifact);
     }
 
     printRunSummary(results, finishedAt - startedAt);
-    await reportTelemetry(telemetry, artifact);
+    await reportTelemetry(telemetry, enrichedArtifacts.runArtifact);
     if (results.some((result) => result.failed)) exitCode = 1;
     if (lifecycle.isStopRequested()) exitCode = Math.max(exitCode, 130);
   } finally {

package/lib/runner/playwright-runner.mjs

@@ -76,5 +76,6 @@ export async function runPlaywrightTask(targetConfig, task, lifecycle, lease) {
     durationMs: fileResult?.durationMs > 0 ? fileResult.durationMs : durationMs,
     startedAt,
     finishedAt,
+    failureDetails: timedOut ? [] : fileResult?.failureDetails || [],
   };
 }

package/lib/runner/reporting.mjs

@@ -23,9 +23,15 @@ export function buildStatusArtifact({
           path: file.path,
           status: file.status,
         };
+        if (file.error) {
+          test.error = file.error;
+        }
         if (file.reason) {
           test.reason = file.reason;
         }
+        if (Array.isArray(file.failureDetails) && file.failureDetails.length > 0) {
+          test.failureDetails = file.failureDetails;
+        }
         tests.push(test);
       }
     }
@@ -72,7 +78,7 @@ export function buildStatusArtifact({
     scope.serviceFilter === null;
 
   return {
-    schemaVersion: 3,
+    schemaVersion: 4,
     source: "testkit",
     notice: "Generated file. Do not edit manually.",
     product: {
@@ -121,7 +127,7 @@ export function buildRunArtifact({
   const dbBackend = summarizeDbBackend(results);
 
   return {
-    schemaVersion: 3,
+    schemaVersion: 4,
     source: "testkit",
     generatedAt: new Date(finishedAt).toISOString(),
     product: {

package/lib/runner/reporting.test.mjs

@@ -78,7 +78,7 @@ describe("runner reporting", () => {
     });
 
     expect(artifact.product.name).toBe("my-product");
-    expect(artifact.schemaVersion).toBe(3);
+    expect(artifact.schemaVersion).toBe(4);
     expect(artifact.run).toMatchObject({
       workers: 2,
       fileTimeoutSeconds: 60,
@@ -149,7 +149,7 @@ describe("runner reporting", () => {
     });
 
     expect(status).toEqual({
-      schemaVersion: 3,
+      schemaVersion: 4,
       source: "testkit",
       notice: "Generated file. Do not edit manually.",
       product: {

package/lib/runner/results.mjs

@@ -1,4 +1,5 @@
 import path from "path";
+import { mergeFailureDetails } from "./failure-details.mjs";
 
 export function buildServiceTrackers(servicePlans, startedAt) {
   const trackers = new Map();
@@ -46,6 +47,7 @@ export function buildServiceTrackers(servicePlans, startedAt) {
               reason: null,
               status: "not_run",
               artifacts: [],
+              failureDetails: [],
             },
           ];
         }),
@@ -59,6 +61,7 @@ export function buildServiceTrackers(servicePlans, startedAt) {
             reason: file.reason,
             status: "skipped",
             artifacts: [],
+            failureDetails: [],
           },
         ]),
       ]),
@@ -121,6 +124,7 @@ export function recordTaskOutcome(trackers, task, outcome, finishedAt = Date.now
     existingFileResult.reason = outcome.reason || null;
     existingFileResult.status = status;
     existingFileResult.artifacts = Array.isArray(outcome.artifacts) ? outcome.artifacts : [];
+    existingFileResult.failureDetails = mergeFailureDetails(outcome.failureDetails);
   } else {
     suite.fileResultsByPath.set(normalizedPath, {
       path: normalizedPath,
@@ -130,6 +134,7 @@ export function recordTaskOutcome(trackers, task, outcome, finishedAt = Date.now
       reason: outcome.reason || null,
       status,
       artifacts: Array.isArray(outcome.artifacts) ? outcome.artifacts : [],
+      failureDetails: mergeFailureDetails(outcome.failureDetails),
     });
   }
   if (status === "failed" && !suite.failedFileSet.has(task.file)) {
@@ -247,6 +252,9 @@ function finalizeSuite(suite) {
       durationMs: file.durationMs,
       error: file.error,
       reason: file.reason,
+      ...(Array.isArray(file.failureDetails) && file.failureDetails.length > 0
+        ? { failureDetails: file.failureDetails }
+        : {}),
       ...(Array.isArray(file.artifacts) && file.artifacts.length > 0
         ? { artifacts: file.artifacts }
         : {}),

package/lib/runner/triage.mjs

@@ -0,0 +1,330 @@
+import fs from "fs";
+import path from "path";
+
+const CLASSIFICATIONS = new Set([
+  "expected_failure",
+  "infra",
+  "product_bug",
+  "stale_test",
+  "test_bug",
+]);
+const STATES = new Set(["closed", "open"]);
+
+export function loadKnownFailuresConfig(productDir, config) {
+  const relativePath = config?.knownFailuresFile;
+  if (!relativePath) return null;
+
+  const absolutePath = path.resolve(productDir, relativePath);
+  if (!fs.existsSync(absolutePath)) {
+    throw new Error(`Known failures file not found: ${relativePath}`);
+  }
+
+  let parsed;
+  try {
+    parsed = JSON.parse(fs.readFileSync(absolutePath, "utf8"));
+  } catch (error) {
+    throw new Error(
+      `Could not parse known failures file ${relativePath}: ${formatErrorMessage(error)}`
+    );
+  }
+
+  return normalizeKnownFailuresDocument(parsed, relativePath);
+}
+
+export function applyKnownFailuresToArtifacts(runArtifact, statusArtifact, knownFailures) {
+  if (!knownFailures) return { runArtifact, statusArtifact };
+
+  const runEntries = extractRunFileEntries(runArtifact);
+  const statusEntries = extractStatusFileEntries(statusArtifact);
+  const fileSummaries = new Map();
+
+  for (const entry of [...runEntries, ...statusEntries]) {
+    const key = buildFileIdentity(entry.service, entry.type, entry.path);
+    if (!fileSummaries.has(key)) {
+      fileSummaries.set(key, {
+        service: entry.service,
+        type: entry.type,
+        path: entry.path,
+        status: entry.status,
+        error: entry.error || null,
+        failureDetails: Array.isArray(entry.failureDetails) ? entry.failureDetails : [],
+      });
+    }
+  }
+
+  const matchesByFileKey = new Map();
+  const matchedByFailedEntryIds = new Set();
+
+  for (const entry of knownFailures.entries) {
+    for (const fileSummary of fileSummaries.values()) {
+      if (!matchesKnownFailureEntry(entry, fileSummary)) continue;
+      const fileKey = buildFileIdentity(fileSummary.service, fileSummary.type, fileSummary.path);
+      if (!matchesByFileKey.has(fileKey)) {
+        matchesByFileKey.set(fileKey, []);
+      }
+      matchesByFileKey.get(fileKey).push(toArtifactTriageEntry(entry));
+      if (fileSummary.status === "failed") {
+        matchedByFailedEntryIds.add(entry.id);
+      }
+    }
+  }
+
+  for (const entry of [...runEntries, ...statusEntries]) {
+    const fileKey = buildFileIdentity(entry.service, entry.type, entry.path);
+    const matches = matchesByFileKey.get(fileKey) || [];
+    if (matches.length === 0) {
+      if (entry.status === "failed") {
+        setEntryTriage(entry, {
+          status: "untriaged",
+          entries: [],
+        });
+      }
+      continue;
+    }
+
+    setEntryTriage(entry, {
+      status: entry.status === "failed" ? "known_failure" : "known_issue_not_reproduced",
+      classifications: [...new Set(matches.map((match) => match.classification))].sort(),
+      entries: matches,
+    });
+  }
+
+  const summaryTests = statusArtifact?.tests || runEntries;
+  const triageSummary = buildTriageSummary(
+    summaryTests,
+    knownFailures.entries,
+    matchedByFailedEntryIds
+  );
+  runArtifact.triageSummary = triageSummary;
+  if (statusArtifact) {
+    statusArtifact.triageSummary = triageSummary;
+  }
+  return { runArtifact, statusArtifact };
+}
+
+export function normalizeKnownFailuresDocument(document, relativePath = "known failures file") {
+  if (!document || typeof document !== "object") {
+    throw new Error(`${relativePath} must contain a JSON object`);
+  }
+  if (document.schemaVersion !== 1) {
+    throw new Error(`${relativePath} schemaVersion must be 1`);
+  }
+  if (!Array.isArray(document.entries)) {
+    throw new Error(`${relativePath} entries must be an array`);
+  }
+
+  const ids = new Set();
+  const entries = document.entries.map((entry, index) => {
+    const normalized = normalizeKnownFailureEntry(entry, `${relativePath} entries[${index}]`);
+    if (ids.has(normalized.id)) {
+      throw new Error(`${relativePath} has duplicate entry id "${normalized.id}"`);
+    }
+    ids.add(normalized.id);
+    return normalized;
+  });
+
+  return {
+    schemaVersion: 1,
+    issueRepo: normalizeOptionalString(document.issueRepo),
+    entries,
+  };
+}
+
+function normalizeKnownFailureEntry(entry, label) {
+  if (!entry || typeof entry !== "object") {
+    throw new Error(`${label} must be an object`);
+  }
+
+  const id = requireNonEmptyString(entry.id, `${label}.id`);
+  const title = requireNonEmptyString(entry.title, `${label}.title`);
+  const classification = requireEnumValue(
+    entry.classification,
+    CLASSIFICATIONS,
+    `${label}.classification`
+  );
+  const state = requireEnumValue(entry.state, STATES, `${label}.state`);
+  const description = requireNonEmptyString(entry.description, `${label}.description`);
+  const whyFailing = requireNonEmptyString(entry.whyFailing, `${label}.whyFailing`);
+  const lastReviewedAt = requireNonEmptyString(entry.lastReviewedAt, `${label}.lastReviewedAt`);
+  if (!Array.isArray(entry.matches) || entry.matches.length === 0) {
+    throw new Error(`${label}.matches must be a non-empty array`);
+  }
+
+  return {
+    id,
+    title,
+    classification,
+    state,
+    issue: normalizeKnownFailureIssue(entry.issue, `${label}.issue`),
+    description,
+    whyFailing,
+    lastReviewedAt,
+    matches: entry.matches.map((match, index) =>
+      normalizeKnownFailureMatch(match, `${label}.matches[${index}]`)
+    ),
+  };
+}
+
+function normalizeKnownFailureIssue(issue, label) {
+  if (!issue || typeof issue !== "object") {
+    throw new Error(`${label} must be an object`);
+  }
+
+  const repo = requireNonEmptyString(issue.repo, `${label}.repo`);
+  const number = issue.number;
+  if (!Number.isInteger(number) || number <= 0) {
+    throw new Error(`${label}.number must be a positive integer`);
+  }
+  const url = requireNonEmptyString(issue.url, `${label}.url`);
+
+  return { repo, number, url };
+}
+
+function normalizeKnownFailureMatch(match, label) {
+  if (!match || typeof match !== "object") {
+    throw new Error(`${label} must be an object`);
+  }
+
+  const pathValue = requireNonEmptyString(match.path, `${label}.path`);
+  const normalized = {
+    path: pathValue,
+  };
+
+  const service = normalizeOptionalString(match.service);
+  if (service) normalized.service = service;
+
+  const type = normalizeOptionalString(match.type);
+  if (type) normalized.type = type;
+
+  const failureKey = normalizeOptionalString(match.failureKey);
+  if (failureKey) normalized.failureKey = failureKey;
+
+  const errorIncludes = normalizeOptionalString(match.errorIncludes);
+  if (errorIncludes) normalized.errorIncludes = errorIncludes;
+
+  return normalized;
+}
+
+function matchesKnownFailureEntry(entry, fileSummary) {
+  return entry.matches.some((match) => matchesKnownFailureMatch(match, fileSummary));
+}
+
+function matchesKnownFailureMatch(match, fileSummary) {
+  if (match.service && match.service !== fileSummary.service) return false;
+  if (match.type && match.type !== fileSummary.type) return false;
+  if (match.path !== fileSummary.path) return false;
+  if (match.failureKey) {
+    const failureKeys = Array.isArray(fileSummary.failureDetails)
+      ? fileSummary.failureDetails.map((detail) => detail.key)
+      : [];
+    if (!failureKeys.includes(match.failureKey)) return false;
+  }
+  if (match.errorIncludes && !String(fileSummary.error || "").includes(match.errorIncludes)) {
+    return false;
+  }
+  return true;
+}
+
+function toArtifactTriageEntry(entry) {
+  return {
+    id: entry.id,
+    title: entry.title,
+    classification: entry.classification,
+    state: entry.state,
+    issue: entry.issue,
+    description: entry.description,
+    whyFailing: entry.whyFailing,
+    lastReviewedAt: entry.lastReviewedAt,
+  };
+}
+
+function buildTriageSummary(tests, entries, matchedEntryIds) {
+  const failedTests = tests.filter((test) => test.status === "failed");
+  const knownFailedTests = failedTests.filter((test) => test.triage?.status === "known_failure");
+  const byClassification = {};
+
+  for (const test of knownFailedTests) {
+    for (const classification of test.triage?.classifications || []) {
+      byClassification[classification] = (byClassification[classification] || 0) + 1;
+    }
+  }
+
+  return {
+    failed: {
+      total: failedTests.length,
+      known: knownFailedTests.length,
+      untriaged: failedTests.length - knownFailedTests.length,
+      byClassification,
+    },
+    entries: {
+      total: entries.length,
+      matchedByFailedTests: matchedEntryIds.size,
+      unmatched: entries.length - matchedEntryIds.size,
+    },
+  };
+}
+
+function extractRunFileEntries(runArtifact) {
+  const entries = [];
+
+  for (const service of runArtifact.services || []) {
+    for (const suite of service.suites || []) {
+      for (const file of suite.files || []) {
+        entries.push({
+          target: file,
+          service: service.name,
+          type: suite.type,
+          path: file.path,
+          status: file.status,
+          error: file.error || null,
+          failureDetails: Array.isArray(file.failureDetails) ? file.failureDetails : [],
+        });
+      }
+    }
+  }
+
+  return entries;
+}
+
+function extractStatusFileEntries(statusArtifact) {
+  return statusArtifact?.tests || [];
+}
+
+function setEntryTriage(entry, triage) {
+  if (entry?.target) {
+    entry.target.triage = triage;
+    return;
+  }
+  entry.triage = triage;
+}
+
+function buildFileIdentity(service, type, filePath) {
+  return `${service}::${type}::${filePath}`;
+}
+
+function requireNonEmptyString(value, label) {
+  const normalized = normalizeOptionalString(value);
+  if (!normalized) {
+    throw new Error(`${label} must be a non-empty string`);
+  }
+  return normalized;
+}
+
+function requireEnumValue(value, allowed, label) {
+  const normalized = requireNonEmptyString(value, label);
+  if (!allowed.has(normalized)) {
+    throw new Error(`${label} must be one of: ${[...allowed].sort().join(", ")}`);
+  }
+  return normalized;
+}
+
+function normalizeOptionalString(value) {
+  if (typeof value !== "string") return null;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : null;
+}
+
+function formatErrorMessage(error) {
+  if (error instanceof Error) return error.message;
+  return String(error);
+}

package/lib/runner/triage.test.mjs

@@ -0,0 +1,156 @@
+import { describe, expect, it } from "vitest";
+import {
+  applyKnownFailuresToArtifacts,
+  normalizeKnownFailuresDocument,
+} from "./triage.mjs";
+
+describe("runner triage", () => {
+  it("matches exact failure keys and enriches both artifacts", () => {
+    const knownFailures = normalizeKnownFailuresDocument({
+      schemaVersion: 1,
+      issueRepo: "acme/repo",
+      entries: [
+        {
+          id: "bad-message",
+          title: "Bad message bug",
+          classification: "product_bug",
+          state: "open",
+          issue: {
+            repo: "acme/repo",
+            number: 12,
+            url: "https://github.com/acme/repo/issues/12",
+          },
+          description: "The API returns the wrong message.",
+          whyFailing: "The endpoint payload is wrong.",
+          lastReviewedAt: "2026-04-27",
+          matches: [
+            {
+              service: "api",
+              type: "int",
+              path: "__testkit__/http/failing.int.testkit.ts",
+              failureKey: "returns the wrong message",
+            },
+          ],
+        },
+      ],
+    });
+
+    const runArtifact = {
+      services: [
+        {
+          name: "api",
+          suites: [
+            {
+              type: "int",
+              files: [
+                {
+                  path: "__testkit__/http/failing.int.testkit.ts",
+                  status: "failed",
+                  error: "Default runtime thresholds failed: checks(rate==1.0)",
+                  failureDetails: [
+                    {
+                      kind: "k6-check",
+                      key: "returns the wrong message",
+                      title: "returns the wrong message",
+                      count: 1,
+                    },
+                  ],
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    };
+    const statusArtifact = {
+      tests: [
+        {
+          service: "api",
+          type: "int",
+          path: "__testkit__/http/failing.int.testkit.ts",
+          status: "failed",
+          error: "Default runtime thresholds failed: checks(rate==1.0)",
+          failureDetails: [
+            {
+              kind: "k6-check",
+              key: "returns the wrong message",
+              title: "returns the wrong message",
+              count: 1,
+            },
+          ],
+        },
+      ],
+    };
+
+    const enriched = applyKnownFailuresToArtifacts(runArtifact, statusArtifact, knownFailures);
+    expect(enriched.statusArtifact.tests[0].triage).toMatchObject({
+      status: "known_failure",
+      classifications: ["product_bug"],
+    });
+    expect(enriched.runArtifact.services[0].suites[0].files[0].triage.entries[0]).toMatchObject({
+      id: "bad-message",
+      issue: {
+        number: 12,
+      },
+    });
+    expect(enriched.statusArtifact.triageSummary).toEqual({
+      failed: {
+        total: 1,
+        known: 1,
+        untriaged: 0,
+        byClassification: {
+          product_bug: 1,
+        },
+      },
+      entries: {
+        total: 1,
+        matchedByFailedTests: 1,
+        unmatched: 0,
+      },
+    });
+  });
+
+  it("marks unmatched failed tests as untriaged", () => {
+    const enriched = applyKnownFailuresToArtifacts(
+      {
+        services: [
+          {
+            name: "api",
+            suites: [
+              {
+                type: "int",
+                files: [
+                  {
+                    path: "__testkit__/http/failing.int.testkit.ts",
+                    status: "failed",
+                    error: "boom",
+                  },
+                ],
+              },
+            ],
+          },
+        ],
+      },
+      {
+        tests: [
+          {
+            service: "api",
+            type: "int",
+            path: "__testkit__/http/failing.int.testkit.ts",
+            status: "failed",
+            error: "boom",
+          },
+        ],
+      },
+      normalizeKnownFailuresDocument({
+        schemaVersion: 1,
+        entries: [],
+      })
+    );
+
+    expect(enriched.statusArtifact.tests[0].triage).toEqual({
+      status: "untriaged",
+      entries: [],
+    });
+  });
+});

package/lib/runtime/index.mjs

@@ -1,6 +1,6 @@
 import rawHttp from "k6/http";
 import { Rate, Trend } from "k6/metrics";
-import { check, fail, group, sleep } from "k6";
+import { fail, sleep } from "k6";
 import {
   formatWaitForTimeoutError,
   normalizeWaitIntervalSeconds,
@@ -8,6 +8,7 @@ import {
   remainingFileTimeoutMs,
   remainingFileTimeoutSeconds,
 } from "../shared/file-timeout.mjs";
+import { check, group } from "../runtime-src/k6/checks.js";
 
 export { check, fail, group, sleep };
 export { Rate, Trend };

package/lib/runtime-src/k6/checks.js

@@ -1,6 +1,9 @@
+import { check as k6Check, group as k6Group } from "k6";
 import { Rate } from "k6/metrics";
+import { emitArtifact } from "./artifacts.js";
 
 export const runtimeFailures = new Rate("testkit_runtime_failures");
+const failureState = createFailureState();
 
 export function singleIterationOptions(overrides = {}) {
   return {
@@ -16,6 +19,41 @@ export function singleIterationOptions(overrides = {}) {
 
 export const defaultOptions = singleIterationOptions();
 
+export function check(value, checks) {
+  let allPassed = true;
+
+  for (const [name, predicate] of Object.entries(checks || {})) {
+    const checkName = normalizeLabel(name, "unnamed check");
+    const passed = k6Check(value, { [checkName]: predicate });
+    if (!passed) {
+      recordFailureDetail({
+        kind: "k6-check",
+        key: buildFailureKey(failureState.groupStack, checkName),
+        title: checkName,
+        checkName,
+        groupPath: [...failureState.groupStack],
+        phase: failureState.phase,
+      });
+      allPassed = false;
+    }
+  }
+
+  return allPassed;
+}
+
+export function group(name, fn) {
+  const groupName = normalizeLabel(name, "unnamed group");
+
+  return k6Group(groupName, () => {
+    failureState.groupStack.push(groupName);
+    try {
+      return fn();
+    } finally {
+      failureState.groupStack.pop();
+    }
+  });
+}
+
 export function json(res) {
   return JSON.parse(res.body);
 }
@@ -46,3 +84,95 @@ export function isSorted(rows, field, direction = "asc") {
 export function recordRuntimeFailure() {
   runtimeFailures.add(1);
 }
+
+export function startFailureCollection(phase) {
+  failureState.phase = normalizeLabel(phase, "exec");
+  failureState.groupStack = [];
+  failureState.detailsByKey = new Map();
+}
+
+export function emitFailureCollectionArtifact() {
+  const failures = [...failureState.detailsByKey.values()]
+    .sort((left, right) => left.key.localeCompare(right.key))
+    .map((detail) => ({ ...detail }));
+
+  if (failures.length > 0) {
+    emitArtifact(
+      "failure-details",
+      {
+        phase: failureState.phase,
+        failures,
+      },
+      {
+        kind: "testkit.failure-details",
+        summary: `${failures.length} failure detail(s)`,
+      }
+    );
+  }
+
+  startFailureCollection(failureState.phase);
+}
+
+export function recordFailureDetail(detail) {
+  const normalized = normalizeFailureDetail(detail);
+  if (!normalized) return;
+
+  const existing = failureState.detailsByKey.get(normalized.key);
+  if (!existing) {
+    failureState.detailsByKey.set(normalized.key, normalized);
+    return;
+  }
+
+  existing.count += normalized.count;
+}
+
+function createFailureState() {
+  return {
+    phase: "exec",
+    groupStack: [],
+    detailsByKey: new Map(),
+  };
+}
+
+function normalizeFailureDetail(detail) {
+  if (!detail || typeof detail !== "object") return null;
+
+  const kind = normalizeLabel(detail.kind, null);
+  const key = normalizeLabel(detail.key, null);
+  const title = normalizeLabel(detail.title, null);
+  if (!kind || !key || !title) return null;
+
+  const normalized = {
+    kind,
+    key,
+    title,
+    count: 1,
+  };
+
+  const checkName = normalizeLabel(detail.checkName, null);
+  if (checkName) normalized.checkName = checkName;
+
+  const phase = normalizeLabel(detail.phase, null);
+  if (phase) normalized.phase = phase;
+
+  const message = normalizeLabel(detail.message, null);
+  if (message) normalized.message = message;
+
+  const groupPath = Array.isArray(detail.groupPath)
+    ? detail.groupPath.map((entry) => normalizeLabel(entry, null)).filter(Boolean)
+    : [];
+  if (groupPath.length > 0) normalized.groupPath = groupPath;
+
+  return normalized;
+}
+
+function buildFailureKey(groupPath, title) {
+  if (!Array.isArray(groupPath) || groupPath.length === 0) return title;
+  return [...groupPath, title].join(" > ");
+}
+
+function normalizeLabel(value, fallback) {
+  if (typeof value !== "string") return fallback;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : fallback;
+}

package/lib/runtime-src/k6/dal-suite.js

@@ -1,5 +1,10 @@
 import { fail } from "k6";
-import { defaultOptions, recordRuntimeFailure } from "./checks.js";
+import {
+  defaultOptions,
+  emitFailureCollectionArtifact,
+  recordRuntimeFailure,
+  startFailureCollection,
+} from "./checks.js";
 import { createDalContext, openDb } from "./dal.js";
 
 export function defineDalSuite(configOrRun, maybeRun) {
@@ -11,14 +16,18 @@ export function defineDalSuite(configOrRun, maybeRun) {
     options: config.options || defaultOptions,
     setup() {
       if (typeof config.setup !== "function") return null;
+      startFailureCollection("setup");
       try {
         return config.setup({ db, dal });
       } catch (error) {
         recordRuntimeFailure();
         fail(formatFatalSuiteError("setup", error));
+      } finally {
+        emitFailureCollectionArtifact();
       }
     },
     exec(setupData) {
+      startFailureCollection("exec");
       try {
         return run({
           db,
@@ -28,6 +37,8 @@ export function defineDalSuite(configOrRun, maybeRun) {
       } catch (error) {
         recordRuntimeFailure();
         fail(formatFatalSuiteError("exec", error));
+      } finally {
+        emitFailureCollectionArtifact();
       }
     },
   };

package/lib/runtime-src/k6/suite.js

@@ -1,5 +1,10 @@
 import { fail } from "k6";
-import { defaultOptions, recordRuntimeFailure } from "./checks.js";
+import {
+  defaultOptions,
+  emitFailureCollectionArtifact,
+  recordRuntimeFailure,
+  startFailureCollection,
+} from "./checks.js";
 import { createHttpClient, getEnv } from "./http.js";
 import {
   clearRuntimeContext,
@@ -16,6 +21,7 @@ export function defineHttpSuite(configOrRun, maybeRun) {
     },
     setup() {
       const resolved = resolveRuntimeConfig(config);
+      startFailureCollection("setup");
       try {
         registerRuntimeContext({ env: resolved.env, http: resolved.client.rawHttp || null });
         if (typeof resolved.auth?.setup !== "function") return null;
@@ -24,11 +30,13 @@ export function defineHttpSuite(configOrRun, maybeRun) {
         recordRuntimeFailure();
         fail(formatFatalSuiteError("setup", error));
       } finally {
+        emitFailureCollectionArtifact();
         clearRuntimeContext();
       }
     },
     exec(setupData) {
       const resolved = resolveRuntimeConfig(config);
+      startFailureCollection("exec");
       try {
         registerRuntimeContext({ env: resolved.env, http: resolved.client.rawHttp || null });
         return run({
@@ -43,6 +51,7 @@ export function defineHttpSuite(configOrRun, maybeRun) {
         recordRuntimeFailure();
         fail(formatFatalSuiteError("exec", error));
       } finally {
+        emitFailureCollectionArtifact();
         clearRuntimeContext();
       }
     },

package/lib/setup/index.d.ts

@@ -90,6 +90,9 @@ export interface TestkitSetup {
   profiles?: {
     http?: Record<string, HttpSuiteConfig>;
   };
+  reporting?: {
+    knownFailuresFile?: string;
+  };
   services?: Record<string, ServiceConfig>;
   telemetry?: {
     enabled?: boolean;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/testkit",
-  "version": "0.1.42",
+  "version": "0.1.43",
   "description": "CLI for discovering and running local HTTP, DAL, and Playwright test suites",
   "type": "module",
   "types": "./lib/index.d.ts",