@elench/testkit 0.1.42 → 0.1.44

package/lib/runner/failure-details.mjs

@@ -0,0 +1,91 @@
+export function normalizeFailureDetail(detail) {
+  if (!detail || typeof detail !== "object") return null;
+
+  const kind = normalizeNonEmptyString(detail.kind);
+  const key = normalizeNonEmptyString(detail.key);
+  const title = normalizeNonEmptyString(detail.title);
+  if (!kind || !key || !title) return null;
+
+  const normalized = {
+    kind,
+    key,
+    title,
+    count: normalizePositiveInteger(detail.count) || 1,
+  };
+
+  const groupPath = normalizeStringArray(detail.groupPath);
+  if (groupPath.length > 0) normalized.groupPath = groupPath;
+
+  const checkName = normalizeNonEmptyString(detail.checkName);
+  if (checkName) normalized.checkName = checkName;
+
+  const suitePath = normalizeStringArray(detail.suitePath);
+  if (suitePath.length > 0) normalized.suitePath = suitePath;
+
+  const phase = normalizeNonEmptyString(detail.phase);
+  if (phase) normalized.phase = phase;
+
+  const message = normalizeNonEmptyString(detail.message);
+  if (message) normalized.message = message;
+
+  return normalized;
+}
+
+export function mergeFailureDetails(details) {
+  const mergedByKey = new Map();
+
+  for (const detail of Array.isArray(details) ? details : []) {
+    const normalized = normalizeFailureDetail(detail);
+    if (!normalized) continue;
+    const dedupeKey = `${normalized.kind}::${normalized.key}`;
+    const existing = mergedByKey.get(dedupeKey);
+    if (!existing) {
+      mergedByKey.set(dedupeKey, { ...normalized });
+      continue;
+    }
+    existing.count += normalized.count;
+    if (!existing.message && normalized.message) {
+      existing.message = normalized.message;
+    }
+  }
+
+  return [...mergedByKey.values()].sort(
+    (left, right) =>
+      left.kind.localeCompare(right.kind) || left.key.localeCompare(right.key)
+  );
+}
+
+export function collectFailureDetailsFromRuntimeArtifacts(artifacts) {
+  const details = [];
+
+  for (const artifact of Array.isArray(artifacts) ? artifacts : []) {
+    if (artifact?.kind !== "testkit.failure-details") continue;
+    const phase = normalizeNonEmptyString(artifact?.data?.phase);
+    for (const detail of Array.isArray(artifact?.data?.failures) ? artifact.data.failures : []) {
+      details.push({
+        ...detail,
+        phase: detail?.phase || phase || null,
+      });
+    }
+  }
+
+  return mergeFailureDetails(details);
+}
+
+function normalizeStringArray(value) {
+  if (!Array.isArray(value)) return [];
+  return value
+    .map((entry) => normalizeNonEmptyString(entry))
+    .filter(Boolean);
+}
+
+function normalizeNonEmptyString(value) {
+  if (typeof value !== "string") return null;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : null;
+}
+
+function normalizePositiveInteger(value) {
+  if (!Number.isInteger(value) || value <= 0) return null;
+  return value;
+}

package/lib/runner/failure-details.test.mjs

@@ -0,0 +1,63 @@
+import { describe, expect, it } from "vitest";
+import {
+  collectFailureDetailsFromRuntimeArtifacts,
+  mergeFailureDetails,
+} from "./failure-details.mjs";
+
+describe("runner failure details", () => {
+  it("merges duplicate failure details by kind and key", () => {
+    expect(
+      mergeFailureDetails([
+        { kind: "k6-check", key: "group > check", title: "check" },
+        { kind: "k6-check", key: "group > check", title: "check" },
+        { kind: "playwright-spec", key: "spec title", title: "spec title" },
+      ])
+    ).toEqual([
+      {
+        kind: "k6-check",
+        key: "group > check",
+        title: "check",
+        count: 2,
+      },
+      {
+        kind: "playwright-spec",
+        key: "spec title",
+        title: "spec title",
+        count: 1,
+      },
+    ]);
+  });
+
+  it("extracts and normalizes runtime failure-detail artifacts", () => {
+    expect(
+      collectFailureDetailsFromRuntimeArtifacts([
+        {
+          kind: "testkit.failure-details",
+          data: {
+            phase: "exec",
+            failures: [
+              {
+                kind: "k6-check",
+                key: "status is 200",
+                title: "status is 200",
+              },
+              {
+                kind: "k6-check",
+                key: "status is 200",
+                title: "status is 200",
+              },
+            ],
+          },
+        },
+      ])
+    ).toEqual([
+      {
+        kind: "k6-check",
+        key: "status is 200",
+        title: "status is 200",
+        count: 2,
+        phase: "exec",
+      },
+    ]);
+  });
+});

package/lib/runner/formatting.mjs

@@ -1,3 +1,5 @@
+import { buildKnownFailureIssueValidationSummaryLines } from "../known-failures/github.mjs";
+
 export function formatDuration(durationMs) {
   const totalSeconds = Math.max(0, Math.round(durationMs / 1000));
   const minutes = Math.floor(totalSeconds / 60);
@@ -46,7 +48,7 @@ export function formatSuiteFramework(framework) {
   return label ? ` [${label}]` : "";
 }
 
-export function buildRunSummaryLines(results, durationMs) {
+export function buildRunSummaryLines(results, durationMs, knownFailureIssueValidation = null) {
   const totalServices = results.length;
   const executedServices = results.filter((result) => !result.skipped);
   const skippedServices = results.filter((result) => result.skipped);
@@ -115,6 +117,13 @@ export function buildRunSummaryLines(results, durationMs) {
     }
   }
 
+  const knownFailureIssueLines = buildKnownFailureIssueValidationSummaryLines(
+    knownFailureIssueValidation
+  );
+  if (knownFailureIssueLines.length > 0) {
+    lines.push(...knownFailureIssueLines);
+  }
+
   if (failedServices.length > 0) {
     lines.push("", `Result: FAILED (${failedServices.length}/${totalServices} services failed)`);
     return lines;

package/lib/runner/formatting.test.mjs

@@ -128,4 +128,40 @@ describe("runner formatting", () => {
     expect(lines.join("\n")).toContain("SKIP api");
     expect(lines.at(-1)).toBe("Result: PASSED");
   });
+
+  it("appends known-failure issue validation summary lines", () => {
+    const lines = buildRunSummaryLines(
+      [
+        {
+          name: "api",
+          skipped: false,
+          failed: false,
+          suiteCount: 1,
+          completedSuiteCount: 1,
+          skippedSuiteCount: 0,
+          failedSuiteCount: 0,
+          totalFileCount: 1,
+          passedFileCount: 1,
+          skippedFileCount: 0,
+          notRunFileCount: 0,
+          durationMs: 1_000,
+          suites: [],
+          errors: [],
+        },
+      ],
+      1_000,
+      {
+        summary: {
+          byCode: {
+            closed_but_failing: 2,
+            title_mismatch: 1,
+          },
+        },
+      }
+    );
+
+    expect(lines.join("\n")).toContain("Known-failure issues:");
+    expect(lines.join("\n")).toContain("2 closed issues still failing");
+    expect(lines.join("\n")).toContain("1 title mismatch");
+  });
 });

package/lib/runner/metadata.mjs

@@ -3,6 +3,7 @@ import os from "os";
 import path from "path";
 import { execFileSync } from "child_process";
 import { fileURLToPath } from "url";
+import { parseGitHubRepoSlug } from "../known-failures/github.mjs";
 
 export function collectGitMetadata(productDir) {
   const read = (args) => {
@@ -13,10 +14,14 @@ export function collectGitMetadata(productDir) {
     }
   };
 
+  const remoteUrl = read(["remote", "get-url", "origin"]);
+
   return {
     branch: read(["rev-parse", "--abbrev-ref", "HEAD"]),
     commitSha: read(["rev-parse", "HEAD"]),
     repoRoot: read(["rev-parse", "--show-toplevel"]),
+    remoteUrl,
+    repoSlug: parseGitHubRepoSlug(remoteUrl),
   };
 }
 

package/lib/runner/orchestrator.mjs

@@ -15,7 +15,12 @@ import {
   summarizeDbBackend,
 } from "./results.mjs";
 import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
+import { applyKnownFailuresToArtifacts, loadKnownFailuresConfig } from "./triage.mjs";
 import { buildRunSummaryLines, formatError } from "./formatting.mjs";
+import {
+  shouldFailKnownFailureIssueValidation,
+  validateKnownFailureIssues,
+} from "../known-failures/github.mjs";
 import {
   loadTimings,
   resetResultArtifacts,
@@ -56,6 +61,10 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     },
     testkitVersion: readPackageMetadata().version,
   };
+  const knownFailures = loadKnownFailuresConfig(
+    productDir,
+    configs[0]?.testkit?.reporting || null
+  );
   const requestedFiles = opts.fileNames || [];
   if (requestedFiles.length > 0) {
     const unmatchedFiles = findUnmatchedRequestedFiles(
@@ -113,6 +122,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
   lifecycle.installSignalHandlers();
   let results = [];
   let finishedAt = Date.now();
+  let knownFailureIssueValidation = null;
 
   try {
     if (executedPlans.length > 0) {
@@ -168,7 +178,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     results = configs.map((config) =>
       finalizeServiceResult(trackers.get(config.name), startedAt, finishedAt)
     );
-    const artifact = buildRunArtifact({
+    const runArtifact = buildRunArtifact({
       productDir,
       results,
       startedAt,
@@ -185,12 +195,8 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
       metadata,
       summarizeDbBackend,
     });
-
-    writeRunArtifact(productDir, artifact);
-    if (opts.writeStatus) {
-      writeStatusArtifact(
-        productDir,
-        buildStatusArtifact({
+    const statusArtifact = opts.writeStatus
+      ? buildStatusArtifact({
           productDir,
           results,
           typeValues,
@@ -200,12 +206,37 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
           serviceFilter: opts.serviceFilter || null,
           metadata,
         })
-      );
+      : null;
+    const enrichedArtifacts = applyKnownFailuresToArtifacts(
+      runArtifact,
+      statusArtifact,
+      knownFailures
+    );
+    knownFailureIssueValidation = await validateKnownFailureIssues({
+      productDir,
+      document: knownFailures,
+      runArtifact: enrichedArtifacts.runArtifact,
+      statusArtifact: enrichedArtifacts.statusArtifact,
+      config: configs[0]?.testkit?.reporting?.issueValidation || null,
+      gitMetadata: metadata.git,
+    });
+    attachKnownFailureIssueValidation(
+      enrichedArtifacts.runArtifact,
+      enrichedArtifacts.statusArtifact,
+      knownFailureIssueValidation
+    );
+
+    writeRunArtifact(productDir, enrichedArtifacts.runArtifact);
+    if (opts.writeStatus) {
+      writeStatusArtifact(productDir, enrichedArtifacts.statusArtifact);
     }
 
-    printRunSummary(results, finishedAt - startedAt);
-    await reportTelemetry(telemetry, artifact);
+    printRunSummary(results, finishedAt - startedAt, knownFailureIssueValidation);
+    await reportTelemetry(telemetry, enrichedArtifacts.runArtifact);
     if (results.some((result) => result.failed)) exitCode = 1;
+    if (shouldFailKnownFailureIssueValidation(knownFailureIssueValidation)) {
+      exitCode = 1;
+    }
     if (lifecycle.isStopRequested()) exitCode = Math.max(exitCode, 130);
   } finally {
     if (lifecycle.isStopRequested()) {
@@ -259,8 +290,8 @@ function collectServicePlans(configs, configMap, typeValues, suiteSelectors, opt
   });
 }
 
-function printRunSummary(results, durationMs) {
-  for (const line of buildRunSummaryLines(results, durationMs)) {
+function printRunSummary(results, durationMs, knownFailureIssueValidation = null) {
+  for (const line of buildRunSummaryLines(results, durationMs, knownFailureIssueValidation)) {
     console.log(line);
   }
 }
@@ -289,3 +320,11 @@ async function reportTelemetry(telemetry, artifact) {
 function normalizePathSeparators(filePath) {
   return filePath.split("\\").join("/");
 }
+
+function attachKnownFailureIssueValidation(runArtifact, statusArtifact, validation) {
+  if (!validation) return;
+  runArtifact.knownFailuresIssueValidation = validation;
+  if (statusArtifact) {
+    statusArtifact.knownFailuresIssueValidation = validation;
+  }
+}

package/lib/runner/playwright-runner.mjs

@@ -76,5 +76,6 @@ export async function runPlaywrightTask(targetConfig, task, lifecycle, lease) {
     durationMs: fileResult?.durationMs > 0 ? fileResult.durationMs : durationMs,
     startedAt,
     finishedAt,
+    failureDetails: timedOut ? [] : fileResult?.failureDetails || [],
   };
 }

package/lib/runner/reporting.mjs

@@ -23,9 +23,15 @@ export function buildStatusArtifact({
           path: file.path,
           status: file.status,
         };
+        if (file.error) {
+          test.error = file.error;
+        }
         if (file.reason) {
           test.reason = file.reason;
         }
+        if (Array.isArray(file.failureDetails) && file.failureDetails.length > 0) {
+          test.failureDetails = file.failureDetails;
+        }
         tests.push(test);
       }
     }
@@ -72,7 +78,7 @@ export function buildStatusArtifact({
     scope.serviceFilter === null;
 
   return {
-    schemaVersion: 3,
+    schemaVersion: 5,
     source: "testkit",
     notice: "Generated file. Do not edit manually.",
     product: {
@@ -121,7 +127,7 @@ export function buildRunArtifact({
   const dbBackend = summarizeDbBackend(results);
 
   return {
-    schemaVersion: 3,
+    schemaVersion: 5,
     source: "testkit",
     generatedAt: new Date(finishedAt).toISOString(),
     product: {

package/lib/runner/reporting.test.mjs

@@ -78,7 +78,7 @@ describe("runner reporting", () => {
     });
 
     expect(artifact.product.name).toBe("my-product");
-    expect(artifact.schemaVersion).toBe(3);
+    expect(artifact.schemaVersion).toBe(5);
     expect(artifact.run).toMatchObject({
       workers: 2,
       fileTimeoutSeconds: 60,
@@ -149,7 +149,7 @@ describe("runner reporting", () => {
     });
 
     expect(status).toEqual({
-      schemaVersion: 3,
+      schemaVersion: 5,
       source: "testkit",
       notice: "Generated file. Do not edit manually.",
       product: {

package/lib/runner/results.mjs

@@ -1,4 +1,5 @@
 import path from "path";
+import { mergeFailureDetails } from "./failure-details.mjs";
 
 export function buildServiceTrackers(servicePlans, startedAt) {
   const trackers = new Map();
@@ -46,6 +47,7 @@ export function buildServiceTrackers(servicePlans, startedAt) {
               reason: null,
               status: "not_run",
               artifacts: [],
+              failureDetails: [],
             },
           ];
         }),
@@ -59,6 +61,7 @@ export function buildServiceTrackers(servicePlans, startedAt) {
             reason: file.reason,
             status: "skipped",
             artifacts: [],
+            failureDetails: [],
           },
         ]),
       ]),
@@ -121,6 +124,7 @@ export function recordTaskOutcome(trackers, task, outcome, finishedAt = Date.now
     existingFileResult.reason = outcome.reason || null;
     existingFileResult.status = status;
     existingFileResult.artifacts = Array.isArray(outcome.artifacts) ? outcome.artifacts : [];
+    existingFileResult.failureDetails = mergeFailureDetails(outcome.failureDetails);
   } else {
     suite.fileResultsByPath.set(normalizedPath, {
       path: normalizedPath,
@@ -130,6 +134,7 @@ export function recordTaskOutcome(trackers, task, outcome, finishedAt = Date.now
       reason: outcome.reason || null,
       status,
       artifacts: Array.isArray(outcome.artifacts) ? outcome.artifacts : [],
+      failureDetails: mergeFailureDetails(outcome.failureDetails),
     });
   }
   if (status === "failed" && !suite.failedFileSet.has(task.file)) {
@@ -247,6 +252,9 @@ function finalizeSuite(suite) {
       durationMs: file.durationMs,
       error: file.error,
       reason: file.reason,
+      ...(Array.isArray(file.failureDetails) && file.failureDetails.length > 0
+        ? { failureDetails: file.failureDetails }
+        : {}),
       ...(Array.isArray(file.artifacts) && file.artifacts.length > 0
         ? { artifacts: file.artifacts }
         : {}),

package/lib/runner/triage.mjs

@@ -0,0 +1,154 @@
+import {
+  buildKnownFailureFileIdentity,
+  findMatchingKnownFailureEntries,
+  loadKnownFailuresConfig,
+} from "../known-failures/index.mjs";
+
+export { loadKnownFailuresConfig };
+
+export function applyKnownFailuresToArtifacts(runArtifact, statusArtifact, knownFailures) {
+  if (!knownFailures) return { runArtifact, statusArtifact };
+
+  const runEntries = extractRunFileEntries(runArtifact);
+  const statusEntries = extractStatusFileEntries(statusArtifact);
+  const fileSummaries = new Map();
+
+  for (const entry of [...runEntries, ...statusEntries]) {
+    const key = buildKnownFailureFileIdentity(entry.service, entry.type, entry.path);
+    if (!fileSummaries.has(key)) {
+      fileSummaries.set(key, {
+        service: entry.service,
+        type: entry.type,
+        path: entry.path,
+        status: entry.status,
+        error: entry.error || null,
+        failureDetails: Array.isArray(entry.failureDetails) ? entry.failureDetails : [],
+      });
+    }
+  }
+
+  const matchesByFileKey = new Map();
+  const matchedByFailedEntryIds = new Set();
+
+  for (const fileSummary of fileSummaries.values()) {
+    const matches = findMatchingKnownFailureEntries(knownFailures, fileSummary);
+    if (matches.length === 0) continue;
+
+    const fileKey = buildKnownFailureFileIdentity(
+      fileSummary.service,
+      fileSummary.type,
+      fileSummary.path
+    );
+    matchesByFileKey.set(fileKey, matches.map((entry) => toArtifactTriageEntry(entry)));
+    if (fileSummary.status === "failed") {
+      for (const entry of matches) {
+        matchedByFailedEntryIds.add(entry.id);
+      }
+    }
+  }
+
+  for (const entry of [...runEntries, ...statusEntries]) {
+    const fileKey = buildKnownFailureFileIdentity(entry.service, entry.type, entry.path);
+    const matches = matchesByFileKey.get(fileKey) || [];
+    if (matches.length === 0) {
+      if (entry.status === "failed") {
+        setEntryTriage(entry, {
+          status: "untriaged",
+          entries: [],
+        });
+      }
+      continue;
+    }
+
+    setEntryTriage(entry, {
+      status: entry.status === "failed" ? "known_failure" : "known_issue_not_reproduced",
+      classifications: [...new Set(matches.map((match) => match.classification))].sort(),
+      entries: matches,
+    });
+  }
+
+  const summaryTests = statusArtifact?.tests || runEntries;
+  const triageSummary = buildTriageSummary(
+    summaryTests,
+    knownFailures.entries,
+    matchedByFailedEntryIds
+  );
+  runArtifact.triageSummary = triageSummary;
+  if (statusArtifact) {
+    statusArtifact.triageSummary = triageSummary;
+  }
+  return { runArtifact, statusArtifact };
+}
+
+function toArtifactTriageEntry(entry) {
+  return {
+    id: entry.id,
+    title: entry.title,
+    classification: entry.classification,
+    state: entry.state,
+    issue: entry.issue,
+    description: entry.description,
+    whyFailing: entry.whyFailing,
+    lastReviewedAt: entry.lastReviewedAt,
+  };
+}
+
+function buildTriageSummary(tests, entries, matchedEntryIds) {
+  const failedTests = tests.filter((test) => test.status === "failed");
+  const knownFailedTests = failedTests.filter((test) => test.triage?.status === "known_failure");
+  const byClassification = {};
+
+  for (const test of knownFailedTests) {
+    for (const classification of test.triage?.classifications || []) {
+      byClassification[classification] = (byClassification[classification] || 0) + 1;
+    }
+  }
+
+  return {
+    failed: {
+      total: failedTests.length,
+      known: knownFailedTests.length,
+      untriaged: failedTests.length - knownFailedTests.length,
+      byClassification,
+    },
+    entries: {
+      total: entries.length,
+      matchedByFailedTests: matchedEntryIds.size,
+      unmatched: entries.length - matchedEntryIds.size,
+    },
+  };
+}
+
+function extractRunFileEntries(runArtifact) {
+  const entries = [];
+
+  for (const service of runArtifact.services || []) {
+    for (const suite of service.suites || []) {
+      for (const file of suite.files || []) {
+        entries.push({
+          target: file,
+          service: service.name,
+          type: suite.type,
+          path: file.path,
+          status: file.status,
+          error: file.error || null,
+          failureDetails: Array.isArray(file.failureDetails) ? file.failureDetails : [],
+        });
+      }
+    }
+  }
+
+  return entries;
+}
+
+function extractStatusFileEntries(statusArtifact) {
+  return statusArtifact?.tests || [];
+}
+
+function setEntryTriage(entry, triage) {
+  if (entry?.target) {
+    entry.target.triage = triage;
+    return;
+  }
+  entry.triage = triage;
+}