@elench/testkit 0.1.42 → 0.1.44

package/README.md

@@ -68,6 +68,14 @@ export default defineTestkitSetup({
     workers: 8,
     fileTimeoutSeconds: 60,
   },
+  reporting: {
+    knownFailuresFile: "testkit.known-failures.json",
+    issueValidation: {
+      provider: "github",
+      mode: "warn",
+      cacheTtlSeconds: 900,
+    },
+  },
   services: {
     api: service({
       ...tsxService({
@@ -141,9 +149,24 @@ for:
 - migrate / seed commands
 - test-local migrate / seed overrides
 - named HTTP suite profiles
+- known-failure annotation merge for enriched status/run artifacts
+- optional GitHub-backed known-failure issue validation
 - repo-declared suite/file skip policies with explicit reasons
 - telemetry upload configuration
 
+If `reporting.knownFailuresFile` is configured, `testkit` enriches
+`.testkit/results/latest.json` and `testkit.status.json` with:
+
+- per-file `failureDetails`
+- per-file `triage` metadata (issue, classification, description)
+- top-level `triageSummary` counts for known vs untriaged failures
+
+If `reporting.issueValidation` is also configured, `testkit` validates known-failure
+issue references against GitHub and adds top-level `knownFailuresIssueValidation`
+data to the run/status artifacts. The most important stale-triage signal is:
+
+- a known-failure test still fails, but the linked GitHub issue is closed
+
 ## Authoring
 
 HTTP suites:

package/lib/config/index.mjs

@@ -15,6 +15,7 @@ import {
   normalizeRuntimeMaxConcurrentTasks,
   normalizeRuntimeInstances,
 } from "../runner/execution-config.mjs";
+import { normalizeKnownFailureIssueValidationConfig } from "../known-failures/github.mjs";
 
 const TESTKIT_K6_BIN = "TESTKIT_K6_BIN";
 const DEFAULT_LOCAL_IMAGE = "pgvector/pgvector:pg16";
@@ -30,6 +31,7 @@ export async function loadConfigs(opts = {}) {
   const productDir = resolveProductDir(process.cwd(), opts.dir);
   const { setup, setupFile } = await loadTestkitSetup(productDir);
   const execution = normalizeRepoExecution(setup.execution);
+  const reporting = normalizeReportingConfig(setup.reporting);
   const explicitServices = setup.services || {};
   const discovery = discoverProject(productDir, explicitServices);
   const serviceNames = new Set([
@@ -46,6 +48,7 @@ export async function loadConfigs(opts = {}) {
         setup,
         setupFile,
         execution,
+        reporting,
         explicitService: explicitServices[name] || {},
         discoveredService: discovery.services[name] || null,
         suites: discovery.suitesByService[name] || {},
@@ -105,6 +108,7 @@ function normalizeServiceConfig({
   setup,
   setupFile,
   execution,
+  reporting,
   explicitService,
   discoveredService,
   suites,
@@ -157,6 +161,7 @@ function normalizeServiceConfig({
     suites,
     testkit: {
       execution,
+      reporting,
       dependsOn: explicitService.dependsOn || [],
       database,
       databaseFrom: explicitService.databaseFrom,
@@ -172,6 +177,22 @@ function normalizeServiceConfig({
   };
 }
 
+function normalizeReportingConfig(value) {
+  if (!value) return null;
+
+  const knownFailuresFile = normalizeOptionalString(value.knownFailuresFile);
+  if (!knownFailuresFile) {
+    throw new Error('testkit.setup.ts reporting.knownFailuresFile must be a non-empty string');
+  }
+
+  const issueValidation = normalizeKnownFailureIssueValidationConfig(value.issueValidation);
+
+  return {
+    knownFailuresFile,
+    issueValidation,
+  };
+}
+
 function normalizeLocalConfig(name, explicitService, discoveredService, productDir) {
   if (Object.prototype.hasOwnProperty.call(explicitService, "local")) {
     if (explicitService.local === false) {
@@ -282,6 +303,12 @@ function normalizeRuntimeConfig(value, serviceName) {
   };
 }
 
+function normalizeOptionalString(value) {
+  if (typeof value !== "string") return null;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : null;
+}
+
 function normalizeLifecycle(value) {
   if (!value) return undefined;
   if (!value.cmd && !value.testkitCmd) {