@elench/testkit 0.1.42 → 0.1.44

package/lib/runner/triage.test.mjs

@@ -0,0 +1,154 @@
+import { describe, expect, it } from "vitest";
+import { normalizeKnownFailuresDocument } from "../known-failures/index.mjs";
+import { applyKnownFailuresToArtifacts } from "./triage.mjs";
+
+describe("runner triage", () => {
+  it("matches exact failure keys and enriches both artifacts", () => {
+    const knownFailures = normalizeKnownFailuresDocument({
+      schemaVersion: 1,
+      issueRepo: "acme/repo",
+      entries: [
+        {
+          id: "bad-message",
+          title: "Bad message bug",
+          classification: "product_bug",
+          state: "open",
+          issue: {
+            repo: "acme/repo",
+            number: 12,
+            url: "https://github.com/acme/repo/issues/12",
+          },
+          description: "The API returns the wrong message.",
+          whyFailing: "The endpoint payload is wrong.",
+          lastReviewedAt: "2026-04-27",
+          matches: [
+            {
+              service: "api",
+              type: "int",
+              path: "__testkit__/http/failing.int.testkit.ts",
+              failureKey: "returns the wrong message",
+            },
+          ],
+        },
+      ],
+    });
+
+    const runArtifact = {
+      services: [
+        {
+          name: "api",
+          suites: [
+            {
+              type: "int",
+              files: [
+                {
+                  path: "__testkit__/http/failing.int.testkit.ts",
+                  status: "failed",
+                  error: "Default runtime thresholds failed: checks(rate==1.0)",
+                  failureDetails: [
+                    {
+                      kind: "k6-check",
+                      key: "returns the wrong message",
+                      title: "returns the wrong message",
+                      count: 1,
+                    },
+                  ],
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    };
+    const statusArtifact = {
+      tests: [
+        {
+          service: "api",
+          type: "int",
+          path: "__testkit__/http/failing.int.testkit.ts",
+          status: "failed",
+          error: "Default runtime thresholds failed: checks(rate==1.0)",
+          failureDetails: [
+            {
+              kind: "k6-check",
+              key: "returns the wrong message",
+              title: "returns the wrong message",
+              count: 1,
+            },
+          ],
+        },
+      ],
+    };
+
+    const enriched = applyKnownFailuresToArtifacts(runArtifact, statusArtifact, knownFailures);
+    expect(enriched.statusArtifact.tests[0].triage).toMatchObject({
+      status: "known_failure",
+      classifications: ["product_bug"],
+    });
+    expect(enriched.runArtifact.services[0].suites[0].files[0].triage.entries[0]).toMatchObject({
+      id: "bad-message",
+      issue: {
+        number: 12,
+      },
+    });
+    expect(enriched.statusArtifact.triageSummary).toEqual({
+      failed: {
+        total: 1,
+        known: 1,
+        untriaged: 0,
+        byClassification: {
+          product_bug: 1,
+        },
+      },
+      entries: {
+        total: 1,
+        matchedByFailedTests: 1,
+        unmatched: 0,
+      },
+    });
+  });
+
+  it("marks unmatched failed tests as untriaged", () => {
+    const enriched = applyKnownFailuresToArtifacts(
+      {
+        services: [
+          {
+            name: "api",
+            suites: [
+              {
+                type: "int",
+                files: [
+                  {
+                    path: "__testkit__/http/failing.int.testkit.ts",
+                    status: "failed",
+                    error: "boom",
+                  },
+                ],
+              },
+            ],
+          },
+        ],
+      },
+      {
+        tests: [
+          {
+            service: "api",
+            type: "int",
+            path: "__testkit__/http/failing.int.testkit.ts",
+            status: "failed",
+            error: "boom",
+          },
+        ],
+      },
+      normalizeKnownFailuresDocument({
+        schemaVersion: 1,
+        entries: [],
+      })
+    );
+
+    expect(enriched.statusArtifact.tests[0].triage).toEqual({
+      status: "untriaged",
+      entries: [],
+    });
+  });
+});

package/lib/runtime/index.mjs

@@ -1,6 +1,6 @@
 import rawHttp from "k6/http";
 import { Rate, Trend } from "k6/metrics";
-import { check, fail, group, sleep } from "k6";
+import { fail, sleep } from "k6";
 import {
   formatWaitForTimeoutError,
   normalizeWaitIntervalSeconds,
@@ -8,6 +8,7 @@ import {
   remainingFileTimeoutMs,
   remainingFileTimeoutSeconds,
 } from "../shared/file-timeout.mjs";
+import { check, group } from "../runtime-src/k6/checks.js";
 
 export { check, fail, group, sleep };
 export { Rate, Trend };

package/lib/runtime-src/k6/checks.js

@@ -1,6 +1,9 @@
+import { check as k6Check, group as k6Group } from "k6";
 import { Rate } from "k6/metrics";
+import { emitArtifact } from "./artifacts.js";
 
 export const runtimeFailures = new Rate("testkit_runtime_failures");
+const failureState = createFailureState();
 
 export function singleIterationOptions(overrides = {}) {
   return {
@@ -16,6 +19,41 @@ export function singleIterationOptions(overrides = {}) {
 
 export const defaultOptions = singleIterationOptions();
 
+export function check(value, checks) {
+  let allPassed = true;
+
+  for (const [name, predicate] of Object.entries(checks || {})) {
+    const checkName = normalizeLabel(name, "unnamed check");
+    const passed = k6Check(value, { [checkName]: predicate });
+    if (!passed) {
+      recordFailureDetail({
+        kind: "k6-check",
+        key: buildFailureKey(failureState.groupStack, checkName),
+        title: checkName,
+        checkName,
+        groupPath: [...failureState.groupStack],
+        phase: failureState.phase,
+      });
+      allPassed = false;
+    }
+  }
+
+  return allPassed;
+}
+
+export function group(name, fn) {
+  const groupName = normalizeLabel(name, "unnamed group");
+
+  return k6Group(groupName, () => {
+    failureState.groupStack.push(groupName);
+    try {
+      return fn();
+    } finally {
+      failureState.groupStack.pop();
+    }
+  });
+}
+
 export function json(res) {
   return JSON.parse(res.body);
 }
@@ -46,3 +84,95 @@ export function isSorted(rows, field, direction = "asc") {
 export function recordRuntimeFailure() {
   runtimeFailures.add(1);
 }
+
+export function startFailureCollection(phase) {
+  failureState.phase = normalizeLabel(phase, "exec");
+  failureState.groupStack = [];
+  failureState.detailsByKey = new Map();
+}
+
+export function emitFailureCollectionArtifact() {
+  const failures = [...failureState.detailsByKey.values()]
+    .sort((left, right) => left.key.localeCompare(right.key))
+    .map((detail) => ({ ...detail }));
+
+  if (failures.length > 0) {
+    emitArtifact(
+      "failure-details",
+      {
+        phase: failureState.phase,
+        failures,
+      },
+      {
+        kind: "testkit.failure-details",
+        summary: `${failures.length} failure detail(s)`,
+      }
+    );
+  }
+
+  startFailureCollection(failureState.phase);
+}
+
+export function recordFailureDetail(detail) {
+  const normalized = normalizeFailureDetail(detail);
+  if (!normalized) return;
+
+  const existing = failureState.detailsByKey.get(normalized.key);
+  if (!existing) {
+    failureState.detailsByKey.set(normalized.key, normalized);
+    return;
+  }
+
+  existing.count += normalized.count;
+}
+
+function createFailureState() {
+  return {
+    phase: "exec",
+    groupStack: [],
+    detailsByKey: new Map(),
+  };
+}
+
+function normalizeFailureDetail(detail) {
+  if (!detail || typeof detail !== "object") return null;
+
+  const kind = normalizeLabel(detail.kind, null);
+  const key = normalizeLabel(detail.key, null);
+  const title = normalizeLabel(detail.title, null);
+  if (!kind || !key || !title) return null;
+
+  const normalized = {
+    kind,
+    key,
+    title,
+    count: 1,
+  };
+
+  const checkName = normalizeLabel(detail.checkName, null);
+  if (checkName) normalized.checkName = checkName;
+
+  const phase = normalizeLabel(detail.phase, null);
+  if (phase) normalized.phase = phase;
+
+  const message = normalizeLabel(detail.message, null);
+  if (message) normalized.message = message;
+
+  const groupPath = Array.isArray(detail.groupPath)
+    ? detail.groupPath.map((entry) => normalizeLabel(entry, null)).filter(Boolean)
+    : [];
+  if (groupPath.length > 0) normalized.groupPath = groupPath;
+
+  return normalized;
+}
+
+function buildFailureKey(groupPath, title) {
+  if (!Array.isArray(groupPath) || groupPath.length === 0) return title;
+  return [...groupPath, title].join(" > ");
+}
+
+function normalizeLabel(value, fallback) {
+  if (typeof value !== "string") return fallback;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : fallback;
+}

package/lib/runtime-src/k6/dal-suite.js

@@ -1,5 +1,10 @@
 import { fail } from "k6";
-import { defaultOptions, recordRuntimeFailure } from "./checks.js";
+import {
+  defaultOptions,
+  emitFailureCollectionArtifact,
+  recordRuntimeFailure,
+  startFailureCollection,
+} from "./checks.js";
 import { createDalContext, openDb } from "./dal.js";
 
 export function defineDalSuite(configOrRun, maybeRun) {
@@ -11,14 +16,18 @@ export function defineDalSuite(configOrRun, maybeRun) {
     options: config.options || defaultOptions,
     setup() {
       if (typeof config.setup !== "function") return null;
+      startFailureCollection("setup");
       try {
         return config.setup({ db, dal });
       } catch (error) {
         recordRuntimeFailure();
         fail(formatFatalSuiteError("setup", error));
+      } finally {
+        emitFailureCollectionArtifact();
       }
     },
     exec(setupData) {
+      startFailureCollection("exec");
       try {
         return run({
           db,
@@ -28,6 +37,8 @@ export function defineDalSuite(configOrRun, maybeRun) {
       } catch (error) {
         recordRuntimeFailure();
         fail(formatFatalSuiteError("exec", error));
+      } finally {
+        emitFailureCollectionArtifact();
       }
     },
   };

package/lib/runtime-src/k6/suite.js

@@ -1,5 +1,10 @@
 import { fail } from "k6";
-import { defaultOptions, recordRuntimeFailure } from "./checks.js";
+import {
+  defaultOptions,
+  emitFailureCollectionArtifact,
+  recordRuntimeFailure,
+  startFailureCollection,
+} from "./checks.js";
 import { createHttpClient, getEnv } from "./http.js";
 import {
   clearRuntimeContext,
@@ -16,6 +21,7 @@ export function defineHttpSuite(configOrRun, maybeRun) {
     },
     setup() {
       const resolved = resolveRuntimeConfig(config);
+      startFailureCollection("setup");
       try {
         registerRuntimeContext({ env: resolved.env, http: resolved.client.rawHttp || null });
         if (typeof resolved.auth?.setup !== "function") return null;
@@ -24,11 +30,13 @@ export function defineHttpSuite(configOrRun, maybeRun) {
         recordRuntimeFailure();
         fail(formatFatalSuiteError("setup", error));
       } finally {
+        emitFailureCollectionArtifact();
         clearRuntimeContext();
       }
     },
     exec(setupData) {
       const resolved = resolveRuntimeConfig(config);
+      startFailureCollection("exec");
       try {
         registerRuntimeContext({ env: resolved.env, http: resolved.client.rawHttp || null });
         return run({
@@ -43,6 +51,7 @@ export function defineHttpSuite(configOrRun, maybeRun) {
         recordRuntimeFailure();
         fail(formatFatalSuiteError("exec", error));
       } finally {
+        emitFailureCollectionArtifact();
         clearRuntimeContext();
       }
     },

package/lib/setup/index.d.ts

@@ -59,6 +59,12 @@ export interface TestkitExecutionConfig {
   fileTimeoutSeconds?: number;
 }
 
+export interface KnownFailureIssueValidationConfig {
+  provider?: "github";
+  mode?: "off" | "warn" | "error";
+  cacheTtlSeconds?: number;
+}
+
 export interface ServiceConfig {
   database?: LocalDatabaseConfig;
   databaseFrom?: string;
@@ -90,6 +96,10 @@ export interface TestkitSetup {
   profiles?: {
     http?: Record<string, HttpSuiteConfig>;
   };
+  reporting?: {
+    knownFailuresFile?: string;
+    issueValidation?: KnownFailureIssueValidationConfig;
+  };
   services?: Record<string, ServiceConfig>;
   telemetry?: {
     enabled?: boolean;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/testkit",
-  "version": "0.1.42",
+  "version": "0.1.44",
   "description": "CLI for discovering and running local HTTP, DAL, and Playwright test suites",
   "type": "module",
   "types": "./lib/index.d.ts",
@@ -17,6 +17,10 @@
       "types": "./lib/runtime/index.d.ts",
       "default": "./lib/runtime/index.mjs"
     },
+    "./known-failures": {
+      "types": "./lib/known-failures/index.d.ts",
+      "default": "./lib/known-failures/index.mjs"
+    },
     "./package.json": "./package.json"
   },
   "bin": {