@elench/testkit 0.1.41 → 0.1.43

package/README.md

@@ -68,6 +68,9 @@ export default defineTestkitSetup({
     workers: 8,
     fileTimeoutSeconds: 60,
   },
+  reporting: {
+    knownFailuresFile: "testkit.known-failures.json",
+  },
   services: {
     api: service({
       ...tsxService({
@@ -83,6 +86,7 @@ export default defineTestkitSetup({
       }),
       runtime: {
         instances: 1,
+        maxConcurrentTasks: 4,
       },
       requirements: {
         files: [
@@ -134,14 +138,23 @@ for:
 - per-file wall clock timeout budget
 - multi-service graphs
 - local runtime instance counts
+- per-runtime concurrent task caps
 - local DB binding configuration
 - explicit per-file or per-suite locks
 - migrate / seed commands
 - test-local migrate / seed overrides
 - named HTTP suite profiles
+- known-failure annotation merge for enriched status/run artifacts
 - repo-declared suite/file skip policies with explicit reasons
 - telemetry upload configuration
 
+If `reporting.knownFailuresFile` is configured, `testkit` enriches
+`.testkit/results/latest.json` and `testkit.status.json` with:
+
+- per-file `failureDetails`
+- per-file `triage` metadata (issue, classification, description)
+- top-level `triageSummary` counts for known vs untriaged failures
+
 ## Authoring
 
 HTTP suites:

package/bin/testkit.mjs

@@ -1,3 +1,8 @@
 #!/usr/bin/env node
 import { run } from "../lib/cli/index.mjs";
-run();
+
+run().catch((error) => {
+  setImmediate(() => {
+    throw error;
+  });
+});

package/lib/cli/index.mjs

@@ -11,7 +11,7 @@ import {
 } from "./args.mjs";
 import * as runner from "../runner/index.mjs";
 
-export function run() {
+export async function run(argv = process.argv) {
   const cli = cac("testkit");
 
   cli
@@ -91,5 +91,7 @@ export function run() {
     });
 
   cli.help();
-  cli.parse();
+  const parsed = cli.parse(argv, { run: false });
+  await cli.runMatchedCommand();
+  return parsed;
 }

package/lib/config/index.mjs

@@ -12,6 +12,7 @@ import {
   DEFAULT_FILE_TIMEOUT_SECONDS,
   normalizeDatabaseBinding,
   normalizeExecutionConfig,
+  normalizeRuntimeMaxConcurrentTasks,
   normalizeRuntimeInstances,
 } from "../runner/execution-config.mjs";
 
@@ -29,6 +30,7 @@ export async function loadConfigs(opts = {}) {
   const productDir = resolveProductDir(process.cwd(), opts.dir);
   const { setup, setupFile } = await loadTestkitSetup(productDir);
   const execution = normalizeRepoExecution(setup.execution);
+  const reporting = normalizeReportingConfig(setup.reporting);
   const explicitServices = setup.services || {};
   const discovery = discoverProject(productDir, explicitServices);
   const serviceNames = new Set([
@@ -45,6 +47,7 @@ export async function loadConfigs(opts = {}) {
         setup,
         setupFile,
         execution,
+        reporting,
         explicitService: explicitServices[name] || {},
         discoveredService: discovery.services[name] || null,
         suites: discovery.suitesByService[name] || {},
@@ -104,6 +107,7 @@ function normalizeServiceConfig({
   setup,
   setupFile,
   execution,
+  reporting,
   explicitService,
   discoveredService,
   suites,
@@ -156,6 +160,7 @@ function normalizeServiceConfig({
     suites,
     testkit: {
       execution,
+      reporting,
       dependsOn: explicitService.dependsOn || [],
       database,
       databaseFrom: explicitService.databaseFrom,
@@ -171,6 +176,19 @@ function normalizeServiceConfig({
   };
 }
 
+function normalizeReportingConfig(value) {
+  if (!value) return null;
+
+  const knownFailuresFile = normalizeOptionalString(value.knownFailuresFile);
+  if (!knownFailuresFile) {
+    throw new Error('testkit.setup.ts reporting.knownFailuresFile must be a non-empty string');
+  }
+
+  return {
+    knownFailuresFile,
+  };
+}
+
 function normalizeLocalConfig(name, explicitService, discoveredService, productDir) {
   if (Object.prototype.hasOwnProperty.call(explicitService, "local")) {
     if (explicitService.local === false) {
@@ -265,6 +283,7 @@ function normalizeRuntimeConfig(value, serviceName) {
   if (!value) {
     return {
       instances: 1,
+      maxConcurrentTasks: Number.POSITIVE_INFINITY,
     };
   }
 
@@ -273,9 +292,19 @@ function normalizeRuntimeConfig(value, serviceName) {
       value.instances ?? 1,
       `Service "${serviceName}" runtime.instances`
     ),
+    maxConcurrentTasks: normalizeRuntimeMaxConcurrentTasks(
+      value.maxConcurrentTasks,
+      `Service "${serviceName}" runtime.maxConcurrentTasks`
+    ),
   };
 }
 
+function normalizeOptionalString(value) {
+  if (typeof value !== "string") return null;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : null;
+}
+
 function normalizeLifecycle(value) {
   if (!value) return undefined;
   if (!value.cmd && !value.testkitCmd) {
@@ -578,6 +607,11 @@ function validateServiceConfig({
   if (runtime.instances < 1) {
     throw new Error(`Service "${name}" runtime.instances must be a positive integer`);
   }
+  if (runtime.maxConcurrentTasks <= 0) {
+    throw new Error(
+      `Service "${name}" runtime.maxConcurrentTasks must be a positive integer when provided`
+    );
+  }
 
   for (const depName of dependsOn || []) {
     if (depName === name) {

package/lib/reporters/playwright.mjs

@@ -1,4 +1,5 @@
 import path from "path";
+import { mergeFailureDetails } from "../runner/failure-details.mjs";
 
 export function parsePlaywrightJsonResults(stdout, cwd) {
   if (!stdout.trim()) {
@@ -16,34 +17,42 @@ export function parsePlaywrightJsonResults(stdout, cwd) {
   }
 
   const fileResults = new Map();
-  visitPlaywrightSuites(parsed.suites || [], null, fileResults, cwd);
+  visitPlaywrightSuites(parsed.suites || [], null, [], fileResults, cwd);
   return {
     fileResults: sanitizePlaywrightFileResults(fileResults),
     errors: (parsed.errors || []).map(formatPlaywrightReporterError).filter(Boolean),
   };
 }
 
-export function visitPlaywrightSuites(suites, inheritedFile, fileResults, cwd) {
+export function visitPlaywrightSuites(suites, inheritedFile, inheritedTitlePath, fileResults, cwd) {
   for (const suite of suites || []) {
     const suiteFile = normalizeReportedFile(extractReporterFile(suite) || inheritedFile, cwd);
+    const suiteTitle = normalizeSuiteTitle(suite?.title, suiteFile);
+    const suiteTitlePath = suiteTitle
+      ? [...inheritedTitlePath, suiteTitle].filter(Boolean)
+      : inheritedTitlePath;
     for (const child of suite.suites || []) {
-      visitPlaywrightSuites([child], suiteFile, fileResults, cwd);
+      visitPlaywrightSuites([child], suiteFile, suiteTitlePath, fileResults, cwd);
     }
     for (const spec of suite.specs || []) {
-      collectPlaywrightSpec(spec, suiteFile, fileResults, cwd);
+      collectPlaywrightSpec(spec, suiteFile, suiteTitlePath, fileResults, cwd);
     }
   }
 }
 
-export function collectPlaywrightSpec(spec, inheritedFile, fileResults, cwd) {
+export function collectPlaywrightSpec(spec, inheritedFile, suiteTitlePath, fileResults, cwd) {
   const file = normalizeReportedFile(extractReporterFile(spec) || inheritedFile, cwd);
   if (!file) return;
+  const specTitle = firstLine(spec?.title || "Playwright spec failed") || "Playwright spec failed";
+  const specPath = [...suiteTitlePath, specTitle].filter(Boolean);
+  const failureKey = specPath.join(" > ");
 
   const current = fileResults.get(file) || {
     failed: false,
     status: "passed",
     error: null,
     durationMs: 0,
+    failureDetails: [],
     passedCount: 0,
     skippedCount: 0,
   };
@@ -62,6 +71,13 @@ export function collectPlaywrightSpec(spec, inheritedFile, fileResults, cwd) {
       current.failed = true;
       current.status = "failed";
       current.error ||= extractPlaywrightFailure(final, spec, test);
+      current.failureDetails.push({
+        kind: "playwright-spec",
+        key: failureKey,
+        title: specTitle,
+        suitePath: suiteTitlePath,
+        message: extractPlaywrightFailure(final, spec, test),
+      });
       continue;
     }
 
@@ -76,6 +92,7 @@ export function collectPlaywrightSpec(spec, inheritedFile, fileResults, cwd) {
   if (!current.failed) {
     current.status = current.passedCount === 0 && current.skippedCount > 0 ? "skipped" : "passed";
   }
+  current.failureDetails = mergeFailureDetails(current.failureDetails);
   fileResults.set(file, current);
 }
 
@@ -155,7 +172,19 @@ function sanitizePlaywrightFileResults(fileResults) {
       status: result.status,
       error: result.error,
       durationMs: result.durationMs,
+      failureDetails: mergeFailureDetails(result.failureDetails),
     });
   }
   return sanitized;
 }
+
+function normalizeSuiteTitle(title, reportedFile) {
+  const normalizedTitle = firstLine(title || "");
+  if (!normalizedTitle) return null;
+  if (!reportedFile) return normalizedTitle;
+  const fileName = reportedFile.split("/").pop();
+  if (normalizedTitle === reportedFile || normalizedTitle === fileName) {
+    return null;
+  }
+  return normalizedTitle;
+}

package/lib/reporters/playwright.test.mjs

@@ -52,6 +52,15 @@ describe("playwright-report", () => {
       status: "failed",
       error: "boom",
       durationMs: 15,
+      failureDetails: [
+        {
+          kind: "playwright-spec",
+          key: "auth works",
+          title: "auth works",
+          count: 1,
+          message: "boom",
+        },
+      ],
     });
   });
 
@@ -86,6 +95,7 @@ describe("playwright-report", () => {
       status: "skipped",
       error: null,
       durationMs: 7,
+      failureDetails: [],
     });
   });
 
@@ -134,6 +144,7 @@ describe("playwright-report", () => {
       status: "passed",
       error: null,
       durationMs: 8,
+      failureDetails: [],
     });
   });
 

package/lib/runner/default-runtime-runner.mjs

@@ -9,9 +9,11 @@ import {
 } from "../shared/file-timeout.mjs";
 import { persistTaskArtifacts } from "./artifacts.mjs";
 import { determineDefaultRuntimeFailure } from "./default-runtime-errors.mjs";
+import { collectFailureDetailsFromRuntimeArtifacts } from "./failure-details.mjs";
 import { RUNTIME_ARTIFACT_MARKER } from "../runtime-src/k6/artifacts.js";
 import { readDatabaseUrl } from "./state-io.mjs";
 import { buildTaskExecutionEnv } from "./template.mjs";
+import { killChildProcess } from "./processes.mjs";
 
 export async function runHttpK6Task(targetConfig, task, lifecycle, lease) {
   const baseUrl = targetConfig.testkit.local?.baseUrl;
@@ -24,7 +26,9 @@ export async function runHttpK6Task(targetConfig, task, lifecycle, lease) {
     serviceName: targetConfig.name,
     sourceFile: path.join(targetConfig.productDir, task.file),
   });
-  console.log(`·· ${targetConfig.runtimeLabel}:${task.suiteName} → ${task.file}`);
+  if (lifecycle.isStopRequested()) {
+    throw new Error(`testkit run interrupted before starting ${task.file}`);
+  }
   return runDefaultRuntimeTask(
     targetConfig,
     task,
@@ -45,7 +49,9 @@ export async function runDalTask(targetConfig, task, lifecycle, lease) {
     serviceName: targetConfig.name,
     sourceFile: path.join(targetConfig.productDir, task.file),
   });
-  console.log(`·· ${targetConfig.runtimeLabel}:${task.suiteName} → ${task.file}`);
+  if (lifecycle.isStopRequested()) {
+    throw new Error(`testkit run interrupted before starting ${task.file}`);
+  }
   return runDefaultRuntimeTask(
     targetConfig,
     task,
@@ -74,11 +80,25 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
         process.env
       ),
       reject: false,
-      cancelSignal: lifecycle.signal,
       forceKillAfterDelay: 5_000,
     }
   );
-  const { result, timedOut } = await settleSubprocess(subprocess, fileTimeoutSeconds);
+  lifecycle.registerProcess(subprocess, () => {
+    killChildProcess(subprocess, "SIGINT");
+  });
+  if (lifecycle.isStopRequested()) {
+    const interruptSubprocess = () => killChildProcess(subprocess, "SIGINT");
+    if (subprocess.pid) interruptSubprocess();
+    else subprocess.once?.("spawn", interruptSubprocess);
+  }
+  console.log(`·· ${targetConfig.runtimeLabel}:${task.suiteName} → ${task.file}`);
+  let result;
+  let timedOut;
+  try {
+    ({ result, timedOut } = await settleSubprocess(subprocess, fileTimeoutSeconds));
+  } finally {
+    lifecycle.unregisterProcess(subprocess.pid);
+  }
 
   const stdout = parseDefaultRuntimeOutput(result.stdout || "");
   const stderr = parseDefaultRuntimeOutput(result.stderr || "");
@@ -86,11 +106,13 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
   if (stderr.visibleOutput) process.stderr.write(stderr.visibleOutput);
 
   const summary = readDefaultRuntimeSummary(summaryFile);
+  const rawRuntimeArtifacts = [...stdout.artifacts, ...stderr.artifacts];
   const runtimeArtifacts = persistTaskArtifacts(
     targetConfig.productDir,
     task,
-    [...stdout.artifacts, ...stderr.artifacts]
+    rawRuntimeArtifacts
   );
+  const failureDetails = collectFailureDetailsFromRuntimeArtifacts(rawRuntimeArtifacts);
   const runtimeError = timedOut
     ? formatFileTimeoutBudgetError(fileTimeoutSeconds)
     : determineDefaultRuntimeFailure(result, summary, getFirstLine);
@@ -104,6 +126,7 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
     startedAt,
     finishedAt,
     artifacts: runtimeArtifacts,
+    failureDetails,
   };
 }
 
@@ -118,7 +141,7 @@ export async function settleSubprocess(subprocess, fileTimeoutSeconds) {
       new Promise((resolve) => {
         timeoutHandle = setTimeout(async () => {
           timedOut = true;
-          subprocess.kill("SIGTERM");
+          killChildProcess(subprocess, "SIGTERM");
           const result = await subprocess.catch((error) => error);
           resolve({ result, timedOut: true });
         }, timeoutMs);

package/lib/runner/execution-config.mjs

@@ -20,6 +20,23 @@ export function normalizeRuntimeInstances(value, label = "runtime.instances") {
   return normalizePositiveInteger(value, label);
 }
 
+export function parseRuntimeMaxConcurrentTasksOption(
+  value,
+  label = "runtime.maxConcurrentTasks"
+) {
+  return parsePositiveInteger(value, label);
+}
+
+export function normalizeRuntimeMaxConcurrentTasks(
+  value,
+  label = "runtime.maxConcurrentTasks"
+) {
+  if (value === undefined || value === null) {
+    return Number.POSITIVE_INFINITY;
+  }
+  return normalizePositiveInteger(value, label);
+}
+
 export function normalizeDatabaseBinding(value, label = "database.binding") {
   const normalized = String(value || "").trim();
   if (!DATABASE_BINDINGS.has(normalized)) {

package/lib/runner/execution-config.test.mjs

@@ -5,8 +5,10 @@ import {
   DEFAULT_FILE_TIMEOUT_SECONDS,
   normalizeDatabaseBinding,
   normalizeExecutionConfig,
+  normalizeRuntimeMaxConcurrentTasks,
   normalizeRuntimeInstances,
   parseFileTimeoutOption,
+  parseRuntimeMaxConcurrentTasksOption,
   parseRuntimeInstancesOption,
   parseWorkersOption,
   resolveExecutionConfig,
@@ -16,9 +18,13 @@ describe("execution-config", () => {
   it("parses worker and runtime-instance options", () => {
     expect(parseWorkersOption("8")).toBe(8);
     expect(parseRuntimeInstancesOption("2")).toBe(2);
+    expect(parseRuntimeMaxConcurrentTasksOption("4")).toBe(4);
     expect(parseFileTimeoutOption("45")).toBe(45);
     expect(() => parseWorkersOption("0")).toThrow('Invalid --workers value "0"');
     expect(() => parseRuntimeInstancesOption("0")).toThrow('Invalid runtime.instances value "0"');
+    expect(() => parseRuntimeMaxConcurrentTasksOption("0")).toThrow(
+      'Invalid runtime.maxConcurrentTasks value "0"'
+    );
     expect(() => parseFileTimeoutOption("0")).toThrow(
       'Invalid --file-timeout-seconds value "0"'
     );
@@ -49,6 +55,8 @@ describe("execution-config", () => {
 
   it("normalizes runtime instances and database bindings", () => {
     expect(normalizeRuntimeInstances(2)).toBe(2);
+    expect(normalizeRuntimeMaxConcurrentTasks(undefined)).toBe(Number.POSITIVE_INFINITY);
+    expect(normalizeRuntimeMaxConcurrentTasks(3)).toBe(3);
     for (const binding of DATABASE_BINDINGS) {
       expect(normalizeDatabaseBinding(binding)).toBe(binding);
     }

package/lib/runner/failure-details.mjs

@@ -0,0 +1,91 @@
+export function normalizeFailureDetail(detail) {
+  if (!detail || typeof detail !== "object") return null;
+
+  const kind = normalizeNonEmptyString(detail.kind);
+  const key = normalizeNonEmptyString(detail.key);
+  const title = normalizeNonEmptyString(detail.title);
+  if (!kind || !key || !title) return null;
+
+  const normalized = {
+    kind,
+    key,
+    title,
+    count: normalizePositiveInteger(detail.count) || 1,
+  };
+
+  const groupPath = normalizeStringArray(detail.groupPath);
+  if (groupPath.length > 0) normalized.groupPath = groupPath;
+
+  const checkName = normalizeNonEmptyString(detail.checkName);
+  if (checkName) normalized.checkName = checkName;
+
+  const suitePath = normalizeStringArray(detail.suitePath);
+  if (suitePath.length > 0) normalized.suitePath = suitePath;
+
+  const phase = normalizeNonEmptyString(detail.phase);
+  if (phase) normalized.phase = phase;
+
+  const message = normalizeNonEmptyString(detail.message);
+  if (message) normalized.message = message;
+
+  return normalized;
+}
+
+export function mergeFailureDetails(details) {
+  const mergedByKey = new Map();
+
+  for (const detail of Array.isArray(details) ? details : []) {
+    const normalized = normalizeFailureDetail(detail);
+    if (!normalized) continue;
+    const dedupeKey = `${normalized.kind}::${normalized.key}`;
+    const existing = mergedByKey.get(dedupeKey);
+    if (!existing) {
+      mergedByKey.set(dedupeKey, { ...normalized });
+      continue;
+    }
+    existing.count += normalized.count;
+    if (!existing.message && normalized.message) {
+      existing.message = normalized.message;
+    }
+  }
+
+  return [...mergedByKey.values()].sort(
+    (left, right) =>
+      left.kind.localeCompare(right.kind) || left.key.localeCompare(right.key)
+  );
+}
+
+export function collectFailureDetailsFromRuntimeArtifacts(artifacts) {
+  const details = [];
+
+  for (const artifact of Array.isArray(artifacts) ? artifacts : []) {
+    if (artifact?.kind !== "testkit.failure-details") continue;
+    const phase = normalizeNonEmptyString(artifact?.data?.phase);
+    for (const detail of Array.isArray(artifact?.data?.failures) ? artifact.data.failures : []) {
+      details.push({
+        ...detail,
+        phase: detail?.phase || phase || null,
+      });
+    }
+  }
+
+  return mergeFailureDetails(details);
+}
+
+function normalizeStringArray(value) {
+  if (!Array.isArray(value)) return [];
+  return value
+    .map((entry) => normalizeNonEmptyString(entry))
+    .filter(Boolean);
+}
+
+function normalizeNonEmptyString(value) {
+  if (typeof value !== "string") return null;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : null;
+}
+
+function normalizePositiveInteger(value) {
+  if (!Number.isInteger(value) || value <= 0) return null;
+  return value;
+}

package/lib/runner/failure-details.test.mjs

@@ -0,0 +1,63 @@
+import { describe, expect, it } from "vitest";
+import {
+  collectFailureDetailsFromRuntimeArtifacts,
+  mergeFailureDetails,
+} from "./failure-details.mjs";
+
+describe("runner failure details", () => {
+  it("merges duplicate failure details by kind and key", () => {
+    expect(
+      mergeFailureDetails([
+        { kind: "k6-check", key: "group > check", title: "check" },
+        { kind: "k6-check", key: "group > check", title: "check" },
+        { kind: "playwright-spec", key: "spec title", title: "spec title" },
+      ])
+    ).toEqual([
+      {
+        kind: "k6-check",
+        key: "group > check",
+        title: "check",
+        count: 2,
+      },
+      {
+        kind: "playwright-spec",
+        key: "spec title",
+        title: "spec title",
+        count: 1,
+      },
+    ]);
+  });
+
+  it("extracts and normalizes runtime failure-detail artifacts", () => {
+    expect(
+      collectFailureDetailsFromRuntimeArtifacts([
+        {
+          kind: "testkit.failure-details",
+          data: {
+            phase: "exec",
+            failures: [
+              {
+                kind: "k6-check",
+                key: "status is 200",
+                title: "status is 200",
+              },
+              {
+                kind: "k6-check",
+                key: "status is 200",
+                title: "status is 200",
+              },
+            ],
+          },
+        },
+      ])
+    ).toEqual([
+      {
+        kind: "k6-check",
+        key: "status is 200",
+        title: "status is 200",
+        count: 2,
+        phase: "exec",
+      },
+    ]);
+  });
+});