@elench/testkit 0.1.34 → 0.1.36

package/lib/runner/reporting.test.mjs

@@ -10,11 +10,13 @@ describe("runner reporting", () => {
         skipped: false,
         suiteCount: 1,
         completedSuiteCount: 1,
+        skippedSuiteCount: 0,
         failedSuiteCount: 0,
         totalFileCount: 3,
         completedFileCount: 3,
         passedFileCount: 3,
         failedFileCount: 0,
+        skippedFileCount: 0,
         notRunFileCount: 0,
         durationMs: 1200,
         totalTaskDurationMs: 2400,
@@ -28,11 +30,13 @@ describe("runner reporting", () => {
         skipped: true,
         suiteCount: 0,
         completedSuiteCount: 0,
+        skippedSuiteCount: 0,
         failedSuiteCount: 0,
         totalFileCount: 0,
         completedFileCount: 0,
         passedFileCount: 0,
         failedFileCount: 0,
+        skippedFileCount: 0,
         notRunFileCount: 0,
         durationMs: 0,
         totalTaskDurationMs: 0,
@@ -49,10 +53,9 @@ describe("runner reporting", () => {
       finishedAt: 4000,
       requestedJobs: 2,
       workerCount: 1,
-      suiteType: "all",
-      suiteNames: [],
+      typeValues: ["all"],
+      suiteSelectors: [],
       fileNames: [],
-      framework: "all",
       shard: null,
       serviceFilter: null,
       metadata: {
@@ -71,11 +74,25 @@ describe("runner reporting", () => {
     });
 
     expect(artifact.product.name).toBe("my-product");
-    expect(artifact.summary.services.total).toBe(1);
+    expect(artifact.schemaVersion).toBe(3);
+    expect(artifact.summary.services).toEqual({
+      total: 1,
+      passed: 1,
+      failed: 0,
+      skipped: 0,
+    });
+    expect(artifact.summary.suites).toEqual({
+      total: 1,
+      completed: 1,
+      passed: 1,
+      failed: 0,
+      skipped: 0,
+    });
     expect(artifact.summary.files).toEqual({
       total: 3,
       passed: 3,
       failed: 0,
+      skipped: 0,
       notRun: 0,
     });
     expect(artifact.services[0].durationMs).toBe(1200);
@@ -93,20 +110,23 @@ describe("runner reporting", () => {
           suites: [
             {
               name: "health",
-              type: "integration",
+              type: "int",
               framework: "k6",
               files: [
                 { path: "tests/api/integration/a.int.testkit.ts", status: "passed" },
-                { path: "tests/api/integration/b.int.testkit.ts", status: "failed" },
+                {
+                  path: "tests/api/integration/b.int.testkit.ts",
+                  status: "skipped",
+                  reason: "Billing is stubbed",
+                },
               ],
             },
           ],
         },
       ],
-      suiteType: "int",
-      suiteNames: ["health"],
+      typeValues: ["int"],
+      suiteSelectors: [{ kind: "plain", name: "health", raw: "health" }],
       fileNames: ["tests/api/integration/b.int.testkit.ts"],
-      framework: "default",
       shard: null,
       serviceFilter: "api",
       metadata: {
@@ -119,7 +139,7 @@ describe("runner reporting", () => {
     });
 
     expect(status).toEqual({
-      schemaVersion: 1,
+      schemaVersion: 3,
       source: "testkit",
       notice: "Generated file. Do not edit manually.",
       product: {
@@ -131,10 +151,9 @@ describe("runner reporting", () => {
       },
       testkitVersion: "0.1.20",
       scope: {
-        suiteType: "int",
-        suiteNames: ["health"],
+        types: ["int"],
+        suiteSelectors: ["health"],
         fileNames: ["tests/api/integration/b.int.testkit.ts"],
-        framework: "default",
         shard: null,
         serviceFilter: "api",
         isFullRun: false,
@@ -144,26 +163,29 @@ describe("runner reporting", () => {
           total: 1,
           passed: 0,
           failed: 1,
+          skipped: 0,
         },
         tests: {
           total: 2,
           passed: 1,
-          failed: 1,
+          failed: 0,
+          skipped: 1,
           notRun: 0,
         },
       },
       tests: [
         {
           service: "api",
-          type: "integration",
+          type: "int",
           path: "tests/api/integration/a.int.testkit.ts",
           status: "passed",
         },
         {
           service: "api",
-          type: "integration",
+          type: "int",
           path: "tests/api/integration/b.int.testkit.ts",
-          status: "failed",
+          status: "skipped",
+          reason: "Billing is stubbed",
         },
       ],
     });
@@ -173,10 +195,9 @@ describe("runner reporting", () => {
     const status = buildStatusArtifact({
       productDir: "/tmp/my-product",
       results: [],
-      suiteType: "all",
-      suiteNames: [],
+      typeValues: ["all"],
+      suiteSelectors: [],
       fileNames: [],
-      framework: "all",
       shard: null,
       serviceFilter: null,
       metadata: {

package/lib/runner/results.mjs

@@ -26,14 +26,15 @@ export function buildServiceTrackers(servicePlans, startedAt) {
       key: `${suite.type}:${suite.name}`,
       name: suite.name,
       type: suite.type,
+      displayType: suite.displayType || suite.type,
       framework: suite.framework,
       orderIndex: suite.orderIndex,
-      fileCount: suite.files.length,
+      fileCount: suite.totalFileCount ?? suite.files.length,
       completedFileCount: 0,
       failedFiles: [],
       failedFileSet: new Set(),
-      fileResultsByPath: new Map(
-        suite.files.map((file) => {
+      fileResultsByPath: new Map([
+        ...suite.files.map((file) => {
           const normalizedPath = normalizePathSeparators(file);
           return [
             normalizedPath,
@@ -42,11 +43,23 @@ export function buildServiceTrackers(servicePlans, startedAt) {
               failed: false,
               durationMs: 0,
               error: null,
+              reason: null,
               status: "not_run",
             },
           ];
-        })
-      ),
+        }),
+        ...(suite.skippedFiles || []).map((file) => [
+          file.path,
+          {
+            path: file.path,
+            failed: false,
+            durationMs: 0,
+            error: null,
+            reason: file.reason,
+            status: "skipped",
+          },
+        ]),
+      ]),
       durationMs: 0,
       error: null,
     }));
@@ -92,25 +105,30 @@ export function recordTaskOutcome(trackers, task, outcome, finishedAt = Date.now
       : Math.max(tracker.lastTaskAt, outcomeFinishedAt);
   tracker.totalTaskDurationMs += outcomeDurationMs;
 
-  suite.completedFileCount += 1;
   suite.durationMs += outcomeDurationMs;
   const normalizedPath = normalizePathSeparators(task.file);
   const existingFileResult = suite.fileResultsByPath.get(normalizedPath);
+  const status = normalizeOutcomeStatus(outcome);
+  if (status !== "skipped") {
+    suite.completedFileCount += 1;
+  }
   if (existingFileResult) {
-    existingFileResult.failed = outcome.failed;
+    existingFileResult.failed = status === "failed";
     existingFileResult.durationMs = outcomeDurationMs;
     existingFileResult.error = outcome.error;
-    existingFileResult.status = outcome.failed ? "failed" : "passed";
+    existingFileResult.reason = outcome.reason || null;
+    existingFileResult.status = status;
   } else {
     suite.fileResultsByPath.set(normalizedPath, {
       path: normalizedPath,
-      failed: outcome.failed,
+      failed: status === "failed",
       durationMs: outcomeDurationMs,
       error: outcome.error,
-      status: outcome.failed ? "failed" : "passed",
+      reason: outcome.reason || null,
+      status,
     });
   }
-  if (outcome.failed && !suite.failedFileSet.has(task.file)) {
+  if (status === "failed" && !suite.failedFileSet.has(task.file)) {
     suite.failedFileSet.add(task.file);
     suite.failedFiles.push(task.file);
   }
@@ -144,7 +162,14 @@ export function finalizeServiceResult(tracker, startedAt, finishedAt) {
       skipped: true,
       suiteCount: 0,
       completedSuiteCount: 0,
+      skippedSuiteCount: 0,
       failedSuiteCount: 0,
+      totalFileCount: 0,
+      completedFileCount: 0,
+      passedFileCount: 0,
+      failedFileCount: 0,
+      skippedFileCount: 0,
+      notRunFileCount: 0,
       durationMs: 0,
       totalTaskDurationMs: 0,
       suites: [],
@@ -157,14 +182,18 @@ export function finalizeServiceResult(tracker, startedAt, finishedAt) {
     .map((suite) => finalizeSuite(suite));
 
   const completedSuiteCount = suites.filter(
-    (suite) => suite.completedFileCount === suite.fileCount
+    (suite) => suite.completedFileCount + suite.skippedFileCount === suite.fileCount
+  ).length;
+  const skippedSuiteCount = suites.filter(
+    (suite) => suite.skippedFileCount === suite.fileCount && suite.fileCount > 0
   ).length;
   const failedSuiteCount = suites.filter((suite) => suite.failedFileCount > 0).length;
   const totalFileCount = suites.reduce((sum, suite) => sum + suite.fileCount, 0);
   const completedFileCount = suites.reduce((sum, suite) => sum + suite.completedFileCount, 0);
   const failedFileCount = suites.reduce((sum, suite) => sum + suite.failedFileCount, 0);
   const passedFileCount = suites.reduce((sum, suite) => sum + suite.passedFileCount, 0);
-  const notRunFileCount = totalFileCount - completedFileCount;
+  const skippedFileCount = suites.reduce((sum, suite) => sum + suite.skippedFileCount, 0);
+  const notRunFileCount = totalFileCount - completedFileCount - skippedFileCount;
   const totalTaskDurationMs =
     tracker.totalTaskDurationMs || suites.reduce((sum, suite) => sum + suite.durationMs, 0);
   const durationMs =
@@ -182,11 +211,13 @@ export function finalizeServiceResult(tracker, startedAt, finishedAt) {
     skipped: false,
     suiteCount: tracker.suiteCount,
     completedSuiteCount,
+    skippedSuiteCount,
     failedSuiteCount,
     totalFileCount,
     completedFileCount,
     passedFileCount,
     failedFileCount,
+    skippedFileCount,
     notRunFileCount,
     durationMs,
     totalTaskDurationMs,
@@ -211,18 +242,23 @@ function finalizeSuite(suite) {
       status: file.status,
       durationMs: file.durationMs,
       error: file.error,
+      reason: file.reason,
     }));
 
   return {
     name: suite.name,
-    type: suite.type,
+    type: suite.displayType,
     framework: formatFrameworkForArtifact(suite.framework),
     failed: suite.failedFiles.length > 0,
     fileCount: suite.fileCount,
     completedFileCount: suite.completedFileCount,
     passedFileCount: files.filter((file) => file.status === "passed").length,
     failedFileCount: suite.failedFiles.length,
-    notRunFileCount: suite.fileCount - suite.completedFileCount,
+    skippedFileCount: files.filter((file) => file.status === "skipped").length,
+    notRunFileCount:
+      suite.fileCount -
+      suite.completedFileCount -
+      files.filter((file) => file.status === "skipped").length,
     failedFiles: suite.failedFiles,
     durationMs: suite.durationMs,
     error: suite.error,
@@ -238,3 +274,8 @@ function formatFrameworkForArtifact(framework) {
   if (framework === "k6") return "default";
   return framework;
 }
+
+function normalizeOutcomeStatus(outcome) {
+  if (outcome?.status === "skipped") return "skipped";
+  return outcome?.failed ? "failed" : "passed";
+}

package/lib/runner/results.test.mjs

@@ -72,6 +72,7 @@ describe("runner results", () => {
         status: "failed",
         durationMs: 250,
         error: "boom",
+        reason: null,
       },
     ]);
   });
@@ -193,6 +194,119 @@ describe("runner results", () => {
     expect(result.totalTaskDurationMs).toBe(3_000);
   });
 
+  it("counts config-skipped files and runtime-skipped files separately from passed work", () => {
+    const trackers = buildServiceTrackers(
+      [
+        {
+          skipped: false,
+          config: {
+            name: "api",
+            testkit: {
+              database: {
+                selectedBackend: null,
+              },
+            },
+          },
+          suites: [
+            {
+              name: "billing",
+              type: "integration",
+              framework: "k6",
+              files: ["tests/billing-live.int.testkit.ts"],
+              skippedFiles: [
+                {
+                  path: "tests/billing-stubbed.int.testkit.ts",
+                  reason: "Billing is stubbed",
+                },
+              ],
+              totalFileCount: 2,
+              orderIndex: 0,
+            },
+            {
+              name: "frontend-smoke",
+              type: "e2e",
+              framework: "playwright",
+              files: ["tests/frontend-smoke.pw.testkit.ts"],
+              orderIndex: 1,
+            },
+          ],
+        },
+      ],
+      1000
+    );
+
+    recordTaskOutcome(
+      trackers,
+      {
+        serviceName: "api",
+        suiteKey: "integration:billing",
+        file: "tests/billing-live.int.testkit.ts",
+      },
+      {
+        failed: false,
+        durationMs: 100,
+        error: null,
+      },
+      1100
+    );
+    recordTaskOutcome(
+      trackers,
+      {
+        serviceName: "api",
+        suiteKey: "e2e:frontend-smoke",
+        file: "tests/frontend-smoke.pw.testkit.ts",
+      },
+      {
+        failed: false,
+        status: "skipped",
+        reason: "Playwright test.skip()",
+        durationMs: 50,
+        error: null,
+      },
+      1150
+    );
+
+    const result = finalizeServiceResult(trackers.get("api"), 1000, 1200);
+
+    expect(result.failed).toBe(false);
+    expect(result.completedSuiteCount).toBe(2);
+    expect(result.skippedSuiteCount).toBe(1);
+    expect(result.totalFileCount).toBe(3);
+    expect(result.completedFileCount).toBe(1);
+    expect(result.passedFileCount).toBe(1);
+    expect(result.failedFileCount).toBe(0);
+    expect(result.skippedFileCount).toBe(2);
+    expect(result.notRunFileCount).toBe(0);
+    expect(result.suites[0].files).toEqual([
+      {
+        path: "tests/billing-live.int.testkit.ts",
+        failed: false,
+        status: "passed",
+        durationMs: 100,
+        error: null,
+        reason: null,
+      },
+      {
+        path: "tests/billing-stubbed.int.testkit.ts",
+        failed: false,
+        status: "skipped",
+        durationMs: 0,
+        error: null,
+        reason: "Billing is stubbed",
+      },
+    ]);
+    expect(result.suites[1].files).toEqual([
+      {
+        path: "tests/frontend-smoke.pw.testkit.ts",
+        failed: false,
+        status: "skipped",
+        durationMs: 50,
+        error: null,
+        reason: "Playwright test.skip()",
+      },
+    ]);
+  });
+
   it("summarizes mixed db backends", () => {
     expect(
       summarizeDbBackend([{ dbBackend: "local" }, { dbBackend: "neon" }])

package/lib/runner/selection.mjs

@@ -1,15 +1,16 @@
 export function findUnmatchedRequestedFiles(
   configs,
-  suiteType,
-  suiteNames,
-  framework,
+  typeValues,
+  suiteSelectors,
   fileNames,
   collectSuites,
   normalizePathSeparators
 ) {
   const matchedFiles = new Set();
   for (const config of configs) {
-    const suites = collectSuites(config, suiteType, suiteNames, framework, []);
+    const suites = collectSuites(config, typeValues, suiteSelectors, [], {
+      ignoreSkipRules: true,
+    });
     for (const suite of suites) {
       for (const file of suite.files) {
         matchedFiles.add(normalizePathSeparators(file));
@@ -22,11 +23,12 @@ export function findUnmatchedRequestedFiles(
   );
 }
 
-export function isFullRunSelection(suiteNames, fileNames, framework, shard, serviceFilter) {
+export function isFullRunSelection(typeValues, suiteSelectors, fileNames, shard, serviceFilter) {
   return (
-    (suiteNames || []).length === 0 &&
+    (typeValues || []).length === 1 &&
+    typeValues[0] === "all" &&
+    (suiteSelectors || []).length === 0 &&
     (fileNames || []).length === 0 &&
-    (framework || "all") === "all" &&
     (shard || null) === null &&
     (serviceFilter || null) === null
   );

package/lib/runner/selection.test.mjs

@@ -5,9 +5,8 @@ describe("runner selection", () => {
   it("finds unmatched requested files", () => {
     const unmatched = findUnmatchedRequestedFiles(
       [{ name: "api" }],
-      "int",
+      ["int"],
       [],
-      "all",
       ["tests/a.int.testkit.ts", "tests/missing.int.testkit.ts"],
       () => [{ files: ["tests/a.int.testkit.ts"] }],
       (value) => value
@@ -17,9 +16,9 @@ describe("runner selection", () => {
   });
 
   it("detects a full run selection", () => {
-    expect(isFullRunSelection([], [], "all", null, null)).toBe(true);
-    expect(isFullRunSelection(["auth"], [], "all", null, null)).toBe(false);
-    expect(isFullRunSelection([], ["a"], "all", null, null)).toBe(false);
-    expect(isFullRunSelection([], [], "playwright", null, null)).toBe(false);
+    expect(isFullRunSelection(["all"], [], [], null, null)).toBe(true);
+    expect(isFullRunSelection(["all"], [{ raw: "auth" }], [], null, null)).toBe(false);
+    expect(isFullRunSelection(["all"], [], ["a"], null, null)).toBe(false);
+    expect(isFullRunSelection(["int"], [], [], null, null)).toBe(false);
   });
 });

package/lib/runner/suite-selection.mjs

@@ -0,0 +1,91 @@
+const USER_TYPES = new Set(["int", "e2e", "dal", "load", "pw", "all"]);
+
+export function normalizeTypeValues(values = []) {
+  const expanded = [];
+  for (const rawValue of values) {
+    if (rawValue == null) continue;
+    for (const part of String(rawValue).split(",")) {
+      const value = part.trim();
+      if (!value) continue;
+      if (!USER_TYPES.has(value)) {
+        throw new Error(
+          `Unknown type "${value}". Expected one of: int, e2e, dal, load, pw, all.`
+        );
+      }
+      expanded.push(value);
+    }
+  }
+
+  if (expanded.length === 0) {
+    return ["all"];
+  }
+
+  const deduped = [...new Set(expanded)];
+  if (deduped.includes("all") && deduped.length > 1) {
+    throw new Error(`"--type all" cannot be combined with other types.`);
+  }
+
+  const order = ["int", "e2e", "dal", "load", "pw", "all"];
+  return deduped.sort((left, right) => order.indexOf(left) - order.indexOf(right));
+}
+
+export function isAllTypeSelection(typeValues = []) {
+  return typeValues.length === 0 || (typeValues.length === 1 && typeValues[0] === "all");
+}
+
+export function parseSuiteSelectors(values = []) {
+  const selectors = [];
+
+  for (const rawValue of values) {
+    if (rawValue == null) continue;
+    for (const part of String(rawValue).split(",")) {
+      const value = part.trim();
+      if (!value) continue;
+
+      const typeMatch = value.match(/^([a-z]+):(.*)$/);
+      if (!typeMatch) {
+        selectors.push({ kind: "plain", name: value, raw: value });
+        continue;
+      }
+
+      const type = typeMatch[1];
+      const name = typeMatch[2].trim();
+      if (!USER_TYPES.has(type) || type === "all") {
+        throw new Error(
+          `Unknown suite selector type "${type}". Expected one of: int, e2e, dal, load, pw.`
+        );
+      }
+      if (!name) {
+        throw new Error(`Invalid suite selector "${value}". Expected "<type>:<suite-name>".`);
+      }
+
+      selectors.push({
+        kind: "typed",
+        type,
+        name,
+        raw: `${type}:${name}`,
+      });
+    }
+  }
+
+  return selectors;
+}
+
+export function suiteSelectionType(type, framework) {
+  if ((framework || "k6") === "playwright") return "pw";
+  if (type === "integration") return "int";
+  return type;
+}
+
+export function matchesSelectedTypes(selectionType, typeValues) {
+  return isAllTypeSelection(typeValues) || typeValues.includes(selectionType);
+}
+
+export function matchesSuiteSelectors(selectionType, suiteName, selectors = []) {
+  if (!selectors || selectors.length === 0) return true;
+
+  return selectors.some((selector) => {
+    if (selector.kind === "plain") return selector.name === suiteName;
+    return selector.type === selectionType && selector.name === suiteName;
+  });
+}

package/lib/runner/suite-selection.test.mjs

@@ -0,0 +1,42 @@
+import { describe, expect, it } from "vitest";
+import {
+  isAllTypeSelection,
+  matchesSelectedTypes,
+  matchesSuiteSelectors,
+  normalizeTypeValues,
+  parseSuiteSelectors,
+  suiteSelectionType,
+} from "./suite-selection.mjs";
+
+describe("runner suite selection", () => {
+  it("normalizes selected type values", () => {
+    expect(normalizeTypeValues([])).toEqual(["all"]);
+    expect(normalizeTypeValues(["int,e2e", "dal"])).toEqual(["int", "e2e", "dal"]);
+    expect(() => normalizeTypeValues(["all", "int"])).toThrow("cannot be combined");
+    expect(() => normalizeTypeValues(["jest"])).toThrow("Unknown type");
+  });
+
+  it("parses suite selectors", () => {
+    expect(parseSuiteSelectors(["auth,dal:queries"])).toEqual([
+      { kind: "plain", name: "auth", raw: "auth" },
+      { kind: "typed", type: "dal", name: "queries", raw: "dal:queries" },
+    ]);
+    expect(() => parseSuiteSelectors(["all:auth"])).toThrow("Unknown suite selector type");
+    expect(() => parseSuiteSelectors(["int:"])).toThrow("Invalid suite selector");
+  });
+
+  it("matches suite types and selectors", () => {
+    expect(isAllTypeSelection(["all"])).toBe(true);
+    expect(matchesSelectedTypes("int", ["int", "dal"])).toBe(true);
+    expect(matchesSelectedTypes("pw", ["int", "dal"])).toBe(false);
+    expect(matchesSuiteSelectors("int", "auth", parseSuiteSelectors(["auth"]))).toBe(true);
+    expect(matchesSuiteSelectors("e2e", "auth", parseSuiteSelectors(["int:auth"]))).toBe(false);
+    expect(matchesSuiteSelectors("int", "auth", parseSuiteSelectors(["int:auth"]))).toBe(true);
+  });
+
+  it("maps discovered suites to user-facing selection types", () => {
+    expect(suiteSelectionType("integration", "k6")).toBe("int");
+    expect(suiteSelectionType("e2e", "playwright")).toBe("pw");
+    expect(suiteSelectionType("dal", "k6")).toBe("dal");
+  });
+});

package/lib/setup/index.d.ts

@@ -18,6 +18,21 @@ export interface LifecycleConfig {
   testkitCwd?: string;
 }
 
+export interface SkipFileRule {
+  path: string;
+  reason: string;
+}
+
+export interface SkipSuiteRule {
+  selector: string;
+  reason: string;
+}
+
+export interface SkipConfig {
+  files?: SkipFileRule[];
+  suites?: SkipSuiteRule[];
+}
+
 export interface ServiceConfig {
   database?: LocalDatabaseConfig;
   databaseFrom?: string;
@@ -38,6 +53,7 @@ export interface ServiceConfig {
   };
   migrate?: LifecycleConfig;
   seed?: LifecycleConfig;
+  skip?: SkipConfig;
 }
 
 export interface TestkitSetup {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/testkit",
-  "version": "0.1.34",
+  "version": "0.1.36",
   "description": "CLI for discovering and running local HTTP, DAL, and Playwright test suites",
   "type": "module",
   "types": "./lib/index.d.ts",