@elench/testkit 0.1.34 → 0.1.36

package/lib/runner/formatting.mjs

@@ -7,12 +7,18 @@ export function formatDuration(durationMs) {
 }
 
 export function formatServiceSummary(result) {
-  const passedSuites = result.completedSuiteCount - result.failedSuiteCount;
+  const skippedSuites = result.skippedSuiteCount || 0;
+  const passedSuites = result.completedSuiteCount - result.failedSuiteCount - skippedSuites;
   const notRunSuites = result.suiteCount - result.completedSuiteCount;
   let detail = `${passedSuites}/${result.suiteCount} suites passed`;
   if ((result.totalFileCount || 0) > 0) {
     detail += `, ${result.passedFileCount}/${result.totalFileCount} files passed`;
   }
+  if (skippedSuites > 0) {
+    detail += `, ${skippedSuites} ${pluralize(skippedSuites, "suite", "suites")} skipped`;
+  } else if ((result.skippedFileCount || 0) > 0) {
+    detail += `, ${result.skippedFileCount} ${pluralize(result.skippedFileCount, "file", "files")} skipped`;
+  }
   if (notRunSuites > 0) {
     detail += `, ${notRunSuites} ${pluralize(notRunSuites, "suite", "suites")} not run`;
   } else if ((result.notRunFileCount || 0) > 0) {
@@ -66,10 +72,18 @@ export function buildRunSummaryLines(results, durationMs) {
     (sum, result) => sum + result.completedSuiteCount,
     0
   );
+  const skippedSuites = executedServices.reduce(
+    (sum, result) => sum + (result.skippedSuiteCount || 0),
+    0
+  );
   const failedSuites = executedServices.reduce((sum, result) => sum + result.failedSuiteCount, 0);
-  const passedSuites = completedSuites - failedSuites;
+  const passedSuites = completedSuites - failedSuites - skippedSuites;
   const totalFiles = executedServices.reduce((sum, result) => sum + (result.totalFileCount || 0), 0);
   const passedFiles = executedServices.reduce((sum, result) => sum + (result.passedFileCount || 0), 0);
+  const skippedFiles = executedServices.reduce(
+    (sum, result) => sum + (result.skippedFileCount || 0),
+    0
+  );
   const lines = [
     "",
     "══ Summary ══",
@@ -77,6 +91,8 @@ export function buildRunSummaryLines(results, durationMs) {
       `services ${passedServices.length}/${executedServices.length} passed`,
       `suites ${passedSuites}/${totalSuites} passed`,
       totalFiles > 0 ? `files ${passedFiles}/${totalFiles} passed` : null,
+      skippedSuites > 0 ? `suites ${skippedSuites} skipped` : null,
+      skippedFiles > 0 ? `files ${skippedFiles} skipped` : null,
       skippedServices.length > 0 ? `${skippedServices.length} skipped` : null,
       `duration ${formatDuration(durationMs)}`,
     ]
@@ -85,8 +101,14 @@ export function buildRunSummaryLines(results, durationMs) {
   ];
 
   for (const result of results) {
-    const status = result.skipped ? "SKIP" : result.failed ? "FAIL" : "PASS";
-    const detail = result.skipped ? "no matching suites" : formatServiceSummary(result);
+    const status = isServiceEffectivelySkipped(result)
+      ? "SKIP"
+      : result.failed
+        ? "FAIL"
+        : "PASS";
+    const detail = result.skipped
+      ? "no matching suites"
+      : formatServiceSummary(result);
     lines.push(
       `${status.padEnd(4)} ${result.name.padEnd(longestServiceName(results))}  ${detail}  ·  ${formatDuration(result.durationMs)}`
     );
@@ -127,3 +149,13 @@ function sanitizeErrorMessage(message) {
 function pluralize(value, singular, plural) {
   return value === 1 ? singular : plural;
 }
+
+function isServiceEffectivelySkipped(result) {
+  if (result.skipped) return true;
+  return (
+    !result.failed &&
+    (result.skippedSuiteCount || 0) > 0 &&
+    (result.skippedSuiteCount || 0) === result.suiteCount &&
+    (result.notRunFileCount || 0) === 0
+  );
+}

package/lib/runner/formatting.test.mjs

@@ -28,14 +28,31 @@ describe("runner formatting", () => {
       formatServiceSummary({
         completedSuiteCount: 2,
         failedSuiteCount: 1,
+        skippedSuiteCount: 0,
         suiteCount: 3,
         totalFileCount: 5,
         passedFileCount: 3,
+        skippedFileCount: 0,
         notRunFileCount: 1,
       })
     ).toBe("1/3 suites passed, 3/5 files passed, 1 suite not run");
   });
 
+  it("formats skipped suites distinctly from passed suites", () => {
+    expect(
+      formatServiceSummary({
+        completedSuiteCount: 2,
+        failedSuiteCount: 0,
+        skippedSuiteCount: 1,
+        suiteCount: 2,
+        totalFileCount: 3,
+        passedFileCount: 1,
+        skippedFileCount: 2,
+        notRunFileCount: 0,
+      })
+    ).toBe("1/2 suites passed, 1/3 files passed, 1 suite skipped");
+  });
+
   it("formats batch descriptors", () => {
     expect(formatBatchDescriptor({ framework: "k6", tasks: [{}, {}] })).toBe(" (2 files)");
     expect(formatBatchDescriptor({ framework: "playwright", tasks: [{}] })).toBe(
@@ -71,9 +88,12 @@ describe("runner formatting", () => {
           failed: true,
           suiteCount: 2,
           completedSuiteCount: 2,
+          skippedSuiteCount: 0,
           failedSuiteCount: 1,
           totalFileCount: 3,
           passedFileCount: 2,
+          skippedFileCount: 0,
+          notRunFileCount: 0,
           durationMs: 20_000,
           suites: [
             {
@@ -97,4 +117,33 @@ describe("runner formatting", () => {
     expect(lines.join("\n")).toContain("worker error: worker broke");
     expect(lines.at(-1)).toBe("Result: FAILED (1/1 services failed)");
   });
+
+  it("marks services with only skipped suites as SKIP", () => {
+    const lines = buildRunSummaryLines(
+      [
+        {
+          name: "api",
+          skipped: false,
+          failed: false,
+          suiteCount: 1,
+          completedSuiteCount: 1,
+          skippedSuiteCount: 1,
+          failedSuiteCount: 0,
+          totalFileCount: 1,
+          passedFileCount: 0,
+          skippedFileCount: 1,
+          notRunFileCount: 0,
+          durationMs: 0,
+          suites: [],
+          errors: [],
+        },
+      ],
+      0
+    );
+
+    expect(lines.join("\n")).toContain("suites 1 skipped");
+    expect(lines.join("\n")).toContain("files 1 skipped");
+    expect(lines.join("\n")).toContain("SKIP api");
+    expect(lines.at(-1)).toBe("Result: PASSED");
+  });
 });

package/lib/runner/orchestrator.mjs

@@ -33,7 +33,7 @@ import { createWorker, runWorker } from "./worker-loop.mjs";
 import { findUnmatchedRequestedFiles, isFullRunSelection } from "./selection.mjs";
 import { uploadTelemetryArtifact } from "../telemetry/index.mjs";
 
-export async function runAll(configs, suiteType, suiteNames, opts, allConfigs = configs) {
+export async function runAll(configs, typeValues, suiteSelectors, opts, allConfigs = configs) {
   const configMap = new Map(allConfigs.map((config) => [config.name, config]));
   const startedAt = Date.now();
   const telemetry = configs[0]?.telemetry || null;
@@ -51,9 +51,8 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
   if (requestedFiles.length > 0) {
     const unmatchedFiles = findUnmatchedRequestedFiles(
       configs,
-      suiteType,
-      suiteNames,
-      opts.framework || "all",
+      typeValues,
+      suiteSelectors,
       requestedFiles,
       collectSuites,
       normalizePathSeparators
@@ -69,9 +68,9 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
     opts.writeStatus &&
     !opts.allowPartialStatus &&
     !isFullRunSelection(
-      suiteNames,
+      typeValues,
+      suiteSelectors,
       requestedFiles,
-      opts.framework || "all",
       opts.shard || null,
       opts.serviceFilter || null
     )
@@ -82,7 +81,7 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
     );
   }
 
-  const servicePlans = collectServicePlans(configs, configMap, suiteType, suiteNames, opts);
+  const servicePlans = collectServicePlans(configs, configMap, typeValues, suiteSelectors, opts);
   const trackers = buildServiceTrackers(servicePlans, startedAt);
   const executedPlans = servicePlans.filter((plan) => !plan.skipped);
   let workerCount = 0;
@@ -144,10 +143,9 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
       finishedAt,
       requestedJobs: opts.jobs || 1,
       workerCount,
-      suiteType,
-      suiteNames,
+      typeValues,
+      suiteSelectors,
       fileNames: requestedFiles,
-      framework: opts.framework || "all",
       shard: opts.shard || null,
       serviceFilter: opts.serviceFilter || null,
       metadata,
@@ -161,10 +159,9 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
         buildStatusArtifact({
           productDir,
           results,
-          suiteType,
-          suiteNames,
+          typeValues,
+          suiteSelectors,
           fileNames: requestedFiles,
-          framework: opts.framework || "all",
           shard: opts.shard || null,
           serviceFilter: opts.serviceFilter || null,
           metadata,
@@ -188,17 +185,17 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
   }
 }
 
-function collectServicePlans(configs, configMap, suiteType, suiteNames, opts) {
+function collectServicePlans(configs, configMap, typeValues, suiteSelectors, opts) {
   return configs.map((config) => {
     console.log(`\n══ ${config.name} ══`);
     const suites = applyShard(
-      collectSuites(config, suiteType, suiteNames, opts.framework, opts.fileNames || []),
+      collectSuites(config, typeValues, suiteSelectors, opts.fileNames || [], opts),
       opts.shard
     );
 
     if (suites.length === 0) {
       console.log(
-        `No test files for ${config.name} type="${suiteType}" suites=${suiteNames.join(",") || "all"} files=${(opts.fileNames || []).join(",") || "all"} — skipping`
+        `No test files for ${config.name} types=${typeValues.join(",") || "all"} suites=${suiteSelectors.map((selector) => selector.raw).join(",") || "all"} files=${(opts.fileNames || []).join(",") || "all"} — skipping`
       );
       return {
         config,

package/lib/runner/planning.mjs

@@ -1,4 +1,9 @@
 import { buildTimingKey, estimateTaskDuration } from "../timing/index.mjs";
+import {
+  matchesSelectedTypes,
+  matchesSuiteSelectors,
+  suiteSelectionType,
+} from "./suite-selection.mjs";
 
 const TYPE_ORDER = ["dal", "integration", "e2e", "load"];
 
@@ -34,44 +39,50 @@ export function resolveRuntimeConfigs(targetConfig, configMap) {
   return ordered;
 }
 
-export function collectSuites(config, suiteType, suiteNames, frameworkFilter, fileNames = []) {
-  const types =
-    suiteType === "all"
-      ? orderedTypes(Object.keys(config.suites))
-      : [suiteType === "int" ? "integration" : suiteType];
-
-  const selectedNames = new Set(suiteNames);
+export function collectSuites(config, typeValues, suiteSelectors, fileNames = [], opts = {}) {
   const selectedFiles = new Set(fileNames.map(normalizePathSeparators));
   const suites = [];
   let orderIndex = 0;
 
-  for (const type of types) {
+  for (const type of orderedTypes(Object.keys(config.suites))) {
     for (const suite of config.suites[type] || []) {
       const framework = suite.framework || "k6";
-      const files =
+      const displayType = suiteSelectionType(type, framework);
+      const selectedSuiteFiles =
         selectedFiles.size === 0
           ? suite.files
           : suite.files.filter((file) => selectedFiles.has(normalizePathSeparators(file)));
-      if (selectedNames.size > 0 && !selectedNames.has(suite.name)) continue;
-      if (frameworkFilter && frameworkFilter !== "all" && framework !== frameworkFilter) continue;
-      if (files.length === 0) continue;
+      if (!matchesSelectedTypes(displayType, typeValues)) continue;
+      if (!matchesSuiteSelectors(displayType, suite.name, suiteSelectors)) continue;
+      if (selectedSuiteFiles.length === 0) continue;
+
+      const { files, skippedFiles } = applySkipRules(
+        config,
+        displayType,
+        suite.name,
+        selectedSuiteFiles,
+        opts
+      );
 
       suites.push({
         ...suite,
         files,
+        skippedFiles,
         framework,
         type,
+        displayType,
         orderIndex,
         sortKey: `${type}:${suite.name}`,
         weight:
           suite.testkit?.weight ||
           (framework === "playwright"
-            ? Math.max(2, files.length)
+            ? Math.max(2, Math.max(1, files.length))
             : Math.max(1, files.length)),
         maxFileConcurrency:
           framework === "k6" || framework === "playwright"
             ? suite.testkit?.maxFileConcurrency || 1
             : 1,
+        totalFileCount: selectedSuiteFiles.length,
       });
       orderIndex += 1;
     }
@@ -261,6 +272,47 @@ function normalizePathSeparators(filePath) {
   return String(filePath).split("\\").join("/");
 }
 
+function applySkipRules(config, displayType, suiteName, files, opts = {}) {
+  if (opts.ignoreSkipRules) {
+    return {
+      files,
+      skippedFiles: [],
+    };
+  }
+  const skip = config.testkit?.skip;
+  if (!skip) {
+    return {
+      files,
+      skippedFiles: [],
+    };
+  }
+
+  const matchingSuiteRules = skip.suites.filter((rule) =>
+    matchesSuiteSelectors(displayType, suiteName, [rule.selector])
+  );
+  const suiteReason = matchingSuiteRules[0]?.reason || null;
+  const runnableFiles = [];
+  const skippedFiles = [];
+
+  for (const file of files) {
+    const normalizedFile = normalizePathSeparators(file);
+    const reason = skip.fileReasonByPath.get(normalizedFile) || suiteReason;
+    if (reason) {
+      skippedFiles.push({
+        path: normalizedFile,
+        reason,
+      });
+      continue;
+    }
+    runnableFiles.push(file);
+  }
+
+  return {
+    files: runnableFiles,
+    skippedFiles,
+  };
+}
+
 export function buildGraphDirName(runtimeNames) {
   const slug = runtimeNames.map(slugSegment).join("__");
   return slug.length > 0 ? slug : "graph";

package/lib/runner/planning.test.mjs

@@ -38,7 +38,7 @@ describe("runner-planning", () => {
     expect(() => resolveRuntimeConfigs(frontend, configMap)).toThrow("Dependency cycle");
   });
 
-  it("collects suites with aliases, weights, and framework filters", () => {
+  it("collects suites with user-facing type selection", () => {
     const config = makeConfig("api", {
       suites: {
         integration: [
@@ -57,15 +57,17 @@ describe("runner-planning", () => {
       },
     });
 
-    expect(collectSuites(config, "int", [], "all")[0]).toMatchObject({
+    expect(collectSuites(config, ["int"], [], [])[0]).toMatchObject({
       name: "health",
       type: "integration",
+      displayType: "int",
       framework: "k6",
       weight: 1,
     });
 
-    expect(collectSuites(config, "all", [], "playwright")[0]).toMatchObject({
+    expect(collectSuites(config, ["pw"], [], [])[0]).toMatchObject({
       name: "auth",
+      displayType: "pw",
       framework: "playwright",
       weight: 2,
     });
@@ -73,9 +75,8 @@ describe("runner-planning", () => {
     expect(
       collectSuites(
         config,
-        "all",
+        ["all"],
         [],
-        "all",
         ["tests/frontend/e2e/signup.pw.testkit.ts"]
       )[0]
     ).toMatchObject({
@@ -84,6 +85,57 @@ describe("runner-planning", () => {
     });
   });
 
+  it("keeps skipped files visible while removing them from runnable work", () => {
+    const config = makeConfig("api", {
+      suites: {
+        integration: [
+          {
+            name: "billing",
+            files: [
+              "__testkit__/billing/a.int.testkit.ts",
+              "__testkit__/billing/b.int.testkit.ts",
+            ],
+          },
+        ],
+      },
+      testkit: {
+        dependsOn: [],
+        skip: {
+          fileReasonByPath: new Map([
+            ["__testkit__/billing/a.int.testkit.ts", "Billing is stubbed"],
+          ]),
+          suites: [],
+        },
+      },
+    });
+
+    expect(collectSuites(config, ["int"], [], [])).toEqual([
+      expect.objectContaining({
+        name: "billing",
+        files: ["__testkit__/billing/b.int.testkit.ts"],
+        skippedFiles: [
+          {
+            path: "__testkit__/billing/a.int.testkit.ts",
+            reason: "Billing is stubbed",
+          },
+        ],
+        totalFileCount: 2,
+      }),
+    ]);
+
+    expect(collectSuites(config, ["int"], [], [], { ignoreSkipRules: true })).toEqual([
+      expect.objectContaining({
+        name: "billing",
+        files: [
+          "__testkit__/billing/a.int.testkit.ts",
+          "__testkit__/billing/b.int.testkit.ts",
+        ],
+        skippedFiles: [],
+        totalFileCount: 2,
+      }),
+    ]);
+  });
+
   it("applies shards, builds graphs, queues tasks, and claims batches", () => {
     const api = makeConfig("api");
     const frontend = makeConfig("frontend");

package/lib/runner/playwright-runner.mjs

@@ -62,7 +62,8 @@ export async function runPlaywrightBatch(targetConfig, batch, lifecycle) {
     if (fileResult) {
       return {
         task,
-        failed: fileResult.failed,
+        failed: fileResult.status === "failed",
+        status: fileResult.status,
         error: fileResult.error,
         durationMs:
           fileResult.durationMs > 0

package/lib/runner/reporting.mjs

@@ -3,26 +3,30 @@ import path from "path";
 export function buildStatusArtifact({
   productDir,
   results,
-  suiteType,
-  suiteNames,
+  typeValues,
+  suiteSelectors,
   fileNames,
-  framework,
   shard,
   serviceFilter,
   metadata,
 }) {
   const executedResults = results.filter((result) => !result.skipped);
+  const effectivelySkippedResults = executedResults.filter(isEffectivelySkippedService);
   const tests = [];
 
   for (const result of executedResults) {
     for (const suite of result.suites) {
       for (const file of suite.files) {
-        tests.push({
+        const test = {
           service: result.name,
           type: suite.type,
           path: file.path,
           status: file.status,
-        });
+        };
+        if (file.reason) {
+          test.reason = file.reason;
+        }
+        tests.push(test);
       }
     }
   }
@@ -37,34 +41,38 @@ export function buildStatusArtifact({
   const summary = {
     services: {
       total: executedResults.length,
-      passed: executedResults.filter((result) => !result.failed).length,
+      passed: executedResults.filter(
+        (result) => !result.failed && !isEffectivelySkippedService(result)
+      ).length,
       failed: executedResults.filter((result) => result.failed).length,
+      skipped: effectivelySkippedResults.length,
     },
     tests: {
       total: tests.length,
       passed: tests.filter((test) => test.status === "passed").length,
       failed: tests.filter((test) => test.status === "failed").length,
+      skipped: tests.filter((test) => test.status === "skipped").length,
       notRun: tests.filter((test) => test.status === "not_run").length,
     },
   };
 
   const scope = {
-    suiteType,
-    suiteNames: [...(suiteNames || [])].sort(),
+    types: [...(typeValues || ["all"])].sort(),
+    suiteSelectors: [...(suiteSelectors || [])].map((selector) => selector.raw).sort(),
     fileNames: [...(fileNames || [])].sort(),
-    framework: formatFrameworkForArtifact(framework || "all"),
     shard: shard || null,
     serviceFilter: serviceFilter || null,
   };
   scope.isFullRun =
-    scope.suiteNames.length === 0 &&
+    scope.types.length === 1 &&
+    scope.types[0] === "all" &&
+    scope.suiteSelectors.length === 0 &&
     scope.fileNames.length === 0 &&
-    scope.framework === "all" &&
     scope.shard === null &&
     scope.serviceFilter === null;
 
   return {
-    schemaVersion: 1,
+    schemaVersion: 3,
     source: "testkit",
     notice: "Generated file. Do not edit manually.",
     product: {
@@ -88,28 +96,30 @@ export function buildRunArtifact({
   finishedAt,
   requestedJobs,
   workerCount,
-  suiteType,
-  suiteNames,
+  typeValues,
+  suiteSelectors,
   fileNames,
-  framework,
   shard,
   serviceFilter,
   metadata,
   summarizeDbBackend,
 }) {
   const executed = results.filter((result) => !result.skipped);
+  const effectivelySkippedServices = executed.filter(isEffectivelySkippedService);
   const failedServices = executed.filter((result) => result.failed);
   const totalSuites = executed.reduce((sum, result) => sum + result.suiteCount, 0);
   const completedSuites = executed.reduce((sum, result) => sum + result.completedSuiteCount, 0);
+  const skippedSuites = executed.reduce((sum, result) => sum + (result.skippedSuiteCount || 0), 0);
   const failedSuites = executed.reduce((sum, result) => sum + result.failedSuiteCount, 0);
   const totalFiles = executed.reduce((sum, result) => sum + (result.totalFileCount || 0), 0);
   const passedFiles = executed.reduce((sum, result) => sum + (result.passedFileCount || 0), 0);
   const failedFiles = executed.reduce((sum, result) => sum + (result.failedFileCount || 0), 0);
+  const skippedFiles = executed.reduce((sum, result) => sum + (result.skippedFileCount || 0), 0);
   const notRunFiles = executed.reduce((sum, result) => sum + (result.notRunFileCount || 0), 0);
   const dbBackend = summarizeDbBackend(results);
 
   return {
-    schemaVersion: 1,
+    schemaVersion: 3,
     source: "testkit",
     generatedAt: new Date(finishedAt).toISOString(),
     product: {
@@ -126,10 +136,9 @@ export function buildRunArtifact({
       requestedJobs,
       workerCount,
       dbBackend,
-      suiteType,
-      suiteNames,
+      types: typeValues,
+      suiteSelectors: suiteSelectors.map((selector) => selector.raw),
       fileNames,
-      framework: formatFrameworkForArtifact(framework),
       shard,
       serviceFilter,
       testkitVersion: metadata.testkitVersion,
@@ -137,19 +146,22 @@ export function buildRunArtifact({
     summary: {
       services: {
         total: executed.length,
-        passed: executed.length - failedServices.length,
+        passed: executed.length - failedServices.length - effectivelySkippedServices.length,
         failed: failedServices.length,
+        skipped: effectivelySkippedServices.length,
       },
       suites: {
         total: totalSuites,
         completed: completedSuites,
-        passed: completedSuites - failedSuites,
+        passed: completedSuites - failedSuites - skippedSuites,
         failed: failedSuites,
+        skipped: skippedSuites,
       },
       files: {
         total: totalFiles,
         passed: passedFiles,
         failed: failedFiles,
+        skipped: skippedFiles,
         notRun: notRunFiles,
       },
     },
@@ -159,11 +171,13 @@ export function buildRunArtifact({
       skipped: result.skipped,
       suiteCount: result.suiteCount,
       completedSuiteCount: result.completedSuiteCount,
+      skippedSuiteCount: result.skippedSuiteCount,
       failedSuiteCount: result.failedSuiteCount,
       totalFileCount: result.totalFileCount,
       completedFileCount: result.completedFileCount,
       passedFileCount: result.passedFileCount,
       failedFileCount: result.failedFileCount,
+      skippedFileCount: result.skippedFileCount,
       notRunFileCount: result.notRunFileCount,
       durationMs: result.durationMs,
       totalTaskDurationMs: result.totalTaskDurationMs,
@@ -174,7 +188,12 @@ export function buildRunArtifact({
   };
 }
 
-function formatFrameworkForArtifact(framework) {
-  if (framework === "k6") return "default";
-  return framework;
+function isEffectivelySkippedService(result) {
+  return (
+    !result.skipped &&
+    !result.failed &&
+    (result.skippedSuiteCount || 0) > 0 &&
+    (result.skippedSuiteCount || 0) === result.suiteCount &&
+    (result.notRunFileCount || 0) === 0
+  );
 }