@elench/testkit 0.1.35 → 0.1.37

package/lib/runner/formatting.test.mjs

@@ -28,14 +28,31 @@ describe("runner formatting", () => {
       formatServiceSummary({
         completedSuiteCount: 2,
         failedSuiteCount: 1,
+        skippedSuiteCount: 0,
         suiteCount: 3,
         totalFileCount: 5,
         passedFileCount: 3,
+        skippedFileCount: 0,
         notRunFileCount: 1,
       })
     ).toBe("1/3 suites passed, 3/5 files passed, 1 suite not run");
   });
 
+  it("formats skipped suites distinctly from passed suites", () => {
+    expect(
+      formatServiceSummary({
+        completedSuiteCount: 2,
+        failedSuiteCount: 0,
+        skippedSuiteCount: 1,
+        suiteCount: 2,
+        totalFileCount: 3,
+        passedFileCount: 1,
+        skippedFileCount: 2,
+        notRunFileCount: 0,
+      })
+    ).toBe("1/2 suites passed, 1/3 files passed, 1 suite skipped");
+  });
+
   it("formats batch descriptors", () => {
     expect(formatBatchDescriptor({ framework: "k6", tasks: [{}, {}] })).toBe(" (2 files)");
     expect(formatBatchDescriptor({ framework: "playwright", tasks: [{}] })).toBe(
@@ -71,9 +88,12 @@ describe("runner formatting", () => {
           failed: true,
           suiteCount: 2,
           completedSuiteCount: 2,
+          skippedSuiteCount: 0,
           failedSuiteCount: 1,
           totalFileCount: 3,
           passedFileCount: 2,
+          skippedFileCount: 0,
+          notRunFileCount: 0,
           durationMs: 20_000,
           suites: [
             {
@@ -97,4 +117,33 @@ describe("runner formatting", () => {
     expect(lines.join("\n")).toContain("worker error: worker broke");
     expect(lines.at(-1)).toBe("Result: FAILED (1/1 services failed)");
   });
+
+  it("marks services with only skipped suites as SKIP", () => {
+    const lines = buildRunSummaryLines(
+      [
+        {
+          name: "api",
+          skipped: false,
+          failed: false,
+          suiteCount: 1,
+          completedSuiteCount: 1,
+          skippedSuiteCount: 1,
+          failedSuiteCount: 0,
+          totalFileCount: 1,
+          passedFileCount: 0,
+          skippedFileCount: 1,
+          notRunFileCount: 0,
+          durationMs: 0,
+          suites: [],
+          errors: [],
+        },
+      ],
+      0
+    );
+
+    expect(lines.join("\n")).toContain("suites 1 skipped");
+    expect(lines.join("\n")).toContain("files 1 skipped");
+    expect(lines.join("\n")).toContain("SKIP api");
+    expect(lines.at(-1)).toBe("Result: PASSED");
+  });
 });

package/lib/runner/orchestrator.mjs

@@ -16,7 +16,13 @@ import {
 } from "./results.mjs";
 import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
 import { buildRunSummaryLines, formatError } from "./formatting.mjs";
-import { loadTimings, saveTimings, writeRunArtifact, writeStatusArtifact } from "./artifacts.mjs";
+import {
+  loadTimings,
+  resetResultArtifacts,
+  saveTimings,
+  writeRunArtifact,
+  writeStatusArtifact,
+} from "./artifacts.mjs";
 import {
   cleanupRunById,
   cleanupRuns,
@@ -39,6 +45,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
   const telemetry = configs[0]?.telemetry || null;
   const productDir = configs[0]?.productDir || process.cwd();
   await cleanupStaleRuns(productDir);
+  resetResultArtifacts(productDir);
   const metadata = {
     git: collectGitMetadata(productDir),
     host: {
@@ -189,7 +196,7 @@ function collectServicePlans(configs, configMap, typeValues, suiteSelectors, opt
   return configs.map((config) => {
     console.log(`\n══ ${config.name} ══`);
     const suites = applyShard(
-      collectSuites(config, typeValues, suiteSelectors, opts.fileNames || []),
+      collectSuites(config, typeValues, suiteSelectors, opts.fileNames || [], opts),
       opts.shard
     );
 

package/lib/runner/planning.mjs

@@ -39,7 +39,7 @@ export function resolveRuntimeConfigs(targetConfig, configMap) {
   return ordered;
 }
 
-export function collectSuites(config, typeValues, suiteSelectors, fileNames = []) {
+export function collectSuites(config, typeValues, suiteSelectors, fileNames = [], opts = {}) {
   const selectedFiles = new Set(fileNames.map(normalizePathSeparators));
   const suites = [];
   let orderIndex = 0;
@@ -48,17 +48,26 @@ export function collectSuites(config, typeValues, suiteSelectors, fileNames = []
     for (const suite of config.suites[type] || []) {
       const framework = suite.framework || "k6";
       const displayType = suiteSelectionType(type, framework);
-      const files =
+      const selectedSuiteFiles =
         selectedFiles.size === 0
           ? suite.files
           : suite.files.filter((file) => selectedFiles.has(normalizePathSeparators(file)));
       if (!matchesSelectedTypes(displayType, typeValues)) continue;
       if (!matchesSuiteSelectors(displayType, suite.name, suiteSelectors)) continue;
-      if (files.length === 0) continue;
+      if (selectedSuiteFiles.length === 0) continue;
+
+      const { files, skippedFiles } = applySkipRules(
+        config,
+        displayType,
+        suite.name,
+        selectedSuiteFiles,
+        opts
+      );
 
       suites.push({
         ...suite,
         files,
+        skippedFiles,
         framework,
         type,
         displayType,
@@ -67,12 +76,13 @@ export function collectSuites(config, typeValues, suiteSelectors, fileNames = []
         weight:
           suite.testkit?.weight ||
           (framework === "playwright"
-            ? Math.max(2, files.length)
+            ? Math.max(2, Math.max(1, files.length))
             : Math.max(1, files.length)),
         maxFileConcurrency:
           framework === "k6" || framework === "playwright"
             ? suite.testkit?.maxFileConcurrency || 1
             : 1,
+        totalFileCount: selectedSuiteFiles.length,
       });
       orderIndex += 1;
     }
@@ -262,6 +272,47 @@ function normalizePathSeparators(filePath) {
   return String(filePath).split("\\").join("/");
 }
 
+function applySkipRules(config, displayType, suiteName, files, opts = {}) {
+  if (opts.ignoreSkipRules) {
+    return {
+      files,
+      skippedFiles: [],
+    };
+  }
+  const skip = config.testkit?.skip;
+  if (!skip) {
+    return {
+      files,
+      skippedFiles: [],
+    };
+  }
+
+  const matchingSuiteRules = skip.suites.filter((rule) =>
+    matchesSuiteSelectors(displayType, suiteName, [rule.selector])
+  );
+  const suiteReason = matchingSuiteRules[0]?.reason || null;
+  const runnableFiles = [];
+  const skippedFiles = [];
+
+  for (const file of files) {
+    const normalizedFile = normalizePathSeparators(file);
+    const reason = skip.fileReasonByPath.get(normalizedFile) || suiteReason;
+    if (reason) {
+      skippedFiles.push({
+        path: normalizedFile,
+        reason,
+      });
+      continue;
+    }
+    runnableFiles.push(file);
+  }
+
+  return {
+    files: runnableFiles,
+    skippedFiles,
+  };
+}
+
 export function buildGraphDirName(runtimeNames) {
   const slug = runtimeNames.map(slugSegment).join("__");
   return slug.length > 0 ? slug : "graph";

package/lib/runner/planning.test.mjs

@@ -85,6 +85,57 @@ describe("runner-planning", () => {
     });
   });
 
+  it("keeps skipped files visible while removing them from runnable work", () => {
+    const config = makeConfig("api", {
+      suites: {
+        integration: [
+          {
+            name: "billing",
+            files: [
+              "__testkit__/billing/a.int.testkit.ts",
+              "__testkit__/billing/b.int.testkit.ts",
+            ],
+          },
+        ],
+      },
+      testkit: {
+        dependsOn: [],
+        skip: {
+          fileReasonByPath: new Map([
+            ["__testkit__/billing/a.int.testkit.ts", "Billing is stubbed"],
+          ]),
+          suites: [],
+        },
+      },
+    });
+
+    expect(collectSuites(config, ["int"], [], [])).toEqual([
+      expect.objectContaining({
+        name: "billing",
+        files: ["__testkit__/billing/b.int.testkit.ts"],
+        skippedFiles: [
+          {
+            path: "__testkit__/billing/a.int.testkit.ts",
+            reason: "Billing is stubbed",
+          },
+        ],
+        totalFileCount: 2,
+      }),
+    ]);
+
+    expect(collectSuites(config, ["int"], [], [], { ignoreSkipRules: true })).toEqual([
+      expect.objectContaining({
+        name: "billing",
+        files: [
+          "__testkit__/billing/a.int.testkit.ts",
+          "__testkit__/billing/b.int.testkit.ts",
+        ],
+        skippedFiles: [],
+        totalFileCount: 2,
+      }),
+    ]);
+  });
+
   it("applies shards, builds graphs, queues tasks, and claims batches", () => {
     const api = makeConfig("api");
     const frontend = makeConfig("frontend");

package/lib/runner/playwright-runner.mjs

@@ -62,7 +62,8 @@ export async function runPlaywrightBatch(targetConfig, batch, lifecycle) {
     if (fileResult) {
       return {
         task,
-        failed: fileResult.failed,
+        failed: fileResult.status === "failed",
+        status: fileResult.status,
         error: fileResult.error,
         durationMs:
           fileResult.durationMs > 0

package/lib/runner/reporting.mjs

@@ -11,17 +11,22 @@ export function buildStatusArtifact({
   metadata,
 }) {
   const executedResults = results.filter((result) => !result.skipped);
+  const effectivelySkippedResults = executedResults.filter(isEffectivelySkippedService);
   const tests = [];
 
   for (const result of executedResults) {
     for (const suite of result.suites) {
       for (const file of suite.files) {
-        tests.push({
+        const test = {
           service: result.name,
           type: suite.type,
           path: file.path,
           status: file.status,
-        });
+        };
+        if (file.reason) {
+          test.reason = file.reason;
+        }
+        tests.push(test);
       }
     }
   }
@@ -36,13 +41,17 @@ export function buildStatusArtifact({
   const summary = {
     services: {
       total: executedResults.length,
-      passed: executedResults.filter((result) => !result.failed).length,
+      passed: executedResults.filter(
+        (result) => !result.failed && !isEffectivelySkippedService(result)
+      ).length,
       failed: executedResults.filter((result) => result.failed).length,
+      skipped: effectivelySkippedResults.length,
     },
     tests: {
       total: tests.length,
       passed: tests.filter((test) => test.status === "passed").length,
       failed: tests.filter((test) => test.status === "failed").length,
+      skipped: tests.filter((test) => test.status === "skipped").length,
       notRun: tests.filter((test) => test.status === "not_run").length,
     },
   };
@@ -63,7 +72,7 @@ export function buildStatusArtifact({
     scope.serviceFilter === null;
 
   return {
-    schemaVersion: 2,
+    schemaVersion: 3,
     source: "testkit",
     notice: "Generated file. Do not edit manually.",
     product: {
@@ -96,18 +105,21 @@ export function buildRunArtifact({
   summarizeDbBackend,
 }) {
   const executed = results.filter((result) => !result.skipped);
+  const effectivelySkippedServices = executed.filter(isEffectivelySkippedService);
   const failedServices = executed.filter((result) => result.failed);
   const totalSuites = executed.reduce((sum, result) => sum + result.suiteCount, 0);
   const completedSuites = executed.reduce((sum, result) => sum + result.completedSuiteCount, 0);
+  const skippedSuites = executed.reduce((sum, result) => sum + (result.skippedSuiteCount || 0), 0);
   const failedSuites = executed.reduce((sum, result) => sum + result.failedSuiteCount, 0);
   const totalFiles = executed.reduce((sum, result) => sum + (result.totalFileCount || 0), 0);
   const passedFiles = executed.reduce((sum, result) => sum + (result.passedFileCount || 0), 0);
   const failedFiles = executed.reduce((sum, result) => sum + (result.failedFileCount || 0), 0);
+  const skippedFiles = executed.reduce((sum, result) => sum + (result.skippedFileCount || 0), 0);
   const notRunFiles = executed.reduce((sum, result) => sum + (result.notRunFileCount || 0), 0);
   const dbBackend = summarizeDbBackend(results);
 
   return {
-    schemaVersion: 2,
+    schemaVersion: 3,
     source: "testkit",
     generatedAt: new Date(finishedAt).toISOString(),
     product: {
@@ -134,19 +146,22 @@ export function buildRunArtifact({
     summary: {
       services: {
         total: executed.length,
-        passed: executed.length - failedServices.length,
+        passed: executed.length - failedServices.length - effectivelySkippedServices.length,
         failed: failedServices.length,
+        skipped: effectivelySkippedServices.length,
       },
       suites: {
         total: totalSuites,
         completed: completedSuites,
-        passed: completedSuites - failedSuites,
+        passed: completedSuites - failedSuites - skippedSuites,
         failed: failedSuites,
+        skipped: skippedSuites,
       },
       files: {
         total: totalFiles,
         passed: passedFiles,
         failed: failedFiles,
+        skipped: skippedFiles,
         notRun: notRunFiles,
       },
     },
@@ -156,11 +171,13 @@ export function buildRunArtifact({
       skipped: result.skipped,
       suiteCount: result.suiteCount,
       completedSuiteCount: result.completedSuiteCount,
+      skippedSuiteCount: result.skippedSuiteCount,
       failedSuiteCount: result.failedSuiteCount,
       totalFileCount: result.totalFileCount,
       completedFileCount: result.completedFileCount,
       passedFileCount: result.passedFileCount,
       failedFileCount: result.failedFileCount,
+      skippedFileCount: result.skippedFileCount,
       notRunFileCount: result.notRunFileCount,
       durationMs: result.durationMs,
       totalTaskDurationMs: result.totalTaskDurationMs,
@@ -170,3 +187,13 @@ export function buildRunArtifact({
     })),
   };
 }
+
+function isEffectivelySkippedService(result) {
+  return (
+    !result.skipped &&
+    !result.failed &&
+    (result.skippedSuiteCount || 0) > 0 &&
+    (result.skippedSuiteCount || 0) === result.suiteCount &&
+    (result.notRunFileCount || 0) === 0
+  );
+}

package/lib/runner/reporting.test.mjs

@@ -10,11 +10,13 @@ describe("runner reporting", () => {
         skipped: false,
         suiteCount: 1,
         completedSuiteCount: 1,
+        skippedSuiteCount: 0,
         failedSuiteCount: 0,
         totalFileCount: 3,
         completedFileCount: 3,
         passedFileCount: 3,
         failedFileCount: 0,
+        skippedFileCount: 0,
         notRunFileCount: 0,
         durationMs: 1200,
         totalTaskDurationMs: 2400,
@@ -28,11 +30,13 @@ describe("runner reporting", () => {
         skipped: true,
         suiteCount: 0,
         completedSuiteCount: 0,
+        skippedSuiteCount: 0,
         failedSuiteCount: 0,
         totalFileCount: 0,
         completedFileCount: 0,
         passedFileCount: 0,
         failedFileCount: 0,
+        skippedFileCount: 0,
         notRunFileCount: 0,
         durationMs: 0,
         totalTaskDurationMs: 0,
@@ -70,11 +74,25 @@ describe("runner reporting", () => {
     });
 
     expect(artifact.product.name).toBe("my-product");
-    expect(artifact.summary.services.total).toBe(1);
+    expect(artifact.schemaVersion).toBe(3);
+    expect(artifact.summary.services).toEqual({
+      total: 1,
+      passed: 1,
+      failed: 0,
+      skipped: 0,
+    });
+    expect(artifact.summary.suites).toEqual({
+      total: 1,
+      completed: 1,
+      passed: 1,
+      failed: 0,
+      skipped: 0,
+    });
     expect(artifact.summary.files).toEqual({
       total: 3,
       passed: 3,
       failed: 0,
+      skipped: 0,
       notRun: 0,
     });
     expect(artifact.services[0].durationMs).toBe(1200);
@@ -96,7 +114,11 @@ describe("runner reporting", () => {
               framework: "k6",
               files: [
                 { path: "tests/api/integration/a.int.testkit.ts", status: "passed" },
-                { path: "tests/api/integration/b.int.testkit.ts", status: "failed" },
+                {
+                  path: "tests/api/integration/b.int.testkit.ts",
+                  status: "skipped",
+                  reason: "Billing is stubbed",
+                },
               ],
             },
           ],
@@ -117,7 +139,7 @@ describe("runner reporting", () => {
     });
 
     expect(status).toEqual({
-      schemaVersion: 2,
+      schemaVersion: 3,
       source: "testkit",
       notice: "Generated file. Do not edit manually.",
       product: {
@@ -141,11 +163,13 @@ describe("runner reporting", () => {
           total: 1,
           passed: 0,
           failed: 1,
+          skipped: 0,
         },
         tests: {
           total: 2,
           passed: 1,
-          failed: 1,
+          failed: 0,
+          skipped: 1,
           notRun: 0,
         },
       },
@@ -160,7 +184,8 @@ describe("runner reporting", () => {
           service: "api",
           type: "int",
           path: "tests/api/integration/b.int.testkit.ts",
-          status: "failed",
+          status: "skipped",
+          reason: "Billing is stubbed",
         },
       ],
     });

package/lib/runner/results.mjs

@@ -29,12 +29,12 @@ export function buildServiceTrackers(servicePlans, startedAt) {
       displayType: suite.displayType || suite.type,
       framework: suite.framework,
       orderIndex: suite.orderIndex,
-      fileCount: suite.files.length,
+      fileCount: suite.totalFileCount ?? suite.files.length,
       completedFileCount: 0,
       failedFiles: [],
       failedFileSet: new Set(),
-      fileResultsByPath: new Map(
-        suite.files.map((file) => {
+      fileResultsByPath: new Map([
+        ...suite.files.map((file) => {
           const normalizedPath = normalizePathSeparators(file);
           return [
             normalizedPath,
@@ -43,11 +43,25 @@ export function buildServiceTrackers(servicePlans, startedAt) {
               failed: false,
               durationMs: 0,
               error: null,
+              reason: null,
               status: "not_run",
+              artifacts: [],
             },
           ];
-        })
-      ),
+        }),
+        ...(suite.skippedFiles || []).map((file) => [
+          file.path,
+          {
+            path: file.path,
+            failed: false,
+            durationMs: 0,
+            error: null,
+            reason: file.reason,
+            status: "skipped",
+            artifacts: [],
+          },
+        ]),
+      ]),
       durationMs: 0,
       error: null,
     }));
@@ -93,25 +107,32 @@ export function recordTaskOutcome(trackers, task, outcome, finishedAt = Date.now
       : Math.max(tracker.lastTaskAt, outcomeFinishedAt);
   tracker.totalTaskDurationMs += outcomeDurationMs;
 
-  suite.completedFileCount += 1;
   suite.durationMs += outcomeDurationMs;
   const normalizedPath = normalizePathSeparators(task.file);
   const existingFileResult = suite.fileResultsByPath.get(normalizedPath);
+  const status = normalizeOutcomeStatus(outcome);
+  if (status !== "skipped") {
+    suite.completedFileCount += 1;
+  }
   if (existingFileResult) {
-    existingFileResult.failed = outcome.failed;
+    existingFileResult.failed = status === "failed";
     existingFileResult.durationMs = outcomeDurationMs;
     existingFileResult.error = outcome.error;
-    existingFileResult.status = outcome.failed ? "failed" : "passed";
+    existingFileResult.reason = outcome.reason || null;
+    existingFileResult.status = status;
+    existingFileResult.artifacts = Array.isArray(outcome.artifacts) ? outcome.artifacts : [];
   } else {
     suite.fileResultsByPath.set(normalizedPath, {
       path: normalizedPath,
-      failed: outcome.failed,
+      failed: status === "failed",
       durationMs: outcomeDurationMs,
       error: outcome.error,
-      status: outcome.failed ? "failed" : "passed",
+      reason: outcome.reason || null,
+      status,
+      artifacts: Array.isArray(outcome.artifacts) ? outcome.artifacts : [],
     });
   }
-  if (outcome.failed && !suite.failedFileSet.has(task.file)) {
+  if (status === "failed" && !suite.failedFileSet.has(task.file)) {
     suite.failedFileSet.add(task.file);
     suite.failedFiles.push(task.file);
   }
@@ -145,7 +166,14 @@ export function finalizeServiceResult(tracker, startedAt, finishedAt) {
       skipped: true,
       suiteCount: 0,
       completedSuiteCount: 0,
+      skippedSuiteCount: 0,
       failedSuiteCount: 0,
+      totalFileCount: 0,
+      completedFileCount: 0,
+      passedFileCount: 0,
+      failedFileCount: 0,
+      skippedFileCount: 0,
+      notRunFileCount: 0,
       durationMs: 0,
       totalTaskDurationMs: 0,
       suites: [],
@@ -158,14 +186,18 @@ export function finalizeServiceResult(tracker, startedAt, finishedAt) {
     .map((suite) => finalizeSuite(suite));
 
   const completedSuiteCount = suites.filter(
-    (suite) => suite.completedFileCount === suite.fileCount
+    (suite) => suite.completedFileCount + suite.skippedFileCount === suite.fileCount
+  ).length;
+  const skippedSuiteCount = suites.filter(
+    (suite) => suite.skippedFileCount === suite.fileCount && suite.fileCount > 0
   ).length;
   const failedSuiteCount = suites.filter((suite) => suite.failedFileCount > 0).length;
   const totalFileCount = suites.reduce((sum, suite) => sum + suite.fileCount, 0);
   const completedFileCount = suites.reduce((sum, suite) => sum + suite.completedFileCount, 0);
   const failedFileCount = suites.reduce((sum, suite) => sum + suite.failedFileCount, 0);
   const passedFileCount = suites.reduce((sum, suite) => sum + suite.passedFileCount, 0);
-  const notRunFileCount = totalFileCount - completedFileCount;
+  const skippedFileCount = suites.reduce((sum, suite) => sum + suite.skippedFileCount, 0);
+  const notRunFileCount = totalFileCount - completedFileCount - skippedFileCount;
   const totalTaskDurationMs =
     tracker.totalTaskDurationMs || suites.reduce((sum, suite) => sum + suite.durationMs, 0);
   const durationMs =
@@ -183,11 +215,13 @@ export function finalizeServiceResult(tracker, startedAt, finishedAt) {
     skipped: false,
     suiteCount: tracker.suiteCount,
     completedSuiteCount,
+    skippedSuiteCount,
     failedSuiteCount,
     totalFileCount,
     completedFileCount,
     passedFileCount,
     failedFileCount,
+    skippedFileCount,
     notRunFileCount,
     durationMs,
     totalTaskDurationMs,
@@ -212,6 +246,10 @@ function finalizeSuite(suite) {
       status: file.status,
       durationMs: file.durationMs,
       error: file.error,
+      reason: file.reason,
+      ...(Array.isArray(file.artifacts) && file.artifacts.length > 0
+        ? { artifacts: file.artifacts }
+        : {}),
     }));
 
   return {
@@ -223,7 +261,11 @@ function finalizeSuite(suite) {
     completedFileCount: suite.completedFileCount,
     passedFileCount: files.filter((file) => file.status === "passed").length,
     failedFileCount: suite.failedFiles.length,
-    notRunFileCount: suite.fileCount - suite.completedFileCount,
+    skippedFileCount: files.filter((file) => file.status === "skipped").length,
+    notRunFileCount:
+      suite.fileCount -
+      suite.completedFileCount -
+      files.filter((file) => file.status === "skipped").length,
     failedFiles: suite.failedFiles,
     durationMs: suite.durationMs,
     error: suite.error,
@@ -239,3 +281,8 @@ function formatFrameworkForArtifact(framework) {
   if (framework === "k6") return "default";
   return framework;
 }
+
+function normalizeOutcomeStatus(outcome) {
+  if (outcome?.status === "skipped") return "skipped";
+  return outcome?.failed ? "failed" : "passed";
+}