@elench/testkit 0.1.34 → 0.1.36

package/README.md

@@ -15,13 +15,11 @@ cd my-product
 npx @elench/testkit
 
 # Filter by type
-npx @elench/testkit int
-npx @elench/testkit dal
-npx @elench/testkit e2e
-
-# Filter by framework
-npx @elench/testkit --framework playwright
-npx @elench/testkit --framework default
+npx @elench/testkit --type int
+npx @elench/testkit --type dal
+npx @elench/testkit --type e2e
+npx @elench/testkit --type int,e2e,dal
+npx @elench/testkit --type pw
 
 # Parallelize with isolated worker stacks
 npx @elench/testkit --jobs 3
@@ -30,14 +28,18 @@ npx @elench/testkit --jobs 3
 npx @elench/testkit --shard 1/3
 
 # Specific service / suite
-npx @elench/testkit frontend e2e -s auth
-npx @elench/testkit api int -s health
+npx @elench/testkit --service frontend --type pw -s navigation
+npx @elench/testkit --service api --type int -s health
+npx @elench/testkit --type int,e2e,dal -s dal:queries
 
 # Exact file
-npx @elench/testkit int --file __testkit__/health/health.int.testkit.ts
+npx @elench/testkit --type int --file __testkit__/health/health.int.testkit.ts
+
+# Temporarily ignore repo-declared skip rules
+npx @elench/testkit --ignore-skip-rules --file __testkit__/billing/billing.int.testkit.ts
 
 # Deterministic git-trackable status snapshot
-npx @elench/testkit int --write-status
+npx @elench/testkit --type int --write-status
 
 # Lifecycle
 npx @elench/testkit status
@@ -84,6 +86,22 @@ export default defineTestkitSetup({
       dependsOn: ["api"],
       envFiles: ["frontend/.env.testkit"],
     }),
+    billing: service({
+      skip: {
+        files: [
+          {
+            path: "__testkit__/invoices/invoices.int.testkit.ts",
+            reason: "Billing is still stubbed locally",
+          },
+        ],
+        suites: [
+          {
+            selector: "pw:lifecycle",
+            reason: "End-to-end billing lifecycle is not implemented yet",
+          },
+        ],
+      },
+    }),
   },
 });
 ```
@@ -96,6 +114,7 @@ for:
 - migrate / seed commands
 - test-local migrate / seed overrides
 - named HTTP suite profiles
+- repo-declared suite/file skip policies with explicit reasons
 - telemetry upload configuration
 
 ## Authoring

package/lib/cli/args.mjs

@@ -1,35 +1,43 @@
 import path from "path";
+import { normalizeTypeValues, parseSuiteSelectors } from "../runner/suite-selection.mjs";
 
-export const SUITE_TYPES = new Set(["int", "integration", "e2e", "dal", "load", "all"]);
+export const POSITIONAL_TYPES = new Set(["int", "e2e", "dal", "load", "pw", "all"]);
 export const LIFECYCLE = new Set(["status", "destroy", "cleanup"]);
-export const RESERVED = new Set([...SUITE_TYPES, ...LIFECYCLE]);
+export const RESERVED = new Set([...POSITIONAL_TYPES, ...LIFECYCLE]);
 
-export function resolveCliSelection({ first, second, serviceNames }) {
-  let service = null;
-  let type = null;
+export function resolveCliSelection({ first, second, third }) {
+  if (second || third) {
+    throw new Error(`Unexpected extra positional arguments. Use --service and --type instead.`);
+  }
+
+  let lifecycle = null;
+  let positionalType = null;
 
-  if (first && serviceNames.has(first)) {
-    service = first;
-    type = second || null;
-  } else if (first && RESERVED.has(first)) {
-    type = first;
+  if (first && LIFECYCLE.has(first)) {
+    lifecycle = first;
+  } else if (first && POSITIONAL_TYPES.has(first)) {
+    positionalType = first;
   } else if (first) {
     throw new Error(
-      `Unknown argument "${first}". Expected a service name (${[...serviceNames].join(", ") || "none found"}) ` +
-      `or suite type (int, e2e, dal, all).`
+      `Unknown argument "${first}". Expected a lifecycle command (status, destroy, cleanup) ` +
+      `or suite type (int, e2e, dal, load, pw, all).`
     );
   }
 
-  return { service, type };
+  return { lifecycle, positionalType };
 }
 
-export function validateFrameworkOption(value) {
-  if (!["all", "default", "playwright"].includes(value)) {
-    throw new Error(
-      `Unknown framework "${value}". Expected one of: all, default, playwright.`
-    );
-  }
-  return value === "default" ? "k6" : value;
+export function parseTypeOption(values, positionalType = null) {
+  const input = [];
+  if (positionalType) input.push(positionalType);
+  if (Array.isArray(values)) input.push(...values);
+  else if (values) input.push(values);
+  return normalizeTypeValues(input);
+}
+
+export function parseSuiteOption(values) {
+  const input = Array.isArray(values) ? values : values ? [values] : [];
+  return parseSuiteSelectors(input);
 }
 
 export function parseJobsOption(value) {

package/lib/cli/args.test.mjs

@@ -2,35 +2,36 @@ import { describe, expect, it } from "vitest";
 import {
   parseJobsOption,
   parseShardOption,
+  parseSuiteOption,
+  parseTypeOption,
   resolveRequestedFiles,
   resolveCliSelection,
-  validateFrameworkOption,
 } from "./args.mjs";
 
 describe("cli-args", () => {
-  it("resolves a service and suite type", () => {
+  it("resolves a positional suite type", () => {
     expect(
       resolveCliSelection({
-        first: "api",
-        second: "int",
-        serviceNames: new Set(["api", "frontend"]),
+        first: "int",
+        second: null,
+        third: null,
       })
     ).toEqual({
-      service: "api",
-      type: "int",
+      lifecycle: null,
+      positionalType: "int",
     });
   });
 
-  it("resolves a reserved suite type without a service", () => {
+  it("resolves lifecycle commands", () => {
     expect(
       resolveCliSelection({
-        first: "e2e",
+        first: "status",
         second: null,
-        serviceNames: new Set(["api"]),
+        third: null,
       })
     ).toEqual({
-      service: null,
-      type: "e2e",
+      lifecycle: "status",
+      positionalType: null,
     });
   });
 
@@ -39,15 +40,19 @@ describe("cli-args", () => {
       resolveCliSelection({
         first: "mystery",
         second: null,
-        serviceNames: new Set(["api"]),
+        third: null,
       })
     ).toThrow('Unknown argument "mystery"');
   });
 
-  it("validates framework names", () => {
-    expect(validateFrameworkOption("playwright")).toBe("playwright");
-    expect(validateFrameworkOption("default")).toBe("k6");
-    expect(() => validateFrameworkOption("jest")).toThrow("Unknown framework");
+  it("parses types and suite selectors", () => {
+    expect(parseTypeOption(["e2e,dal"], "int")).toEqual(["int", "e2e", "dal"]);
+    expect(() => parseTypeOption(["all", "int"])).toThrow("cannot be combined");
+
+    expect(parseSuiteOption(["auth,dal:queries"])).toEqual([
+      { kind: "plain", name: "auth", raw: "auth" },
+      { kind: "typed", type: "dal", name: "queries", raw: "dal:queries" },
+    ]);
   });
 
   it("parses and validates jobs", () => {

package/lib/cli/index.mjs

@@ -3,9 +3,10 @@ import { loadConfigs } from "../config/index.mjs";
 import {
   parseJobsOption,
   parseShardOption,
+  parseSuiteOption,
+  parseTypeOption,
   resolveRequestedFiles,
   resolveCliSelection,
-  validateFrameworkOption,
 } from "./args.mjs";
 import * as runner from "../runner/index.mjs";
 
@@ -13,7 +14,11 @@ export function run() {
   const cli = cac("testkit");
 
   cli
-    .command("[first] [second] [third]", "Run test suites (int, e2e, dal, all)")
+    .command("[first] [second] [third]", "Run test suites")
+    .option("--service <name>", "Run only one service")
+    .option("-t, --type <name>", "Run specific suite type(s): int, e2e, dal, load, pw, all", {
+      default: [],
+    })
     .option("-s, --suite <name>", "Run specific suite(s)", { default: [] })
     .option("-f, --file <path>", "Run specific file(s)", { default: [] })
     .option("--dir <path>", "Explicit product directory")
@@ -21,76 +26,60 @@ export function run() {
       default: "1",
     })
     .option("--shard <i/n>", "Run only shard i of n at suite granularity")
-    .option("--framework <name>", "Filter by framework (default, playwright, all)", {
-      default: "all",
-    })
     .option("--write-status", "Write a deterministic testkit.status.json snapshot")
     .option("--allow-partial-status", "Allow --write-status for filtered runs")
+    .option(
+      "--ignore-skip-rules",
+      "Run files even if testkit.setup.ts marks them skipped"
+    )
     .action(async (first, second, third, options) => {
-      // Resolve: service filter, suite type, and --dir.
-      //
-      // From product dir:
-      //   testkit                          → all services, all types
-      //   testkit int -s health            → all services, int, health
-      //   testkit api int                  → one service, int
-      //   testkit api                      → one service, all types
-      //
-      // From workspace root:
-      //   testkit --dir my-product int     → all services, int
-      //   testkit --dir my-product api int → one service, int
-
-      // Now resolve service vs type from remaining args
-      const allConfigs = await loadConfigs({ dir: options.dir });
-      const serviceNames = new Set(allConfigs.map((config) => config.name));
-      const { service, type } = resolveCliSelection({
+      const { lifecycle, positionalType } = resolveCliSelection({
         first,
         second,
-        serviceNames,
+        third,
       });
-      const configs = service
-        ? allConfigs.filter((config) => config.name === service)
+      const allConfigs = await loadConfigs({ dir: options.dir });
+      const configs = options.service
+        ? allConfigs.filter((config) => config.name === options.service)
         : allConfigs;
-      if (service && configs.length === 0) {
+      if (options.service && configs.length === 0) {
         const available = allConfigs.map((config) => config.name).join(", ");
-        throw new Error(`Service "${service}" not found. Available: ${available}`);
+        throw new Error(`Service "${options.service}" not found. Available: ${available}`);
       }
 
       // Lifecycle commands
-      if (type === "cleanup") {
+      if (lifecycle === "cleanup") {
         await runner.cleanup(allConfigs[0]?.productDir || process.cwd());
         return;
       }
 
-      if (type === "status" || type === "destroy") {
+      if (lifecycle === "status" || lifecycle === "destroy") {
         for (const config of configs) {
           if (configs.length > 1) console.log(`\n── ${config.name} ──`);
-          if (type === "status") runner.showStatus(config);
+          if (lifecycle === "status") runner.showStatus(config);
           else await runner.destroy(config);
         }
         return;
       }
 
-      const framework = validateFrameworkOption(options.framework);
-
       const jobs = parseJobsOption(options.jobs);
       const shard = parseShardOption(options.shard);
-
-      const suiteType = type || "all";
-      const suiteNames = Array.isArray(options.suite) ? options.suite : [options.suite].filter(Boolean);
+      const typeValues = parseTypeOption(options.type, positionalType);
+      const suiteSelectors = parseSuiteOption(options.suite);
       const rawFileNames = Array.isArray(options.file) ? options.file : [options.file].filter(Boolean);
       const productDir = allConfigs[0]?.productDir || process.cwd();
       const fileNames = resolveRequestedFiles(rawFileNames, productDir, process.cwd());
       await runner.runAll(
         configs,
-        suiteType,
-        suiteNames,
+        typeValues,
+        suiteSelectors,
         {
           ...options,
-          framework,
+          typeValues,
           fileNames,
           jobs,
           shard,
-          serviceFilter: service,
+          serviceFilter: options.service || null,
         },
         allConfigs
       );

package/lib/config/index.mjs

@@ -3,6 +3,11 @@ import path from "path";
 import { fileURLToPath } from "url";
 import { discoverProject } from "./discovery.mjs";
 import { loadTestkitSetup } from "./setup-loader.mjs";
+import {
+  matchesSuiteSelectors,
+  parseSuiteSelectors,
+  suiteSelectionType,
+} from "../runner/suite-selection.mjs";
 
 const TESTKIT_K6_BIN = "TESTKIT_K6_BIN";
 const DEFAULT_LOCAL_IMAGE = "pgvector/pgvector:pg16";
@@ -100,6 +105,11 @@ function normalizeServiceConfig({
   const database = normalizeDatabaseConfig(explicitService, name);
   const migrate = normalizeLifecycle(explicitService.migrate);
   const seed = normalizeLifecycle(explicitService.seed);
+  const skip = normalizeSkipConfig(explicitService.skip, {
+    name,
+    productDir,
+    suites,
+  });
 
   if (!explicitService.databaseFrom && !database && (migrate || seed)) {
     throw new Error(
@@ -134,6 +144,7 @@ function normalizeServiceConfig({
       serviceEnv,
       migrate,
       seed,
+      skip,
       local,
     },
   };
@@ -234,6 +245,96 @@ function normalizeLifecycle(value) {
   };
 }
 
+function normalizeSkipConfig(value, { name, productDir, suites }) {
+  if (!value) return undefined;
+
+  const discoveredFiles = new Set();
+  const discoveredSuites = [];
+  for (const [type, typedSuites] of Object.entries(suites || {})) {
+    for (const suite of typedSuites || []) {
+      const displayType = suiteSelectionType(type, suite.framework || "k6");
+      discoveredSuites.push({
+        type,
+        displayType,
+        name: suite.name,
+      });
+      for (const file of suite.files || []) {
+        discoveredFiles.add(normalizePath(file));
+      }
+    }
+  }
+
+  const seenFiles = new Set();
+  const files = [];
+  for (const rule of value.files || []) {
+    if (!rule || typeof rule !== "object") {
+      throw new Error(`Service "${name}" skip.files entries must be objects`);
+    }
+    const filePath = normalizePath(rule.path);
+    const reason = normalizeSkipReason(rule.reason, `Service "${name}" skip.files["${filePath}"]`);
+    if (!filePath) {
+      throw new Error(`Service "${name}" skip.files entries require a non-empty path`);
+    }
+    if (seenFiles.has(filePath)) {
+      throw new Error(`Service "${name}" defines duplicate skip.files path "${filePath}"`);
+    }
+    if (!discoveredFiles.has(filePath)) {
+      throw new Error(
+        `Service "${name}" skip.files path "${filePath}" did not match any discovered suite file`
+      );
+    }
+    seenFiles.add(filePath);
+    files.push({ path: filePath, reason });
+  }
+
+  const parsedSelectors = (value.suites || []).flatMap((rule, index) => {
+    if (!rule || typeof rule !== "object") {
+      throw new Error(`Service "${name}" skip.suites entries must be objects`);
+    }
+    const selector = String(rule.selector || "").trim();
+    if (!selector) {
+      throw new Error(`Service "${name}" skip.suites[${index}] requires a non-empty selector`);
+    }
+    const reason = normalizeSkipReason(
+      rule.reason,
+      `Service "${name}" skip.suites["${selector}"]`
+    );
+    const parsed = parseSuiteSelectors([selector]);
+    if (parsed.length !== 1) {
+      throw new Error(`Service "${name}" skip.suites["${selector}"] is invalid`);
+    }
+    return [{ selector: parsed[0], reason }];
+  });
+
+  const seenSelectors = new Set();
+  const suitesWithReasons = [];
+  for (const rule of parsedSelectors) {
+    if (seenSelectors.has(rule.selector.raw)) {
+      throw new Error(
+        `Service "${name}" defines duplicate skip.suites selector "${rule.selector.raw}"`
+      );
+    }
+    const matched = discoveredSuites.some((suite) =>
+      matchesSuiteSelectors(suite.displayType, suite.name, [rule.selector])
+    );
+    if (!matched) {
+      throw new Error(
+        `Service "${name}" skip.suites selector "${rule.selector.raw}" did not match any discovered suite`
+      );
+    }
+    seenSelectors.add(rule.selector.raw);
+    suitesWithReasons.push(rule);
+  }
+
+  if (files.length === 0 && suitesWithReasons.length === 0) return undefined;
+
+  return {
+    files,
+    fileReasonByPath: new Map(files.map((rule) => [rule.path, rule.reason])),
+    suites: suitesWithReasons,
+  };
+}
+
 function inferEnvFiles(productDir, explicitService, local) {
   if (explicitService.envFile || explicitService.envFiles) {
     const files = [];
@@ -256,6 +357,14 @@ function inferEnvFiles(productDir, explicitService, local) {
     .filter((candidate) => fs.existsSync(resolveServiceCwd(productDir, candidate)));
 }
 
+function normalizeSkipReason(reason, label) {
+  const normalized = String(reason || "").trim();
+  if (!normalized) {
+    throw new Error(`${label} requires a non-empty reason`);
+  }
+  return normalized;
+}
+
 function loadServiceEnv(productDir, envFiles) {
   const env = {};
   for (const envFile of envFiles) {
@@ -361,6 +470,13 @@ function normalizeTelemetryConfig(telemetry) {
   };
 }
 
+function normalizePath(value) {
+  return String(value || "")
+    .split(path.sep)
+    .join("/")
+    .replace(/^\.\/+/, "");
+}
+
 function resolveProductDir(cwd, explicitDir) {
   const dir = explicitDir ? path.resolve(cwd, explicitDir) : cwd;
   if (!fs.existsSync(dir)) {

package/lib/reporters/playwright.mjs

@@ -18,7 +18,7 @@ export function parsePlaywrightJsonResults(stdout, cwd) {
   const fileResults = new Map();
   visitPlaywrightSuites(parsed.suites || [], null, fileResults, cwd);
   return {
-    fileResults,
+    fileResults: sanitizePlaywrightFileResults(fileResults),
     errors: (parsed.errors || []).map(formatPlaywrightReporterError).filter(Boolean),
   };
 }
@@ -41,8 +41,11 @@ export function collectPlaywrightSpec(spec, inheritedFile, fileResults, cwd) {
 
   const current = fileResults.get(file) || {
     failed: false,
+    status: "passed",
     error: null,
     durationMs: 0,
+    passedCount: 0,
+    skippedCount: 0,
   };
 
   for (const test of spec.tests || []) {
@@ -53,16 +56,26 @@ export function collectPlaywrightSpec(spec, inheritedFile, fileResults, cwd) {
     );
 
     const final = choosePlaywrightFinalResult(results);
-    const failed =
-      test.outcome === "unexpected" ||
-      !isPlaywrightPassingStatus(final?.status);
+    const status = classifyPlaywrightTestOutcome(test, final);
 
-    if (failed) {
+    if (status === "failed") {
       current.failed = true;
+      current.status = "failed";
       current.error ||= extractPlaywrightFailure(final, spec, test);
+      continue;
     }
+
+    if (status === "skipped") {
+      current.skippedCount += 1;
+      continue;
+    }
+
+    current.passedCount += 1;
   }
 
+  if (!current.failed) {
+    current.status = current.passedCount === 0 && current.skippedCount > 0 ? "skipped" : "passed";
+  }
   fileResults.set(file, current);
 }
 
@@ -75,6 +88,16 @@ export function isPlaywrightPassingStatus(status) {
   return !status || ["passed", "skipped", "expected"].includes(status);
 }
 
+export function classifyPlaywrightTestOutcome(test, finalResult) {
+  if (test?.outcome === "unexpected" || !isPlaywrightPassingStatus(finalResult?.status)) {
+    return "failed";
+  }
+  if (test?.outcome === "skipped" || finalResult?.status === "skipped") {
+    return "skipped";
+  }
+  return "passed";
+}
+
 export function extractPlaywrightFailure(finalResult, spec, test) {
   const fromResult =
     finalResult?.error?.message ||
@@ -123,3 +146,16 @@ function formatError(error) {
   if (error instanceof Error) return error.message;
   return String(error);
 }
+
+function sanitizePlaywrightFileResults(fileResults) {
+  const sanitized = new Map();
+  for (const [file, result] of fileResults.entries()) {
+    sanitized.set(file, {
+      failed: result.failed,
+      status: result.status,
+      error: result.error,
+      durationMs: result.durationMs,
+    });
+  }
+  return sanitized;
+}

package/lib/reporters/playwright.test.mjs

@@ -49,11 +49,94 @@ describe("playwright-report", () => {
     expect(parsed.errors).toEqual(["reporter failure"]);
     expect(parsed.fileResults.get("tests/auth.spec.js")).toEqual({
       failed: true,
+      status: "failed",
       error: "boom",
       durationMs: 15,
     });
   });
 
+  it("marks files as skipped when every Playwright test is skipped", () => {
+    const stdout = JSON.stringify({
+      suites: [
+        {
+          file: "/tmp/tests/billing.spec.js",
+          specs: [
+            {
+              title: "billing is stubbed",
+              tests: [
+                {
+                  outcome: "skipped",
+                  results: [
+                    {
+                      status: "skipped",
+                      duration: 7,
+                    },
+                  ],
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    });
+
+    const parsed = parsePlaywrightJsonResults(stdout, "/tmp");
+    expect(parsed.fileResults.get("tests/billing.spec.js")).toEqual({
+      failed: false,
+      status: "skipped",
+      error: null,
+      durationMs: 7,
+    });
+  });
+
+  it("keeps file status passed when one spec passes and another is skipped", () => {
+    const stdout = JSON.stringify({
+      suites: [
+        {
+          file: "/tmp/tests/mixed.spec.js",
+          specs: [
+            {
+              title: "passes",
+              tests: [
+                {
+                  outcome: "expected",
+                  results: [
+                    {
+                      status: "passed",
+                      duration: 5,
+                    },
+                  ],
+                },
+              ],
+            },
+            {
+              title: "skips",
+              tests: [
+                {
+                  outcome: "skipped",
+                  results: [
+                    {
+                      status: "skipped",
+                      duration: 3,
+                    },
+                  ],
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    });
+
+    const parsed = parsePlaywrightJsonResults(stdout, "/tmp");
+    expect(parsed.fileResults.get("tests/mixed.spec.js")).toEqual({
+      failed: false,
+      status: "passed",
+      error: null,
+      durationMs: 8,
+    });
+  });
+
   it("chooses the final result and extracts failures", () => {
     expect(
       choosePlaywrightFinalResult([