@elench/testkit 0.1.10 → 0.1.12

package/README.md

@@ -1,44 +1,43 @@
 # @elench/testkit
 
-CLI that reads `testkit.manifest.json` from a product repo, spins up ephemeral infrastructure (Neon DB branch + Fly machine) per service, runs k6 tests, and tears down.
+CLI that reads `runner.manifest.json` plus `testkit.config.json` from a product repo, starts the required local services, provisions Neon branches when configured, and runs manifest-defined suites across `k6` and Playwright.
 
 ## Prerequisites
 
 ```bash
 sudo snap install k6
 sudo apt-get install -y jq
-curl -L https://fly.io/install.sh | sh
-fly auth login
 ```
 
-## Setup
-
-Add platform secrets to each product's `.env`:
-
-```bash
-NEON_API_KEY='...'
-FLY_API_TOKEN='...'
-```
-
-Product secrets (`CLERK_SECRET_KEY`, etc.) are also loaded from the same `.env`.
+For DAL suites, `@elench/testkit` ships its own `k6` SQL binary. For suites using a Neon branch, set `NEON_API_KEY` in the product `.env`, shell, or `.envrc`.
 
 ## Usage
 
 ```bash
 cd bourne
 
-# Run all suites
+# Run every testkit-managed suite
 npx @elench/testkit
 
-# Specific type / suite
-npx @elench/testkit int -s health
+# Filter by type
+npx @elench/testkit int
+npx @elench/testkit dal
 npx @elench/testkit e2e
 
-# Specific service (multi-service products)
-npx @elench/testkit avocado_api int -s health
+# Filter by framework
+npx @elench/testkit --framework playwright
+npx @elench/testkit --framework k6
 
-# Build from source first
-npx @elench/testkit int --build
+# Parallelize with isolated worker stacks
+npx @elench/testkit --jobs 3
+
+# Run a deterministic shard
+npx @elench/testkit --shard 1/3
+npx @elench/testkit --jobs 2 --shard 2/3
+
+# Specific service / suite
+npx @elench/testkit frontend e2e -s auth
+npx @elench/testkit bourne int -s health
 
 # Lifecycle
 npx @elench/testkit status
@@ -47,17 +46,50 @@ npx @elench/testkit destroy
 
 ## How it works
 
-1. **Config** — reads `testkit.manifest.json` for per-service infra config + test suites
-2. **Neon** — discovers or creates a `<service>-test` branch, truncates tables between runs
-3. **Fly** — discovers or creates a machine on the service's test app, updates env vars
-4. **k6** — runs matched test files with `BASE_URL` and `MACHINE_ID` injected
-5. **DAL** — DAL tests use a bundled k6-sql binary (`vendor/k6`) — no external binary needed
-6. **Cleanup** — stops the Fly machine (preserved for next run)
+1. **Discovery** — reads `runner.manifest.json` for services, suites, files, and frameworks
+2. **Config** — reads `testkit.config.json` for local runtime, migration, dependency, and database settings
+3. **Database** — provisions a Neon branch when a service declares one
+4. **Seed** — runs optional product seed commands against the provisioned database
+5. **Runtime** — starts required local services, waits for readiness, and injects test env
+6. **Execution** — distributes suites across isolated worker stacks, runs `k6` suites file-by-file, and runs Playwright suites suite-by-suite
+7. **Cleanup** — stops local processes and preserves `.testkit/` state for inspection or later destroy
+
+## File roles
+
+- `runner.manifest.json`: canonical test inventory
+- `testkit.config.json`: local execution and provisioning config
 
-Multi-service products run all services in parallel. Each service gets its own Neon branch and Fly machine.
+## Parallel execution
+
+`@elench/testkit` can run suites in parallel with `--jobs <n>`.
+
+Each worker gets its own:
+- Neon branch
+- `.testkit` state subtree
+- local service ports
+
+This keeps suites isolated while still reusing one stack per worker across multiple assigned suites.
+
+Use `--shard <i/n>` to split the selected suite set deterministically before worker scheduling.
+
+## Suite metadata
+
+`runner.manifest.json` remains the source of truth for suites. Optional per-suite `testkit` metadata can tune execution:
+
+```json
+{
+  "name": "health",
+  "files": ["tests/example.js"],
+  "testkit": {
+    "maxFileConcurrency": 2,
+    "weight": 3
+  }
+}
+```
 
-State is persisted in `.testkit/` (or `.testkit/<service>/` for multi-service) so subsequent runs reuse existing infrastructure.
+- `maxFileConcurrency`: k6-only opt-in for running files within the suite concurrently
+- `weight`: optional scheduling weight when distributing suites across workers
 
-## Manifest schema
+## Schema
 
-See [testkit-manifest-schema.md](testkit-manifest-schema.md).
+See [testkit-config-schema.md](testkit-config-schema.md).

package/lib/cli.mjs

@@ -12,8 +12,14 @@ export function run() {
   cli
     .command("[first] [second] [third]", "Run test suites (int, e2e, dal, all)")
     .option("-s, --suite <name>", "Run specific suite(s)", { default: [] })
-    .option("--build", "Build image from source first", { default: false })
     .option("--dir <path>", "Explicit product directory")
+    .option("--jobs <n>", "Number of isolated worker stacks per service", {
+      default: "1",
+    })
+    .option("--shard <i/n>", "Run only shard i of n at suite granularity")
+    .option("--framework <name>", "Filter by framework (k6, playwright, all)", {
+      default: "all",
+    })
     .action(async (first, second, third, options) => {
       // Resolve: service filter, suite type, and --dir.
       //
@@ -57,7 +63,14 @@ export function run() {
         );
       }
 
-      const configs = loadConfigs({ dir: options.dir, service });
+      const allConfigs = loadConfigs({ dir: options.dir });
+      const configs = service
+        ? allConfigs.filter((config) => config.name === service)
+        : allConfigs;
+      if (service && configs.length === 0) {
+        const available = allConfigs.map((config) => config.name).join(", ");
+        throw new Error(`Service "${service}" not found. Available: ${available}`);
+      }
 
       // Lifecycle commands
       if (type === "status" || type === "destroy") {
@@ -69,9 +82,48 @@ export function run() {
         return;
       }
 
+      if (!["all", "k6", "playwright"].includes(options.framework)) {
+        throw new Error(
+          `Unknown framework "${options.framework}". Expected one of: all, k6, playwright.`
+        );
+      }
+
+      const jobs = Number.parseInt(String(options.jobs), 10);
+      if (!Number.isInteger(jobs) || jobs <= 0) {
+        throw new Error(`Invalid --jobs value "${options.jobs}". Expected a positive integer.`);
+      }
+
+      let shard = null;
+      if (options.shard) {
+        const match = String(options.shard).match(/^(\d+)\/(\d+)$/);
+        if (!match) {
+          throw new Error(
+            `Invalid --shard value "${options.shard}". Expected the form "i/n", e.g. 1/3.`
+          );
+        }
+        const index = Number.parseInt(match[1], 10);
+        const total = Number.parseInt(match[2], 10);
+        if (index <= 0 || total <= 0 || index > total) {
+          throw new Error(
+            `Invalid --shard value "${options.shard}". Expected 1 <= i <= n.`
+          );
+        }
+        shard = { index, total };
+      }
+
       const suiteType = type || "all";
       const suiteNames = Array.isArray(options.suite) ? options.suite : [options.suite].filter(Boolean);
-      await runner.runAll(configs, suiteType, suiteNames, options);
+      await runner.runAll(
+        configs,
+        suiteType,
+        suiteNames,
+        {
+          ...options,
+          jobs,
+          shard,
+        },
+        allConfigs
+      );
     });
 
   cli.help();