@elench/testkit 0.1.16 → 0.1.18

package/README.md

@@ -1,6 +1,6 @@
 # @elench/testkit
 
-CLI that reads `runner.manifest.json` plus `testkit.config.json` from a product repo, starts the required local services, provisions the selected database backend, and runs manifest-defined suites across `k6` and Playwright.
+CLI that reads `runner.manifest.json` plus `testkit.config.json` from a product repo, starts the required local services, provisions local Postgres isolation, and runs manifest-defined suites across `k6` and Playwright.
 
 ## Prerequisites
 
@@ -9,14 +9,14 @@ sudo snap install k6
 sudo apt-get install -y jq
 ```
 
-For DAL suites, `@elench/testkit` ships its own `k6` SQL binary. For suites using a Neon branch, set `NEON_API_KEY` in the product `.env`, shell, or `.envrc`.
+For DAL suites, `@elench/testkit` ships its own `k6` SQL binary.
 
-Local database mode uses Docker-managed `postgres:16` containers. `testkit` creates and reuses template databases automatically, then clones per-worker databases from those templates for fast reruns.
+Database isolation uses Docker-managed local Postgres containers. `testkit` creates and reuses template databases automatically, then clones per-worker databases from those templates for fast reruns. The default image is `pgvector/pgvector:pg16`.
 
 ## Usage
 
 ```bash
-cd bourne
+cd my-product
 
 # Run every testkit-managed suite
 npx @elench/testkit
@@ -26,10 +26,6 @@ npx @elench/testkit int
 npx @elench/testkit dal
 npx @elench/testkit e2e
 
-# Force a database backend
-npx @elench/testkit --db-backend local
-npx @elench/testkit --db-backend neon
-
 # Filter by framework
 npx @elench/testkit --framework playwright
 npx @elench/testkit --framework k6
@@ -43,26 +39,35 @@ npx @elench/testkit --jobs 2 --shard 2/3
 
 # Specific service / suite
 npx @elench/testkit frontend e2e -s auth
-npx @elench/testkit bourne int -s health
+npx @elench/testkit api int -s health
 
 # Lifecycle
 npx @elench/testkit status
 npx @elench/testkit destroy
 ```
 
+From outside the product repo, use `--dir` explicitly:
+
+```bash
+npx @elench/testkit --dir my-product int
+npx @elench/testkit --dir my-product api int -s health
+```
+
 ## How it works
 
 1. **Discovery** — reads `runner.manifest.json` for services, suites, files, and frameworks
 2. **Config** — reads `testkit.config.json` for local runtime, migration, dependency, and database settings
    Per-service `.env` files declared in config are loaded when present.
-3. **Database** — provisions the selected backend (`neon` or Docker-managed `local`) when a service declares one
+3. **Database** — provisions Docker-managed local Postgres when a service declares one
 4. **Seed** — runs optional product seed commands against the provisioned database
 5. **Runtime** — starts required local services, waits for readiness, and injects test env
-6. **Execution** — distributes suites across isolated worker stacks, runs `k6` suites file-by-file, and runs Playwright suites suite-by-suite
+6. **Execution** — schedules file-level execution tasks across a global worker pool, reuses warm dependency graphs when possible, runs `k6` file-by-file, and batches Playwright files per worker
 7. **Cleanup** — stops local processes and preserves `.testkit/` state for inspection or later destroy
 
 Each run ends with a compact summary that shows per-service pass/fail status, suite totals, failed suites, and total duration.
 
+`testkit` also writes `.testkit/results/latest.json` for every run. When `testkit.config.json` includes a top-level `telemetry` block, it can best-effort upload that artifact to a configured HTTP endpoint with bearer auth.
+
 ## File roles
 
 - `runner.manifest.json`: canonical test inventory
@@ -70,26 +75,31 @@ Each run ends with a compact summary that shows per-service pass/fail status, su
 
 `testkit.config.json` can also declare:
 
+- `telemetry` for optional generic HTTP result upload
 - `envFile` / `envFiles` for service-specific environment loading
 - `databaseFrom` for dependency services that must share another service's `DATABASE_URL`
-- `database.defaultBackend` / `database.backends` to switch between Neon and local Postgres
+- `database.provider` for local Postgres settings
 - `database.template.inputs` to define the local template cache invalidation inputs
-- `migrate.backends` / `seed.backends` for backend-specific command overrides
+- `migrate.backends` / `seed.backends` for optional local-only command overrides
 
 ## Parallel execution
 
-`@elench/testkit` can run suites in parallel with `--jobs <n>`.
+`@elench/testkit` can run test work in parallel with `--jobs <n>`.
+
+`--jobs` is global for the whole run, not per service.
 
 Each worker gets its own:
-- Neon branch
-- or cloned local Postgres database
+
+- cloned local Postgres database
 - `.testkit` state subtree
 - local service ports
 
-This keeps suites isolated while still reusing one stack per worker across multiple assigned suites.
+Workers prefer to stay on the same dependency graph so service stacks can be reused across assigned work, including dependent services such as `frontend -> api`.
 
 Use `--shard <i/n>` to split the selected suite set deterministically before worker scheduling.
 
+`testkit` also writes `.testkit/timings.json` and uses those file timings on later runs for longest-first balancing.
+
 ## Suite metadata
 
 `runner.manifest.json` remains the source of truth for suites. Optional per-suite `testkit` metadata can tune execution:
@@ -105,9 +115,24 @@ Use `--shard <i/n>` to split the selected suite set deterministically before wor
 }
 ```
 
-- `maxFileConcurrency`: k6-only opt-in for running files within the suite concurrently
-- `weight`: optional scheduling weight when distributing suites across workers
+- `maxFileConcurrency`: k6-only opt-in for batching multiple files from the same suite onto one worker
+- `weight`: optional fallback scheduling weight when no file timing history exists yet
 
 ## Schema
 
 See [testkit-config-schema.md](testkit-config-schema.md).
+
+## Development Tests
+
+```bash
+npm test
+npm run test:unit
+npm run test:integration
+npm run test:system
+```
+
+`test:system` runs real end-to-end fixtures from `test/fixtures/system/`. It requires:
+
+- Docker with a running daemon
+- `k6` on `PATH`
+- Playwright Chromium browser assets available to `@playwright/test`

package/bin/testkit.mjs

@@ -1,3 +1,3 @@
 #!/usr/bin/env node
-import { run } from "../lib/cli.mjs";
+import { run } from "../lib/cli/index.mjs";
 run();

package/lib/cli/args.mjs

@@ -0,0 +1,57 @@
+export const SUITE_TYPES = new Set(["int", "integration", "e2e", "dal", "load", "all"]);
+export const LIFECYCLE = new Set(["status", "destroy"]);
+export const RESERVED = new Set([...SUITE_TYPES, ...LIFECYCLE]);
+
+export function resolveCliSelection({ first, second, serviceNames }) {
+  let service = null;
+  let type = null;
+
+  if (first && serviceNames.has(first)) {
+    service = first;
+    type = second || null;
+  } else if (first && RESERVED.has(first)) {
+    type = first;
+  } else if (first) {
+    throw new Error(
+      `Unknown argument "${first}". Expected a service name (${[...serviceNames].join(", ") || "none found"}) ` +
+      `or suite type (int, e2e, dal, all).`
+    );
+  }
+
+  return { service, type };
+}
+
+export function validateFrameworkOption(value) {
+  if (!["all", "k6", "playwright"].includes(value)) {
+    throw new Error(
+      `Unknown framework "${value}". Expected one of: all, k6, playwright.`
+    );
+  }
+}
+
+export function parseJobsOption(value) {
+  const jobs = Number.parseInt(String(value), 10);
+  if (!Number.isInteger(jobs) || jobs <= 0) {
+    throw new Error(`Invalid --jobs value "${value}". Expected a positive integer.`);
+  }
+  return jobs;
+}
+
+export function parseShardOption(value) {
+  if (!value) return null;
+
+  const match = String(value).match(/^(\d+)\/(\d+)$/);
+  if (!match) {
+    throw new Error(
+      `Invalid --shard value "${value}". Expected the form "i/n", e.g. 1/3.`
+    );
+  }
+
+  const index = Number.parseInt(match[1], 10);
+  const total = Number.parseInt(match[2], 10);
+  if (index <= 0 || total <= 0 || index > total) {
+    throw new Error(`Invalid --shard value "${value}". Expected 1 <= i <= n.`);
+  }
+
+  return { index, total };
+}

package/lib/cli/args.test.mjs

@@ -0,0 +1,62 @@
+import { describe, expect, it } from "vitest";
+import {
+  parseJobsOption,
+  parseShardOption,
+  resolveCliSelection,
+  validateFrameworkOption,
+} from "./args.mjs";
+
+describe("cli-args", () => {
+  it("resolves a service and suite type", () => {
+    expect(
+      resolveCliSelection({
+        first: "api",
+        second: "int",
+        serviceNames: new Set(["api", "frontend"]),
+      })
+    ).toEqual({
+      service: "api",
+      type: "int",
+    });
+  });
+
+  it("resolves a reserved suite type without a service", () => {
+    expect(
+      resolveCliSelection({
+        first: "e2e",
+        second: null,
+        serviceNames: new Set(["api"]),
+      })
+    ).toEqual({
+      service: null,
+      type: "e2e",
+    });
+  });
+
+  it("rejects unknown positional arguments", () => {
+    expect(() =>
+      resolveCliSelection({
+        first: "mystery",
+        second: null,
+        serviceNames: new Set(["api"]),
+      })
+    ).toThrow('Unknown argument "mystery"');
+  });
+
+  it("validates framework names", () => {
+    expect(() => validateFrameworkOption("playwright")).not.toThrow();
+    expect(() => validateFrameworkOption("jest")).toThrow("Unknown framework");
+  });
+
+  it("parses and validates jobs", () => {
+    expect(parseJobsOption("3")).toBe(3);
+    expect(() => parseJobsOption("0")).toThrow("Invalid --jobs value");
+  });
+
+  it("parses and validates shards", () => {
+    expect(parseShardOption("2/5")).toEqual({ index: 2, total: 5 });
+    expect(parseShardOption(null)).toBeNull();
+    expect(() => parseShardOption("2-of-5")).toThrow("Invalid --shard value");
+    expect(() => parseShardOption("3/2")).toThrow("Expected 1 <= i <= n");
+  });
+});

package/lib/cli/index.mjs

@@ -0,0 +1,88 @@
+import { cac } from "cac";
+import { loadConfigs, getServiceNames } from "../config/index.mjs";
+import {
+  parseJobsOption,
+  parseShardOption,
+  RESERVED,
+  resolveCliSelection,
+  validateFrameworkOption,
+} from "./args.mjs";
+import * as runner from "../runner/index.mjs";
+
+export function run() {
+  const cli = cac("testkit");
+
+  cli
+    .command("[first] [second] [third]", "Run test suites (int, e2e, dal, all)")
+    .option("-s, --suite <name>", "Run specific suite(s)", { default: [] })
+    .option("--dir <path>", "Explicit product directory")
+    .option("--jobs <n>", "Number of isolated worker stacks for the whole run", {
+      default: "1",
+    })
+    .option("--shard <i/n>", "Run only shard i of n at suite granularity")
+    .option("--framework <name>", "Filter by framework (k6, playwright, all)", {
+      default: "all",
+    })
+    .action(async (first, second, third, options) => {
+      // Resolve: service filter, suite type, and --dir.
+      //
+      // From product dir:
+      //   testkit                          → all services, all types
+      //   testkit int -s health            → all services, int, health
+      //   testkit api int                  → one service, int
+      //   testkit api                      → one service, all types
+      //
+      // From workspace root:
+      //   testkit --dir my-product int     → all services, int
+      //   testkit --dir my-product api int → one service, int
+
+      // Now resolve service vs type from remaining args
+      const serviceNames = new Set(getServiceNames(options.dir));
+      const { service, type } = resolveCliSelection({
+        first,
+        second,
+        serviceNames,
+      });
+
+      const allConfigs = loadConfigs({ dir: options.dir });
+      const configs = service
+        ? allConfigs.filter((config) => config.name === service)
+        : allConfigs;
+      if (service && configs.length === 0) {
+        const available = allConfigs.map((config) => config.name).join(", ");
+        throw new Error(`Service "${service}" not found. Available: ${available}`);
+      }
+
+      // Lifecycle commands
+      if (type === "status" || type === "destroy") {
+        for (const config of configs) {
+          if (configs.length > 1) console.log(`\n── ${config.name} ──`);
+          if (type === "status") runner.showStatus(config);
+          else await runner.destroy(config);
+        }
+        return;
+      }
+
+      validateFrameworkOption(options.framework);
+
+      const jobs = parseJobsOption(options.jobs);
+      const shard = parseShardOption(options.shard);
+
+      const suiteType = type || "all";
+      const suiteNames = Array.isArray(options.suite) ? options.suite : [options.suite].filter(Boolean);
+      await runner.runAll(
+        configs,
+        suiteType,
+        suiteNames,
+        {
+          ...options,
+          jobs,
+          shard,
+        },
+        allConfigs
+      );
+    });
+
+  cli.help();
+  cli.parse();
+}

package/lib/config/index.mjs

@@ -0,0 +1,294 @@
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import {
+  getServiceEnvFiles as getServiceEnvFilesModel,
+  isDalSuiteType as isDalSuiteTypeModel,
+  isObject as isObjectModel,
+  normalizeTelemetryConfig as normalizeTelemetryConfigModel,
+  normalizeTemplateConfig as normalizeTemplateConfigModel,
+  parseDotenvString,
+  requireString as requireStringModel,
+  resolveLifecycleConfig as resolveLifecycleConfigModel,
+  resolveSelectedDatabase as resolveSelectedDatabaseModel,
+  validateConfigCoverage as validateConfigCoverageModel,
+  validateDatabaseProviderConfig as validateDatabaseProviderConfigModel,
+  validateHttpUrl as validateHttpUrlModel,
+  validateLifecycleConfig as validateLifecycleConfigModel,
+  validateRunnerManifest,
+  validateServiceConfig as validateServiceConfigModel,
+  validateTelemetryConfig as validateTelemetryConfigModel,
+  validateTemplateConfig as validateTemplateConfigModel,
+} from "./model.mjs";
+
+const RUNNER_MANIFEST = "runner.manifest.json";
+const TESTKIT_CONFIG = "testkit.config.json";
+export function parseDotenv(filePath) {
+  if (!fs.existsSync(filePath)) return {};
+  return parseDotenvString(fs.readFileSync(filePath, "utf8"));
+}
+
+export function getServiceNames(cwd) {
+  const dir = cwd || process.cwd();
+  const runnerPath = path.join(dir, RUNNER_MANIFEST);
+  if (!fs.existsSync(runnerPath)) return [];
+  const runner = JSON.parse(fs.readFileSync(runnerPath, "utf8"));
+  if (!isObject(runner.services)) return [];
+  return Object.keys(runner.services);
+}
+
+export function loadConfigs(opts = {}) {
+  const productDir = resolveProductDir(process.cwd(), opts.dir);
+  const runner = loadRunnerManifest(productDir);
+  const config = loadTestkitConfig(productDir);
+  validateConfigCoverage(runner, config);
+
+  const serviceEntries = Object.entries(runner.services);
+  const filtered = opts.service
+    ? serviceEntries.filter(([name]) => name === opts.service)
+    : serviceEntries;
+
+  if (opts.service && filtered.length === 0) {
+    const available = serviceEntries.map(([name]) => name).join(", ");
+    throw new Error(`Service "${opts.service}" not found. Available: ${available}`);
+  }
+
+  return filtered.map(([name, runnerService]) => {
+    const serviceConfig = config.services[name];
+    if (!serviceConfig) {
+      throw new Error(
+        `Service "${name}" exists in ${RUNNER_MANIFEST} but is missing from ${TESTKIT_CONFIG}`
+      );
+    }
+
+    const resolvedDatabase = resolveSelectedDatabase(name, serviceConfig);
+    const serviceEnv = loadServiceEnv(productDir, serviceConfig);
+    const selectedBackend = resolvedDatabase?.selectedBackend;
+    const resolvedMigrate = resolveLifecycleConfig(serviceConfig.migrate, selectedBackend);
+    const resolvedSeed = resolveLifecycleConfig(serviceConfig.seed, selectedBackend);
+    validateMergedService(
+      name,
+      runnerService,
+      serviceConfig,
+      resolvedDatabase,
+      resolvedMigrate,
+      resolvedSeed,
+      productDir
+    );
+
+    return {
+      name,
+      productDir,
+      stateDir: path.join(productDir, ".testkit", name),
+      telemetry: normalizeTelemetryConfig(config.telemetry),
+      suites: runnerService.suites,
+      testkit: {
+        ...serviceConfig,
+        database: resolvedDatabase,
+        migrate: resolvedMigrate,
+        seed: resolvedSeed,
+        envFiles: getServiceEnvFiles(serviceConfig),
+        serviceEnv,
+      },
+    };
+  });
+}
+
+export function resolveDalBinary() {
+  const thisFile = fileURLToPath(import.meta.url);
+  const abs = path.resolve(path.dirname(thisFile), "..", "..", "vendor", "k6");
+  if (!fs.existsSync(abs)) {
+    throw new Error(`Bundled DAL k6 binary not found: ${abs}`);
+  }
+  return abs;
+}
+
+export function resolveServiceCwd(productDir, maybeRelative) {
+  return path.resolve(productDir, maybeRelative || ".");
+}
+
+function loadRunnerManifest(productDir) {
+  const manifestPath = path.join(productDir, RUNNER_MANIFEST);
+  const raw = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
+  validateRunnerManifest(raw, RUNNER_MANIFEST, manifestPath);
+  return raw;
+}
+
+function loadTestkitConfig(productDir) {
+  const configPath = path.join(productDir, TESTKIT_CONFIG);
+  const raw = JSON.parse(fs.readFileSync(configPath, "utf8"));
+
+  if (!isObject(raw.services)) {
+    throw new Error(`${TESTKIT_CONFIG} must have a "services" object (${configPath})`);
+  }
+
+  if (raw.telemetry !== undefined) {
+    validateTelemetryConfigModel(raw.telemetry, configPath, TESTKIT_CONFIG);
+  }
+
+  for (const [serviceName, service] of Object.entries(raw.services)) {
+    validateServiceConfigModel(serviceName, service, configPath);
+  }
+
+  return raw;
+}
+
+function validateConfigCoverage(runner, config) {
+  return validateConfigCoverageModel(runner, config, TESTKIT_CONFIG, RUNNER_MANIFEST);
+}
+
+function resolveProductDir(cwd, explicitDir) {
+  if (explicitDir) {
+    const dir = path.resolve(cwd, explicitDir);
+    ensureProductFiles(dir);
+    return dir;
+  }
+
+  ensureProductFiles(cwd);
+  return cwd;
+}
+
+function ensureProductFiles(dir) {
+  const missing = [RUNNER_MANIFEST, TESTKIT_CONFIG].filter(
+    (file) => !fs.existsSync(path.join(dir, file))
+  );
+  if (missing.length > 0) {
+    throw new Error(
+      `Expected ${missing.join(" and ")} in ${dir}. Either cd into a product directory or use --dir.`
+    );
+  }
+}
+
+function validateMergedService(
+  name,
+  runnerService,
+  serviceConfig,
+  resolvedDatabase,
+  resolvedMigrate,
+  resolvedSeed,
+  productDir
+) {
+  const usesLocalExecution = Object.values(runnerService.suites).some((suites) =>
+    suites.some(
+      (suite) =>
+        (suite.framework && suite.framework !== "k6") ||
+        !isDalSuiteType(suite, runnerService, suites)
+    )
+  );
+
+  if (usesLocalExecution && !isObject(serviceConfig.local)) {
+    throw new Error(
+      `Service "${name}" defines non-DAL suites but has no local runtime in ${TESTKIT_CONFIG}`
+    );
+  }
+
+  if (serviceConfig.dependsOn) {
+    for (const dep of serviceConfig.dependsOn) {
+      if (dep === name) {
+        throw new Error(`Service "${name}" cannot depend on itself`);
+      }
+    }
+  }
+
+  if (
+    resolvedDatabase?.provider === "local" &&
+    (serviceConfig.migrate || serviceConfig.seed) &&
+    resolvedDatabase.template.inputs.length === 0
+  ) {
+    throw new Error(
+      `Service "${name}" uses local database provisioning with migrations or seeds, so database.template.inputs must list the files/directories that define the template cache`
+    );
+  }
+
+  if (serviceConfig.local?.cwd) {
+    const cwdPath = resolveServiceCwd(productDir, serviceConfig.local.cwd);
+    if (!fs.existsSync(cwdPath)) {
+      throw new Error(
+        `Service "${name}" local.cwd does not exist: ${serviceConfig.local.cwd}`
+      );
+    }
+  }
+
+  if (resolvedMigrate?.cwd) {
+    const cwdPath = resolveServiceCwd(productDir, resolvedMigrate.cwd);
+    if (!fs.existsSync(cwdPath)) {
+      throw new Error(
+        `Service "${name}" migrate.cwd does not exist: ${resolvedMigrate.cwd}`
+      );
+    }
+  }
+
+  if (resolvedSeed?.cwd) {
+    const cwdPath = resolveServiceCwd(productDir, resolvedSeed.cwd);
+    if (!fs.existsSync(cwdPath)) {
+      throw new Error(
+        `Service "${name}" seed.cwd does not exist: ${resolvedSeed.cwd}`
+      );
+    }
+  }
+
+}
+
+function validateServiceConfig(name, service, configPath) {
+  return validateServiceConfigModel(name, service, configPath);
+}
+
+function validateTelemetryConfig(telemetry, configPath) {
+  return validateTelemetryConfigModel(telemetry, configPath, TESTKIT_CONFIG);
+}
+
+function loadServiceEnv(productDir, serviceConfig) {
+  const env = {};
+  for (const envFile of getServiceEnvFiles(serviceConfig)) {
+    Object.assign(env, parseDotenv(resolveServiceCwd(productDir, envFile)));
+  }
+  return env;
+}
+
+function getServiceEnvFiles(serviceConfig) {
+  return getServiceEnvFilesModel(serviceConfig);
+}
+
+function resolveSelectedDatabase(name, serviceConfig) {
+  return resolveSelectedDatabaseModel(name, serviceConfig);
+}
+
+function validateDatabaseProviderConfig(name, db, label) {
+  return validateDatabaseProviderConfigModel(name, db, label);
+}
+
+function validateTemplateConfig(name, template, label) {
+  return validateTemplateConfigModel(name, template, label);
+}
+
+function normalizeTemplateConfig(template) {
+  return normalizeTemplateConfigModel(template);
+}
+
+function validateLifecycleConfig(name, value, label) {
+  return validateLifecycleConfigModel(name, value, label);
+}
+
+function resolveLifecycleConfig(value, selectedBackend) {
+  return resolveLifecycleConfigModel(value, selectedBackend);
+}
+
+function requireString(obj, key, label) {
+  return requireStringModel(obj, key, label);
+}
+
+function isDalSuiteType(suite, runnerService, suitesForType) {
+  return isDalSuiteTypeModel(suite, runnerService, suitesForType);
+}
+
+function isObject(value) {
+  return isObjectModel(value);
+}
+
+function normalizeTelemetryConfig(telemetry) {
+  return normalizeTelemetryConfigModel(telemetry);
+}
+
+function validateHttpUrl(value, label) {
+  return validateHttpUrlModel(value, label);
+}

package/lib/config/index.test.mjs

@@ -0,0 +1,12 @@
+import fs from "fs";
+import path from "path";
+import { describe, expect, it } from "vitest";
+import { resolveDalBinary } from "./index.mjs";
+
+describe("config-index", () => {
+  it("resolves the bundled DAL k6 binary from the package root", () => {
+    const binaryPath = resolveDalBinary();
+    expect(path.basename(binaryPath)).toBe("k6");
+    expect(fs.existsSync(binaryPath)).toBe(true);
+  });
+});