@elench/testkit 0.1.2 → 0.1.3

package/infra/neon-up.sh

@@ -51,9 +51,17 @@ if [ -z "$BRANCH_ID" ]; then
 
   BRANCH_ID=$(echo "$RESPONSE" | jq -r '.branch.id')
   if [ -z "$BRANCH_ID" ] || [ "$BRANCH_ID" = "null" ]; then
-    echo "ERROR: Failed to create branch"
-    echo "$RESPONSE" | jq .
-    exit 1
+    # Create failed — may be a 409 race (another parallel service created it).
+    # Re-discover by name before giving up.
+    BRANCH_ID=$(curl -sf "$NEON_API/branches" \
+      -H "Authorization: Bearer $NEON_API_KEY" \
+      | jq -r --arg name "$BRANCH_NAME" '.branches[] | select(.name == $name) | .id' | head -1)
+
+    if [ -z "$BRANCH_ID" ] || [ "$BRANCH_ID" = "null" ]; then
+      echo "ERROR: Failed to create or discover branch '$BRANCH_NAME'"
+      exit 1
+    fi
+    echo "Branch created by another process — discovered: $BRANCH_ID"
   fi
   echo "$BRANCH_ID" > "$STATE_DIR/neon_branch_id"
 

package/lib/cli.mjs

@@ -1,50 +1,77 @@
 import { cac } from "cac";
-import { loadConfig, inferProduct } from "./config.mjs";
+import { loadConfigs, getServiceNames, isSiblingProduct } from "./config.mjs";
 import * as runner from "./runner.mjs";
 
 const SUITE_TYPES = new Set(["int", "integration", "e2e", "dal", "load", "all"]);
 const LIFECYCLE = new Set(["status", "destroy"]);
+const RESERVED = new Set([...SUITE_TYPES, ...LIFECYCLE]);
 
 export function run() {
   const cli = cac("testkit");
 
   cli
-    .command("[first] [second]", "Run test suites (int, e2e, dal, all)")
+    .command("[first] [second] [third]", "Run test suites (int, e2e, dal, all)")
     .option("-s, --suite <name>", "Run specific suite(s)", { default: [] })
     .option("--build", "Build image from source first", { default: false })
     .option("--dir <path>", "Explicit product directory")
-    .action(async (first, second, options) => {
-      // Resolve product and type from positional args.
-      // Supports:  testkit outreach int -s health
-      //            testkit int -s health          (infer product from cwd)
-      //            testkit outreach               (type defaults to "all")
-      //            testkit                        (infer product, type "all")
-      let product, type;
-
-      if (first && !SUITE_TYPES.has(first) && !LIFECYCLE.has(first)) {
-        // First arg is a product name
-        product = first;
-        type = second;
-      } else {
-        // First arg is a type/lifecycle or missing — infer product from cwd
-        product = inferProduct();
-        type = first;   // second is ignored (no product was given)
+    .action(async (first, second, third, options) => {
+      // Resolve: service filter, suite type, and --dir.
+      //
+      // From product dir:
+      //   testkit                          → all services, all types
+      //   testkit int -s health            → all services, int, health
+      //   testkit avocado_api int          → one service, int
+      //   testkit avocado_api              → one service, all types
+      //
+      // From workspace root:
+      //   testkit --dir outreach int       → all services, int
+      //   testkit --dir avocado avocado_api int  → one service, int
+      //
+      // Legacy (sibling dir as first arg):
+      //   testkit outreach int             → --dir outreach, int
+
+      let service = null;
+      let type = null;
+
+      // If first arg is a sibling dir with a manifest, treat it as --dir
+      if (first && !RESERVED.has(first) && !options.dir && isSiblingProduct(first)) {
+        options.dir = first;
+        first = second;
+        second = third;
+        third = undefined;
       }
 
-      // Handle lifecycle subcommands
-      if (type === "status") {
-        const config = loadConfig(product, options);
-        return runner.showStatus(config);
+      // Now resolve service vs type from remaining args
+      const serviceNames = new Set(getServiceNames(options.dir));
+
+      if (first && serviceNames.has(first)) {
+        service = first;
+        type = second || null;
+      } else if (first && RESERVED.has(first)) {
+        type = first;
+      } else if (first) {
+        // Unknown arg — might be a service name that doesn't exist
+        throw new Error(
+          `Unknown argument "${first}". Expected a service name (${[...serviceNames].join(", ") || "none found"}) ` +
+          `or suite type (int, e2e, dal, all).`
+        );
       }
-      if (type === "destroy") {
-        const config = loadConfig(product, options);
-        return runner.destroy(config);
+
+      const configs = loadConfigs({ dir: options.dir, service });
+
+      // Lifecycle commands
+      if (type === "status" || type === "destroy") {
+        for (const config of configs) {
+          if (configs.length > 1) console.log(`\n── ${config.name} ──`);
+          if (type === "status") runner.showStatus(config);
+          else await runner.destroy(config);
+        }
+        return;
       }
 
-      const config = loadConfig(product, options);
       const suiteType = type || "all";
       const suiteNames = Array.isArray(options.suite) ? options.suite : [options.suite].filter(Boolean);
-      await runner.run(config, suiteType, suiteNames, options);
+      await runner.runAll(configs, suiteType, suiteNames, options);
     });
 
   cli.help();

package/lib/config.mjs

@@ -25,66 +25,89 @@ export function parseDotenv(filePath) {
 }
 
 /**
- * Infer the product name from ./runner.manifest.json in the cwd.
- * Used when the CLI is invoked without an explicit product argument.
+ * Read the manifest and return the service names.
+ * Used by the CLI to resolve ambiguous positional args.
  */
-export function inferProduct(cwd) {
+export function getServiceNames(cwd) {
   const dir = cwd || process.cwd();
   const manifestPath = path.join(dir, "runner.manifest.json");
-  if (!fs.existsSync(manifestPath)) {
-    throw new Error(
-      `No runner.manifest.json in current directory. ` +
-      `Either cd into a product directory or pass the product name as the first argument.`
-    );
-  }
+  if (!fs.existsSync(manifestPath)) return [];
   const m = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
-  if (typeof m.product !== "string" || !m.product) {
-    throw new Error(`runner.manifest.json is missing a "product" field`);
-  }
-  return m.product;
+  if (!isObject(m.services)) return [];
+  return Object.keys(m.services);
 }
 
 /**
- * Find the product directory and load the manifest + env.
- * Returns { productDir, manifest, stateDir }.
+ * Load configs for all (or one) service from the manifest.
+ * Returns an array of { name, productDir, stateDir, manifest } objects.
+ *
+ * opts.service — filter to a single service name
+ * opts.dir     — explicit product directory
  */
-export function loadConfig(product, opts = {}) {
-  const cwd = opts.cwd || process.cwd();
-  const productDir = resolveProductDir(product, cwd, opts.dir);
+export function loadConfigs(opts = {}) {
+  const cwd = process.cwd();
+  const productDir = resolveProductDir(cwd, opts.dir);
   const manifestPath = path.join(productDir, "runner.manifest.json");
-  const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
+  const raw = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
 
-  validateManifest(manifest, manifestPath);
+  if (!isObject(raw.services)) {
+    throw new Error(`Manifest must have a "services" object (${manifestPath})`);
+  }
 
   // Load product .env into process.env (product wins on conflict)
   const dotenv = parseDotenv(path.join(productDir, ".env"));
   Object.assign(process.env, dotenv);
 
-  // Only require NEON_API_KEY unconditionally — Fly token is checked on demand
   if (!process.env.NEON_API_KEY) {
     throw new Error(
-      `NEON_API_KEY not found. Set it in your shell environment, .envrc, or ${product}/.env`
+      `NEON_API_KEY not found. Set it in your shell environment, .envrc, or .env`
     );
   }
 
-  // Validate manifest secrets are available
-  const tk = manifest.testkit;
-  for (const key of [...(tk.fly?.secrets || []), ...(tk.k6?.secrets || []), ...(tk.dal?.secrets || [])]) {
-    if (!process.env[key]) {
-      throw new Error(`Secret "${key}" (from manifest) not found in env or ${product}/.env`);
-    }
+  const entries = Object.entries(raw.services);
+
+  // Filter by service name if requested
+  const filtered = opts.service
+    ? entries.filter(([name]) => name === opts.service)
+    : entries;
+
+  if (opts.service && filtered.length === 0) {
+    const available = entries.map(([n]) => n).join(", ");
+    throw new Error(`Service "${opts.service}" not found. Available: ${available}`);
   }
 
-  return { productDir, manifest, stateDir: path.join(productDir, ".testkit") };
+  return filtered.map(([name, svc]) => {
+    validateService(name, svc, manifestPath);
+
+    // Check service secrets are available
+    const tk = svc.testkit;
+    for (const key of [...(tk.fly?.secrets || []), ...(tk.k6?.secrets || []), ...(tk.dal?.secrets || [])]) {
+      if (!process.env[key]) {
+        throw new Error(`Secret "${key}" (service "${name}") not found in env or .env`);
+      }
+    }
+
+    // Each service gets its own state dir: .testkit/<service>/
+    const stateDir = entries.length === 1
+      ? path.join(productDir, ".testkit")
+      : path.join(productDir, ".testkit", name);
+
+    return {
+      name,
+      productDir,
+      stateDir,
+      manifest: { testkit: tk, suites: svc.suites },
+    };
+  });
 }
 
 /**
  * Require FLY_API_TOKEN. Call this only when HTTP tests need Fly infrastructure.
  */
-export function requireFlyToken(product) {
+export function requireFlyToken(serviceName) {
   if (!process.env.FLY_API_TOKEN) {
     throw new Error(
-      `FLY_API_TOKEN not found. Set it in your shell environment, .envrc, or ${product}/.env`
+      `FLY_API_TOKEN not found. Set it in your shell environment, .envrc, or .env`
     );
   }
 }
@@ -105,157 +128,120 @@ export function resolveDalBinary(config) {
   return abs;
 }
 
-function resolveProductDir(product, cwd, explicitDir) {
+// ── Directory resolution ────────────────────────────────────────────────
+
+function resolveProductDir(cwd, explicitDir) {
   if (explicitDir) {
     const p = path.resolve(cwd, explicitDir);
     if (fs.existsSync(path.join(p, "runner.manifest.json"))) return p;
     throw new Error(`No runner.manifest.json in ${p}`);
   }
 
-  // Try ./bourne/runner.manifest.json (workspace root)
-  const sibling = path.join(cwd, product);
-  if (fs.existsSync(path.join(sibling, "runner.manifest.json"))) return sibling;
-
-  // Try ./runner.manifest.json (already in product dir)
-  const here = path.join(cwd, "runner.manifest.json");
-  if (fs.existsSync(here)) {
-    const m = JSON.parse(fs.readFileSync(here, "utf8"));
-    if (m.product === product) return cwd;
-  }
+  // Check cwd
+  if (fs.existsSync(path.join(cwd, "runner.manifest.json"))) return cwd;
 
   throw new Error(
-    `Could not find runner.manifest.json for "${product}". ` +
-    `Run from the workspace root or use --dir.`
+    `No runner.manifest.json in current directory. ` +
+    `Either cd into a product directory or use --dir.`
   );
 }
 
-// --- Schema validation ---
+/**
+ * Check if the given name (relative to cwd) is a sibling directory with a manifest.
+ * Used by the CLI to distinguish dir args from service/type args.
+ */
+export function isSiblingProduct(name) {
+  const candidate = path.join(process.cwd(), name);
+  return fs.existsSync(path.join(candidate, "runner.manifest.json"));
+}
 
-function validateManifest(manifest, manifestPath) {
-  const errors = [];
-  const ctx = `in ${manifestPath}`;
+// ── Per-service validation ──────────────────────────────────────────────
 
-  requireString(errors, manifest, "product", ctx);
+function validateService(name, svc, manifestPath) {
+  const errors = [];
+  const ctx = `service "${name}" in ${manifestPath}`;
 
-  if (!isObject(manifest.testkit)) {
-    errors.push(`Manifest error: testkit is required and must be an object (${ctx})`);
+  if (!isObject(svc.testkit)) {
+    errors.push(`${ctx}: testkit is required and must be an object`);
   } else {
-    const tk = manifest.testkit;
+    const tk = svc.testkit;
 
-    // testkit.neon
     if (!isObject(tk.neon)) {
-      errors.push(`Manifest error: testkit.neon is required and must be an object (${ctx})`);
+      errors.push(`${ctx}: testkit.neon is required`);
     } else {
-      requireString(errors, tk.neon, "testkit.neon.projectId", ctx, "projectId");
-      requireString(errors, tk.neon, "testkit.neon.dbName", ctx, "dbName");
+      requireString(errors, tk.neon, `${ctx}: testkit.neon.projectId`, "projectId");
+      requireString(errors, tk.neon, `${ctx}: testkit.neon.dbName`, "dbName");
     }
 
-    // testkit.fly
     if (!isObject(tk.fly)) {
-      errors.push(`Manifest error: testkit.fly is required and must be an object (${ctx})`);
+      errors.push(`${ctx}: testkit.fly is required`);
     } else {
-      requireString(errors, tk.fly, "testkit.fly.app", ctx, "app");
-      requireString(errors, tk.fly, "testkit.fly.org", ctx, "org");
-      optionalString(errors, tk.fly, "testkit.fly.region", ctx, "region");
-      optionalString(errors, tk.fly, "testkit.fly.port", ctx, "port");
-      optionalStringRecord(errors, tk.fly, "testkit.fly.env", ctx, "env");
-      optionalStringArray(errors, tk.fly, "testkit.fly.secrets", ctx, "secrets");
+      requireString(errors, tk.fly, `${ctx}: testkit.fly.app`, "app");
+      requireString(errors, tk.fly, `${ctx}: testkit.fly.org`, "org");
+      optionalStringArray(errors, tk.fly, `${ctx}: testkit.fly.secrets`, "secrets");
     }
 
-    // testkit.k6
-    if (tk.k6 !== undefined) {
-      if (!isObject(tk.k6)) {
-        errors.push(`Manifest error: testkit.k6 must be an object (${ctx})`);
-      } else {
-        optionalStringArray(errors, tk.k6, "testkit.k6.secrets", ctx, "secrets");
-      }
+    if (tk.k6 !== undefined && !isObject(tk.k6)) {
+      errors.push(`${ctx}: testkit.k6 must be an object`);
     }
 
-    // testkit.dal
     if (tk.dal !== undefined) {
       if (!isObject(tk.dal)) {
-        errors.push(`Manifest error: testkit.dal must be an object (${ctx})`);
-      } else {
-        // k6Binary is required if suites.dal exists
-        if (manifest.suites?.dal) {
-          requireString(errors, tk.dal, "testkit.dal.k6Binary", ctx, "k6Binary");
-        }
-        optionalStringArray(errors, tk.dal, "testkit.dal.secrets", ctx, "secrets");
+        errors.push(`${ctx}: testkit.dal must be an object`);
+      } else if (svc.suites?.dal) {
+        requireString(errors, tk.dal, `${ctx}: testkit.dal.k6Binary`, "k6Binary");
       }
     }
+
+    if (svc.suites?.dal && !tk.dal) {
+      errors.push(`${ctx}: testkit.dal is required when suites.dal exists`);
+    }
   }
 
-  // suites
-  if (!isObject(manifest.suites)) {
-    errors.push(`Manifest error: suites is required and must be an object (${ctx})`);
+  if (!isObject(svc.suites)) {
+    errors.push(`${ctx}: suites is required and must be an object`);
   } else {
-    const keys = Object.keys(manifest.suites);
-    if (keys.length === 0) {
-      errors.push(`Manifest error: suites must have at least one key (${ctx})`);
-    }
-    for (const type of keys) {
-      const arr = manifest.suites[type];
+    for (const type of Object.keys(svc.suites)) {
+      const arr = svc.suites[type];
       if (!Array.isArray(arr)) {
-        errors.push(`Manifest error: suites.${type} must be an array (${ctx})`);
+        errors.push(`${ctx}: suites.${type} must be an array`);
         continue;
       }
       for (let i = 0; i < arr.length; i++) {
         const suite = arr[i];
-        const prefix = `suites.${type}[${i}]`;
-        requireString(errors, suite, `${prefix}.name`, ctx, "name");
-        requireNonEmptyStringArray(errors, suite, `${prefix}.files`, ctx, "files");
+        const prefix = `${ctx}: suites.${type}[${i}]`;
+        requireString(errors, suite, `${prefix}.name`, "name");
+        requireNonEmptyStringArray(errors, suite, `${prefix}.files`, "files");
       }
     }
   }
 
-  // dal.k6Binary required if suites.dal exists but testkit.dal is missing
-  if (manifest.suites?.dal && !manifest.testkit?.dal) {
-    errors.push(`Manifest error: testkit.dal is required when suites.dal exists (${ctx})`);
-  }
-
-  if (errors.length) {
-    throw new Error(errors.join("\n"));
-  }
+  if (errors.length) throw new Error(errors.join("\n"));
 }
 
+// ── Helpers ─────────────────────────────────────────────────────────────
+
 function isObject(v) {
   return v !== null && typeof v === "object" && !Array.isArray(v);
 }
 
-function requireString(errors, obj, path, ctx, key) {
-  const k = key || path;
-  const val = obj[k];
-  if (typeof val !== "string" || val.length === 0) {
-    errors.push(`Manifest error: ${path} is required (${ctx})`);
+function requireString(errors, obj, msg, key) {
+  if (typeof obj[key] !== "string" || obj[key].length === 0) {
+    errors.push(`${msg} is required`);
   }
 }
 
-function optionalString(errors, obj, path, ctx, key) {
-  const val = obj[key];
-  if (val !== undefined && typeof val !== "string") {
-    errors.push(`Manifest error: ${path} must be a string (${ctx})`);
-  }
-}
-
-function optionalStringArray(errors, obj, path, ctx, key) {
+function optionalStringArray(errors, obj, msg, key) {
   const val = obj[key];
   if (val === undefined) return;
   if (!Array.isArray(val) || !val.every(v => typeof v === "string")) {
-    errors.push(`Manifest error: ${path} must be a string[] (${ctx})`);
-  }
-}
-
-function optionalStringRecord(errors, obj, path, ctx, key) {
-  const val = obj[key];
-  if (val === undefined) return;
-  if (!isObject(val) || !Object.values(val).every(v => typeof v === "string")) {
-    errors.push(`Manifest error: ${path} must be a Record<string, string> (${ctx})`);
+    errors.push(`${msg} must be a string[]`);
   }
 }
 
-function requireNonEmptyStringArray(errors, obj, path, ctx, key) {
+function requireNonEmptyStringArray(errors, obj, msg, key) {
   const val = obj[key];
   if (!Array.isArray(val) || val.length === 0 || !val.every(v => typeof v === "string")) {
-    errors.push(`Manifest error: ${path} is required and must be a non-empty string[] (${ctx})`);
+    errors.push(`${msg} is required and must be a non-empty string[]`);
   }
 }

package/lib/runner.mjs

@@ -6,6 +6,39 @@ import { requireFlyToken, resolveDalBinary } from "./config.mjs";
 
 const HTTP_SUITE_TYPES = new Set(["integration", "e2e", "load"]);
 
+/**
+ * Run multiple service configs in parallel.
+ * Single service: runs directly. Multiple: Promise.allSettled.
+ */
+export async function runAll(configs, suiteType, suiteNames, opts) {
+  if (configs.length === 1) {
+    const failed = await runService(configs[0], suiteType, suiteNames, opts);
+    if (failed) process.exit(1);
+    return;
+  }
+
+  // Multiple services — run in parallel
+  const results = await Promise.allSettled(
+    configs.map(async (config) => {
+      console.log(`\n══ ${config.name} ══`);
+      return runService(config, suiteType, suiteNames, opts);
+    })
+  );
+
+  // Summarize
+  const summary = configs.map((c, i) => {
+    const r = results[i];
+    const ok = r.status === "fulfilled" && !r.value;
+    return `  ${ok ? "✓" : "✗"} ${c.name}`;
+  });
+  console.log(`\n── Summary ──\n${summary.join("\n")}`);
+
+  const anyFailed = results.some(
+    (r) => r.status === "rejected" || r.value === true
+  );
+  if (anyFailed) process.exit(1);
+}
+
 /**
  * Build the Docker image and push to Fly registry.
  */
@@ -111,7 +144,6 @@ export async function runTests(config, files) {
       await execaCommand(`k6 run ${envStr} ${absFile}`, { stdio: "inherit" });
     } catch (err) {
       failed = true;
-      // Continue running remaining files — report all failures, don't stop at first
     }
   }
   return { failed };
@@ -158,15 +190,18 @@ export function showStatus(config) {
   }
   for (const file of fs.readdirSync(stateDir)) {
     if (file === "fly-env.sh") continue;
-    const val = fs.readFileSync(path.join(stateDir, file), "utf8").trim();
+    const filePath = path.join(stateDir, file);
+    if (fs.statSync(filePath).isDirectory()) continue;
+    const val = fs.readFileSync(filePath, "utf8").trim();
     console.log(`  ${file}: ${val}`);
   }
 }
 
 /**
- * Main entry: orchestrate HTTP and/or DAL test flows.
+ * Run a single service: orchestrate HTTP and/or DAL test flows.
+ * Returns true if any tests failed.
  */
-export async function run(config, suiteType, suiteNames, opts) {
+async function runService(config, suiteType, suiteNames, opts) {
   const { manifest, stateDir } = config;
 
   // Collect test files from manifest, partitioned by flow
@@ -189,7 +224,8 @@ export async function run(config, suiteType, suiteNames, opts) {
   }
 
   if (!httpFiles.length && !dalFiles.length) {
-    throw new Error(`No test files found for type="${suiteType}" suites=${suiteNames.join(",") || "all"}`);
+    console.log(`No test files for ${config.name} type="${suiteType}" suites=${suiteNames.join(",") || "all"} — skipping`);
+    return false;
   }
 
   fs.mkdirSync(stateDir, { recursive: true });
@@ -199,7 +235,7 @@ export async function run(config, suiteType, suiteNames, opts) {
 
   // HTTP flow: build → neon → fly → run → fly down
   if (httpFiles.length) {
-    requireFlyToken(manifest.product);
+    requireFlyToken(config.name);
     if (opts.build) await build(config);
     await neonUp(config);
     neonReady = true;
@@ -219,5 +255,5 @@ export async function run(config, suiteType, suiteNames, opts) {
     if (result?.failed) failed = true;
   }
 
-  if (failed) process.exit(1);
+  return failed;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/testkit",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "CLI for running k6 tests against real, ephemeral infrastructure",
   "type": "module",
   "bin": {