@elench/testkit 0.1.1 → 0.1.2

package/infra/fly-build.sh

@@ -29,6 +29,16 @@ if [ -f "$STATE_DIR/fly_image" ]; then
   fi
 fi
 
+# Skip if image already exists in registry (e.g. built by a previous run without state)
+if git -C "$API_DIR" diff --quiet 2>/dev/null; then
+  fly auth docker 2>/dev/null
+  if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
+    echo "Image $IMAGE already in registry — skipping build"
+    echo "$IMAGE" > "$STATE_DIR/fly_image"
+    exit 0
+  fi
+fi
+
 # Ensure the Fly app exists (registry requires it)
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 bash "$SCRIPT_DIR/fly-app-ensure.sh"

package/infra/fly-up.sh

@@ -1,7 +1,8 @@
 #!/usr/bin/env bash
 # Ensures a Fly machine is running with the correct env vars and image.
 # Reuses an existing stopped machine if available; creates a new one if not.
-# Requires: FLY_APP, FLY_REGION, FLY_IMAGE
+# Requires: FLY_APP, FLY_REGION
+# Image required only for --build or new machines (FLY_IMAGE / FLY_IMAGE_DEFAULT / .state/fly_image)
 # Optional: DB_URL_ENV_NAME (default: DATABASE_URL), FLY_PORT, FLY_ENV_FILE
 set -eo pipefail
 
@@ -15,12 +16,11 @@ FLY_REGION="${FLY_REGION:-lhr}"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 bash "$SCRIPT_DIR/fly-app-ensure.sh"
 
-# Image resolution: explicit $FLY_IMAGE → .state/fly_image → $FLY_IMAGE_DEFAULT → error
+# Resolve image if available (not required yet — may not be needed for reuse)
 if [ -z "$FLY_IMAGE" ] && [ -f "$STATE_DIR/fly_image" ]; then
   FLY_IMAGE="$(cat "$STATE_DIR/fly_image")"
 fi
 FLY_IMAGE="${FLY_IMAGE:-$FLY_IMAGE_DEFAULT}"
-FLY_IMAGE="${FLY_IMAGE:?FLY_IMAGE required — run fly-build.sh first or set FLY_IMAGE}"
 
 if [ ! -f "$STATE_DIR/database_url" ]; then
   echo "ERROR: No database_url — run neon-up.sh first"
@@ -40,33 +40,56 @@ fi
 
 MACHINE_ID=""
 
-# ── Try to reuse existing machine ────────────────────────────────────────
+# ── 1. Try state file ────────────────────────────────────────────────────
 if [ -f "$STATE_DIR/fly_machine_id" ]; then
   EXISTING_ID=$(cat "$STATE_DIR/fly_machine_id")
   if fly machines status "$EXISTING_ID" --app "$FLY_APP" >/dev/null 2>&1; then
-    echo "Reusing machine $EXISTING_ID — updating config"
-
-    # Update env vars + image (handles DATABASE_URL change and code changes)
-    # shellcheck disable=SC2086
-    fly machines update "$EXISTING_ID" --app "$FLY_APP" \
-      --env "$DB_URL_ENV_NAME=$DATABASE_URL" \
-      $FLY_ENV_FLAGS \
-      --image "$FLY_IMAGE" \
-      --yes 2>&1
-
-    # Start if not already running (update may auto-restart)
-    fly machines start "$EXISTING_ID" --app "$FLY_APP" 2>/dev/null || true
-    fly machines wait "$EXISTING_ID" --app "$FLY_APP" --state started --wait-timeout 60s
-
     MACHINE_ID="$EXISTING_ID"
   else
-    echo "Stored machine $EXISTING_ID gone — creating new"
+    echo "Stored machine $EXISTING_ID gone — will discover or create"
     rm -f "$STATE_DIR/fly_machine_id"
   fi
 fi
 
-# ── Create new machine if needed ─────────────────────────────────────────
+# ── 2. Discover existing machine on the app ──────────────────────────────
+if [ -z "$MACHINE_ID" ]; then
+  DISCOVERED_ID=$(fly machines list --app "$FLY_APP" --json 2>/dev/null \
+    | jq -r '.[0].id // empty')
+  if [ -n "$DISCOVERED_ID" ]; then
+    MACHINE_ID="$DISCOVERED_ID"
+    echo "Discovered existing machine: $MACHINE_ID"
+  fi
+fi
+
+# ── 3. Update and start existing machine ─────────────────────────────────
+if [ -n "$MACHINE_ID" ]; then
+  echo "Reusing machine $MACHINE_ID — updating config"
+
+  # Build update flags: always update DATABASE_URL + target env
+  UPDATE_FLAGS="--env $DB_URL_ENV_NAME=$DATABASE_URL $FLY_ENV_FLAGS"
+
+  # Only update image if one was provided (--build was used)
+  if [ -n "$FLY_IMAGE" ]; then
+    UPDATE_FLAGS="$UPDATE_FLAGS --image $FLY_IMAGE"
+  fi
+
+  # shellcheck disable=SC2086
+  fly machines update "$MACHINE_ID" --app "$FLY_APP" \
+    $UPDATE_FLAGS \
+    --yes 2>&1
+
+  # Start if not already running (update may auto-restart)
+  fly machines start "$MACHINE_ID" --app "$FLY_APP" 2>/dev/null || true
+  fly machines wait "$MACHINE_ID" --app "$FLY_APP" --state started --wait-timeout 60s
+
+  echo "$MACHINE_ID" > "$STATE_DIR/fly_machine_id"
+fi
+
+# ── 4. Create new machine if needed ──────────────────────────────────────
 if [ -z "$MACHINE_ID" ]; then
+  # Image is required for new machines
+  FLY_IMAGE="${FLY_IMAGE:?FLY_IMAGE required — run with --build or set FLY_IMAGE}"
+
   echo "Creating Fly machine (app: $FLY_APP, region: $FLY_REGION)"
 
   # shellcheck disable=SC2086

package/infra/neon-up.sh

@@ -11,7 +11,7 @@ NEON_API="https://console.neon.tech/api/v2/projects/$NEON_PROJECT_ID"
 
 mkdir -p "$STATE_DIR"
 
-# ── Check for existing branch ────────────────────────────────────────────
+# ── 1. Check state file ──────────────────────────────────────────────────
 BRANCH_ID=""
 if [ -f "$STATE_DIR/neon_branch_id" ]; then
   STORED_ID=$(cat "$STATE_DIR/neon_branch_id")
@@ -20,12 +20,25 @@ if [ -f "$STATE_DIR/neon_branch_id" ]; then
     BRANCH_ID="$STORED_ID"
     echo "Neon branch exists: $BRANCH_ID"
   else
-    echo "Stored branch $STORED_ID gone — creating new"
+    echo "Stored branch $STORED_ID gone — will discover or create"
     rm -f "$STATE_DIR/neon_branch_id"
   fi
 fi
 
-# ── Create branch if needed ──────────────────────────────────────────────
+# ── 2. Discover existing branch by name ──────────────────────────────────
+if [ -z "$BRANCH_ID" ]; then
+  EXISTING_ID=$(curl -sf "$NEON_API/branches" \
+    -H "Authorization: Bearer $NEON_API_KEY" \
+    | jq -r --arg name "$BRANCH_NAME" '.branches[] | select(.name == $name) | .id' | head -1)
+
+  if [ -n "$EXISTING_ID" ] && [ "$EXISTING_ID" != "null" ]; then
+    BRANCH_ID="$EXISTING_ID"
+    echo "$BRANCH_ID" > "$STATE_DIR/neon_branch_id"
+    echo "Discovered existing Neon branch '$BRANCH_NAME': $BRANCH_ID"
+  fi
+fi
+
+# ── 3. Create branch if needed ───────────────────────────────────────────
 if [ -z "$BRANCH_ID" ]; then
   echo "Creating Neon branch: $BRANCH_NAME"
   RESPONSE=$(curl -sf -X POST "$NEON_API/branches" \

package/lib/cli.mjs

@@ -1,16 +1,36 @@
 import { cac } from "cac";
-import { loadConfig } from "./config.mjs";
+import { loadConfig, inferProduct } from "./config.mjs";
 import * as runner from "./runner.mjs";
 
+const SUITE_TYPES = new Set(["int", "integration", "e2e", "dal", "load", "all"]);
+const LIFECYCLE = new Set(["status", "destroy"]);
+
 export function run() {
   const cli = cac("testkit");
 
   cli
-    .command("<product> [type]", "Run test suites")
+    .command("[first] [second]", "Run test suites (int, e2e, dal, all)")
     .option("-s, --suite <name>", "Run specific suite(s)", { default: [] })
     .option("--build", "Build image from source first", { default: false })
     .option("--dir <path>", "Explicit product directory")
-    .action(async (product, type, options) => {
+    .action(async (first, second, options) => {
+      // Resolve product and type from positional args.
+      // Supports:  testkit outreach int -s health
+      //            testkit int -s health          (infer product from cwd)
+      //            testkit outreach               (type defaults to "all")
+      //            testkit                        (infer product, type "all")
+      let product, type;
+
+      if (first && !SUITE_TYPES.has(first) && !LIFECYCLE.has(first)) {
+        // First arg is a product name
+        product = first;
+        type = second;
+      } else {
+        // First arg is a type/lifecycle or missing — infer product from cwd
+        product = inferProduct();
+        type = first;   // second is ignored (no product was given)
+      }
+
       // Handle lifecycle subcommands
       if (type === "status") {
         const config = loadConfig(product, options);

package/lib/config.mjs

@@ -24,6 +24,26 @@ export function parseDotenv(filePath) {
   return env;
 }
 
+/**
+ * Infer the product name from ./runner.manifest.json in the cwd.
+ * Used when the CLI is invoked without an explicit product argument.
+ */
+export function inferProduct(cwd) {
+  const dir = cwd || process.cwd();
+  const manifestPath = path.join(dir, "runner.manifest.json");
+  if (!fs.existsSync(manifestPath)) {
+    throw new Error(
+      `No runner.manifest.json in current directory. ` +
+      `Either cd into a product directory or pass the product name as the first argument.`
+    );
+  }
+  const m = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
+  if (typeof m.product !== "string" || !m.product) {
+    throw new Error(`runner.manifest.json is missing a "product" field`);
+  }
+  return m.product;
+}
+
 /**
  * Find the product directory and load the manifest + env.
  * Returns { productDir, manifest, stateDir }.
@@ -34,26 +54,22 @@ export function loadConfig(product, opts = {}) {
   const manifestPath = path.join(productDir, "runner.manifest.json");
   const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
 
-  if (!manifest.testkit) {
-    throw new Error(`No "testkit" section in ${manifestPath}`);
-  }
+  validateManifest(manifest, manifestPath);
 
   // Load product .env into process.env (product wins on conflict)
   const dotenv = parseDotenv(path.join(productDir, ".env"));
   Object.assign(process.env, dotenv);
 
-  // Validate required infra keys
-  for (const key of ["NEON_API_KEY", "FLY_API_TOKEN"]) {
-    if (!process.env[key]) {
-      throw new Error(
-        `${key} not found. Set it in your shell environment, .envrc, or ${product}/.env`
-      );
-    }
+  // Only require NEON_API_KEY unconditionally — Fly token is checked on demand
+  if (!process.env.NEON_API_KEY) {
+    throw new Error(
+      `NEON_API_KEY not found. Set it in your shell environment, .envrc, or ${product}/.env`
+    );
   }
 
   // Validate manifest secrets are available
   const tk = manifest.testkit;
-  for (const key of [...(tk.fly?.secrets || []), ...(tk.k6?.secrets || [])]) {
+  for (const key of [...(tk.fly?.secrets || []), ...(tk.k6?.secrets || []), ...(tk.dal?.secrets || [])]) {
     if (!process.env[key]) {
       throw new Error(`Secret "${key}" (from manifest) not found in env or ${product}/.env`);
     }
@@ -62,6 +78,33 @@ export function loadConfig(product, opts = {}) {
   return { productDir, manifest, stateDir: path.join(productDir, ".testkit") };
 }
 
+/**
+ * Require FLY_API_TOKEN. Call this only when HTTP tests need Fly infrastructure.
+ */
+export function requireFlyToken(product) {
+  if (!process.env.FLY_API_TOKEN) {
+    throw new Error(
+      `FLY_API_TOKEN not found. Set it in your shell environment, .envrc, or ${product}/.env`
+    );
+  }
+}
+
+/**
+ * Resolve the custom k6 binary for DAL tests. Returns the absolute path.
+ */
+export function resolveDalBinary(config) {
+  const { productDir, manifest } = config;
+  const rel = manifest.testkit.dal?.k6Binary;
+  if (!rel) {
+    throw new Error("testkit.dal.k6Binary is required for DAL tests");
+  }
+  const abs = path.resolve(productDir, rel);
+  if (!fs.existsSync(abs)) {
+    throw new Error(`DAL k6 binary not found: ${abs}`);
+  }
+  return abs;
+}
+
 function resolveProductDir(product, cwd, explicitDir) {
   if (explicitDir) {
     const p = path.resolve(cwd, explicitDir);
@@ -85,3 +128,134 @@ function resolveProductDir(product, cwd, explicitDir) {
     `Run from the workspace root or use --dir.`
   );
 }
+
+// --- Schema validation ---
+
+function validateManifest(manifest, manifestPath) {
+  const errors = [];
+  const ctx = `in ${manifestPath}`;
+
+  requireString(errors, manifest, "product", ctx);
+
+  if (!isObject(manifest.testkit)) {
+    errors.push(`Manifest error: testkit is required and must be an object (${ctx})`);
+  } else {
+    const tk = manifest.testkit;
+
+    // testkit.neon
+    if (!isObject(tk.neon)) {
+      errors.push(`Manifest error: testkit.neon is required and must be an object (${ctx})`);
+    } else {
+      requireString(errors, tk.neon, "testkit.neon.projectId", ctx, "projectId");
+      requireString(errors, tk.neon, "testkit.neon.dbName", ctx, "dbName");
+    }
+
+    // testkit.fly
+    if (!isObject(tk.fly)) {
+      errors.push(`Manifest error: testkit.fly is required and must be an object (${ctx})`);
+    } else {
+      requireString(errors, tk.fly, "testkit.fly.app", ctx, "app");
+      requireString(errors, tk.fly, "testkit.fly.org", ctx, "org");
+      optionalString(errors, tk.fly, "testkit.fly.region", ctx, "region");
+      optionalString(errors, tk.fly, "testkit.fly.port", ctx, "port");
+      optionalStringRecord(errors, tk.fly, "testkit.fly.env", ctx, "env");
+      optionalStringArray(errors, tk.fly, "testkit.fly.secrets", ctx, "secrets");
+    }
+
+    // testkit.k6
+    if (tk.k6 !== undefined) {
+      if (!isObject(tk.k6)) {
+        errors.push(`Manifest error: testkit.k6 must be an object (${ctx})`);
+      } else {
+        optionalStringArray(errors, tk.k6, "testkit.k6.secrets", ctx, "secrets");
+      }
+    }
+
+    // testkit.dal
+    if (tk.dal !== undefined) {
+      if (!isObject(tk.dal)) {
+        errors.push(`Manifest error: testkit.dal must be an object (${ctx})`);
+      } else {
+        // k6Binary is required if suites.dal exists
+        if (manifest.suites?.dal) {
+          requireString(errors, tk.dal, "testkit.dal.k6Binary", ctx, "k6Binary");
+        }
+        optionalStringArray(errors, tk.dal, "testkit.dal.secrets", ctx, "secrets");
+      }
+    }
+  }
+
+  // suites
+  if (!isObject(manifest.suites)) {
+    errors.push(`Manifest error: suites is required and must be an object (${ctx})`);
+  } else {
+    const keys = Object.keys(manifest.suites);
+    if (keys.length === 0) {
+      errors.push(`Manifest error: suites must have at least one key (${ctx})`);
+    }
+    for (const type of keys) {
+      const arr = manifest.suites[type];
+      if (!Array.isArray(arr)) {
+        errors.push(`Manifest error: suites.${type} must be an array (${ctx})`);
+        continue;
+      }
+      for (let i = 0; i < arr.length; i++) {
+        const suite = arr[i];
+        const prefix = `suites.${type}[${i}]`;
+        requireString(errors, suite, `${prefix}.name`, ctx, "name");
+        requireNonEmptyStringArray(errors, suite, `${prefix}.files`, ctx, "files");
+      }
+    }
+  }
+
+  // dal.k6Binary required if suites.dal exists but testkit.dal is missing
+  if (manifest.suites?.dal && !manifest.testkit?.dal) {
+    errors.push(`Manifest error: testkit.dal is required when suites.dal exists (${ctx})`);
+  }
+
+  if (errors.length) {
+    throw new Error(errors.join("\n"));
+  }
+}
+
+function isObject(v) {
+  return v !== null && typeof v === "object" && !Array.isArray(v);
+}
+
+function requireString(errors, obj, path, ctx, key) {
+  const k = key || path;
+  const val = obj[k];
+  if (typeof val !== "string" || val.length === 0) {
+    errors.push(`Manifest error: ${path} is required (${ctx})`);
+  }
+}
+
+function optionalString(errors, obj, path, ctx, key) {
+  const val = obj[key];
+  if (val !== undefined && typeof val !== "string") {
+    errors.push(`Manifest error: ${path} must be a string (${ctx})`);
+  }
+}
+
+function optionalStringArray(errors, obj, path, ctx, key) {
+  const val = obj[key];
+  if (val === undefined) return;
+  if (!Array.isArray(val) || !val.every(v => typeof v === "string")) {
+    errors.push(`Manifest error: ${path} must be a string[] (${ctx})`);
+  }
+}
+
+function optionalStringRecord(errors, obj, path, ctx, key) {
+  const val = obj[key];
+  if (val === undefined) return;
+  if (!isObject(val) || !Object.values(val).every(v => typeof v === "string")) {
+    errors.push(`Manifest error: ${path} must be a Record<string, string> (${ctx})`);
+  }
+}
+
+function requireNonEmptyStringArray(errors, obj, path, ctx, key) {
+  const val = obj[key];
+  if (!Array.isArray(val) || val.length === 0 || !val.every(v => typeof v === "string")) {
+    errors.push(`Manifest error: ${path} is required and must be a non-empty string[] (${ctx})`);
+  }
+}

package/lib/runner.mjs

@@ -2,6 +2,9 @@ import fs from "fs";
 import path from "path";
 import { runScript } from "./exec.mjs";
 import { execaCommand } from "execa";
+import { requireFlyToken, resolveDalBinary } from "./config.mjs";
+
+const HTTP_SUITE_TYPES = new Set(["integration", "e2e", "load"]);
 
 /**
  * Build the Docker image and push to Fly registry.
@@ -84,7 +87,7 @@ export async function destroy(config) {
 }
 
 /**
- * Run k6 test files with the correct env flags.
+ * Run k6 test files against the Fly machine (HTTP tests).
  */
 export async function runTests(config, files) {
   const { productDir, stateDir, manifest } = config;
@@ -114,6 +117,36 @@ export async function runTests(config, files) {
   return { failed };
 }
 
+/**
+ * Run k6 DAL test files directly against Neon (no Fly machine).
+ */
+export async function runDalTests(config, files) {
+  const { productDir, stateDir, manifest } = config;
+  const tk = manifest.testkit;
+  const k6Binary = resolveDalBinary(config);
+
+  // Read DATABASE_URL from neon state
+  const databaseUrl = fs.readFileSync(path.join(stateDir, "database_url"), "utf8").trim();
+
+  // Build -e flags
+  const envFlags = [`-e DATABASE_URL=${databaseUrl}`];
+  for (const key of tk.dal?.secrets || []) {
+    envFlags.push(`-e ${key}=${process.env[key]}`);
+  }
+  const envStr = envFlags.join(" ");
+
+  let failed = false;
+  for (const file of files) {
+    const absFile = path.join(productDir, file);
+    try {
+      await execaCommand(`${k6Binary} run ${envStr} ${absFile}`, { stdio: "inherit" });
+    } catch (err) {
+      failed = true;
+    }
+  }
+  return { failed };
+}
+
 /**
  * Show state directory contents.
  */
@@ -131,41 +164,60 @@ export function showStatus(config) {
 }
 
 /**
- * Main entry: build → neon → fly → k6 → stop.
+ * Main entry: orchestrate HTTP and/or DAL test flows.
  */
 export async function run(config, suiteType, suiteNames, opts) {
   const { manifest, stateDir } = config;
 
-  // Collect test files from manifest
+  // Collect test files from manifest, partitioned by flow
   const types = suiteType === "all"
     ? Object.keys(manifest.suites)
     : [suiteType === "int" ? "integration" : suiteType];
 
-  let files = [];
+  let httpFiles = [];
+  let dalFiles = [];
   for (const type of types) {
     const suites = manifest.suites[type] || [];
     for (const suite of suites) {
       if (suiteNames.length && !suiteNames.includes(suite.name)) continue;
-      files.push(...suite.files);
+      if (type === "dal") {
+        dalFiles.push(...suite.files);
+      } else {
+        httpFiles.push(...suite.files);
+      }
     }
   }
 
-  if (!files.length) {
+  if (!httpFiles.length && !dalFiles.length) {
     throw new Error(`No test files found for type="${suiteType}" suites=${suiteNames.join(",") || "all"}`);
   }
 
   fs.mkdirSync(stateDir, { recursive: true });
 
-  if (opts.build) await build(config);
-  await neonUp(config);
-  await flyUp(config);
+  let failed = false;
+  let neonReady = false;
+
+  // HTTP flow: build → neon → fly → run → fly down
+  if (httpFiles.length) {
+    requireFlyToken(manifest.product);
+    if (opts.build) await build(config);
+    await neonUp(config);
+    neonReady = true;
+    await flyUp(config);
+    try {
+      const result = await runTests(config, httpFiles);
+      if (result?.failed) failed = true;
+    } finally {
+      await flyDown(config);
+    }
+  }
 
-  let result;
-  try {
-    result = await runTests(config, files);
-  } finally {
-    await flyDown(config);
+  // DAL flow: neon (idempotent) → run DAL tests
+  if (dalFiles.length) {
+    if (!neonReady) await neonUp(config);
+    const result = await runDalTests(config, dalFiles);
+    if (result?.failed) failed = true;
   }
 
-  if (result?.failed) process.exit(1);
+  if (failed) process.exit(1);
 }

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@elench/testkit",
-  "version": "0.1.1",
+  "version": "0.1.2",
+  "description": "CLI for running k6 tests against real, ephemeral infrastructure",
   "type": "module",
   "bin": {
     "testkit": "bin/testkit.mjs"