@elench/testkit 0.1.48 → 0.1.50

package/lib/runner/results.mjs

@@ -114,7 +114,7 @@ export function recordTaskOutcome(trackers, task, outcome, finishedAt = Date.now
   const normalizedPath = normalizePathSeparators(task.file);
   const existingFileResult = suite.fileResultsByPath.get(normalizedPath);
   const status = normalizeOutcomeStatus(outcome);
-  if (status !== "skipped") {
+  if (status !== "skipped" && status !== "not_run") {
     suite.completedFileCount += 1;
   }
   if (existingFileResult) {
@@ -291,6 +291,7 @@ function formatFrameworkForArtifact(framework) {
 }
 
 function normalizeOutcomeStatus(outcome) {
+  if (outcome?.status === "not_run") return "not_run";
   if (outcome?.status === "skipped") return "skipped";
   return outcome?.failed ? "failed" : "passed";
 }

package/lib/runner/results.test.mjs

@@ -307,6 +307,67 @@ describe("runner results", () => {
     ]);
   });
 
+  it("preserves not-run task outcomes in suite and service summaries", () => {
+    const trackers = buildServiceTrackers(
+      [
+        {
+          skipped: false,
+          config: {
+            name: "api",
+            testkit: {
+              database: {
+                selectedBackend: null,
+              },
+            },
+          },
+          suites: [
+            {
+              name: "health",
+              type: "integration",
+              framework: "k6",
+              files: ["tests/a.int.testkit.ts", "tests/b.int.testkit.ts"],
+              orderIndex: 0,
+            },
+          ],
+        },
+      ],
+      1000
+    );
+
+    recordTaskOutcome(
+      trackers,
+      {
+        serviceName: "api",
+        suiteKey: "integration:health",
+        file: "tests/a.int.testkit.ts",
+      },
+      {
+        failed: false,
+        status: "not_run",
+        reason: "Graph initialization failed",
+        durationMs: 0,
+        error: null,
+      },
+      1000
+    );
+
+    const result = finalizeServiceResult(trackers.get("api"), 1000, 1200);
+
+    expect(result.completedFileCount).toBe(0);
+    expect(result.passedFileCount).toBe(0);
+    expect(result.failedFileCount).toBe(0);
+    expect(result.skippedFileCount).toBe(0);
+    expect(result.notRunFileCount).toBe(2);
+    expect(result.suites[0].files).toContainEqual({
+      path: "tests/a.int.testkit.ts",
+      failed: false,
+      status: "not_run",
+      durationMs: 0,
+      error: null,
+      reason: "Graph initialization failed",
+    });
+  });
+
   it("summarizes mixed db backends", () => {
     expect(
       summarizeDbBackend([{ dbBackend: "local" }, { dbBackend: "neon" }])

package/lib/runner/runtime-contexts.mjs

@@ -2,6 +2,7 @@ import fs from "fs";
 import path from "path";
 import { prepareDatabaseRuntime } from "../database/index.mjs";
 import { taskNeedsLocalRuntime } from "./planning.mjs";
+import { prepareRuntimeServices } from "./runtime-preparation.mjs";
 import { writeGraphMetadata } from "./state.mjs";
 import { startLocalServices, stopLocalServices } from "./services.mjs";
 import { resolveRuntimeInstanceConfigs } from "./template.mjs";
@@ -39,6 +40,7 @@ export async function ensureRuntimeInstanceReady(context, task, lifecycle) {
     if (!context.preparationPromise) {
       context.preparationPromise = (async () => {
         await prepareDatabases(context.runtimeConfigs);
+        await prepareRuntimeServices(context.runtimeConfigs);
         context.prepared = true;
       })().finally(() => {
         context.preparationPromise = null;

package/lib/runner/runtime-manager.mjs

@@ -1,6 +1,7 @@
 import fs from "fs";
 import path from "path";
 import { buildRuntimeIds } from "./execution-config.mjs";
+import { formatError } from "./formatting.mjs";
 import {
   cleanupRuntimeInstanceContext,
   createRuntimeInstanceContext,
@@ -11,6 +12,7 @@ export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {}
   const graphByKey = new Map(graphs.map((graph) => [graph.key, graph]));
   const pools = new Map();
   const locks = new Map();
+  const failedGraphs = new Map();
   let nextLeaseCounter = 1;
   const runtimeHooks = {
     cleanupRuntimeInstanceContext,
@@ -22,6 +24,9 @@ export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {}
 
   return {
     canAcquire(task) {
+      if (failedGraphs.has(task.graphKey)) {
+        return false;
+      }
       const pool = getPool(pools, graphByKey, task, productDir, lifecycle);
       if (!locksAvailable(locks, task.locks || [])) {
         return false;
@@ -64,6 +69,9 @@ export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {}
         releaseLocks(locks, task.locks || [], leaseId);
         cleanupLeaseDir({ leaseDir: path.join(slot.context?.runtimeDir || productDir, "leases", leaseId) });
         await drainRuntimeSlot(slot, lifecycle, runtimeHooks);
+        if (!lifecycle.isStopRequested()) {
+          markGraphFailed(failedGraphs, task.graphKey, formatError(error));
+        }
         throw error;
       }
     },
@@ -104,6 +112,27 @@ export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {}
         }))
         .sort((left, right) => String(left.graphKey).localeCompare(String(right.graphKey)));
     },
+    drainFailedTasks(queue) {
+      const drained = [];
+      for (let index = queue.length - 1; index >= 0; index -= 1) {
+        const task = queue[index];
+        const message = failedGraphs.get(task.graphKey);
+        if (!message) continue;
+
+        queue.splice(index, 1);
+        drained.push({
+          task,
+          reason: `Not run because runtime graph failed to initialize: ${message}`,
+          message,
+          graph: graphByKey.get(task.graphKey) || {
+            key: task.graphKey,
+            targetNames: [task.targetName],
+          },
+        });
+      }
+
+      return drained.reverse();
+    },
   };
 }
 
@@ -218,6 +247,11 @@ function releaseLocks(lockMap, lockNames, leaseId) {
   }
 }
 
+function markGraphFailed(failedGraphs, graphKey, message) {
+  if (!graphKey || failedGraphs.has(graphKey)) return;
+  failedGraphs.set(graphKey, message);
+}
+
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }

package/lib/runner/runtime-manager.test.mjs

@@ -203,4 +203,50 @@ describe("runtime-manager", () => {
     await manager.release(leaseTwo);
     await manager.cleanupAll();
   });
+
+  it("blocks a graph after fatal runtime initialization failure and drains remaining tasks", async () => {
+    const productDir = makeTempDir("testkit-runtime-manager-");
+    const manager = createRuntimeManager({
+      productDir,
+      lifecycle: makeLifecycle(),
+      graphs: [
+        {
+          key: "api",
+          dirName: "api",
+          targetNames: ["api"],
+          instanceCount: 1,
+          maxConcurrentTasks: 1,
+        },
+      ],
+      hooks: {
+        ...makeHooks({ created: [], ready: [], cleaned: [] }),
+        async ensureRuntimeInstanceReady() {
+          throw new Error("prepare failed");
+        },
+      },
+    });
+
+    const firstTask = makeTask(1, { file: "tests/a.int.testkit.ts", targetName: "api" });
+    const secondTask = makeTask(2, { file: "tests/b.int.testkit.ts", targetName: "api" });
+
+    await expect(manager.acquire(firstTask)).rejects.toThrow(/prepare failed/);
+    expect(manager.canAcquire(secondTask)).toBe(false);
+
+    const queue = [secondTask];
+    expect(manager.drainFailedTasks(queue)).toEqual([
+      {
+        task: secondTask,
+        reason: "Not run because runtime graph failed to initialize: prepare failed",
+        message: "prepare failed",
+        graph: {
+          key: "api",
+          dirName: "api",
+          targetNames: ["api"],
+          instanceCount: 1,
+          maxConcurrentTasks: 1,
+        },
+      },
+    ]);
+    expect(queue).toEqual([]);
+  });
 });

package/lib/runner/runtime-preparation.mjs

@@ -0,0 +1,107 @@
+import crypto from "crypto";
+import fs from "fs";
+import path from "path";
+import { resolveServiceCwd } from "../config/index.mjs";
+import { appendFileToHash, appendInputToHash } from "../database/fingerprint.mjs";
+import { readDatabaseUrl } from "./state-io.mjs";
+import { buildExecutionEnv } from "./template.mjs";
+import {
+  collectConfiguredInputs,
+  runConfiguredSteps,
+} from "./template-steps.mjs";
+
+const MANIFEST_FILE = "prepare-manifest.json";
+
+export async function prepareRuntimeServices(runtimeConfigs) {
+  for (const config of runtimeConfigs) {
+    await prepareRuntimeService(config);
+  }
+}
+
+export async function prepareRuntimeService(config) {
+  const prepare = config.testkit.runtime.prepare;
+  if (!prepare || prepare.steps.length === 0) return;
+
+  const prepareDir = config.testkit.prepareDir;
+  const fingerprint = await computeRuntimePrepareFingerprint(config);
+  const manifestPath = path.join(prepareDir, MANIFEST_FILE);
+  const existingManifest = readPrepareManifest(manifestPath);
+  if (existingManifest?.fingerprint === fingerprint) {
+    return;
+  }
+
+  fs.rmSync(prepareDir, { recursive: true, force: true });
+  fs.mkdirSync(prepareDir, { recursive: true });
+
+  const env = {
+    ...buildExecutionEnv(config, config.testkit.local?.env || {}, process.env),
+  };
+  const databaseUrl = readDatabaseUrl(config.stateDir);
+  if (databaseUrl) {
+    env.DATABASE_URL = databaseUrl;
+  }
+
+  try {
+    await runConfiguredSteps({
+      config,
+      steps: prepare.steps,
+      env,
+      labelPrefix: "runtime:prepare",
+    });
+    writePrepareManifest(manifestPath, {
+      fingerprint,
+      preparedAt: new Date().toISOString(),
+      runtimeId: config.runtimeId || null,
+      serviceName: config.name,
+    });
+  } catch (error) {
+    fs.rmSync(prepareDir, { recursive: true, force: true });
+    throw error;
+  }
+}
+
+export async function computeRuntimePrepareFingerprint(config) {
+  const hash = crypto.createHash("sha256");
+  hash.update(
+    JSON.stringify({
+      prepare: config.testkit.runtime.prepare || null,
+      serviceEnv: config.testkit.serviceEnv || {},
+      local: config.testkit.local
+        ? {
+            baseUrl: config.testkit.local.baseUrl,
+            cwd: config.testkit.local.cwd,
+            env: config.testkit.local.env || {},
+            readyUrl: config.testkit.local.readyUrl,
+            start: config.testkit.local.start,
+          }
+        : null,
+    })
+  );
+
+  for (const envFile of config.testkit.envFiles || []) {
+    appendFileToHash(hash, config.productDir, resolveServiceCwd(config.productDir, envFile));
+  }
+  for (const input of collectConfiguredInputs(config.productDir, config.testkit.runtime.prepare)) {
+    appendResolvedInputToHash(hash, config.productDir, input);
+  }
+
+  return hash.digest("hex");
+}
+
+function appendResolvedInputToHash(hash, productDir, absPath) {
+  const relative = path.relative(productDir, absPath);
+  appendInputToHash(hash, productDir, relative);
+}
+
+function readPrepareManifest(manifestPath) {
+  if (!fs.existsSync(manifestPath)) return null;
+  try {
+    return JSON.parse(fs.readFileSync(manifestPath, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+function writePrepareManifest(manifestPath, manifest) {
+  fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
+}

package/lib/runner/runtime-preparation.test.mjs

@@ -0,0 +1,141 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { afterEach, describe, expect, it } from "vitest";
+import { prepareRuntimeService } from "./runtime-preparation.mjs";
+import { resolveRuntimeInstanceConfigs } from "./template.mjs";
+
+const tempDirs = [];
+
+afterEach(() => {
+  while (tempDirs.length > 0) {
+    fs.rmSync(tempDirs.pop(), { recursive: true, force: true });
+  }
+});
+
+function makeTempDir(prefix) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+  tempDirs.push(dir);
+  return dir;
+}
+
+function writeFile(filePath, contents) {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, contents);
+}
+
+function makeResolvedRuntimeConfig(productDir, prepareConfig) {
+  const rawConfig = {
+    name: "api",
+    productDir,
+    testkit: {
+      envFiles: [],
+      serviceEnv: {},
+      runtime: {
+        instances: 1,
+        maxConcurrentTasks: 1,
+        prepare: prepareConfig,
+      },
+      local: {
+        cwd: ".",
+        start: "node {prepareDir}/server.mjs",
+        port: 3010,
+        baseUrl: "http://127.0.0.1:{port}",
+        readyUrl: "http://127.0.0.1:{port}/health",
+        env: {},
+      },
+    },
+  };
+
+  const runtimeDir = path.join(productDir, ".testkit", "_graphs", "api", "runtimes", "runtime-1");
+  fs.mkdirSync(runtimeDir, { recursive: true });
+  return resolveRuntimeInstanceConfigs([rawConfig], "runtime-1", runtimeDir, {
+    graphDirName: "api",
+    portNamespaceIndex: 0,
+    portNamespaceStride: 1,
+  })[0];
+}
+
+describe("runtime preparation", () => {
+  it("runs prepare steps once and reuses the manifest until inputs change", async () => {
+    const productDir = makeTempDir("testkit-runtime-prepare-");
+    writeFile(path.join(productDir, "src", "message.txt"), "hello one\n");
+    writeFile(
+      path.join(productDir, "scripts", "prepare.mjs"),
+      [
+        'import fs from "fs";',
+        'import path from "path";',
+        'const prepareDir = process.argv[2];',
+        'const productDir = process.cwd();',
+        'const message = fs.readFileSync(path.join(productDir, "src", "message.txt"), "utf8").trim();',
+        'const counterDir = path.join(productDir, ".runtime", "prepared");',
+        'const counterFile = path.join(counterDir, "build-count.txt");',
+        'const current = fs.existsSync(counterFile) ? Number.parseInt(fs.readFileSync(counterFile, "utf8"), 10) || 0 : 0;',
+        'fs.mkdirSync(counterDir, { recursive: true });',
+        'fs.mkdirSync(prepareDir, { recursive: true });',
+        'fs.writeFileSync(counterFile, `${current + 1}\\n`);',
+        'fs.writeFileSync(path.join(prepareDir, "message.txt"), `${message}\\n`);',
+      ].join("\n")
+    );
+
+    const config = makeResolvedRuntimeConfig(productDir, {
+      inputs: ["src/message.txt"],
+      steps: [
+        {
+          kind: "command",
+          cmd: "node scripts/prepare.mjs {prepareDir}",
+        },
+      ],
+    });
+
+    await prepareRuntimeService(config);
+    expect(
+      fs.readFileSync(path.join(productDir, ".runtime", "prepared", "build-count.txt"), "utf8").trim()
+    ).toBe("1");
+    expect(fs.readFileSync(path.join(config.testkit.prepareDir, "message.txt"), "utf8").trim()).toBe(
+      "hello one"
+    );
+
+    await prepareRuntimeService(config);
+    expect(
+      fs.readFileSync(path.join(productDir, ".runtime", "prepared", "build-count.txt"), "utf8").trim()
+    ).toBe("1");
+
+    fs.writeFileSync(path.join(productDir, "src", "message.txt"), "hello two\n");
+    await prepareRuntimeService(config);
+    expect(
+      fs.readFileSync(path.join(productDir, ".runtime", "prepared", "build-count.txt"), "utf8").trim()
+    ).toBe("2");
+    expect(fs.readFileSync(path.join(config.testkit.prepareDir, "message.txt"), "utf8").trim()).toBe(
+      "hello two"
+    );
+  });
+
+  it("cleans incomplete prepared output when a prepare step fails", async () => {
+    const productDir = makeTempDir("testkit-runtime-prepare-fail-");
+    writeFile(
+      path.join(productDir, "scripts", "prepare-fail.mjs"),
+      [
+        'import fs from "fs";',
+        'import path from "path";',
+        'const prepareDir = process.argv[2];',
+        'fs.mkdirSync(prepareDir, { recursive: true });',
+        'fs.writeFileSync(path.join(prepareDir, "partial.txt"), "partial\\n");',
+        'throw new Error("prepare failed");',
+      ].join("\n")
+    );
+
+    const config = makeResolvedRuntimeConfig(productDir, {
+      steps: [
+        {
+          kind: "command",
+          cmd: "node scripts/prepare-fail.mjs {prepareDir}",
+        },
+      ],
+    });
+
+    await expect(prepareRuntimeService(config)).rejects.toThrow("Command failed with exit code 1");
+    expect(fs.existsSync(config.testkit.prepareDir)).toBe(false);
+    expect(fs.existsSync(path.join(productDir, ".testkit", "_graphs", "api", "runtimes", "runtime-1", "services", "api", "prepared", "prepare-manifest.json"))).toBe(false);
+  });
+});

package/lib/runner/template-steps.mjs

@@ -0,0 +1,191 @@
+import crypto from "crypto";
+import fs from "fs";
+import path from "path";
+import { build } from "esbuild";
+import { execa, execaCommand } from "execa";
+import { fileURLToPath, pathToFileURL } from "url";
+import { resolveServiceCwd } from "../config/index.mjs";
+
+const PACKAGE_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..");
+const ROOT_ENTRY = path.join(PACKAGE_ROOT, "lib", "index.mjs");
+const SETUP_ENTRY = path.join(PACKAGE_ROOT, "lib", "setup", "index.mjs");
+const RUNTIME_ENTRY = path.join(PACKAGE_ROOT, "lib", "runtime", "index.mjs");
+const KNOWN_FAILURES_ENTRY = path.join(PACKAGE_ROOT, "lib", "known-failures", "index.mjs");
+
+export async function runConfiguredSteps({ config, steps = [], env, labelPrefix }) {
+  if (steps.length === 0) return;
+
+  for (const [index, step] of steps.entries()) {
+    const label = `${labelPrefix}:${config.name}:${index + 1}`;
+    console.log(`\n── ${label} ──`);
+    await runConfiguredStep(config, step, env);
+  }
+}
+
+export function collectConfiguredInputs(productDir, { inputs = [], steps = [] } = {}) {
+  const collected = new Set();
+  for (const input of inputs) {
+    collected.add(resolveConfiguredPath(productDir, null, input));
+  }
+  for (const step of steps) {
+    if (step.kind === "sql-file") {
+      collected.add(resolveConfiguredPath(productDir, step.cwd, step.path));
+    }
+    if (step.kind === "module") {
+      collected.add(
+        resolveConfiguredPath(productDir, step.cwd, parseModuleSpecifier(step.specifier).modulePath)
+      );
+    }
+    for (const input of step.inputs || []) {
+      collected.add(resolveConfiguredPath(productDir, step.cwd, input));
+    }
+  }
+  return [...collected].sort();
+}
+
+export function resolveConfiguredCwd(productDir, stepCwd) {
+  return resolveServiceCwd(productDir, stepCwd || ".");
+}
+
+export function resolveConfiguredPath(productDir, stepCwd, targetPath) {
+  return path.resolve(resolveConfiguredCwd(productDir, stepCwd), targetPath);
+}
+
+async function runConfiguredStep(config, step, env) {
+  if (step.kind === "command") {
+    await execaCommand(step.cmd, {
+      cwd: resolveConfiguredCwd(config.productDir, step.cwd),
+      env,
+      stdio: "inherit",
+      shell: true,
+    });
+    return;
+  }
+
+  if (step.kind === "sql-file") {
+    await execa(
+      "psql",
+      [
+        env.DATABASE_URL,
+        "-v",
+        "ON_ERROR_STOP=1",
+        "-X",
+        "-f",
+        resolveConfiguredPath(config.productDir, step.cwd, step.path),
+      ],
+      {
+        cwd: resolveConfiguredCwd(config.productDir, step.cwd),
+        env,
+        stdio: "inherit",
+      }
+    );
+    return;
+  }
+
+  if (step.kind === "module") {
+    const moduleRef = await loadConfiguredModule(config.productDir, step);
+    const { exportName } = parseModuleSpecifier(step.specifier);
+    const fn = moduleRef[exportName];
+    if (typeof fn !== "function") {
+      throw new Error(
+        `Template module step "${step.specifier}" did not export a function named "${exportName}"`
+      );
+    }
+
+    await withProcessContext(resolveConfiguredCwd(config.productDir, step.cwd), env, async () => {
+      await fn({
+        databaseUrl: env.DATABASE_URL || null,
+        productDir: config.productDir,
+        cwd: resolveConfiguredCwd(config.productDir, step.cwd),
+        serviceName: config.name,
+        env: { ...env },
+        runtimeId: config.runtimeId || null,
+        stateDir: config.stateDir,
+        prepareDir: config.testkit.prepareDir || null,
+      });
+    });
+    return;
+  }
+
+  throw new Error(`Unsupported template step kind "${step.kind}"`);
+}
+
+async function loadConfiguredModule(productDir, step) {
+  const { modulePath } = parseModuleSpecifier(step.specifier);
+  const absoluteModulePath = resolveConfiguredPath(productDir, step.cwd, modulePath);
+  const bundleDir = path.join(productDir, ".testkit", "_template-steps");
+  fs.mkdirSync(bundleDir, { recursive: true });
+
+  const cacheKey = buildModuleCacheKey(absoluteModulePath);
+  const outputFile = path.join(
+    bundleDir,
+    `${path.basename(modulePath).replace(/\W+/g, "-")}-${cacheKey.slice(0, 12)}.mjs`
+  );
+
+  await build({
+    absWorkingDir: productDir,
+    bundle: true,
+    entryPoints: [absoluteModulePath],
+    format: "esm",
+    legalComments: "none",
+    outfile: outputFile,
+    platform: "node",
+    sourcemap: "inline",
+    target: "es2020",
+    plugins: [testkitAliasPlugin()],
+  });
+
+  return import(`${pathToFileURL(outputFile).href}?v=${cacheKey}`);
+}
+
+function buildModuleCacheKey(modulePath) {
+  const content = fs.readFileSync(modulePath, "utf8");
+  return crypto.createHash("sha256").update(modulePath).update("\0").update(content).digest("hex");
+}
+
+function testkitAliasPlugin() {
+  return {
+    name: "testkit-template-step-alias",
+    setup(buildApi) {
+      buildApi.onResolve({ filter: /^@elench\/testkit(?:\/.*)?$/ }, (args) => ({
+        namespace: "file",
+        path: resolvePackageSubpath(args.path),
+      }));
+    },
+  };
+}
+
+function resolvePackageSubpath(specifier) {
+  const subpath = specifier.slice("@elench/testkit".length);
+  if (!subpath) return ROOT_ENTRY;
+  if (subpath === "/setup") return SETUP_ENTRY;
+  if (subpath === "/runtime") return RUNTIME_ENTRY;
+  if (subpath === "/known-failures") return KNOWN_FAILURES_ENTRY;
+
+  throw new Error(`Unsupported @elench/testkit import "${specifier}" while loading template step`);
+}
+
+function parseModuleSpecifier(specifier) {
+  const [modulePath, exportName] = String(specifier).split("#", 2);
+  return {
+    modulePath,
+    exportName: exportName || "default",
+  };
+}
+
+async function withProcessContext(cwd, env, fn) {
+  const previousCwd = process.cwd();
+  const previousEnv = process.env;
+  process.chdir(cwd);
+  process.env = {
+    ...previousEnv,
+    ...env,
+  };
+
+  try {
+    return await fn();
+  } finally {
+    process.chdir(previousCwd);
+    process.env = previousEnv;
+  }
+}