@elench/testkit 0.1.40 → 0.1.42

package/lib/runner/results.test.mjs

@@ -54,7 +54,7 @@ describe("runner results", () => {
     const tracker = trackers.get("api");
     addTrackerError(tracker, "worker failed");
     addTrackerError(tracker, "worker failed");
-    recordGraphError(trackers, { assignedTargets: ["api"] }, "graph failed", 1300);
+    recordGraphError(trackers, { targetNames: ["api"] }, "graph failed", 1300);
 
     const result = finalizeServiceResult(tracker, 1000, 1500);
     expect(result.failed).toBe(true);

package/lib/runner/runtime-contexts.mjs

@@ -3,29 +3,29 @@ import path from "path";
 import { execaCommand } from "execa";
 import { resolveServiceCwd } from "../config/index.mjs";
 import { prepareDatabaseRuntime } from "../database/index.mjs";
-import { buildExecutionEnv, resolveStackRuntimeConfigs } from "./template.mjs";
+import { taskNeedsLocalRuntime } from "./planning.mjs";
 import { writeGraphMetadata } from "./state.mjs";
 import { startLocalServices, stopLocalServices } from "./services.mjs";
+import { buildExecutionEnv, resolveRuntimeInstanceConfigs } from "./template.mjs";
 
-export function createStackContext(stackId, graph, productDir) {
+export function createRuntimeInstanceContext(runtimeId, graph, productDir) {
   const graphDir = path.join(productDir, ".testkit", "_graphs", graph.dirName);
-  const stackStateDir = path.join(graphDir, "stacks", stackId);
-  fs.mkdirSync(stackStateDir, { recursive: true });
+  const runtimeDir = path.join(graphDir, "runtimes", runtimeId);
+  fs.mkdirSync(runtimeDir, { recursive: true });
   writeGraphMetadata(graphDir, graph);
 
-  const runtimeConfigs = resolveStackRuntimeConfigs(
-    graph.rootConfig,
-    graph.runtimeConfigs,
-    stackId,
-    stackStateDir
-  );
+  const runtimeConfigs = resolveRuntimeInstanceConfigs(graph.runtimeConfigs, runtimeId, runtimeDir, {
+    graphDirName: graph.dirName,
+    portNamespaceIndex: graph.portNamespaceIndex,
+    portNamespaceStride: graph.portNamespaceStride,
+  });
 
   return {
     graphKey: graph.key,
-    assignedTargets: [...graph.assignedTargets],
+    targetNames: [...graph.targetNames],
     graphDir,
-    stackId,
-    stackStateDir,
+    runtimeDir,
+    runtimeId,
     runtimeConfigs,
     configByName: new Map(runtimeConfigs.map((config) => [config.name, config])),
     prepared: false,
@@ -36,7 +36,7 @@ export function createStackContext(stackId, graph, productDir) {
   };
 }
 
-export async function ensureStackContextReady(context, batch, lifecycle) {
+export async function ensureRuntimeInstanceReady(context, task, lifecycle) {
   if (!context.prepared) {
     if (!context.preparationPromise) {
       context.preparationPromise = (async () => {
@@ -49,7 +49,7 @@ export async function ensureStackContextReady(context, batch, lifecycle) {
     await context.preparationPromise;
   }
 
-  if (batchNeedsLocalRuntime(batch) && !context.started) {
+  if (taskNeedsLocalRuntime(task) && !context.started) {
     if (!context.startupPromise) {
       context.startupPromise = (async () => {
         context.startedServices = await startLocalServices(context.runtimeConfigs, lifecycle);
@@ -64,16 +64,16 @@ export async function ensureStackContextReady(context, batch, lifecycle) {
   return context;
 }
 
-export async function deactivateStackContext(context, lifecycle) {
+export async function deactivateRuntimeInstanceContext(context, lifecycle) {
   if (!context?.started) return;
   await stopLocalServices(context.startedServices, lifecycle);
   context.started = false;
   context.startedServices = [];
 }
 
-export async function cleanupStackContext(context, lifecycle) {
+export async function cleanupRuntimeInstanceContext(context, lifecycle) {
   if (!context) return;
-  await deactivateStackContext(context, lifecycle);
+  await deactivateRuntimeInstanceContext(context, lifecycle);
 }
 
 export async function prepareDatabases(runtimeConfigs) {
@@ -94,7 +94,7 @@ async function runMigrate(config, databaseUrl) {
   const env = buildExecutionEnv(config, {}, process.env);
   if (databaseUrl) env.DATABASE_URL = databaseUrl;
 
-  console.log(`\n── migrate:${config.stackLabel}:${config.name} ──`);
+  console.log(`\n── migrate:${config.runtimeLabel}:${config.name} ──`);
   await execaCommand(migrate.cmd, {
     cwd: resolveServiceCwd(config.productDir, migrate.cwd),
     env,
@@ -110,7 +110,7 @@ async function runSeed(config, databaseUrl) {
   const env = buildExecutionEnv(config, {}, process.env);
   if (databaseUrl) env.DATABASE_URL = databaseUrl;
 
-  console.log(`\n── seed:${config.stackLabel}:${config.name} ──`);
+  console.log(`\n── seed:${config.runtimeLabel}:${config.name} ──`);
   await execaCommand(seed.cmd, {
     cwd: resolveServiceCwd(config.productDir, seed.cwd),
     env,
@@ -118,7 +118,3 @@ async function runSeed(config, databaseUrl) {
     shell: true,
   });
 }
-
-function batchNeedsLocalRuntime(batch) {
-  return batch.type !== "dal";
-}

package/lib/runner/runtime-manager.mjs

@@ -0,0 +1,228 @@
+import fs from "fs";
+import path from "path";
+import { buildRuntimeIds } from "./execution-config.mjs";
+import {
+  cleanupRuntimeInstanceContext,
+  createRuntimeInstanceContext,
+  ensureRuntimeInstanceReady,
+} from "./runtime-contexts.mjs";
+
+export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {} }) {
+  const graphByKey = new Map(graphs.map((graph) => [graph.key, graph]));
+  const pools = new Map();
+  const locks = new Map();
+  let nextLeaseCounter = 1;
+  const runtimeHooks = {
+    cleanupRuntimeInstanceContext,
+    createRuntimeInstanceContext,
+    ensureRuntimeInstanceReady,
+    sleep,
+    ...hooks,
+  };
+
+  return {
+    canAcquire(task) {
+      const pool = getPool(pools, graphByKey, task, productDir, lifecycle);
+      if (!locksAvailable(locks, task.locks || [])) {
+        return false;
+      }
+      return claimableRuntimeSlot(pool, task) !== null;
+    },
+    async acquire(task) {
+      const pool = getPool(pools, graphByKey, task, productDir, lifecycle);
+      if (lifecycle.isStopRequested()) {
+        throw lifecycle.signal.reason || new Error("testkit run interrupted");
+      }
+
+      const leaseId = `lease-${task.id}-${nextLeaseCounter}`;
+      nextLeaseCounter += 1;
+
+      if (!tryAcquireLocks(locks, task.locks || [], leaseId)) {
+        throw new Error(`Task ${task.id} was claimed before its locks were available`);
+      }
+
+      const slot = claimRuntimeSlot(pool, task);
+      if (!slot) {
+        releaseLocks(locks, task.locks || [], leaseId);
+        throw new Error(`Task ${task.id} was claimed before runtime capacity was available`);
+      }
+
+      try {
+        const context = await getReadyContext(slot, productDir, task, lifecycle, runtimeHooks);
+        const leaseDir = path.join(context.runtimeDir, "leases", leaseId);
+        fs.mkdirSync(leaseDir, { recursive: true });
+        return {
+          leaseId,
+          leaseDir,
+          lockNames: task.locks || [],
+          resourceCost: task.resourceCost || 1,
+          slot,
+          context,
+        };
+      } catch (error) {
+        releaseRuntimeSlot(slot, task);
+        releaseLocks(locks, task.locks || [], leaseId);
+        cleanupLeaseDir({ leaseDir: path.join(slot.context?.runtimeDir || productDir, "leases", leaseId) });
+        await drainRuntimeSlot(slot, lifecycle, runtimeHooks);
+        throw error;
+      }
+    },
+    async release(lease, options = {}) {
+      if (!lease?.slot) return;
+      releaseLocks(locks, lease.lockNames || [], lease.leaseId);
+      releaseRuntimeSlot(lease.slot, { resourceCost: lease.resourceCost || 1 });
+      cleanupLeaseDir(lease);
+      if (options.invalidate) {
+        lease.slot.draining = true;
+      }
+      if (lease.slot.draining && lease.slot.activeLeaseCount === 0) {
+        await drainRuntimeSlot(lease.slot, lifecycle, runtimeHooks);
+      }
+    },
+    async cleanupAll() {
+      for (const pool of pools.values()) {
+        for (const slot of pool.slots) {
+          slot.draining = true;
+          await drainRuntimeSlot(slot, lifecycle, runtimeHooks);
+        }
+      }
+    },
+    getStats() {
+      return [...pools.values()]
+        .map((pool) => ({
+          graphKey: pool.slots[0]?.graph.key || null,
+          targetNames: [...(pool.slots[0]?.graph.targetNames || [])],
+          maxConcurrentTasks: Number.isFinite(pool.slots[0]?.graph.maxConcurrentTasks)
+            ? pool.slots[0]?.graph.maxConcurrentTasks
+            : null,
+          runtimeCount: pool.slots.length,
+          runtimes: pool.slots.map((slot) => ({
+            runtimeId: slot.runtimeId,
+            peakLeaseCount: slot.peakLeaseCount,
+            peakResourceUnits: slot.peakResourceUnits,
+          })),
+        }))
+        .sort((left, right) => String(left.graphKey).localeCompare(String(right.graphKey)));
+    },
+  };
+}
+
+function getPool(pools, graphByKey, task, productDir, lifecycle) {
+  const existing = pools.get(task.graphKey);
+  if (existing) return existing;
+
+  const graph = graphByKey.get(task.graphKey);
+  if (!graph) {
+    throw new Error(`Unknown graph "${task.graphKey}"`);
+  }
+
+  const pool = {
+    productDir,
+    lifecycle,
+    slots: buildRuntimeIds(graph.instanceCount).map((runtimeId) => ({
+      graph,
+      runtimeId,
+      context: null,
+      contextPromise: null,
+      activeLeaseCount: 0,
+      activeResourceUnits: 0,
+      peakLeaseCount: 0,
+      peakResourceUnits: 0,
+      draining: false,
+    })),
+  };
+  pools.set(task.graphKey, pool);
+  return pool;
+}
+
+function claimRuntimeSlot(pool, task) {
+  const resourceCost = task.resourceCost || 1;
+  const available = pool.slots.filter((slot) => !slot.draining);
+  if (available.length === 0) return null;
+
+  const slot = [...available]
+    .filter((candidate) => slotHasCapacity(candidate, resourceCost))
+    .sort(
+      (left, right) =>
+        left.activeResourceUnits - right.activeResourceUnits ||
+        left.activeLeaseCount - right.activeLeaseCount ||
+        left.runtimeId.localeCompare(right.runtimeId)
+    )[0];
+  if (!slot) return null;
+  slot.activeLeaseCount += 1;
+  slot.activeResourceUnits += resourceCost;
+  slot.peakLeaseCount = Math.max(slot.peakLeaseCount, slot.activeLeaseCount);
+  slot.peakResourceUnits = Math.max(slot.peakResourceUnits, slot.activeResourceUnits);
+  return slot;
+}
+
+function releaseRuntimeSlot(slot, task = {}) {
+  slot.activeLeaseCount = Math.max(0, slot.activeLeaseCount - 1);
+  slot.activeResourceUnits = Math.max(0, slot.activeResourceUnits - (task.resourceCost || 1));
+}
+
+async function getReadyContext(slot, productDir, task, lifecycle, runtimeHooks) {
+  if (!slot.context) {
+    slot.context = runtimeHooks.createRuntimeInstanceContext(slot.runtimeId, slot.graph, productDir);
+    lifecycle.trackGraphContext(slot.context);
+  }
+  if (!slot.contextPromise) {
+    slot.contextPromise = Promise.resolve(slot.context);
+  }
+  await slot.contextPromise;
+  await runtimeHooks.ensureRuntimeInstanceReady(slot.context, task, lifecycle);
+  return slot.context;
+}
+
+async function drainRuntimeSlot(slot, lifecycle, runtimeHooks) {
+  if (!slot.context || slot.activeLeaseCount > 0) return;
+  await runtimeHooks.cleanupRuntimeInstanceContext(slot.context, lifecycle);
+  slot.context = null;
+  slot.contextPromise = null;
+  slot.draining = false;
+}
+
+function tryAcquireLocks(lockMap, lockNames, leaseId) {
+  const uniqueLocks = [...new Set(lockNames)].sort();
+  for (const lockName of uniqueLocks) {
+    if (lockMap.has(lockName)) {
+      return false;
+    }
+  }
+  for (const lockName of uniqueLocks) {
+    lockMap.set(lockName, leaseId);
+  }
+  return true;
+}
+
+function locksAvailable(lockMap, lockNames) {
+  return [...new Set(lockNames)].every((lockName) => !lockMap.has(lockName));
+}
+
+function claimableRuntimeSlot(pool, task) {
+  const resourceCost = task.resourceCost || 1;
+  return (
+    pool.slots.find((slot) => !slot.draining && slotHasCapacity(slot, resourceCost)) || null
+  );
+}
+
+function slotHasCapacity(slot, resourceCost) {
+  return slot.activeResourceUnits + resourceCost <= slot.graph.maxConcurrentTasks;
+}
+
+function releaseLocks(lockMap, lockNames, leaseId) {
+  for (const lockName of [...new Set(lockNames)].sort()) {
+    if (lockMap.get(lockName) === leaseId) {
+      lockMap.delete(lockName);
+    }
+  }
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function cleanupLeaseDir(lease) {
+  if (!lease?.leaseDir) return;
+  fs.rmSync(lease.leaseDir, { recursive: true, force: true });
+}

package/lib/runner/runtime-manager.test.mjs

@@ -0,0 +1,206 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { afterEach, describe, expect, it } from "vitest";
+import { createRuntimeManager } from "./runtime-manager.mjs";
+
+const cleanups = [];
+
+afterEach(() => {
+  while (cleanups.length > 0) {
+    cleanups.pop()();
+  }
+});
+
+function makeTempDir(prefix) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+  cleanups.push(() => fs.rmSync(dir, { recursive: true, force: true }));
+  return dir;
+}
+
+function makeLifecycle() {
+  return {
+    signal: new AbortController().signal,
+    isStopRequested() {
+      return false;
+    },
+    trackGraphContext() {},
+  };
+}
+
+function makeTask(id, extras = {}) {
+  return {
+    id,
+    graphKey: "api",
+    locks: [],
+    ...extras,
+  };
+}
+
+function makeHooks(events) {
+  return {
+    createRuntimeInstanceContext(runtimeId, graph, productDir) {
+      const runtimeDir = path.join(productDir, ".testkit", "_graphs", graph.dirName, "runtimes", runtimeId);
+      fs.mkdirSync(runtimeDir, { recursive: true });
+      events.created.push(runtimeId);
+      return {
+        graphKey: graph.key,
+        runtimeDir,
+        runtimeId,
+        targetNames: [...graph.targetNames],
+      };
+    },
+    async ensureRuntimeInstanceReady(context) {
+      events.ready.push(context.runtimeId);
+    },
+    async cleanupRuntimeInstanceContext(context) {
+      events.cleaned.push(context.runtimeId);
+      fs.rmSync(context.runtimeDir, { recursive: true, force: true });
+    },
+    async sleep() {
+      await new Promise((resolve) => setTimeout(resolve, 0));
+    },
+  };
+}
+
+describe("runtime-manager", () => {
+  it("spreads concurrent leases across runtime instances", async () => {
+    const productDir = makeTempDir("testkit-runtime-manager-");
+    const events = { created: [], ready: [], cleaned: [] };
+    const manager = createRuntimeManager({
+      productDir,
+      lifecycle: makeLifecycle(),
+      graphs: [
+        {
+          key: "api",
+          dirName: "api",
+          targetNames: ["api"],
+          instanceCount: 2,
+          maxConcurrentTasks: 1,
+        },
+      ],
+      hooks: makeHooks(events),
+    });
+
+    const leaseOne = await manager.acquire(makeTask(1));
+    const leaseTwo = await manager.acquire(makeTask(2));
+
+    expect(leaseOne.slot.runtimeId).toBe("runtime-1");
+    expect(leaseTwo.slot.runtimeId).toBe("runtime-2");
+    expect(events.created).toEqual(["runtime-1", "runtime-2"]);
+
+    await manager.release(leaseOne);
+    await manager.release(leaseTwo);
+    await manager.cleanupAll();
+  });
+
+  it("marks conflicting locks unavailable until the first lease releases", async () => {
+    const productDir = makeTempDir("testkit-runtime-manager-");
+    const manager = createRuntimeManager({
+      productDir,
+      lifecycle: makeLifecycle(),
+      graphs: [
+        {
+          key: "api",
+          dirName: "api",
+          targetNames: ["api"],
+          instanceCount: 1,
+          maxConcurrentTasks: 1,
+        },
+      ],
+      hooks: makeHooks({ created: [], ready: [], cleaned: [] }),
+    });
+
+    const firstLease = await manager.acquire(makeTask(1, { locks: ["shared-lock"] }));
+    expect(manager.canAcquire(makeTask(2, { locks: ["shared-lock"] }))).toBe(false);
+
+    await manager.release(firstLease);
+    const secondLease = await manager.acquire(makeTask(2, { locks: ["shared-lock"] }));
+
+    await manager.release(secondLease);
+    await manager.cleanupAll();
+  });
+
+  it("does not drain an invalidated runtime slot until all active leases release", async () => {
+    const productDir = makeTempDir("testkit-runtime-manager-");
+    const events = { created: [], ready: [], cleaned: [] };
+    const manager = createRuntimeManager({
+      productDir,
+      lifecycle: makeLifecycle(),
+      graphs: [
+        {
+          key: "api",
+          dirName: "api",
+          targetNames: ["api"],
+          instanceCount: 1,
+          maxConcurrentTasks: 2,
+        },
+      ],
+      hooks: makeHooks(events),
+    });
+
+    const leaseOne = await manager.acquire(makeTask(1));
+    const leaseTwo = await manager.acquire(makeTask(2));
+
+    await manager.release(leaseOne, { invalidate: true });
+    expect(events.cleaned).toEqual([]);
+
+    await manager.release(leaseTwo);
+    expect(events.cleaned).toEqual(["runtime-1"]);
+  });
+
+  it("removes lease directories on release", async () => {
+    const productDir = makeTempDir("testkit-runtime-manager-");
+    const manager = createRuntimeManager({
+      productDir,
+      lifecycle: makeLifecycle(),
+      graphs: [
+        {
+          key: "api",
+          dirName: "api",
+          targetNames: ["api"],
+          instanceCount: 1,
+          maxConcurrentTasks: 1,
+        },
+      ],
+      hooks: makeHooks({ created: [], ready: [], cleaned: [] }),
+    });
+
+    const lease = await manager.acquire(makeTask(1));
+    expect(fs.existsSync(lease.leaseDir)).toBe(true);
+
+    await manager.release(lease);
+    expect(fs.existsSync(lease.leaseDir)).toBe(false);
+
+    await manager.cleanupAll();
+  });
+
+  it("exposes runtime capacity through canAcquire", async () => {
+    const productDir = makeTempDir("testkit-runtime-manager-");
+    const manager = createRuntimeManager({
+      productDir,
+      lifecycle: makeLifecycle(),
+      graphs: [
+        {
+          key: "api",
+          dirName: "api",
+          targetNames: ["api"],
+          instanceCount: 1,
+          maxConcurrentTasks: 2,
+        },
+      ],
+      hooks: makeHooks({ created: [], ready: [], cleaned: [] }),
+    });
+
+    expect(manager.canAcquire(makeTask(1))).toBe(true);
+    const leaseOne = await manager.acquire(makeTask(1));
+    expect(manager.canAcquire(makeTask(2))).toBe(true);
+    const leaseTwo = await manager.acquire(makeTask(2));
+    expect(manager.canAcquire(makeTask(3))).toBe(false);
+
+    await manager.release(leaseOne);
+    expect(manager.canAcquire(makeTask(3))).toBe(true);
+    await manager.release(leaseTwo);
+    await manager.cleanupAll();
+  });
+});

package/lib/runner/services.mjs

@@ -1,7 +1,7 @@
 import { resolveServiceCwd } from "../config/index.mjs";
 import { buildExecutionEnv, numericPortFromUrl } from "./template.mjs";
 import { DEFAULT_READY_TIMEOUT_MS, assertLocalServicePortsAvailable, isPortInUse, waitForReady } from "./readiness.mjs";
-import { pipeOutput, startDetachedCommand, stopChildProcess, sleep } from "./processes.mjs";
+import { killChildProcess, pipeOutput, startDetachedCommand, stopChildProcess, sleep } from "./processes.mjs";
 import { readDatabaseUrl } from "./state-io.mjs";
 
 export async function startLocalServices(runtimeConfigs, lifecycle) {
@@ -36,20 +36,22 @@ export async function startLocalService(config, lifecycle) {
 
   await assertLocalServicePortsAvailable(config, isPortInUse);
 
-  console.log(`Starting ${config.stackLabel}:${config.name}: ${config.testkit.local.start}`);
+  console.log(`Starting ${config.runtimeLabel}:${config.name}: ${config.testkit.local.start}`);
   const child = startDetachedCommand(config.testkit.local.start, cwd, env);
 
   const outputDrains = [
-    pipeOutput(child.stdout, `[${config.stackLabel}:${config.name}]`),
-    pipeOutput(child.stderr, `[${config.stackLabel}:${config.name}]`),
+    pipeOutput(child.stdout, `[${config.runtimeLabel}:${config.name}]`),
+    pipeOutput(child.stderr, `[${config.runtimeLabel}:${config.name}]`),
   ];
-  lifecycle.registerService(config, child, cwd);
+  lifecycle.registerService(config, child, cwd, () => {
+    killChildProcess(child, "SIGTERM");
+  });
 
   const readyTimeoutMs = config.testkit.local.readyTimeoutMs || DEFAULT_READY_TIMEOUT_MS;
 
   try {
     await waitForReady({
-      name: `${config.stackLabel}:${config.name}`,
+      name: `${config.runtimeLabel}:${config.name}`,
       url: config.testkit.local.readyUrl,
       timeoutMs: readyTimeoutMs,
       process: child,

package/lib/runner/state.mjs

@@ -46,8 +46,7 @@ export function writeGraphMetadata(graphDir, graph) {
   fs.mkdirSync(graphDir, { recursive: true });
   const metadata = {
     runtimeServices: graph.runtimeNames,
-    assignedTargets: [...graph.assignedTargets].sort(),
-    rootService: graph.rootConfig.name,
+    targetServices: [...(graph.targetNames || [])].sort(),
   };
   fs.writeFileSync(
     path.join(graphDir, GRAPH_METADATA),

package/lib/runner/state.test.mjs

@@ -48,14 +48,12 @@ describe("runner-state", () => {
     const graphDir = path.join(productDir, ".testkit", "_graphs", "api__frontend");
     writeGraphMetadata(graphDir, {
       runtimeNames: ["api", "frontend"],
-      assignedTargets: ["frontend", "api"],
-      rootConfig: { name: "api" },
+      targetNames: ["frontend", "api"],
     });
 
     expect(readGraphMetadata(graphDir)).toEqual({
       runtimeServices: ["api", "frontend"],
-      assignedTargets: ["api", "frontend"],
-      rootService: "api",
+      targetServices: ["api", "frontend"],
     });
 
     expect(findGraphDirsForService(productDir, "frontend")).toEqual([graphDir]);