@elench/testkit 0.1.39 → 0.1.41

package/lib/runner/playwright-runner.mjs

@@ -1,16 +1,15 @@
 import path from "path";
 import { execa } from "execa";
-import {
-  parsePlaywrightJsonResults,
-} from "../reporters/playwright.mjs";
-import { resolveServiceCwd, } from "../config/index.mjs";
-import { formatPlaywrightBatchFiles } from "./formatting.mjs";
-import { printBufferedOutput } from "./processes.mjs";
+import { parsePlaywrightJsonResults } from "../reporters/playwright.mjs";
+import { resolveServiceCwd } from "../config/index.mjs";
+import { formatFileTimeoutBudgetError } from "../shared/file-timeout.mjs";
+import { settleSubprocess } from "./default-runtime-runner.mjs";
 import { ensurePlaywrightTestConfig } from "./playwright-config.mjs";
-import { buildPlaywrightEnv } from "./template.mjs";
+import { printBufferedOutput } from "./processes.mjs";
 import { normalizePathSeparators } from "./state.mjs";
+import { buildPlaywrightEnv } from "./template.mjs";
 
-export async function runPlaywrightBatch(targetConfig, batch, lifecycle) {
+export async function runPlaywrightTask(targetConfig, task, lifecycle, lease) {
   const local = targetConfig.testkit.local;
   if (!local?.baseUrl) {
     throw new Error(
@@ -18,69 +17,48 @@ export async function runPlaywrightBatch(targetConfig, batch, lifecycle) {
     );
   }
 
-  console.log(
-    `\n── ${targetConfig.stackLabel} playwright:${targetConfig.name} (${batch.tasks.length} file${batch.tasks.length === 1 ? "" : "s"})${formatPlaywrightBatchFiles(batch)} ──`
-  );
+  console.log(`\n── ${targetConfig.runtimeLabel} playwright:${targetConfig.name} (${task.file}) ──`);
 
   const cwd = resolveServiceCwd(targetConfig.productDir, local.cwd);
-  const requestedFiles = batch.tasks.map((task) =>
-    path.relative(cwd, path.join(targetConfig.productDir, task.file))
-  );
-  const playwrightConfigPath = ensurePlaywrightTestConfig(targetConfig, cwd, requestedFiles);
+  const requestedFile = path.relative(cwd, path.join(targetConfig.productDir, task.file));
+  const playwrightConfigPath = ensurePlaywrightTestConfig(targetConfig, cwd, [requestedFile], lease);
   const startedAt = Date.now();
-  const result = await execa(
+  const fileTimeoutSeconds = targetConfig.testkit.execution.fileTimeoutSeconds;
+  const subprocess = execa(
     "npx",
     ["playwright", "test", "--config", playwrightConfigPath, "--reporter=json"],
     {
       cwd,
-      env: buildPlaywrightEnv(targetConfig, local.baseUrl, process.env),
+      env: buildPlaywrightEnv(targetConfig, local.baseUrl, lease, process.env),
       reject: false,
       cancelSignal: lifecycle.signal,
       forceKillAfterDelay: 5_000,
     }
   );
+  const { result, timedOut } = await settleSubprocess(subprocess, fileTimeoutSeconds);
 
   if (result.stderr) {
-    printBufferedOutput(result.stderr, `[${targetConfig.stackLabel}:${targetConfig.name}:playwright]`);
+    printBufferedOutput(result.stderr, `[${targetConfig.runtimeLabel}:${targetConfig.name}:playwright]`);
   }
 
-  const parsed = parsePlaywrightJsonResults(result.stdout, cwd);
+  const parsed = parsePlaywrightJsonResults(result.stdout || "", cwd);
   const finishedAt = Date.now();
-  const batchDurationMs = finishedAt - startedAt;
-  const genericError =
-    result.exitCode === 0
+  const durationMs = finishedAt - startedAt;
+  const relativeFile = normalizePathSeparators(requestedFile);
+  const fileResult = parsed.fileResults.get(relativeFile);
+  const genericError = timedOut
+    ? formatFileTimeoutBudgetError(fileTimeoutSeconds)
+    : result.exitCode === 0
       ? parsed.errors[0] || null
-      : parsed.errors[0] ||
-        result.stderr.trim() ||
-        `Playwright exited with code ${result.exitCode}`;
-
-  return batch.tasks.map((task) => {
-    const relativeFile = normalizePathSeparators(
-      path.relative(cwd, path.join(targetConfig.productDir, task.file))
-    );
-    const fileResult = parsed.fileResults.get(relativeFile);
-    if (fileResult) {
-      return {
-        task,
-        failed: fileResult.status === "failed",
-        status: fileResult.status,
-        error: fileResult.error,
-        durationMs:
-          fileResult.durationMs > 0
-            ? fileResult.durationMs
-            : Math.round(batchDurationMs / Math.max(1, batch.tasks.length)),
-        startedAt,
-        finishedAt,
-      };
-    }
+      : parsed.errors[0] || result.stderr.trim() || `Playwright exited with code ${result.exitCode}`;
 
-    return {
-      task,
-      failed: result.exitCode !== 0,
-      error: result.exitCode !== 0 ? genericError : null,
-      durationMs: Math.round(batchDurationMs / Math.max(1, batch.tasks.length)),
-      startedAt,
-      finishedAt,
-    };
-  });
+  return {
+    task,
+    failed: timedOut ? true : fileResult ? fileResult.status === "failed" : result.exitCode !== 0,
+    status: timedOut ? "failed" : fileResult?.status || (result.exitCode === 0 ? "passed" : "failed"),
+    error: timedOut ? genericError : fileResult?.error || genericError,
+    durationMs: fileResult?.durationMs > 0 ? fileResult.durationMs : durationMs,
+    startedAt,
+    finishedAt,
+  };
 }

package/lib/runner/readiness.mjs

@@ -54,11 +54,11 @@ export async function assertLocalServicePortsAvailable(config, isPortInUse) {
       const owner = findPortOwner(config.productDir, socket);
       const ownerDetail = owner
         ? owner.active
-          ? ` Active testkit run ${formatRunSummary(owner.manifest)} owns ${key} via ${owner.service.stackLabel}:${owner.service.serviceName}.`
+          ? ` Active testkit run ${formatRunSummary(owner.manifest)} owns ${key} via ${owner.service.runtimeLabel}:${owner.service.serviceName}.`
           : ` Stale testkit run ${formatRunSummary(owner.manifest)} owns ${key}.`
         : "";
       throw new Error(
-        `Cannot start "${config.stackLabel}:${config.name}" because ${key} is already in use. ` +
+        `Cannot start "${config.runtimeLabel}:${config.name}" because ${key} is already in use. ` +
           `Stop the existing process and rerun testkit.${ownerDetail}`
       );
     }

package/lib/runner/reporting.mjs

@@ -96,7 +96,7 @@ export function buildRunArtifact({
   finishedAt,
   execution,
   workerCount,
-  stackCount,
+  runtimeInstanceCount,
   typeValues,
   suiteSelectors,
   fileNames,
@@ -135,9 +135,9 @@ export function buildRunArtifact({
       finishedAt: new Date(finishedAt).toISOString(),
       durationMs: finishedAt - startedAt,
       workers: execution.workers,
+      fileTimeoutSeconds: execution.fileTimeoutSeconds,
       workerCount,
-      stackMode: execution.stackMode,
-      stackCount,
+      runtimeInstanceCount,
       dbBackend,
       types: typeValues,
       suiteSelectors: suiteSelectors.map((selector) => selector.raw),

package/lib/runner/reporting.test.mjs

@@ -53,11 +53,10 @@ describe("runner reporting", () => {
       finishedAt: 4000,
       execution: {
         workers: 2,
-        stackMode: "shared",
-        stackCount: 1,
+        fileTimeoutSeconds: 60,
       },
       workerCount: 1,
-      stackCount: 1,
+      runtimeInstanceCount: 2,
       typeValues: ["all"],
       suiteSelectors: [],
       fileNames: [],
@@ -82,9 +81,9 @@ describe("runner reporting", () => {
     expect(artifact.schemaVersion).toBe(3);
     expect(artifact.run).toMatchObject({
       workers: 2,
+      fileTimeoutSeconds: 60,
       workerCount: 1,
-      stackMode: "shared",
-      stackCount: 1,
+      runtimeInstanceCount: 2,
     });
     expect(artifact.summary.services).toEqual({
       total: 1,

package/lib/runner/results.mjs

@@ -142,7 +142,7 @@ export function recordTaskOutcome(trackers, task, outcome, finishedAt = Date.now
 }
 
 export function recordGraphError(trackers, graph, message, now = Date.now()) {
-  const targetNames = graph?.assignedTargets || [];
+  const targetNames = graph?.targetNames || [];
   for (const targetName of targetNames) {
     const tracker = trackers.get(targetName);
     if (tracker && !tracker.skipped) {

package/lib/runner/results.test.mjs

@@ -54,7 +54,7 @@ describe("runner results", () => {
     const tracker = trackers.get("api");
     addTrackerError(tracker, "worker failed");
     addTrackerError(tracker, "worker failed");
-    recordGraphError(trackers, { assignedTargets: ["api"] }, "graph failed", 1300);
+    recordGraphError(trackers, { targetNames: ["api"] }, "graph failed", 1300);
 
     const result = finalizeServiceResult(tracker, 1000, 1500);
     expect(result.failed).toBe(true);

package/lib/runner/runtime-contexts.mjs

@@ -3,29 +3,29 @@ import path from "path";
 import { execaCommand } from "execa";
 import { resolveServiceCwd } from "../config/index.mjs";
 import { prepareDatabaseRuntime } from "../database/index.mjs";
-import { buildExecutionEnv, resolveStackRuntimeConfigs } from "./template.mjs";
+import { taskNeedsLocalRuntime } from "./planning.mjs";
 import { writeGraphMetadata } from "./state.mjs";
 import { startLocalServices, stopLocalServices } from "./services.mjs";
+import { buildExecutionEnv, resolveRuntimeInstanceConfigs } from "./template.mjs";
 
-export function createStackContext(stackId, graph, productDir) {
+export function createRuntimeInstanceContext(runtimeId, graph, productDir) {
   const graphDir = path.join(productDir, ".testkit", "_graphs", graph.dirName);
-  const stackStateDir = path.join(graphDir, "stacks", stackId);
-  fs.mkdirSync(stackStateDir, { recursive: true });
+  const runtimeDir = path.join(graphDir, "runtimes", runtimeId);
+  fs.mkdirSync(runtimeDir, { recursive: true });
   writeGraphMetadata(graphDir, graph);
 
-  const runtimeConfigs = resolveStackRuntimeConfigs(
-    graph.rootConfig,
-    graph.runtimeConfigs,
-    stackId,
-    stackStateDir
-  );
+  const runtimeConfigs = resolveRuntimeInstanceConfigs(graph.runtimeConfigs, runtimeId, runtimeDir, {
+    graphDirName: graph.dirName,
+    portNamespaceIndex: graph.portNamespaceIndex,
+    portNamespaceStride: graph.portNamespaceStride,
+  });
 
   return {
     graphKey: graph.key,
-    assignedTargets: [...graph.assignedTargets],
+    targetNames: [...graph.targetNames],
     graphDir,
-    stackId,
-    stackStateDir,
+    runtimeDir,
+    runtimeId,
     runtimeConfigs,
     configByName: new Map(runtimeConfigs.map((config) => [config.name, config])),
     prepared: false,
@@ -36,7 +36,7 @@ export function createStackContext(stackId, graph, productDir) {
   };
 }
 
-export async function ensureStackContextReady(context, batch, lifecycle) {
+export async function ensureRuntimeInstanceReady(context, task, lifecycle) {
   if (!context.prepared) {
     if (!context.preparationPromise) {
       context.preparationPromise = (async () => {
@@ -49,7 +49,7 @@ export async function ensureStackContextReady(context, batch, lifecycle) {
     await context.preparationPromise;
   }
 
-  if (batchNeedsLocalRuntime(batch) && !context.started) {
+  if (taskNeedsLocalRuntime(task) && !context.started) {
     if (!context.startupPromise) {
       context.startupPromise = (async () => {
         context.startedServices = await startLocalServices(context.runtimeConfigs, lifecycle);
@@ -64,16 +64,16 @@ export async function ensureStackContextReady(context, batch, lifecycle) {
   return context;
 }
 
-export async function deactivateStackContext(context, lifecycle) {
+export async function deactivateRuntimeInstanceContext(context, lifecycle) {
   if (!context?.started) return;
   await stopLocalServices(context.startedServices, lifecycle);
   context.started = false;
   context.startedServices = [];
 }
 
-export async function cleanupStackContext(context, lifecycle) {
+export async function cleanupRuntimeInstanceContext(context, lifecycle) {
   if (!context) return;
-  await deactivateStackContext(context, lifecycle);
+  await deactivateRuntimeInstanceContext(context, lifecycle);
 }
 
 export async function prepareDatabases(runtimeConfigs) {
@@ -94,7 +94,7 @@ async function runMigrate(config, databaseUrl) {
   const env = buildExecutionEnv(config, {}, process.env);
   if (databaseUrl) env.DATABASE_URL = databaseUrl;
 
-  console.log(`\n── migrate:${config.stackLabel}:${config.name} ──`);
+  console.log(`\n── migrate:${config.runtimeLabel}:${config.name} ──`);
   await execaCommand(migrate.cmd, {
     cwd: resolveServiceCwd(config.productDir, migrate.cwd),
     env,
@@ -110,7 +110,7 @@ async function runSeed(config, databaseUrl) {
   const env = buildExecutionEnv(config, {}, process.env);
   if (databaseUrl) env.DATABASE_URL = databaseUrl;
 
-  console.log(`\n── seed:${config.stackLabel}:${config.name} ──`);
+  console.log(`\n── seed:${config.runtimeLabel}:${config.name} ──`);
   await execaCommand(seed.cmd, {
     cwd: resolveServiceCwd(config.productDir, seed.cwd),
     env,
@@ -118,7 +118,3 @@ async function runSeed(config, databaseUrl) {
     shell: true,
   });
 }
-
-function batchNeedsLocalRuntime(batch) {
-  return batch.type !== "dal";
-}

package/lib/runner/runtime-manager.mjs

@@ -0,0 +1,181 @@
+import fs from "fs";
+import path from "path";
+import { buildRuntimeIds } from "./execution-config.mjs";
+import {
+  cleanupRuntimeInstanceContext,
+  createRuntimeInstanceContext,
+  ensureRuntimeInstanceReady,
+} from "./runtime-contexts.mjs";
+
+export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {} }) {
+  const graphByKey = new Map(graphs.map((graph) => [graph.key, graph]));
+  const pools = new Map();
+  const locks = new Map();
+  let nextLeaseCounter = 1;
+  const runtimeHooks = {
+    cleanupRuntimeInstanceContext,
+    createRuntimeInstanceContext,
+    ensureRuntimeInstanceReady,
+    sleep,
+    ...hooks,
+  };
+
+  return {
+    async acquire(task) {
+      const pool = getPool(pools, graphByKey, task, productDir, lifecycle);
+
+      while (true) {
+        if (lifecycle.isStopRequested()) {
+          throw lifecycle.signal.reason || new Error("testkit run interrupted");
+        }
+
+        const leaseId = `lease-${task.id}-${nextLeaseCounter}`;
+        nextLeaseCounter += 1;
+
+        if (!tryAcquireLocks(locks, task.locks || [], leaseId)) {
+          await runtimeHooks.sleep(10);
+          continue;
+        }
+
+        const slot = claimRuntimeSlot(pool);
+        if (!slot) {
+          releaseLocks(locks, task.locks || [], leaseId);
+          await runtimeHooks.sleep(10);
+          continue;
+        }
+
+        try {
+          const context = await getReadyContext(slot, productDir, task, lifecycle, runtimeHooks);
+          const leaseDir = path.join(context.runtimeDir, "leases", leaseId);
+          fs.mkdirSync(leaseDir, { recursive: true });
+          return {
+            leaseId,
+            leaseDir,
+            lockNames: task.locks || [],
+            slot,
+            context,
+          };
+        } catch (error) {
+          releaseRuntimeSlot(slot);
+          releaseLocks(locks, task.locks || [], leaseId);
+          cleanupLeaseDir({ leaseDir: path.join(slot.context?.runtimeDir || productDir, "leases", leaseId) });
+          await drainRuntimeSlot(slot, lifecycle, runtimeHooks);
+          throw error;
+        }
+      }
+    },
+    async release(lease, options = {}) {
+      if (!lease?.slot) return;
+      releaseLocks(locks, lease.lockNames || [], lease.leaseId);
+      releaseRuntimeSlot(lease.slot);
+      cleanupLeaseDir(lease);
+      if (options.invalidate) {
+        lease.slot.draining = true;
+      }
+      if (lease.slot.draining && lease.slot.activeLeaseCount === 0) {
+        await drainRuntimeSlot(lease.slot, lifecycle, runtimeHooks);
+      }
+    },
+    async cleanupAll() {
+      for (const pool of pools.values()) {
+        for (const slot of pool.slots) {
+          slot.draining = true;
+          await drainRuntimeSlot(slot, lifecycle, runtimeHooks);
+        }
+      }
+    },
+  };
+}
+
+function getPool(pools, graphByKey, task, productDir, lifecycle) {
+  const existing = pools.get(task.graphKey);
+  if (existing) return existing;
+
+  const graph = graphByKey.get(task.graphKey);
+  if (!graph) {
+    throw new Error(`Unknown graph "${task.graphKey}"`);
+  }
+
+  const pool = {
+    productDir,
+    lifecycle,
+    slots: buildRuntimeIds(graph.instanceCount).map((runtimeId) => ({
+      graph,
+      runtimeId,
+      context: null,
+      contextPromise: null,
+      activeLeaseCount: 0,
+      draining: false,
+    })),
+  };
+  pools.set(task.graphKey, pool);
+  return pool;
+}
+
+function claimRuntimeSlot(pool) {
+  const available = pool.slots.filter((slot) => !slot.draining);
+  if (available.length === 0) return null;
+
+  const slot = [...available].sort(
+    (left, right) =>
+      left.activeLeaseCount - right.activeLeaseCount ||
+      left.runtimeId.localeCompare(right.runtimeId)
+  )[0];
+  slot.activeLeaseCount += 1;
+  return slot;
+}
+
+function releaseRuntimeSlot(slot) {
+  slot.activeLeaseCount = Math.max(0, slot.activeLeaseCount - 1);
+}
+
+async function getReadyContext(slot, productDir, task, lifecycle, runtimeHooks) {
+  if (!slot.context) {
+    slot.context = runtimeHooks.createRuntimeInstanceContext(slot.runtimeId, slot.graph, productDir);
+    lifecycle.trackGraphContext(slot.context);
+  }
+  if (!slot.contextPromise) {
+    slot.contextPromise = Promise.resolve(slot.context);
+  }
+  await slot.contextPromise;
+  await runtimeHooks.ensureRuntimeInstanceReady(slot.context, task, lifecycle);
+  return slot.context;
+}
+
+async function drainRuntimeSlot(slot, lifecycle, runtimeHooks) {
+  if (!slot.context || slot.activeLeaseCount > 0) return;
+  await runtimeHooks.cleanupRuntimeInstanceContext(slot.context, lifecycle);
+  slot.context = null;
+  slot.contextPromise = null;
+  slot.draining = false;
+}
+
+function tryAcquireLocks(lockMap, lockNames, leaseId) {
+  const uniqueLocks = [...new Set(lockNames)].sort();
+  for (const lockName of uniqueLocks) {
+    if (lockMap.has(lockName)) {
+      return false;
+    }
+  }
+  for (const lockName of uniqueLocks) {
+    lockMap.set(lockName, leaseId);
+  }
+  return true;
+}
+
+function releaseLocks(lockMap, lockNames, leaseId) {
+  for (const lockName of [...new Set(lockNames)].sort()) {
+    if (lockMap.get(lockName) === leaseId) {
+      lockMap.delete(lockName);
+    }
+  }
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function cleanupLeaseDir(lease) {
+  if (!lease?.leaseDir) return;
+  fs.rmSync(lease.leaseDir, { recursive: true, force: true });
+}

package/lib/runner/runtime-manager.test.mjs

@@ -0,0 +1,181 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { afterEach, describe, expect, it } from "vitest";
+import { createRuntimeManager } from "./runtime-manager.mjs";
+
+const cleanups = [];
+
+afterEach(() => {
+  while (cleanups.length > 0) {
+    cleanups.pop()();
+  }
+});
+
+function makeTempDir(prefix) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+  cleanups.push(() => fs.rmSync(dir, { recursive: true, force: true }));
+  return dir;
+}
+
+function makeLifecycle() {
+  return {
+    signal: new AbortController().signal,
+    isStopRequested() {
+      return false;
+    },
+    trackGraphContext() {},
+  };
+}
+
+function makeTask(id, extras = {}) {
+  return {
+    id,
+    graphKey: "api",
+    locks: [],
+    ...extras,
+  };
+}
+
+function makeHooks(events) {
+  return {
+    createRuntimeInstanceContext(runtimeId, graph, productDir) {
+      const runtimeDir = path.join(productDir, ".testkit", "_graphs", graph.dirName, "runtimes", runtimeId);
+      fs.mkdirSync(runtimeDir, { recursive: true });
+      events.created.push(runtimeId);
+      return {
+        graphKey: graph.key,
+        runtimeDir,
+        runtimeId,
+        targetNames: [...graph.targetNames],
+      };
+    },
+    async ensureRuntimeInstanceReady(context) {
+      events.ready.push(context.runtimeId);
+    },
+    async cleanupRuntimeInstanceContext(context) {
+      events.cleaned.push(context.runtimeId);
+      fs.rmSync(context.runtimeDir, { recursive: true, force: true });
+    },
+    async sleep() {
+      await new Promise((resolve) => setTimeout(resolve, 0));
+    },
+  };
+}
+
+describe("runtime-manager", () => {
+  it("spreads concurrent leases across runtime instances", async () => {
+    const productDir = makeTempDir("testkit-runtime-manager-");
+    const events = { created: [], ready: [], cleaned: [] };
+    const manager = createRuntimeManager({
+      productDir,
+      lifecycle: makeLifecycle(),
+      graphs: [
+        {
+          key: "api",
+          dirName: "api",
+          targetNames: ["api"],
+          instanceCount: 2,
+        },
+      ],
+      hooks: makeHooks(events),
+    });
+
+    const leaseOne = await manager.acquire(makeTask(1));
+    const leaseTwo = await manager.acquire(makeTask(2));
+
+    expect(leaseOne.slot.runtimeId).toBe("runtime-1");
+    expect(leaseTwo.slot.runtimeId).toBe("runtime-2");
+    expect(events.created).toEqual(["runtime-1", "runtime-2"]);
+
+    await manager.release(leaseOne);
+    await manager.release(leaseTwo);
+    await manager.cleanupAll();
+  });
+
+  it("blocks conflicting locks until the first lease releases", async () => {
+    const productDir = makeTempDir("testkit-runtime-manager-");
+    const manager = createRuntimeManager({
+      productDir,
+      lifecycle: makeLifecycle(),
+      graphs: [
+        {
+          key: "api",
+          dirName: "api",
+          targetNames: ["api"],
+          instanceCount: 1,
+        },
+      ],
+      hooks: makeHooks({ created: [], ready: [], cleaned: [] }),
+    });
+
+    const firstLease = await manager.acquire(makeTask(1, { locks: ["shared-lock"] }));
+    let secondSettled = false;
+    const secondPromise = manager.acquire(makeTask(2, { locks: ["shared-lock"] })).then((lease) => {
+      secondSettled = true;
+      return lease;
+    });
+
+    await new Promise((resolve) => setTimeout(resolve, 20));
+    expect(secondSettled).toBe(false);
+
+    await manager.release(firstLease);
+    const secondLease = await secondPromise;
+    expect(secondSettled).toBe(true);
+
+    await manager.release(secondLease);
+    await manager.cleanupAll();
+  });
+
+  it("does not drain an invalidated runtime slot until all active leases release", async () => {
+    const productDir = makeTempDir("testkit-runtime-manager-");
+    const events = { created: [], ready: [], cleaned: [] };
+    const manager = createRuntimeManager({
+      productDir,
+      lifecycle: makeLifecycle(),
+      graphs: [
+        {
+          key: "api",
+          dirName: "api",
+          targetNames: ["api"],
+          instanceCount: 1,
+        },
+      ],
+      hooks: makeHooks(events),
+    });
+
+    const leaseOne = await manager.acquire(makeTask(1));
+    const leaseTwo = await manager.acquire(makeTask(2));
+
+    await manager.release(leaseOne, { invalidate: true });
+    expect(events.cleaned).toEqual([]);
+
+    await manager.release(leaseTwo);
+    expect(events.cleaned).toEqual(["runtime-1"]);
+  });
+
+  it("removes lease directories on release", async () => {
+    const productDir = makeTempDir("testkit-runtime-manager-");
+    const manager = createRuntimeManager({
+      productDir,
+      lifecycle: makeLifecycle(),
+      graphs: [
+        {
+          key: "api",
+          dirName: "api",
+          targetNames: ["api"],
+          instanceCount: 1,
+        },
+      ],
+      hooks: makeHooks({ created: [], ready: [], cleaned: [] }),
+    });
+
+    const lease = await manager.acquire(makeTask(1));
+    expect(fs.existsSync(lease.leaseDir)).toBe(true);
+
+    await manager.release(lease);
+    expect(fs.existsSync(lease.leaseDir)).toBe(false);
+
+    await manager.cleanupAll();
+  });
+});