mixdog 1.4.0 → 1.4.2

package/scripts/lib/gate-scheduler.mjs

@@ -1,42 +1,148 @@
+import { randomUUID } from 'node:crypto';
+import { readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
 import { availableParallelism } from 'node:os';
-import { testConcurrencyArg } from './run-node-tests.mjs';
 
-// A local gate has one budget, not a full machine's worth of workers per lane.
-// Six is the interactive-desktop default; an explicit budget still wins.
-export function gateTestBudget(env = process.env, parallelism = availableParallelism()) {
-  const argument = testConcurrencyArg([], env);
-  return argument ? Number(argument.split('=')[1]) : Math.min(6, parallelism);
+// The local gate runs every lane from one shared pool of workers: each lane's
+// files are cut into shards, and whichever worker is free takes the longest
+// shard left, whatever its lane. Shard sizes come from the previous run's
+// measured per-file times, so the split stays balanced without hand-kept
+// weights, and no worker idles while another lane still has work.
+
+// The gate runs its whole process tree at below-normal priority, so a
+// foreground app always wins the CPU. Priority does not bound what the
+// desktop shell shares with every process (handles, desktop heap, work
+// queues; explorer.exe has failed with "no work queue available" under many
+// concurrent test processes), so the default also caps the worker count:
+// half the cores, leaving the other half for what the budget does not count
+// (each shard's runner, the processes tests spawn, the desktop build's own
+// threads), and never more than DEFAULT_MAX_WORKERS. Each worker is three
+// node processes (shard runner, its node --test batch, the test file) plus
+// whatever the test spawns. Measured on a 20-core Windows PC with other work
+// running: at 10 workers, test-file processes began dying within ~20 ms with
+// no output (every remaining file of a shard, in 2 of 2 runs); at 8 and 6
+// they did not.
+// An explicit budget still wins: --test-concurrency (gate argument), then
+// MIXDOG_TEST_CONCURRENCY.
+const DEFAULT_MAX_WORKERS = 8;
+export function gateTestBudget(argv = [], env = process.env, parallelism = availableParallelism()) {
+  const index = argv.findIndex((arg) => arg === '--test-concurrency' || arg.startsWith('--test-concurrency='));
+  let explicit = env.MIXDOG_TEST_CONCURRENCY || undefined;
+  // A flag without a value is an error, not "unspecified".
+  if (index >= 0) explicit = argv[index].includes('=') ? argv[index].split('=')[1] : (argv[index + 1] ?? '');
+  if (explicit === undefined) return Math.max(1, Math.min(DEFAULT_MAX_WORKERS, Math.floor(parallelism / 2)));
+  const budget = Number(explicit);
+  if (!Number.isInteger(budget) || budget < 1) throw new Error(`gate test budget must be a positive integer (got "${explicit}")`);
+  return budget;
+}
+
+// Estimates for work with no recorded time yet: the next run records it.
+export const DEFAULT_ESTIMATE_MS = { fast: 2_000, slow: 25_000, command: 60_000 };
+
+// Previous runs' measured times ({ version, ms: { key: ms } }). A missing,
+// unreadable or malformed file counts as no history, and an entry that is
+// not a finite positive number is dropped (its work gets the default).
+export function readTimingHistory(file) {
+  let ms;
+  try {
+    ms = JSON.parse(readFileSync(file, 'utf8'))?.ms;
+  } catch {
+    return {};
+  }
+  if (!ms || typeof ms !== 'object' || Array.isArray(ms)) return {};
+  return Object.fromEntries(
+    Object.entries(ms).filter(([, value]) => typeof value === 'number' && Number.isFinite(value) && value > 0)
+  );
 }
 
-// Splits the budget across legs in proportion to leg.weight (default 1), at least
-// one worker each. Spare workers go one at a time to the leg with the highest
-// weight per worker, so longer legs get more. Needs budget >= legs.length.
-export function allocateWorkers(legs, budget) {
-  const shares = legs.map(() => 1);
-  for (let spare = budget - legs.length; spare > 0; spare--) {
-    let best = 0;
-    for (let i = 1; i < legs.length; i++) {
-      if ((legs[i].weight ?? 1) / shares[i] > (legs[best].weight ?? 1) / shares[best]) best = i;
+// Writes the history through a temp file of its own (concurrent gates in
+// other worktrees share the file) and renames it into place. Never throws:
+// returns the error, or null once saved; the timings only plan the next run.
+export function saveTimingHistory(file, ms) {
+  const temp = `${file}.${process.pid}.${randomUUID()}.tmp`;
+  try {
+    writeFileSync(temp, `${JSON.stringify({ version: 1, ms })}\n`);
+    renameSync(temp, file);
+    return null;
+  } catch (error) {
+    try {
+      rmSync(temp, { force: true });
+    } catch {
+      // The save error is the one reported.
     }
-    shares[best]++;
+    return error;
   }
-  return shares;
 }
+// Enough shards per worker that the pool's tail (the shards still running
+// when the queue is empty) stays short; the floor bounds the per-shard runner
+// start-up and process sweep (~1-2 s each).
+const SHARDS_PER_WORKER = 8;
+const MIN_SHARD_MS = 15_000;
 
-// The test runners read their worker count when spawned, so it cannot grow
-// mid-run: shares are static. With fewer workers than legs, each worker runs
-// legs one after another.
-export async function runGateLegs(legs, budget, runLeg) {
-  if (!Number.isInteger(budget) || budget < 1) throw new Error('gate test budget must be a positive integer');
-  if (budget < legs.length) {
-    let next = 0;
-    await Promise.all(
-      Array.from({ length: budget }, async () => {
-        while (next < legs.length) await runLeg(legs[next++], 1);
-      })
-    );
-    return;
+// test.mjs selects every lane file whose path contains a given filter, so a
+// file whose path contains another file's path has to share that file's shard
+// or it would run twice. Returns the indivisible groups.
+export function filterGroups(files) {
+  const parent = new Map(files.map((file) => [file, file]));
+  const find = (file) => (parent.get(file) === file ? file : find(parent.get(file)));
+  for (const inner of files) {
+    for (const outer of files) if (outer !== inner && outer.includes(inner)) parent.set(find(outer), find(inner));
   }
-  const shares = allocateWorkers(legs, budget);
-  await Promise.all(legs.map((leg, index) => runLeg(leg, shares[index])));
+  const groups = new Map();
+  for (const file of files) groups.set(find(file), [...(groups.get(find(file)) ?? []), file]);
+  return [...groups.values()];
+}
+
+// Cuts each lane ({ name, files, estimateOf }) into shards of about one
+// target size, longest-first into the least-loaded shard, so a file longer
+// than the target runs alone. Returns [{ lane, files, estimate }].
+export function planShards(lanes, budget) {
+  const total = lanes.reduce((sum, lane) => sum + lane.files.reduce((s, file) => s + lane.estimateOf(file), 0), 0);
+  const target = Math.max(MIN_SHARD_MS, total / (budget * SHARDS_PER_WORKER));
+  const shards = [];
+  for (const lane of lanes) {
+    const items = filterGroups(lane.files)
+      .map((files) => ({ files, estimate: files.reduce((s, file) => s + lane.estimateOf(file), 0) }))
+      .sort((a, b) => b.estimate - a.estimate);
+    const laneTotal = items.reduce((sum, item) => sum + item.estimate, 0);
+    const bins = Array.from({ length: Math.min(items.length, Math.max(1, Math.round(laneTotal / target))) }, () => ({
+      lane: lane.name,
+      files: [],
+      estimate: 0,
+    }));
+    for (const item of items) {
+      const bin = bins.reduce((least, candidate) => (candidate.estimate < least.estimate ? candidate : least));
+      bin.files.push(...item.files);
+      bin.estimate += item.estimate;
+    }
+    shards.push(...bins);
+  }
+  return shards;
+}
+
+// Per-file time of a finished shard: what its own tests reported plus an
+// equal share of the rest of the shard's wall time (process start-up, module
+// load, the runner's own work). Returns { file: ms }.
+export function measuredFileTimes(files, reportedMs, wallMs) {
+  const reported = files.reduce((sum, file) => sum + (reportedMs[file] ?? 0), 0);
+  const overhead = Math.max(0, wallMs - reported) / files.length;
+  return Object.fromEntries(files.map((file) => [file, (reportedMs[file] ?? 0) + overhead]));
+}
+
+// Runs every unit on `budget` workers, longest estimate first. A failed unit
+// marked stopOnFail stops new starts (running units finish). Returns the
+// units never started.
+export async function runPool(units, budget, run) {
+  if (!Number.isInteger(budget) || budget < 1) throw new Error('gate test budget must be a positive integer');
+  const queue = [...units].sort((a, b) => b.estimate - a.estimate);
+  let stopped = false;
+  await Promise.all(
+    Array.from({ length: Math.min(budget, queue.length) }, async () => {
+      while (!stopped && queue.length > 0) {
+        const unit = queue.shift();
+        unit.ok = await run(unit);
+        if (!unit.ok && unit.stopOnFail) stopped = true;
+      }
+    })
+  );
+  return queue;
 }

package/scripts/lib/gate-timing-reporter.mjs

@@ -0,0 +1,16 @@
+// node:test reporter the local gate adds to each shard it runs: one JSON line
+// per test file, { file, ms }, the file's top-level test time (relative to the
+// package the run started in, like `scripts/test.mjs --list`). The gate turns
+// these into the per-file estimates its next run plans shards from.
+import { relative } from 'node:path';
+
+export default async function* gateTimingReporter(source) {
+  const byFile = new Map();
+  for await (const event of source) {
+    if (event.type !== 'test:pass' && event.type !== 'test:fail') continue;
+    if (event.data.nesting !== 0 || !event.data.file) continue;
+    const file = relative(process.cwd(), String(event.data.file)).replaceAll('\\', '/');
+    byFile.set(file, (byFile.get(file) ?? 0) + (Number(event.data.details?.duration_ms) || 0));
+  }
+  for (const [file, ms] of byFile) yield `${JSON.stringify({ file, ms })}\n`;
+}

package/scripts/lib/isolated-test-env.mjs

@@ -2,15 +2,20 @@
 // evaluates imports in order) so the data/home dirs resolve to a scratch dir
 // before any module under test reads them, even under plain `node --test` from a
 // shell that exports the signed-in profile's MIXDOG_DATA_DIR.
-import { mkdtempSync, rmSync } from 'node:fs';
+import { mkdtempSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
+import { registerOwnedTempDir } from './scoped-temp.mjs';
+import { pruneStaleTempDirs, removeTempDirSync } from './stale-temp.mjs';
 
-const root = mkdtempSync(join(tmpdir(), 'mixdog-test-profile-'));
+// The exit handler below never runs for a killed process, so each start sweeps
+// what earlier killed processes left.
+const PREFIX = 'mixdog-test-profile-';
+pruneStaleTempDirs([PREFIX]);
+const root = mkdtempSync(join(tmpdir(), PREFIX));
+registerOwnedTempDir(root);
 process.env.MIXDOG_DATA_DIR = join(root, 'data');
 process.env.MIXDOG_HOME = join(root, 'home');
 process.on('exit', () => {
-  try {
-    rmSync(root, { recursive: true, force: true });
-  } catch {}
+  removeTempDirSync(root);
 });

package/scripts/lib/run-node-tests.mjs

@@ -1,8 +1,9 @@
 import { spawn } from 'node:child_process';
-import { appendFile, mkdtemp, readdir, readFile, rm, stat } from 'node:fs/promises';
+import { appendFile, mkdtemp, readFile, rm } from 'node:fs/promises';
 import { availableParallelism, tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { setTimeout as delay } from 'node:timers/promises';
+import { pruneStaleTempDirs } from './stale-temp.mjs';
 import { FAILURE_RECORDS_ENV, spawnLedgerPath } from './test-failure-records.mjs';
 import { classifyRerunOutcomes, formatFailureSummary } from './test-failure-summary.mjs';
 
@@ -20,28 +21,14 @@ const HEAP_FLAG = /^--(?:max-old-space-size|max-semi-space-size)=\d+$/;
 // An argument costs its own characters plus the separator and its quotes.
 const argCost = (arg) => arg.length + 3;
 
-// Full logs are kept for reading after the run; older ones are stale.
+// A green run removes its log directory; a failed run keeps it (the failure
+// summary points at it) until the start-of-run prune finds it stale. The
+// failure and scratch directories are removed when their run ends, so only a
+// killed runner leaves them for that same prune.
 const TEST_LOG_PREFIX = 'mixdog-test-output-';
-const TEST_LOG_RETENTION_MS = 3 * 24 * 60 * 60 * 1000;
-
-async function pruneStaleTestLogs(root) {
-  let names = [];
-  try {
-    names = await readdir(root);
-  } catch {
-    return;
-  }
-  const cutoff = Date.now() - TEST_LOG_RETENTION_MS;
-  for (const name of names) {
-    if (!name.startsWith(TEST_LOG_PREFIX)) continue;
-    const path = join(root, name);
-    try {
-      if ((await stat(path)).mtimeMs < cutoff) await rm(path, { recursive: true, force: true });
-    } catch {
-      /* another run may own or be removing it */
-    }
-  }
-}
+const TEST_FAILURES_PREFIX = 'mixdog-test-failures-';
+const TEST_SCRATCH_PREFIX = 'mixdog-test-scratch-';
+export const RUNNER_TEMP_PREFIXES = [TEST_LOG_PREFIX, TEST_FAILURES_PREFIX, TEST_SCRATCH_PREFIX];
 
 // Batches keep the file order Node was given; a single path larger than the
 // budget still gets its own batch rather than being dropped. An empty list
@@ -95,7 +82,8 @@ const concurrencyFlag = (arg) => arg === '--test-concurrency' || arg.startsWith(
 function concurrencyOf(runArgs, fileArgs) {
   const args = [...runArgs, ...fileArgs];
   const index = args.findIndex(concurrencyFlag);
-  const value = index < 0 ? null : positiveInt(args[index].includes('=') ? args[index].split('=')[1] : args[index + 1]);
+  let value = null;
+  if (index >= 0) value = positiveInt(args[index].includes('=') ? args[index].split('=')[1] : args[index + 1]);
   return value ?? Math.max(1, availableParallelism() - 1);
 }
 
@@ -202,17 +190,29 @@ async function processSnapshot() {
     });
 }
 
-const CLOCK_SLACK_MS = 2000;
+// How far a snapshot's creation time may sit from Date.now() read around the
+// spawn. Windows reports creation to the microsecond from the same system
+// clock, so only a clock tick separates them; POSIX ps reports whole elapsed
+// seconds. No wider: Windows hands a freed pid to the next process within
+// milliseconds, and a 2 s window made a process another run had just started
+// pass for one of this run's exited spawns, so it was killed with them.
+const clockSlackMs = (platform) => (platform === 'win32' ? 50 : 2000);
 
 // The processes this run owns, out of a snapshot: an anchor (the batch, or a
 // child a test process recorded in its spawn ledger) still alive under its
-// pid, the orphans an exited anchor left (they still name it as their parent;
-// a later process that reuses its pid only has younger children), every
-// descendant of those, and anything executing from the run's scratch root
-// (e.g. the postgres the memory runtime downloads into MIXDOG_DATA_DIR).
+// pid, the orphans an exited anchor left (they still name it as their parent),
+// every descendant of those, and anything executing from the run's scratch
+// root (e.g. the postgres the memory runtime downloads into MIXDOG_DATA_DIR).
+// An anchor is { pid, before, since, exited? }: the clock just before and just
+// after its spawn, and when its exit was seen. A pid alone never identifies a
+// process: an anchor seen to exit no longer holds its pid, so whatever holds
+// it now is another process; a live holder was created within its spawn; an
+// orphan was created while its anchor lived (a later process that reuses the
+// pid only has younger children); a descendant is younger than its parent.
 // Command-line text never makes a process the run's: an editor or terminal
 // opened on the scratch root is not.
-export function ownedProcesses(snapshot, anchors, { batchStart, scratchDir, platform = process.platform }) {
+export function ownedProcesses(snapshot, anchors, { scratchDir, platform = process.platform }) {
+  const slack = clockSlackMs(platform);
   const byPid = new Map(snapshot.map((entry) => [entry.pid, entry]));
   const childrenOf = new Map();
   for (const entry of snapshot) childrenOf.set(entry.ppid, [...(childrenOf.get(entry.ppid) ?? []), entry]);
@@ -223,14 +223,19 @@ export function ownedProcesses(snapshot, anchors, { batchStart, scratchDir, plat
     owned.set(entry.pid, entry);
     queue.push(entry);
   };
-  for (const { pid, since } of anchors) {
+  for (const { pid, before, since, exited } of anchors) {
     const holder = byPid.get(pid);
-    if (holder && holder.created >= batchStart - CLOCK_SLACK_MS && holder.created <= since + CLOCK_SLACK_MS) {
+    if (holder && exited === undefined && holder.created >= before - slack && holder.created <= since + slack) {
       claim(holder);
       continue;
     }
     for (const orphan of childrenOf.get(pid) ?? []) {
-      if (orphan.created >= since - CLOCK_SLACK_MS && (!holder || orphan.created < holder.created)) claim(orphan);
+      if (
+        orphan.created >= before - slack &&
+        orphan.created <= (exited ?? Number.POSITIVE_INFINITY) + slack &&
+        (!holder || orphan.created < holder.created)
+      )
+        claim(orphan);
     }
   }
   const fold = (path) => (platform === 'win32' ? path.toLowerCase() : path);
@@ -242,26 +247,30 @@ export function ownedProcesses(snapshot, anchors, { batchStart, scratchDir, plat
   while (queue.length > 0) {
     const parent = queue.pop();
     for (const entry of childrenOf.get(parent.pid) ?? []) {
-      if (entry.created >= parent.created - CLOCK_SLACK_MS) claim(entry);
+      if (entry.created >= parent.created - slack) claim(entry);
     }
   }
   return [...owned.values()];
 }
 
+// One anchor per recorded spawn (scripts/lib/test-spawn-ledger.mjs): its
+// "<pid> <before> <since>" line, with "<exited>" from the line that repeats
+// it once the exit was seen.
 async function readSpawnLedger(path) {
   const text = await readFile(path, 'utf8').catch(() => '');
-  return text
-    .split('\n')
-    .filter(Boolean)
-    .map((line) => {
-      const [pid, since] = line.split(' ').map(Number);
-      return { pid, since };
-    });
+  const spawns = new Map();
+  for (const line of text.split('\n').filter(Boolean)) {
+    const [pid, before, since, exited] = line.split(' ').map(Number);
+    const key = `${pid} ${before} ${since}`;
+    spawns.set(key, { ...(spawns.get(key) ?? { pid, before, since }), ...(exited ? { exited } : {}) });
+  }
+  return [...spawns.values()];
 }
 
 // A process a test leaves running must not outlive its batch: it can hold a
 // port, a database, or files in the scratch root the run removes at the end.
-async function reapLeftovers(child, { batchStart, ledger, scratchDir }) {
+// `batch` is the batch's own anchor, its exit already seen.
+async function reapLeftovers(child, { batch, ledger, scratchDir }) {
   // The batch itself has exited: only its process group can remain.
   if (process.platform !== 'win32' && child.pid) {
     try {
@@ -270,12 +279,12 @@ async function reapLeftovers(child, { batchStart, ledger, scratchDir }) {
       /* the group is empty */
     }
   }
-  const anchors = [...(child.pid ? [{ pid: child.pid, since: batchStart }] : []), ...(await readSpawnLedger(ledger))];
-  await killOwned(anchors, { batchStart, scratchDir });
+  const anchors = [...(child.pid ? [batch] : []), ...(await readSpawnLedger(ledger))];
+  await killOwned(anchors, { scratchDir });
 }
 
-async function killOwned(anchors, { batchStart, scratchDir }) {
-  const leftovers = ownedProcesses(await processSnapshot(), anchors, { batchStart, scratchDir });
+async function killOwned(anchors, { scratchDir }) {
+  const leftovers = ownedProcesses(await processSnapshot(), anchors, { scratchDir });
   if (leftovers.length === 0) return;
   const pids = leftovers.map((entry) => entry.pid);
   console.error(`Killing ${pids.length} process(es) a test left running: ${pids.join(' ')}`);
@@ -292,12 +301,13 @@ async function killOwned(anchors, { batchStart, scratchDir }) {
   }
 }
 
-// Resolves { code, hung } once the batch exited and its leftovers are gone.
+// Resolves { code, hung, pid } once the batch exited and its leftovers are gone.
 // `scratchDir` '' = leave the scratch-root sweep to the caller (side-by-side batches).
 function spawnBatch(args, env, { files, ledger, scratchDir, timeoutMs }) {
   return new Promise((resolve) => {
-    const batchStart = Date.now();
+    const before = Date.now();
     const child = spawn(process.execPath, args, { stdio: 'inherit', env, detached: process.platform !== 'win32' });
+    const since = Date.now();
     let hung = '';
     const watchdog = setTimeout(() => {
       hung = `hung after ${minutes(timeoutMs)} min: ${files.join(' ') || '(discovered files)'}`;
@@ -315,10 +325,11 @@ function spawnBatch(args, env, { files, ledger, scratchDir, timeoutMs }) {
     const signals = process.platform === 'win32' ? [] : ['SIGINT', 'SIGTERM'];
     for (const signal of signals) process.on(signal, forward);
     const settle = async (code) => {
+      const batch = { pid: child.pid, before, since, exited: Date.now() };
       clearTimeout(watchdog);
       for (const signal of signals) process.off(signal, forward);
-      await reapLeftovers(child, { batchStart, ledger, scratchDir });
-      resolve({ code: hung ? 1 : code, hung });
+      await reapLeftovers(child, { batch, ledger, scratchDir });
+      resolve({ code: hung ? 1 : code, hung, pid: child.pid });
     };
     child.on('error', (error) => {
       console.error(error);
@@ -331,14 +342,14 @@ function spawnBatch(args, env, { files, ledger, scratchDir, timeoutMs }) {
 // rm's own maxRetries re-walks the tree on every nested retry, so one held
 // file a few directories down multiplied its delay into hours. Each pass here
 // is a single bounded walk.
-async function removeScratch(dir) {
+async function removeTempDir(dir) {
   for (let attempt = 1; ; attempt += 1) {
     try {
       await rm(dir, { recursive: true, force: true });
       return;
     } catch (error) {
       if (attempt === 5) {
-        console.error(`Test scratch left behind (${error.code}): ${dir}`);
+        console.error(`Test temp dir left behind (${error.code}): ${dir}`);
         return;
       }
       await delay(200 * attempt);
@@ -372,21 +383,37 @@ async function readFailureRecords(path) {
 // log can still name what failed.
 //
 // `testTimeoutMs` is the per-file --test-timeout default (see testTimeoutArg).
-export async function runNodeTests(
-  nodeArgs,
-  fileArgs,
-  { argBudget = ARG_BUDGET, coverage = false, rerunFailed = 0, testTimeoutMs = DEFAULT_TEST_TIMEOUT_MS } = {}
-) {
-  await pruneStaleTestLogs(tmpdir());
-  const logPath = join(await mkdtemp(join(tmpdir(), TEST_LOG_PREFIX)), 'full.log');
+export async function runNodeTests(nodeArgs, fileArgs, options = {}) {
+  pruneStaleTempDirs(RUNNER_TEMP_PREFIXES);
+  const outputDir = await mkdtemp(join(tmpdir(), TEST_LOG_PREFIX));
+  const logPath = join(outputDir, 'full.log');
   console.error(`Full test log: ${logPath}`);
   // Failure records live outside the log directory: the log directory holds
   // exactly one merged log, and these records are an internal artifact.
-  const failureDir = await mkdtemp(join(tmpdir(), 'mixdog-test-failures-'));
+  const failureDir = await mkdtemp(join(tmpdir(), TEST_FAILURES_PREFIX));
   // Test processes get a private temp root that is removed after the run, so
   // every fixture directory a test forgets to delete goes with it instead of
   // piling up in the system temp directory.
-  const scratchDir = await mkdtemp(join(tmpdir(), 'mixdog-test-scratch-'));
+  const scratchDir = await mkdtemp(join(tmpdir(), TEST_SCRATCH_PREFIX));
+  let status = 1;
+  try {
+    status = await executeRun(nodeArgs, fileArgs, options, { logPath, failureDir, scratchDir });
+  } finally {
+    await removeTempDir(failureDir);
+    // Leftover processes are already killed; what a process outside the run
+    // still holds stays behind.
+    await removeTempDir(scratchDir);
+    // A throw keeps the log too: the run did not pass.
+    if (status === 0) await removeTempDir(outputDir);
+  }
+}
+
+async function executeRun(
+  nodeArgs,
+  fileArgs,
+  { argBudget = ARG_BUDGET, coverage = false, rerunFailed = 0, testTimeoutMs = DEFAULT_TEST_TIMEOUT_MS },
+  { logPath, failureDir, scratchDir }
+) {
   // The explicit entry also accepts Node options alongside its file paths.
   const concurrencyArg = testConcurrencyArg([...nodeArgs, ...fileArgs]);
   const timeoutArg = testTimeoutArg([...nodeArgs, ...fileArgs], process.env, testTimeoutMs);
@@ -458,7 +485,7 @@ export async function runNodeTests(
   );
   // Peers were still running while each batch exited, so the scratch root's
   // own processes (e.g. a postgres a test started) are swept once, here.
-  if (overlapped) await killOwned([], { batchStart: Date.now(), scratchDir });
+  if (overlapped) await killOwned([], { scratchDir });
   // Logs and records merge in batch order, whatever order the batches finished.
   for (const [index, outcome] of outcomes.entries()) {
     const batchLog = batchLogOf(index);
@@ -509,15 +536,13 @@ export async function runNodeTests(
     status = 0;
   process.exitCode = status;
 
-  await rm(failureDir, { recursive: true, force: true });
-  // Leftover processes are already killed; what a process outside the run
-  // still holds stays behind.
-  await removeScratch(scratchDir);
   if (coverageDir) {
     const { reportCoverage } = await import('./coverage.mjs');
     await reportCoverage(coverageDir);
     await rm(coverageDir, { recursive: true, force: true });
   }
   // Last line of the run, after every reporter and the coverage report.
-  process.stdout.write(`\n${formatFailureSummary({ failed, flaky, rerunFailed, hung, exitCode: status })}`);
+  const pids = outcomes.filter((outcome) => outcome.code !== 0 && outcome.pid).map((outcome) => outcome.pid);
+  process.stdout.write(`\n${formatFailureSummary({ failed, flaky, rerunFailed, hung, exitCode: status, pids })}`);
+  return status;
 }

package/scripts/lib/scoped-temp.d.mts

@@ -0,0 +1,16 @@
+export function makeTempDirSync(prefix: string): string;
+export function registerOwnedTempDir(dir: string): string;
+export function redirectSystemTemp(prefix?: string): string;
+export function removeTempDirSync(dir: string): void;
+export function removeTempDir(dir: string): Promise<void>;
+export function tempDirFor(t: { after(fn: () => unknown): void }, prefix: string): string;
+export interface ScopedTempDir {
+  path: string;
+  join(...parts: string[]): string;
+  remove(): Promise<void>;
+  removeSync(): void;
+  [Symbol.asyncDispose](): Promise<void>;
+  [Symbol.dispose](): void;
+}
+export function scopedTempDir(prefix: string): ScopedTempDir;
+export function withTempDir<T>(prefix: string, fn: (dir: string) => T | Promise<T>): Promise<T>;