mixdog 1.0.12 → 1.1.1

package/scripts/lib/run-node-tests.mjs

@@ -2,10 +2,12 @@ import { spawn } from 'node:child_process';
 import { appendFile, mkdtemp, readdir, readFile, rm, stat } from 'node:fs/promises';
 import { availableParallelism, tmpdir } from 'node:os';
 import { join } from 'node:path';
-import { FAILURE_RECORDS_ENV } from './test-failure-records.mjs';
+import { setTimeout as delay } from 'node:timers/promises';
+import { FAILURE_RECORDS_ENV, spawnLedgerPath } from './test-failure-records.mjs';
 import { classifyRerunOutcomes, formatFailureSummary } from './test-failure-summary.mjs';
 
 const SUMMARY_REPORTER = new URL('./test-summary-reporter.mjs', import.meta.url).href;
+const SPAWN_LEDGER = new URL('./test-spawn-ledger.mjs', import.meta.url).href;
 
 // A whole-suite run passes ~700 paths to one node process. Windows caps a
 // command line at 32767 characters (CreateProcess) and POSIX caps argv+env at
@@ -68,30 +70,282 @@ const positiveInt = (value) => {
 // `node --test` defaults to availableParallelism()-1 file processes. When the
 // run is one of several concurrent shell commands (the Mixdog shell tool
 // exports the admission cap as MIXDOG_SHELL_CONCURRENCY_CAP), that multiplies
-// across peers, so each run takes its share of the cores. Precedence: an
+// across peers, so a capped run leaves `cap` cores to them: cores - cap, never
+// below 4. Dividing the cores by the cap left a 20-core machine with 2 file
+// processes and tripled the wall time (desktop 289 s vs 63.5 s). Precedence: an
 // explicit --test-concurrency flag, then MIXDOG_TEST_CONCURRENCY, then the
-// share; with none of them Node's default stands and standalone runs are
-// unchanged. Returns the flag to add, or null.
+// capped share; with none of them Node's default (cores-1) stands.
+// Returns the flag to add, or null.
 export function testConcurrencyArg(nodeArgs, env = process.env, parallelism = availableParallelism()) {
-  if (nodeArgs.some((arg) => arg === '--test-concurrency' || arg.startsWith('--test-concurrency='))) return null;
+  if (nodeArgs.some(concurrencyFlag)) return null;
   const explicit = positiveInt(env.MIXDOG_TEST_CONCURRENCY);
   if (explicit) return `--test-concurrency=${explicit}`;
   const cap = positiveInt(env.MIXDOG_SHELL_CONCURRENCY_CAP);
   if (!cap) return null;
-  return `--test-concurrency=${Math.max(2, Math.floor(parallelism / cap))}`;
+  return `--test-concurrency=${Math.max(4, parallelism - cap)}`;
 }
 
-function spawnBatch(args, env) {
+// Every test file process is bounded: node:test fails a file (and records it
+// like any other failure) that is still running after --test-timeout, so a hang
+// costs minutes instead of waiting for the batch watchdog. A --test-timeout
+// flag wins, then MIXDOG_TEST_TIMEOUT_MS (0 turns the bound off), then the
+// caller's default. Returns the flag to add, or null.
+export function testTimeoutArg(nodeArgs, env = process.env, defaultMs = DEFAULT_TEST_TIMEOUT_MS) {
+  if (nodeArgs.some((arg) => arg === '--test-timeout' || arg.startsWith('--test-timeout='))) return null;
+  const raw = env.MIXDOG_TEST_TIMEOUT_MS;
+  if (raw !== undefined && String(raw).trim() === '0') return null;
+  const ms = positiveInt(raw) ?? defaultMs;
+  return ms ? `--test-timeout=${ms}` : null;
+}
+
+const concurrencyFlag = (arg) => arg === '--test-concurrency' || arg.startsWith('--test-concurrency=');
+
+// The file-process count a run was given (flag or env), else Node's default.
+function concurrencyOf(runArgs, fileArgs) {
+  const args = [...runArgs, ...fileArgs];
+  const index = args.findIndex(concurrencyFlag);
+  const value = index < 0 ? null : positiveInt(args[index].includes('=') ? args[index].split('=')[1] : args[index + 1]);
+  return value ?? Math.max(1, availableParallelism() - 1);
+}
+
+function withoutConcurrency(args) {
+  return args.filter((arg, i) => !concurrencyFlag(arg) && !(i > 0 && args[i - 1] === '--test-concurrency'));
+}
+
+// What keeps a test process off the signed-in user's machine state.
+export function testIsolationEnv(scratchDir) {
+  return {
+    TEMP: scratchDir,
+    TMP: scratchDir,
+    TMPDIR: scratchDir,
+    // A fixture must never write into the signed-in app's data (~/.mixdog):
+    // schedule/webhook logs, design-library bindings and the like resolve the
+    // default data dir. The desktop preload already does this per process.
+    MIXDOG_DATA_DIR: join(scratchDir, 'data'),
+    // Any session a test composes would otherwise read the real home's
+    // AGENTS.md / CLAUDE.md whenever the stored setting is on; tests that
+    // exercise the feature clear this and pass their own home.
+    MIXDOG_BOOT_INSTRUCTION_FILES: '0',
+  };
+}
+
+export const DEFAULT_TEST_TIMEOUT_MS = 120_000;
+export const LONG_TEST_TIMEOUT_MS = 600_000;
+
+// A batch that has not exited by then is killed with its process tree and
+// fails the run; MIXDOG_TEST_BATCH_TIMEOUT_MS overrides the default.
+const DEFAULT_BATCH_TIMEOUT_MS = 60 * 60 * 1000;
+const minutes = (ms) => Number((ms / 60_000).toFixed(2));
+
+// Runs a helper command to completion and returns its stdout.
+function capture(command, args) {
   return new Promise((resolve) => {
-    const child = spawn(process.execPath, args, { stdio: 'inherit', env });
+    const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'ignore'], windowsHide: true });
+    let out = '';
+    child.stdout.setEncoding('utf8');
+    child.stdout.on('data', (chunk) => {
+      out += chunk;
+    });
+    child.on('error', (error) => {
+      console.error(`${command} failed: ${error.message}`);
+      resolve('');
+    });
+    child.on('close', () => resolve(out));
+  });
+}
+
+// Kills a process and every descendant still linked to it. On POSIX a batch
+// leads its own process group (spawned detached), so the group goes with it.
+async function killTree(pid) {
+  if (process.platform === 'win32') {
+    await capture('taskkill', ['/T', '/F', '/PID', String(pid)]);
+    return;
+  }
+  for (const target of [-pid, pid]) {
+    try {
+      process.kill(target, 'SIGKILL');
+    } catch {
+      /* already gone */
+    }
+  }
+}
+
+// Every live process: { pid, ppid, created (epoch ms), exe ('' when unknown) }.
+// POSIX ps only reports elapsed seconds, so its creation times are ±1 s.
+async function processSnapshot() {
+  if (process.platform === 'win32') {
+    const out = await capture('powershell', [
+      '-NoProfile',
+      '-NonInteractive',
+      '-Command',
+      "Get-CimInstance Win32_Process | Where-Object CreationDate | ForEach-Object { '{0} {1} {2} {3}' -f $_.ProcessId, $_.ParentProcessId, ([DateTimeOffset]$_.CreationDate).ToUnixTimeMilliseconds(), $_.ExecutablePath }",
+    ]);
+    return out
+      .split(/\r?\n/)
+      .filter(Boolean)
+      .map((line) => {
+        const [pid, ppid, created, ...exe] = line.split(' ');
+        return { pid: Number(pid), ppid: Number(ppid), created: Number(created), exe: exe.join(' ') };
+      });
+  }
+  const now = Date.now();
+  const out = await capture('ps', ['-A', '-o', 'pid=,ppid=,etime=']);
+  return out
+    .split('\n')
+    .map((line) => line.trim().split(/\s+/))
+    .filter((fields) => fields.length === 3)
+    .map(([pid, ppid, etime]) => {
+      // [[dd-]hh:]mm:ss
+      const [days, clock] = etime.includes('-') ? etime.split('-') : ['0', etime];
+      const seconds = clock.split(':').reduce((total, part) => total * 60 + Number(part), Number(days) * 86_400);
+      return { pid: Number(pid), ppid: Number(ppid), created: now - seconds * 1000, exe: '' };
+    });
+}
+
+const CLOCK_SLACK_MS = 2000;
+
+// The processes this run owns, out of a snapshot: an anchor (the batch, or a
+// child a test process recorded in its spawn ledger) still alive under its
+// pid, the orphans an exited anchor left (they still name it as their parent;
+// a later process that reuses its pid only has younger children), every
+// descendant of those, and anything executing from the run's scratch root
+// (e.g. the postgres the memory runtime downloads into MIXDOG_DATA_DIR).
+// Command-line text never makes a process the run's: an editor or terminal
+// opened on the scratch root is not.
+export function ownedProcesses(snapshot, anchors, { batchStart, scratchDir, platform = process.platform }) {
+  const byPid = new Map(snapshot.map((entry) => [entry.pid, entry]));
+  const childrenOf = new Map();
+  for (const entry of snapshot) childrenOf.set(entry.ppid, [...(childrenOf.get(entry.ppid) ?? []), entry]);
+  const owned = new Map();
+  const queue = [];
+  const claim = (entry) => {
+    if (entry.pid === process.pid || owned.has(entry.pid)) return;
+    owned.set(entry.pid, entry);
+    queue.push(entry);
+  };
+  for (const { pid, since } of anchors) {
+    const holder = byPid.get(pid);
+    if (holder && holder.created >= batchStart - CLOCK_SLACK_MS && holder.created <= since + CLOCK_SLACK_MS) {
+      claim(holder);
+      continue;
+    }
+    for (const orphan of childrenOf.get(pid) ?? []) {
+      if (orphan.created >= since - CLOCK_SLACK_MS && (!holder || orphan.created < holder.created)) claim(orphan);
+    }
+  }
+  const fold = (path) => (platform === 'win32' ? path.toLowerCase() : path);
+  // An empty scratchDir skips the sweep: batches running side by side share
+  // one scratch root, so one batch's exit must not claim its peers' processes.
+  const scratchPrefix = scratchDir ? fold(join(scratchDir, '/')) : null;
+  if (scratchPrefix)
+    for (const entry of snapshot) if (entry.exe && fold(entry.exe).startsWith(scratchPrefix)) claim(entry);
+  while (queue.length > 0) {
+    const parent = queue.pop();
+    for (const entry of childrenOf.get(parent.pid) ?? []) {
+      if (entry.created >= parent.created - CLOCK_SLACK_MS) claim(entry);
+    }
+  }
+  return [...owned.values()];
+}
+
+async function readSpawnLedger(path) {
+  const text = await readFile(path, 'utf8').catch(() => '');
+  return text
+    .split('\n')
+    .filter(Boolean)
+    .map((line) => {
+      const [pid, since] = line.split(' ').map(Number);
+      return { pid, since };
+    });
+}
+
+// A process a test leaves running must not outlive its batch: it can hold a
+// port, a database, or files in the scratch root the run removes at the end.
+async function reapLeftovers(child, { batchStart, ledger, scratchDir }) {
+  // The batch itself has exited: only its process group can remain.
+  if (process.platform !== 'win32' && child.pid) {
+    try {
+      process.kill(-child.pid, 'SIGKILL');
+    } catch {
+      /* the group is empty */
+    }
+  }
+  const anchors = [...(child.pid ? [{ pid: child.pid, since: batchStart }] : []), ...(await readSpawnLedger(ledger))];
+  await killOwned(anchors, { batchStart, scratchDir });
+}
+
+async function killOwned(anchors, { batchStart, scratchDir }) {
+  const leftovers = ownedProcesses(await processSnapshot(), anchors, { batchStart, scratchDir });
+  if (leftovers.length === 0) return;
+  const pids = leftovers.map((entry) => entry.pid);
+  console.error(`Killing ${pids.length} process(es) a test left running: ${pids.join(' ')}`);
+  if (process.platform === 'win32') {
+    await capture('taskkill', ['/T', '/F', ...pids.flatMap((pid) => ['/PID', String(pid)])]);
+    return;
+  }
+  for (const pid of pids) {
+    try {
+      process.kill(pid, 'SIGKILL');
+    } catch {
+      /* already gone */
+    }
+  }
+}
+
+// Resolves { code, hung } once the batch exited and its leftovers are gone.
+// `scratchDir` '' = leave the scratch-root sweep to the caller (side-by-side batches).
+function spawnBatch(args, env, { files, ledger, scratchDir, timeoutMs }) {
+  return new Promise((resolve) => {
+    const batchStart = Date.now();
+    const child = spawn(process.execPath, args, { stdio: 'inherit', env, detached: process.platform !== 'win32' });
+    let hung = '';
+    const watchdog = setTimeout(() => {
+      hung = `hung after ${minutes(timeoutMs)} min: ${files.join(' ') || '(discovered files)'}`;
+      console.error(`Test batch ${hung}; killing its process tree`);
+      void killTree(child.pid);
+    }, timeoutMs);
+    // A detached POSIX batch no longer gets the terminal's signals itself.
+    const forward = (signal) => {
+      try {
+        process.kill(-child.pid, signal);
+      } catch {
+        /* already gone */
+      }
+    };
+    const signals = process.platform === 'win32' ? [] : ['SIGINT', 'SIGTERM'];
+    for (const signal of signals) process.on(signal, forward);
+    const settle = async (code) => {
+      clearTimeout(watchdog);
+      for (const signal of signals) process.off(signal, forward);
+      await reapLeftovers(child, { batchStart, ledger, scratchDir });
+      resolve({ code: hung ? 1 : code, hung });
+    };
     child.on('error', (error) => {
       console.error(error);
-      resolve(1);
+      void settle(1);
     });
-    child.on('exit', (code, signal) => resolve(code ?? (signal ? 1 : 0)));
+    child.on('exit', (code, signal) => void settle(code ?? (signal ? 1 : 0)));
   });
 }
 
+// rm's own maxRetries re-walks the tree on every nested retry, so one held
+// file a few directories down multiplied its delay into hours. Each pass here
+// is a single bounded walk.
+async function removeScratch(dir) {
+  for (let attempt = 1; ; attempt += 1) {
+    try {
+      await rm(dir, { recursive: true, force: true });
+      return;
+    } catch (error) {
+      if (attempt === 5) {
+        console.error(`Test scratch left behind (${error.code}): ${dir}`);
+        return;
+      }
+      await delay(200 * attempt);
+    }
+  }
+}
+
 // The failure reporter writes one JSON line per failed test; a spawn that
 // died before Node opened its destination leaves no file at all.
 async function readFailureRecords(path) {
@@ -116,10 +370,12 @@ async function readFailureRecords(path) {
 // flag was passed, does not fail the run; a test that fails every re-run
 // still exits 1. Every run ends with the failure summary block, so a filtered
 // log can still name what failed.
+//
+// `testTimeoutMs` is the per-file --test-timeout default (see testTimeoutArg).
 export async function runNodeTests(
   nodeArgs,
   fileArgs,
-  { argBudget = ARG_BUDGET, coverage = false, rerunFailed = 0 } = {}
+  { argBudget = ARG_BUDGET, coverage = false, rerunFailed = 0, testTimeoutMs = DEFAULT_TEST_TIMEOUT_MS } = {}
 ) {
   await pruneStaleTestLogs(tmpdir());
   const logPath = join(await mkdtemp(join(tmpdir(), TEST_LOG_PREFIX)), 'full.log');
@@ -133,9 +389,13 @@ export async function runNodeTests(
   const scratchDir = await mkdtemp(join(tmpdir(), 'mixdog-test-scratch-'));
   // The explicit entry also accepts Node options alongside its file paths.
   const concurrencyArg = testConcurrencyArg([...nodeArgs, ...fileArgs]);
+  const timeoutArg = testTimeoutArg([...nodeArgs, ...fileArgs], process.env, testTimeoutMs);
   const runArgs = [
     ...nodeArgs,
     ...(concurrencyArg ? [concurrencyArg] : []),
+    ...(timeoutArg ? [timeoutArg] : []),
+    // Node's test runner passes --import on to every test file process.
+    `--import=${SPAWN_LEDGER}`,
     `--test-reporter=${SUMMARY_REPORTER}`,
     '--test-reporter-destination=stdout',
     '--test-reporter=spec',
@@ -144,8 +404,19 @@ export async function runNodeTests(
     argCost(process.execPath) +
     runArgs.reduce((total, arg) => total + argCost(arg), 0) +
     argCost(`--test-reporter-destination=${logPath}`) +
-    argCost('.999'); // the per-batch log suffix
+    argCost('.999') + // the per-batch log suffix
+    argCost('--test-concurrency=999'); // the per-batch share when batches overlap
   const batches = chunkFileArgs(fileArgs, Math.max(argBudget - fixedCost, 1));
+  // Batches run side by side (a sequential second batch idled the machine
+  // behind the first one's slowest file), so the run's concurrency is split
+  // across them instead of multiplied.
+  const overlapped = batches.length > 1;
+  const totalConcurrency = overlapped ? concurrencyOf(runArgs, fileArgs) : 0;
+  const batchRunArgs = () => {
+    if (!overlapped) return runArgs;
+    const share = Math.max(1, Math.ceil(totalConcurrency / batches.length));
+    return [...withoutConcurrency(runArgs), `--test-concurrency=${share}`];
+  };
   const coverageDir = coverage ? await mkdtemp(join(tmpdir(), 'mixdog-v8-coverage-')) : '';
   // Node 24's test runner rebuilds each test process's execArgv from its option
   // table and drops V8 flags on the way, so a heap cap passed to the runner no
@@ -157,13 +428,7 @@ export async function runNodeTests(
   // already budgeted, and a third reporter would warn about listeners.
   const childEnv = (recordsPath) => ({
     ...process.env,
-    TEMP: scratchDir,
-    TMP: scratchDir,
-    TMPDIR: scratchDir,
-    // A fixture must never write into the signed-in app's data (~/.mixdog):
-    // schedule/webhook logs, design-library bindings and the like resolve the
-    // default data dir. The desktop preload already does this per process.
-    MIXDOG_DATA_DIR: join(scratchDir, 'data'),
+    ...testIsolationEnv(scratchDir),
     // Only the immutable validator binary is shared; fixture data stays private.
     // A gate supplies one directory across lanes; a standalone run owns its own.
     MIXDOG_TEST_OOXML_CACHE_DIR: process.env.MIXDOG_TEST_OOXML_CACHE_DIR || join(scratchDir, 'ooxml-cache'),
@@ -171,15 +436,35 @@ export async function runNodeTests(
     ...(coverageDir ? { NODE_V8_COVERAGE: coverageDir } : {}),
     ...(nodeOptions ? { NODE_OPTIONS: nodeOptions } : {}),
   });
+  const batchOptions = (files, recordsPath, sweepScratch = true) => ({
+    files,
+    ledger: spawnLedgerPath(recordsPath),
+    scratchDir: sweepScratch ? scratchDir : '',
+    timeoutMs: positiveInt(process.env.MIXDOG_TEST_BATCH_TIMEOUT_MS) ?? DEFAULT_BATCH_TIMEOUT_MS,
+  });
   let status = 0;
   const failures = [];
-  for (const [index, batch] of batches.entries()) {
-    const batchLog = batches.length === 1 ? logPath : `${logPath}.${index + 1}`;
-    const batchFailures = join(failureDir, `batch-${index + 1}.jsonl`);
-    const code = await spawnBatch(
-      [...runArgs, `--test-reporter-destination=${batchLog}`, ...batch],
-      childEnv(batchFailures)
-    );
+  const hung = [];
+  const batchLogOf = (index) => (batches.length === 1 ? logPath : `${logPath}.${index + 1}`);
+  const batchFailuresOf = (index) => join(failureDir, `batch-${index + 1}.jsonl`);
+  const outcomes = await Promise.all(
+    batches.map((batch, index) =>
+      spawnBatch(
+        [...batchRunArgs(), `--test-reporter-destination=${batchLogOf(index)}`, ...batch],
+        childEnv(batchFailuresOf(index)),
+        batchOptions(batch, batchFailuresOf(index), !overlapped)
+      )
+    )
+  );
+  // Peers were still running while each batch exited, so the scratch root's
+  // own processes (e.g. a postgres a test started) are swept once, here.
+  if (overlapped) await killOwned([], { batchStart: Date.now(), scratchDir });
+  // Logs and records merge in batch order, whatever order the batches finished.
+  for (const [index, outcome] of outcomes.entries()) {
+    const batchLog = batchLogOf(index);
+    const batchFailures = batchFailuresOf(index);
+    const { code } = outcome;
+    if (outcome.hung) hung.push(outcome.hung);
     if (batchLog !== logPath) {
       await appendFile(logPath, await readFile(batchLog));
       await rm(batchLog, { force: true });
@@ -196,10 +481,13 @@ export async function runNodeTests(
     console.error(`Re-running ${files.length} failed file(s): attempt ${attempt}/${rerunFailed}`);
     const attemptLog = `${logPath}.rerun-${attempt}`;
     const attemptFailures = join(failureDir, `rerun-${attempt}.jsonl`);
-    lastAttemptStatus = await spawnBatch(
+    const outcome = await spawnBatch(
       [...runArgs, `--test-reporter-destination=${attemptLog}`, ...files],
-      childEnv(attemptFailures)
+      childEnv(attemptFailures),
+      batchOptions(files, attemptFailures)
     );
+    lastAttemptStatus = outcome.code;
+    if (outcome.hung) hung.push(outcome.hung);
     await appendFile(logPath, await readFile(attemptLog));
     await rm(attemptLog, { force: true });
     attempts.push({ files, failures: await readFailureRecords(attemptFailures) });
@@ -209,17 +497,27 @@ export async function runNodeTests(
   // Flaky tolerance applies only to an ordinary test-failure exit that the
   // re-runs fully explained: nothing failed twice and the last re-run itself
   // came back clean.
-  if (rerunFailed > 0 && status === 1 && failed.length === 0 && flaky.length > 0 && lastAttemptStatus === 0) status = 0;
+  // A hung batch is never explained by re-runs.
+  if (
+    rerunFailed > 0 &&
+    status === 1 &&
+    hung.length === 0 &&
+    failed.length === 0 &&
+    flaky.length > 0 &&
+    lastAttemptStatus === 0
+  )
+    status = 0;
   process.exitCode = status;
 
   await rm(failureDir, { recursive: true, force: true });
-  // A process a test left running may still hold a file; what it holds stays.
-  await rm(scratchDir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }).catch(() => {});
+  // Leftover processes are already killed; what a process outside the run
+  // still holds stays behind.
+  await removeScratch(scratchDir);
   if (coverageDir) {
     const { reportCoverage } = await import('./coverage.mjs');
     await reportCoverage(coverageDir);
     await rm(coverageDir, { recursive: true, force: true });
   }
   // Last line of the run, after every reporter and the coverage report.
-  process.stdout.write(`\n${formatFailureSummary({ failed, flaky, rerunFailed })}`);
+  process.stdout.write(`\n${formatFailureSummary({ failed, flaky, rerunFailed, hung, exitCode: status })}`);
 }

package/scripts/lib/test-failure-records.mjs

@@ -13,6 +13,11 @@ import { relative } from 'node:path';
 
 export const FAILURE_RECORDS_ENV = 'MIXDOG_TEST_FAILURE_RECORDS';
 
+// Every child process a test process starts is logged beside its spawn's
+// failure records (scripts/lib/test-spawn-ledger.mjs), so the runner can tell
+// a process the run started from one that merely names its directories.
+export const spawnLedgerPath = (recordsPath) => (recordsPath ? `${recordsPath}.spawns` : '');
+
 // Ancestor names are tracked per file: node:test runs files concurrently, so
 // one flat stack would mix their nestings.
 export function failureRecord(event, stacks) {

package/scripts/lib/test-failure-summary.mjs

@@ -43,8 +43,19 @@ export function policyLine(rerunFailed) {
     : 'policy: --rerun-failed is off, so every failure fails the run (pass --rerun-failed <n> to classify flaky)';
 }
 
-export function formatFailureSummary({ failed = [], flaky = [], rerunFailed = 0 }) {
-  const lines = [`${SUMMARY_TAG} ${failed.length} failed, ${flaky.length} flaky (every failed test is named below)`];
+// `hung` holds one "hung after N min: <files>" message per batch the runner's
+// watchdog killed; such a batch fails the run whatever its tests reported.
+// A non-zero `exitCode` with no failed test, flaky test or hung batch behind it
+// (a bad node flag, a crashed runner) is its own failure, never "0 failed".
+export function formatFailureSummary({ failed = [], flaky = [], rerunFailed = 0, hung = [], exitCode = 0 }) {
+  const hungCount = hung.length > 0 ? `, ${hung.length} hung batch(es)` : '';
+  const unexplained = exitCode !== 0 && failed.length === 0 && hung.length === 0;
+  const lines = [
+    unexplained
+      ? `${SUMMARY_TAG} RUN FAILED: exit code ${exitCode} with no failed test recorded (bad argument or runner error; see output above)`
+      : `${SUMMARY_TAG} ${failed.length} failed, ${flaky.length} flaky${hungCount} (every failed test is named below)`,
+  ];
+  for (const message of hung) lines.push(`${SUMMARY_TAG} HUNG ${message}`);
   for (const record of failed) {
     const origin = record.firstSeenOnRerun ? ` (first seen on re-run ${record.firstSeenOnRerun})` : '';
     lines.push(`${SUMMARY_TAG} FAILED ${record.file} > ${record.name}${origin}`);
@@ -54,6 +65,6 @@ export function formatFailureSummary({ failed = [], flaky = [], rerunFailed = 0
       `${SUMMARY_TAG} FLAKY ${record.file} > ${record.name} (failed, then passed on re-run ${record.passedOnAttempt}/${rerunFailed})`
     );
   }
-  if (failed.length > 0 || flaky.length > 0) lines.push(`${SUMMARY_TAG} ${policyLine(rerunFailed)}`);
+  if (failed.length > 0 || flaky.length > 0 || hung.length > 0) lines.push(`${SUMMARY_TAG} ${policyLine(rerunFailed)}`);
   return `${lines.join('\n')}\n`;
 }

package/scripts/lib/test-spawn-ledger.mjs

@@ -0,0 +1,19 @@
+// Preloaded (--import) into every test process the runner starts: records
+// "<pid> <spawn time ms>" for each child process, so a process a test leaves
+// running can be traced back to the run after its parent exited
+// (scripts/lib/run-node-tests.mjs). Without the runner's environment it does
+// nothing.
+import { ChildProcess } from 'node:child_process';
+import { appendFileSync } from 'node:fs';
+import { FAILURE_RECORDS_ENV, spawnLedgerPath } from './test-failure-records.mjs';
+
+const ledger = spawnLedgerPath(process.env[FAILURE_RECORDS_ENV]);
+if (ledger) {
+  // spawn, exec, execFile and fork all start their process here.
+  const spawn = ChildProcess.prototype.spawn;
+  ChildProcess.prototype.spawn = function recordedSpawn(...args) {
+    const result = spawn.apply(this, args);
+    if (this.pid) appendFileSync(ledger, `${this.pid} ${Date.now()}\n`);
+    return result;
+  };
+}

package/scripts/merge-mac-update-feed.mjs

@@ -0,0 +1,107 @@
+#!/usr/bin/env node
+// electron-builder writes one latest-mac.yml per macOS build, and each runner
+// builds a single architecture, so each feed lists only its own zip and dmg.
+// electron-updater on macOS always reads the single latest-mac.yml of the
+// latest release and picks the zip whose name matches the machine's
+// architecture, so the release needs one feed that lists both builds.
+//
+// The publish job runs without installing dependencies, so this reads the
+// fixed shape electron-builder emits (top-level scalars plus a `files:` list
+// of `- url:` items) directly and fails on anything else.
+
+import { readFileSync, writeFileSync } from 'node:fs';
+import { pathToFileURL } from 'node:url';
+
+export const MAC_ARCHES = Object.freeze(['arm64', 'x64']);
+// Legacy top-level path/sha512 (read only by electron-updater 1.x) point at
+// the x64 zip, which also runs on Apple silicon through Rosetta.
+const PRIMARY_ARCH = 'x64';
+const TOP_LEVEL_KEYS = ['version', 'path', 'sha512', 'releaseDate'];
+
+function zipArch(url) {
+  const match = /^mixdog-desktop-mac-([a-z0-9]+)\.zip$/.exec(url);
+  return match ? match[1] : null;
+}
+
+/** Parse one electron-builder latest-mac.yml. File items keep their raw lines. */
+export function parseMacFeed(text, label) {
+  const top = {};
+  const files = [];
+  let inFiles = false;
+  for (const line of text.split(/\r?\n/)) {
+    if (!line.trim()) continue;
+    if (/^\S/.test(line)) {
+      const match = /^([A-Za-z][A-Za-z0-9]*):(?:\s+(.*))?$/.exec(line);
+      if (!match) throw new Error(`${label}: unsupported line: ${line}`);
+      inFiles = match[1] === 'files';
+      if (!inFiles) top[match[1]] = line;
+    } else if (inFiles && /^ {2}- url: \S+$/.test(line)) {
+      files.push({ url: line.replace(/^ {2}- url:\s+/, ''), lines: [line] });
+    } else if (inFiles && files.length > 0 && /^ {4}[A-Za-z][A-Za-z0-9]*: \S.*$/.test(line)) {
+      files[files.length - 1].lines.push(line);
+    } else {
+      throw new Error(`${label}: unsupported line: ${line}`);
+    }
+  }
+  for (const key of TOP_LEVEL_KEYS) {
+    if (!top[key]) throw new Error(`${label}: missing ${key}`);
+  }
+  const value = (key) => top[key].slice(key.length + 1).trim();
+  return { version: value('version'), top, files };
+}
+
+/** Merge the per-architecture feeds into one feed listing both zips. */
+export function mergeMacFeeds(inputs, expectedVersion) {
+  const feeds = inputs.map(({ label, text }) => {
+    const feed = parseMacFeed(text, label);
+    if (expectedVersion && feed.version !== expectedVersion) {
+      throw new Error(`${label}: version ${feed.version} is not ${expectedVersion}`);
+    }
+    const zips = feed.files.filter((file) => zipArch(file.url));
+    if (zips.length !== 1) throw new Error(`${label}: expected exactly one mac zip, found ${zips.length}`);
+    const [zip] = zips;
+    for (const field of ['sha512', 'size']) {
+      if (!zip.lines.some((line) => line.startsWith(`    ${field}: `))) {
+        throw new Error(`${label}: ${zip.url} has no ${field}`);
+      }
+    }
+    return { ...feed, arch: zipArch(zip.url) };
+  });
+  const arches = feeds.map((feed) => feed.arch).sort();
+  if (JSON.stringify(arches) !== JSON.stringify(MAC_ARCHES)) {
+    throw new Error(`Expected one feed per ${MAC_ARCHES.join(' + ')}, found: ${arches.join(', ')}`);
+  }
+  if (feeds[0].version !== feeds[1].version) {
+    throw new Error(`Feed versions differ: ${feeds[0].version} and ${feeds[1].version}`);
+  }
+  feeds.sort((a, b) => MAC_ARCHES.indexOf(a.arch) - MAC_ARCHES.indexOf(b.arch));
+  const entries = feeds.flatMap((feed) => feed.files);
+  const urls = entries.map((file) => file.url);
+  if (new Set(urls).size !== urls.length) throw new Error(`Duplicate feed files: ${urls.join(', ')}`);
+  const primary = feeds.find((feed) => feed.arch === PRIMARY_ARCH);
+  return [
+    primary.top.version,
+    'files:',
+    ...entries.flatMap((file) => file.lines),
+    primary.top.path,
+    primary.top.sha512,
+    primary.top.releaseDate,
+    '',
+  ].join('\n');
+}
+
+if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) {
+  const args = process.argv.slice(2);
+  const option = (name) => args.find((arg) => arg.startsWith(`--${name}=`))?.slice(name.length + 3);
+  const out = option('out');
+  const inputs = args.filter((arg) => !arg.startsWith('--'));
+  if (!out || inputs.length === 0) {
+    throw new Error('usage: merge-mac-update-feed.mjs --out=latest-mac.yml [--version=X.Y.Z] <feed.yml>...');
+  }
+  const merged = mergeMacFeeds(
+    inputs.map((path) => ({ label: path, text: readFileSync(path, 'utf8') })),
+    option('version')
+  );
+  writeFileSync(out, merged);
+  console.log(`Wrote ${out} from ${inputs.length} feeds.`);
+}