oxmutate 1.2.5 → 1.4.0

package/dist/coordinator.d.ts

@@ -23,6 +23,8 @@ export interface MutantResult extends NativeMutant {
     flaky?: boolean | undefined;
     /** Reused from the incremental cache, not run. */
     reused?: boolean | undefined;
+    /** Paths this mutant's run wrote outside the sandbox, found in its unit and put back (diskguard.ts). */
+    wrote?: string[] | undefined;
 }
 export interface TestInfo {
     id: string;
@@ -50,6 +52,8 @@ export interface RunStats {
     canaryFiles: number;
     /** Contexts restarted because a mutant left state behind (the per-execution canary fired). */
     pollutionRestarts: number;
+    /** Units after which the project on disk had changed (diskguard.ts): put back, and their verdicts and those of the units running beside them re-run. */
+    diskWrites: number;
     /** Survivors of the warm run that a whole-file re-run killed (cross-test effects). */
     survivorsOverturned: number;
     /** Watchdog timeouts that finished when re-run with 3x the budget: the machine was slow, not the mutant. */
@@ -62,6 +66,8 @@ export interface RunStats {
     flakyMutants: number;
     units: number;
     serversSpawned: number;
+    /** Mutants whose Timeout their worker called itself, the wait able to end by nothing (quiescence.ts): no watchdog wait, no respawn. */
+    deadlocks: number;
 }
 export interface RunResult {
     root: string;
@@ -86,4 +92,11 @@ export interface RunResult {
 }
 export declare class InstrumentFault extends Error {
 }
+/**
+ * No single test execution is waited on past this (owner, 2026-11-04: "we should not wait longer than
+ * that"). Every watchdog budget is capped at it per test the run executes, the 3x re-run of a busy stop
+ * included. A hang is mostly called by its worker in about half a second (quiescence.ts); the cap
+ * bounds the waits it cannot judge.
+ */
+export declare const MAX_TEST_MS = 5000;
 export declare function run(options: Options): Promise<RunResult>;

package/dist/coordinator.js

@@ -7,6 +7,7 @@ import { checkSurvivors } from './checker.js';
 import { runCommandMutants } from './command-runner.js';
 import { disables } from './config.js';
 import { rangeDisables } from './directives.js';
+import { guardDisk, isClean } from './diskguard.js';
 import { globToRegExp, matchFiles } from './files.js';
 import { changedLines, parseMutateSpec, shardOf, touches, withinRange } from './lanes.js';
 import { native } from './native.js';
@@ -27,7 +28,16 @@ function readRecords(file) {
 }
 const sha = (s) => createHash('sha256').update(s).digest('hex').slice(0, 16);
 const UNIT_PAIRS = 400;
+const DEADLOCK_REASON = 'its test deadlocked: nothing left in the worker could resume it (a hang, called in under a second)';
+function timeoutReason(v) {
+    if (v.deadlock)
+        return DEADLOCK_REASON;
+    if (v.idle)
+        return 'its test waited past the budget with the worker idle: a hang, not a slow test';
+    return 'stopped by the watchdog twice, the second time with 3x the budget';
+}
 const RERUN_SLACK = 3;
+export const MAX_TEST_MS = 5000;
 const MS_PER_MINUTE = 60_000;
 const PERCENT = 100;
 const allCovering = (m) => [...m.coveredBy, ...(m.specCoveredBy ?? [])];
@@ -44,7 +54,7 @@ export async function run(options) {
     const work = mkdtempSync(path.join(tmpdir(), 'oxmutate-'));
     const stats = {
         files: 0, mutants: 0, instrumentMs: 0, dryRunMs: 0, mutationMs: 0, staticMs: 0, staticRecollections: 0, isolatedMs: 0, wallMs: 0, testExecutions: 0,
-        watchdogKills: 0, idleStalls: 0, crashes: 0, canaryFiles: 0, pollutionRestarts: 0, survivorsOverturned: 0, timeoutsOverturned: 0, compileErrors: 0, isolatedRuns: 0, reused: 0, flakyTests: 0, flakyMutants: 0, units: 0, serversSpawned: 0,
+        watchdogKills: 0, idleStalls: 0, deadlocks: 0, crashes: 0, canaryFiles: 0, pollutionRestarts: 0, diskWrites: 0, survivorsOverturned: 0, timeoutsOverturned: 0, compileErrors: 0, isolatedRuns: 0, reused: 0, flakyTests: 0, flakyMutants: 0, units: 0, serversSpawned: 0,
     };
     let unitSeq = 0;
     const fileFor = (kind, ext) => path.join(work, `${kind}-${String(unitSeq++)}.${ext}`);
@@ -272,7 +282,7 @@ export async function run(options) {
         const hitLimitOf = (id) => Math.max(1000, (hits.get(id) ?? 1) * 100);
         const hitLimits = Object.fromEntries(toRun.map((m) => [m.id, hitLimitOf(m.id)]));
         const maxTestMs = (id) => { const m = byId.get(id); return Math.max(0, ...(m ? allCovering(m) : []).map((t) => testInfo.get(t)?.ms ?? 0)); };
-        const warmBudget = (id) => options.timeoutMS + Math.max(options.timeoutFactor, 3) * maxTestMs(id) + 250;
+        const warmBudget = (id) => Math.min(MAX_TEST_MS, options.timeoutMS + Math.max(options.timeoutFactor, 3) * maxTestMs(id) + 250);
         const fileOrder = new Map();
         for (const m of toRun) {
             const cost = new Map();
@@ -283,11 +293,12 @@ export async function run(options) {
             }
             fileOrder.set(m.id, [...cost].sort((a, b) => a[1] - b[1]).map(([f]) => f));
         }
+        const testMsOf = (plan) => Object.fromEntries(Object.keys(plan).map((t) => [t, testInfo.get(t)?.ms ?? 0]));
         const planUnits = new Map();
         const unfinished = new Set();
         const makeUnit = (file, plan, depth = 0) => {
             const unit = { mode: 'mutant', files: [file], results: fileFor('unit', 'jsonl'), progress: fileFor('unit', 'progress'), plan: fileFor('plan', 'json') };
-            writeFileSync(unit.plan, JSON.stringify({ tests: plan, hitLimits, skip: [], specTests: [...specOf.keys()], runOnce: dry.runOnce }));
+            writeFileSync(unit.plan, JSON.stringify({ tests: plan, hitLimits, skip: [], specTests: [...specOf.keys()], runOnce: dry.runOnce, testMs: testMsOf(plan) }));
             planUnits.set(unit.results, { file, plan, depth });
             stats.units++;
             return unit;
@@ -322,12 +333,58 @@ export async function run(options) {
             return units.sort((a, b) => weight(b) - weight(a));
         };
         const records = [];
+        const disk = env['OXMUTATE_NO_DISKGUARD'] ? undefined : await guardDisk(root);
+        const inFlight = new Set();
+        const tainted = new Set();
+        const writers = new Map();
+        const diskWrittenDuring = async (unit) => {
+            const change = await disk?.check();
+            const dirty = change !== undefined && !isClean(change);
+            if (dirty) {
+                stats.diskWrites++;
+                if (!disk?.restore(change)) {
+                    throw new InstrumentFault(`a mutant wrote outside the sandbox and the project could not be put back: ${[...change.changed, ...change.added, ...change.removed].join(', ')}. fix: restore them (git checkout / git clean), and see which test writes into the project`);
+                }
+                for (const other of inFlight)
+                    tainted.add(other);
+            }
+            if (!dirty && !tainted.delete(unit.results))
+                return undefined;
+            const entry = planUnits.get(unit.results);
+            if (!entry)
+                return [];
+            const ids = [...new Set(Object.values(entry.plan).flat())];
+            const paths = change ? [...change.changed, ...change.added, ...change.removed] : [];
+            if (dirty && ids.length === 1) {
+                const id = ids[0];
+                writers.set(id, paths);
+                if (inFlight.size === 0)
+                    records.push(...readRecords(unit.results));
+                else
+                    unfinished.add(id);
+                return [];
+            }
+            if (entry.depth >= 40) {
+                for (const id of ids)
+                    unfinished.add(id);
+                return [];
+            }
+            if (ids.length < 2)
+                return [makeUnit(entry.file, entry.plan, entry.depth + 1)];
+            const half = new Set(ids.slice(0, Math.ceil(ids.length / 2)));
+            const pick = (inHalf) => Object.fromEntries(Object.entries(entry.plan).map(([t, l]) => [t, l.filter((id) => half.has(id) === inHalf)]).filter(([, l]) => l.length > 0));
+            return [makeUnit(entry.file, pick(true), entry.depth + 1), makeUnit(entry.file, pick(false), entry.depth + 1)];
+        };
         const runaways = new Set();
         const crashedIds = new Set();
         const pool = createPool({ root, vitestConfig: options.vitestConfig, workers: 1, instrumentedMap: mapFile, nodeArgs: options.testRunnerNodeArgs, inlineDeps: true }, options.concurrency, work, 'mutant');
         pools.push(pool);
         const killedSoFar = (spec) => new Set([...runaways, ...crashedIds, ...records.filter((r) => r.type === 'mutant' && r.status !== 'Survived' && Boolean(r.spec) === spec).map((r) => r.id ?? '')]);
-        const afterUnit = (unit, outcome) => {
+        const afterUnit = async (unit, outcome) => {
+            inFlight.delete(unit.results);
+            const wrote = await diskWrittenDuring(unit);
+            if (wrote)
+                return wrote;
             const raw = readRecords(unit.results);
             const suspect = new Set(raw.filter((r) => r.type === 'polluted').flatMap((r) => (r.suspects ?? []).map((id) => `${id}|${r.test ?? ''}`)));
             const recs = raw.filter((r) => !(r.type === 'mutant' && suspect.has(`${r.id ?? ''}|${r.test ?? ''}`)));
@@ -370,6 +427,7 @@ export async function run(options) {
         progress.phase('warm runs', killedCount);
         const m0 = performance.now();
         const beforeUnit = (unit) => {
+            inFlight.add(unit.results);
             const entry = planUnits.get(unit.results);
             if (!entry)
                 return true;
@@ -381,10 +439,12 @@ export async function run(options) {
                 if (keep.length)
                     left[t] = keep;
             }
-            if (Object.keys(left).length === 0)
+            if (Object.keys(left).length === 0) {
+                inFlight.delete(unit.results);
                 return false;
+            }
             entry.plan = left;
-            writeFileSync(unit.plan, JSON.stringify({ tests: left, hitLimits, skip: [], specTests: [...specOf.keys()], runOnce: dry.runOnce }));
+            writeFileSync(unit.plan, JSON.stringify({ tests: left, hitLimits, skip: [], specTests: [...specOf.keys()], runOnce: dry.runOnce, testMs: testMsOf(left) }));
             return true;
         };
         await pool.drain(unitsForRound(toRun, (m) => fileOrder.get(m.id) ?? []), (_, id) => warmBudget(id), afterUnit, beforeUnit);
@@ -414,6 +474,8 @@ export async function run(options) {
             }
             else if (killers.length) {
                 m.status = killers.some((r) => r.status === 'Killed') ? 'Killed' : 'Timeout';
+                if (m.status === 'Timeout' && killers.some((r) => r.deadlock))
+                    m.statusReason = DEADLOCK_REASON;
                 m.killedBy = [...new Set(killers.map((r) => r.test ?? ''))];
                 m.testsCompleted = recs.length;
             }
@@ -452,13 +514,20 @@ export async function run(options) {
             const covering = byCost([...new Set(tests)]);
             return [...covering, ...byAffinity(m.file, [...loaders].filter((f) => !covering.includes(f)))];
         };
+        const testsRun = (id) => {
+            const m = byId.get(id);
+            const fs = m ? filesFor(m) : allTestFiles;
+            const covering = new Set(m && !m.static ? m.coveredBy : []);
+            return Math.max(1, dry.tests.filter((t) => fs.includes(t.file) && (covering.size === 0 || covering.has(t.id))).length);
+        };
         const isolatedBudget = (id) => {
             const m = byId.get(id);
             const fs = m ? filesFor(m) : allTestFiles;
             const covering = new Set(m && !m.static ? m.coveredBy : []);
             const ms = dry.tests.filter((t) => fs.includes(t.file) && (covering.size === 0 || covering.has(t.id))).reduce((sum, t) => sum + t.ms, 0);
-            return 1500 + options.timeoutMS + options.timeoutFactor * ms * 2;
+            return Math.min(MAX_TEST_MS * testsRun(id), 1500 + options.timeoutMS + options.timeoutFactor * ms * 2);
         };
+        const isolatedCapped = (id, slack) => Math.min(slack * isolatedBudget(id), MAX_TEST_MS * testsRun(id));
         const decideStaticLive = async (items) => {
             if (items.length === 0)
                 return [];
@@ -512,7 +581,7 @@ export async function run(options) {
             const open = (m) => !killed.has(m.id) && !back.has(m.id);
             const rounds = [(m) => filesFor(m).slice(0, 1), (m) => (open(m) ? filesFor(m).slice(1) : [])];
             for (const round of rounds)
-                await staticPool.drain(unitsFor(round), (_, id) => 2 * isolatedBudget(id), afterStatic, keepUndecided);
+                await staticPool.drain(unitsFor(round), (_, id) => isolatedCapped(id, 2), afterStatic, keepUndecided);
             const undecided = [];
             for (const m of items) {
                 const k = killed.get(m.id);
@@ -544,14 +613,14 @@ export async function run(options) {
             }
             const units = bins.sort((x, y) => y.load - x.load).map((b) => mk(b.batch));
             stats.isolatedRuns += items.length;
-            await pool.drain(units, (_, id) => slack * isolatedBudget(id), (unit, outcome) => {
+            await pool.drain(units, (_, id) => isolatedCapped(id, slack), (unit, outcome) => {
                 for (const rec of readRecords(unit.results)) {
                     if (rec.type !== 'isolated' || !rec.id)
                         continue;
                     const prev = verdicts.get(rec.id);
                     const testsCompleted = (prev?.testsCompleted ?? 0) + (rec.ran ?? 0);
                     if (rec.timeout)
-                        verdicts.set(rec.id, { status: 'Timeout', testsCompleted });
+                        verdicts.set(rec.id, { status: 'Timeout', testsCompleted, ...(rec.deadlock && { idle: true, deadlock: true }) });
                     else if (rec.killed)
                         verdicts.set(rec.id, { status: 'Killed', killedBy: [...(prev?.killedBy ?? []), ...(rec.failed ?? [])], testsCompleted });
                     else
@@ -595,7 +664,7 @@ export async function run(options) {
                 m.killedBy = v.killedBy;
                 m.testsCompleted = v.testsCompleted;
                 if (v.status === 'Timeout')
-                    m.statusReason = v.idle ? 'its test waited past the budget with the worker idle: a hang, not a slow test' : 'stopped by the watchdog twice, the second time with 3x the budget';
+                    m.statusReason = timeoutReason(v);
             }
         }
         const survivors = toRun.filter((m) => m.status === 'Survived' && new Set(m.coveredBy).size > 1);
@@ -651,6 +720,12 @@ export async function run(options) {
         stats.isolatedMs = performance.now() - i0;
         stats.watchdogKills = pool.stats.killed;
         stats.idleStalls = pool.stats.idle;
+        for (const [id, paths] of writers) {
+            const m = byId.get(id);
+            if (m)
+                m.wrote = paths;
+        }
+        stats.deadlocks = mutants.filter((m) => m.statusReason === DEADLOCK_REASON).length;
         stats.crashes = pool.stats.crashed;
         if (options.incremental) {
             mkdirSync(path.dirname(options.incrementalFile), { recursive: true });

package/dist/diskguard.js

@@ -0,0 +1,92 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { ambientRuntime, run } from 'bellpull';
+export const MAX_SNAPSHOT_BYTES = 64 * 1024 * 1024;
+export const isClean = (c) => c.changed.length + c.added.length + c.removed.length === 0;
+async function git(cwd, args) {
+    const r = await run('git', args, { cwd, runtime: ambientRuntime() }).catch(() => undefined);
+    return r?.ok ? r.stdout : undefined;
+}
+export async function guardDisk(root, maxSnapshotBytes = MAX_SNAPSHOT_BYTES) {
+    const listed = await git(root, ['ls-files', '-z', '--cached', '--', '.']);
+    if (listed === undefined)
+        return undefined;
+    const files = listed.split('\0').filter(Boolean);
+    const tracked = new Map();
+    let bytes = 0;
+    for (const rel of files) {
+        const abs = path.join(root, rel);
+        if (!existsSync(abs))
+            continue;
+        const st = statSync(abs);
+        if (!st.isFile())
+            continue;
+        bytes += st.size;
+        tracked.set(rel, { size: st.size, mtimeMs: st.mtimeMs });
+    }
+    const restorable = bytes <= maxSnapshotBytes;
+    if (restorable)
+        for (const [rel, t] of tracked)
+            t.content = readFileSync(path.join(root, rel));
+    const dirs = new Map();
+    for (const rel of tracked.keys()) {
+        const dir = path.dirname(rel);
+        if (!dirs.has(dir))
+            dirs.set(dir, new Set(readdirSync(path.join(root, dir))));
+    }
+    const knownIgnored = new Map();
+    const ignored = async (rels) => {
+        const ask = rels.filter((rel) => !knownIgnored.has(rel));
+        if (ask.length) {
+            const r = await run('git', ['check-ignore', '--', ...ask], { cwd: root, runtime: ambientRuntime() }).catch(() => undefined);
+            const yes = new Set((r?.stdout ?? '').split('\n').filter(Boolean));
+            for (const rel of ask)
+                knownIgnored.set(rel, yes.has(rel));
+        }
+        return new Set(rels.filter((rel) => knownIgnored.get(rel)));
+    };
+    return {
+        async check() {
+            const changed = [];
+            const removed = [];
+            for (const [rel, t] of tracked) {
+                const abs = path.join(root, rel);
+                if (!existsSync(abs)) {
+                    removed.push(rel);
+                    continue;
+                }
+                const st = statSync(abs);
+                if (st.size !== t.size || st.mtimeMs !== t.mtimeMs)
+                    changed.push(rel);
+            }
+            const fresh = [];
+            for (const [dir, before] of dirs) {
+                const abs = path.join(root, dir);
+                if (!existsSync(abs))
+                    continue;
+                for (const name of readdirSync(abs))
+                    if (!before.has(name))
+                        fresh.push(path.join(dir, name));
+            }
+            const skip = await ignored(fresh);
+            return { changed, added: fresh.filter((rel) => !skip.has(rel)), removed };
+        },
+        restore(change) {
+            if (!restorable)
+                return false;
+            for (const rel of [...change.changed, ...change.removed]) {
+                const t = tracked.get(rel);
+                if (!t?.content)
+                    return false;
+                mkdirSync(path.dirname(path.join(root, rel)), { recursive: true });
+                writeFileSync(path.join(root, rel), t.content);
+                const st = statSync(path.join(root, rel));
+                t.mtimeMs = st.mtimeMs;
+                t.size = st.size;
+            }
+            for (const rel of change.added)
+                rmSync(path.join(root, rel), { recursive: true, force: true });
+            return true;
+        },
+    };
+}

package/dist/quiescence.js

@@ -0,0 +1,189 @@
+import { syncBuiltinESMExports } from 'node:module';
+import { performance } from 'node:perf_hooks';
+import timers from 'node:timers';
+import promiseTimers from 'node:timers/promises';
+import { promisify } from 'node:util';
+import { env, stderr } from './runtime.js';
+export const SAMPLE_MS = 100;
+export const QUIET_SELF_SHARE = 0.05;
+export const QUIET_OTHER_SHARE = 0.25;
+export const MIN_CONFIRM_MS = 500;
+export class DeadlockError extends Error {
+    quietMs;
+    constructor(quietMs) {
+        super(`oxmutate: deadlocked for ${String(Math.round(quietMs))} ms: nothing left in this worker could resume the test`);
+        this.quietMs = quietMs;
+        this.name = 'DeadlockError';
+    }
+}
+export const isDeadlock = (e) => e instanceof DeadlockError;
+const { threadCpuUsage: threadCpu } = process;
+const threadCpuUsage = threadCpu?.bind(process);
+const { setInterval: realSetInterval, clearInterval: realClearInterval } = globalThis;
+const realNow = performance.now.bind(performance);
+const US_PER_MS = 1000;
+function resources() {
+    const counts = new Map();
+    for (const r of process.getActiveResourcesInfo())
+        counts.set(r, (counts.get(r) ?? 0) + 1);
+    return counts;
+}
+export const withinFloor = (now, floor) => [...now].every(([kind, n]) => n <= (floor.get(kind) ?? 0));
+export const quiet = (r) => r.selfShare < QUIET_SELF_SHARE && r.otherShare < QUIET_OTHER_SHARE && r.rpcInFlight === 0 && r.pendingTimers === 0 && r.withinFloor;
+let rpcInFlight = 0;
+let pendingTimers = 0;
+const live = new Map();
+const done = (id) => {
+    if (live.delete(id))
+        pendingTimers--;
+};
+const WRAPPED = Symbol.for('oxmutate.timers');
+const isWrapped = (fn) => typeof fn === 'function' && WRAPPED in fn;
+const mark = (fn) => Object.assign(fn, { [WRAPPED]: true });
+const clearing = (clear) => (handle) => {
+    if (handle !== undefined && handle !== null)
+        done(Number(handle));
+    clear(handle);
+};
+function pending(handle) {
+    live.set(Number(handle), true);
+    pendingTimers++;
+    return handle;
+}
+function oneShot(start, cb, args) {
+    const handle = start(() => {
+        done(Number(handle));
+        cb(...args);
+    });
+    return pending(handle);
+}
+function counted(p) {
+    pendingTimers++;
+    return p.finally(() => pendingTimers--);
+}
+function standIn(wrapper, original) {
+    const custom = Reflect.get(original, promisify.custom);
+    if (custom !== undefined)
+        Reflect.defineProperty(wrapper, promisify.custom, { value: custom });
+    return mark(wrapper);
+}
+export function wrapTimers(api) {
+    if (isWrapped(api.setTimeout))
+        return;
+    const { setTimeout: later, setImmediate: soon, setInterval: every, clearTimeout: unlater, clearImmediate: unsoon, clearInterval: unevery } = api;
+    const timeout = (cb, ms, ...args) => oneShot((fire) => later(fire, ms), cb, args);
+    const immediate = (cb, ...args) => oneShot((fire) => soon(fire), cb, args);
+    const interval = (cb, ms, ...args) => pending(every(cb, ms, ...args));
+    api.setTimeout = standIn(timeout, later);
+    api.setImmediate = standIn(immediate, soon);
+    api.setInterval = standIn(interval, every);
+    api.clearTimeout = clearing(unlater);
+    api.clearImmediate = clearing(unsoon);
+    api.clearInterval = clearing(unevery);
+}
+export function wrapPromiseTimers(api) {
+    if (isWrapped(api.setTimeout))
+        return;
+    const { setTimeout: later, setImmediate: soon } = api;
+    api.setTimeout = mark((...a) => counted(later(...a)));
+    api.setImmediate = mark((...a) => counted(soon(...a)));
+}
+export function trackTimers() {
+    wrapTimers(globalThis);
+    wrapTimers(timers);
+    wrapPromiseTimers(promiseTimers);
+    syncBuiltinESMExports();
+}
+export const pendingTimerCount = () => pendingTimers;
+export function trackRpc(state) {
+    const rpc = state?.rpc;
+    if (!rpc || rpc.__oxmutateTracked)
+        return;
+    state.rpc = new Proxy(rpc, {
+        get(target, key, receiver) {
+            if (key === '__oxmutateTracked')
+                return true;
+            const value = Reflect.get(target, key, receiver);
+            if (typeof value !== 'function' || typeof key !== 'string' || key.startsWith('$'))
+                return value;
+            const counted = (...args) => {
+                const result = Reflect.apply(value, target, args);
+                if (!(result instanceof Promise))
+                    return result;
+                rpcInFlight++;
+                return result.finally(() => rpcInFlight--);
+            };
+            return Object.defineProperties(counted, Object.getOwnPropertyDescriptors(value));
+        },
+    });
+}
+let floor;
+export const floorTaken = () => floor;
+export const rpcInFlightCount = () => rpcInFlight;
+export function takeFloor() {
+    floor ??= resources();
+}
+export function confirmStep(o) {
+    if (!o.quiet)
+        return { quietSince: undefined };
+    const since = o.quietSince ?? o.now - o.span;
+    const open = o.now - since;
+    return open >= o.confirmMs ? { quietSince: since, deadlockMs: open } : { quietSince: since };
+}
+export function watch() {
+    let armed;
+    let quietSince;
+    let self = threadCpuUsage?.();
+    let all = process.cpuUsage();
+    let at = realNow();
+    const timer = realSetInterval(() => {
+        const now = realNow();
+        const span = Math.max(1, now - at);
+        const selfDelta = threadCpuUsage?.(self);
+        const allDelta = process.cpuUsage(all);
+        self = threadCpuUsage?.();
+        all = process.cpuUsage();
+        at = now;
+        if (!armed || !selfDelta || !floor)
+            return;
+        const selfUs = selfDelta.user + selfDelta.system;
+        const reading = {
+            selfShare: selfUs / US_PER_MS / span,
+            otherShare: Math.max(0, allDelta.user + allDelta.system - selfUs) / US_PER_MS / span,
+            rpcInFlight,
+            pendingTimers,
+            withinFloor: withinFloor(resources(), floor),
+        };
+        if (env['OXMUTATE_DEBUG_QUIET'])
+            stderr.write(`quiet? ${JSON.stringify(reading)} res=${JSON.stringify([...resources()])} floor=${JSON.stringify([...floor])}\n`);
+        const next = confirmStep({ quietSince, quiet: quiet(reading), now, span, confirmMs: armed.confirmMs });
+        quietSince = next.quietSince;
+        if (next.deadlockMs !== undefined) {
+            const { reject } = armed;
+            armed = undefined;
+            reject(new DeadlockError(next.deadlockMs));
+        }
+    }, SAMPLE_MS);
+    timer.unref();
+    return {
+        async guard(fn, confirmMs) {
+            const pending = Promise.resolve().then(fn);
+            if (!threadCpuUsage)
+                return pending;
+            const deadlock = new Promise((_, reject) => {
+                quietSince = undefined;
+                armed = { reject, confirmMs: Math.max(MIN_CONFIRM_MS, confirmMs) };
+            });
+            try {
+                return await Promise.race([pending, deadlock]);
+            }
+            finally {
+                armed = undefined;
+                pending.catch(() => undefined);
+            }
+        },
+        stop() {
+            realClearInterval(timer);
+        },
+    };
+}

package/dist/reports.js

@@ -95,6 +95,8 @@ export function describe(m) {
 function corrections(s) {
     return [
         s.watchdogKills && `${String(s.watchdogKills)} runaway(s) stopped`,
+        s.deadlocks && `${String(s.deadlocks)} deadlock(s) called by their worker`,
+        s.diskWrites && `${String(s.diskWrites)} write(s) outside the sandbox put back, their units re-run`,
         s.pollutionRestarts && `${String(s.pollutionRestarts)} polluted context(s) restarted (in ${String(s.canaryFiles)} file(s))`,
         s.flakyMutants && `${String(s.flakyMutants)} flaky verdict(s)`,
         s.survivorsOverturned && `${String(s.survivorsOverturned)} survivor(s) killed by a whole-file re-run`,

package/dist/runner.d.ts

@@ -50,6 +50,12 @@ export default class OxmutateRunner extends OxmutateRunner_base {
      * mutant is later re-run against the files that load its module, not the whole suite (Stryker
      * re-runs everything for each one).
      */
+    /**
+     * Before any test file is imported: count the timers it and the code under test will create, count
+     * Vitest's calls to its main thread, and take the resource floor a deadlock is measured against
+     * (quiescence.ts).
+     */
+    onBeforeCollect(paths: string[]): unknown;
     onCollectStart(file: AnyTask): unknown;
     onBeforeTryTask(test: AnyTask, options: {
         retry: number;

package/dist/runner.js

@@ -1,7 +1,9 @@
 import { appendFileSync, readFileSync, writeFileSync } from 'node:fs';
 import { createRequire } from 'node:module';
+import { availableParallelism, loadavg } from 'node:os';
 import path from 'node:path';
 import { BEAT_ENV, markTest, realNow, startHeartbeat } from './heartbeat.js';
+import { isDeadlock, takeFloor, trackRpc, trackTimers, watch } from './quiescence.js';
 import { resolveEsm } from './resolve.js';
 import { ACTIVE_MUTANT_ENV, FILE_KEY, NAMESPACE, cwd, env } from './runtime.js';
 const fromProject = async (specifier, dir = cwd()) => import(resolveEsm(specifier, dir));
@@ -51,6 +53,11 @@ if (mode === 'isolated' && env['OXMUTATE_HIT_LIMIT']) {
     ns.hitLimit = Number(env['OXMUTATE_HIT_LIMIT']);
 }
 const ranAway = () => ns.hitCount !== undefined && ns.hitLimit !== undefined && ns.hitCount > ns.hitLimit;
+const deadlocks = mode === 'dry' ? undefined : watch();
+let deadlocked = false;
+let tornDown = false;
+const confirmFor = (testId) => 2 * (plan.testMs?.[testId ?? ''] ?? 0) * Math.max(1, (loadavg()[0] ?? 0) / availableParallelism());
+const guarded = (testId, fn) => (deadlocks ? deadlocks.guard(fn, confirmFor(testId)) : Promise.resolve().then(fn));
 const PROTOTYPES = {
     Object: Object.prototype, Array: Array.prototype, String: String.prototype, Number: Number.prototype,
     Boolean: Boolean.prototype, Function: Function.prototype, Promise: Promise.prototype, Map: Map.prototype,
@@ -197,14 +204,19 @@ async function runCollected(file, runnable) {
                 const cleanups = await beforeEach(t, t.context);
                 ran++;
                 try {
-                    await fn();
+                    await guarded(t.id, fn);
                 }
                 catch (e) {
+                    if (isDeadlock(e))
+                        deadlocked = true;
                     failed.push(t.id);
                     note(t, e);
+                    if (isDeadlock(e))
+                        return;
                 }
                 finally {
-                    await afterEach(t, t.context, cleanups);
+                    if (!deadlocked)
+                        await afterEach(t, t.context, cleanups);
                 }
             }
         }
@@ -229,6 +241,14 @@ function fullName(t) {
     return parts.join(' > ');
 }
 export default class OxmutateRunner extends Base {
+    onBeforeCollect(paths) {
+        if (deadlocks) {
+            trackTimers();
+            trackRpc(this.workerState);
+            takeFloor();
+        }
+        return super.onBeforeCollect?.(paths);
+    }
     onCollectStart(file) {
         if (mode === 'dry')
             ns.currentTestId = `${FILE_KEY}${file.filepath ?? file.file?.filepath ?? ''}`;
@@ -248,15 +268,22 @@ export default class OxmutateRunner extends Base {
                 return;
             markTest(true);
             try {
-                await fn();
+                await guarded(test.id, fn);
                 if (mode === 'dry')
                     await probe(test, fn);
             }
+            catch (e) {
+                if (isDeadlock(e))
+                    deadlocked = true;
+                throw e;
+            }
             finally {
                 markTest(false);
             }
             return;
         }
+        if (tornDown)
+            return;
         let current = test;
         let run = fn;
         let context = test.context;
@@ -267,7 +294,7 @@ export default class OxmutateRunner extends Base {
         let cleanups = [];
         let attempt = 0;
         const teardown = async () => {
-            await afterEach(current, context, cleanups);
+            await guarded(current.id, () => afterEach(current, context, cleanups));
             cleanups = [];
         };
         const setup = async () => {
@@ -280,7 +307,7 @@ export default class OxmutateRunner extends Base {
                 context = copy.context;
             }
             invoked = true;
-            cleanups = await beforeEach(current, context);
+            cleanups = await guarded(current.id, () => beforeEach(current, context));
             await this.retryReset(test, ++attempt);
         };
         const fresh = async () => {
@@ -288,62 +315,96 @@ export default class OxmutateRunner extends Base {
             await setup();
         };
         let unchecked = [];
+        const pollutedBy = (e) => {
+            polluted = true;
+            canaryEvery = 1;
+            emit({ type: 'polluted', id: unchecked[0], suspects: unchecked.slice(1), test: test.id, file: test.file?.filepath, error: e instanceof Error ? e.message.slice(0, POLLUTION_ERROR_CHARS) : String(e) });
+            return false;
+        };
         const canary = async () => {
             if (!unchecked.length)
                 return true;
-            await fresh();
             try {
-                await run();
+                await fresh();
+            }
+            catch (e) {
+                if (!isDeadlock(e))
+                    throw e;
+                return pollutedBy(e);
+            }
+            try {
+                await guarded(test.id, run);
                 unchecked = [];
                 canaryEvery = Math.min(MAX_CANARY_EVERY, canaryEvery * 2);
                 return true;
             }
             catch (e) {
-                polluted = true;
-                canaryEvery = 1;
-                emit({ type: 'polluted', id: unchecked[0], suspects: unchecked.slice(1), test: test.id, file: test.file?.filepath, error: e instanceof Error ? e.message.slice(0, POLLUTION_ERROR_CHARS) : String(e) });
-                return false;
+                return pollutedBy(e);
             }
         };
         const runArmed = async (id) => {
-            await teardown();
+            try {
+                await teardown();
+            }
+            catch (e) {
+                if (!isDeadlock(e))
+                    throw e;
+                pollutedBy(e);
+                return { status: 'Polluted', ms: 0 };
+            }
             const t0 = performance.now();
             ns.activeMutant = id;
             ns.hitCount = 0;
             ns.hitLimit = plan.hitLimits[id] ?? Number.MAX_SAFE_INTEGER;
             let status = 'Survived';
+            let deadlock = false;
             try {
                 await setup();
-                await run();
+                await guarded(current.id, run);
             }
             catch (e) {
-                status = isHitLimit(e) ? 'Timeout' : 'Killed';
+                deadlock = isDeadlock(e);
+                status = isHitLimit(e) || deadlock ? 'Timeout' : 'Killed';
             }
             finally {
                 ns.activeMutant = undefined;
                 ns.hitCount = undefined;
                 ns.hitLimit = undefined;
             }
-            return { status, ms: performance.now() - t0 };
+            return { status, ms: performance.now() - t0, ...(deadlock && { deadlock: true }) };
         };
         for (const id of ids) {
             if (done.has(id))
                 continue;
             if (progressPath)
                 writeFileSync(progressPath, `${id} ${String(realNow())}\n`);
-            let { status, ms } = await runArmed(id);
+            let { status, ms, deadlock } = await runArmed(id);
+            if (status === 'Polluted')
+                break;
             if (status !== 'Survived' && unchecked.length > 0) {
                 if (!(await canary()))
                     break;
-                ({ status, ms } = await runArmed(id));
+                ({ status, ms, deadlock } = await runArmed(id));
+                if (status === 'Polluted')
+                    break;
             }
             if (status !== 'Survived')
                 done.add(id);
-            emit({ type: 'mutant', id, test: test.id, status, ms, ...(spec ? { spec: true } : {}) });
+            emit({ type: 'mutant', id, test: test.id, status, ms, ...(spec ? { spec: true } : {}), ...(deadlock && { deadlock: true }) });
             unchecked.push(id);
+            if (deadlock) {
+                pollutedBy(new Error('torn down: a deadlocked run is still pending in this worker'));
+                tornDown = true;
+                break;
+            }
             if (unchecked.length >= canaryEvery && !(await canary()))
                 break;
         }
+        if (tornDown) {
+            if (progressPath)
+                writeFileSync(progressPath, '');
+            return;
+        }
         if (!polluted)
             await canary();
         if (progressPath && ids.length)
@@ -407,11 +468,12 @@ export default class OxmutateRunner extends Base {
             ns.activeMutant = armed;
             ns.hitCount = armed ? 0 : undefined;
             ns.hitLimit = armed ? (staticLimits[armed] ?? Number.MAX_SAFE_INTEGER) : undefined;
+            deadlocked = false;
             try {
                 vi.resetModules();
                 const [file] = await collectTests([filepath], this);
                 const result = file ? await runCollected(file, runnable) : { failed: [filepath], ran: 0 };
-                return { ...result, timeout: ranAway() };
+                return { ...result, timeout: ranAway() || deadlocked, ...(deadlocked ? { deadlock: true } : {}) };
             }
             finally {
                 ns.activeMutant = undefined;
@@ -431,6 +493,8 @@ export default class OxmutateRunner extends Base {
             if (progressPath)
                 writeFileSync(progressPath, `${id} ${String(realNow())}\n`);
             held.push({ id, verdict: await evaluate(id) });
+            if (held.at(-1)?.verdict.deadlock)
+                return back('torn down: a deadlocked run is still pending in this worker');
             const left = snapshotDiff(before, snapshot());
             if (left.length)
                 return back('the mutant left a trace', left);
@@ -444,7 +508,7 @@ export default class OxmutateRunner extends Base {
             if (after.length)
                 return back('the unarmed re-collection left a trace', after);
             for (const { id: done, verdict } of held)
-                emit({ type: 'isolated', id: done, killed: verdict.failed.length > 0 && !verdict.timeout, failed: verdict.failed, ran: verdict.ran, ...(verdict.timeout && { timeout: true }) });
+                emit({ type: 'isolated', id: done, killed: verdict.failed.length > 0 && !verdict.timeout, failed: verdict.failed, ran: verdict.ran, ...(verdict.timeout && { timeout: true }), ...(verdict.deadlock && { deadlock: true }) });
             held = [];
             every = Math.min(STATIC_CANARY_MAX, every * 2);
         }
@@ -528,8 +592,8 @@ export default class OxmutateRunner extends Base {
                 if (f.result?.state === 'fail' && !failed.length)
                     failed.push(f.id);
             }
-            const timeout = ranAway();
-            emit({ type: 'isolated', id: env[ACTIVE_MUTANT_ENV] ?? ns.activeMutant, killed: failed.length > 0 && !timeout, failed, ran, ...(timeout && { timeout }) });
+            const timeout = ranAway() || deadlocked;
+            emit({ type: 'isolated', id: env[ACTIVE_MUTANT_ENV] ?? ns.activeMutant, killed: failed.length > 0 && !timeout, failed, ran, ...(timeout && { timeout }), ...(deadlocked && { deadlock: true }) });
         }
         return super.onAfterRunFiles?.(files);
     }

package/dist/scheduler.js

@@ -172,7 +172,7 @@ export function createPool(config, size, workDir, tag) {
                     if (outcome.killedFor || outcome.crashed) {
                         servers[slot] = replacement();
                     }
-                    const more = after(unit, outcome);
+                    const more = await after(unit, outcome);
                     queue.unshift(...more);
                     busy--;
                     signal();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "oxmutate",
-  "version": "1.2.5",
+  "version": "1.4.0",
   "description": "Mutation testing for JavaScript and TypeScript on the oxc engine, run warm inside the project's own Vitest, with CI lanes, SARIF and an agent surface. No dependency outside the burgee family. A Stryker migration path.",
   "license": "MIT",
   "repository": {