oxmutate 1.3.0 → 1.5.0

package/dist/coordinator.d.ts

@@ -23,6 +23,8 @@ export interface MutantResult extends NativeMutant {
     flaky?: boolean | undefined;
     /** Reused from the incremental cache, not run. */
     reused?: boolean | undefined;
+    /** Paths this mutant's run wrote outside the sandbox, found in its unit and put back (diskguard.ts). */
+    wrote?: string[] | undefined;
 }
 export interface TestInfo {
     id: string;
@@ -48,8 +50,13 @@ export interface RunStats {
     idleStalls: number;
     crashes: number;
     canaryFiles: number;
+    /** Of the verdicts the incremental cache would have reused, how many were run afresh to check it, and how many disagreed on detected or not. */
+    auditSampled: number;
+    auditDisagreed: number;
     /** Contexts restarted because a mutant left state behind (the per-execution canary fired). */
     pollutionRestarts: number;
+    /** Units after which the project on disk had changed (diskguard.ts): put back, and their verdicts and those of the units running beside them re-run. */
+    diskWrites: number;
     /** Survivors of the warm run that a whole-file re-run killed (cross-test effects). */
     survivorsOverturned: number;
     /** Watchdog timeouts that finished when re-run with 3x the budget: the machine was slow, not the mutant. */

package/dist/coordinator.js

@@ -7,10 +7,12 @@ import { checkSurvivors } from './checker.js';
 import { runCommandMutants } from './command-runner.js';
 import { disables } from './config.js';
 import { rangeDisables } from './directives.js';
+import { guardDisk, isClean } from './diskguard.js';
 import { globToRegExp, matchFiles } from './files.js';
 import { changedLines, parseMutateSpec, shardOf, touches, withinRange } from './lanes.js';
 import { native } from './native.js';
 import { startProgress } from './progress.js';
+import { VERSION } from './reports.js';
 import { FILE_KEY, env, stderr } from './runtime.js';
 import { createPool } from './scheduler.js';
 import { summarizeSpec, tracesFor } from './spec-oracle.js';
@@ -26,7 +28,26 @@ function readRecords(file) {
         .map((l) => JSON.parse(l));
 }
 const sha = (s) => createHash('sha256').update(s).digest('hex').slice(0, 16);
+const LOCKFILES = ['package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'bun.lock', 'bun.lockb'];
+function lockfileHash(root) {
+    const found = LOCKFILES.map((f) => path.join(root, f)).find((f) => existsSync(f));
+    if (found)
+        return sha(readFileSync(found, 'utf8'));
+    const above = path.dirname(root);
+    return above === root ? 'none' : lockfileHash(above);
+}
+const verdictOptions = (o) => ({
+    timeoutMS: o.timeoutMS,
+    timeoutFactor: o.timeoutFactor,
+    excludedMutations: [...o.excludedMutations].sort(),
+    ignoreStatic: o.ignoreStatic,
+    strykerCompat: o.strykerCompat,
+    testOverrides: o.testOverrides,
+    vitestConfig: o.vitestConfig && existsSync(o.vitestConfig) ? sha(readFileSync(o.vitestConfig, 'utf8')) : null,
+});
 const UNIT_PAIRS = 400;
+const detected = (s) => s === 'Killed' || s === 'Timeout';
+const AUDIT_SHARE = 0.05;
 const DEADLOCK_REASON = 'its test deadlocked: nothing left in the worker could resume it (a hang, called in under a second)';
 function timeoutReason(v) {
     if (v.deadlock)
@@ -53,7 +74,7 @@ export async function run(options) {
     const work = mkdtempSync(path.join(tmpdir(), 'oxmutate-'));
     const stats = {
         files: 0, mutants: 0, instrumentMs: 0, dryRunMs: 0, mutationMs: 0, staticMs: 0, staticRecollections: 0, isolatedMs: 0, wallMs: 0, testExecutions: 0,
-        watchdogKills: 0, idleStalls: 0, deadlocks: 0, crashes: 0, canaryFiles: 0, pollutionRestarts: 0, survivorsOverturned: 0, timeoutsOverturned: 0, compileErrors: 0, isolatedRuns: 0, reused: 0, flakyTests: 0, flakyMutants: 0, units: 0, serversSpawned: 0,
+        watchdogKills: 0, idleStalls: 0, deadlocks: 0, crashes: 0, canaryFiles: 0, pollutionRestarts: 0, diskWrites: 0, auditSampled: 0, auditDisagreed: 0, survivorsOverturned: 0, timeoutsOverturned: 0, compileErrors: 0, isolatedRuns: 0, reused: 0, flakyTests: 0, flakyMutants: 0, units: 0, serversSpawned: 0,
     };
     let unitSeq = 0;
     const fileFor = (kind, ext) => path.join(work, `${kind}-${String(unitSeq++)}.${ext}`);
@@ -168,7 +189,7 @@ export async function run(options) {
             if (fileErrors.length) {
                 throw new InstrumentFault(`${String(fileErrors.length)} test file(s) failed to load during the dry run, so their tests would be missing from every verdict: ${fileErrors.slice(0, 3).map((e) => `${path.relative(root, e.file)}: ${e.error}`).join('; ')}`);
             }
-            return { tests: recs.flatMap((r) => r.tests ?? []), perTest, static: stat, runOnce: recs.flatMap((r) => r.runOnce ?? []) };
+            return { tests: recs.flatMap((r) => r.tests ?? []), perTest, static: stat, runOnce: recs.flatMap((r) => r.runOnce ?? []), modules: Object.assign({}, ...recs.map((r) => r.modules ?? {})) };
         };
         progress.phase('dry run');
         const d0 = performance.now();
@@ -262,13 +283,23 @@ export async function run(options) {
             }
             return h;
         };
-        const fingerprint = (m) => sha([sha(sources[m.file] ?? ''), ...[...new Set(m.coveredBy.map((t) => testInfo.get(t)?.file ?? ''))].sort().map(fileHash)].join(':'));
+        const envKey = sha(JSON.stringify({ oxmutate: VERSION, node: process.version, lock: lockfileHash(root), options: verdictOptions(options) }));
+        const fingerprint = (m) => {
+            const testFiles = new Set([...m.coveredBy.map((t) => testInfo.get(t)?.file ?? ''), ...(staticFiles.get(m.id) ?? [])]);
+            const graph = new Set([...testFiles].flatMap((f) => dry.modules[f] ?? []).filter((f) => !testFiles.has(f) && f !== path.join(root, m.file)));
+            return sha([envKey, sha(sources[m.file] ?? ''), ...[...testFiles].sort().map(fileHash), '|', ...[...graph].sort().map((f) => `${f}=${fileHash(f)}`)].join(':'));
+        };
+        const audited = new Map();
         const cache = options.incremental && existsSync(options.incrementalFile) ? JSON.parse(readFileSync(options.incrementalFile, 'utf8')).mutants : {};
         for (const m of mutants) {
             if (m.status === 'Ignored' || (!m.static && m.coveredBy.length === 0))
                 continue;
             const hit = cache[m.id];
             if (hit && hit.fp === fingerprint(m)) {
+                if (Math.random() < AUDIT_SHARE) {
+                    audited.set(m.id, hit.status);
+                    continue;
+                }
                 m.status = hit.status;
                 m.killedBy = hit.killedBy;
                 m.testsCompleted = hit.testsCompleted;
@@ -332,12 +363,58 @@ export async function run(options) {
             return units.sort((a, b) => weight(b) - weight(a));
         };
         const records = [];
+        const disk = env['OXMUTATE_NO_DISKGUARD'] ? undefined : await guardDisk(root);
+        const inFlight = new Set();
+        const tainted = new Set();
+        const writers = new Map();
+        const diskWrittenDuring = async (unit) => {
+            const change = await disk?.check();
+            const dirty = change !== undefined && !isClean(change);
+            if (dirty) {
+                stats.diskWrites++;
+                if (!disk?.restore(change)) {
+                    throw new InstrumentFault(`a mutant wrote outside the sandbox and the project could not be put back: ${[...change.changed, ...change.added, ...change.removed].join(', ')}. fix: restore them (git checkout / git clean), and see which test writes into the project`);
+                }
+                for (const other of inFlight)
+                    tainted.add(other);
+            }
+            if (!dirty && !tainted.delete(unit.results))
+                return undefined;
+            const entry = planUnits.get(unit.results);
+            if (!entry)
+                return [];
+            const ids = [...new Set(Object.values(entry.plan).flat())];
+            const paths = change ? [...change.changed, ...change.added, ...change.removed] : [];
+            if (dirty && ids.length === 1) {
+                const id = ids[0];
+                writers.set(id, paths);
+                if (inFlight.size === 0)
+                    records.push(...readRecords(unit.results));
+                else
+                    unfinished.add(id);
+                return [];
+            }
+            if (entry.depth >= 40) {
+                for (const id of ids)
+                    unfinished.add(id);
+                return [];
+            }
+            if (ids.length < 2)
+                return [makeUnit(entry.file, entry.plan, entry.depth + 1)];
+            const half = new Set(ids.slice(0, Math.ceil(ids.length / 2)));
+            const pick = (inHalf) => Object.fromEntries(Object.entries(entry.plan).map(([t, l]) => [t, l.filter((id) => half.has(id) === inHalf)]).filter(([, l]) => l.length > 0));
+            return [makeUnit(entry.file, pick(true), entry.depth + 1), makeUnit(entry.file, pick(false), entry.depth + 1)];
+        };
         const runaways = new Set();
         const crashedIds = new Set();
         const pool = createPool({ root, vitestConfig: options.vitestConfig, workers: 1, instrumentedMap: mapFile, nodeArgs: options.testRunnerNodeArgs, inlineDeps: true }, options.concurrency, work, 'mutant');
         pools.push(pool);
         const killedSoFar = (spec) => new Set([...runaways, ...crashedIds, ...records.filter((r) => r.type === 'mutant' && r.status !== 'Survived' && Boolean(r.spec) === spec).map((r) => r.id ?? '')]);
-        const afterUnit = (unit, outcome) => {
+        const afterUnit = async (unit, outcome) => {
+            inFlight.delete(unit.results);
+            const wrote = await diskWrittenDuring(unit);
+            if (wrote)
+                return wrote;
             const raw = readRecords(unit.results);
             const suspect = new Set(raw.filter((r) => r.type === 'polluted').flatMap((r) => (r.suspects ?? []).map((id) => `${id}|${r.test ?? ''}`)));
             const recs = raw.filter((r) => !(r.type === 'mutant' && suspect.has(`${r.id ?? ''}|${r.test ?? ''}`)));
@@ -380,6 +457,7 @@ export async function run(options) {
         progress.phase('warm runs', killedCount);
         const m0 = performance.now();
         const beforeUnit = (unit) => {
+            inFlight.add(unit.results);
             const entry = planUnits.get(unit.results);
             if (!entry)
                 return true;
@@ -391,8 +469,10 @@ export async function run(options) {
                 if (keep.length)
                     left[t] = keep;
             }
-            if (Object.keys(left).length === 0)
+            if (Object.keys(left).length === 0) {
+                inFlight.delete(unit.results);
                 return false;
+            }
             entry.plan = left;
             writeFileSync(unit.plan, JSON.stringify({ tests: left, hitLimits, skip: [], specTests: [...specOf.keys()], runOnce: dry.runOnce, testMs: testMsOf(left) }));
             return true;
@@ -670,11 +750,20 @@ export async function run(options) {
         stats.isolatedMs = performance.now() - i0;
         stats.watchdogKills = pool.stats.killed;
         stats.idleStalls = pool.stats.idle;
+        for (const [id, paths] of writers) {
+            const m = byId.get(id);
+            if (m)
+                m.wrote = paths;
+        }
         stats.deadlocks = mutants.filter((m) => m.statusReason === DEADLOCK_REASON).length;
         stats.crashes = pool.stats.crashed;
         if (options.incremental) {
             mkdirSync(path.dirname(options.incrementalFile), { recursive: true });
-            const next = { ...cache };
+            stats.auditSampled = audited.size;
+            stats.auditDisagreed = [...audited].filter(([id, was]) => detected(byId.get(id)?.status) !== detected(was)).length;
+            if (stats.auditDisagreed > 0)
+                stderr.write(`oxmutate: ${String(stats.auditDisagreed)} of ${String(audited.size)} audited cached verdicts disagree with a fresh run; the incremental cache is rebuilt from this run\n`);
+            const next = stats.auditDisagreed > 0 ? {} : { ...cache };
             for (const m of mutants) {
                 if (['Killed', 'Survived', 'Timeout'].includes(m.status) && !m.flaky)
                     next[m.id] = { fp: fingerprint(m), status: m.status, killedBy: m.killedBy, testsCompleted: m.testsCompleted };

package/dist/diskguard.js

@@ -0,0 +1,92 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { ambientRuntime, run } from 'bellpull';
+export const MAX_SNAPSHOT_BYTES = 64 * 1024 * 1024;
+export const isClean = (c) => c.changed.length + c.added.length + c.removed.length === 0;
+async function git(cwd, args) {
+    const r = await run('git', args, { cwd, runtime: ambientRuntime() }).catch(() => undefined);
+    return r?.ok ? r.stdout : undefined;
+}
+export async function guardDisk(root, maxSnapshotBytes = MAX_SNAPSHOT_BYTES) {
+    const listed = await git(root, ['ls-files', '-z', '--cached', '--', '.']);
+    if (listed === undefined)
+        return undefined;
+    const files = listed.split('\0').filter(Boolean);
+    const tracked = new Map();
+    let bytes = 0;
+    for (const rel of files) {
+        const abs = path.join(root, rel);
+        if (!existsSync(abs))
+            continue;
+        const st = statSync(abs);
+        if (!st.isFile())
+            continue;
+        bytes += st.size;
+        tracked.set(rel, { size: st.size, mtimeMs: st.mtimeMs });
+    }
+    const restorable = bytes <= maxSnapshotBytes;
+    if (restorable)
+        for (const [rel, t] of tracked)
+            t.content = readFileSync(path.join(root, rel));
+    const dirs = new Map();
+    for (const rel of tracked.keys()) {
+        const dir = path.dirname(rel);
+        if (!dirs.has(dir))
+            dirs.set(dir, new Set(readdirSync(path.join(root, dir))));
+    }
+    const knownIgnored = new Map();
+    const ignored = async (rels) => {
+        const ask = rels.filter((rel) => !knownIgnored.has(rel));
+        if (ask.length) {
+            const r = await run('git', ['check-ignore', '--', ...ask], { cwd: root, runtime: ambientRuntime() }).catch(() => undefined);
+            const yes = new Set((r?.stdout ?? '').split('\n').filter(Boolean));
+            for (const rel of ask)
+                knownIgnored.set(rel, yes.has(rel));
+        }
+        return new Set(rels.filter((rel) => knownIgnored.get(rel)));
+    };
+    return {
+        async check() {
+            const changed = [];
+            const removed = [];
+            for (const [rel, t] of tracked) {
+                const abs = path.join(root, rel);
+                if (!existsSync(abs)) {
+                    removed.push(rel);
+                    continue;
+                }
+                const st = statSync(abs);
+                if (st.size !== t.size || st.mtimeMs !== t.mtimeMs)
+                    changed.push(rel);
+            }
+            const fresh = [];
+            for (const [dir, before] of dirs) {
+                const abs = path.join(root, dir);
+                if (!existsSync(abs))
+                    continue;
+                for (const name of readdirSync(abs))
+                    if (!before.has(name))
+                        fresh.push(path.join(dir, name));
+            }
+            const skip = await ignored(fresh);
+            return { changed, added: fresh.filter((rel) => !skip.has(rel)), removed };
+        },
+        restore(change) {
+            if (!restorable)
+                return false;
+            for (const rel of [...change.changed, ...change.removed]) {
+                const t = tracked.get(rel);
+                if (!t?.content)
+                    return false;
+                mkdirSync(path.dirname(path.join(root, rel)), { recursive: true });
+                writeFileSync(path.join(root, rel), t.content);
+                const st = statSync(path.join(root, rel));
+                t.mtimeMs = st.mtimeMs;
+                t.size = st.size;
+            }
+            for (const rel of change.added)
+                rmSync(path.join(root, rel), { recursive: true, force: true });
+            return true;
+        },
+    };
+}

package/dist/reports.js

@@ -96,12 +96,13 @@ function corrections(s) {
     return [
         s.watchdogKills && `${String(s.watchdogKills)} runaway(s) stopped`,
         s.deadlocks && `${String(s.deadlocks)} deadlock(s) called by their worker`,
+        s.diskWrites && `${String(s.diskWrites)} write(s) outside the sandbox put back, their units re-run`,
         s.pollutionRestarts && `${String(s.pollutionRestarts)} polluted context(s) restarted (in ${String(s.canaryFiles)} file(s))`,
         s.flakyMutants && `${String(s.flakyMutants)} flaky verdict(s)`,
         s.survivorsOverturned && `${String(s.survivorsOverturned)} survivor(s) killed by a whole-file re-run`,
         s.timeoutsOverturned && `${String(s.timeoutsOverturned)} timeout(s) that finished given 3x the time`,
         s.compileErrors && `${String(s.compileErrors)} survivor(s) that do not type-check, out of the score`,
-        s.reused && `${String(s.reused)} verdict(s) reused`,
+        s.reused && `${String(s.reused)} verdict(s) reused${s.auditSampled ? ` (${String(s.auditSampled)} audited, ${String(s.auditDisagreed)} disagreeing)` : ''}`,
         s.crashes && `${String(s.crashes)} worker crash(es)`,
     ].filter((c) => Boolean(c));
 }

package/dist/runner.d.ts

@@ -35,6 +35,7 @@ declare const OxmutateRunner_base: new (config: unknown) => {
         evaluatedModules?: {
             getModuleById: (id: string) => unknown;
             invalidateModule: (node: unknown) => void;
+            fileToModulesMap?: Map<string, unknown>;
         };
     };
     _currentSpecification?: unknown;

package/dist/runner.js

@@ -120,6 +120,7 @@ function takeCopy(test) {
     const file = collections.get(test.file?.filepath ?? '')?.[n];
     return file && find(file, test.id);
 }
+const dryModules = new Map();
 const PROBE_ID = '__oxmutate_probe__';
 async function probe(test, fn) {
     const file = test.file?.filepath;
@@ -425,6 +426,10 @@ export default class OxmutateRunner extends Base {
         return super.importFile?.(filepath, source);
     }
     async onAfterRunSuite(suite) {
+        if (mode === 'dry' && suite.filepath) {
+            const files = [...(this.workerState?.evaluatedModules?.fileToModulesMap?.keys() ?? [])];
+            dryModules.set(suite.filepath, files.filter((f) => path.isAbsolute(f) && !f.split(path.sep).includes('node_modules')));
+        }
         const copies = suite.filepath ? collections.get(suite.filepath) : undefined;
         if (suite.filepath && copies) {
             collections.delete(suite.filepath);
@@ -574,7 +579,7 @@ export default class OxmutateRunner extends Base {
                 if (result?.state === 'fail' && !f.tasks?.length)
                     fileErrors.push({ file: filepath, error: (result.errors?.[0]?.message ?? 'failed to collect').slice(0, FILE_ERROR_CHARS) });
             }
-            emit({ type: 'dry', tests, coverage: ns.mutantCoverage, fileErrors, runOnce: [...runOnce] });
+            emit({ type: 'dry', tests, coverage: ns.mutantCoverage, fileErrors, runOnce: [...runOnce], modules: Object.fromEntries(dryModules) });
         }
         else if (mode === 'isolated') {
             const failed = [];

package/dist/scheduler.js

@@ -172,7 +172,7 @@ export function createPool(config, size, workDir, tag) {
                     if (outcome.killedFor || outcome.crashed) {
                         servers[slot] = replacement();
                     }
-                    const more = after(unit, outcome);
+                    const more = await after(unit, outcome);
                     queue.unshift(...more);
                     busy--;
                     signal();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "oxmutate",
-  "version": "1.3.0",
+  "version": "1.5.0",
   "description": "Mutation testing for JavaScript and TypeScript on the oxc engine, run warm inside the project's own Vitest, with CI lanes, SARIF and an agent surface. No dependency outside the burgee family. A Stryker migration path.",
   "license": "MIT",
   "repository": {