mixdog 0.9.172 → 0.9.173

package/scripts/lib/coverage.mjs

@@ -0,0 +1,309 @@
+// V8 coverage, folded into the one artifact that answers a single question:
+// does the suite actually execute this function? Grepping test files for a
+// symbol answers it wrongly whenever a helper is reached only through a public
+// entry point, so the evidence comes from Node itself — NODE_V8_COVERAGE makes
+// every test process dump raw V8 coverage JSON at exit, and this module folds
+// those dumps (one per batch, one per test child) into per-function lines.
+//
+// No dependency is involved: V8 emits the raw data, `node:fs` reads it back.
+import { createHash } from 'node:crypto';
+import { mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises';
+import { dirname, isAbsolute, join, relative, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+export const ARTIFACT_PATH = '.runtime/coverage/coverage.json';
+
+// The runner's own lane patterns: *.test.mjs and *-test.mjs (plus the jsx/ts
+// spellings used by apps/desktop). A test executing itself proves nothing.
+const TEST_FILE = /[.-]test\.[cm]?[jt]sx?$/;
+// tsx transpiles TypeScript before V8 sees it, so the offsets V8 reports for a
+// .ts file describe the transpiled output instead of the file on disk.
+const TYPESCRIPT_FILE = /\.[cm]?tsx?$/;
+
+export const EXCLUSION_REASONS = {
+  'transpiled-by-tsx':
+    'TypeScript executed through tsx: V8 offsets describe the transpiled output, not this file, and this round does not translate them through source maps',
+  'offsets-beyond-source':
+    'recorded offsets run past the file on disk, so the executed source was generated or transformed rather than read from here',
+  'source-unreadable': 'the file could not be read back while folding coverage',
+};
+
+// The identity a later query uses to tell whether the file still is what was
+// measured. Content, not mtime or size: a checkout, a formatter rewriting a
+// file byte-identically or a restored backup all move mtime without changing a
+// single line, and an edit that swaps two lines keeps the size — either way the
+// recorded line ranges would be wrong or wrongly distrusted. Folding already
+// reads every source in full to translate offsets, so hashing it costs nothing
+// and answers exactly. (A file absent from the artifact has no recorded
+// content to compare, so its mtime against the collection time is the only
+// evidence there is; `queryCoverage` uses that for the unknown-file case.)
+export function sourceDigest(source) {
+  return createHash('sha256').update(source).digest('hex');
+}
+
+// Sources are normalized the same way at folding and at query time, so the
+// digest compares like with like.
+async function currentSource(rootPath, path) {
+  try {
+    return (await readFile(join(rootPath, path), 'utf8')).replace(/^\uFEFF/, '');
+  } catch {
+    return null;
+  }
+}
+
+async function modifiedAfter(rootPath, path, generatedAt) {
+  const collected = Date.parse(generatedAt ?? '');
+  if (Number.isNaN(collected)) return false;
+  try {
+    return (await stat(join(rootPath, path))).mtimeMs > collected;
+  } catch {
+    return false;
+  }
+}
+
+function lineStarts(source) {
+  const starts = [0];
+  for (let index = 0; index < source.length; index++) {
+    if (source.charCodeAt(index) === 10) starts.push(index + 1);
+  }
+  return starts;
+}
+
+// 1-based line holding a 0-based character offset.
+function lineOf(starts, offset) {
+  let low = 0;
+  let high = starts.length - 1;
+  while (low < high) {
+    const mid = (low + high + 1) >> 1;
+    if (starts[mid] <= offset) low = mid;
+    else high = mid - 1;
+  }
+  return low + 1;
+}
+
+// V8 keeps the script's own top-level range as a nameless function spanning the
+// whole source. It is the module body, not a function anyone can call, so it
+// becomes the file's `loaded` flag instead of an entry every line sits inside.
+const isModuleBody = (entry, length) => entry.name === '' && entry.startOffset === 0 && entry.endOffset >= length;
+
+function repoRelative(url, rootPath) {
+  if (!url.startsWith('file:')) return '';
+  const parsed = new URL(url);
+  parsed.search = '';
+  parsed.hash = '';
+  const absolute = fileURLToPath(parsed);
+  const path = relative(rootPath, absolute).replaceAll('\\', '/');
+  if (path === '' || path.startsWith('../') || isAbsolute(path)) return '';
+  return path;
+}
+
+// Raw V8 dumps -> { path -> [{ name, startOffset, endOffset, executed }] }.
+// Every dump contributes: a function executed in any batch is executed.
+function collectFunctions(reports, rootPath, excluded) {
+  const byFile = new Map();
+  for (const report of reports) {
+    for (const script of report?.result ?? []) {
+      const path = repoRelative(script.url ?? '', rootPath);
+      if (path === '' || path.split('/').includes('node_modules') || TEST_FILE.test(path)) continue;
+      if (TYPESCRIPT_FILE.test(path)) {
+        excluded.set(path, 'transpiled-by-tsx');
+        continue;
+      }
+      const functions = byFile.get(path) ?? new Map();
+      byFile.set(path, functions);
+      for (const entry of script.functions ?? []) {
+        const range = entry.ranges?.[0];
+        if (!range) continue;
+        const key = `${entry.functionName}\u0000${range.startOffset}\u0000${range.endOffset}`;
+        const merged = functions.get(key);
+        if (merged) merged.executed ||= range.count > 0;
+        else
+          functions.set(key, {
+            name: entry.functionName,
+            startOffset: range.startOffset,
+            endOffset: range.endOffset,
+            executed: range.count > 0,
+          });
+      }
+    }
+  }
+  return byFile;
+}
+
+// The artifact: per repository source file, every function V8 covered, with
+// offsets translated to 1-based lines and the single bit that matters.
+export async function foldCoverage(reports, { root = process.cwd() } = {}) {
+  const rootPath = resolve(root);
+  const excluded = new Map();
+  const collected = collectFunctions(reports, rootPath, excluded);
+  const files = {};
+  for (const path of [...collected.keys()].sort()) {
+    const entries = [...collected.get(path).values()].sort(
+      (left, right) => left.startOffset - right.startOffset || left.endOffset - right.endOffset
+    );
+    let source;
+    try {
+      source = (await readFile(join(rootPath, path), 'utf8')).replace(/^\uFEFF/, '');
+    } catch {
+      excluded.set(path, 'source-unreadable');
+      continue;
+    }
+    if (entries.some((entry) => entry.endOffset > source.length)) {
+      excluded.set(path, 'offsets-beyond-source');
+      continue;
+    }
+    const starts = lineStarts(source);
+    const body = entries.find((entry) => isModuleBody(entry, source.length));
+    files[path] = {
+      loaded: body ? body.executed : entries.some((entry) => entry.executed),
+      sha256: sourceDigest(source),
+      functions: entries
+        .filter((entry) => entry !== body)
+        .map((entry) => ({
+          name: entry.name || '(anonymous)',
+          startLine: lineOf(starts, entry.startOffset),
+          endLine: lineOf(starts, Math.max(entry.startOffset, entry.endOffset - 1)),
+          executed: entry.executed,
+        })),
+    };
+  }
+  const covered = Object.values(files);
+  return {
+    version: 2,
+    generatedAt: new Date().toISOString(),
+    root: rootPath.replaceAll('\\', '/'),
+    summary: {
+      filesWithExecutedFunctions: covered.filter((file) => file.functions.some((fn) => fn.executed)).length,
+      filesWithoutExecutedFunctions: covered.filter((file) => !file.functions.some((fn) => fn.executed)).length,
+      excludedFiles: excluded.size,
+    },
+    files,
+    exclusionReasons: EXCLUSION_REASONS,
+    excluded: [...excluded.keys()].sort().map((path) => ({ path, reason: excluded.get(path) })),
+  };
+}
+
+export async function readRawCoverage(directory) {
+  const names = (await readdir(directory)).filter((name) => name.endsWith('.json')).sort();
+  const reports = [];
+  for (const name of names) reports.push(JSON.parse(await readFile(join(directory, name), 'utf8')));
+  return reports;
+}
+
+export async function writeCoverageArtifact(artifact, { root = process.cwd(), artifactPath = ARTIFACT_PATH } = {}) {
+  const target = resolve(root, artifactPath);
+  await mkdir(dirname(target), { recursive: true });
+  await writeFile(target, `${JSON.stringify(artifact, null, 2)}\n`);
+  return target;
+}
+
+export async function loadCoverageArtifact({ root = process.cwd(), artifactPath = ARTIFACT_PATH } = {}) {
+  return JSON.parse(await readFile(resolve(root, artifactPath), 'utf8'));
+}
+
+// Is this line inside a function the suite executed? The innermost enclosing
+// function answers; a line the file's functions do not cover is reported as
+// such rather than borrowed from the module body around it.
+//
+// A recorded answer is only worth giving while the file still is the file that
+// was measured, so every query re-reads the source and compares its digest
+// first: a changed file answers `stale`, never EXECUTED or NOT EXECUTED. An
+// artifact predating this check has no digest to match, which is itself a
+// mismatch, so it announces itself too.
+export async function queryCoverage(artifact, file, line, { root = process.cwd() } = {}) {
+  const rootPath = resolve(root);
+  const path = String(file).replaceAll('\\', '/').replace(/^\.\//, '');
+  const generatedAt = artifact.generatedAt;
+  const entry = artifact.files?.[path];
+  const excluded = artifact.excluded?.find((item) => item.path === path);
+  if (!entry) {
+    if (excluded) return { path, line, status: 'excluded', reason: excluded.reason, generatedAt };
+    // Nothing was recorded for this file, so there is no content to compare
+    // against; a file touched after the collection is the case where a test
+    // was added since, which is exactly how this answer misled a reader.
+    const changed = await modifiedAfter(rootPath, path, generatedAt);
+    return { path, line, status: 'unknown-file', generatedAt, changedSinceCollection: changed };
+  }
+  const source = await currentSource(rootPath, path);
+  if (source === null) return { path, line, status: 'stale', change: 'missing', generatedAt };
+  if (sourceDigest(source) !== entry.sha256) return { path, line, status: 'stale', change: 'modified', generatedAt };
+  const inside = entry.functions.filter((fn) => line >= fn.startLine && line <= fn.endLine);
+  const innermost = inside.reduce(
+    (best, fn) => (best === null || fn.endLine - fn.startLine <= best.endLine - best.startLine ? fn : best),
+    null
+  );
+  if (!innermost) return { path, line, status: 'outside-functions', loaded: entry.loaded, generatedAt };
+  return {
+    path,
+    line,
+    status: innermost.executed ? 'executed' : 'not-executed',
+    loaded: entry.loaded,
+    generatedAt,
+    function: innermost,
+  };
+}
+
+// Every answer a reader must not act on says what to do instead.
+const RECOLLECT = 're-run `node scripts/test.mjs --coverage`';
+
+export function formatCoverageAnswer(result) {
+  const at = `${result.path}:${result.line}`;
+  const collected = result.generatedAt ? `collected ${result.generatedAt}` : 'collected at an unrecorded time';
+  if (result.status === 'stale')
+    return result.change === 'missing'
+      ? `STALE ${at} — the file is gone since coverage was ${collected}, so nothing here can be answered: ${RECOLLECT}`
+      : `STALE ${at} — the file changed since coverage was ${collected}, so its recorded lines no longer describe it: ${RECOLLECT}`;
+  if (result.status === 'excluded')
+    return `EXCLUDED ${at} — not in the artifact: ${EXCLUSION_REASONS[result.reason] ?? result.reason}`;
+  if (result.status === 'unknown-file')
+    return result.changedSinceCollection
+      ? `STALE ${at} — no coverage was recorded for this file and it changed after coverage was ${collected}, so a test may cover it now: ${RECOLLECT}`
+      : `NO COVERAGE ${at} — the run ${collected} recorded no coverage for this file; if a test was added since, ${RECOLLECT}`;
+  if (result.status === 'outside-functions')
+    return `NO FUNCTION ${at} — outside every covered function (module body ${result.loaded ? 'ran' : 'never ran'})`;
+  const where = `${result.function.name} (lines ${result.function.startLine}-${result.function.endLine})`;
+  return result.status === 'executed'
+    ? `EXECUTED ${at} — inside ${where}, which the suite ran`
+    : `NOT EXECUTED ${at} — inside ${where}, which no test ran`;
+}
+
+// The whole-artifact signal: how much of what was measured no longer exists as
+// measured, and when it was measured, so a caller can decide to re-collect
+// before asking anything at all.
+export async function coverageStaleness(artifact, { root = process.cwd() } = {}) {
+  const rootPath = resolve(root);
+  const changed = [];
+  const missing = [];
+  for (const [path, entry] of Object.entries(artifact.files ?? {})) {
+    const source = await currentSource(rootPath, path);
+    if (source === null) missing.push(path);
+    else if (sourceDigest(source) !== entry.sha256) changed.push(path);
+  }
+  return {
+    generatedAt: artifact.generatedAt,
+    files: Object.keys(artifact.files ?? {}).length,
+    changed: changed.sort(),
+    missing: missing.sort(),
+    changedCount: changed.length + missing.length,
+    stale: changed.length + missing.length > 0,
+  };
+}
+
+export function formatStaleness(state) {
+  const collected = state.generatedAt ? `collected ${state.generatedAt}` : 'collected at an unrecorded time';
+  return state.stale
+    ? `STALE coverage ${collected}: ${state.changedCount} of ${state.files} files changed since (${state.missing.length} gone); ${RECOLLECT}`
+    : `FRESH coverage ${collected}: all ${state.files} files unchanged since`;
+}
+
+// Called once per `--coverage` run, after the last batch has exited.
+export async function reportCoverage(rawDirectory, { root = process.cwd(), log = console.error } = {}) {
+  const artifact = await foldCoverage(await readRawCoverage(rawDirectory), { root });
+  const target = await writeCoverageArtifact(artifact, { root });
+  const { filesWithExecutedFunctions, filesWithoutExecutedFunctions, excludedFiles } = artifact.summary;
+  log(
+    `Coverage: ${filesWithExecutedFunctions} files with executed functions, ${filesWithoutExecutedFunctions} without, ${excludedFiles} excluded`
+  );
+  log(`Coverage artifact: ${target}`);
+  return target;
+}

package/scripts/lib/embedding-model-bench-core.mjs

@@ -97,6 +97,8 @@ export function buildEvaluation(caseFile, kase) {
     : [];
   if (typeof query !== 'string' || !query.trim() || targets.length === 0) return null;
   const temporal = parsePeriod(String(kase?.args?.period || ''), true);
+  const startMs = Number(temporal?.startMs);
+  const endMs = Number(temporal?.endMs);
   return {
     id: String(kase.id),
     caseFile,
@@ -114,8 +116,8 @@ export function buildEvaluation(caseFile, kase) {
             .toLowerCase()
         )
         .filter(Boolean),
-      startMs: Number.isFinite(Number(temporal?.startMs)) ? Number(temporal.startMs) : null,
-      endMs: Number.isFinite(Number(temporal?.endMs)) ? Number(temporal.endMs) : null,
+      startMs: Number.isFinite(startMs) ? startMs : null,
+      endMs: Number.isFinite(endMs) ? endMs : null,
     },
   };
 }

package/scripts/lib/recall-bench-eval.mjs

@@ -184,12 +184,12 @@ export function scoreWithinPeriod(items, temporal, toleranceMs = 60_000) {
   const endMs = Number(temporal?.endMs);
   const hasStart = Number.isFinite(startMs);
   const hasEnd = Number.isFinite(endMs);
-  const offenders = items.filter(
+  const timestamped = items.filter((item) => item.timestampMs !== null);
+  const offenders = timestamped.filter(
     (item) =>
-      item.timestampMs !== null &&
-      ((hasStart && item.timestampMs < startMs - toleranceMs) || (hasEnd && item.timestampMs > endMs + toleranceMs))
+      (hasStart && item.timestampMs < startMs - toleranceMs) || (hasEnd && item.timestampMs > endMs + toleranceMs)
   );
-  return { checked: items.filter((item) => item.timestampMs !== null).length, offenders, ok: offenders.length === 0 };
+  return { checked: timestamped.length, offenders, ok: offenders.length === 0 };
 }
 
 export function evaluateCase(kase, outcome, quality, recency, allContain, withinPeriod, pageOrder) {

package/scripts/lib/run-node-tests.mjs

@@ -1,28 +1,152 @@
 import { spawn } from 'node:child_process';
-import { mkdtemp } from 'node:fs/promises';
+import { appendFile, mkdtemp, readFile, rm } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
+import { FAILURE_RECORDS_ENV } from './test-failure-records.mjs';
+import { classifyRerunOutcomes, formatFailureSummary } from './test-failure-summary.mjs';
 
 const SUMMARY_REPORTER = new URL('./test-summary-reporter.mjs', import.meta.url).href;
 
+// A whole-suite run passes ~700 paths to one node process. Windows caps a
+// command line at 32767 characters (CreateProcess) and POSIX caps argv+env at
+// ARG_MAX, so the file list is spawned in batches that stay under the cap
+// instead of dying with spawn ENAMETOOLONG before any test runs.
+export const ARG_BUDGET = process.platform === 'win32' ? 30_000 : 1_000_000;
+
+// An argument costs its own characters plus the separator and its quotes.
+const argCost = (arg) => arg.length + 3;
+
+// Batches keep the file order Node was given; a single path larger than the
+// budget still gets its own batch rather than being dropped. An empty list
+// stays one batch: Node's own discovery is what an argument-less run means.
+export function chunkFileArgs(fileArgs, budget) {
+  const chunks = [[]];
+  let cost = 0;
+  for (const file of fileArgs) {
+    const current = chunks.at(-1);
+    if (current.length > 0 && cost + argCost(file) > budget) {
+      chunks.push([file]);
+      cost = argCost(file);
+      continue;
+    }
+    current.push(file);
+    cost += argCost(file);
+  }
+  return chunks;
+}
+
+function spawnBatch(args, env) {
+  return new Promise((resolve) => {
+    const child = spawn(process.execPath, args, { stdio: 'inherit', env });
+    child.on('error', (error) => {
+      console.error(error);
+      resolve(1);
+    });
+    child.on('exit', (code, signal) => resolve(code ?? (signal ? 1 : 0)));
+  });
+}
+
+// The failure reporter writes one JSON line per failed test; a spawn that
+// died before Node opened its destination leaves no file at all.
+async function readFailureRecords(path) {
+  const text = await readFile(path, 'utf8').catch(() => '');
+  return text
+    .split('\n')
+    .filter(Boolean)
+    .map((line) => JSON.parse(line));
+}
+
 // Both discovery and explicit-file runs use the same output contract. Node
-// still owns test selection, diagnostics, skip/todo counts and exit status.
-export async function runNodeTests(nodeArgs, fileArgs) {
+// still owns test selection, diagnostics, skip/todo counts and exit status;
+// a batched run keeps one full log and fails on the first failing batch.
+//
+// With `coverage`, every batch inherits one fresh NODE_V8_COVERAGE directory,
+// so V8 writes a dump per test process into it and the batches fold into a
+// single artifact. Without it nothing here changes — not the spawn options,
+// not the environment, not the output.
+//
+// `rerunFailed` (default 0 = off) re-runs the files that failed, up to n
+// times. A test that passes on a re-run is reported as flaky and, because the
+// flag was passed, does not fail the run; a test that fails every re-run
+// still exits 1. Every run ends with the failure summary block, so a filtered
+// log can still name what failed.
+export async function runNodeTests(
+  nodeArgs,
+  fileArgs,
+  { argBudget = ARG_BUDGET, coverage = false, rerunFailed = 0 } = {}
+) {
   const logPath = join(await mkdtemp(join(tmpdir(), 'mixdog-test-output-')), 'full.log');
   console.error(`Full test log: ${logPath}`);
-  const child = spawn(process.execPath, [
+  // Failure records live outside the log directory: the log directory holds
+  // exactly one merged log, and these records are an internal artifact.
+  const failureDir = await mkdtemp(join(tmpdir(), 'mixdog-test-failures-'));
+  const runArgs = [
     ...nodeArgs,
     `--test-reporter=${SUMMARY_REPORTER}`,
     '--test-reporter-destination=stdout',
     '--test-reporter=spec',
-    `--test-reporter-destination=${logPath}`,
-    ...fileArgs,
-  ], { stdio: 'inherit' });
-  child.on('error', (error) => {
-    console.error(error);
-    process.exitCode = 1;
-  });
-  child.on('exit', (code, signal) => {
-    process.exitCode = code ?? (signal ? 1 : 0);
+  ];
+  const fixedCost =
+    argCost(process.execPath) +
+    runArgs.reduce((total, arg) => total + argCost(arg), 0) +
+    argCost(`--test-reporter-destination=${logPath}`) +
+    argCost('.999'); // the per-batch log suffix
+  const batches = chunkFileArgs(fileArgs, Math.max(argBudget - fixedCost, 1));
+  const coverageDir = coverage ? await mkdtemp(join(tmpdir(), 'mixdog-v8-coverage-')) : '';
+  // Failure records travel by environment, not argv: the command line is
+  // already budgeted, and a third reporter would warn about listeners.
+  const childEnv = (recordsPath) => ({
+    ...process.env,
+    [FAILURE_RECORDS_ENV]: recordsPath,
+    ...(coverageDir ? { NODE_V8_COVERAGE: coverageDir } : {}),
   });
+  let status = 0;
+  const failures = [];
+  for (const [index, batch] of batches.entries()) {
+    const batchLog = batches.length === 1 ? logPath : `${logPath}.${index + 1}`;
+    const batchFailures = join(failureDir, `batch-${index + 1}.jsonl`);
+    const code = await spawnBatch(
+      [...runArgs, `--test-reporter-destination=${batchLog}`, ...batch],
+      childEnv(batchFailures)
+    );
+    if (batchLog !== logPath) {
+      await appendFile(logPath, await readFile(batchLog));
+      await rm(batchLog, { force: true });
+    }
+    failures.push(...(await readFailureRecords(batchFailures)));
+    if (status === 0) status = code;
+  }
+
+  const attempts = [];
+  let lastAttemptStatus = 0;
+  let pending = failures;
+  for (let attempt = 1; attempt <= rerunFailed && pending.length > 0; attempt += 1) {
+    const files = [...new Set(pending.map((record) => record.file))];
+    console.error(`Re-running ${files.length} failed file(s): attempt ${attempt}/${rerunFailed}`);
+    const attemptLog = `${logPath}.rerun-${attempt}`;
+    const attemptFailures = join(failureDir, `rerun-${attempt}.jsonl`);
+    lastAttemptStatus = await spawnBatch(
+      [...runArgs, `--test-reporter-destination=${attemptLog}`, ...files],
+      childEnv(attemptFailures)
+    );
+    await appendFile(logPath, await readFile(attemptLog));
+    await rm(attemptLog, { force: true });
+    attempts.push({ files, failures: await readFailureRecords(attemptFailures) });
+    pending = classifyRerunOutcomes(failures, attempts).failed;
+  }
+  const { failed, flaky } = classifyRerunOutcomes(failures, attempts);
+  // Flaky tolerance applies only to an ordinary test-failure exit that the
+  // re-runs fully explained: nothing failed twice and the last re-run itself
+  // came back clean.
+  if (rerunFailed > 0 && status === 1 && failed.length === 0 && flaky.length > 0 && lastAttemptStatus === 0) status = 0;
+  process.exitCode = status;
+
+  await rm(failureDir, { recursive: true, force: true });
+  if (coverageDir) {
+    const { reportCoverage } = await import('./coverage.mjs');
+    await reportCoverage(coverageDir);
+    await rm(coverageDir, { recursive: true, force: true });
+  }
+  // Last line of the run, after every reporter and the coverage report.
+  process.stdout.write(`\n${formatFailureSummary({ failed, flaky, rerunFailed })}`);
 }

package/scripts/lib/test-failure-records.mjs

@@ -0,0 +1,46 @@
+// Structured failure records for the run-level failure summary.
+//
+// Node keeps ownership of diagnostics, totals and exit status; this only
+// records "which test failed, in which file" as one JSON line each, so the
+// runner can name every failure after all other output even when the run log
+// was filtered (scripts/lib/run-node-tests.mjs).
+//
+// It rides the existing summary reporter instead of adding a third node:test
+// reporter: a third reporter pushes the shared TestsStream past its listener
+// cap and every run would start with a MaxListenersExceededWarning.
+import { appendFileSync } from 'node:fs';
+import { relative } from 'node:path';
+
+export const FAILURE_RECORDS_ENV = 'MIXDOG_TEST_FAILURE_RECORDS';
+
+// Ancestor names are tracked per file: node:test runs files concurrently, so
+// one flat stack would mix their nestings.
+export function failureRecord(event, stacks) {
+  const file = event.data?.file ? relative(process.cwd(), String(event.data.file)).replaceAll('\\', '/') : '';
+  if (event.type === 'test:start') {
+    const stack = stacks.get(file) ?? [];
+    stack.length = event.data.nesting;
+    stack[event.data.nesting] = event.data.name;
+    stacks.set(file, stack);
+    return null;
+  }
+  if (event.type !== 'test:fail') return null;
+  // A failing todo does not fail the run, and a suite reports its children's
+  // failures as its own; only the leaves that decide the exit status belong
+  // in the summary.
+  if (event.data.todo) return null;
+  if (event.data.details?.error?.failureType === 'subtestsFailed') return null;
+  const ancestors = (stacks.get(file) ?? []).slice(0, event.data.nesting);
+  return { file, name: [...ancestors, event.data.name].join(' > ') };
+}
+
+// The runner sets FAILURE_RECORDS_ENV to a per-spawn JSONL path. Without it
+// nothing is recorded, so a direct `node --test` run is unaffected.
+export function createFailureRecorder(path = process.env[FAILURE_RECORDS_ENV]) {
+  if (!path) return () => {};
+  const stacks = new Map();
+  return (event) => {
+    const record = failureRecord(event, stacks);
+    if (record) appendFileSync(path, `${JSON.stringify(record)}\n`);
+  };
+}

package/scripts/lib/test-failure-summary.mjs

@@ -0,0 +1,59 @@
+// Run-level failure classification and the final summary block.
+//
+// A filtered run log loses a failure: `ℹ fail 1` says a test failed but never
+// which one. Every line here carries SUMMARY_TAG and names its file and test,
+// so the block survives `grep`/`Select-String` filtering, and the runner
+// prints it after all other output (scripts/lib/run-node-tests.mjs).
+export const SUMMARY_TAG = '[failure-summary]';
+
+const keyOf = (record) => `${record.file}\u0000${record.name}`;
+const byLocation = (a, b) => (a.file === b.file ? a.name.localeCompare(b.name) : a.file.localeCompare(b.file));
+
+// A test that failed and then passed inside the re-run budget is flaky; one
+// that fails every attempt stays failed. A failure that only appears during a
+// re-run joins the failed set from the attempt that produced it, so a re-run
+// can never hide a test it just broke.
+//
+// attempts: [{ files: [rerun file paths], failures: [{ file, name }] }]
+export function classifyRerunOutcomes(initialFailures, attempts = []) {
+  const pending = new Map(initialFailures.map((record) => [keyOf(record), { ...record }]));
+  const flaky = new Map();
+  for (const [index, attempt] of attempts.entries()) {
+    const attemptNumber = index + 1;
+    const rerunFiles = new Set(attempt.files);
+    const stillFailing = new Set(attempt.failures.map(keyOf));
+    for (const [key, record] of [...pending]) {
+      if (!rerunFiles.has(record.file) || stillFailing.has(key)) continue;
+      pending.delete(key);
+      flaky.set(key, { ...record, passedOnAttempt: attemptNumber });
+    }
+    for (const record of attempt.failures) {
+      const key = keyOf(record);
+      if (pending.has(key) || flaky.has(key)) continue;
+      pending.set(key, { ...record, firstSeenOnRerun: attemptNumber });
+    }
+  }
+  return { failed: [...pending.values()].sort(byLocation), flaky: [...flaky.values()].sort(byLocation) };
+}
+
+// Exit-code policy, stated wherever the classification is shown.
+export function policyLine(rerunFailed) {
+  return rerunFailed > 0
+    ? `policy: --rerun-failed ${rerunFailed} is set, so a flaky test does not fail the run; a test that fails every re-run still exits 1`
+    : 'policy: --rerun-failed is off, so every failure fails the run (pass --rerun-failed <n> to classify flaky)';
+}
+
+export function formatFailureSummary({ failed = [], flaky = [], rerunFailed = 0 }) {
+  const lines = [`${SUMMARY_TAG} ${failed.length} failed, ${flaky.length} flaky (every failed test is named below)`];
+  for (const record of failed) {
+    const origin = record.firstSeenOnRerun ? ` (first seen on re-run ${record.firstSeenOnRerun})` : '';
+    lines.push(`${SUMMARY_TAG} FAILED ${record.file} > ${record.name}${origin}`);
+  }
+  for (const record of flaky) {
+    lines.push(
+      `${SUMMARY_TAG} FLAKY ${record.file} > ${record.name} (failed, then passed on re-run ${record.passedOnAttempt}/${rerunFailed})`
+    );
+  }
+  if (failed.length > 0 || flaky.length > 0) lines.push(`${SUMMARY_TAG} ${policyLine(rerunFailed)}`);
+  return `${lines.join('\n')}\n`;
+}

package/scripts/lib/test-summary-reporter.mjs

@@ -1,14 +1,18 @@
 import { Readable } from 'node:stream';
 import { spec } from 'node:test/reporters';
+import { createFailureRecorder } from './test-failure-records.mjs';
 import timingReporter from './test-timing-reporter.mjs';
 
 // Filter structured success events, never arbitrary stdout/stderr text.
 // Node's own reporter still owns failure diagnostics, warnings and totals.
-// The runner writes an unfiltered spec report alongside this summary.
+// The runner writes an unfiltered spec report alongside this summary, and
+// every failure is also recorded for the run's final failure summary.
 export default async function* summaryReporter(source) {
   const timings = [];
+  const recordFailure = createFailureRecorder();
   async function* events() {
     for await (const event of source) {
+      recordFailure(event);
       if ((event.type === 'test:pass' || event.type === 'test:fail') && event.data.nesting === 0) {
         timings.push({
           type: event.type,
@@ -17,8 +21,8 @@ export default async function* summaryReporter(source) {
       }
       // A start without its matching pass leaves a pending name in spec's
       // parent stack. Suppress both; fail events carry their own location.
-      if (event.type !== 'test:start' &&
-          (event.type !== 'test:pass' || event.data.skip || event.data.todo)) yield event;
+      if (event.type !== 'test:start' && (event.type !== 'test:pass' || event.data.skip || event.data.todo))
+        yield event;
     }
   }
   yield* Readable.from(events()).pipe(new spec());

package/scripts/mouse-probe.mjs

@@ -29,10 +29,8 @@ process.on('SIGINT', cleanup);
 process.stdin.on('data', (buf) => {
   const s = buf.toString('utf8');
   if (s === 'q' || s === '\x03') return cleanup();
-  let m;
-  MOUSE.lastIndex = 0;
   let sawMouse = false;
-  while ((m = MOUSE.exec(s)) !== null) {
+  for (const m of s.matchAll(MOUSE)) {
     sawMouse = true;
     const b = Number(m[1]);
     const mods = [b & 4 ? 'shift' : '', b & 8 ? 'alt' : '', b & 16 ? 'ctrl' : ''].filter(Boolean).join('+') || 'none';