@joshuafolkken/kit 1.2116.0 → 1.3000.0

package/scripts/test/test-pinned-type-check.ts

@@ -0,0 +1,110 @@
+import { writeFile } from 'node:fs/promises'
+import path from 'node:path'
+import type { BaseTree } from '#scripts/git/base-tree'
+import { import_graph } from '#scripts/lint/import-graph'
+import { execa } from 'execa'
+import { test_pinned_reach } from './test-pinned-reach'
+
+// The type-check stage of a `josh test:pinned` probe (joshuafolkken/kit#5385): a hunk its direct
+// importers' tests leave green is type-checked with the hunk reverted before the whole reach runs. A
+// change only the types pin — a key added to a loose schema, a constant added to a union — reads red
+// here, so no full-reach probe runs for it.
+//
+// **The check is scoped to the file and its direct importers** through a throwaway tsconfig that
+// extends the tree's own, never the whole project. Its build info lives beside the tree, so a lane's
+// later hunks reuse it. The scope is checked before the revert too, and a red there answers nothing
+// (`test-pinned-probe.ts`), so a red is the revert's alone — a tree whose types do not build without
+// generated files, as a SvelteKit consumer's at HEAD, reads green. A file no graph reaches, or one
+// TypeScript does not check, is scoped to nothing and reads green — the whole reach answers it as
+// before.
+
+const CONFIG_NAME = 'tsconfig.josh-pinned.json'
+const BASE_CONFIG = './tsconfig.json'
+const BUILD_INFO = 'josh-pinned.tsbuildinfo'
+const TSC_BIN = path.join('node_modules', '.bin', 'tsc')
+const TYPED_FILE = /\.[cm]?tsx?$/u
+const TYPE_ERROR = /error TS\d+/u
+
+// Off for the scoped check: a revert that leaves an import, local or parameter from another hunk
+// unread, or code unreachable, is no type contract pinning the hunk, so it must not read red.
+const LINT_OPTIONS_OFF = {
+	noUnusedLocals: false,
+	noUnusedParameters: false,
+	allowUnreachableCode: true,
+}
+
+function is_typed(file: string): boolean {
+	return TYPED_FILE.test(file)
+}
+
+// The file and each typed file of the work tree that imports it directly, relative to the work tree;
+// empty where the file is not typed or no graph can be built.
+async function scope_of(file: string): Promise<Array<string>> {
+	if (!is_typed(file)) return []
+	const graph = await import_graph.create(process.cwd())
+	if (graph === undefined) return []
+	const target = path.resolve(file)
+	const universe = await test_pinned_reach.code_universe()
+	const readers = universe.filter(
+		(reader) => is_typed(reader) && graph.imports(reader).includes(target),
+	)
+
+	return [file, ...readers.map((reader) => path.relative(process.cwd(), reader))]
+}
+
+// The throwaway tsconfig: the tree's own, narrowed to `scope`, its build info at `build_info`.
+function config_text(scope: ReadonlyArray<string>, build_info: string): string {
+	const compiler_options = {
+		...LINT_OPTIONS_OFF,
+		noEmit: true,
+		incremental: true,
+		tsBuildInfoFile: build_info,
+	}
+	const config = {
+		extends: BASE_CONFIG,
+		compilerOptions: compiler_options,
+		include: [],
+		files: scope,
+	}
+
+	return `${JSON.stringify(config, undefined, '\t')}\n`
+}
+
+// The scoped check's outcome: `ran` is whether tsc started and exited on its own, so a tsc that
+// could not start or was killed is told from a clean pass (joshuafolkken/kit#5505).
+interface TypeCheck {
+	ran: boolean
+	red: boolean
+}
+
+// Type-checks `scope` in the tree as it stands. `--pretty false` keeps a color-forcing environment
+// from splitting `TYPE_ERROR` with escape codes (joshuafolkken/kit#5505).
+async function check(tree: BaseTree, scope: ReadonlyArray<string>): Promise<TypeCheck> {
+	const build_info = path.join(tree.parent, BUILD_INFO)
+
+	await writeFile(path.join(tree.tree, CONFIG_NAME), config_text(scope, build_info))
+	const result = await execa(
+		path.join(tree.tree, TSC_BIN),
+		['-p', CONFIG_NAME, '--pretty', 'false'],
+		{
+			cwd: tree.tree,
+			reject: false,
+			all: true,
+		},
+	)
+
+	return { ran: result.exitCode !== undefined, red: TYPE_ERROR.test(result.all) }
+}
+
+// Whether `scope` reads a type error in the tree as it stands. A tsc that cannot start reads no error.
+async function is_red(tree: BaseTree, scope: ReadonlyArray<string>): Promise<boolean> {
+	if (scope.length === 0) return false
+
+	const outcome = await check(tree, scope)
+
+	return outcome.red
+}
+
+const test_pinned_type_check = { check, config_text, is_red, scope_of, CONFIG_NAME }
+
+export { test_pinned_type_check }

package/scripts/test/test-pinned.ts

@@ -6,6 +6,7 @@ import { git_spawn } from '#scripts/git/git-spawn'
 import { review_code_tree } from '#scripts/review/review-code-tree'
 import { test_pinned_cache, type LoadedCache, type Reached } from './test-pinned-cache'
 import { test_pinned_cache_logic, type Known, type Recheck } from './test-pinned-cache-logic'
+import { test_pinned_carry } from './test-pinned-carry'
 import { test_pinned_emit } from './test-pinned-emit'
 import { test_pinned_functions } from './test-pinned-functions'
 import type { PinnedOrigin } from './test-pinned-ledger'
@@ -15,19 +16,20 @@ import {
 	type HunkOutcome,
 	type PinnedReport,
 } from './test-pinned-logic'
+import { test_pinned_moves } from './test-pinned-moves'
 import { test_pinned_probe, type CheckOptions, type Probe } from './test-pinned-probe'
 import { test_pinned_reach, type Reach } from './test-pinned-reach'
+import { test_pinned_snapshot } from './test-pinned-snapshot'
 import type { Staged } from './test-pinned-stage'
 import { test_red } from './test-red'
 import type { RedProbe } from './test-red-logic'
 
 // `josh test:pinned` (joshuafolkken/kit#4516) — put each changed hunk of a non-test code file back to
 // the merge-base's lines, one at a time, run that file's related tests, and list the hunks no test went
-// red on: the "reverting it stays green" finding a review used to stop a lane on.
+// red on, so reviews can identify changes whose behavior the tests do not pin.
 //
-// **The reverts happen in a detached worktree at HEAD, never the caller's checkout** (`base-tree.ts`).
-// The changed files are copied in from the working tree first, so what is reverted is the change as it
-// stands, committed or not, and each hunk's file is restored before the next one runs.
+// The entire read runs inside an immutable snapshot (`test-pinned-snapshot.ts`). Each probe gets a
+// detached tree at that snapshot's HEAD, and each hunk's file is restored before the next one runs.
 
 const HEAD = 'HEAD'
 
@@ -42,17 +44,21 @@ interface ProbeRead extends StubRead {
 }
 
 interface RunOptions {
+	cache_file?: string
 	max_hunks: number
 	is_cached_only: boolean
 	is_first_unpinned?: boolean
 	rewired?: number
+	over_cap?: number
 	origin?: PinnedOrigin | undefined
 }
 
-// One run's probing: the checked hunks, what the cache knows of each, and the cache they are written to.
+// One run's probing: the checked hunks, what the cache knows of each, each file's test count, and the
+// cache they are written to.
 interface ProbeRun {
 	hunks: ReadonlyArray<Hunk>
 	known: ReadonlyArray<Known>
+	counts: ReadonlyMap<string, number>
 	cache: LoadedCache
 	keys: ReadonlyArray<string>
 	options: RunOptions
@@ -81,15 +87,6 @@ async function read_hunks(base: string, files: ReadonlyArray<string>): Promise<A
 	)
 }
 
-// The key of each hunk, read off the working file its lines sit in now.
-async function keys_of(hunks: ReadonlyArray<Hunk>): Promise<Array<string>> {
-	const contents = await Promise.all(hunks.map(async (hunk) => await readFile(hunk.file, 'utf8')))
-
-	return test_pinned_cache_logic.distinct(
-		hunks.map((hunk, index) => test_pinned_cache_logic.hunk_key(hunk, contents[index] ?? '')),
-	)
-}
-
 // The verdict the cache already holds; one left to recheck holds none yet.
 function held_of(known: Known): RedProbe | undefined {
 	return test_pinned_cache_logic.is_recheck(known) ? undefined : known
@@ -106,13 +103,21 @@ function probes_of(hunks: ReadonlyArray<Hunk>, known: ReadonlyArray<Known>): Arr
 		.filter((probe) => held_of(known[probe.index]) === undefined)
 }
 
-// The probes in the order they run: the fewest tests reached first (`test_pinned_logic.by_reach`),
-// each file's count read once and carried on its probes, which a wide one runs in stages by
-// (`test-pinned-stage.ts`); a file with no graph to read counts as 0.
-async function by_reach(probes: ReadonlyArray<Probe>, reach: Reach): Promise<Array<Probe>> {
-	const files = test_pinned_cache_logic.distinct(probes.map((probe) => probe.hunk.file))
+// Each hunk file's count of the test files it reaches, read once; a file with no graph to read counts
+// as 0.
+async function counts_of(
+	hunks: ReadonlyArray<Hunk>,
+	reach: Reach,
+): Promise<ReadonlyMap<string, number>> {
+	const files = test_pinned_cache_logic.distinct(hunks.map((hunk) => hunk.file))
 	const related = await Promise.all(files.map(async (file) => await reach.related(file)))
-	const counts = new Map(files.map((file, index) => [file, related[index]?.length ?? 0]))
+
+	return new Map(files.map((file, index) => [file, related[index]?.length ?? 0]))
+}
+
+// The probes in the order they run: the fewest tests reached first (`test_pinned_logic.by_reach`),
+// each file's count carried on its probes, which a wide one runs in stages by (`test-pinned-stage.ts`).
+function by_reach(probes: ReadonlyArray<Probe>, counts: ReadonlyMap<string, number>): Array<Probe> {
 	const counted = probes.map((probe) => ({ ...probe, reach: counts.get(probe.hunk.file) }))
 
 	return test_pinned_logic.by_reach(counted, counts)
@@ -127,19 +132,31 @@ function check_options(held: number, options: RunOptions): CheckOptions {
 	return {
 		held,
 		rewired: options.rewired ?? 0,
+		over_cap: options.over_cap ?? 0,
 		is_first_unpinned: options.is_first_unpinned === true,
 		origin: options.origin,
 	}
 }
 
+async function with_spawned(probes: ReadonlyArray<Probe>, reach: Reach): Promise<Array<Probe>> {
+	const files = [...new Set(probes.map((probe) => probe.hunk.file))]
+	const spawned = await Promise.all(files.map(async (file) => await reach.spawned(file)))
+	const by_file = new Map(files.map((file, index) => [file, spawned[index]]))
+
+	return probes.map((probe) => ({ ...probe, spawned: by_file.get(probe.hunk.file) }))
+}
+
 // Each hunk's verdict: the cached one where it still holds, else a fresh run's, in the hunks' order.
 // The held ones are written first and each fresh one as it is reached (`test_pinned_cache.writer`).
 // Under `is_first_unpinned` a hunk the run stopped before keeps no verdict.
 async function verdicts_on(paths: ReadonlyArray<string>, probing: ProbeRun): Promise<Verdicts> {
 	const verdicts = probing.known.map((entry) => held_of(entry))
 	const reached = reached_of(verdicts)
-	const probes = await by_reach(probes_of(probing.hunks, probing.known), probing.cache.reach)
-	const writer = test_pinned_cache.writer(probing.cache, probing.keys)
+	const probes = await with_spawned(
+		by_reach(probes_of(probing.hunks, probing.known), probing.counts),
+		probing.cache.reach,
+	)
+	const writer = test_pinned_cache.writer({ ...probing.cache, hunks: probing.hunks }, probing.keys)
 	const options = check_options(reached.length, probing.options)
 
 	await writer.record(reached)
@@ -175,18 +192,21 @@ function cut_of(
 	return cut.length === 0 ? {} : { cut }
 }
 
+// The hunks over the test-file cap (joshuafolkken/kit#5503); none, no `over_cap` key.
+function over_cap_of(over_cap: ReadonlyArray<Hunk>): Pick<HunkOutcome, 'over_cap'> {
+	return over_cap.length === 0 ? {} : { over_cap }
+}
+
 // A fresh cached verdict is reused without a run (`test-pinned-cache.ts`); every hunk's verdict is
 // stored again, so a rerun on the same tree starts no vitest. `is_cached_only` runs no probe: only the
 // verdicts the cache holds are read, and nothing is written.
-async function outcome_on(
-	base: string,
+async function checked_outcome(
 	paths: ReadonlyArray<string>,
-	hunks: ReadonlyArray<Hunk>,
+	read: Pick<ProbeRun, 'hunks' | 'counts' | 'cache'>,
 	options: RunOptions,
 ): Promise<HunkOutcome> {
-	if (hunks.length === 0) return NO_OUTCOME
-	const cache = await test_pinned_cache.load(base, hunks)
-	const keys = await keys_of(hunks)
+	const { hunks, cache } = read
+	const keys = test_pinned_carry.keys(cache, hunks)
 	const known = await test_pinned_cache.known(
 		cache,
 		keys,
@@ -200,11 +220,39 @@ async function outcome_on(
 		)
 	}
 
-	const { verdicts, staged } = await verdicts_on(paths, { hunks, known, cache, keys, options })
+	const { verdicts, staged } = await verdicts_on(paths, { ...read, known, keys, options })
 
 	return { ...outcome_of(hunks, verdicts), ...cut_of(hunks, verdicts), staged }
 }
 
+async function hinted_hunks(outcome: HunkOutcome, reach: Reach): Promise<HunkOutcome['unpinned']> {
+	return await Promise.all(
+		outcome.unpinned.map(async (hunk) => ({
+			...hunk,
+			related_tests: await reach.related(hunk.file),
+		})),
+	)
+}
+
+// A hunk over the test-file cap is never probed; both modes report it alike, and the fresh run's
+// ledger row counts it.
+async function outcome_on(
+	cache: LoadedCache,
+	paths: ReadonlyArray<string>,
+	changed: ReadonlyArray<Hunk>,
+	options: RunOptions,
+): Promise<HunkOutcome> {
+	if (changed.length === 0) return NO_OUTCOME
+	const counts = await counts_of(changed, cache.reach)
+	const { checked, unchecked } = test_pinned_logic.capped(changed, counts)
+	const capped_options = { ...options, over_cap: unchecked.length }
+	const outcome = await checked_outcome(paths, { hunks: checked, counts, cache }, capped_options)
+
+	const unpinned = await hinted_hunks(outcome, cache.reach)
+
+	return { ...outcome, unpinned, ...over_cap_of(unchecked) }
+}
+
 // A target the base's tree does not hold is added — committed on the branch or still untracked.
 async function added_scripts(base: string, targets: ReadonlyArray<string>): Promise<Array<string>> {
 	const held = new Set(await changed_paths.read_held_at(base, targets))
@@ -229,11 +277,12 @@ async function stub_hunks_of(added: ReadonlyArray<string>): Promise<StubRead> {
 // (`test-pinned-emit.ts`), so neither is `unchecked`; without `typescript` every hunk is kept, so the
 // run checks more rather than less.
 async function runtime_hunks(hunks: ReadonlyArray<Hunk>): Promise<ProbeRead> {
+	const grouped = test_pinned_moves.group(hunks)
 	const api = review_code_tree.load_typescript(process.cwd())
-	if (api === undefined) return { hunks: [...hunks], unparsed: [], rewired: 0 }
-	const contents = await Promise.all(hunks.map(async (hunk) => await readFile(hunk.file, 'utf8')))
+	if (api === undefined) return { hunks: grouped, unparsed: [], rewired: 0 }
+	const contents = await Promise.all(grouped.map(async (hunk) => await readFile(hunk.file, 'utf8')))
 
-	return { ...test_pinned_emit.probed_of(api, hunks, contents), unparsed: [] }
+	return { ...test_pinned_emit.probed_of(api, grouped, contents), unparsed: [] }
 }
 
 async function read_probes(base: string, targets: ReadonlyArray<string>): Promise<ProbeRead> {
@@ -265,8 +314,10 @@ async function report_on(
 		changed.filter((file) => test_pinned_logic.is_target(file)),
 	)
 	const probes = await read_probes(base, targets)
-	const { checked, unchecked } = test_pinned_logic.split(probes.hunks, has_runner ? max_hunks : 0)
-	const outcome = await outcome_on(base, paths, checked, { ...options, rewired: probes.rewired })
+	const cache = await test_pinned_cache.load(base, probes.hunks, options.cache_file)
+	const { checked, unchecked } = await test_pinned_carry.select(cache, has_runner ? max_hunks : 0)
+	if (!options.is_cached_only) test_pinned_carry.record(cache, unchecked)
+	const outcome = await outcome_on(cache, paths, checked, { ...options, rewired: probes.rewired })
 
 	return { ...outcome, unchecked, unparsed: probes.unparsed, max_hunks, has_runner }
 }
@@ -276,7 +327,13 @@ async function report(options: RunOptions): Promise<PinnedReport> {
 	const has_runner = test_red.has_runner()
 	if (base === undefined) return { ...NO_REPORT, max_hunks: options.max_hunks, has_runner }
 
-	return await report_on(base, options, has_runner)
+	const root = process.cwd()
+	const result = await test_pinned_snapshot.with_snapshot(async (snapshot) => ({
+		...(await report_on(base, { ...options, cache_file: snapshot.cache_file }, has_runner)),
+		tree_hash: snapshot.tree_hash,
+	}))
+
+	return { ...result, is_stale: test_pinned_snapshot.current_hash(root) !== result.tree_hash }
 }
 
 // No merge-base (not a repository, no default branch) checks nothing, as `josh test:red` does. The cap

package/scripts/test/test-red-logic.ts

@@ -13,13 +13,16 @@ import { test_declared_logic } from './test-declared-logic'
 // of them has to fail there.
 
 type RedVerdict = 'red' | 'green' | 'no-test' | 'stopped' | 'test-only'
+type PinnedBy = 'type-check'
 
 // One run's verdict and the test files that failed in it. `ran` is the number of test files the run
-// ran, absent on a verdict read back from a cache (joshuafolkken/kit#5078).
+// ran, absent on a verdict read back from a cache (joshuafolkken/kit#5078). `pinned_by` marks a `red`
+// the type check reached, not a test (joshuafolkken/kit#5385).
 interface RedProbe {
 	verdict: RedVerdict
 	failed: ReadonlyArray<string>
 	ran?: number
+	pinned_by?: PinnedBy
 }
 
 // The declaration a bug-fix Issue carries under its background heading, in the same slot as the
@@ -91,6 +94,39 @@ function failed_files(report: string, roots: ReadonlyArray<string>): Array<strin
 		.flatMap((result) => relative_in(roots, result.name))
 }
 
+// joshuafolkken/kit#6814: a probe that outruns its timeout writes no JSON report, so a reporter of its
+// own appends one line per test file vitest queued and ended. The files queued and never ended are the
+// ones still running when the probe was killed — the names the timeout failure carries.
+const QUEUED_EVENT = 'queued'
+const END_EVENT = 'end'
+
+function progress_reporter(progress: string): string {
+	const target = JSON.stringify(progress)
+
+	return [
+		"import { appendFileSync } from 'node:fs'",
+		'export default class {',
+		`\tonTestModuleQueued(module) { appendFileSync(${target}, \`${QUEUED_EVENT} \${module.moduleId}\\n\`) }`,
+		`\tonTestModuleEnd(module) { appendFileSync(${target}, \`${END_EVENT} \${module.moduleId}\\n\`) }`,
+		'}',
+		'',
+	].join('\n')
+}
+
+function files_of(lines: ReadonlyArray<string>, event: string): Array<string> {
+	const prefix = `${event} `
+
+	return lines.filter((line) => line.startsWith(prefix)).map((line) => line.slice(prefix.length))
+}
+
+function unfinished_files(progress: string, roots: ReadonlyArray<string>): Array<string> {
+	const lines = progress.split('\n')
+	const ended = new Set(files_of(lines, END_EVENT))
+	const running = files_of(lines, QUEUED_EVENT).filter((file) => !ended.has(file))
+
+	return [...new Set(running)].flatMap((file) => relative_in(roots, file))
+}
+
 // `no-test` when nothing ran and vitest passed with no tests; `stopped` when nothing ran and it failed
 // anyway — its config could not load, so no test was ever asked (joshuafolkken/kit#5043). Otherwise a
 // failing run is `red` — the test caught the pre-fix tree — and a passing one is `green`.
@@ -114,10 +150,12 @@ const test_red_logic = {
 	is_bug_fix,
 	is_refused,
 	is_test_only,
+	progress_reporter,
 	ran_file_count,
+	unfinished_files,
 	unit_test_files,
 	verdict_for,
 }
 
-export type { RedProbe, RedVerdict }
+export type { PinnedBy, RedProbe, RedVerdict }
 export { test_red_logic }

package/scripts/test/test-red.ts

@@ -1,5 +1,5 @@
 import { existsSync } from 'node:fs'
-import { readFile, realpath, rm } from 'node:fs/promises'
+import { readFile, realpath, rm, writeFile } from 'node:fs/promises'
 import path from 'node:path'
 import { base_tree, type BaseTree } from '#scripts/git/base-tree'
 import { change_base } from '#scripts/git/change-base'
@@ -18,6 +18,12 @@ import { vitest_cache_directory } from './vitest-cache-directory'
 
 const TREE_PREFIX = 'josh-test-red-'
 const REPORT_NAME = 'report.json'
+const PROGRESS_NAME = 'progress.log'
+const REPORTER_NAME = 'progress-reporter.mjs'
+// A probe this long is spinning, not slow: it stops well inside the harness cap a detached `josh ship`
+// runs `test:pinned` under (joshuafolkken/kit#6814).
+const PROBE_TIMEOUT_MS = 300_000
+const KILL_SIGNAL = 'SIGKILL'
 // Beside the tree, so the run's dependency optimizer never rewrites the borrowed `node_modules/.vite`
 // a gate beside it imports from (`vitest-cache-directory.ts`).
 const CACHE_NAME = 'vite-cache'
@@ -30,6 +36,11 @@ interface RedRun {
 	files: ReadonlyArray<string>
 }
 
+interface VitestRun {
+	is_failed: boolean
+	timed_out: boolean
+}
+
 // The changed Vitest files that still exist — a deleted test has nothing to run.
 function changed_tests(paths: ReadonlyArray<string>): Array<string> {
 	return test_red_logic.unit_test_files(paths).filter((file) => existsSync(file))
@@ -45,23 +56,57 @@ async function read_report(report: string): Promise<string> {
 	return existsSync(report) ? await readFile(report, 'utf8') : ''
 }
 
-// The verdict of one run over `selection` — `run <files>`, or `related <file> --run` for
-// `josh test:pinned`. `--passWithNoTests` keeps "the filter matched nothing" from reading as a
-// failure, and the JSON report — written beside the tree, never into it — says whether anything ran at
-// all, so a file the old config excludes is `no-test` rather than a `green` that refuses a correct fix.
-// A test that ran and broke, or a file that could not load against the old code, makes it `red`, and
-// the files it broke in come back beside the verdict. The report is removed first, so a run that writes
-// none never reads the one before it.
-async function probe_in(
-	{ parent, tree }: BaseTree,
-	selection: ReadonlyArray<string>,
-): Promise<RedProbe> {
-	const report = path.join(parent, REPORT_NAME)
-	const vitest_args = [...selection, '--passWithNoTests', ...reporter_arguments(report)]
+// The reporter is written beside the tree, never into it, and both outputs are removed first, so a run
+// that writes none never reads the one before it.
+async function prepare_outputs(parent: string, outputs: ReadonlyArray<string>): Promise<string> {
+	const reporter = path.join(parent, REPORTER_NAME)
+
+	await Promise.all(
+		outputs.map(async (output) => {
+			await rm(output, { force: true })
+		}),
+	)
+	await writeFile(reporter, test_red_logic.progress_reporter(path.join(parent, PROGRESS_NAME)))
+
+	return reporter
+}
+
+// The run is detached so its pid leads a process group holding vitest and its fork workers: a worker
+// spinning in a test never answers its parent, so only signalling the whole group stops it. A detached
+// run sits outside its caller's group and execa's exit cleanup, so every group still running is killed
+// when this process exits — a `test:pinned` or `test:red` stopped by SIGTERM, SIGINT or SIGHUP ends
+// through `process.exit` (`base-tree.ts`).
+const live_groups = new Set<number | undefined>()
+
+function kill_group(pid: number | undefined): void {
+	if (pid === undefined) return
+
+	try {
+		process.kill(-pid, KILL_SIGNAL)
+	} catch {
+		// It ended before the signal; there is nothing left to stop.
+	}
+}
+
+function stop_live_probes(): void {
+	for (const pid of live_groups) kill_group(pid)
+}
 
-	await rm(report, { force: true })
-	const result = await execa(path.join(tree, VITEST_BIN), vitest_args, {
+process.on('exit', stop_live_probes)
+
+// What `vitest_in` reads of the run: the group leader's pid and how the run ended.
+type VitestProcess = Promise<{
+	exitCode?: number | undefined
+	signal?: NodeJS.Signals | undefined
+}> & { pid?: number | undefined }
+
+function spawn_vitest(
+	{ parent, tree }: BaseTree,
+	vitest_args: ReadonlyArray<string>,
+): VitestProcess {
+	return execa(path.join(tree, VITEST_BIN), vitest_args, {
 		cwd: tree,
+		detached: true,
 		env: {
 			...git_location_environment.location_free_environment(),
 			[vitest_cache_directory.CACHE_DIRECTORY_KEY]: path.join(parent, CACHE_NAME),
@@ -69,12 +114,79 @@ async function probe_in(
 		extendEnv: true,
 		reject: false,
 	})
+}
+
+// A timeout counts only when its kill ended the run: one that finished just as the timer fired is read
+// by its own exit code.
+async function vitest_in(
+	base: BaseTree,
+	vitest_args: ReadonlyArray<string>,
+	timeout_ms: number,
+): Promise<VitestRun> {
+	const state = { timed_out: false }
+	const subprocess = spawn_vitest(base, vitest_args)
+
+	live_groups.add(subprocess.pid)
+	const timer = setTimeout(() => {
+		state.timed_out = true
+		kill_group(subprocess.pid)
+	}, timeout_ms)
+	const result = await subprocess
+
+	clearTimeout(timer)
+	live_groups.delete(subprocess.pid)
+
+	return {
+		is_failed: result.exitCode !== 0,
+		timed_out: state.timed_out && result.signal === KILL_SIGNAL,
+	}
+}
+
+async function timeout_error(
+	progress: string,
+	roots: ReadonlyArray<string>,
+	timeout_ms: number,
+): Promise<Error> {
+	const running = test_red_logic.unfinished_files(await read_report(progress), roots)
+	const names = running.length > 0 ? running.join(', ') : 'no test file had been queued'
+
+	return new Error(
+		`vitest probe passed its ${String(timeout_ms)} ms timeout, still running: ${names}`,
+	)
+}
+
+// The verdict of one run over `selection` — `run <files>`, or `related <file> --run` for
+// `josh test:pinned`. `--passWithNoTests` keeps "the filter matched nothing" from reading as a
+// failure, and the JSON report — written beside the tree, never into it — says whether anything ran at
+// all, so a file the old config excludes is `no-test` rather than a `green` that refuses a correct fix.
+// A test that ran and broke, or a file that could not load against the old code, makes it `red`, and
+// the files it broke in come back beside the verdict. A run past `timeout_ms` is killed and throws,
+// naming the test files still running (joshuafolkken/kit#6814).
+async function probe_in(
+	base: BaseTree,
+	selection: ReadonlyArray<string>,
+	timeout_ms = PROBE_TIMEOUT_MS,
+): Promise<RedProbe> {
+	const report = path.join(base.parent, REPORT_NAME)
+	const progress = path.join(base.parent, PROGRESS_NAME)
+	const reporter = await prepare_outputs(base.parent, [report, progress])
+	const vitest_args = [
+		...selection,
+		'--passWithNoTests',
+		...reporter_arguments(report),
+		`--reporter=${reporter}`,
+	]
+	const outcome = await vitest_in(base, vitest_args, timeout_ms)
+	const roots = [base.tree, await realpath(base.tree)]
+
+	if (outcome.timed_out) throw await timeout_error(progress, roots, timeout_ms)
+
 	const text = await read_report(report)
 	const ran = test_red_logic.ran_file_count(text)
 
 	return {
-		verdict: test_red_logic.verdict_for(ran, result.exitCode !== 0),
-		failed: test_red_logic.failed_files(text, [tree, await realpath(tree)]),
+		verdict: test_red_logic.verdict_for(ran, outcome.is_failed),
+		failed: test_red_logic.failed_files(text, roots),
 		ran,
 	}
 }
@@ -123,7 +235,7 @@ function has_runner(): boolean {
 	return existsSync(VITEST_BIN)
 }
 
-const test_red = { has_runner, main, probe_in, run }
+const test_red = { has_runner, main, probe_in, run, stop_live_probes }
 
 export type { RedRun }
 export { test_red }