@joshuafolkken/kit 1.1471.0 → 1.1784.0

package/scripts/gate/check-cpu.ts

@@ -0,0 +1,60 @@
+import {
+	run_board_usage_rusage,
+	type RusageReader,
+} from '#scripts/run/board/run-board-usage-rusage'
+
+// **The CPU a check's process tree spent, read off the gate's own reaped-children time**
+// (joshuafolkken/kit#4362). Each check is a `pnpm josh <target>` child that waits for its tool before it
+// exits, so once the gate has reaped it the whole tree's CPU sits in the gate's `proc_pid_rusage`
+// child times (`run-board-usage-rusage.ts`, shared rather than read twice). The reader folds the
+// gate's own time in, so `process.cpuUsage()` takes it back out, and each finished check takes the
+// growth since the previous one finished.
+//
+// **A reading is an attribution, not an isolation**: any other child the gate reaped in between — a
+// `ps` sample, a git call — lands on the next check that finishes, and two checks reaped in one tick
+// split their CPU unevenly. Both are small beside a check's tools. Off macOS, or without `node:ffi`,
+// there is no reader and every check records none.
+
+const US_PER_MS = 1000
+
+// The CPU milliseconds spent by the children reaped since the previous call, `undefined` where the
+// gate's own usage cannot be read.
+type CheckCpu = () => number | undefined
+
+function own_cpu_ms(): number {
+	const usage = process.cpuUsage()
+
+	return (usage.user + usage.system) / US_PER_MS
+}
+
+function reaped_cpu_ms(read: RusageReader | undefined, pid: number): number | undefined {
+	const reading = read?.(pid)
+
+	return reading === undefined ? undefined : reading.cpu_ms - own_cpu_ms()
+}
+
+// One gate's reader, its baseline taken now — before any check is reaped.
+function check_cpu_reader(
+	read: RusageReader | undefined = run_board_usage_rusage.rusage_reader(
+		process.platform,
+		run_board_usage_rusage.load_builtin,
+	),
+	pid: number = process.pid,
+): CheckCpu {
+	const previous = { reaped_ms: reaped_cpu_ms(read, pid) }
+
+	return function since_previous(): number | undefined {
+		const reaped_ms = reaped_cpu_ms(read, pid)
+		const before = previous.reaped_ms
+
+		previous.reaped_ms = reaped_ms
+		if (reaped_ms === undefined || before === undefined) return undefined
+
+		return Math.max(0, reaped_ms - before)
+	}
+}
+
+const check_cpu = { check_cpu_reader }
+
+export type { CheckCpu }
+export { check_cpu }

package/scripts/gate/check-inputs.ts

@@ -0,0 +1,73 @@
+import path from 'node:path'
+import { PROJECT_ROOT } from '#scripts/init/init-paths'
+import { test_related } from '#scripts/test/test-related'
+import {
+	verification_inputs,
+	type InputDeclaration,
+	type PrettierIgnored,
+} from './verification-inputs'
+
+// The file-level answer of a check's declaration (`verification-inputs.ts`): whether any of the paths
+// that moved is one the check reads, so a gate re-run reuses a check none of whose inputs moved
+// (`check-reuse.ts`). It lives apart from the declarations because following imports reads the
+// test-read map (`test-related.ts`), whose own since-green narrowing reads the declarations.
+//
+// **The unit step's inputs are the test-read map's, not a guess** (joshuafolkken/kit#4314): it is
+// reusable only when `josh test:related` would run no test for what moved — every moved file one no
+// recorded test reads. No map, or one recorded on a tree git can no longer diff against, reaches every
+// test, so the step runs.
+
+function prettier_ignored(root: string): PrettierIgnored {
+	const ignore_path = verification_inputs.PRETTIER_IGNORE_FILES.map((name) => path.join(root, name))
+
+	return async function is_ignored(relative: string): Promise<boolean> {
+		try {
+			const prettier = await import('prettier')
+			const info = await prettier.getFileInfo(path.join(root, relative), {
+				ignorePath: ignore_path,
+			})
+
+			return info.ignored
+		} catch {
+			return false
+		}
+	}
+}
+
+async function reads_any(
+	declaration: InputDeclaration,
+	paths: ReadonlyArray<string>,
+	is_ignored: PrettierIgnored,
+	root: string,
+): Promise<boolean> {
+	if (declaration.reads === 'recorded') return await test_related.reaches_tests(root, paths)
+
+	const { is_input } = declaration
+
+	if (is_input === undefined) return true
+
+	const answers = await Promise.all(
+		paths.map(async (relative) => await is_input(relative, is_ignored)),
+	)
+
+	return answers.includes(true)
+}
+
+// Whether any of `paths` (repository-root relative) is an input of the check `label`. An undeclared
+// check answers `true` for every set of paths, so the only way to reuse it is never.
+async function touches_inputs(
+	label: string,
+	paths: ReadonlyArray<string>,
+	is_ignored: PrettierIgnored,
+	root: string = PROJECT_ROOT,
+): Promise<boolean> {
+	const declaration = verification_inputs.declaration_of(label)
+
+	if (declaration === undefined || verification_inputs.configures(label, paths)) return true
+
+	return await reads_any(declaration, paths, is_ignored, root)
+}
+
+const check_inputs = { prettier_ignored, touches_inputs }
+
+export { check_inputs }

package/scripts/gate/check-reuse.ts

@@ -0,0 +1,141 @@
+import { git_command } from '#scripts/git/git-command'
+import { PROJECT_ROOT } from '#scripts/init/init-paths'
+import {
+	file_map_stamp,
+	type FileMapStamp,
+	type FileMapStampAccess,
+} from '#scripts/josh/file-map-stamp'
+import { stamp_file } from '#scripts/josh/stamp-file'
+import { review_tree } from '#scripts/review/review-tree'
+import { check_inputs } from './check-inputs'
+import { gate_report, type GateStepResult } from './gate-report'
+import type { GateTree } from './gate-tree'
+import { moved_paths } from './moved-paths'
+import { verification_inputs, type PrettierIgnored } from './verification-inputs'
+
+// One green record per declared check, so a gate re-run starts only the checks whose inputs moved
+// (`verification-inputs.ts` declares what each reads). The whole-gate record (`gate-skip.ts`) is refused
+// the moment the merge base moves; a main sync that brought in only `.claude/**/*.md` then re-ran lint
+// and the type check over sources neither had a reason to read again. What moved since a record is
+// `moved-paths.ts`.
+//
+// **A record is reuse of a result, not a check dropped**: no base, an empty changed-file map, an
+// unreadable base diff, no root or a fingerprint of the check's tools and environment that moved
+// answers "not reusable", and an undeclared check is never reusable. The records are written and read
+// only where the CLI entry resolved their destination (`CheckRecords`), so a suite driving the gate
+// never plants a green record over the live tree.
+
+const CHECK_PREFIX = 'josh-gate-check-'
+
+function check_stamp(label: string): FileMapStampAccess {
+	return file_map_stamp.create(CHECK_PREFIX, PROJECT_ROOT, () =>
+		verification_inputs.fingerprint([label]),
+	)
+}
+
+// Where each check's record lives — a parameter so a test plants records in a directory of its own.
+type CheckRecords = (label: string) => string
+
+function default_records(label: string): string {
+	return stamp_file.stamp_path(`${CHECK_PREFIX}${label}-`, PROJECT_ROOT)
+}
+
+interface ReuseContext {
+	tree: GateTree
+	records: CheckRecords
+	is_ignored: PrettierIgnored
+	root: string
+}
+
+async function is_stamp_reusable(
+	label: string,
+	stamp: FileMapStamp,
+	context: ReuseContext,
+): Promise<boolean> {
+	const moved = await moved_paths.moved_since(stamp, context.tree)
+
+	if (moved === undefined || !check_stamp(label).describes_inputs(stamp)) return false
+
+	return !(await check_inputs.touches_inputs(label, moved, context.is_ignored, context.root))
+}
+
+// The time the check's record was taken, or `undefined` when the check has to run.
+async function reused_at(label: string, context: ReuseContext): Promise<string | undefined> {
+	const stamp = verification_inputs.is_declared(label)
+		? check_stamp(label).read(context.records(label))
+		: undefined
+
+	if (stamp === undefined) return undefined
+
+	return (await is_stamp_reusable(label, stamp, context)) ? stamp.taken_at : undefined
+}
+
+async function read_root(): Promise<string | undefined> {
+	try {
+		return await git_command.repository_root()
+	} catch {
+		return undefined
+	}
+}
+
+// Each reusable check's label, mapped to when it was last green.
+async function reusable_checks(
+	labels: ReadonlyArray<string>,
+	tree: GateTree,
+	records: CheckRecords = default_records,
+): Promise<ReadonlyMap<string, string>> {
+	const root = await read_root()
+
+	if (root === undefined) return new Map()
+
+	const context = { tree, records, root, is_ignored: check_inputs.prettier_ignored(root) }
+	const answers = await Promise.all(
+		labels.map(async (label) => [label, await reused_at(label, context)] as const),
+	)
+
+	return new Map(answers.flatMap(([label, at]) => (at === undefined ? [] : [[label, at] as const])))
+}
+
+function is_recordable(result: GateStepResult): boolean {
+	return (
+		verification_inputs.is_declared(result.label) &&
+		!gate_report.is_gate_step_failed(result) &&
+		gate_report.has_nothing_to_say(result)
+	)
+}
+
+function write_records(
+	results: ReadonlyArray<GateStepResult>,
+	tree: GateTree,
+	records: CheckRecords,
+): void {
+	const recordable = results.filter((result) => is_recordable(result))
+
+	for (const { label } of recordable) {
+		check_stamp(label).write(tree.files, records(label), tree.base)
+	}
+}
+
+// Every scoped check that passed silently, recorded against the tree it read — a red gate's green
+// checks included, since each record answers for its own check alone. Nothing is written when the tree
+// moved while the checks ran, and a failed write leaves no record, the safe direction either way.
+async function record_green_checks(
+	results: ReadonlyArray<GateStepResult>,
+	tree: GateTree,
+	records: CheckRecords = default_records,
+): Promise<void> {
+	try {
+		const after = await review_tree.read_changed_tree()
+
+		if (JSON.stringify(after) !== JSON.stringify(tree.files)) return
+
+		write_records(results, { files: after, base: tree.base }, records)
+	} catch {
+		/* no record is the safe answer */
+	}
+}
+
+const check_reuse = { DEFAULT_RECORDS: default_records, record_green_checks, reusable_checks }
+
+export type { CheckRecords }
+export { check_reuse }

package/scripts/gate/command-memory.ts

@@ -0,0 +1,192 @@
+import os from 'node:os'
+import { bash_triggers } from '#scripts/rules/bash-triggers'
+import { git_argv, type ProgramCall } from '#scripts/rules/git-argv'
+import { shell_segments } from '#scripts/rules/shell-segments'
+import { time_shell } from '#scripts/time-runtime/time-shell'
+import { core_weights } from './core-weights'
+
+// The heavy tools a shell line launches, and the memory they hold (joshuafolkken/kit#4582).
+//
+// **`reserved-run` counted a wrapped command's cores and no memory**: a lane ran
+// `reserved-run 4 -- sh -c '… eslint . --concurrency 4 …'`, typed eslint held a TypeScript program per
+// worker, and the 12 GB process was admitted beside four other lanes until the machine swapped. The
+// estimate here is what the wrapper now reserves, and the same reading is what `lane-direct-tool`
+// refuses a multi-worker eslint on.
+//
+// **Every spelling is read down to the program.** `git_argv.program_of` skips the wrapper words, a
+// quoted script handed to a shell is read as its own line (`bash_triggers.shell_script_of`), and the
+// command after `pnpm josh reserved-run <weight> --` is read as the call it wraps, marked reserved. A
+// heredoc body is text, so it is dropped before the line is cut — unless a shell reads it
+// (`bash <<'EOF'`), when it is read as a line of its own, the `sh -c` escape spelled another way.
+//
+// **The estimate is the largest tool's, not the sum**: the commands of one line run one after another,
+// so only one of them is resident at a time. A line with no known tool is `undefined`, the 0 MB it
+// counted before — `reserved-run 1 -- pnpm josh audit` reserves its own, and is not counted twice.
+
+interface ToolCall extends ProgramCall {
+	// Inside a `pnpm josh reserved-run … --`, the budgeted path.
+	is_reserved: boolean
+}
+
+// A line still to be cut into its programs.
+interface PendingLine {
+	line: string
+	is_reserved: boolean
+}
+
+const JOSH_PROGRAM = 'josh'
+const RESERVED_RUN = 'reserved-run'
+const COMMAND_SEPARATOR = '--'
+const ESLINT = 'eslint'
+const VITEST = 'vitest'
+const AUTO_WORKERS = 'auto'
+const LINE_BREAK = '\n'
+// A heredoc fed to a shell — `bash <<'EOF'`, `sh -s <<EOF` — whose body the shell runs.
+const SHELL_HEREDOC = /\b(?:ba|z|da|k)?sh(?:\s+-\w+)*\s*<<-?\s*(['"]?)(\w+)\1/u
+const HEREDOC_DELIMITER_GROUP = 2
+// ESLint 10's `auto` caps its workers at half the available cores (`eslint/lib/eslint/eslint.js`).
+const AUTO_CORE_SHARE = 2
+const SINGLE_WORKER = 1
+// The flag each multi-worker tool takes its worker count from.
+const WORKER_FLAGS: Readonly<Partial<Record<string, string>>> = {
+	[ESLINT]: '--concurrency',
+	[VITEST]: '--maxWorkers',
+}
+
+// The command `pnpm josh reserved-run <weight> [--memory <MB>] -- <command>` wraps, or `undefined`.
+function wrapped_command_of(call: ProgramCall): string | undefined {
+	if (call.name !== JOSH_PROGRAM || call.args[0] !== RESERVED_RUN) return undefined
+
+	const separator = call.args.indexOf(COMMAND_SEPARATOR)
+
+	return separator === -1 ? undefined : call.args.slice(separator + 1).join(' ')
+}
+
+// One command of a line: the program it launches, or the line it hands on — a `reserved-run`'s
+// wrapped command, or a shell's quoted script.
+function reading_of(segment: string, is_reserved: boolean): ToolCall | PendingLine | undefined {
+	const call = git_argv.program_of(segment)
+	const wrapped = call === undefined ? undefined : wrapped_command_of(call)
+
+	if (wrapped !== undefined) return { line: wrapped, is_reserved: true }
+
+	const script = bash_triggers.shell_script_of(segment)
+
+	if (script !== undefined) return { line: script, is_reserved }
+
+	return call === undefined ? undefined : { ...call, is_reserved }
+}
+
+// The lines up to the delimiter alone on its line, or to the end when it never closes.
+function body_until(lines: ReadonlyArray<string>, delimiter: string): string {
+	const end = lines.findIndex((body_line) => body_line.trim() === delimiter)
+
+	return (end === -1 ? lines : lines.slice(0, end)).join(LINE_BREAK)
+}
+
+// The body of every heredoc the line feeds to a shell, each as a line of its own: `bash <<'EOF'` runs
+// its body, where `cat <<EOF` only prints it.
+function shell_heredoc_bodies(line: string): Array<string> {
+	const lines = line.split(LINE_BREAK)
+
+	return lines.flatMap((opening, index) => {
+		const delimiter = SHELL_HEREDOC.exec(opening)?.[HEREDOC_DELIMITER_GROUP]
+
+		return delimiter === undefined ? [] : body_until(lines.slice(index + 1), delimiter)
+	})
+}
+
+function is_tool_call(reading: ToolCall | PendingLine): reading is ToolCall {
+	return 'name' in reading
+}
+
+function is_pending_line(reading: ToolCall | PendingLine): reading is PendingLine {
+	return !is_tool_call(reading)
+}
+
+// Every program the line launches, a shell's quoted script and a `reserved-run`'s command included.
+// A queue rather than recursion: each handed-on line is shorter than the one it came from.
+function tool_calls_of(line: string): Array<ToolCall> {
+	const pending: Array<PendingLine> = [{ line, is_reserved: false }]
+	const calls: Array<ToolCall> = []
+
+	for (let next = pending.shift(); next !== undefined; next = pending.shift()) {
+		const { is_reserved } = next
+		const readings = shell_segments
+			.quote_aware_commands_of(time_shell.without_heredoc_bodies(next.line))
+			.flatMap((segment) => reading_of(segment, is_reserved) ?? [])
+
+		const bodies = shell_heredoc_bodies(next.line).map((body) => ({ line: body, is_reserved }))
+
+		calls.push(...readings.filter((reading) => is_tool_call(reading)))
+		pending.push(...readings.filter((reading) => is_pending_line(reading)), ...bodies)
+	}
+
+	return calls
+}
+
+// `--flag <value>` or `--flag=<value>`.
+function flag_value(args: ReadonlyArray<string>, flag: string): string | undefined {
+	const prefix = `${flag}=`
+	const index = args.findIndex((argument) => argument === flag || argument.startsWith(prefix))
+	const argument = args[index]
+
+	if (argument === undefined) return undefined
+
+	return argument === flag ? args[index + 1] : argument.slice(prefix.length)
+}
+
+// The workers a tool opens when its flag is missing. **vitest's default is the machine less one**
+// (`maxWorkers`, run mode): `vitest.config.ts` shares the machine only between marked runs, so a lone
+// direct run opens them all. ESLint's `--concurrency` defaults to `off`, one worker.
+function default_workers(name: string, cores: number): number {
+	return name === VITEST ? Math.max(SINGLE_WORKER, cores - 1) : SINGLE_WORKER
+}
+
+// A worker flag's count: half the cores for `auto`, else the number — one for `off` and an unreadable
+// value alike.
+function count_of(value: string, cores: number): number {
+	if (value === AUTO_WORKERS) return Math.max(SINGLE_WORKER, Math.floor(cores / AUTO_CORE_SHARE))
+
+	const count = Number(value)
+
+	return Number.isSafeInteger(count) && count > SINGLE_WORKER ? count : SINGLE_WORKER
+}
+
+function workers_of(call: ToolCall, cores: number = os.availableParallelism()): number {
+	const flag = WORKER_FLAGS[call.name]
+	const value = flag === undefined ? undefined : flag_value(call.args, flag)
+
+	return value === undefined ? default_workers(call.name, cores) : count_of(value, cores)
+}
+
+function memory_of(call: ToolCall, cores: number): number | undefined {
+	const row = core_weights.TOOL_MEMORY[call.name]
+
+	return row === undefined ? undefined : core_weights.MEMORY_MB[row] * workers_of(call, cores)
+}
+
+// The MB the line's largest tool holds, or `undefined` when it launches none the table knows.
+function estimated_memory_mb(
+	line: string,
+	cores: number = os.availableParallelism(),
+): number | undefined {
+	const amounts = tool_calls_of(line).flatMap((call) => memory_of(call, cores) ?? [])
+
+	return amounts.length === 0 ? undefined : Math.max(...amounts)
+}
+
+// True when the line runs eslint with more than one worker, wrapped or not.
+function runs_multi_worker_eslint(
+	line: string,
+	cores: number = os.availableParallelism(),
+): boolean {
+	return tool_calls_of(line).some(
+		(call) => call.name === ESLINT && workers_of(call, cores) > SINGLE_WORKER,
+	)
+}
+
+const command_memory = { estimated_memory_mb, runs_multi_worker_eslint, tool_calls_of }
+
+export { command_memory }
+export type { ToolCall }