@joshuafolkken/kit 1.1455.0 → 1.1514.0

package/scripts/gate/gate-run-steps.ts

@@ -0,0 +1,150 @@
+import { availableParallelism } from 'node:os'
+import { bounded_pool } from '#scripts/lib/bounded-pool'
+import { buffered_process } from '#scripts/lib/buffered-process'
+import { unit_worker_share } from '#scripts/test/unit-worker-share'
+import { core_budget } from './core-budget'
+import { gate_plan, type GatePlan } from './gate-plan'
+import type { GateStep, GateStepResult } from './gate-report'
+import { build_gate_step } from './gate-steps'
+import { project_checks } from './project-checks'
+
+// Running the gate's checks: each one under its place in the machine-wide budget, a reused one
+// (`check-reuse.ts`) without starting it. `verification-gate.ts` decides what runs and records the
+// outcome; this module only runs it.
+
+async function run_gate_step(step: GateStep): Promise<GateStepResult> {
+	if (step.skip_reason !== undefined) {
+		return {
+			label: step.label,
+			command: step.command_args.join(' '),
+			output: project_checks.skip_notice('check', step.skip_reason),
+			exit_code: 0,
+			elapsed_ms: 0,
+		}
+	}
+
+	const result = await buffered_process.run_buffered_process(
+		step.command_args,
+		step.cwd === undefined ? {} : { cwd: step.cwd },
+	)
+
+	return { label: step.label, command: step.command_args.join(' '), ...result }
+}
+
+// This gate's share of the machine, resolved once at the gate's entry and carried down as one value.
+// `available_cores` is the budget every check reserves against; `is_reserved`
+// is false for a gate nested inside another gate's unit suite — this repository's own gate tests — so
+// the outer gate reserves and every gate beneath it does not. Captured before this gate sets
+// `JOSH_UNIT_RUN_MARKED`, which is why it is read at entry rather than re-derived here.
+interface BudgetContext {
+	available_cores: number
+	is_reserved: boolean
+}
+
+function default_budget(): BudgetContext {
+	return {
+		available_cores: availableParallelism(),
+		is_reserved: !unit_worker_share.is_nested_run() && !core_budget.is_held(),
+	}
+}
+
+// A gate step paired with the cores it reserves from the machine-wide budget while it runs.
+// The weight travels with the step so `run_gate_steps` never has to re-pair a
+// step with its check by index.
+interface ReservedStep {
+	step: GateStep
+	weight: number
+	memory_mb: number
+	// When the check's own green record was taken, for a check `check-reuse.ts` found reusable.
+	reused_at: string | undefined
+}
+
+// The checks this gate reuses, each mapped to when it was last green.
+type ReusedChecks = ReadonlyMap<string, string>
+
+async function build_reserved_steps(
+	plan: GatePlan,
+	available_cores: number,
+	reused: ReusedChecks,
+): Promise<ReadonlyArray<ReservedStep>> {
+	return await Promise.all(
+		plan.checks.map(async (check) => ({
+			step: await build_gate_step(check, process.cwd(), plan),
+			weight: gate_plan.check_weight(check, plan, available_cores),
+			memory_mb: gate_plan.check_memory(check, plan, available_cores),
+			reused_at: reused.get(check.label),
+		})),
+	)
+}
+
+// A reused check starts no process and claims no core: its result is the record's, and the empty
+// output is what that record was written from (`gate_report.has_nothing_to_say`).
+function reused_result(step: GateStep, reused_at: string): GateStepResult {
+	return {
+		label: step.label,
+		command: step.command_args.join(' '),
+		output: '',
+		exit_code: 0,
+		elapsed_ms: 0,
+		reused_at,
+	}
+}
+
+// **A nested gate takes no reservation, and the budget is what makes that necessary rather than
+// merely tidy**. The unit step spawns the suite, which runs this repository's
+// own gate tests; each of those is a gate inside the outer gate's unit reservation, and a place it
+// claimed would wait for cores the outer gate cannot free until the suite — this test included —
+// finishes.
+async function run_reserved_step(
+	reserved: ReservedStep,
+	budget: BudgetContext,
+): Promise<GateStepResult> {
+	if (reserved.reused_at !== undefined) return reused_result(reserved.step, reserved.reused_at)
+	if (!budget.is_reserved) return await run_gate_step(reserved.step)
+
+	return await core_budget.with_core_reservation(
+		reserved.weight,
+		async (admission) => ({
+			...(await run_gate_step(reserved.step)),
+			budget_note: core_budget.describe_overflow(admission),
+		}),
+		{
+			budget: budget.available_cores,
+			memory_mb: reserved.memory_mb,
+			command: `gate ${reserved.step.label}`,
+		},
+	)
+}
+
+// `bounded_pool` rather than a bare `Promise.all`, so the plan's `concurrency` is what decides how
+// many run at once. **The all-failures-in-one-pass property survives the
+// change** because no check ever rejects: `buffered_process` reports a non-zero exit as a value, so
+// the pool's first-failure abort — written for callers that spawn real Claude sessions — never
+// fires here and every queued check still runs. Results come back in input order however they
+// finished, which is what keeps the printed sections in declaration order. Each step first claims its
+// place in the machine-wide budget and waits while the budget is full.
+async function run_gate_steps(
+	plan: GatePlan,
+	budget: BudgetContext = default_budget(),
+	reused: ReusedChecks = new Map(),
+): Promise<ReadonlyArray<GateStepResult>> {
+	const reserved = await build_reserved_steps(plan, budget.available_cores, reused)
+
+	// **The checks run under the held mark, so a check's own dispatch claims nothing**.
+	// Each check is a `pnpm josh <target>` child, and a target that declares
+	// a weight would otherwise claim a second place for the cores reserved for it here. A nested gate
+	// sets the mark too, since its checks are covered by the outer gate's reservation.
+	return await core_budget.with_held_mark(
+		async () =>
+			await bounded_pool.bounded_map(
+				reserved,
+				plan.concurrency,
+				async (entry) => await run_reserved_step(entry, budget),
+			),
+	)
+}
+
+const gate_run_steps = { default_budget, run_gate_step, run_gate_steps }
+
+export type { BudgetContext, ReusedChecks }
+export { gate_run_steps }

package/scripts/gate/held-mark.ts

@@ -0,0 +1,55 @@
+// **The environment mark that says a parent already holds this process's cores**
+// `josh gate` reserves per check and then spawns `pnpm josh lint`. Without
+// the mark, that child's own dispatch would claim a second place for the cores its parent already
+// holds. Every holder sets it, and every child inherits it through the environment and claims nothing.
+const HELD_KEY = 'JOSH_CORE_RESERVED'
+const HELD_VALUE = '1'
+// The ledger key of the claim a command holds for its children (`core_budget.with_command_reservation`),
+// so a child reading the machine can tell that claim's admitted, not-yet-busy weight from running load.
+const HELD_RESERVATION_KEY = 'JOSH_CORE_RESERVATION'
+
+function is_held(): boolean {
+	return process.env[HELD_KEY] === HELD_VALUE
+}
+
+// The ledger key of the one claim a parent command holds for this process, `undefined` where none does.
+function held_reservation(): string | undefined {
+	const key = process.env[HELD_RESERVATION_KEY] ?? ''
+
+	return key === '' ? undefined : key
+}
+
+// A function of its own so the restoring write does not sit after an `await`, where
+// `require-atomic-updates` reads any write to `process` as a possible race — the reason `josh.ts`
+// gives `record_exit_code`.
+function set_held_mark(value: string, reservation: string): void {
+	process.env[HELD_KEY] = value
+	process.env[HELD_RESERVATION_KEY] = reservation
+}
+
+// Marks this process, and every child it spawns, as running inside a held reservation — `reservation`
+// names its ledger key where one claim covers them all. The previous values come back afterwards, so a
+// nested holder never clears the mark its outer holder set.
+async function with_held_mark<T>(run: () => Promise<T>, reservation = ''): Promise<T> {
+	const previous = process.env[HELD_KEY] ?? ''
+	const previous_reservation = process.env[HELD_RESERVATION_KEY] ?? ''
+
+	set_held_mark(HELD_VALUE, reservation)
+
+	try {
+		return await run()
+	} finally {
+		set_held_mark(previous, previous_reservation)
+	}
+}
+
+const held_mark = {
+	HELD_KEY,
+	HELD_RESERVATION_KEY,
+	HELD_VALUE,
+	held_reservation,
+	is_held,
+	with_held_mark,
+}
+
+export { held_mark }

package/scripts/gate/machine-capacity.ts

@@ -24,7 +24,8 @@ import { execa } from 'execa'
 // admits without looking at memory, exactly as before.
 //
 // **`run:board` draws its machine gauges from the same reading**, so what the
-// board shows a person is what the gate admits against. The board takes a sample with no window and
+// board shows a person is what the gate admits against — the gate reading the free memory as none
+// while the board's pressure and swap gauges show why. The board takes a sample with no window and
 // draws the difference from its previous one. `lane:sample` records the memory half of it, so the
 // lane-limit measurement sets the same figures beside the lane count.
 
@@ -50,29 +51,50 @@ const PAGE_SIZE_ARGUMENTS = ['PAGESIZE']
 // every second. `kern.memorystatus_level` is the kernel's own memory-pressure figure: the percentage of
 // memory still available, compressible and purgeable pages included — what `os.freemem()` leaves out on
 // macOS, where it reads near zero on a machine with gigabytes to spare. The swapper counters are the
-// pages swapped in and out since boot, `vm_stat`'s `Swapins` / `Swapouts`. The names stay in the output
+// pages swapped in and out since boot, `vm_stat`'s `Swapins` / `Swapouts`; `vm.compressor_input_bytes` is
+// the bytes the compressor has taken in since boot, `vm_stat`'s `Compressions`. The names stay in the output
 // (no `-n`), so a name an older macOS lacks drops only its own line.
 const LEVEL_NAME = 'kern.memorystatus_level'
 const SWAP_IN_NAME = 'vm.compressor.swapper.swapins_total'
 const SWAP_OUT_NAME = 'vm.compressor.swapper.swapouts_total'
+const COMPRESSED_NAME = 'vm.compressor_input_bytes'
 const PAGE_SIZE_NAME = 'hw.pagesize'
 // The kernel's memory-pressure verdict — 1 normal, 2 warning, 4 critical — the one Activity Monitor
 // colors its "memory pressure" graph by.
 const PRESSURE_NAME = 'kern.memorystatus_vm_pressure_level'
-const SYSCTL_ARGUMENTS = [LEVEL_NAME, SWAP_IN_NAME, SWAP_OUT_NAME, PAGE_SIZE_NAME, PRESSURE_NAME]
+const PRESSURE_WARNING = 2
+// The swap traffic across one window that reads as pressure — 64 MB a second paged to disk. A machine
+// that swapped once still pages a few megabytes back in as old pages are touched.
+const SWAP_PRESSURE_MB = 16
+// The compressor traffic across one window that reads as pressure. An idle machine compresses nothing
+// across a window; the 2026-10-16 thrash compressed about 240,000 16 KB pages every three seconds, about
+// 310 MB a window, with no swap figure to show for it.
+const COMPRESSION_PRESSURE_MB = 64
+const SYSCTL_ARGUMENTS = [
+	LEVEL_NAME,
+	SWAP_IN_NAME,
+	SWAP_OUT_NAME,
+	COMPRESSED_NAME,
+	PAGE_SIZE_NAME,
+	PRESSURE_NAME,
+]
 const SYSCTL_LINE = /^(?<name>[\w.]+): (?<value>\d+)$/gmu
 
+// `memory_pressured` says why `available_mb` read none, so a claim that waits can name the pressure.
 interface MachineReading {
 	busy_cores: number | undefined
 	available_mb: number | undefined
+	memory_pressured?: boolean | undefined
 }
 
 // The memory half of a reading. `swapped_mb` is a counter — every page swapped in or out since boot —
-// so a rate is the difference between two readings. `pressure_level` is read on macOS only.
+// so a rate is the difference between two readings, and so is `compressed_mb`, every byte compressed
+// since boot. `pressure_level` and `compressed_mb` are read on macOS only.
 interface MemoryReading {
 	available_mb: number | undefined
 	swapped_mb: number | undefined
 	pressure_level: number | undefined
+	compressed_mb?: number | undefined
 }
 
 // What the ledger's admitted reservations declare — the part of the machine's load already accounted for.
@@ -84,6 +106,7 @@ interface LedgerLoad {
 interface MachineBudget {
 	cores: number
 	memory_mb: number
+	memory_pressured?: boolean | undefined
 }
 
 interface CpuTimes {
@@ -141,10 +164,10 @@ function swapped_pages(
 	return pages_in === undefined || pages_out === undefined ? undefined : pages_in + pages_out
 }
 
-function swapped_mb(pages: number | undefined, page_bytes: number | undefined): number | undefined {
-	if (pages === undefined || page_bytes === undefined) return undefined
+function mb_of(units: number | undefined, unit_bytes: number | undefined): number | undefined {
+	if (units === undefined || unit_bytes === undefined) return undefined
 
-	return (pages * page_bytes) / BYTES_PER_MB
+	return (units * unit_bytes) / BYTES_PER_MB
 }
 
 // What was swapped between two readings of the counter. A counter that went back — a reboot between
@@ -163,11 +186,12 @@ function parse_darwin_memory(output: string, total_bytes: number): MemoryReading
 	return {
 		available_mb:
 			level === undefined ? undefined : (level / PERCENT) * (total_bytes / BYTES_PER_MB),
-		swapped_mb: swapped_mb(
+		swapped_mb: mb_of(
 			swapped_pages(values.get(SWAP_IN_NAME), values.get(SWAP_OUT_NAME)),
 			values.get(PAGE_SIZE_NAME),
 		),
 		pressure_level: values.get(PRESSURE_NAME),
+		compressed_mb: mb_of(values.get(COMPRESSED_NAME), 1),
 	}
 }
 
@@ -187,7 +211,7 @@ function pages_of(pattern: RegExp, vmstat: string): number | undefined {
 function parse_linux_swapped(vmstat: string, page_bytes: number): number | undefined {
 	const pages = swapped_pages(pages_of(LINUX_SWAP_IN, vmstat), pages_of(LINUX_SWAP_OUT, vmstat))
 
-	return swapped_mb(pages, page_bytes)
+	return mb_of(pages, page_bytes)
 }
 
 async function settled<T>(read: () => Promise<T>): Promise<T | undefined> {
@@ -256,16 +280,47 @@ async function default_sleep(ms: number): Promise<void> {
 	})
 }
 
+// Whether a counter gained more than `limit` MB across the window; an unread counter gained nothing.
+function moved_over(from: number | undefined, to: number | undefined, limit: number): boolean {
+	return (swapped_between(from, to) ?? 0) > limit
+}
+
+function is_pressured(before: MemoryReading, after: MemoryReading): boolean {
+	return (
+		(after.pressure_level ?? 0) >= PRESSURE_WARNING ||
+		moved_over(before.swapped_mb, after.swapped_mb, SWAP_PRESSURE_MB) ||
+		moved_over(before.compressed_mb, after.compressed_mb, COMPRESSION_PRESSURE_MB)
+	)
+}
+
+// **Memory under compression and swap pressure has nothing free to admit against.** The level counts
+// compressible pages as available, so on 2026-10-16 it read 5.4 GB free while the compressor held
+// 5.9 GB and compressed and decompressed about 3.3 GB every three seconds, and 5.2 GB of reservations
+// were admitted on top. A reading the kernel calls pressured, or one with swap or compressor traffic
+// across the window, answers no free memory: what the ledger holds keeps running and a heavy claim waits
+// for the pressure to pass.
+function pressured_available(before: MemoryReading, after: MemoryReading): number | undefined {
+	if (after.available_mb === undefined) return undefined
+
+	return is_pressured(before, after) ? 0 : after.available_mb
+}
+
+// Memory is read at both ends of the window, so the swap counter's difference is the activity during it.
 async function read_machine(
 	sleep: (ms: number) => Promise<void> = default_sleep,
 ): Promise<MachineReading> {
 	const before = cpu_times()
-	const [memory] = await Promise.all([read_memory(), sleep(SAMPLE_WINDOW_MS)])
+	const [memory_before] = await Promise.all([read_memory(), sleep(SAMPLE_WINDOW_MS)])
+	const memory_after = await read_memory()
 	const busy_cores = is_unquoted()
 		? busy_cores_between(before, cpu_times(), cpus().length)
 		: undefined
 
-	return { busy_cores, available_mb: memory.available_mb }
+	return {
+		busy_cores,
+		available_mb: pressured_available(memory_before, memory_after),
+		memory_pressured: is_pressured(memory_before, memory_after),
+	}
 }
 
 async function read_sample(): Promise<MachineSample> {
@@ -291,6 +346,7 @@ function machine_budget(cores: number, reading: MachineReading, ledger: LedgerLo
 		cores: cores - external_cores(reading.busy_cores, ledger.cores),
 		memory_mb:
 			reading.available_mb === undefined ? Infinity : reading.available_mb + ledger.memory_mb,
+		memory_pressured: reading.memory_pressured,
 	}
 }
 
@@ -298,14 +354,17 @@ const machine_capacity = {
 	BASELINE_CORES,
 	busy_cores_between,
 	busy_share_between,
+	COMPRESSION_PRESSURE_MB,
 	external_cores,
 	machine_budget,
 	parse_darwin_memory,
 	parse_linux_available,
 	parse_linux_swapped,
+	pressured_available,
 	read_machine,
 	read_memory,
 	read_sample,
+	SWAP_PRESSURE_MB,
 	swapped_between,
 }
 

package/scripts/gate/pre-commit-type-check.ts

@@ -3,6 +3,7 @@ import { fileURLToPath } from 'node:url'
 import type { FileMapStamp } from '#scripts/josh/file-map-stamp'
 import { resolve_spawn_exit } from '#scripts/lib/spawn-exit'
 import { execa } from 'execa'
+import { core_budget } from './core-budget'
 import { gate_skip } from './gate-skip'
 import { gate_tree, type GateTree } from './gate-tree'
 import { hook_gate_reuse } from './hook-gate-reuse'
@@ -143,6 +144,18 @@ async function run_type_check(): Promise<number> {
 	return resolve_spawn_exit(PNPM, result)
 }
 
+// **The type check claims the place `josh check` claims** (joshuafolkken/kit#3944), so lefthook's
+// parallel commit hooks stop reaching other lanes' admission as foreign load. Only a miss reserves:
+// a reused record runs nothing, and a claim there could make a commit wait out a full ledger for a
+// check it then skips.
+async function run_reserved_type_check(): Promise<number> {
+	return await core_budget.with_command_reservation(
+		core_budget.CORE_WEIGHTS.type_check,
+		run_type_check,
+		{ memory_mb: core_budget.MEMORY_MB.type_check, command: 'pre-commit type check' },
+	)
+}
+
 function unexpected_arguments(extra_arguments: ReadonlyArray<string>): ReadonlyArray<string> {
 	return extra_arguments.filter((argument) => argument !== gate_skip.FORCE_FLAG)
 }
@@ -161,7 +174,7 @@ async function run_pre_commit_type_check(
 
 	const reusable = await reusable_green_commit(await read_commit_tree(), extra_arguments, source)
 
-	if (reusable === undefined) return await run_type_check()
+	if (reusable === undefined) return await run_reserved_type_check()
 
 	process.stdout.write(`${format_skip(reusable.taken_at)}\n`)