@joshuafolkken/kit 1.1514.0 → 1.1784.0

package/scripts/run/board/run-board-usage-read.ts

@@ -1,18 +1,27 @@
 import { cpus, totalmem } from 'node:os'
-import path from 'node:path'
 import { lane_paths } from '#scripts/lane/lane-paths'
+import type { TableRead } from '#scripts/lane/lane-process-table'
 import { lane_registry } from '#scripts/lane/lane-registry'
+import { process_directories } from '#scripts/lane/process-directories'
 import { PROBE_TIMEOUT_MS } from '#scripts/lib/timeouts'
 import { execa } from 'execa'
 import type { ProcessReading, UsageMark } from './run-board-usage'
-import { run_board_usage_rusage, type RusageReader } from './run-board-usage-rusage'
+import { run_board_usage_lanes, type DirectoryRead, type LaneLookup } from './run-board-usage-lanes'
+import { run_board_usage_listing, type ProcessListing } from './run-board-usage-listing'
+import {
+	run_board_usage_rusage,
+	type RusageReader,
+	type RusageReading,
+} from './run-board-usage-rusage'
+import { run_board_usage_top, type TopReading, type TopReadings } from './run-board-usage-top'
 
 // How `run:board` reads which lane each process belongs to and what it uses.
-// **One `ps` a second, and `lsof` only for a process not seen before.** `ps` lists every process's
+// **One `ps` a second, and `lsof` only for a process not seen before.** `ps` — the snapshot the lane
+// count reads in the same refresh (`lane-process-table.ts`), never one of its own — lists every process's
 // cumulative CPU time and resident memory in about 15 ms; a process is put in the lane its working
-// directory sits under (`.kit-lanes/<N>`), looked up once and remembered, so a full `lsof` (about
-// 220 ms) is paid on the first sample alone. The working directory rather than the parent chain: a
-// process detached with `josh ship --detach` is re-parented and no longer traceable to its lane.
+// directory sits under (`.kit-lanes/<N>`), else its nearest ancestor's. A directory is looked up once
+// and remembered — asked again only when an `lsof` cut short left it out — so a full `lsof` (about
+// 220 ms) is paid on the first sample alone (`run-board-usage-lanes.ts`).
 // **Only this repository's lanes root counts** — another repository's lane of the same number is a
 // different lane. **macOS alone:** Linux's `ps` prints `time` in whole seconds, too coarse for a
 // one-second difference, so there the board draws no column rather than a figure flickering to 0.
@@ -21,194 +30,148 @@ import { run_board_usage_rusage, type RusageReader } from './run-board-usage-rus
 // Where `node:ffi` reaches the kernel's reaped-children times (`run-board-usage-rusage.ts`), each
 // process's CPU time is read with them folded in and its parent is kept, so such a child lands in its
 // parent's lane; elsewhere the figure stays `ps`'s own time, a lower bound.
+// **Memory is `top`'s MEM, compressed memory its CMPRS** — the footprint `node:ffi` reads each second
+// where it can, else `top`'s last reading (`run-board-usage-top.ts`, at most once every 30 s), else
+// `ps`'s resident set; compressed memory only `top` has, so it is 0 until `top` has answered.
 
-const PS_ARGUMENTS = ['-A', '-o', 'pid=,ppid=,time=,rss=']
-const LSOF_ARGUMENTS = ['-a', '-d', 'cwd', '-Fn', '-p']
+const LSOF_ARGUMENTS = [...process_directories.CWD_ARGUMENTS, '-p']
 const PID_SEPARATOR = ','
-const PS_FIELDS = 4
-const DAY_SEPARATOR = '-'
-const CLOCK_SEPARATOR = ':'
-const SECONDS_PER_DAY = 86_400
-const SECONDS_PER_CLOCK_PART = 60
-const MS_PER_SECOND = 1000
-const BYTES_PER_KB = 1024
-// `lsof -F`'s field lines: `p<pid>` opens a process, `n<path>` names its directory.
-const LSOF_PROCESS = /^(?=p)/mu
-const LSOF_NAME = 'n'
 const { load_builtin, rusage_reader } = run_board_usage_rusage
-
-// One process's line of `ps`'s listing.
-interface ProcessListing {
-	parent: number
-	reading: ProcessReading
-}
-
-interface UsagePorts {
-	// `ps`'s listing.
-	list: () => Promise<string>
-	// Each process's CPU time with its reaped children's, where `node:ffi` reaches it.
+const { list_top, top_reader } = run_board_usage_top
+const { listings_of } = run_board_usage_listing
+const { lanes_for } = run_board_usage_lanes
+
+// The working directories and the lanes root come with `LaneLookup`.
+interface UsagePorts extends LaneLookup {
+	// The process snapshot the lane count reads too (`lane-process-table.ts`).
+	list: TableRead
+	// Each process's CPU time with its reaped children's and its footprint, where `node:ffi` reaches it.
 	rusage: RusageReader | undefined
-	// The working directory of each process that answered.
-	directories: (pids: ReadonlyArray<number>) => Promise<ReadonlyMap<number, string>>
+	// `top`'s last memory and compressed-memory reading.
+	top: () => Promise<TopReadings>
 	now: () => number
-	// The board's own lanes root.
-	root: string
 }
 
-// `ps`'s `time`, `[DD-][HH:]MM:SS[.ss]`, in milliseconds.
-function cpu_ms_of(text: string): number {
-	const [days, clock = ''] = text.includes(DAY_SEPARATOR) ? text.split(DAY_SEPARATOR) : ['0', text]
-	const seconds = clock
-		.split(CLOCK_SEPARATOR)
-		.reduce((sum, part) => sum * SECONDS_PER_CLOCK_PART + Number(part), 0)
-
-	return (Number(days) * SECONDS_PER_DAY + seconds) * MS_PER_SECOND
+// What one reading is made from: `ps`'s line, `node:ffi`'s reading and `top`'s — `ps`'s resident set
+// and no compressed memory standing in for a process `top` has not listed.
+interface ReadingSources {
+	listing: ProcessListing
+	rusage: RusageReading | undefined
+	top: TopReading
+	earlier: ProcessReading | undefined
 }
 
-function listing_of(line: string): [number, ProcessListing] | undefined {
-	const fields = line.trim().split(/\s+/u)
-	const [pid, parent, time = '', rss] = fields
-	const reading = { cpu_ms: cpu_ms_of(time), rss_bytes: Number(rss) * BYTES_PER_KB }
-	const is_number = !Number.isNaN(reading.cpu_ms + reading.rss_bytes + Number(parent))
-
-	if (!is_number || fields.length !== PS_FIELDS) return undefined
-
-	return [Number(pid), { parent: Number(parent), reading }]
+function top_of(top: TopReadings, pid: number, listing: ProcessListing): TopReading {
+	return top.get(pid) ?? { memory_bytes: listing.rss_bytes, compressed_bytes: 0 }
 }
 
-function parse_ps(output: string): Map<number, ProcessListing> {
-	const listings = output.split('\n').map((line) => listing_of(line))
+// A process whose `node:ffi` read failed keeps the larger of `ps`'s own time and its last reading: once
+// reaped, its whole folded count lands in its parent's, so what comes off there has to be that count —
+// its own time alone would count its reaped children a second time.
+function cpu_ms_from({ listing, rusage, earlier }: ReadingSources): number {
+	if (rusage !== undefined) return rusage.cpu_ms
 
-	return new Map(listings.filter((listing) => listing !== undefined))
+	return Math.max(listing.cpu_ms, earlier?.cpu_ms ?? 0)
 }
 
-// A process whose read failed keeps the larger of `ps`'s own time and its last reading: once reaped,
-// its whole folded count lands in its parent's, so what comes off there has to be that count — its own
-// time alone would count its reaped children a second time.
-function folded_reading(
-	reading: ProcessReading,
-	cpu_ms: number | undefined,
-	earlier: ProcessReading | undefined,
-): ProcessReading {
-	if (cpu_ms !== undefined) return { ...reading, cpu_ms }
-
-	return { ...reading, cpu_ms: Math.max(reading.cpu_ms, earlier?.cpu_ms ?? 0) }
-}
+function reading_of(sources: ReadingSources): ProcessReading {
+	const { rusage, top } = sources
+	const memory_bytes = rusage?.footprint_bytes ?? top.memory_bytes
 
-type Readings = Pick<UsageMark, 'folded' | 'parents' | 'processes'>
+	return { cpu_ms: cpu_ms_from(sources), memory_bytes, compressed_bytes: top.compressed_bytes }
+}
 
-function own_readings(listings: ReadonlyMap<number, ProcessListing>): Readings {
-	const processes = new Map([...listings].map(([pid, { reading }]) => [pid, reading] as const))
+type Readings = Pick<UsageMark, 'folded' | 'names' | 'parents' | 'processes'>
 
-	return { processes, parents: new Map(), folded: new Set() }
+// What the readings are made from besides `ps`'s listing.
+interface ReadingInputs {
+	rusage: RusageReader | undefined
+	top: TopReadings
+	earlier: ReadonlyMap<number, ProcessReading>
 }
 
-// `ps`'s own-time readings, each replaced by its reaped-children-folded time where `rusage` reads one.
-// Every process keeps its parent once there is a reader: one whose own read failed — gone by then, or
-// another user's — still lands in its parent's count when reaped, so what was counted for it comes off.
+// `ps`'s own-time readings, each replaced by its reaped-children-folded time where `rusage` reads one;
+// `ps` alone counts no reaped child, so nothing is folded there. Every process keeps its parent: one
+// whose own read failed — gone by then, or another user's — still lands in its parent's count when
+// reaped, so what was counted for it comes off.
 function readings_of(
 	listings: ReadonlyMap<number, ProcessListing>,
-	rusage: RusageReader | undefined,
-	earlier: ReadonlyMap<number, ProcessReading> = new Map(),
+	inputs: ReadingInputs,
 ): Readings {
-	if (rusage === undefined) return own_readings(listings)
-
 	const processes = new Map<number, ProcessReading>()
 	const folded = new Set<number>()
-	const parents = new Map([...listings].map(([pid, { parent }]) => [pid, parent] as const))
 
-	for (const [pid, { reading }] of listings) {
-		const cpu_ms = rusage(pid)
+	for (const [pid, listing] of listings) {
+		const rusage = inputs.rusage?.(pid)
+		const top = top_of(inputs.top, pid, listing)
 
-		processes.set(pid, folded_reading(reading, cpu_ms, earlier.get(pid)))
-		if (cpu_ms !== undefined) folded.add(pid)
+		processes.set(pid, reading_of({ listing, rusage, top, earlier: inputs.earlier.get(pid) }))
+		if (rusage !== undefined) folded.add(pid)
 	}
 
-	return { processes, parents, folded }
-}
-
-function lsof_block(block: string): [number, string] | undefined {
-	const [process_line = '', ...rest] = block.trim().split('\n')
-	const name = rest.find((line) => line.startsWith(LSOF_NAME))
-
-	return name === undefined ? undefined : [Number(process_line.slice(1)), name.slice(1)]
-}
-
-function parse_lsof(output: string): Map<number, string> {
-	const blocks = output.split(LSOF_PROCESS).map((block) => lsof_block(block))
-
-	return new Map(blocks.filter((block) => block !== undefined))
-}
-
-// The lane of `root` a directory sits under — itself or a directory above it, the outermost first.
-function lane_of(directory: string, root: string): string | undefined {
-	const parts = directory.split(path.sep)
-	const ancestors = parts.map((_, index) => parts.slice(0, index + 1).join(path.sep)).slice(1)
-	const environment = { [lane_paths.LANE_ROOT_KEY]: root }
-
-	return ancestors.map((ancestor) => lane_paths.lane_issue_of(ancestor, environment)).find(Boolean)
-}
+	const parents = new Map([...listings].map(([pid, { parent }]) => [pid, parent] as const))
+	const names = new Map([...listings].map(([pid, { command }]) => [pid, command] as const))
 
-function directory_lane(directory: string | undefined, root: string): string | undefined {
-	return directory === undefined ? undefined : lane_of(directory, root)
+	return { processes, parents, names, folded }
 }
 
-// The lanes of the processes still listed: a gone one dropped, and one not seen before looked up.
-async function lanes_for(
-	known: ReadonlyMap<number, string | undefined>,
-	processes: ReadonlyMap<number, ProcessReading>,
+// The snapshot's listings and `top`'s reading, read together; a snapshot `ps` did not answer fails the
+// sample.
+async function listed_of(
 	ports: UsagePorts,
-): Promise<Map<number, string | undefined>> {
-	const kept = new Map([...known].filter(([pid]) => processes.has(pid)))
-	const fresh = [...processes.keys()].filter((pid) => !kept.has(pid))
-
-	if (fresh.length === 0) return kept
+): Promise<{ listed: Map<number, ProcessListing>; top: TopReadings }> {
+	const [table, top] = await Promise.all([ports.list(), ports.top()])
 
-	const directories = await ports.directories(fresh)
-	const found = fresh.map((pid) => [pid, directory_lane(directories.get(pid), ports.root)] as const)
+	if (table === undefined) throw new Error('ps did not answer')
 
-	return new Map([...kept, ...found])
+	return { listed: listings_of(table), top }
 }
 
 async function sample(before: UsageMark | undefined, ports: UsagePorts): Promise<UsageMark> {
 	const at_ms = ports.now()
-	const readings = readings_of(parse_ps(await ports.list()), ports.rusage, before?.processes)
-	const lanes = await lanes_for(before?.lanes ?? new Map(), readings.processes, ports)
+	const { listed, top } = await listed_of(ports)
+	const earlier = before?.processes ?? new Map()
+	const readings = readings_of(listed, { rusage: ports.rusage, top, earlier })
+	const lanes = await lanes_for(before, readings.parents, ports)
 
-	return { at_ms, ...readings, lanes, cores: cpus().length, total_bytes: totalmem() }
+	return { at_ms, ...readings, ...lanes, cores: cpus().length, total_bytes: totalmem() }
 }
 
-async function darwin_directories(
-	pids: ReadonlyArray<number>,
-): Promise<ReadonlyMap<number, string>> {
-	// `lsof` exits non-zero when any one process is gone, and still lists the rest. A budget kill is
-	// not that: it lists nothing, and `lanes_for` never asks again about a process it read as being in
-	// no lane, so the sample fails instead of pinning every live process to none.
+// The answers of an `lsof` cut short, its last dropped: that one may end on a directory missing its
+// tail, `.kit-lanes/43` for `.kit-lanes/4346`.
+function whole_answers(directories: Map<number, string>): Map<number, string> {
+	return new Map([...directories].slice(0, -1))
+}
+
+async function darwin_directories(pids: ReadonlyArray<number>): Promise<DirectoryRead> {
+	// `lsof` exits non-zero when any one process is gone, and still lists the rest. One killed on its
+	// budget or by a signal, or one that never started — a spawn failing under load has no exit code —
+	// did not run to its end, and every process it did not answer for is asked again on the next sample
+	// (`run-board-usage-lanes.ts`).
 	const lsof_arguments = [...LSOF_ARGUMENTS, pids.join(PID_SEPARATOR)]
 	const result = await execa('lsof', lsof_arguments, { reject: false, timeout: PROBE_TIMEOUT_MS })
+	const is_cut_short = result.isTerminated || (result.failed && result.exitCode === undefined)
+	const directories = process_directories.parse(result.stdout)
 
-	if (result.timedOut) throw new Error('lsof timed out')
+	if (is_cut_short) return { directories: whole_answers(directories), is_complete: false }
 
-	return parse_lsof(result.stdout)
+	return { directories, is_complete: true }
 }
 
-const PLATFORM_DIRECTORIES: Readonly<Partial<Record<NodeJS.Platform, UsagePorts['directories']>>> =
+const PLATFORM_DIRECTORIES: Readonly<Partial<Record<NodeJS.Platform, LaneLookup['directories']>>> =
 	{ darwin: darwin_directories }
 
-async function list_live(): Promise<string> {
-	const { stdout } = await execa('ps', PS_ARGUMENTS, { timeout: PROBE_TIMEOUT_MS })
-
-	return stdout
-}
-
 async function board_lane_root(): Promise<string> {
 	return lane_paths.lane_root(await lane_registry.main_repository_root())
 }
 
-// What one board's reader resolves once and keeps, since neither moves while the board is open.
+// What one reading is taken through: the board's snapshot, and what its reader resolves once and keeps,
+// since none of those moves while the board is open.
 interface ReaderSetup {
+	list: TableRead
 	root: Promise<string>
 	rusage: RusageReader | undefined
+	top: UsagePorts['top']
 }
 
 // `undefined` where the processes cannot be read — no `ps`, no way to a working directory, no
@@ -223,33 +186,40 @@ async function read_usage(
 	if (directories === undefined) return undefined
 
 	try {
-		const { rusage } = setup
-		const ports = { list: list_live, rusage, directories, now: Date.now, root: await setup.root }
+		const { list, rusage, top } = setup
+		const root = await setup.root
 
-		return await sample(before, ports)
+		return await sample(before, { list, rusage, top, directories, now: Date.now, root })
 	} catch {
 		return undefined
 	}
 }
 
-// One board's reader: the `node:ffi` reader is opened with it, and the lanes root resolved on the first
-// sample.
-function usage_reader(
-	platform: NodeJS.Platform = process.platform,
-): (before: UsageMark | undefined) => Promise<UsageMark | undefined> {
+type UsageReader = (
+	before: UsageMark | undefined,
+	table: TableRead,
+) => Promise<UsageMark | undefined>
+
+// One board's reader: the `node:ffi` and `top` readers are opened with it, and the lanes root resolved
+// on the first sample. Each reading takes the refresh's snapshot rather than a `ps` of its own.
+function usage_reader(platform: NodeJS.Platform = process.platform): UsageReader {
 	const rusage = rusage_reader(platform, load_builtin)
+	const top = top_reader({ list: list_top, now: Date.now })
 	const resolved: { root?: Promise<string> } = {}
 
-	async function read(before: UsageMark | undefined): Promise<UsageMark | undefined> {
+	async function read(
+		before: UsageMark | undefined,
+		table: TableRead,
+	): Promise<UsageMark | undefined> {
 		resolved.root ??= board_lane_root()
 
-		return await read_usage(before, { root: resolved.root, rusage }, platform)
+		return await read_usage(before, { list: table, root: resolved.root, rusage, top }, platform)
 	}
 
 	return read
 }
 
-const run_board_usage_read = { lane_of, parse_lsof, parse_ps, sample, usage_reader }
+const run_board_usage_read = { sample, usage_reader }
 
 export { run_board_usage_read }
-export type { UsagePorts }
+export type { UsagePorts, UsageReader }

package/scripts/run/board/run-board-usage-rusage.ts

@@ -3,7 +3,9 @@ import type { dlopen } from 'node:ffi'
 // A process's CPU time with the children it has reaped folded in, so a vitest worker or a gate's
 // eslint born and gone between two `run:board` samples still reaches its lane through its parent.
 // macOS's `ps -S` does not fold them; the kernel keeps them in `proc_pid_rusage`'s
-// `ri_child_user_time` / `ri_child_system_time`, reached here through `node:ffi`.
+// `ri_child_user_time` / `ri_child_system_time`, reached here through `node:ffi`. The same call's
+// `ri_phys_footprint` is the process's memory as `top`'s MEM and Activity Monitor's "Memory" count it —
+// compressed pages included, which a resident set leaves out (joshuafolkken/kit#4171).
 //
 // **`node:ffi` is experimental** — added in Node 26 (kit's `engines` is `^22.19.0 || ^24.0.0 ||
 // >=26.0.0`) and liable to change. Where it is missing, or the platform is not macOS, or a call
@@ -23,8 +25,14 @@ interface Timebase {
 	denom: number
 }
 
-// The CPU milliseconds of one process, `undefined` where it cannot be read (gone, or another user's).
-type RusageReader = (pid: number) => number | undefined
+// One process's CPU milliseconds and its physical footprint in bytes.
+interface RusageReading {
+	cpu_ms: number
+	footprint_bytes: number
+}
+
+// One process's reading, `undefined` where it cannot be read (gone, or another user's).
+type RusageReader = (pid: number) => RusageReading | undefined
 
 const PROC_LIBRARY = '/usr/lib/libproc.dylib'
 const SYSTEM_LIBRARY = '/usr/lib/libSystem.B.dylib'
@@ -35,6 +43,8 @@ const USER_TIME_OFFSET = 16
 const SYSTEM_TIME_OFFSET = 24
 const CHILD_USER_TIME_OFFSET = 96
 const CHILD_SYSTEM_TIME_OFFSET = 104
+// `rusage_info_v2`'s `ri_phys_footprint`.
+const PHYS_FOOTPRINT_OFFSET = 72
 const TIME_OFFSETS = [
 	USER_TIME_OFFSET,
 	SYSTEM_TIME_OFFSET,
@@ -93,10 +103,12 @@ function reader_of(ffi: Ffi): RusageReader {
 	const { proc_pid_rusage } = ffi.dlopen(PROC_LIBRARY, PROC_PID_RUSAGE).functions
 	const buffer = Buffer.alloc(RUSAGE_INFO_V2_BYTES)
 
-	return function read(pid: number): number | undefined {
-		const status = proc_pid_rusage(pid, RUSAGE_INFO_V2, buffer)
+	return function read(pid: number): RusageReading | undefined {
+		if (proc_pid_rusage(pid, RUSAGE_INFO_V2, buffer) !== 0) return undefined
+
+		const footprint_bytes = Number(buffer.readBigUInt64LE(PHYS_FOOTPRINT_OFFSET))
 
-		return status === 0 ? cpu_ms_of(buffer, timebase) : undefined
+		return { cpu_ms: cpu_ms_of(buffer, timebase), footprint_bytes }
 	}
 }
 
@@ -121,4 +133,4 @@ function rusage_reader(platform: NodeJS.Platform, load: ModuleLoad): RusageReade
 const run_board_usage_rusage = { cpu_ms_of, load_builtin, quiet_ffi, rusage_reader }
 
 export { run_board_usage_rusage }
-export type { RusageReader, Timebase }
+export type { RusageReader, RusageReading, Timebase }