@joshuafolkken/kit 1.1514.0 → 1.1784.0

package/scripts/run/board/run-board-process-groups.ts

@@ -0,0 +1,195 @@
+import { issue_cite } from '#scripts/issue/issue-cite'
+import { stamped_memo } from '#scripts/lib/stamped-memo'
+import { run_board_app_name } from './run-board-app-name'
+import {
+	run_board_usage,
+	type ProcessFigures,
+	type UsageFigure,
+	type UsageMark,
+} from './run-board-usage'
+
+// Which part of the machine each process's usage is put under on `run:board`'s 💻 processes section:
+// the backlogrun itself — its supervisor, the session that holds the run, its lanes — the apps outside
+// it, or what a closed lane left running (joshuafolkken/kit#4171).
+// **A lane's processes are found by working directory, the rest by ancestry.** A process under
+// `.kit-lanes/<N>` is that lane's while the lane is open and left over once it is closed — a detached
+// `eslint_d` is re-parented to launchd, so no ancestry reaches it. The supervisor and the session are
+// found by walking each process's parents up to the nearest of the two; anything else is an app, named
+// by its bundle or its command, so a Claude session that is not this run's is an app too.
+
+// How many apps the section names, the largest by memory first.
+const APP_LIMIT = 4
+
+// The backlogrun's own processes: its supervisor (`run-wake-cli --loop`, `backlog:drive` beneath it)
+// and the session that holds the run. Either is `undefined` where it is not running.
+interface ProcessRoots {
+	supervisor?: number | undefined
+	parent?: number | undefined
+}
+
+// What the grouping reads besides the figures: the open lanes by issue number, the processes a close
+// could not end, each with why (`lane-reap-settle.ts`), and the issues whose lane is not ended.
+interface GroupSources {
+	mark: UsageMark
+	figures: ProcessFigures
+	open_lanes: ReadonlySet<number>
+	roots: ProcessRoots
+	stuck: ReadonlyMap<number, string>
+	closing: ReadonlySet<string>
+}
+
+interface NamedFigure {
+	name: string
+	// How many separate runs of it there are — a process whose parent is of the same name is one of
+	// that one's helpers, not another run.
+	instances: number
+	figure: UsageFigure
+}
+
+interface ProcessGroups {
+	backlogrun: {
+		total: UsageFigure
+		supervisor: UsageFigure
+		parent: UsageFigure
+		lanes: { count: number; figure: UsageFigure }
+	}
+	apps: { total: UsageFigure; top: ReadonlyArray<NamedFigure> }
+	leftover: { total: UsageFigure; entries: ReadonlyArray<NamedFigure> }
+}
+
+type Membership =
+	{ kind: 'lane' | 'leftover'; lane: number } | { kind: 'supervisor' | 'parent' } | { kind: 'app' }
+
+// Each command's app name, kept across redraws: the command alone decides it, and one gone from the
+// table is forgotten (joshuafolkken/kit#4263).
+const app_names = stamped_memo.create<string>()
+
+function app_of(command: string): string {
+	return app_names.of(command, stamped_memo.NO_STAMP, () => run_board_app_name.app_of(command))
+}
+
+// A process and its ancestors, the nearest first — at most once round the parents.
+function ancestry(pid: number, mark: UsageMark): Array<number> {
+	const line = [pid]
+
+	for (let ancestor = mark.parents.get(pid); ancestor !== undefined;) {
+		if (line.length > mark.parents.size) break
+		line.push(ancestor)
+		ancestor = mark.parents.get(ancestor)
+	}
+
+	return line
+}
+
+// The nearest of the backlogrun's roots a process is, or descends from.
+function root_of(pid: number, mark: UsageMark, roots: ProcessRoots): Membership | undefined {
+	const root = ancestry(pid, mark).find((ancestor) =>
+		[roots.supervisor, roots.parent].includes(ancestor),
+	)
+
+	if (root === undefined) return undefined
+
+	return { kind: root === roots.supervisor ? 'supervisor' : 'parent' }
+}
+
+function membership_of(pid: number, sources: GroupSources): Membership {
+	const lane = sources.mark.lanes.get(pid)
+
+	if (lane === undefined) return root_of(pid, sources.mark, sources.roots) ?? { kind: 'app' }
+
+	const number = Number(lane)
+
+	return { kind: sources.open_lanes.has(number) ? 'lane' : 'leftover', lane: number }
+}
+
+function name_of(pid: number, mark: UsageMark): string {
+	return app_of(mark.names.get(pid) ?? '')
+}
+
+function figures_where(
+	sources: GroupSources,
+	memberships: ReadonlyMap<number, Membership>,
+	kind: Membership['kind'],
+): Map<number, UsageFigure> {
+	return new Map([...sources.figures].filter(([pid]) => memberships.get(pid)?.kind === kind))
+}
+
+// A group's members by name, each named group's instances counted, the largest by memory first.
+function named_figures(
+	figures: ProcessFigures,
+	mark: UsageMark,
+	label_of: (pid: number) => string,
+): Array<NamedFigure> {
+	const sums = run_board_usage.grouped_sums(figures, label_of)
+	const runs = [...figures.keys()].filter((pid) => {
+		const parent = mark.parents.get(pid)
+
+		return parent === undefined || !figures.has(parent) || label_of(parent) !== label_of(pid)
+	})
+	const named = [...sums].map(([name, figure]) => {
+		const instances = runs.filter((pid) => label_of(pid) === name).length
+
+		return { name, instances, figure }
+	})
+
+	return named.toSorted((left, right) => right.figure.memory_bytes - left.figure.memory_bytes)
+}
+
+// A leftover that outlived a close's SIGKILL is named with why; one whose lane is not ended yet is
+// being reaped (`↻`) — the supervisor closes that lane again each pass. An ended lane's leftover is
+// closed again by nothing, so it carries no mark (joshuafolkken/kit#4300).
+function leftover_label(pid: number, sources: GroupSources): string {
+	const issue = sources.mark.lanes.get(pid) ?? ''
+	const lane = `${name_of(pid, sources.mark)} ${issue_cite.plain(issue)}`
+	const reason = sources.stuck.get(pid)
+
+	if (reason !== undefined) return `${lane} ⚠ ${reason} (pid ${String(pid)})`
+
+	return sources.closing.has(issue) ? `${lane} ↻` : lane
+}
+
+function backlogrun_of(
+	sources: GroupSources,
+	memberships: ReadonlyMap<number, Membership>,
+): ProcessGroups['backlogrun'] {
+	const [supervisor, parent, lanes] = [
+		run_board_usage.sum_of(figures_where(sources, memberships, 'supervisor').values()),
+		run_board_usage.sum_of(figures_where(sources, memberships, 'parent').values()),
+		run_board_usage.usage_of(sources.mark, figures_where(sources, memberships, 'lane')),
+	]
+	const lane_figure = run_board_usage.sum_of(lanes.values())
+
+	return {
+		total: run_board_usage.sum_of([supervisor, parent, lane_figure]),
+		supervisor,
+		parent,
+		lanes: { count: lanes.size, figure: lane_figure },
+	}
+}
+
+function groups_of(sources: GroupSources): ProcessGroups {
+	const { figures, mark } = sources
+
+	app_names.keep(new Set(mark.names.values()))
+
+	const memberships = new Map([...figures.keys()].map((pid) => [pid, membership_of(pid, sources)]))
+	const apps = figures_where(sources, memberships, 'app')
+	const leftover = figures_where(sources, memberships, 'leftover')
+
+	return {
+		backlogrun: backlogrun_of(sources, memberships),
+		apps: {
+			total: run_board_usage.sum_of(apps.values()),
+			top: named_figures(apps, mark, (pid) => name_of(pid, mark)).slice(0, APP_LIMIT),
+		},
+		leftover: {
+			total: run_board_usage.sum_of(leftover.values()),
+			entries: named_figures(leftover, mark, (pid) => leftover_label(pid, sources)),
+		},
+	}
+}
+
+const run_board_process_groups = { app_of, groups_of }
+
+export { run_board_process_groups }
+export type { GroupSources, NamedFigure, ProcessGroups, ProcessRoots }

package/scripts/run/board/run-board-read.ts

@@ -1,18 +1,26 @@
+import { backlog_drive_memory } from '#scripts/backlog/backlog-drive-memory'
 import type { NamedPlan } from '#scripts/backlog/backlog-plan'
 import { backlog_ready } from '#scripts/backlog/backlog-ready'
 import { backlogrun_launch } from '#scripts/backlog/backlogrun-launch'
+import { lane_held_stats } from '#scripts/lane/lane-held-stats'
+import { lane_ledger } from '#scripts/lane/lane-ledger'
+import type { TableRead } from '#scripts/lane/lane-process-table'
 import { lane_registry, type LaneInfo } from '#scripts/lane/lane-registry'
-import { lane_spent } from '#scripts/lane/lane-spent'
-import { run_carry } from '#scripts/run/carry/run-carry'
+import { lane_state } from '#scripts/lane/lane-state'
+import { stamped_memo } from '#scripts/lib/stamped-memo'
+import { run_carry, type RunCarry } from '#scripts/run/carry/run-carry'
 import { run_carry_ended } from '#scripts/run/carry/run-carry-ended'
 import { run_event_stream, type RunEvent } from '#scripts/run/event/run-event-stream'
 import { run_stranded_detect } from '#scripts/run/run-stranded-detect'
 import { run_wake } from '#scripts/run/wake/run-wake'
+import { run_board_lane_state, type BoardLanes } from './run-board-lane-state'
 import {
 	run_board_operation_read,
 	type OperationSources,
 	type SessionOperations,
 } from './run-board-operation-read'
+import type { ProcessRoots } from './run-board-process-groups'
+import { run_board_roots } from './run-board-roots'
 import { run_board_scope } from './run-board-scope'
 import { run_board_span, type BoardSpan } from './run-board-span'
 import { run_board_usual, type UsualTimes } from './run-board-usual'
@@ -27,6 +35,9 @@ interface LocalRead {
 	scope: NamedPlan
 	events: ReadonlyArray<RunEvent>
 	lanes: ReadonlyArray<string>
+	// The open lanes as their lane state records sort them — this run's, an earlier run's and the ones
+	// with no record — with what each is doing (`run-board-lane-state.ts`).
+	lane_states: BoardLanes
 	// The session a stopped run is resumed from; `undefined` otherwise.
 	resume: string | undefined
 	// The run is carried but nothing drives it — its owner and its supervisor are both gone
@@ -37,37 +48,89 @@ interface LocalRead {
 	// The open lanes no child is working in at this read (`lane-spent.ts`); one is drawn stopped once it
 	// has stayed so (`run-board-idle.ts`).
 	idle_lanes?: ReadonlyArray<string> | undefined
-	// The open lanes that last took a child before the run started — an earlier run's, until this run
-	// dispatches into one again.
-	earlier_lanes?: ReadonlyArray<string> | undefined
+	// The earlier run's lanes that still hold a place under the limit — counted in `occupied_count`, so
+	// the lanes section draws them beside this run's own.
+	earlier_occupied?: ReadonlyArray<string> | undefined
 	// How many more lanes the limit allows beside the live ones, as `lane:limit` counts them;
 	// `undefined` where the limit could not be read, so nothing is said about it.
 	free_lanes?: number | undefined
-	// The issues whose lanes hold a place under the limit (`backlog-occupied.ts`), named beside a row
-	// that waits for a free lane.
-	occupied_lanes?: ReadonlyArray<string> | undefined
+	// The lane limit those free lanes are counted under; `undefined` where it could not be read.
+	lane_limit?: number | undefined
+	// How many lanes hold a place under the limit (`backlog-occupied.ts`), counted beside a row that
+	// waits for a free lane.
+	occupied_count?: number | undefined
 	// How long each phase usually takes, from the repository's lane ledger.
 	usual?: UsualTimes | undefined
+	// The driver holds its offered issue until the machine has memory for one more lane — the ledger's
+	// open hold is `backlog-drive-memory.ts`'s.
+	is_memory_held?: boolean | undefined
+	// The run's supervisor and parent session, named in the 💻 section (`run-board-roots.ts`).
+	roots?: ProcessRoots | undefined
+	// The processes a lane's close could not end, by pid, each with why (`lane-reap-settle.ts`).
+	stuck?: ReadonlyMap<number, string> | undefined
+	// The issues whose lane record is not ended — a close the supervisor still retries.
+	closing?: ReadonlySet<string> | undefined
+	// The run's record, whose declared order holds a queued row (`backlog-drive-named.ts`).
+	carry?: RunCarry | undefined
 }
 
-const NO_SPAN: SpanHere = { span: undefined, resume: undefined, transcript: undefined }
+const NO_SPAN: SpanHere = {
+	span: undefined,
+	resume: undefined,
+	transcript: undefined,
+	roots: {},
+	carry: undefined,
+	is_stranded: false,
+}
 
 interface SpanHere {
 	span: BoardSpan | undefined
 	resume: string | undefined
 	// The parent session's transcript, where the run's record names one.
 	transcript: string | undefined
+	roots: ProcessRoots
+	carry: RunCarry | undefined
+	is_stranded: boolean
 }
 
-function span_here(repository: string): SpanHere {
+// The main work tree, resolved once: it does not move while the board is open, and resolving it runs a
+// `git` subprocess the redraw would wait on (joshuafolkken/kit#4263). A failed resolve is forgotten,
+// so a `git` that timed out once is asked again on the next redraw rather than failing every one.
+const main_roots = stamped_memo.create<Promise<string>>()
+
+async function main_root(): Promise<string> {
+	try {
+		return await main_roots.of(
+			process.cwd(),
+			stamped_memo.NO_STAMP,
+			async () => await lane_registry.main_repository_root(),
+		)
+	} catch (error) {
+		main_roots.keep(new Set())
+		throw error
+	}
+}
+
+// Every process the run's record names is asked of `table`, the redraw's own `ps` snapshot — only once
+// a run is found, so a repository with no run waits on no `ps`.
+async function span_here(repository: string, table: TableRead | undefined): Promise<SpanHere> {
 	const read = run_carry.read_carry(run_carry.carry_path(repository))
 	const ended = run_carry_ended.read_ended(run_carry_ended.ended_path(repository))
 	const launch = backlogrun_launch.pending_launch(repository)
+	const span = run_board_span.span_of(read, ended, launch)
+
+	if (span === undefined) return NO_SPAN
+
+	const same_process = await run_board_roots.same_in(table)
+	const stranded_ports = run_stranded_detect.ports_with(same_process)
 
 	return {
-		span: run_board_span.span_of(read, ended, launch),
+		span,
 		resume: run_board_span.resume_of(read, ended, launch),
 		transcript: read.kind === 'carried' ? read.carry.owner_transcript : undefined,
+		roots: run_board_roots.roots_of(repository, read, same_process),
+		carry: read.kind === 'carried' ? read.carry : undefined,
+		is_stranded: run_stranded_detect.is_stranded_at(repository, stranded_ports),
 	}
 }
 
@@ -87,23 +150,83 @@ function sources_of(
 	}
 }
 
-// joshuafolkken/kit#3969: a lane whose last dispatch is unknown is not called earlier.
-function earlier_lanes(lanes: ReadonlyArray<LaneInfo>, started_ms: number): Array<string> {
-	return lanes
-		.filter((lane) => lane.dispatched_ms !== undefined && lane.dispatched_ms < started_ms)
-		.map((lane) => lane.issue)
+// The open lanes sorted by their records, each folded from the whole stream, not the run's span: an
+// earlier run's lane launched before the span began.
+async function lanes_here(
+	lanes: ReadonlyArray<LaneInfo>,
+	repository: string,
+	events: ReadonlyArray<RunEvent>,
+	started_ms: number,
+): Promise<BoardLanes> {
+	const issues = lanes.map((lane) => lane.issue)
+	const records = await lane_state.read_states(issues, repository)
+
+	return run_board_lane_state.lanes_of({ events, lanes: issues, records, started_ms })
+}
+
+// The lanes not yet ended and the processes their closes could not end, off their records — once a
+// retried close ends the lane, its survivors are gone.
+async function unfinished_here(repository: string): Promise<Pick<LocalRead, 'closing' | 'stuck'>> {
+	const records = await lane_state.read_records(repository)
+	const unfinished = records.filter((record) => record.state !== 'ended')
+
+	return {
+		closing: new Set(unfinished.map(({ issue }) => issue)),
+		stuck: new Map(
+			unfinished.flatMap(({ stuck = [] }) => stuck.map(({ pid, reason }) => [pid, reason])),
+		),
+	}
+}
+
+type LaneRoomRead = Pick<
+	LocalRead,
+	'free_lanes' | 'lane_limit' | 'occupied_count' | 'earlier_occupied' | 'idle_lanes'
+>
+
+// The lanes against the limit, read once so the free count, the lanes it counts, the earlier run's
+// among them and the idle ones agree.
+async function lane_room_of(
+	lanes: ReadonlyArray<LaneInfo>,
+	sorted: BoardLanes,
+	table: TableRead | undefined,
+): Promise<LaneRoomRead> {
+	const room = await backlog_ready.lane_room(lanes, table)
+	const count = await backlog_ready.count_in(room)
+	const occupied = room.occupied.map((lane) => lane.issue)
+
+	return {
+		free_lanes: count?.free,
+		lane_limit: count?.limit,
+		occupied_count: occupied.length,
+		earlier_occupied: run_board_lane_state.earlier_occupied(occupied, sorted),
+		idle_lanes: room.idle,
+	}
+}
+
+// The repository's lane ledger, read once for the usual times and the driver's open hold.
+function ledger_here(repository: string): Pick<LocalRead, 'usual' | 'is_memory_held'> {
+	const entries = lane_ledger.read_entries(lane_ledger.target_of(repository))
+	const now_ms = Date.now()
+
+	return {
+		usual: run_board_usual.usual_of(entries, now_ms),
+		is_memory_held:
+			lane_held_stats.open_reason(entries, now_ms) === backlog_drive_memory.HOLD_REASON,
+	}
 }
 
-// `undefined` when no run has started here.
-async function read_local(): Promise<LocalRead | undefined> {
+// `undefined` when no run has started here. The lane count reads `table`, the redraw's process
+// snapshot, so it agrees with the usage columns.
+async function read_local(table?: TableRead): Promise<LocalRead | undefined> {
 	const repository = await run_carry.repository_directory()
-	const { span, resume, transcript } = repository === undefined ? NO_SPAN : span_here(repository)
+	const { span, resume, transcript, roots, carry, is_stranded } =
+		repository === undefined ? NO_SPAN : await span_here(repository, table)
 
 	if (repository === undefined || span === undefined) return undefined
 
 	const events = run_event_stream.read_events(run_event_stream.target_of(repository))
-	const lanes = await lane_registry.list_lanes()
-	const occupied = await backlog_ready.occupied_lanes(lanes)
+	const lanes = await lane_registry.list_lanes(await main_root())
+	const lane_states = await lanes_here(lanes, repository, events, span.started_ms)
 
 	return {
 		started_ms: span.started_ms,
@@ -111,14 +234,15 @@ async function read_local(): Promise<LocalRead | undefined> {
 		scope: run_board_scope.scope_of(span.invocation),
 		events: run_board_span.events_of(events, span),
 		lanes: lanes.map((lane) => lane.issue),
+		lane_states,
 		resume,
-		is_stranded: run_stranded_detect.is_stranded_at(repository),
+		roots,
+		...(await unfinished_here(repository)),
+		carry,
+		is_stranded,
 		operations: run_board_operation_read.read(sources_of(repository, lanes, transcript)),
-		idle_lanes: lane_spent.idle_lanes(lanes),
-		earlier_lanes: earlier_lanes(lanes, span.started_ms),
-		free_lanes: await backlog_ready.free_lanes_beside(lanes),
-		occupied_lanes: occupied.map((lane) => lane.issue),
-		usual: run_board_usual.read(repository),
+		...(await lane_room_of(lanes, lane_states, table)),
+		...ledger_here(repository),
 	}
 }