@joshuafolkken/kit 1.1514.0 → 1.1784.0

package/scripts/run/board/run-board-gather-worker.ts

@@ -0,0 +1,29 @@
+import { setTimeout as sleep } from 'node:timers/promises'
+import { parentPort, type MessagePort } from 'node:worker_threads'
+import { run_board_gather } from './run-board-gather'
+import { run_board_gather_live } from './run-board-gather-live'
+import { run_board_gather_thread } from './run-board-gather-thread'
+import { run_board_spin } from './run-board-spin'
+
+// The live board's worker thread: one round of the reads off this machine a redraw apart, each handed
+// to the board as it lands (`run-board-gather-thread.ts`). A round that throws is handed over as
+// failed, and the next round reads again.
+
+const { FRESH_GATHER } = run_board_gather
+const { gather_round } = run_board_gather_thread
+
+async function gather_forever(port: MessagePort): Promise<void> {
+	const ports = run_board_gather_live.ports_of(true)
+	let state = FRESH_GATHER
+
+	for (;;) {
+		// eslint-disable-next-line no-await-in-loop -- polling: each round folds the state the last one left
+		state = await gather_round(state, ports, (value) => {
+			port.postMessage(value)
+		})
+		// eslint-disable-next-line no-await-in-loop -- polling: each round waits a redraw
+		await sleep(run_board_spin.REDRAW_MS)
+	}
+}
+
+if (parentPort !== null) await gather_forever(parentPort)

package/scripts/run/board/run-board-gather.ts

@@ -0,0 +1,187 @@
+import type { BudgetWait } from '#scripts/gate/core-wait'
+import type { MachineSample } from '#scripts/gate/machine-capacity'
+import { lane_await } from '#scripts/lane/lane-await'
+import { lane_process_table, type TableRead } from '#scripts/lane/lane-process-table'
+import { MS_PER_SECOND } from '#scripts/lib/timeouts'
+import type { BoardUsage } from './run-board-header'
+import { run_board_idle, type IdleSince } from './run-board-idle'
+import { run_board_machine, type MachineGauges, type MachineMark } from './run-board-machine'
+import type { LocalRead } from './run-board-read'
+import { run_board_state, type Pace } from './run-board-state'
+import type { UsageMark } from './run-board-usage'
+import { run_board_usage_board } from './run-board-usage-board'
+
+// What `run:board` reads off this machine: the run's own stream and lanes, the process table and the
+// machine's gauges. A live board reads them on a worker thread (`run-board-gather-worker.ts`), so no
+// read stops the spinner; a single frame has no spinner to stop and reads them where it draws. **Two
+// speeds**: the run's own stream and lanes are re-read no more often than `LOCAL_READ_MS`, because each
+// read parses the whole stream and asks git twice, and a board left open all night would otherwise grow
+// heavier as the run grew longer; the machine is sampled once a second — one `sysctl` takes a few
+// milliseconds, and its gauges are the differences between consecutive samples.
+
+const MACHINE_SAMPLE_MS = MS_PER_SECOND
+const LOCAL_READ_SECONDS = 5
+const LOCAL_READ_MS = LOCAL_READ_SECONDS * MS_PER_SECOND
+const { is_due } = run_board_state
+
+interface GatherPorts {
+	// Every process on the machine, one `ps` (`lane-process-table.ts`); a read takes it at most once
+	// and hands it to both readers below, so the lane count and the usage columns read one instant.
+	read_table: TableRead
+	// `undefined` when no run has started here.
+	read_local: (table: TableRead) => Promise<LocalRead | undefined>
+	read_machine: () => Promise<MachineSample>
+	// The claims waiting in the core budget, read on the machine's second; none where it is not given.
+	read_waits?: (() => Array<BudgetWait>) | undefined
+	// Every process and its lane, the last reading's lanes reused (`run-board-usage-read.ts`).
+	read_usage: (before: UsageMark | undefined, table: TableRead) => Promise<UsageMark | undefined>
+	// Whether the reads go on — a live board — so a usage reading has the last one to compare against.
+	is_live: boolean
+	now: () => number
+	sleep: (ms: number) => Promise<void>
+}
+
+// What a frame draws from the reads, as a worker hands it over.
+interface GatherRead {
+	// The last local read, drawn until the next is due; `undefined` when no run had started.
+	local: LocalRead | undefined
+	read_ms: number | undefined
+	// When each lane the reads found idle was first read so (`run-board-idle.ts`).
+	idle_since: IdleSince
+	gauges: MachineGauges | undefined
+	// Each lane's and each process group's usage, `undefined` where none could be read.
+	usages: BoardUsage | undefined
+	budget_waits: ReadonlyArray<BudgetWait>
+	// When the reads last failed; `undefined` once a read has answered since.
+	read_failed_ms?: number | undefined
+	// Why they failed, the first line of the error; cleared with `read_failed_ms`.
+	read_error?: string | undefined
+}
+
+interface GatherState extends GatherRead {
+	// The last machine sample, which the next one is compared against.
+	machine: MachineMark | undefined
+	// The read the last sample was taken on, which the next sample is due a second after.
+	sampled_ms: number | undefined
+	// The last process reading, which the next one is compared against.
+	usage: UsageMark | undefined
+}
+
+// The reads a frame draws from, taken as it asks.
+type Gatherer = (pace: Pace) => Promise<GatherRead>
+
+const FRESH_GATHER: GatherState = {
+	local: undefined,
+	read_ms: undefined,
+	idle_since: run_board_idle.NO_IDLE,
+	gauges: undefined,
+	usages: undefined,
+	budget_waits: [],
+	machine: undefined,
+	sampled_ms: undefined,
+	usage: undefined,
+}
+
+// The process snapshot and the pace the reads are taken at, shared by the reads.
+interface Snapshot {
+	pace: Pace
+	table: TableRead
+}
+
+interface SnapshotAt extends Snapshot {
+	now_ms: number
+}
+
+async function read_at(
+	state: GatherState,
+	ports: GatherPorts,
+	at: SnapshotAt,
+): Promise<GatherState> {
+	const local = await ports.read_local(at.table)
+	const idle_since = run_board_idle.since_of(state.idle_since, local?.idle_lanes ?? [], at.now_ms)
+
+	return { ...state, local, read_ms: at.now_ms, idle_since }
+}
+
+// A single frame has no earlier read to confirm an idle lane against (`run-board-idle.ts`).
+function is_unconfirmed(before: GatherState, read: GatherState, pace: Pace): boolean {
+	return pace === 'settled' && before.read_ms === undefined && read.idle_since.size > 0
+}
+
+// The reads over a snapshot taken now.
+function fresh_snapshot(ports: GatherPorts, pace: Pace): Snapshot {
+	return { pace, table: lane_process_table.shared(ports.read_table) }
+}
+
+// The lane read and the reads after it share one snapshot; a confirmation taken after the wait reads
+// a snapshot of its own and hands it on, so no lane is counted from the state before the wait.
+async function reread(
+	state: GatherState,
+	ports: GatherPorts,
+	snapshot: Snapshot,
+): Promise<{ read: GatherState; snapshot: Snapshot }> {
+	const now_ms = ports.now()
+
+	if (!is_due(state.read_ms, LOCAL_READ_MS, now_ms)) return { read: state, snapshot }
+
+	const read = await read_at(state, ports, { now_ms, ...snapshot })
+
+	if (!is_unconfirmed(state, read, snapshot.pace)) return { read, snapshot }
+
+	await ports.sleep(lane_await.RECONFIRM_MS)
+
+	const later = fresh_snapshot(ports, snapshot.pace)
+
+	return { read: await read_at(read, ports, { now_ms: ports.now(), ...later }), snapshot: later }
+}
+
+// The sample is timed when it is taken, not when the reads began: it is due by the reads' clock, so a
+// sample taken a few milliseconds in does not slip to the read after the second.
+async function resample(
+	state: GatherState,
+	ports: GatherPorts,
+	at: SnapshotAt,
+): Promise<GatherState> {
+	if (!is_due(state.sampled_ms, MACHINE_SAMPLE_MS, at.now_ms)) return state
+
+	const at_ms = ports.now()
+	const machine = { sample: await ports.read_machine(), at_ms }
+	const gauges = run_board_machine.gauges_of(state.machine, machine)
+	const budget_waits = ports.read_waits?.() ?? []
+	const usages = await run_board_usage_board.read_usages(state, ports, at.table)
+
+	return { ...state, ...usages, machine, gauges, budget_waits, sampled_ms: at.now_ms }
+}
+
+// One round of the reads. The process snapshot is read once, by whichever of the lane read and the
+// sample asks first.
+async function gather(state: GatherState, ports: GatherPorts, pace: Pace): Promise<GatherState> {
+	const { read, snapshot } = await reread(state, ports, fresh_snapshot(ports, pace))
+
+	return await resample(read, ports, { now_ms: ports.now(), ...snapshot })
+}
+
+// What a frame draws, without the marks the next reads compare against.
+function read_of(state: GatherState): GatherRead {
+	const { local, read_ms, idle_since, gauges, usages, budget_waits } = state
+	const { read_failed_ms, read_error } = state
+
+	return { local, read_ms, idle_since, gauges, usages, budget_waits, read_failed_ms, read_error }
+}
+
+// A single frame's reads, taken afresh each time it is drawn.
+function frame_gatherer(ports: GatherPorts): Gatherer {
+	return async (pace) => read_of(await gather(FRESH_GATHER, ports, pace))
+}
+
+const run_board_gather = {
+	FRESH_GATHER,
+	LOCAL_READ_MS,
+	MACHINE_SAMPLE_MS,
+	frame_gatherer,
+	gather,
+	read_of,
+}
+
+export { run_board_gather }
+export type { GatherPorts, GatherRead, GatherState, Gatherer }

package/scripts/run/board/run-board-github.ts

@@ -1,24 +1,32 @@
-import { run_progress } from '#scripts/run/progress/run-progress'
+import { MS_PER_SECOND } from '#scripts/lib/timeouts'
 import { run_progress_cli } from '#scripts/run/progress/run-progress-cli'
 import type { ClosedAnswer } from './run-board-closed'
 import { run_board_fetch, type Fetch } from './run-board-fetch'
 import { run_board_idle } from './run-board-idle'
-import { run_board_layout, type BoardPlan } from './run-board-layout'
+import type { BoardPlan } from './run-board-layout'
 import type { LocalRead } from './run-board-read'
+import type { Known } from './run-board-snapshot'
 import { run_board_state, type BoardPorts, type BoardState, type Pace } from './run-board-state'
-import { run_board_status, type ItemStatus } from './run-board-status'
+import {
+	run_board_status,
+	type ItemState,
+	type ItemStatus,
+	type OpenRead,
+} from './run-board-status'
 
 // The GitHub reads a `run:board` redraw draws over: the plan, and the run's children that left the
-// open listing. Each is taken no more often than `run:progress` re-reads after a decline. **A live
+// open listing or are drawn stopped. Each is taken no more often than `run:progress` re-reads after a decline. **A live
 // board never waits on them**: each read is launched and folded in on the redraw after it lands, so
 // the clock keeps moving while GitHub answers; one plain frame waits for them. A failed plan read
 // keeps the previous plan on screen and says when it failed. A run that ended keeps its plan as it
 // last read.
 
-const PLAN_RETRY_MS = run_progress_cli.DECLINE_RETRY_SECONDS * run_progress.MS_PER_SECOND
+const PLAN_RETRY_MS = run_progress_cli.DECLINE_RETRY_SECONDS * MS_PER_SECOND
 // A rejected closed read answers nothing closed and not whole, so it is asked again.
 const UNANSWERED: ClosedAnswer = { closed: new Map(), is_whole: false }
 const { is_due } = run_board_state
+const STOPPED_STATE: ItemState = 'stopped'
+const RUNNING_STATE: ItemState = 'running'
 
 // What a step of the GitHub reads works against.
 interface Gather {
@@ -28,19 +36,39 @@ interface Gather {
 	pace: Pace
 }
 
-// A title the board once knew stays known, so an issue that left the open listing keeps its name.
-function with_titles(plan: BoardPlan, previous: BoardPlan | undefined): BoardPlan {
-	const titles = new Map([...(previous?.context.titles ?? []), ...plan.context.titles])
+// A title the board once knew stays known, so an issue that left the open listing keeps its name —
+// one the last board's snapshot named among them.
+function titles_of(state: BoardState): ReadonlyMap<number, string> {
+	return state.plan?.context.titles ?? state.known.titles
+}
+
+function with_titles(plan: BoardPlan, state: BoardState): BoardPlan {
+	const titles = new Map([...titles_of(state), ...plan.context.titles])
 
 	return { ...plan, context: { ...plan.context, titles } }
 }
 
+// A lane this run started after the last plan read was asked, on an issue the plan has no title for
+// (joshuafolkken/kit#4577). The read it launches is asked after the lane started, so a title still
+// missing after it — past the listing's cap, or a failed read — waits the usual interval.
+function has_untitled_lane(state: BoardState, gather: Gather): boolean {
+	const asked_ms = state.attempted_ms ?? Infinity
+	const titles = titles_of(state)
+
+	return [...gather.local.lane_states.own].some(
+		([issue, status]) => !titles.has(issue) && (status.started_ms ?? -Infinity) > asked_ms,
+	)
+}
+
 // An ended run's plan is read until it lands once, then held; a read in flight is not launched again.
+// A lane with no title is read for at once rather than after the interval.
 function is_plan_due(state: BoardState, gather: Gather): boolean {
 	if (state.plan_fetch !== undefined) return false
 	if (gather.local.ended_ms !== undefined && state.fetched_ms !== undefined) return false
 
-	return is_due(state.attempted_ms, PLAN_RETRY_MS, gather.now_ms)
+	return (
+		is_due(state.attempted_ms, PLAN_RETRY_MS, gather.now_ms) || has_untitled_lane(state, gather)
+	)
 }
 
 function launch_plan(state: BoardState, gather: Gather): BoardState {
@@ -67,41 +95,61 @@ function fold_plan(state: BoardState): BoardState {
 
 	return {
 		...settled,
-		plan: with_titles(answer.value, state.plan),
+		plan: with_titles(answer.value, state),
 		fetched_ms: state.attempted_ms,
 		failed_ms: undefined,
 	}
 }
 
-// joshuafolkken/kit#3969: the planned issues whose open lane may be this run's though the stream never
-// saw it launched. An earlier run's lane is not, so its issue waits until this run launches it.
-function lanes_in_run(plan: BoardPlan, local: LocalRead): ReadonlySet<number> {
-	const earlier = new Set(local.earlier_lanes?.map(Number))
+function statuses_for(state: BoardState, local: LocalRead): ReadonlyMap<number, ItemStatus> {
+	const stopped = run_board_idle.stopped_of(state.idle_since, state.read_ms)
 
-	return new Set([...run_board_layout.numbers_of(plan)].filter((issue) => !earlier.has(issue)))
+	return run_board_status.statuses_of(local.events, local.lane_states, stopped)
 }
 
-function statuses_for(
-	state: BoardState,
-	plan: BoardPlan,
-	local: LocalRead,
-): ReadonlyMap<number, ItemStatus> {
-	const in_run = lanes_in_run(plan, local)
-	const stopped = run_board_idle.stopped_of(state.idle_since, state.read_ms)
-
-	return run_board_status.statuses_of(local.events, local.lanes, in_run, stopped)
+// The open listing the statuses are settled against — none before a plan has landed.
+function open_read_of(state: BoardState, plan: BoardPlan | undefined): OpenRead {
+	return {
+		open_numbers: plan?.context.open_numbers,
+		read_ms: state.fetched_ms,
+		closed: state.closed,
+		labels: plan?.labels ?? new Map(),
+	}
 }
 
-// The children the run touched that the open listing no longer holds and the board has not read yet.
-// A listing cut short proves nothing missing, so it asks about none.
-function unread_closed(state: BoardState, local: LocalRead): Array<number> {
-	const open = state.plan?.context.open_numbers
+function is_unlisted(issue: number, read: OpenRead): boolean {
+	const open = read.open_numbers
 
-	if (open === undefined || state.plan === undefined) return []
+	return open !== undefined && !open.has(issue)
+}
 
-	const touched = [...statuses_for(state, state.plan, local).keys()]
+// A child still drawn stopped or running holds no closed read of this launch — any it holds closed
+// before it relaunched — so it is asked again. A running child is asked too while the plan read fails:
+// the kept plan still lists it, and an empty listing — failed, or a repository whose last open issue
+// was this child — would otherwise leave it running forever (joshuafolkken/kit#4537).
+function is_closed_candidate(
+	issue: number,
+	status: ItemStatus,
+	read: OpenRead,
+	is_plan_failed: boolean,
+): boolean {
+	if (status.state === STOPPED_STATE) return true
+	if (status.state === RUNNING_STATE) return is_plan_failed || is_unlisted(issue, read)
+
+	return is_unlisted(issue, read) && !read.closed.has(issue)
+}
+
+// The children the board has not read closed yet that the open listing no longer holds, or that it
+// draws stopped: a child that merged itself and left its lane is drawn stopped until GitHub says it
+// merged (joshuafolkken/kit#4401), and a listing cut short proves nothing missing.
+function unread_closed(state: BoardState, local: LocalRead): Array<number> {
+	const read = open_read_of(state, state.plan)
+	const drawn = run_board_status.settle_closed(statuses_for(state, local), read)
+	const is_plan_failed = state.failed_ms !== undefined
 
-	return touched.filter((issue) => !open.has(issue) && !state.closed.has(issue))
+	return [...drawn]
+		.filter(([issue, status]) => is_closed_candidate(issue, status, read, is_plan_failed))
+		.map(([issue]) => issue)
 }
 
 // An ended run asks until one ask after it ended has answered whole, then holds what it has, as its
@@ -114,18 +162,25 @@ function has_asked_since_end(state: BoardState, local: LocalRead): boolean {
 	return closed_answered_ms >= local.ended_ms
 }
 
-function is_closed_due(state: BoardState, gather: Gather): boolean {
+function has_new_child(state: BoardState, unread: ReadonlyArray<number>): boolean {
+	return unread.some((issue) => !state.closed_asked.has(issue))
+}
+
+function is_closed_due(state: BoardState, gather: Gather, unread: ReadonlyArray<number>): boolean {
 	if (state.closed_fetch !== undefined || has_asked_since_end(state, gather.local)) return false
 
-	return is_due(state.closed_attempted_ms, PLAN_RETRY_MS, gather.now_ms)
+	return (
+		has_new_child(state, unread) || is_due(state.closed_attempted_ms, PLAN_RETRY_MS, gather.now_ms)
+	)
 }
 
 // A closed child is read once and its answer kept; one that has not answered closed is asked again no
-// sooner than the plan is, and an ended run's not again once an ask after the end has answered whole.
+// sooner than the plan is — unless a child the last ask did not hold has joined it — and an ended
+// run's not again once an ask after the end has answered whole.
 function launch_closed(state: BoardState, gather: Gather): BoardState {
 	const unread = unread_closed(state, gather.local)
 
-	if (unread.length === 0 || !is_closed_due(state, gather)) return state
+	if (unread.length === 0 || !is_closed_due(state, gather, unread)) return state
 
 	async function read(): Promise<ClosedAnswer> {
 		return await gather.ports.read_closed(unread)
@@ -133,7 +188,12 @@ function launch_closed(state: BoardState, gather: Gather): BoardState {
 
 	const closed_fetch = run_board_fetch.launch(read, UNANSWERED)
 
-	return { ...state, closed_fetch, closed_attempted_ms: gather.now_ms }
+	return {
+		...state,
+		closed_fetch,
+		closed_attempted_ms: gather.now_ms,
+		closed_asked: new Set(unread),
+	}
 }
 
 // A whole answer counts as of the moment it was asked.
@@ -158,6 +218,22 @@ async function settle(pending: Fetch<unknown> | undefined, pace: Pace): Promise<
 	if (pending !== undefined && pace === 'settled') await pending.landed
 }
 
+// The titles and labels the board names its rows by: the plan's once it has landed, the snapshot's before.
+function known_of(state: BoardState): Known {
+	if (state.plan === undefined) return state.known
+
+	return { titles: state.plan.context.titles, labels: state.plan.labels }
+}
+
+// Each read that landed rewrites the snapshot whole, for the board opened next on this run.
+function keep_snapshot(before: BoardState, after: BoardState, gather: Gather): void {
+	if (after.plan === before.plan && after.closed === before.closed) return
+
+	const run_started_ms = gather.local.started_ms
+
+	gather.ports.snapshot?.save({ run_started_ms, ...known_of(after), closed: after.closed })
+}
+
 // Folds what landed and launches what is due — the plan first, since the closed children are read off
 // the plan's listing.
 async function gather_github(state: BoardState, gather: Gather): Promise<BoardState> {
@@ -169,10 +245,14 @@ async function gather_github(state: BoardState, gather: Gather): Promise<BoardSt
 
 	await settle(closing.closed_fetch, gather.pace)
 
-	return fold_closed(closing)
+	const gathered = fold_closed(closing)
+
+	keep_snapshot(state, gathered, gather)
+
+	return gathered
 }
 
-const run_board_github = { PLAN_RETRY_MS, gather_github, statuses_for }
+const run_board_github = { PLAN_RETRY_MS, gather_github, known_of, open_read_of, statuses_for }
 
 export { run_board_github }
 export type { Gather }