@joshuafolkken/kit 1.872.0

package/scripts/backlog/backlog-drive-restore.ts

@@ -0,0 +1,67 @@
+import { run_event_stream, type RunEvent } from '#scripts/run/run-event-stream'
+
+const ISSUE_IN_EVENT = /#([1-9]\d*)\b/u
+const SETTLED_KINDS: ReadonlySet<string> = new Set([
+	run_event_stream.EVENT_KIND.MERGE,
+	run_event_stream.EVENT_KIND.PARK,
+	run_event_stream.EVENT_KIND.SPLIT,
+	run_event_stream.EVENT_KIND.OUTAGE,
+])
+const DONE_KINDS: ReadonlySet<string> = new Set([
+	run_event_stream.EVENT_KIND.MERGE,
+	run_event_stream.EVENT_KIND.PARK,
+	run_event_stream.EVENT_KIND.SPLIT,
+])
+
+function issue_of(event: RunEvent): string | undefined {
+	return ISSUE_IN_EVENT.exec(event.text)?.[1]
+}
+
+function update_active(active: Set<string>, event: RunEvent): void {
+	const issue = issue_of(event)
+
+	if (issue === undefined) return
+	if (event.kind === run_event_stream.EVENT_KIND.CHILD_LAUNCH) active.add(issue)
+	if (SETTLED_KINDS.has(event.kind)) active.delete(issue)
+}
+
+function active_issues(events: ReadonlyArray<RunEvent>): ReadonlySet<string> {
+	const active = new Set<string>()
+
+	for (const event of events) update_active(active, event)
+
+	return active
+}
+
+function settled_issues(events: ReadonlyArray<RunEvent>): ReadonlySet<number> {
+	const settled = events.filter((event) => DONE_KINDS.has(event.kind))
+
+	return new Set(
+		settled
+			.map((event) => issue_of(event))
+			.filter((issue) => issue !== undefined)
+			.map(Number),
+	)
+}
+
+function parked_issues(events: ReadonlyArray<RunEvent>): ReadonlySet<number> {
+	return new Set(
+		events
+			.filter((event) => event.kind === run_event_stream.EVENT_KIND.PARK)
+			.map((event) => issue_of(event))
+			.filter((issue) => issue !== undefined)
+			.map(Number),
+	)
+}
+
+function restore(
+	lanes: ReadonlyArray<string>,
+	events: ReadonlyArray<RunEvent>,
+	merged_issues: ReadonlyArray<number>,
+): ReadonlyArray<string> {
+	const active = active_issues(events)
+
+	return lanes.filter((issue) => active.has(issue) && !merged_issues.includes(Number(issue)))
+}
+
+export const backlog_drive_restore = { restore, settled_issues, parked_issues }

package/scripts/backlog/backlog-drive.ts

@@ -0,0 +1,322 @@
+// The body of `josh backlog:drive`: the backlogrun parent's mechanical transitions, run as one resident
+// wait instead of a model turn per event (joshuafolkken/kit#2499).
+//
+// **What it replaces is the parent's turn between two commands.** Measured after #2492 the parent still
+// spent about half of its backlogrun requests on nothing but glue: a child's process ends, the model
+// wakes, runs `run:merge`, reads a number, runs `lane:launch`, restarts `lane:await`, ends its turn — or,
+// headless, polls in the foreground because its turn cannot end. Every one of those steps is a command
+// whose answer is already a token, so the loop here only reads tokens and never decides what one means.
+//
+// **It decides nothing another command answers.** Collecting a child is `run:merge`'s, choosing the next
+// issue and the budget are `backlog:offer`'s, opening a lane is `lane:launch`'s, and a child's exit is
+// `lane:await`'s re-confirmed process check. What is left is which of them runs next, and when a token is
+// one the model has to read — a control verdict, a refused launch, a watch, the end of the run — the
+// loop stops and hands that token back rather than acting on it.
+//
+// **Everything it touches is a port**, the pattern `run-wake-loop.ts` set, so the sequencing is pinned
+// without a process, a network or a clock.
+
+// The `run:merge` tokens that need the parent: a hand-off, a person's stop, a tripped guard, a foreign
+// owner, an unreadable state. `resumed` is not among them — the same lane is awaited again.
+const HANDOFF_TOKENS: ReadonlySet<string> = new Set([
+	'over',
+	'human-review',
+	'stop',
+	'environment',
+	'busy',
+	'retry',
+])
+const RESUMED_TOKEN = 'resumed'
+const RUN_VERDICT = 'run'
+const STOP_VERDICT = 'stop'
+const WATCH_VERDICT = 'watch'
+// The verdicts the loop acts on without the parent. Any other is handed back as it was printed.
+const KNOWN_VERDICTS: ReadonlySet<string> = new Set([
+	RUN_VERDICT,
+	STOP_VERDICT,
+	WATCH_VERDICT,
+	'wait',
+])
+const NO_RETRIES = 0
+
+interface OfferRead {
+	verdict: string
+	issues: ReadonlyArray<string>
+	retries: number
+	answer?: string | undefined
+	reason?: string | undefined
+	is_retrospective_done?: boolean
+	is_finish?: boolean | undefined
+}
+
+interface DriveState {
+	// The children this run has in flight — seeded from the open lanes, so a restarted loop resumes them.
+	in_flight: Array<string>
+	// Every child collected, fed back to `backlog:offer` because GitHub closes a merged issue late.
+	exclude: Array<string>
+	// When the run last did something, handed to `backlog:budget` as `--active`.
+	active: string
+	retries: number
+	// A `stop` verdict: start nothing more, collect what is in flight, then end.
+	is_stopping: boolean
+	stop_reason?: string | undefined
+	stop_is_finish?: boolean | undefined
+}
+
+interface DriveEnd {
+	// Why the loop handed control back: `merge`, `launch`, `offer`, `watch`, `stop`, `window` or a verdict.
+	reason: string
+	// The token the parent reads, where one command printed it.
+	token: string | undefined
+	issue: string | undefined
+	detail?: string
+	is_finish?: boolean | undefined
+}
+
+interface DrivePorts {
+	// Whether a child's process has confirmed-ended — `lane:await`'s re-confirmed check in production.
+	is_finished: (issue: string) => boolean
+	// `run:merge <N>`'s first stdout token.
+	merge: (issue: string) => Promise<string>
+	// `backlog:offer`'s answer for the state, or `undefined` when it exited non-zero.
+	offer: (state: DriveState) => Promise<OfferRead | undefined>
+	launch: (issue: string) => Promise<boolean>
+	now: () => Date
+}
+
+// An end carries the state reached so far, so the resume line keeps what the pass collected before it.
+type PassResult =
+	{ kind: 'continue'; state: DriveState } | { kind: 'end'; end: DriveEnd; state: DriveState }
+
+function ended(reason: string, state: DriveState, token?: string, issue?: string): PassResult {
+	const detail = reason === STOP_VERDICT ? state.stop_reason : undefined
+
+	return {
+		kind: 'end',
+		end: {
+			reason,
+			token,
+			issue,
+			...(detail !== undefined && { detail }),
+			...(reason === STOP_VERDICT && { is_finish: state.stop_is_finish }),
+		},
+		state,
+	}
+}
+
+function is_drain(read: OfferRead, state: DriveState): boolean {
+	return (
+		read.verdict === WATCH_VERDICT && state.in_flight.length === 0 && !read.is_retrospective_done
+	)
+}
+
+function needs_retrospective(read: OfferRead, state: DriveState): boolean {
+	return (
+		read.verdict === STOP_VERDICT &&
+		read.answer === 'exhausted' &&
+		state.in_flight.length === 0 &&
+		!read.is_retrospective_done
+	)
+}
+
+function initial_state(in_flight: ReadonlyArray<string>, active: string): DriveState {
+	return {
+		in_flight: [...in_flight],
+		exclude: [],
+		active,
+		retries: NO_RETRIES,
+		is_stopping: false,
+	}
+}
+
+// One finished child: collected, excluded, and — unless its cut was resumed in place — out of flight.
+async function collect(issue: string, state: DriveState, ports: DrivePorts): Promise<PassResult> {
+	const token = await ports.merge(issue)
+
+	// An empty first line is a `run:merge` that failed before printing one: never read as a collection.
+	if (token === '') return ended('merge', state, undefined, issue)
+	if (HANDOFF_TOKENS.has(token)) return ended('merge', state, token, issue)
+
+	const in_flight =
+		token === RESUMED_TOKEN ? state.in_flight : state.in_flight.filter((item) => item !== issue)
+	const exclude = state.exclude.includes(issue) ? state.exclude : [...state.exclude, issue]
+
+	return {
+		kind: 'continue',
+		state: { ...state, in_flight, exclude, active: ports.now().toISOString() },
+	}
+}
+
+async function collect_finished(state: DriveState, ports: DrivePorts): Promise<PassResult> {
+	const finished = state.in_flight.filter((item) => ports.is_finished(item))
+	let current = state
+
+	for (const issue of finished) {
+		const result = await collect(issue, current, ports)
+
+		if (result.kind === 'end') return { ...result, state: current }
+
+		current = result.state
+	}
+
+	return { kind: 'continue', state: current }
+}
+
+async function launch_all(
+	issues: ReadonlyArray<string>,
+	state: DriveState,
+	ports: DrivePorts,
+): Promise<PassResult> {
+	const in_flight = [...state.in_flight]
+
+	for (const issue of issues) {
+		const is_launched = await ports.launch(issue)
+
+		if (!is_launched) return ended('launch', { ...state, in_flight }, undefined, issue)
+
+		in_flight.push(issue)
+	}
+
+	return { kind: 'continue', state: { ...state, in_flight, active: ports.now().toISOString() } }
+}
+
+// A watch with nothing in flight is the drain: `run:step` owes the retrospective there, so it is the
+// parent's. With children still running the watch is only waiting on them.
+function on_verdict(read: OfferRead, state: DriveState): PassResult {
+	if (!KNOWN_VERDICTS.has(read.verdict)) return ended(read.verdict, state, read.verdict)
+	if (needs_retrospective(read, state)) return ended('retrospective', state, 'retrospective')
+
+	if (read.verdict === STOP_VERDICT) {
+		return {
+			kind: 'continue',
+			state: {
+				...state,
+				is_stopping: true,
+				stop_reason: read.reason,
+				stop_is_finish: read.is_finish,
+			},
+		}
+	}
+
+	if (is_drain(read, state)) return ended(WATCH_VERDICT, state, WATCH_VERDICT)
+
+	return { kind: 'continue', state }
+}
+
+async function ask_offer(state: DriveState, ports: DrivePorts): Promise<PassResult> {
+	const read = await ports.offer(state)
+
+	if (read === undefined) return ended('offer', state)
+
+	const next = { ...state, retries: read.retries }
+
+	if (read.verdict === RUN_VERDICT) return await launch_all(read.issues, next, ports)
+
+	return on_verdict(read, next)
+}
+
+// One pass: collect every child that ended, then — unless a `stop` has been read — ask what to start.
+// `should_offer` is the caller's throttle on the one network read; a collection always asks, because a
+// lane just freed.
+function is_offering(before: DriveState, after: DriveState, should_offer: boolean): boolean {
+	if (after.is_stopping) return false
+
+	return should_offer || after.in_flight.length !== before.in_flight.length
+}
+
+// A stopping run with nothing left in flight has ended.
+function settle(result: PassResult): PassResult {
+	if (result.kind === 'end') return result
+
+	const is_done = result.state.is_stopping && result.state.in_flight.length === 0
+
+	return is_done ? ended(STOP_VERDICT, result.state, STOP_VERDICT) : result
+}
+
+async function run_pass(
+	state: DriveState,
+	should_offer: boolean,
+	ports: DrivePorts,
+): Promise<PassResult> {
+	const collected = await collect_finished(state, ports)
+
+	if (collected.kind === 'end') return collected
+	if (!is_offering(state, collected.state, should_offer)) return settle(collected)
+
+	return settle(await ask_offer(collected.state, ports))
+}
+
+interface LoopConfig {
+	poll_ms: number
+	// The floor between two `backlog:offer` asks while nothing was collected — the same one-minute
+	// cadence the wake supervisor polls the backlog on, so an idle stretch is not a network hammer.
+	offer_ms: number
+	// How long one invocation may wait before returning `window`, or `undefined` for no bound. A headless
+	// parent's foreground call is capped by its shell's timeout, so it passes one under that cap.
+	window_ms: number | undefined
+}
+
+interface LoopPorts extends DrivePorts {
+	sleep: (milliseconds: number) => Promise<void>
+	finish: (end: DriveEnd) => Promise<void>
+	// Called after every pass, ended ones included, so the caller can keep the resume line current.
+	on_state: (state: DriveState) => void
+}
+
+interface LoopRun {
+	state: DriveState
+	started_ms: number
+	offered_ms: number
+}
+
+const WINDOW_END: DriveEnd = { reason: 'window', token: undefined, issue: undefined }
+
+function is_window_spent(run: LoopRun, config: LoopConfig, now_ms: number): boolean {
+	if (config.window_ms === undefined) return false
+
+	return now_ms - run.started_ms >= config.window_ms
+}
+
+type LoopStep = { kind: 'end'; end: DriveEnd } | { kind: 'next'; run: LoopRun }
+
+// One pass of the loop: the offer is due on the first pass and then no oftener than `offer_ms`, bar the
+// passes that collected a child. A `next` carries the run to sleep on and pass again.
+async function loop_step(run: LoopRun, config: LoopConfig, ports: LoopPorts): Promise<LoopStep> {
+	const now_ms = ports.now().getTime()
+	const is_due = now_ms - run.offered_ms >= config.offer_ms
+	const result = await run_pass(run.state, is_due, ports)
+
+	ports.on_state(result.state)
+
+	if (result.kind === 'end') return result
+
+	const next = { ...run, state: result.state, offered_ms: is_due ? now_ms : run.offered_ms }
+
+	return is_window_spent(next, config, ports.now().getTime())
+		? { kind: 'end', end: WINDOW_END }
+		: { kind: 'next', run: next }
+}
+
+// Pass, sleep, repeat — until a pass hands a token back or the window is spent.
+async function run_loop(
+	state: DriveState,
+	config: LoopConfig,
+	ports: LoopPorts,
+): Promise<DriveEnd> {
+	const first: LoopRun = { state, started_ms: ports.now().getTime(), offered_ms: -Infinity }
+	let step = await loop_step(first, config, ports)
+
+	while (step.kind === 'next') {
+		await ports.sleep(config.poll_ms)
+		step = await loop_step(step.run, config, ports)
+	}
+
+	if (step.end.reason === STOP_VERDICT) await ports.finish(step.end)
+
+	return step.end
+}
+
+const backlog_drive = { HANDOFF_TOKENS, collect, initial_state, on_verdict, run_loop, run_pass }
+
+export type { DriveEnd, DrivePorts, DriveState, LoopConfig, LoopPorts, OfferRead, PassResult }
+export { backlog_drive }

package/scripts/backlog/backlog-named.ts

@@ -0,0 +1,164 @@
+// The named-issue prefix of a `backlogrun` invocation (joshuafolkken/kit#1984). `backlogrun #N1 #N2 …`
+// runs the issues typed after the keyword in the order they were typed, one at a time, and only once
+// they are all done does the opted-in backlog drain. This module is the single source of that order
+// and of what a failure partway through it does, so the plan a person reads (`backlog:plan`) and the
+// loop that runs it (`backlogrun.md`) cannot disagree.
+//
+// **It replaces the old `queue` keyword.** `queue #N1 #N2 …` was a separate entry point for exactly
+// this sequential run; folding it in means one command carries both the named prefix and the backlog
+// that follows it, under one authorization and one carried budget.
+//
+// **A named item is a single issue or an epic** (joshuafolkken/kit#1985, folding in the old `epicrun`
+// keyword). A single-issue item is one `fullrun`; an epic item runs every child in dependency order,
+// across the free lanes, and is done only once every child has been processed — merged or parked. The
+// run does not advance to the next named item until the epic item is complete, which is `epicrun`'s
+// whole-epic execution at the granularity of one named item.
+
+const NAMED_KIND = 'named'
+const BACKLOG_KIND = 'backlog'
+// What follows the named list when `--only` was typed: nothing. `end` is `backlog`'s counterpart in
+// `after_failure.next`, so the loop reads one field to know whether the pool is next or the run is over.
+const END_KIND = 'end'
+const NEXT_INDEX_OFFSET = 1
+const NOT_FOUND = -1
+
+// `backlogrun --only` with no named issues has nothing to run — the flag says "run the named list and
+// stop", and an empty named list makes that the empty run. Refused before anything starts rather than
+// begun and found empty, the same way a budget-only invocation is still a valid bare `backlogrun`.
+const NOTHING_TO_RUN =
+	'`--only` runs the named issues and stops, but none were named, so there is nothing to run. Name the issues (`backlogrun #N1 #N2 … --only`), or drop `--only` to drain the opted-in backlog.'
+
+// One named item: a single issue (`is_epic: false`) or an epic (`is_epic: true`). The parser only sees
+// `#N`, so which one it is is decided by the caller from GitHub state; the pure functions here carry
+// the flag rather than re-derive it, so the plan and the loop agree on how each item runs.
+interface NamedItem {
+	issue: number
+	is_epic: boolean
+}
+
+interface NamedStep {
+	kind: typeof NAMED_KIND
+	issue: number
+	is_epic: boolean
+}
+
+interface BacklogStep {
+	kind: typeof BACKLOG_KIND
+}
+
+type PlanStep = NamedStep | BacklogStep
+
+// A named issue that could not finish parks like any other child; what is specific to a sequential run
+// is that the issues declared *after* it are not started — the order was the point of naming them. What
+// follows is the pool by default, or nothing under `--only`, so the run then drains the opted-in
+// backlog or ends (joshuafolkken/kit#1984).
+//
+// **This skip only applies to a single-issue item.** An epic item does not park as a unit — its
+// children park individually and the epic item completes anyway (`epic_item_outcome`), so a parked
+// epic child, which is never in the named list, skips nothing and the run advances to the next item
+// (joshuafolkken/kit#1985).
+interface FailureOutcome {
+	parked: number
+	skipped: ReadonlyArray<number>
+	next: typeof BACKLOG_KIND | typeof END_KIND
+}
+
+// The completion of one epic named item. It runs every child and finishes once each has been
+// processed; the children that parked are carried here so the completion report can list them, and the
+// item is complete regardless — a parked child never holds up the next named item
+// (joshuafolkken/kit#1985).
+interface EpicItemOutcome {
+	epic: number
+	parked: ReadonlyArray<number>
+	complete: true
+}
+
+// One child of an epic item after it ran: merged (`parked: false`) or parked (`parked: true`).
+interface EpicChildOutcome {
+	issue: number
+	parked: boolean
+}
+
+// The plan, or the refusal that stops it before it starts. `startup` is the single decision a
+// `backlogrun` begins from, so the refusal and the order cannot drift into two places.
+type Startup =
+	{ kind: 'plan'; steps: ReadonlyArray<PlanStep> } | { kind: 'refused'; reason: string }
+
+const BACKLOG_STEP: BacklogStep = { kind: BACKLOG_KIND }
+
+function named_step(item: NamedItem): NamedStep {
+	return { kind: NAMED_KIND, issue: item.issue, is_epic: item.is_epic }
+}
+
+// The execution plan: the named items in the order they were declared, then the opted-in backlog —
+// unless `--only`, which stops after the named list. Each named step carries whether it is an epic, so
+// the loop knows to run one `fullrun` or a whole epic. A bare `backlogrun` names none, so the plan is
+// the backlog alone, which is the behavior that entry point always had.
+function plan(named: ReadonlyArray<NamedItem>, is_only = false): ReadonlyArray<PlanStep> {
+	const steps = named.map((item) => named_step(item))
+
+	return is_only ? steps : [...steps, BACKLOG_STEP]
+}
+
+// The plan a `backlogrun` starts from, or the refusal that keeps it from starting. Only `--only` with
+// no named items is refused; everything else is a plan, a bare `backlogrun` (the backlog alone)
+// included.
+function startup(named: ReadonlyArray<NamedItem>, is_only = false): Startup {
+	if (is_only && named.length === 0) return { kind: 'refused', reason: NOTHING_TO_RUN }
+
+	return { kind: 'plan', steps: plan(named, is_only) }
+}
+
+// A single-issue named item failed midway. It parks, the items declared after it are skipped (starting
+// them would break the order they were named in), and the run proceeds to the backlog — or ends under
+// `--only`, which never had a pool to fall through to. A failure whose number is not a *single-issue*
+// named item skips nothing: an issue never named, and an epic child (which is not in the named list),
+// both leave the order intact, so the run simply continues. An epic named item completes through
+// `epic_item_outcome` rather than parking as a unit, so its number never skips the list either.
+function after_failure(
+	named: ReadonlyArray<NamedItem>,
+	failed: number,
+	is_only = false,
+): FailureOutcome {
+	const index = named.findIndex((item) => item.issue === failed && !item.is_epic)
+	const skipped =
+		index === NOT_FOUND ? [] : named.slice(index + NEXT_INDEX_OFFSET).map((item) => item.issue)
+
+	return { parked: failed, skipped, next: is_only ? END_KIND : BACKLOG_KIND }
+}
+
+// An epic named item is complete once every child has been processed — merged or parked. The parked
+// children are carried out so the completion report can list them, and the item is complete either
+// way: a parked epic child does not hold up the next named item, which is `epicrun`'s park-and-continue
+// at the granularity of one named epic item (joshuafolkken/kit#1985).
+function epic_item_outcome(
+	epic: number,
+	children: ReadonlyArray<EpicChildOutcome>,
+): EpicItemOutcome {
+	const parked = children.filter((child) => child.parked).map((child) => child.issue)
+
+	return { epic, parked, complete: true }
+}
+
+const backlog_named = {
+	BACKLOG_KIND,
+	END_KIND,
+	NAMED_KIND,
+	NOTHING_TO_RUN,
+	after_failure,
+	epic_item_outcome,
+	plan,
+	startup,
+}
+
+export type {
+	BacklogStep,
+	EpicChildOutcome,
+	EpicItemOutcome,
+	FailureOutcome,
+	NamedItem,
+	NamedStep,
+	PlanStep,
+	Startup,
+}
+export { backlog_named }