@joshuafolkken/kit 1.1514.0 → 1.1784.0

package/scripts/lane/lane-session-broker.ts

@@ -1,10 +1,14 @@
 import { randomUUID } from 'node:crypto'
-import { mkdirSync, readFileSync, renameSync, rmSync, unlinkSync, writeFileSync } from 'node:fs'
+import { mkdirSync, readFileSync, rmSync, unlinkSync } from 'node:fs'
 import path from 'node:path'
 import { setTimeout as sleep } from 'node:timers/promises'
 import { git_common_directory } from '#scripts/git/git-common-directory'
+import { issue_cite } from '#scripts/issue/issue-cite'
 import { stamp_file } from '#scripts/josh/stamp-file'
+import { atomic_write } from '#scripts/lib/atomic-write'
 import { json_value } from '#scripts/lib/json-value'
+import { run_event_stream } from '#scripts/run/event/run-event-stream'
+import { run_event_stream_emit } from '#scripts/run/event/run-event-stream-emit'
 import { z } from 'zod'
 import { lane_broker_server, type BrokerServer } from './lane-broker-server'
 import type { LaneInfo } from './lane-registry'
@@ -28,11 +32,23 @@ const POLL_MS = 100
 const LEGACY_HEARTBEAT_MS = 500
 const LEGACY_HEARTBEAT_LIMIT_MS = 3000
 const MINUTE_MS = 60_000
-const REQUEST_LIMIT_MINUTES = 30
-const REQUEST_LIMIT_MS = REQUEST_LIMIT_MINUTES * MINUTE_MS
+// **A relaunch is answered once the session has started, a review once it has finished**: waiting out a
+// review's limit on a relaunch left the stopped lane idle for half an hour (joshuafolkken/kit#4344).
+const REQUEST_LIMIT_MINUTES: Readonly<Record<BrokerAction['kind'], number>> = {
+	review: 30,
+	relaunch: 2,
+}
 const request_schema = z
 	.object({ session: z.uuid(), nonce: z.uuid(), action: lane_session_action.action_schema })
 	.strict()
+// **A request this broker cannot read is still answered**: the supervisor reloads on new code only at
+// its next pass boundary (`run-wake-code.ts`), so until then a lane's ship can run a newer main than
+// the broker's, and a field the ship added made the request unreadable and
+// the ship waited out the whole limit on a server that had dropped it (joshuafolkken/kit#4230).
+// **The reason names the fields the schema refused**: a guess at the supervisor's age hid a field the
+// broker's schema had never learned even on one main (joshuafolkken/kit#4489).
+const envelope_schema = z.looseObject({ session: z.uuid(), nonce: z.uuid() })
+const UNREADABLE_REASON = 'the broker could not read the request'
 const response_schema = z
 	.object({
 		session: z.uuid(),
@@ -44,7 +60,7 @@ const response_schema = z
 // **A failure's reason sits beside the response, never in it**: a ship branched before the reason parses
 // the response strictly, so a new field would read as no server (joshuafolkken/kit#3974).
 const reason_schema = z.object({ nonce: z.string(), reason: z.string() }).strict()
-type BrokerRequest = z.infer<typeof request_schema>
+type BrokerEnvelope = z.infer<typeof envelope_schema>
 type BrokerResponse = z.infer<typeof response_schema>
 
 const { failure } = lane_session_action
@@ -73,19 +89,56 @@ function read_record<T>(target: string, schema: z.ZodType<T>): T | undefined {
 }
 
 function write_record(target: string, body: BrokerResponse | z.infer<typeof reason_schema>): void {
-	const temporary = `${target}.${String(process.pid)}.tmp`
+	atomic_write.replace_file(target, JSON.stringify(body))
+}
+
+function refused_field(issue: z.core.$ZodIssue): ReadonlyArray<string> {
+	const at = issue.path.map(String)
+
+	if (issue.code !== 'unrecognized_keys') return [`${at.join('.')}: ${issue.message}`]
+
+	return issue.keys.map((key) => `unrecognized key: ${[...at, key].join('.')}`)
+}
+
+function unreadable(error: z.ZodError): BrokerAnswer {
+	const fields = error.issues.flatMap((issue) => refused_field(issue))
+
+	return failure(`${UNREADABLE_REASON} — ${fields.join('; ')}`)
+}
+
+async function answer_request(lane: LaneInfo, received: BrokerEnvelope): Promise<BrokerAnswer> {
+	const parsed = request_schema.safeParse(received)
+
+	return parsed.success
+		? await lane_session_action.answer(lane, parsed.data.action)
+		: unreadable(parsed.error)
+}
+
+// The requests this process's servers are answering now — what the supervisor's reload waits out
+// (`run-wake-cli.ts`), since closing a server mid-review fails that review.
+const answering = { count: 0 }
 
-	writeFileSync(temporary, JSON.stringify(body))
-	renameSync(temporary, target)
+function is_answering(): boolean {
+	return answering.count > 0
+}
+
+async function answer_counted(lane: LaneInfo, received: BrokerEnvelope): Promise<BrokerAnswer> {
+	answering.count += 1
+
+	try {
+		return await answer_request(lane, received)
+	} finally {
+		answering.count -= 1
+	}
 }
 
 async function process_request(
 	lane: LaneInfo,
 	response: string,
-	received: BrokerRequest,
+	received: BrokerEnvelope,
 	state: { current: BrokerResponse },
 ): Promise<void> {
-	const answer = await lane_session_action.answer(lane, received.action)
+	const answer = await answer_counted(lane, received)
 	const { session, nonce } = received
 
 	if (!answer.is_success) write_record(reason_path(response), { nonce, reason: answer.reason })
@@ -103,7 +156,7 @@ async function serve(
 	const target = request_path(lane.directory, lane.issue)
 
 	while (state.is_running) {
-		const received = read_record(target, request_schema)
+		const received = read_record(target, envelope_schema)
 
 		if (received?.session === session && received.nonce !== last_nonce) {
 			last_nonce = received.nonce
@@ -233,8 +286,10 @@ async function wait_response(
 	target: string,
 	session: string,
 	nonce: string,
+	kind: BrokerAction['kind'],
 ): Promise<BrokerAnswer> {
-	const deadline = Date.now() + REQUEST_LIMIT_MS
+	const limit_minutes = REQUEST_LIMIT_MINUTES[kind]
+	const deadline = Date.now() + limit_minutes * MINUTE_MS
 
 	while (Date.now() < deadline) {
 		const verdict = response_verdict(target, session, nonce)
@@ -244,7 +299,7 @@ async function wait_response(
 		await sleep(POLL_MS)
 	}
 
-	return failure(`the broker gave no answer within ${String(REQUEST_LIMIT_MINUTES)} minutes`)
+	return failure(`the broker gave no answer within ${String(limit_minutes)} minutes`)
 }
 
 function live_response(target: string): BrokerResponse | undefined {
@@ -255,12 +310,18 @@ function live_response(target: string): BrokerResponse | undefined {
 		: undefined
 }
 
+// The session of the server answering for this lane now, or `undefined` while none does. A new session
+// means the server that held the lane before has ended.
+function serving_session(directory: string, issue: string): string | undefined {
+	const target = response_path(directory, issue)
+
+	return target === undefined ? undefined : live_response(target)?.session
+}
+
 // Whether a server answers for this lane now — what a Claude lane's ship asks before it hands a session
 // over rather than starting the session itself.
 function is_serving(directory: string, issue: string): boolean {
-	const target = response_path(directory, issue)
-
-	return target !== undefined && live_response(target) !== undefined
+	return serving_session(directory, issue) !== undefined
 }
 
 // **A request nobody waits for any more is taken back**: a server busy with an earlier request reads the
@@ -270,11 +331,17 @@ function withdraw(pending: string, nonce: string): void {
 	if (read_record(pending, request_schema)?.nonce === nonce) stamp_file.remove_stamp(pending)
 }
 
-async function request(
-	directory: string,
-	issue: string,
-	action: BrokerAction,
-): Promise<BrokerAnswer> {
+// A failure is put on the run's stream as well as the ship's output, so a broker that cannot serve the
+// lane is seen from `run:board` rather than only from the ship log (joshuafolkken/kit#4230).
+async function record_failure(issue: string, answer: BrokerAnswer): Promise<void> {
+	if (answer.is_success) return
+
+	const text = `${issue_cite.plain(issue)} lane broker: ${answer.reason}`
+
+	await run_event_stream_emit.emit(run_event_stream.EVENT_KIND.NOTE, text)
+}
+
+async function send(directory: string, issue: string, action: BrokerAction): Promise<BrokerAnswer> {
 	const target = response_path(directory, issue)
 	const response = target === undefined ? undefined : live_response(target)
 
@@ -285,19 +352,33 @@ async function request(
 	const pending = request_path(directory, issue)
 
 	stamp_file.replace_stamp(pending, body)
-	const answer = await wait_response(target, response.session, nonce)
+	const answer = await wait_response(target, response.session, nonce, action.kind)
 
 	withdraw(pending, nonce)
 
 	return answer
 }
 
+async function request(
+	directory: string,
+	issue: string,
+	action: BrokerAction,
+): Promise<BrokerAnswer> {
+	const answer = await send(directory, issue, action)
+
+	await record_failure(issue, answer)
+
+	return answer
+}
+
 const lane_session_broker = {
+	is_answering,
 	is_serving,
 	request,
 	request_path,
 	response_path,
 	review_paths: lane_session_action.review_paths,
+	serving_session,
 	with_server,
 }
 

package/scripts/lane/lane-snapshot-trace.ts

@@ -0,0 +1,84 @@
+import {
+	PROCESS_ALIVE,
+	PROCESS_NONE,
+	PROCESS_UNKNOWN,
+	type ProcessTrace,
+} from '#scripts/run/run-liveness'
+import { lane_child_invocation } from './lane-child-invocation'
+import { lane_handoff } from './lane-handoff'
+import { lane_process_table, type ProcessTable, type TableRead } from './lane-process-table'
+import type { TreeProbes } from './lane-reap'
+
+// `lane_await.process_trace_default` answered for many lanes from one process snapshot
+// (joshuafolkken/kit#4245). That probe starts a synchronous `pgrep` per lane and reads the ship record's
+// start time through another `ps`, so a reader asking about every lane each few seconds — `run:board`,
+// the backlogrun drive — blocked its event loop for as long as the lanes took between them.
+//
+// **The answers are the probe's own.** A command line matching the lane's pattern, or a detached ship
+// still running for it, is `alive`; a snapshot that matched neither is `none`; a snapshot that could not
+// be read is `unknown`, as an unanswered `pgrep` is. The ship record is read only for a lane no process
+// matched, which is the only lane its answer can change.
+
+type TraceRead = (issue: string) => ProcessTrace
+
+function is_matched(probes: TreeProbes, issue: string): boolean {
+	return probes.matching(lane_child_invocation.process_pattern(issue)).length > 0
+}
+
+/** Each of `issues` traced from `rows`, the snapshot `undefined` where it could not be read. */
+async function traces_of(
+	issues: ReadonlyArray<string>,
+	rows: ProcessTable | undefined,
+): Promise<TraceRead> {
+	if (rows === undefined) return () => PROCESS_UNKNOWN
+
+	const probes = lane_process_table.probes_of(rows)
+	const unmatched = issues.filter((issue) => !is_matched(probes, issue))
+	const is_shipping =
+		unmatched.length === 0
+			? (): boolean => false
+			: await lane_handoff.ship_reader(lane_process_table.start_reader(rows))
+	const idle = new Set(unmatched.filter((issue) => !is_shipping(issue)))
+
+	return (issue) => (idle.has(issue) ? PROCESS_NONE : PROCESS_ALIVE)
+}
+
+function never_merge_waiting(): boolean {
+	return false
+}
+
+/**
+ * Whether each of `issues` has a detached ship waiting on its merge (`lane_handoff.merge_wait_reader`),
+ * read from `rows`. The record is read only where the snapshot shows that lane's ship supervisor, so a
+ * count with no ship running starts no process (joshuafolkken/kit#4466).
+ */
+async function merge_waits_of(
+	issues: ReadonlyArray<string>,
+	rows: ProcessTable | undefined,
+): Promise<(issue: string) => boolean> {
+	if (rows === undefined) return never_merge_waiting
+
+	const probes = lane_process_table.probes_of(rows)
+	const shipping = issues.filter(
+		(issue) => probes.matching(lane_child_invocation.ship_pattern(issue)).length > 0,
+	)
+
+	if (shipping.length === 0) return never_merge_waiting
+
+	return await lane_handoff.merge_wait_reader(lane_process_table.start_reader(rows))
+}
+
+/** How many of `issues` have a detached ship in this checkout waiting on its merge, from a fresh snapshot. */
+async function count_merge_waiting(
+	issues: ReadonlyArray<string>,
+	table: TableRead = lane_process_table.read_table,
+): Promise<number> {
+	const is_waiting = await merge_waits_of(issues, await table())
+
+	return issues.filter((issue) => is_waiting(issue)).length
+}
+
+const lane_snapshot_trace = { count_merge_waiting, merge_waits_of, traces_of }
+
+export { lane_snapshot_trace }
+export type { TraceRead }

package/scripts/lane/lane-spans.ts

@@ -0,0 +1,51 @@
+// The spans an issue's starts open and its ends close, shared by `josh lane:stats`' gaps
+// (`lane-gap-stats.ts`) and `run:board`'s in-lane time (`run-board-lane-time.ts`)
+// (joshuafolkken/kit#4463).
+
+// One instant an issue's wait can end at. `issue` is absent on a stage whose writer held none.
+interface IssueAt {
+	issue: number | undefined
+	at_ms: number
+}
+
+interface IssuePair {
+	starts: ReadonlyArray<IssueAt>
+	bounds: ReadonlyArray<IssueAt>
+	ends: ReadonlyArray<IssueAt>
+}
+
+interface Span {
+	from_ms: number
+	end_ms: number | undefined
+}
+
+// The issue's next start of any kind closes the window: an end after it belongs to that later start,
+// so a start never ended stays not ended rather than borrowing a later run's end.
+function next_start_ms(
+	starts: ReadonlyArray<IssueAt>,
+	issue: number | undefined,
+	from_ms: number,
+): number {
+	const later = starts.filter((other) => other.issue === issue && other.at_ms > from_ms)
+
+	return Math.min(...later.map((other) => other.at_ms))
+}
+
+// Each start until the first end of its own issue, before that issue's next start in `bounds`.
+function issue_spans(pair: IssuePair): Array<Span> {
+	return pair.starts.map((start) => {
+		const until_ms = next_start_ms(pair.bounds, start.issue, start.at_ms)
+
+		return {
+			from_ms: start.at_ms,
+			end_ms: pair.ends.find(
+				(end) => end.issue === start.issue && end.at_ms >= start.at_ms && end.at_ms < until_ms,
+			)?.at_ms,
+		}
+	})
+}
+
+const lane_spans = { issue_spans }
+
+export { lane_spans }
+export type { IssueAt, IssuePair, Span }

package/scripts/lane/lane-spent.ts

@@ -89,8 +89,11 @@ const LIVE_PROBES: SpentProbes = {
 	unpushed_commits: read_unpushed_commits,
 }
 
+// The one probe the in-use readings take, so a reader holding a process snapshot passes its own.
+type TraceProbes = Pick<SpentProbes, 'process_trace'>
+
 // A stranded lane has no work tree for a child to be in, so it is never in use.
-function trace_of(lane: LaneInfo, probes: SpentProbes): ProcessTrace {
+function trace_of(lane: LaneInfo, probes: TraceProbes): ProcessTrace {
 	return lane.is_stranded ? PROCESS_NONE : probes.process_trace(lane.issue)
 }
 
@@ -178,7 +181,7 @@ function issues_traced(traced: ReadonlyArray<TracedLane>, trace: ProcessTrace):
 	return traced.filter(([, answer]) => answer === trace).map(([issue]) => issue)
 }
 
-function traced_of(lanes: ReadonlyArray<LaneInfo>, probes: SpentProbes): Array<TracedLane> {
+function traced_of(lanes: ReadonlyArray<LaneInfo>, probes: TraceProbes): Array<TracedLane> {
 	return lanes.map((lane): TracedLane => [lane.issue, trace_of(lane, probes)])
 }
 
@@ -195,7 +198,7 @@ async function lanes_in_use(probes: SpentProbes = LIVE_PROBES): Promise<LaneUse>
 /** The lanes no child is working in, read as such — never one whose probe did not answer. */
 function idle_lanes(
 	lanes: ReadonlyArray<LaneInfo>,
-	probes: SpentProbes = LIVE_PROBES,
+	probes: TraceProbes = LIVE_PROBES,
 ): Array<string> {
 	return issues_traced(traced_of(lanes, probes), PROCESS_NONE)
 }

package/scripts/lane/lane-stage-stats.ts

@@ -1,3 +1,4 @@
+import { MS_PER_MINUTE } from '#scripts/lib/timeouts'
 import { run_ship_stage } from '#scripts/run/ship/run-ship-stage'
 import { lane_ledger, type DispatchEntry, type LedgerEntry, type StageEntry } from './lane-ledger'
 import { lane_stats, type StatsWindow } from './lane-stats'
@@ -14,9 +15,10 @@ import { lane_stats, type StatsWindow } from './lane-stats'
 // dispatched, and a ship resumed after a fix cannot record the implementation twice.
 //
 // **`ci-wait` is `followup`'s own lap, inside the `followup` stage** — a row of its own so the wait for
-// CI reads beside `review`, the round-one review it would overlap.
+// CI reads beside `review`, the round-one review it would overlap. `coderabbit-wait` and
+// `coderabbit-rate-limit` follow it: how much of `followup`'s waiting CodeRabbit was still running or
+// rate-limited, a row recorded only by a run that saw it.
 
-const MS_PER_MINUTE = 60_000
 const DECIMALS = 1
 const NOT_MEASURED = 'not measured'
 const IMPLEMENT = 'implement'
@@ -25,6 +27,7 @@ const { STAGE } = run_ship_stage
 const STAGES = [
 	IMPLEMENT,
 	STAGE.PREFLIGHT,
+	lane_ledger.PRE_DETACH_STAGE,
 	STAGE.REVIEW,
 	STAGE.SYNC,
 	STAGE.GATE,
@@ -32,6 +35,8 @@ const STAGES = [
 	STAGE.ROUND_TWO,
 	STAGE.FOLLOWUP,
 	lane_ledger.CI_WAIT_STAGE,
+	lane_ledger.CODERABBIT_WAIT_STAGE,
+	lane_ledger.CODERABBIT_RATE_LIMIT_STAGE,
 	STAGE.REPORT,
 ] as const