@joshuafolkken/kit 1.1387.0 → 1.1453.0

package/scripts/lane/lane-session-broker.ts

@@ -0,0 +1,363 @@
+import { randomUUID } from 'node:crypto'
+import { mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'node:fs'
+import path from 'node:path'
+import { setTimeout as sleep } from 'node:timers/promises'
+import { agent_argv } from '#scripts/agent/agent-argv'
+import { agent_role_profile, type AgentProvider } from '#scripts/agent/agent-role-profile'
+import { git_common_directory } from '#scripts/git/git-common-directory'
+import { stamp_file } from '#scripts/josh/stamp-file'
+import { error_text } from '#scripts/lib/error-message'
+import { json_value } from '#scripts/lib/json-value'
+import { detached_launch } from '#scripts/run/detached-launch'
+import { run_ship_review } from '#scripts/run/ship/run-ship-review'
+import { run_ship_stage } from '#scripts/run/ship/run-ship-stage'
+import { z } from 'zod'
+import { lane_broker_server, type BrokerServer } from './lane-broker-server'
+import { lane_child_invocation, type ShipStop } from './lane-child-invocation'
+import { lane_child_marker } from './lane-child-marker'
+import type { LaneInfo } from './lane-registry'
+import { lane_relaunch } from './lane-relaunch'
+
+// The sessions a lane child's detached `josh ship` cannot start itself, started for it by a process
+// that can: the round-1 and round-2 reviewer, and the fresh child a stopped ship hands back to. An
+// OpenAI lane's own supervisor serves it, because a nested Codex cannot write outside its sandbox; a
+// Claude lane is served by the `run:wake` supervisor (`claude-lane-broker.ts`), because Claude Code removes its
+// token from what its Bash tool starts, so a token-only credential never reaches the ship the child
+// launched (joshuafolkken/kit#3817). **The request names an action, never a command**: the server
+// rebuilds the argv from the registered lane, so a child can ask for nothing but these two sessions.
+
+const REQUEST_PREFIX = '.josh-review-request-'
+const RESPONSE_PREFIX = '.josh-review-response-'
+const BRIEF_PREFIX = 'josh-ship-review-brief-'
+const FINDINGS_PREFIX = 'josh-ship-review-findings-'
+const LOG_PREFIX = 'josh-ship-review-log-'
+const CACHE_DIRECTORY = ['node_modules', '.cache']
+const POLL_MS = 100
+// The heartbeat is kept for a ship branched before the server record, and read only where a
+// supervisor started before it left no such record (`lane-broker-server.ts`).
+const LEGACY_HEARTBEAT_MS = 500
+const LEGACY_HEARTBEAT_LIMIT_MS = 3000
+const MINUTE_MS = 60_000
+const REQUEST_LIMIT_MINUTES = 30
+const REQUEST_LIMIT_MS = REQUEST_LIMIT_MINUTES * MINUTE_MS
+const ANTHROPIC_PROVIDER: AgentProvider = 'anthropic'
+const stop_schema = z
+	.object({
+		stage: z.enum(run_ship_stage.STAGE),
+		title: z.string(),
+		flags: z.array(z.string()).readonly(),
+		is_review: z.boolean(),
+		conflicts: z.array(z.string()).readonly().default([]),
+	})
+	.strict()
+const action_schema = z.discriminatedUnion('kind', [
+	z.object({ kind: z.literal('review'), round: z.enum(['1', '2']) }).strict(),
+	z.object({ kind: z.literal('relaunch'), stop: stop_schema }).strict(),
+])
+const request_schema = z
+	.object({ session: z.uuid(), nonce: z.uuid(), action: action_schema })
+	.strict()
+const response_schema = z
+	.object({
+		session: z.uuid(),
+		nonce: z.string(),
+		is_success: z.boolean(),
+		heartbeat_at: z.number().int().positive(),
+	})
+	.strict()
+type BrokerRequest = z.infer<typeof request_schema>
+type BrokerResponse = z.infer<typeof response_schema>
+type ReviewRound = Extract<z.infer<typeof action_schema>, { kind: 'review' }>['round']
+type BrokerAction = { kind: 'review'; round: ReviewRound } | { kind: 'relaunch'; stop: ShipStop }
+
+function review_paths(directory: string): { brief: string; findings: string; log: string } {
+	return {
+		brief: stamp_file.stamp_path(BRIEF_PREFIX, directory, '.md'),
+		findings: stamp_file.stamp_path(FINDINGS_PREFIX, directory, '.txt'),
+		log: stamp_file.stamp_path(LOG_PREFIX, directory, '.log'),
+	}
+}
+
+function request_path(directory: string, issue: string): string {
+	return path.join(directory, ...CACHE_DIRECTORY, `${REQUEST_PREFIX}${issue}.json`)
+}
+
+function response_path(directory: string, issue: string): string | undefined {
+	const common = git_common_directory.resolve(directory)
+	if (common === undefined || path.basename(common) !== '.git') return undefined
+
+	return path.join(path.dirname(common), ...CACHE_DIRECTORY, `${RESPONSE_PREFIX}${issue}.json`)
+}
+
+function read_record<T>(target: string, schema: z.ZodType<T>): T | undefined {
+	try {
+		return json_value.parse_with(readFileSync(target, 'utf8'), schema)
+	} catch {
+		return undefined
+	}
+}
+
+// The reviewer runs on the lane's own provider, so a Claude lane's server never starts a Codex reviewer
+// and an OpenAI lane's never starts a Claude one.
+function provider_of(lane: LaneInfo): AgentProvider {
+	return lane.profile?.provider ?? ANTHROPIC_PROVIDER
+}
+
+// **The reviewer carries the lane's mark explicitly**: the supervisor serving a Claude lane is no lane child,
+// so a reviewer inheriting its environment would descend from the carry record's owner unmarked and be
+// read as the `backlogrun` parent by every hook.
+
+async function launch_review(lane: LaneInfo, round: ReviewRound): Promise<boolean> {
+	const target = review_paths(lane.directory)
+	const prompt_of =
+		round === '1' ? run_ship_review.reviewer_prompt : run_ship_review.verification_prompt
+	const built = agent_argv.resolve_in(
+		prompt_of(target.brief, target.findings),
+		agent_role_profile.REVIEWER,
+		lane.directory,
+	)
+	if (built.kind === 'rejected' || built.profile.provider !== provider_of(lane)) return false
+
+	const result = await detached_launch.launch_attached(
+		{
+			argv: built.argv,
+			cwd: lane.directory,
+			log_path: target.log,
+			profile: built.profile,
+			env: lane_child_marker.env_for(lane.issue),
+		},
+		console.error,
+	)
+
+	return result.kind === 'completed' && result.exit_code === 0
+}
+
+// An OpenAI lane's successor is its supervisor's to start, never a relaunch's.
+function relaunch(lane: LaneInfo, stop: ShipStop): boolean {
+	if (lane_relaunch.is_openai_lane(lane)) return false
+
+	const invocation = lane_child_invocation.ship_stop_invocation(lane.issue, stop)
+	const phase = agent_role_profile.IMPLEMENTATION_PHASE
+	const result = lane_relaunch.relaunch(lane, invocation, phase, console.error)
+
+	return result.kind === 'launched'
+}
+
+async function perform(lane: LaneInfo, action: BrokerAction): Promise<boolean> {
+	if (action.kind === 'relaunch') return relaunch(lane, action.stop)
+
+	return await launch_review(lane, action.round)
+}
+
+function write_response(target: string, body: BrokerResponse): void {
+	const temporary = `${target}.${String(process.pid)}.tmp`
+
+	writeFileSync(temporary, JSON.stringify(body))
+	renameSync(temporary, target)
+}
+
+async function launched(lane: LaneInfo, action: BrokerAction): Promise<boolean> {
+	try {
+		return await perform(lane, action)
+	} catch (error) {
+		error_text.trace_swallowed('lane_session_broker.launched', error)
+
+		return false
+	}
+}
+
+async function process_request(
+	lane: LaneInfo,
+	response: string,
+	received: BrokerRequest,
+	state: { current: BrokerResponse },
+): Promise<void> {
+	const is_success = await launched(lane, received.action)
+
+	state.current = {
+		session: received.session,
+		nonce: received.nonce,
+		is_success,
+		heartbeat_at: Date.now(),
+	}
+	write_response(response, state.current)
+}
+
+async function serve(
+	lane: LaneInfo,
+	session: string,
+	response: string,
+	state: { is_running: boolean; current: BrokerResponse },
+): Promise<void> {
+	let last_nonce = ''
+	const target = request_path(lane.directory, lane.issue)
+
+	while (state.is_running) {
+		const received = read_record(target, request_schema)
+
+		if (received?.session === session && received.nonce !== last_nonce) {
+			last_nonce = received.nonce
+			// eslint-disable-next-line no-await-in-loop -- requests are served one at a time, in arrival order
+			await process_request(lane, response, received, state)
+		}
+
+		// eslint-disable-next-line no-await-in-loop -- polling: each read waits on the state the previous one saw
+		await sleep(POLL_MS)
+	}
+}
+
+// The server record is written before the response, so a ship that finds the response finds it too.
+function prepare(lane: LaneInfo): { response: string; current: BrokerResponse } {
+	const response = response_path(lane.directory, lane.issue)
+	if (response === undefined) throw new Error('review broker has no trusted response location')
+
+	const session = randomUUID()
+
+	mkdirSync(path.dirname(response), { recursive: true })
+	mkdirSync(path.dirname(request_path(lane.directory, lane.issue)), { recursive: true })
+	lane_broker_server.write(response, session)
+	const current: BrokerResponse = {
+		session,
+		nonce: '',
+		is_success: false,
+		heartbeat_at: Date.now(),
+	}
+
+	write_response(response, current)
+
+	return { response, current }
+}
+
+function start_heartbeat(state: { current: BrokerResponse }, response: string): NodeJS.Timeout {
+	const heartbeat = setInterval(() => {
+		state.current = { ...state.current, heartbeat_at: Date.now() }
+		write_response(response, state.current)
+	}, LEGACY_HEARTBEAT_MS)
+
+	heartbeat.unref()
+
+	return heartbeat
+}
+
+// **The heartbeat outlives the request being served.** A ship waiting on a reviewer reads a stale
+// heartbeat as a dead server, so stopping it first would fail a review the server is still running.
+async function stop_server(
+	state: { is_running: boolean },
+	heartbeat: NodeJS.Timeout,
+	serving: Promise<void>,
+	response: string,
+): Promise<void> {
+	state.is_running = false
+	await serving
+	clearInterval(heartbeat)
+	unlinkSync(response)
+	lane_broker_server.remove(response)
+}
+
+async function with_server<T>(lane: LaneInfo, run: () => Promise<T>): Promise<T> {
+	const { response, current } = prepare(lane)
+	const state = { is_running: true, current }
+	const heartbeat = start_heartbeat(state, response)
+	const serving = serve(lane, current.session, response, state)
+
+	try {
+		return await run()
+	} finally {
+		await stop_server(state, heartbeat, serving, response)
+	}
+}
+
+function is_fresh_heartbeat(heartbeat_at: number): boolean {
+	const now = Date.now()
+
+	return heartbeat_at <= now && heartbeat_at >= now - LEGACY_HEARTBEAT_LIMIT_MS
+}
+
+// Live by the server record (`lane-broker-server.ts`); a response with none was written by a
+// supervisor started before it, read by the heartbeat that supervisor still writes.
+function is_live_response(
+	target: string,
+	response: BrokerResponse | undefined,
+	is_live: (server: BrokerServer) => boolean,
+): response is BrokerResponse {
+	if (response === undefined) return false
+
+	const server = lane_broker_server.read(target, response.session)
+
+	return server === undefined ? is_fresh_heartbeat(response.heartbeat_at) : is_live(server)
+}
+
+function session_response(target: string, session: string): BrokerResponse | undefined {
+	const response = read_record(target, response_schema)
+
+	return response?.session === session ? response : undefined
+}
+
+function response_verdict(
+	target: string,
+	session: string,
+	nonce: string,
+): 'wait' | 'success' | 'failure' {
+	const response = session_response(target, session)
+
+	if (!is_live_response(target, response, lane_broker_server.is_running)) return 'failure'
+	if (response.nonce !== nonce) return 'wait'
+
+	return response.is_success ? 'success' : 'failure'
+}
+
+async function wait_response(target: string, session: string, nonce: string): Promise<boolean> {
+	const deadline = Date.now() + REQUEST_LIMIT_MS
+
+	while (Date.now() < deadline) {
+		const verdict = response_verdict(target, session, nonce)
+		if (verdict !== 'wait') return verdict === 'success'
+
+		// eslint-disable-next-line no-await-in-loop -- polling: each read waits on the state the previous one saw
+		await sleep(POLL_MS)
+	}
+
+	return false
+}
+
+function live_response(target: string): BrokerResponse | undefined {
+	const response = read_record(target, response_schema)
+
+	return is_live_response(target, response, lane_broker_server.is_serving_process)
+		? response
+		: undefined
+}
+
+// Whether a server answers for this lane now — what a Claude lane's ship asks before it hands a session
+// over rather than starting the session itself.
+function is_serving(directory: string, issue: string): boolean {
+	const target = response_path(directory, issue)
+
+	return target !== undefined && live_response(target) !== undefined
+}
+
+async function request(directory: string, issue: string, action: BrokerAction): Promise<boolean> {
+	const target = response_path(directory, issue)
+	const response = target === undefined ? undefined : live_response(target)
+
+	if (target === undefined || response === undefined) return false
+
+	const nonce = randomUUID()
+	const body = { session: response.session, nonce, action }
+
+	stamp_file.replace_stamp(request_path(directory, issue), body)
+
+	return await wait_response(target, response.session, nonce)
+}
+
+const lane_session_broker = {
+	is_serving,
+	request,
+	request_path,
+	response_path,
+	review_paths,
+	with_server,
+}
+
+export type { BrokerAction, ReviewRound }
+export { lane_session_broker }

package/scripts/lane/lane-spent.ts

@@ -0,0 +1,195 @@
+import { backlog_drive_restore } from '#scripts/backlog/backlog-drive-restore'
+import { git_gh_issue_read } from '#scripts/gh/git-gh-issue-read'
+import { git_command } from '#scripts/git/git-command'
+import { issue_state } from '#scripts/issue/issue-state'
+import { run_event_stream_emit } from '#scripts/run/event/run-event-stream-emit'
+import { run_hold } from '#scripts/run/hold/run-hold'
+import {
+	PROCESS_ALIVE,
+	PROCESS_NONE,
+	PROCESS_UNKNOWN,
+	type ProcessTrace,
+} from '#scripts/run/run-liveness'
+import { lane_await } from './lane-await'
+import { lane_registry, type LaneInfo } from './lane-registry'
+
+// Whether a lane is in use, and whether it is spent — the one reading `lane:prune` discards a lane
+// on and `josh backlogrun` keeps a watching run on.
+//
+// **In use means a child is working in it, or nobody could tell.** A lane a past run left behind has
+// no process, and counting it as open kept a watching run from being replaced for as long as the lane
+// sat there. An unanswered probe is in use: both callers act on "nobody is here", and the lane a
+// wrong answer costs is one with a child mid-change.
+//
+// **Spent is all five: no child, the issue closed, not in flight, a clean tree, no commit off every
+// remote.** `lane:close` forces the removal, so each condition names something a close cannot give
+// back — and a condition that could not be read keeps the lane, with the reason said.
+//
+// **A lane the running `backlogrun` still has in flight is its `run:merge`'s.** A child that merged
+// while no driver watched leaves a lane that reads spent on every other count; closed here, the run
+// would have no `merge` event, no ledger line and no merged count for it (`run-tidy-lanes.ts`).
+
+const STATE_FIELD = 'state'
+const CLOSED_STATE = 'CLOSED'
+
+interface SpentProbes {
+	process_trace: (issue: string) => ProcessTrace
+	issue_state: (issue: string) => Promise<string | undefined>
+	// The issues the running `backlogrun` launched and has not settled.
+	in_flight: () => Promise<ReadonlySet<string>>
+	is_tree_dirty: (directory: string) => Promise<boolean>
+	unpushed_commits: (branch: string) => Promise<number | undefined>
+}
+
+interface KeptLane {
+	issue: string
+	reason: string
+}
+
+// The lanes that are in use, apart: a probe that did not answer is not a child seen working.
+interface LaneUse {
+	working: Array<string>
+	unread: Array<string>
+}
+
+// One probe answer per lane: a second sample can disagree with the first, and a lane bucketed on
+// two answers can land in neither list.
+type TracedLane = readonly [issue: string, trace: ProcessTrace]
+
+interface SpentSort {
+	spent: Array<LaneInfo>
+	kept: Array<KeptLane>
+}
+
+type Check = (lane: LaneInfo, probes: SpentProbes) => Promise<string | undefined>
+
+async function read_issue_state(issue: string): Promise<string | undefined> {
+	const json = await git_gh_issue_read.issue_view_json(issue, STATE_FIELD)
+
+	return json === undefined ? undefined : issue_state.parse_issue_state(json)?.state
+}
+
+async function read_unpushed_commits(branch: string): Promise<number | undefined> {
+	try {
+		return await git_command.commit_count_unpushed(branch)
+	} catch {
+		return undefined
+	}
+}
+
+async function read_in_flight(): Promise<ReadonlySet<string>> {
+	return backlog_drive_restore.active_issues(await run_event_stream_emit.current_events())
+}
+
+const LIVE_PROBES: SpentProbes = {
+	process_trace: lane_await.process_trace_default,
+	issue_state: read_issue_state,
+	in_flight: read_in_flight,
+	is_tree_dirty: run_hold.is_tree_dirty,
+	unpushed_commits: read_unpushed_commits,
+}
+
+// A stranded lane has no work tree for a child to be in, so it is never in use.
+function trace_of(lane: LaneInfo, probes: SpentProbes): ProcessTrace {
+	return lane.is_stranded ? PROCESS_NONE : probes.process_trace(lane.issue)
+}
+
+async function child_reason(lane: LaneInfo, probes: SpentProbes): Promise<string | undefined> {
+	const trace = probes.process_trace(lane.issue)
+
+	if (trace === PROCESS_NONE) return undefined
+
+	return trace === PROCESS_ALIVE
+		? 'a child is still working in it'
+		: 'whether a child is working in it could not be read'
+}
+
+async function issue_reason(lane: LaneInfo, probes: SpentProbes): Promise<string | undefined> {
+	const state = await probes.issue_state(lane.issue)
+
+	if (state === CLOSED_STATE) return undefined
+
+	return state === undefined ? 'its issue could not be read' : 'its issue is still open'
+}
+
+async function run_reason(lane: LaneInfo, probes: SpentProbes): Promise<string | undefined> {
+	const in_flight = await probes.in_flight()
+
+	return in_flight.has(lane.issue)
+		? 'the running backlogrun has not collected it yet (`pnpm josh run:merge` does)'
+		: undefined
+}
+
+// `is_tree_dirty` answers dirty for a status it could not read, which is the side that keeps the lane.
+async function tree_reason(lane: LaneInfo, probes: SpentProbes): Promise<string | undefined> {
+	return (await probes.is_tree_dirty(lane.directory)) ? 'it has uncommitted changes' : undefined
+}
+
+async function commit_reason(lane: LaneInfo, probes: SpentProbes): Promise<string | undefined> {
+	const count = await probes.unpushed_commits(lane.branch)
+
+	if (count === 0) return undefined
+
+	return count === undefined
+		? 'its commits could not be counted'
+		: `it has ${String(count)} commit(s) that are on no remote`
+}
+
+// In the order a person acts on them: a running child outranks everything, and an open issue is why
+// the lane exists at all.
+const CHECKS: ReadonlyArray<Check> = [
+	child_reason,
+	issue_reason,
+	run_reason,
+	tree_reason,
+	commit_reason,
+]
+
+// Why the lane is kept, or `undefined` for a spent one.
+async function kept_reason(
+	lane: LaneInfo,
+	probes: SpentProbes = LIVE_PROBES,
+): Promise<string | undefined> {
+	const reasons = await Promise.all(CHECKS.map(async (check) => await check(lane, probes)))
+
+	return reasons.find((reason) => reason !== undefined)
+}
+
+// Every live lane as spent or kept. A stranded lane is neither: `lane:prune` closes it already.
+async function sort_lanes(
+	lanes: ReadonlyArray<LaneInfo>,
+	probes: SpentProbes = LIVE_PROBES,
+): Promise<SpentSort> {
+	const live = lanes.filter((lane) => !lane.is_stranded)
+	const reasons = await Promise.all(live.map(async (lane) => await kept_reason(lane, probes)))
+	const sorted: SpentSort = { spent: [], kept: [] }
+
+	for (const [index, lane] of live.entries()) {
+		const reason = reasons[index]
+
+		if (reason === undefined) sorted.spent.push(lane)
+		else sorted.kept.push({ issue: lane.issue, reason })
+	}
+
+	return sorted
+}
+
+function issues_traced(traced: ReadonlyArray<TracedLane>, trace: ProcessTrace): Array<string> {
+	return traced.filter(([, answer]) => answer === trace).map(([issue]) => issue)
+}
+
+/** The lanes in use — what a run being replaced must not be stopped over. */
+async function lanes_in_use(probes: SpentProbes = LIVE_PROBES): Promise<LaneUse> {
+	const lanes = await lane_registry.list_lanes()
+	const traced = lanes.map((lane): TracedLane => [lane.issue, trace_of(lane, probes)])
+
+	return {
+		working: issues_traced(traced, PROCESS_ALIVE),
+		unread: issues_traced(traced, PROCESS_UNKNOWN),
+	}
+}
+
+const lane_spent = { kept_reason, lanes_in_use, sort_lanes }
+
+export type { KeptLane, LaneUse, SpentProbes, SpentSort }
+export { lane_spent }

package/scripts/lane/lane-stage-stats.ts

@@ -112,11 +112,15 @@ function stage_row(stage: string, measured: Measured): string {
 	])
 }
 
+function measured_of(entries: ReadonlyArray<LedgerEntry>): Measured {
+	const stages = entries.filter(is_stage)
+
+	return { implement: implement_durations(entries.filter(is_dispatch), stages), stages }
+}
+
 // The period's table: `entries` is the whole ledger, filtered here to `window`.
 function table(entries: ReadonlyArray<LedgerEntry>, window: StatsWindow): string {
-	const period = lane_stats.in_window(entries, window)
-	const stages = period.filter(is_stage)
-	const measured = { implement: implement_durations(period.filter(is_dispatch), stages), stages }
+	const measured = measured_of(lane_stats.in_window(entries, window))
 
 	return [
 		lane_stats.to_row(COLUMNS),
@@ -125,10 +129,30 @@ function table(entries: ReadonlyArray<LedgerEntry>, window: StatsWindow): string
 	].join('\n')
 }
 
+// Each stage's median over every entry given, a stage nothing recorded left out rather than read as
+// `0` — how long a stage usually takes, which `run:board` measures a running child's phase against.
+function medians(entries: ReadonlyArray<LedgerEntry>): ReadonlyMap<string, number> {
+	const measured = measured_of(entries)
+
+	return new Map(
+		STAGES.flatMap((stage) => {
+			const median = lane_stats.median(durations_of(stage, measured))
+
+			return median === undefined ? [] : [[stage, median] as const]
+		}),
+	)
+}
+
 const lane_stage_stats = {
 	IMPLEMENT,
 	NOT_MEASURED,
 	STAGES,
+	cell,
+	is_dispatch,
+	is_number,
+	is_stage,
+	medians,
+	started_ms,
 	table,
 }