@joshuafolkken/kit 1.1514.0 → 1.2114.0

package/scripts/run/ship/run-ship-place.ts

@@ -0,0 +1,103 @@
+import { setTimeout as sleep } from 'node:timers/promises'
+import { backlog_ready } from '#scripts/backlog/backlog-ready'
+import { git_common_directory } from '#scripts/git/git-common-directory'
+import { lane_capacity } from '#scripts/lane/lane-capacity'
+import { lane_handoff } from '#scripts/lane/lane-handoff'
+import { lane_registry, type LaneInfo } from '#scripts/lane/lane-registry'
+import { run_ship_stage, type Stage } from './run-ship-stage'
+
+// **A followup stop past the lane limit waits for a place, then relaunches**: its lane held no place
+// while it waited (`backlog-occupied.ts`), so the next launch may have taken it, and a child started at
+// once could run lanes past the limit (joshuafolkken/kit#4466). The stopped lane counts again while this
+// supervisor waits, so the driver launches nothing into the place it waits for. An unreadable limit or
+// lane count relaunches at once, and so does a place that does not free within the wait.
+//
+// **One relaunch may run a lane past the limit**: a followup relaunch is a few minutes' work on a lane
+// about to merge, so it does not queue behind a fresh lane's whole run (joshuafolkken/kit#5172). The
+// waiter counts in the lane use, so the lanes it sees at the limit plus one include itself, and a second
+// waiter sees the first one's lane over that and waits.
+//
+// **Waiters queue in issue order rather than block each other**: every waiter counts in the lane use,
+// so three of them past a limit of two would each see the lanes full and wait out the hour together. A
+// waiter leaves out the waiters behind it, so the first ones that fit relaunch and the rest wait on them.
+// The mark is the ship stage record's (`run_ship_stage.mark_place_waiting`), read only while the ship
+// runs, so a dead supervisor's mark queues nobody.
+
+// An hour of lane counts, half a minute apart: about two lanes' median run (`lane:stats`).
+const PLACE_POLL_MS = 30_000
+const PLACE_POLLS = 120
+// The one relaunch allowed past the lane limit.
+const RELAUNCH_OVERSHOOT = 1
+
+function is_behind(lane: LaneInfo, issue: string): boolean {
+	return lane.issue.localeCompare(issue, undefined, { numeric: true }) > 0
+}
+
+async function waiters_behind(issue: string): Promise<number> {
+	const lanes = await lane_registry.list_lanes()
+
+	return lanes.filter(
+		(lane) => is_behind(lane, issue) && lane_handoff.is_place_waiting(lane.issue, lane.directory),
+	).length
+}
+
+async function is_over_limit(issue: string): Promise<boolean> {
+	const limit = await lane_capacity.lane_limit()
+
+	if (limit.kind !== 'limit') return false
+
+	const use = await backlog_ready.lane_use()
+	const behind = await waiters_behind(issue)
+
+	return use.occupied - behind > lane_capacity.seated_limit(limit.limit) + RELAUNCH_OVERSHOOT
+}
+
+async function is_place_taken(issue: string): Promise<boolean> {
+	try {
+		return await is_over_limit(issue)
+	} catch {
+		return false
+	}
+}
+
+// Best-effort: a mark that could not be written leaves this waiter counted by the others, as before.
+function mark(lane: LaneInfo, is_waiting: boolean): void {
+	const repository = git_common_directory.repository(lane.directory)
+
+	if (repository === undefined) return
+
+	try {
+		run_ship_stage.mark_place_waiting(
+			run_ship_stage.record_path(repository, lane.issue),
+			is_waiting,
+		)
+	} catch {
+		// The queue only orders waiters; an unmarked one still waits on the lane count.
+	}
+}
+
+async function poll(issue: string): Promise<void> {
+	for (let polled = 0; polled < PLACE_POLLS; polled += 1) {
+		// eslint-disable-next-line no-await-in-loop -- polling: each check waits on a lane the previous one saw busy
+		if (!(await is_place_taken(issue))) return
+
+		// eslint-disable-next-line no-await-in-loop -- polling: the wait between two lane counts
+		await sleep(PLACE_POLL_MS)
+	}
+}
+
+async function await_place(lane: LaneInfo, stage: Stage): Promise<void> {
+	if (stage !== run_ship_stage.STAGE.FOLLOWUP) return
+
+	mark(lane, true)
+
+	try {
+		await poll(lane.issue)
+	} finally {
+		mark(lane, false)
+	}
+}
+
+const run_ship_place = { await_place }
+
+export { run_ship_place }

package/scripts/run/ship/run-ship-planned.ts

@@ -0,0 +1,92 @@
+import path from 'node:path'
+import { changed_paths } from '#scripts/git/changed-paths'
+import { git_branch } from '#scripts/git/git-branch'
+import { git_issue } from '#scripts/git/git-issue'
+import { lane_plan, type LanePlan } from '#scripts/lane/lane-plan'
+import { test_declared_logic } from '#scripts/test/test-declared-logic'
+
+// `josh ship`'s `planned` precondition (joshuafolkken/kit#5152): the branch's code files that the
+// lane's `lane:plan` record never listed. Nothing else compares a lane's edits with its plan, so an
+// out-of-scope fix landed with no one noticing (joshuafolkken/kit#5122). Tests, documents and data
+// files (a `cspell.config.yaml` term, a lock file) never count, and a branch with no plan record, or a
+// yielded one — a `fullrun` a person drove — passes as before.
+
+// The precondition's name in the preflight's answer and in a `stop` record's `preconditions`.
+const PRECONDITION = 'unplanned'
+const SEPARATOR = '/'
+const NON_CODE_PREFIXES = ['docs/', '.josh/']
+const CODE_EXTENSIONS = new Set(['.cjs', '.js', '.mjs', '.sh', '.svelte', '.ts'])
+
+// A claim ending in `/` is a directory and holds every path under it — `backlog_overlap`'s reading.
+function is_planned(claims: ReadonlyArray<string>, file: string): boolean {
+	return claims.some((claim) =>
+		claim.endsWith(SEPARATOR) ? file.startsWith(claim) : claim === file,
+	)
+}
+
+function is_code(file: string): boolean {
+	return (
+		CODE_EXTENSIONS.has(path.extname(file)) &&
+		NON_CODE_PREFIXES.every((prefix) => !file.startsWith(prefix))
+	)
+}
+
+// The code files of `changed` that `claims` does not hold.
+function unplanned_of(
+	claims: ReadonlyArray<string>,
+	changed: ReadonlyArray<string>,
+): Array<string> {
+	return test_declared_logic
+		.runtime_files(changed)
+		.filter((file) => is_code(file) && !is_planned(claims, file))
+}
+
+// The two ways out, each named with its command: the re-run replaces the record, so it lists the
+// planned paths as well as the new ones.
+function problem_text(
+	issue: string,
+	claims: ReadonlyArray<string>,
+	files: ReadonlyArray<string>,
+): string {
+	const replan = ['pnpm josh lane:plan', issue, ...claims, ...files].join(' ')
+
+	return `unplanned: ${files.join(', ')} — the lane's lane:plan record does not list these code files. In scope: run \`${replan}\` and comment the reason on the Issue in one line, then re-detach; if it answers \`yield\`, another lane holds the file — follow what it prints instead. Out of scope: revert the change and file it with \`pnpm josh issue:file\`.`
+}
+
+// A yielded plan outlives its lane on purpose (`lane-plan.ts`) and cleared no edit, so it reads as none.
+function held_claims(plan: LanePlan | undefined): ReadonlyArray<string> | undefined {
+	return plan?.yielded === true ? undefined : plan?.paths
+}
+
+// The branch's unplanned code files and the text naming the two ways out, `undefined` where none is —
+// read by the ship's preflight and, ahead of it, by `josh check:related` (joshuafolkken/kit#5203).
+interface Unplanned {
+	issue: number
+	files: ReadonlyArray<string>
+	text: string
+}
+
+async function unplanned(): Promise<Unplanned | undefined> {
+	const issue = git_issue.branch_number(await git_branch.current())
+
+	if (issue === undefined) return undefined
+	const claims = held_claims(lane_plan.read(Number(issue), process.cwd()))
+
+	if (claims === undefined) return undefined
+	const files = unplanned_of(claims, await changed_paths.read_changed_paths(false))
+
+	if (files.length === 0) return undefined
+
+	return { issue: Number(issue), files, text: problem_text(issue, claims, files) }
+}
+
+async function problems(): Promise<Array<string>> {
+	const found = await unplanned()
+
+	return found === undefined ? [] : [found.text]
+}
+
+const run_ship_planned = { PRECONDITION, problems, unplanned, unplanned_of }
+
+export { run_ship_planned }
+export type { Unplanned }

package/scripts/run/ship/run-ship-pre-detach-pinned.ts

@@ -0,0 +1,197 @@
+import { randomUUID } from 'node:crypto'
+import { check_reuse } from '#scripts/gate/check-reuse'
+import { PROJECT_ROOT } from '#scripts/init/init-paths'
+import type { JoshResult } from '#scripts/josh/josh-run'
+import { buffered_process } from '#scripts/lib/buffered-process'
+import { error_text } from '#scripts/lib/error-message'
+import { review_round_reuse } from '#scripts/review/review-round-reuse'
+import { run_review_pinned } from '#scripts/run/review/run-review-pinned'
+import { run_review_pinned_accepted } from '#scripts/run/review/run-review-pinned-accepted'
+import { run_review_pinned_join } from '#scripts/run/review/run-review-pinned-join'
+import { run_review_pinned_steps } from '#scripts/run/review/run-review-pinned-steps'
+import { run_review_pinned_tree } from '#scripts/run/review/run-review-pinned-tree'
+import { test_pinned } from '#scripts/test/test-pinned'
+import { test_pinned_logic } from '#scripts/test/test-pinned-logic'
+import { test_pinned_record, type PinnedSummary } from '#scripts/test/test-pinned-record'
+import { run_ship_pre_detach, type PreDetachResult } from './run-ship-pre-detach'
+import type { ShipArguments } from './run-ship-steps'
+
+// `josh test:pinned` among the pre-detach checks of a `josh ship --review` (joshuafolkken/kit#4785). The
+// child's `run:review` checked the tree it saw; a round-1 fix made after it reached no `test:pinned`
+// until the supervisor's review stage stopped on it, after the detach. Here it stops in the
+// foreground, with the text and the `--unpinned-ok` judgment `run:review --join` uses
+// (`run-review-pinned.ts`).
+//
+// **On the tree `run:review` checked it runs no `test:pinned`**: it reads that background check's
+// answer (`run-review-pinned-join.ts`). On another tree it waits that check out, so its verdicts
+// reach the cache (joshuafolkken/kit#4895), then runs its own as a `josh` command, so it reserves its
+// own cores (`DIRECT_UNIT_CLAIM`) beside the other checks, and it probes only the hunks whose stored
+// verdict the per-checkout cache no longer holds (`test-pinned-cache-logic.ts`). Its answer comes
+// back through the record `--record` leaves (`test-pinned-record.ts`). Under
+// `--unpinned-ok "<reason>"` — given, or carried from this branch's record (`with_recorded_reason`) —
+// it runs only when the supervisor reuses round 1 (`pinned_check`).
+// Without `--review` nothing after the detach asks `test:pinned`, and `--unpinned-ok` is refused there
+// (`run-ship-args.ts`), so the check is not asked.
+
+const SUCCESS_EXIT_CODE = 0
+const FAILURE_EXIT_CODE = 1
+const SKIPPED_NOTE =
+	'test:pinned not run before the detach; round 1 lists the unpinned places under --unpinned-ok: '
+const CARRIED_NOTE = 'josh ship: --unpinned-ok carried from an earlier run on this branch: '
+
+type CheckOptions = Pick<ShipArguments, 'number' | 'unpinned_ok'>
+type PinnedOptions = CheckOptions & Pick<ShipArguments, 'is_review'>
+
+// A check that left no answer could not run, and fails open as the one `run:review` launches does.
+// Its arguments follow the reason (`test_pinned_record.launch_argv`).
+async function own_summary(reason: string | undefined): Promise<PinnedSummary> {
+	const token = randomUUID()
+	const argv = ['josh', ...test_pinned_record.launch_argv(token, reason, 'pre-detach')]
+	const result = await buffered_process.run_buffered_process(argv)
+	const answer = test_pinned_record.read(token)
+
+	test_pinned_record.clear(token)
+
+	return answer ?? test_pinned_record.not_run_summary(result.output)
+}
+
+function remaining_note(remaining: number | undefined): string {
+	const hunks =
+		remaining === undefined ? 'the remaining hunks' : `${String(remaining)} remaining hunks`
+
+	return `test:pinned: probing ${hunks} in the background — the next ship's pre-detach reads their verdicts\n`
+}
+
+// **A stop at the first unpinned hunk leaves the rest probing in the background**
+// (joshuafolkken/kit#5327): the ship still stops on it, and the next pre-detach joins that probe on
+// this tree, or waits it out on a fixed one so its verdicts reach the cache and only the hunks the fix
+// moved are probed again. A launch that failed or threw adds no note, and never fails the check.
+async function is_rest_launched(): Promise<boolean> {
+	try {
+		const tree = await run_review_pinned_tree.key()
+
+		run_review_pinned_steps.clear()
+
+		return run_review_pinned_steps.launch_remaining(PROJECT_ROOT, tree).kind === 'launched'
+	} catch (error) {
+		error_text.trace_swallowed('run_ship_pre_detach_pinned.is_rest_launched', error)
+
+		return false
+	}
+}
+
+async function continued(answer: PinnedSummary): Promise<PinnedSummary> {
+	if (answer.is_cut !== true || !(await is_rest_launched())) return answer
+
+	return { ...answer, text: `${answer.text}${remaining_note(answer.remaining)}` }
+}
+
+// **The tree `run:review` checked is not checked twice** (joshuafolkken/kit#4888): its background
+// check's answer is read, waited for while it still runs; only another tree, or a check that left no
+// answer, runs one here. **A reason never judges a check that stopped at its first unpinned hunk**
+// (joshuafolkken/kit#5153): the hunks it left are probed here before the reason accepts the list.
+async function summary(reason: string | undefined): Promise<PinnedSummary> {
+	const answer = await run_review_pinned_join.answer_on_tree()
+	const is_whole = answer?.is_cut !== true || reason === undefined
+
+	return await continued(answer !== undefined && is_whole ? answer : await own_summary(reason))
+}
+
+// The verdicts an earlier probe left for this very tree, read from the cache without a probe
+// (joshuafolkken/kit#5078); `undefined` when a hunk moved since, or the tree could not be read.
+async function held_summary(): Promise<PinnedSummary | undefined> {
+	try {
+		const report = await test_pinned.held_report(test_pinned_logic.DEFAULT_MAX_HUNKS)
+
+		return report === undefined ? undefined : test_pinned_record.summary_of(report)
+	} catch (error) {
+		error_text.trace_swallowed('run_ship_pre_detach_pinned.held_summary', error)
+
+		return undefined
+	}
+}
+
+// **A re-ship with a new reason probes nothing the cache already answers** (joshuafolkken/kit#5078):
+// the earlier probe's verdicts are judged under the reason, with no wait on a check elsewhere. A moved
+// hunk falls to `summary`, whose run probes only the hunks the cache no longer holds.
+async function summary_for(reason: string | undefined): Promise<PinnedSummary> {
+	const held = reason === undefined ? undefined : await held_summary()
+
+	return held ?? (await summary(reason))
+}
+
+// Whether the supervisor runs a fresh round 1, whose `run:review --join` lists and records the unpinned
+// places under the reason. A reused round 1 — its delta left to round 2, or after the ship's own
+// round 1 stopped, the lane's fix included — runs no `test:pinned` (`review-round-reuse.ts`). A question git could not answer probes.
+async function is_listed_after_detach(number: string): Promise<boolean> {
+	try {
+		return !(await review_round_reuse.is_reusable_here(Number(number)))
+	} catch (error) {
+		error_text.trace_swallowed('run_ship_pre_detach_pinned.is_listed_after_detach', error)
+
+		return false
+	}
+}
+
+// **A reason accepts any answer, so no probe is waited on** (joshuafolkken/kit#4833) when a fresh
+// round 1 lists the places after the detach.
+async function pinned_check(options: CheckOptions): Promise<JoshResult> {
+	const reason = run_review_pinned.ok_reason_of(options.unpinned_ok)
+
+	if (reason !== undefined && (await is_listed_after_detach(options.number))) {
+		return { code: SUCCESS_EXIT_CODE, out: `${SKIPPED_NOTE}${reason}\n` }
+	}
+
+	const verdict = await run_review_pinned_accepted.judge(await summary_for(reason), reason)
+
+	return {
+		code: verdict.kind === 'stop' ? FAILURE_EXIT_CODE : SUCCESS_EXIT_CODE,
+		out: verdict.text,
+	}
+}
+
+// **A ship with no `--review` reads no background probe, so it stops the one left running**
+// (joshuafolkken/kit#5327) rather than leave it holding cores unread. A clear that throws never fails
+// the ship.
+function cleared(): void {
+	try {
+		run_review_pinned_steps.clear()
+	} catch (error) {
+		error_text.trace_swallowed('run_ship_pre_detach_pinned.cleared', error)
+	}
+}
+
+// The pre-detach checks this ship runs, `test:pinned` among them under `--review`.
+function checks_for(options: PinnedOptions): () => Promise<PreDetachResult> {
+	if (!options.is_review) {
+		return async () => {
+			cleared()
+
+			return await run_ship_pre_detach.checks()
+		}
+	}
+
+	return async () =>
+		await run_ship_pre_detach.checks(process.cwd(), check_reuse.DEFAULT_RECORDS, async () => {
+			return await pinned_check(options)
+		})
+}
+
+// **A reason an earlier run on this branch accepted is this ship's `--unpinned-ok`** when it gives
+// none (joshuafolkken/kit#4974): the pre-detach check skips its probe under it, and the supervisor's
+// round 1 and its relaunch carry it, so a place the record lacks is listed and recorded after the
+// detach rather than stopping the ship there.
+async function with_recorded_reason(args: ShipArguments): Promise<ShipArguments> {
+	if (!args.is_review || args.unpinned_ok.length > 0) return args
+
+	const reason = await run_review_pinned_accepted.recorded_reason()
+
+	if (reason === undefined) return args
+	console.info(`${CARRIED_NOTE}${reason}`)
+
+	return { ...args, unpinned_ok: [run_review_pinned.UNPINNED_OK_FLAG, reason] }
+}
+
+const run_ship_pre_detach_pinned = { checks_for, pinned_check, with_recorded_reason }
+
+export { run_ship_pre_detach_pinned }

package/scripts/run/ship/run-ship-pre-detach-timed.ts

@@ -0,0 +1,42 @@
+import { lane_ledger_write } from '#scripts/lane/lane-ledger-write'
+import { run_event_stream } from '#scripts/run/event/run-event-stream'
+import { run_event_stream_emit } from '#scripts/run/event/run-event-stream-emit'
+import { run_ship_pre_detach, type PreDetachResult } from './run-ship-pre-detach'
+import { run_ship_stage, type Phase } from './run-ship-stage'
+
+// The pre-detach checks, timed into the lane ledger and marked on the event stream as a ship stage
+// is, so their minutes are no longer folded into the lane's implementation time
+// (joshuafolkken/kit#4382). They are not a `Stage`: nothing resumes past them, and `run:board` draws no
+// phase from a word it does not know.
+
+const SUCCESS_EXIT_CODE = 0
+const { PHASE } = run_ship_stage
+
+async function emit_phase(issue: string, phase: Phase): Promise<void> {
+	const text = run_ship_stage.event_text(issue, lane_ledger_write.PRE_DETACH_STAGE, phase)
+
+	await run_event_stream_emit.emit(run_event_stream.EVENT_KIND.SHIP_STAGE, text)
+}
+
+async function checks(
+	issue: string,
+	run: () => Promise<PreDetachResult> = run_ship_pre_detach.checks,
+): Promise<PreDetachResult> {
+	await emit_phase(issue, PHASE.START)
+	const started_ms = performance.now()
+	const result = await run()
+	const elapsed_ms = performance.now() - started_ms
+
+	await lane_ledger_write.record_stage({
+		stage: lane_ledger_write.PRE_DETACH_STAGE,
+		elapsed_ms,
+		issue: Number(issue),
+	})
+	await emit_phase(issue, result.code === SUCCESS_EXIT_CODE ? PHASE.DONE : PHASE.FAILED)
+
+	return result
+}
+
+const run_ship_pre_detach_timed = { checks }
+
+export { run_ship_pre_detach_timed }