@joshuafolkken/kit 1.1784.0 → 1.2114.0

package/scripts/followup/followup-main-move.ts

@@ -0,0 +1,113 @@
+import { git_command } from '#scripts/git/git-command'
+import { main_merge_guard } from '#scripts/git/main-merge-guard'
+import type { ObservationEntry } from '#scripts/observations/observation-record'
+import { followup_failed_tests } from './followup-failed-tests'
+
+// **A test red outside the change that main has since changed is merged, not rerun**
+// (joshuafolkken/kit#5316): a date-bound test fixed on main went red on a lane still on the old base,
+// and the rerun on that base was red again before `merge_again` (`run-ship-sync.ts`) took main in. So
+// before a rerun, a failed test whose file or subject module (`followup_failed_tests.subjects_of`) main
+// moved since the branch point skips it: `followup` fails as before and ship's merge, gate and push go
+// first. The rerun right is unspent, and after the merge the same file is no longer incoming, so a
+// test still red is rerun once as before.
+
+const OBSERVATION_SLUG = 'followup-main-merge-first'
+const OBSERVATION_DEPTH = 'd1'
+const LATEST_COMMIT = 1
+const SHORT_SHA_LENGTH = 12
+
+interface MainMove {
+	files: ReadonlyArray<string>
+	commit: string
+}
+
+// A failed test file beside the path main moved for it: the file itself or its subject module.
+interface MovedTest {
+	file: string
+	moved: string
+}
+
+function moved_tests(
+	test_files: ReadonlyArray<string>,
+	incoming: ReadonlySet<string>,
+): Array<MovedTest> {
+	return test_files.flatMap((file) => {
+		const moved = followup_failed_tests.subjects_of(file).find((subject) => incoming.has(subject))
+
+		return moved === undefined ? [] : { file, moved }
+	})
+}
+
+// The main-side commit is the latest on main's own line that touched the first moved path.
+async function latest_commit(file: string, main_reference: string): Promise<string> {
+	const [commit] = await git_command.log_first_parent(LATEST_COMMIT, file, main_reference)
+
+	return commit ?? main_reference
+}
+
+// **The failed test files main moved since the branch point**, `undefined` when main did not move or
+// moved none of them. The fetch is what makes `origin/<default>` current, as `main_merge.merge` does.
+async function read_main_move(test_files: ReadonlyArray<string>): Promise<MainMove | undefined> {
+	if (test_files.length === 0) return undefined
+
+	const default_branch = await git_command.get_default_branch()
+
+	await git_command.fetch_branch(default_branch)
+
+	const incoming = new Set(await main_merge_guard.incoming_paths(default_branch))
+	const moved = moved_tests(test_files, incoming)
+	const [first] = moved
+
+	if (first === undefined) return undefined
+
+	return {
+		files: moved.map(({ file }) => file),
+		commit: await latest_commit(first.moved, `origin/${default_branch}`),
+	}
+}
+
+function observation_of(move: MainMove): ObservationEntry {
+	return {
+		slug: OBSERVATION_SLUG,
+		depth: OBSERVATION_DEPTH,
+		where: move.files.join(', '),
+		what: `followup put the main merge before the rerun; the failed test moved on main at ${move.commit.slice(0, SHORT_SHA_LENGTH)}`,
+	}
+}
+
+// A fetch or read that fails is reported and keeps the rerun: the merge is the shortcut, the rerun the
+// recovery that stood before it.
+async function read_or_rerun(test_files: ReadonlyArray<string>): Promise<MainMove | undefined> {
+	try {
+		return await read_main_move(test_files)
+	} catch (error) {
+		console.info(`⚠️ Could not compare the failed tests with main, rerunning: ${String(error)}`)
+
+		return undefined
+	}
+}
+
+// **Whether the rerun is skipped for a merge of main**, with the reason posted on the issue.
+async function should_merge_first(
+	test_files: ReadonlyArray<string>,
+	issue: number | undefined,
+): Promise<boolean> {
+	const move = await read_or_rerun(test_files)
+
+	if (move === undefined) return false
+
+	console.info(
+		`🔀 Not rerunning: main changed the failed tests since the branch point (${move.files.join(', ')}); merge main first.`,
+	)
+	await followup_failed_tests.record_entry(observation_of(move), issue)
+
+	return true
+}
+
+const followup_main_move = {
+	read_main_move,
+	should_merge_first,
+}
+
+export type { MainMove }
+export { followup_main_move, OBSERVATION_SLUG }

package/scripts/followup/git-followup-finish.ts

@@ -7,6 +7,7 @@ import { run_progress_clock } from '#scripts/run/progress/run-progress-clock'
 import { parse_completed_issue_number } from './followup-issue-number'
 import { git_followup_cleanup, type CleanupStep } from './git-followup-cleanup'
 import { git_followup_pending } from './git-followup-pending'
+import { git_followup_stages } from './git-followup-stages'
 
 // **The count of unreleased merges, not the project version**. The local `package.json` version
 // reads as "the version this run just shipped", but children do not bump, so that number names the
@@ -17,38 +18,69 @@ import { git_followup_pending } from './git-followup-pending'
 // Printed after the merge, and the count is read from a freshly fetched default branch — so this
 // run's own merge is already in it and nothing here says otherwise, unlike the Telegram sent one step
 // earlier from a tree the merge had not reached.
-async function print_pending_release(): Promise<void> {
+async function read_pending_release(): Promise<ReadonlyArray<string>> {
 	const line = await git_followup_pending.pending_release_line({ is_merge_pending: false })
 
-	if (line !== undefined) console.info(line)
+	return line === undefined ? [] : [line]
+}
+
+async function print_pending_release(): Promise<void> {
+	const lines = await read_pending_release()
+
+	for (const line of lines) console.info(line)
 }
 
 // #821: surface what to run next right where the completion is read. Printed before the project
 // version line, which stays the final line of the console output by contract.
-async function print_next_issues(completed_issue_number: string | undefined): Promise<void> {
-	const lines = await git_next_issues.fetch_next_issue_lines(
+//
+// Merged runs only, like the epic auto-close: on `--no-merge` the linked issue is still open
+// and still the current task, so a "next" list would hide the one issue that matters.
+async function read_next_issues(
+	completed_issue_number: string | undefined,
+	should_merge: boolean,
+): Promise<ReadonlyArray<string>> {
+	if (!should_merge) return []
+
+	return await git_next_issues.fetch_next_issue_lines(
 		parse_completed_issue_number(completed_issue_number),
 	)
+}
+
+async function print_next_issues(completed_issue_number: string | undefined): Promise<void> {
+	const lines = await read_next_issues(completed_issue_number, true)
+
 	for (const line of lines) console.info(line)
 }
 
-// The tail printed once the workflow itself has finished. `print_pending_release` stays last by
-// contract, so anything added here goes above it.
+// The tail printed once the workflow itself has finished. `print_pending_release`'s line stays last
+// by contract, so anything added here goes above it.
+//
+// **The two reads need nothing from one another, so they go out together** (#5131), and the time
+// they take is printed as a `completion` stage row: the stage block is already out by now, and an
+// untimed tail is the fixed cost `josh ship --log` could not see. The row goes above the completion
+// lines, because the pending-release line is the one that has to stay last.
 async function print_completion(
 	issue_number: string | undefined,
 	should_merge: boolean,
 ): Promise<void> {
+	const started_ms = performance.now()
+	const [next_lines, release_lines] = await Promise.all([
+		read_next_issues(issue_number, should_merge),
+		read_pending_release(),
+	])
+	const duration_ms = performance.now() - started_ms
+
+	console.info(
+		git_followup_stages.format_stage({ name: git_followup_stages.STAGE.completion, duration_ms }),
+	)
 	console.info('')
 	console.info('✅ PR followup completed.')
-	// Merged runs only, like the epic auto-close: on `--no-merge` the linked issue is still open
-	// and still the current task, so a "next" list would hide the one issue that matters.
-	if (should_merge) await print_next_issues(issue_number)
-	await print_pending_release()
+	for (const line of [...next_lines, ...release_lines]) console.info(line)
 }
 
 // A **merged** run ends here, and the round-1 review snapshot's lifetime is one run.
 // `--no-merge` is not the end of one — the pull request is still open and
-// the issue is still the current task, the same line `print_next_issues` and the epic auto-close
+// the issue is still the current task, the same line `read_next_issues` and the epic auto-close
 // already draw — and clearing there would be the unsafe direction: the next round-1 brief would find
 // no record and write a fresh one against the already-fixed tree, which is the arm-A skip over
 // unreviewed fix code the record exists to prevent.

package/scripts/followup/git-followup-pending.ts

@@ -14,8 +14,8 @@ import { version_targets } from '#scripts/version/version-targets'
 // command's own modules rather than re-derived here, and it doubles as the visible signal that a
 // release is owed — a count that keeps climbing is a release nobody has run.
 
-// **The Telegram is sent before the merge**, deliberately: `git-pr-followup.ts` cuts its wrap-up
-// there because that is the last point at which the run can still fail safely. So the count taken
+// **The Telegram's body is read before the merge**, deliberately, though the send goes out beside it
+// (#5080): `git-pr-followup.ts` reads this line before it issues the merge. So the count taken
 // there cannot include this run's own merge, and a bare number would be understated by exactly one.
 // Saying so is the honest form of the same measurement. The console line printed after the merge
 // passes `false` and carries no note, because by then the merge is on the remote and the fetch below
@@ -61,7 +61,7 @@ async function read_tip(options: PendingReadOptions): Promise<string> {
 // remove.
 // The manifest read is **inside** the `try` with the rest. `readFileSync` can throw after
 // `existsSync` passed — EACCES, EISDIR, a racing write — and this function is called from
-// `notify_completion`, which runs *before* `pr_merge`: an escaping rejection there would abort the
+// `completion_sender`, which runs *before* `pr_merge`: an escaping rejection there would abort the
 // run and lose a merge over a cosmetic line.
 async function read_pending(options: PendingReadOptions): Promise<number | undefined> {
 	try {

package/scripts/followup/git-followup-stages.ts

@@ -1,6 +1,7 @@
 import { PrConflictError } from '#scripts/gh/git-pr-checks-eval'
-import { lane_ledger } from '#scripts/lane/lane-ledger'
+import { lane_ledger_write } from '#scripts/lane/lane-ledger-write'
 import { time_format } from '#scripts/time-runtime/time-format'
+import { followup_ci_path, type CiPath } from './followup-ci-path'
 import { followup_coderabbit_wait, type CodeRabbitWait } from './followup-coderabbit-wait'
 
 // Where `pnpm josh followup` spends its own wall clock.
@@ -11,8 +12,9 @@ import { followup_coderabbit_wait, type CodeRabbitWait } from './followup-codera
 // epic auto-close. Untimed, every proposal to shorten it is a guess, and a guess made under time
 // pressure falls on whichever stretch is easiest to cut rather than on the one that is long.
 //
-// **Those nine are seven printed rows**: the four reads in front of the wait go out together and so
-// do the two steps after the merge, and a lap measures an interval rather than a call. `STAGE` below is the list; the two names carrying `and` are the batches.
+// **Those nine are six printed rows**: the four reads in front of the wait go out together, so do the
+// Telegram and the merge, and so do the two steps after the merge, and a lap measures an interval
+// rather than a call. `STAGE` below is the list; the three names carrying `and` are the batches.
 //
 // **Laps, not a wrapper per stage.** The alternative was a `measure(name, () => …)` closure around
 // each call, which reads well and rewrites every call site in the file it instruments — nine
@@ -52,21 +54,29 @@ const NO_DURATION = 0
 // The stages, named once. They are the units a proposal to shorten `followup` would be written
 // against, so a rename has to move the printer and whatever reads it together.
 //
-// **Two names say `and` because two stages became one**. Requests that need
+// **Three names say `and` because each is two stages become one**. Requests that need
 // nothing from one another are now issued together, and a lap records an interval rather than a
 // call — so a batch is one interval, and what it holds has to be readable from its name or the
 // shortening becomes invisible to the very measurement that found it. `closes-and-context` is the
 // former `closes-check` and `context`; `completion-and-epic-close` the former `completion-comment`
-// and `epic-close`. **`checks-wait` is untouched, and deliberately so**: it is the merge gate, and
-// this change only ever moved time that was not a wait.
+// and `epic-close`; `telegram-and-merge` the former `telegram` and `merge` (#5080). **`checks-wait` is
+// untouched, and deliberately so**: it is the merge gate, and this change only ever moved time that was
+// not a wait.
 const STAGE = {
 	closes_and_context: 'closes-and-context',
 	checks_wait: 'checks-wait',
+	// Only on a run that retriggered a rate-limited CodeRabbit: the window and the review after it
+	// (joshuafolkken/kit#5289), kept out of `checks-wait` so the `ci-wait` row is the checks alone.
+	coderabbit_retrigger: 'coderabbit-retrigger',
 	coderabbit_comments: 'coderabbit-comments',
 	ai_review_comments: 'ai-review-comments',
+	// The interval of a run that merged nothing, where the send has nothing to go beside.
 	telegram: 'telegram',
-	merge: 'merge',
+	telegram_and_merge: 'telegram-and-merge',
 	completion_and_epic_close: 'completion-and-epic-close',
+	// The completion tail `git-followup-finish.ts` prints after this block (#5131): the next-issue list
+	// and the unreleased-merge count. Printed as its own row there, since the block is already out.
+	completion: 'completion',
 	// The lap that was still running when a stage threw. `followup` exits non-zero on an AI-review
 	// blocker and on a red check, and those are the runs whose wait is longest — reporting nothing for
 	// them would leave the measurement blind to exactly the invocations worth measuring.
@@ -93,15 +103,31 @@ interface FollowupStage {
 // negative lap.
 //
 // `coderabbit` rides on the log because every check wait of the run — the merge gate's and the one the
-// merge goes back to (#4036) — adds to the same CodeRabbit time.
+// merge goes back to (#4036) — adds to the same CodeRabbit time. `ci_path` is the first wait's
+// per-check timing, which the `ci-wait` row carries (`followup-ci-path.ts`), with `issue` once the run
+// knows it, so `lane:breakdown` finds the lane's row.
 interface StageLog {
 	stages: Array<FollowupStage>
 	last_ms: number
 	coderabbit: CodeRabbitWait
+	ci_path: CiPath
+	issue?: number
 }
 
 function new_log(): StageLog {
-	return { stages: [], last_ms: performance.now(), coderabbit: followup_coderabbit_wait.new_wait() }
+	return {
+		stages: [],
+		last_ms: performance.now(),
+		coderabbit: followup_coderabbit_wait.new_wait(),
+		ci_path: followup_ci_path.new_path(),
+	}
+}
+
+// The issue the `ci-wait` row is written under; a number that does not parse leaves the row without one.
+function hold_issue(log: StageLog, issue_number: string | undefined): void {
+	const issue = Number(issue_number)
+
+	if (Number.isSafeInteger(issue) && issue > 0) log.issue = issue
 }
 
 function lap(log: StageLog, name: string): void {
@@ -151,9 +177,11 @@ async function record_ci_wait(log: StageLog): Promise<void> {
 
 	if (wait === undefined) return
 
-	await lane_ledger.record_stage({
-		stage: lane_ledger.CI_WAIT_STAGE,
+	await lane_ledger_write.record_stage({
+		stage: lane_ledger_write.CI_WAIT_STAGE,
 		elapsed_ms: wait.duration_ms,
+		...(log.issue !== undefined && { issue: log.issue }),
+		...followup_ci_path.record_of(log.ci_path),
 	})
 }
 
@@ -167,8 +195,10 @@ const git_followup_stages = {
 	STAGE_LINE_PREFIX,
 	STAGE_TOTAL_PREFIX,
 	new_log,
+	hold_issue,
 	lap,
 	total_ms,
+	format_stage,
 	format_stages,
 	print_stages,
 	record_waits,

package/scripts/followup/git-pr-followup-checks.ts

@@ -6,10 +6,15 @@ import {
 	WATCH_CONFIRMED_STABLE_READS,
 	type SnapshotObserver,
 } from '#scripts/gh/git-pr-checks'
-import { is_coderabbit_check } from '#scripts/gh/git-pr-checks-eval'
+import {
+	FAILURE_MESSAGE_PREFIX,
+	is_coderabbit_check,
+	PrConflictError,
+} from '#scripts/gh/git-pr-checks-eval'
 import { CHECK_STATUS_PASS, type PrStateSnapshot } from '#scripts/gh/git-pr-checks-parse'
 import { issue_watch_gate } from '#scripts/issue/issue-watch-gate'
 import { json_value } from '#scripts/lib/json-value'
+import { followup_job_rerun } from './followup-job-rerun'
 
 // The check wait and the notes read off its snapshot — split out of `git-pr-followup.ts` when it
 // reached its line limit. `git-pr-followup` re-exports each name it always
@@ -71,12 +76,15 @@ async function handle_watch_failure(branch_name: string, error: unknown): Promis
 	console.info(`⚠️ ${WATCH_FAILED_NOTE}: ${git_gh_helpers.get_error_message_with_stderr(error)}`)
 }
 
-// **Returns whether the watch confirmed completion**: `true` only when
-// `pr_checks_watch` saw every check finish, `false` when it timed out with checks still pending or
-// fell through to polling on a swallowed failure. `run_checks` reads it to decide how many stable poll
-// reads the wait still needs. A timed-out watch (`timed_out: true`) never spanned the pending→settled
-// window, so it must keep the full stable-read window rather than claim the confirmation. A rethrown
-// failure — the empty-rollup case — never reaches a caller, so it needs no answer here.
+// **Returns whether the watch observed the checks**: `true` when `pr_checks_watch` returned — every
+// check finished, or the budget ran out with one still pending — and `false` when it fell through to
+// polling on a swallowed failure. `run_checks` reads it to decide how many stable poll reads the wait
+// still needs. **A timed-out watch counts too** (#5080): it read the checks for two minutes without
+// two settled reads in a row, so the checks were registered and not yet settled while it watched, and
+// the poll's first settled read closes that pending→settled window — the window the second stable read
+// exists for. Holding it to two cost ten seconds on every lane whose CI outlasts the two-minute
+// budget. A rethrown failure — the empty-rollup case — never reaches a caller, so it needs no answer
+// here.
 async function watch_before_polling(
 	branch_name: string,
 	observe: SnapshotObserver | undefined,
@@ -85,9 +93,9 @@ async function watch_before_polling(
 	console.info('📊 Watching PR checks...')
 
 	try {
-		const result = await git_gh_command.pr_checks_watch(branch_name, observe)
+		await git_gh_command.pr_checks_watch(branch_name, observe)
 
-		return !result.timed_out
+		return true
 	} catch (error) {
 		await handle_watch_failure(branch_name, error)
 
@@ -95,21 +103,23 @@ async function watch_before_polling(
 	}
 }
 
-// **A confirmed watch lowers the poll's stable-read requirement to one**. The
-// watch is the first confirmation that every check settled, so the poll that follows need only agree
-// once rather than twice — which removes the extra interval `followup` spent re-proving a settled run.
+// **A returned watch lowers the poll's stable-read requirement to one**. The
+// watch saw every check settle or saw them pending, so the poll that follows need only agree once
+// rather than twice — which removes the extra interval `followup` spent re-proving a settled run.
 // A skipped or fallen-through watch keeps `DEFAULT_STABLE_READS`, and a check that turned red is still
 // caught: the poll evaluates the full merge gate and throws on failure before any stable read counts.
 // `observe` is shown each poll's snapshot, the watch's included — the run's `pr-checks` line
 // (joshuafolkken/kit#3770).
-async function run_checks(input: {
+interface RunChecksInput {
 	branch_name: string
 	is_skip_watch: boolean
 	observe?: SnapshotObserver
-}): Promise<PrStateSnapshot> {
-	const is_watch_confirmed =
+}
+
+async function wait_for_checks(input: RunChecksInput): Promise<PrStateSnapshot> {
+	const is_watch_returned =
 		!input.is_skip_watch && (await watch_before_polling(input.branch_name, input.observe))
-	const stable_reads = is_watch_confirmed ? WATCH_CONFIRMED_STABLE_READS : DEFAULT_STABLE_READS
+	const stable_reads = is_watch_returned ? WATCH_CONFIRMED_STABLE_READS : DEFAULT_STABLE_READS
 
 	return await git_pr_checks.wait_for_pr_success(
 		input.branch_name,
@@ -119,6 +129,35 @@ async function run_checks(input: {
 	)
 }
 
+// A merge conflict is not a check, so no rerun can clear it.
+function is_checks_failure(error: unknown): boolean {
+	if (error instanceof PrConflictError) return false
+
+	return error instanceof Error && error.message.startsWith(FAILURE_MESSAGE_PREFIX)
+}
+
+// **A job the runner stopped after every step passed, or one red only on tests outside the change, is
+// rerun once and the wait judged again** (`followup-job-rerun.ts`). The second wait has no rerun of its own, so a job that stops the same way
+// twice — or any other failure — ends the run as before.
+async function rejudge_after_rerun(
+	input: RunChecksInput,
+	error: unknown,
+): Promise<PrStateSnapshot> {
+	if (!is_checks_failure(error)) throw error
+	const rerun = await followup_job_rerun.rerun_failed_jobs(input.branch_name)
+	if (rerun.length === 0) throw error
+
+	return await wait_for_checks(input)
+}
+
+async function run_checks(input: RunChecksInput): Promise<PrStateSnapshot> {
+	try {
+		return await wait_for_checks(input)
+	} catch (error) {
+		return await rejudge_after_rerun(input, error)
+	}
+}
+
 // Temporary: record every CodeRabbit check that was not passing when the merge gate
 // opened, so a merge shipped without CodeRabbit review stays auditable.
 function read_coderabbit_skip_notes(snapshot: PrStateSnapshot): Array<string> {

package/scripts/followup/git-pr-followup-wrapup.ts

@@ -4,6 +4,7 @@ import type { PrStateSnapshot } from '#scripts/gh/git-pr-checks'
 import { git_pr_merge_checked, type WaitForGreen } from '#scripts/gh/git-pr-merge-checked'
 import { issue_body_or_comment } from '#scripts/issue/issue-body-or-comment'
 import { issue_watch_gate } from '#scripts/issue/issue-watch-gate'
+import { settled_together } from '#scripts/lib/settled-together'
 import { git_notify, type GitNotifyConfig } from '#scripts/notify/git-notify'
 import { git_followup_cleanup } from './git-followup-cleanup'
 import { git_followup_issue_close } from './git-followup-issue-close'
@@ -47,6 +48,8 @@ interface WrapupInput {
 	// that head moved before the merge (`git-pr-merge-checked.ts`, #4036).
 	checked?: PrStateSnapshot | undefined
 	wait_for_green: WaitForGreen
+	// The completion Telegram, its body already read, sent beside the merge.
+	send_completion: () => Promise<void>
 }
 
 function build_notify_body(input: {
@@ -232,13 +235,29 @@ async function run_tail_steps(input: WrapupInput): Promise<void> {
 // The last look at the issue before the merge: the CI poll checked it on every iteration, but a
 // person's update can land between that poll and this step, and only the seconds after this read are
 // left unwatched.
-async function run_wrapup(input: WrapupInput, log: StageLog): Promise<void> {
-	if (input.should_merge && input.is_merged !== true) {
-		await issue_watch_gate.assert_read_here()
-		await git_pr_merge_checked.merge_checked(input)
-		lap(log, STAGE.merge)
+async function merge_step(input: WrapupInput): Promise<void> {
+	await issue_watch_gate.assert_read_here()
+	await git_pr_merge_checked.merge_checked(input)
+}
+
+// **The completion Telegram goes out beside the merge** (#5080): neither reads what the other writes,
+// and in sequence the send was about four seconds in front of every merge. The send is tolerant, so
+// its failure still never stops the merge; both settle before a merge failure is thrown, so a refused
+// merge does not cut a send in flight. Answers the stage the interval was.
+async function send_beside_merge(input: WrapupInput): Promise<string> {
+	if (!input.should_merge || input.is_merged === true) {
+		await input.send_completion()
+
+		return STAGE.telegram
 	}
 
+	await settled_together(input.send_completion(), merge_step(input))
+
+	return STAGE.telegram_and_merge
+}
+
+async function run_wrapup(input: WrapupInput, log: StageLog): Promise<void> {
+	lap(log, await send_beside_merge(input))
 	await run_tail_steps(input)
 
 	lap(log, STAGE.completion_and_epic_close)