@joshuafolkken/kit 1.1784.0 → 1.2114.0

package/scripts/followup/followup-coderabbit-wait.ts

@@ -1,6 +1,6 @@
 import type { SnapshotObserver } from '#scripts/gh/git-pr-checks'
 import type { PrStateSnapshot } from '#scripts/gh/git-pr-checks-parse'
-import { lane_ledger } from '#scripts/lane/lane-ledger'
+import { lane_ledger_write } from '#scripts/lane/lane-ledger-write'
 import type { CodeRabbitState } from '#scripts/run/event/run-event-checks'
 import { followup_checks_event } from './followup-checks-event'
 
@@ -57,26 +57,29 @@ function observer_of(wait: CodeRabbitWait): SnapshotObserver {
 	}
 }
 
-// Both observers on every poll — the board's event line and this timer.
-function both(first: SnapshotObserver, second: SnapshotObserver): SnapshotObserver {
-	return async function observe_both(snapshot: PrStateSnapshot): Promise<void> {
-		await first(snapshot)
-		await second(snapshot)
+// Every observer on every poll, started in order — the board's event line and the wait's timers.
+function all(...observers: ReadonlyArray<SnapshotObserver>): SnapshotObserver {
+	return async function observe_all(snapshot: PrStateSnapshot): Promise<void> {
+		await Promise.all(
+			observers.map(async (observer) => {
+				await observer(snapshot)
+			}),
+		)
 	}
 }
 
 async function record_part(stage: string, elapsed_ms: number): Promise<void> {
-	if (elapsed_ms > NO_TIME) await lane_ledger.record_stage({ stage, elapsed_ms })
+	if (elapsed_ms > NO_TIME) await lane_ledger_write.record_stage({ stage, elapsed_ms })
 }
 
 // **A run that never saw CodeRabbit waiting records nothing**, so a row's run count is how many
 // followups CodeRabbit held up, and its median is over those.
 async function record(wait: CodeRabbitWait): Promise<void> {
-	await record_part(lane_ledger.CODERABBIT_WAIT_STAGE, wait.wait_ms)
-	await record_part(lane_ledger.CODERABBIT_RATE_LIMIT_STAGE, wait.rate_limited_ms)
+	await record_part(lane_ledger_write.CODERABBIT_WAIT_STAGE, wait.wait_ms)
+	await record_part(lane_ledger_write.CODERABBIT_RATE_LIMIT_STAGE, wait.rate_limited_ms)
 }
 
-const followup_coderabbit_wait = { both, new_wait, observe, observer_of, record }
+const followup_coderabbit_wait = { all, new_wait, observe, observer_of, record }
 
 export type { CodeRabbitWait }
 export { followup_coderabbit_wait }

package/scripts/followup/followup-failed-tests.ts

@@ -0,0 +1,160 @@
+import path from 'node:path'
+import { git_gh_issue_write } from '#scripts/gh/git-gh-issue-write'
+import { observation_ledger_home } from '#scripts/observations/observation-ledger-home'
+import {
+	observation_record,
+	REFUSED_VERDICT,
+	type ObservationEntry,
+} from '#scripts/observations/observation-record'
+import { review_comment_round } from '#scripts/review/review-comment-round'
+import { vitest_failure_summary } from '#scripts/test/vitest-failure-summary'
+
+// A CI job that ended `failure` because a unit test went red can still owe nothing to the change: a test
+// the diff never touched fails under runner load (joshuafolkken/kit#5152, a frame-budget test on a busy
+// runner). `followup` reruns such a job once, as it does a stalled one (`followup-job-rerun.ts`), and
+// these are the readings that decide it: the failed tests a job's log names, and whether any of them
+// belongs to the change.
+//
+// **A test belongs to the change when its own file or its subject module moved** — `foo.claim.test.ts`
+// tests `foo.claim.ts` and `foo.ts`. A test reaching a moved module only through an import is missed,
+// and costs one rerun: the second red still stops `followup`, since a rerun attempt is never rerun.
+
+// GitHub Actions stamps every log line `<ISO timestamp> `, which vitest's ` FAIL  ` line does not begin with.
+const LOG_TIMESTAMP = /^\d{4}-\d{2}-\d{2}T\S+Z /u
+const TEST_SUFFIX = /\.test(?<extension>\.[^./]+)$/u
+const SEGMENT_SEPARATOR = '.'
+const OBSERVATION_SLUG = 'followup-unrelated-test-rerun'
+const OBSERVATION_DEPTH = 'd1'
+const NAME_SEPARATOR = '; '
+const UNNAMED_ISSUE =
+	'Did not record the failed tests outside the change: the branch names no issue to post them on.'
+
+const RERUN_VERDICT = 'rerun'
+const KEPT_VERDICT = 'kept'
+const UNNAMED_VERDICT = 'unnamed'
+
+type TestVerdict = typeof RERUN_VERDICT | typeof KEPT_VERDICT | typeof UNNAMED_VERDICT
+
+interface FailedTests {
+	files: ReadonlyArray<string>
+	names: ReadonlyArray<string>
+}
+
+interface UnrelatedFailure {
+	job: string
+	tests: FailedTests
+}
+
+// The failed tests vitest named in a job's log — empty when the job failed on anything but a test.
+function read_failed_tests(log: string): FailedTests {
+	const lines = log.split('\n').map((line) => line.replace(LOG_TIMESTAMP, ''))
+	const names = vitest_failure_summary.failed_tests(lines.join('\n'))
+
+	return { files: vitest_failure_summary.failed_files(names), names }
+}
+
+// The test file itself, then the module each leading run of its name's segments names.
+function subjects_of(test_file: string): ReadonlyArray<string> {
+	const extension = TEST_SUFFIX.exec(test_file)?.groups?.['extension']
+
+	if (extension === undefined) return [test_file]
+
+	const directory = path.posix.dirname(test_file)
+	const segments = path.posix.basename(test_file, `.test${extension}`).split(SEGMENT_SEPARATOR)
+	const modules = segments.map((_, index) =>
+		path.posix.join(
+			directory,
+			`${segments.slice(0, index + 1).join(SEGMENT_SEPARATOR)}${extension}`,
+		),
+	)
+
+	return [test_file, ...modules]
+}
+
+function is_in_change(test_file: string, changed: ReadonlySet<string>): boolean {
+	return subjects_of(test_file).some((subject) => changed.has(subject))
+}
+
+// **`rerun` only when the log named a failed test and none of them belongs to the change**, `kept`
+// when one does. A job red with no test named — a lint, a build, a job gathering others' results — is
+// `unnamed`: rerun only as one waiting on a job red on tests outside the change (`followup-job-needs.ts`).
+function verdict_of(tests: FailedTests, changed: ReadonlySet<string>): TestVerdict {
+	if (tests.files.length === 0) return UNNAMED_VERDICT
+
+	return tests.files.every((file) => !is_in_change(file, changed)) ? RERUN_VERDICT : KEPT_VERDICT
+}
+
+// The ledger line a rerun leaves, so a test that keeps failing beside unrelated changes is counted
+// rather than retried into silence. A ledger line's fields are split on ` | `, so the `|<project>|`
+// prefix a workspace test name carries is rewritten.
+function observation_of(failures: ReadonlyArray<UnrelatedFailure>): ObservationEntry {
+	const files = new Set(failures.flatMap(({ tests }) => tests.files))
+	const names = new Set(failures.flatMap(({ tests }) => tests.names))
+	const listed = [...names].map((name) => name.replaceAll('|', '/'))
+	const jobs = failures.map(({ job }) => job)
+
+	return {
+		slug: OBSERVATION_SLUG,
+		depth: OBSERVATION_DEPTH,
+		where: [...files].join(', '),
+		what: `followup reran ${jobs.join(', ')}; failed outside the change: ${listed.join(NAME_SEPARATOR)}`,
+	}
+}
+
+// **The line is posted on the issue, never appended to the tree** (joshuafolkken/kit#5197): the rerun
+// happens during the CI wait, after `prepare_merge` committed the ledger, so an appended line would
+// either be lost with the lane or need a push of its own that restarts CI. `review-comment-round.ts`
+// carries it into the ledger once the issue closes; the count is still read off this tree's ledger.
+async function post(entry: ObservationEntry, issue: number): Promise<string> {
+	const assessment = await observation_record.assess({
+		entry,
+		checkout: observation_ledger_home.ledger_root(),
+		now: new Date(),
+	})
+
+	if (assessment.verdict === REFUSED_VERDICT) return `${REFUSED_VERDICT}: ${assessment.reason}`
+
+	await git_gh_issue_write.issue_comment(
+		String(issue),
+		review_comment_round.observation_body_of([assessment.line]),
+	)
+
+	return assessment.verdict
+}
+
+// A post that fails is reported and does not stop the step it describes; a branch naming no issue has
+// nowhere to post.
+async function record_entry(entry: ObservationEntry, issue: number | undefined): Promise<void> {
+	if (issue === undefined) {
+		console.info(`⚠️ ${UNNAMED_ISSUE}`)
+
+		return
+	}
+
+	try {
+		console.info(`📝 Recorded the ${entry.slug} observation: ${await post(entry, issue)}`)
+	} catch (error) {
+		console.info(`⚠️ Could not record the ${entry.slug} observation: ${String(error)}`)
+	}
+}
+
+async function record(
+	failures: ReadonlyArray<UnrelatedFailure>,
+	issue: number | undefined,
+): Promise<void> {
+	if (failures.length === 0) return
+
+	await record_entry(observation_of(failures), issue)
+}
+
+const followup_failed_tests = {
+	observation_of,
+	read_failed_tests,
+	record,
+	record_entry,
+	subjects_of,
+	verdict_of,
+}
+
+export type { FailedTests, TestVerdict, UnrelatedFailure }
+export { followup_failed_tests, KEPT_VERDICT, OBSERVATION_SLUG, RERUN_VERDICT, UNNAMED_VERDICT }

package/scripts/followup/followup-job-needs.ts

@@ -0,0 +1,100 @@
+import { readFile } from 'node:fs/promises'
+import { yaml_document } from '#scripts/lib/yaml-document'
+import { z } from 'zod'
+
+// A job that only gathers others' results — kit's `Checks`, which `needs` the unit shards — ends
+// `failure` beside a shard red on a test the change never touched, and its log names no test
+// (joshuafolkken/kit#5171). It fails because of that shard alone, so it is rerun with it; the rerun
+// endpoint reruns a failed job's dependents in any case. **The workflow's own `needs` says which job
+// waits on which**, read from the checkout the pull request's head is.
+//
+// A job is known by the name a check run carries, which the workflow writes as a template: an
+// expression in it matches anything, and a name with none gains the ` (<matrix values>)` suffix GitHub
+// appends to a matrix job. A job no pattern matches — a reusable workflow's, a name built otherwise —
+// is waited on by nothing here, so its failure is kept.
+
+const EXPRESSION = /\$\{\{[^}]*\}\}/u
+const EXPRESSION_MARK = '${{'
+const ANY_TEXT = '.+'
+const MATRIX_SUFFIX = String.raw`(?: \(.+\))?`
+const REGEX_SPECIAL = /[$()*+.?[\\\]^{|}]/gu
+
+const needs_schema = z.array(z.string())
+
+const job_schema = z.looseObject({
+	name: z.string().optional(),
+	needs: z.union([z.string(), needs_schema]).optional(),
+})
+
+const workflow_schema = z.looseObject({ jobs: z.record(z.string(), job_schema) })
+
+type WorkflowJobs = z.infer<typeof workflow_schema>['jobs']
+
+interface JobNeeds {
+	pattern: RegExp
+	needs: ReadonlyArray<RegExp>
+}
+
+type NeedsGraph = ReadonlyArray<JobNeeds>
+
+function escape(text: string): string {
+	return text.replaceAll(REGEX_SPECIAL, String.raw`\$&`)
+}
+
+function name_pattern(id: string, name: string = id): RegExp {
+	const source = name.includes(EXPRESSION_MARK)
+		? name
+				.split(EXPRESSION)
+				.map((part) => escape(part))
+				.join(ANY_TEXT)
+		: `${escape(name)}${MATRIX_SUFFIX}`
+
+	return new RegExp(`^${source}$`, 'u')
+}
+
+function needs_list(needs: string | ReadonlyArray<string> = []): ReadonlyArray<string> {
+	return typeof needs === 'string' ? [needs] : needs
+}
+
+function pattern_of(jobs: WorkflowJobs, id: string): RegExp {
+	return name_pattern(id, jobs[id]?.name)
+}
+
+// The jobs of a workflow, each with the names of the jobs it needs; a workflow of another shape has
+// none. YAML that does not parse throws, which `read_graph` answers as none.
+function parse_graph(workflow: string): NeedsGraph {
+	const parsed = workflow_schema.safeParse(yaml_document.parse_yaml(workflow))
+
+	if (!parsed.success) return []
+
+	const { jobs } = parsed.data
+
+	return Object.entries(jobs).map(([id, job]) => ({
+		pattern: pattern_of(jobs, id),
+		needs: needs_list(job.needs).map((need) => pattern_of(jobs, need)),
+	}))
+}
+
+// **Whether the job named `job_name` needs one of the jobs named in `upstream`.**
+function is_waiting_on(
+	graph: NeedsGraph,
+	job_name: string,
+	upstream: ReadonlyArray<string>,
+): boolean {
+	const needs = graph.filter(({ pattern }) => pattern.test(job_name)).flatMap((job) => job.needs)
+
+	return upstream.some((name) => needs.some((pattern) => pattern.test(name)))
+}
+
+async function read_graph(workflow_path: string): Promise<NeedsGraph> {
+	try {
+		return parse_graph(await readFile(workflow_path, 'utf8'))
+	} catch {
+		return []
+	}
+}
+
+const followup_job_needs = { is_waiting_on, parse_graph, read_graph }
+
+export type { NeedsGraph }
+export { followup_job_needs }

package/scripts/followup/followup-job-rerun.ts

@@ -0,0 +1,303 @@
+import { git_gh_api_path } from '#scripts/gh/git-gh-api-path'
+import { git_gh_command } from '#scripts/gh/git-gh-command'
+import { git_gh_exec } from '#scripts/gh/git-gh-exec'
+import { git_gh_helpers } from '#scripts/gh/git-gh-helpers'
+import { changed_paths } from '#scripts/git/changed-paths'
+import { git_branch } from '#scripts/git/git-branch'
+import { poll } from '#scripts/lib/poll'
+import { z } from 'zod'
+import {
+	followup_failed_tests,
+	KEPT_VERDICT,
+	RERUN_VERDICT,
+	UNNAMED_VERDICT,
+	type FailedTests,
+	type TestVerdict,
+} from './followup-failed-tests'
+import { followup_job_needs, type NeedsGraph } from './followup-job-needs'
+import { followup_main_move } from './followup-main-move'
+
+// A GitHub Actions job whose every step succeeded can still end `cancelled` or `timed_out`: the runner
+// stops reporting after the last step and the job's `timeout-minutes` cuts it off (joshuafolkken/kit#4813,
+// two 3-second jobs cut at 2m0s). Nothing in the change caused it, so `followup` reruns such a job once
+// — as `git-gh-retry.ts` retries a GitHub 5xx — rather than failing a pull request that is green.
+//
+// **A job red only on tests outside the change is rerun the same way** (joshuafolkken/kit#5171): a test
+// the diff never touched failing under runner load stopped `followup` and waited for a relaunched
+// session to type the rerun by hand. Only the run's first attempt qualifies, so a test that fails again
+// stops `followup` as before, and each such rerun leaves an observation (`followup-failed-tests.ts`).
+// A job that failed only for waiting on such a job is rerun with it (`followup-job-needs.ts`).
+//
+// The rerun endpoint reruns **every** failed job of a run, and GitHub refuses it while the run is still
+// going. So each run is waited out first, and it is rerun only when every job that failed in it
+// qualifies: a run holding a job that failed on the change itself is never rerun, or a test the change
+// broke could be retried into a merge.
+
+const FAILURE_CONCLUSION = 'failure'
+const FAILED_CONCLUSIONS = [FAILURE_CONCLUSION, 'cancelled', 'timed_out'] as const
+const FAILED_JOB_CONCLUSIONS: ReadonlySet<string> = new Set(FAILED_CONCLUSIONS)
+const ACTIONS_APP_SLUG = 'github-actions'
+const PASSING_STEP_CONCLUSIONS: ReadonlySet<string> = new Set(['success', 'skipped'])
+const RUN_COMPLETED = 'completed'
+const FIRST_ATTEMPT = 1
+
+// The longest `timeout-minutes` in `ci.yml` is 10; a run still going past that is not one a rerun helps.
+const COMPLETE_ATTEMPTS = 40
+const COMPLETE_INTERVAL_MS = 15_000
+
+// The rerun answers once the new attempt exists, but its check runs can trail it; a re-judge that read
+// the failed run again would fail on the very job just rerun.
+const REGISTER_ATTEMPTS = 10
+const REGISTER_INTERVAL_MS = 3000
+
+const failed_check_schema = z.looseObject({
+	id: z.number(),
+	conclusion: z.enum(FAILED_CONCLUSIONS),
+	app: z.looseObject({ slug: z.literal(ACTIONS_APP_SLUG) }),
+})
+
+const rollup_schema = z.looseObject({ statusCheckRollup: z.array(z.unknown()) })
+
+// REST answers `null` for a step that never finished; a test fixture may leave the field out.
+const step_schema = z.looseObject({ conclusion: z.string().nullish() })
+
+const job_schema = z.looseObject({
+	id: z.number(),
+	name: z.string(),
+	run_id: z.number(),
+	run_attempt: z.number().nullish(),
+	conclusion: z.string().nullish(),
+	steps: z.array(step_schema),
+})
+
+// `path` is the workflow file's, relative to the repository root.
+const run_schema = z.looseObject({ status: z.string(), path: z.string().optional() })
+
+const jobs_page_schema = z.looseObject({ jobs: z.array(job_schema) })
+
+type Job = z.infer<typeof job_schema>
+
+// A failed job and what its reading says, with the failed tests when its log named some.
+interface Judged {
+	job: Job
+	verdict: TestVerdict
+	tests?: FailedTests
+}
+
+function actions_api_path(rest: string): string {
+	return `${git_gh_api_path.repo_api_path()}/actions/${rest}`
+}
+
+async function read_actions<T>(rest: string, schema: z.ZodType<T>): Promise<T> {
+	const json = await git_gh_exec.exec_gh_api({ path: actions_api_path(rest) })
+
+	return schema.parse(JSON.parse(json))
+}
+
+async function read_failed_check_ids(branch_name: string): Promise<Array<number>> {
+	const { snapshot_json } = await git_gh_command.pr_get_checks_snapshot(branch_name)
+	const rollup = rollup_schema.parse(JSON.parse(snapshot_json)).statusCheckRollup
+
+	return rollup.flatMap((item) => {
+		const parsed = failed_check_schema.safeParse(item)
+
+		return parsed.success ? parsed.data.id : []
+	})
+}
+
+// A check run of GitHub Actions has its job's id, which is how the job — and so its run — is found.
+async function read_failed_run_ids(branch_name: string): Promise<Array<number>> {
+	const ids = await read_failed_check_ids(branch_name)
+	const jobs = await Promise.all(
+		ids.map(async (id) => await read_actions(`jobs/${String(id)}`, job_schema)),
+	)
+
+	return [...new Set(jobs.map((job) => job.run_id))]
+}
+
+async function is_run_completed(run_id: number): Promise<boolean> {
+	const run = await read_actions(`runs/${String(run_id)}`, run_schema)
+
+	return run.status === RUN_COMPLETED
+}
+
+async function await_run_completed(run_id: number): Promise<boolean> {
+	return await poll.poll_until(async () => await is_run_completed(run_id), {
+		attempts: COMPLETE_ATTEMPTS,
+		interval_ms: COMPLETE_INTERVAL_MS,
+	})
+}
+
+function has_every_step_passed(job: Job): boolean {
+	if (job.steps.length === 0) return false
+
+	return job.steps.every((step) => PASSING_STEP_CONCLUSIONS.has(step.conclusion ?? ''))
+}
+
+function is_first_failure(job: Job): boolean {
+	return (
+		job.conclusion === FAILURE_CONCLUSION && (job.run_attempt ?? FIRST_ATTEMPT) <= FIRST_ATTEMPT
+	)
+}
+
+// A job whose every step passed stopped on the runner; one red on its first attempt is judged by the
+// failed tests its log names. The log endpoint redirects to the plain text, which `gh api` follows; a
+// vitest log is colored.
+async function judge_job(job: Job, changed: ReadonlySet<string>): Promise<Judged> {
+	if (has_every_step_passed(job)) return { job, verdict: RERUN_VERDICT }
+	if (!is_first_failure(job)) return { job, verdict: KEPT_VERDICT }
+
+	const log = await git_gh_exec.exec_gh_api({
+		path: actions_api_path(`jobs/${String(job.id)}/logs`),
+		should_allow_escapes: true,
+	})
+	const tests = followup_failed_tests.read_failed_tests(log)
+
+	return { job, verdict: followup_failed_tests.verdict_of(tests, changed), tests }
+}
+
+function is_red_on_tests({ verdict, tests }: Judged): boolean {
+	return verdict === RERUN_VERDICT && tests !== undefined
+}
+
+// A job red with no test named qualifies only as one waiting on a job red on tests outside the change.
+function settle_unnamed(judged: ReadonlyArray<Judged>, graph: NeedsGraph): Array<Judged> {
+	const red_on_tests = judged.filter((entry) => is_red_on_tests(entry)).map(({ job }) => job.name)
+
+	return judged.map((entry) => {
+		if (entry.verdict !== UNNAMED_VERDICT) return entry
+
+		const is_waiting = followup_job_needs.is_waiting_on(graph, entry.job.name, red_on_tests)
+
+		return is_waiting ? { job: entry.job, verdict: RERUN_VERDICT } : entry
+	})
+}
+
+// The workflow is read only when a job named no test; a run without its `path` waits on nothing.
+async function read_needs_graph(
+	run_id: number,
+	judged: ReadonlyArray<Judged>,
+): Promise<NeedsGraph> {
+	if (judged.every(({ verdict }) => verdict !== UNNAMED_VERDICT)) return []
+
+	const run = await read_actions(`runs/${String(run_id)}`, run_schema)
+
+	return run.path === undefined ? [] : await followup_job_needs.read_graph(run.path)
+}
+
+// **The jobs the rerun would retry**, or `undefined` when the run must not be rerun: still going, or
+// holding a failed job that does not qualify.
+async function read_rerun_jobs(
+	run_id: number,
+	changed: ReadonlySet<string>,
+): Promise<Array<Judged> | undefined> {
+	if (!(await await_run_completed(run_id))) return undefined
+	const { jobs } = await read_actions(
+		`runs/${String(run_id)}/jobs?filter=latest&per_page=100`,
+		jobs_page_schema,
+	)
+	const failed = jobs.filter((job) => FAILED_JOB_CONCLUSIONS.has(job.conclusion ?? ''))
+	const judged = await Promise.all(failed.map(async (job) => await judge_job(job, changed)))
+	const settled = settle_unnamed(judged, await read_needs_graph(run_id, judged))
+
+	return settled.every(({ verdict }) => verdict === RERUN_VERDICT) ? settled : undefined
+}
+
+// One request per run, not retried: a rerun that landed before a transient failure would be refused
+// the second time.
+async function rerun_runs(run_ids: ReadonlyArray<number>): Promise<void> {
+	await Promise.all(
+		run_ids.map(
+			async (run_id) =>
+				await git_gh_exec.exec_gh_api({
+					path: actions_api_path(`runs/${String(run_id)}/rerun-failed-jobs`),
+					method: 'POST',
+					should_retry: false,
+				}),
+		),
+	)
+}
+
+async function await_rerun_registered(
+	branch_name: string,
+	jobs: ReadonlyArray<Job>,
+): Promise<void> {
+	const rerun_ids = new Set(jobs.map((job) => job.id))
+
+	await poll.poll_until(
+		async () => {
+			const failed = await read_failed_check_ids(branch_name)
+
+			return failed.every((id) => !rerun_ids.has(id))
+		},
+		{ attempts: REGISTER_ATTEMPTS, interval_ms: REGISTER_INTERVAL_MS },
+	)
+}
+
+// Every run must qualify: rerunning some would still leave the pull request red on the others. The
+// change is read once, and only when a run failed.
+async function find_rerun_jobs(branch_name: string): Promise<Array<Judged>> {
+	const run_ids = await read_failed_run_ids(branch_name)
+
+	if (run_ids.length === 0) return []
+
+	const changed = new Set(await changed_paths.read_changed_paths(false))
+	const per_run = await Promise.all(
+		run_ids.map(async (run_id) => await read_rerun_jobs(run_id, changed)),
+	)
+
+	return per_run.every((jobs) => jobs !== undefined) ? per_run.flat() : []
+}
+
+async function rerun(branch_name: string, entries: ReadonlyArray<Judged>): Promise<Array<string>> {
+	const jobs = entries.map(({ job }) => job)
+	const names = jobs.map((job) => job.name)
+
+	console.info(
+		`🔁 Rerunning CI jobs that stopped after every step passed or failed only outside the change: ${names.join(', ')}`,
+	)
+	await followup_failed_tests.record(
+		entries.flatMap(({ job, tests }) => (tests === undefined ? [] : { job: job.name, tests })),
+		git_branch.issue_from_branch(branch_name),
+	)
+	await rerun_runs([...new Set(jobs.map((job) => job.run_id))])
+	await await_rerun_registered(branch_name, jobs)
+
+	return names
+}
+
+// A failed test main has since changed is merged before any rerun (`followup-main-move.ts`).
+async function should_merge_first(
+	branch_name: string,
+	entries: ReadonlyArray<Judged>,
+): Promise<boolean> {
+	const files = [...new Set(entries.flatMap(({ tests }) => tests?.files ?? []))]
+
+	return await followup_main_move.should_merge_first(
+		files,
+		git_branch.issue_from_branch(branch_name),
+	)
+}
+
+// **Returns the names of the jobs it reran**, empty when there was none to rerun or main has to be
+// merged first. A read or a rerun that fails is reported and answers empty: the rerun is a recovery,
+// so the original failure stands.
+async function rerun_failed_jobs(branch_name: string): Promise<Array<string>> {
+	try {
+		const entries = await find_rerun_jobs(branch_name)
+
+		if (entries.length === 0 || (await should_merge_first(branch_name, entries))) return []
+
+		return await rerun(branch_name, entries)
+	} catch (error) {
+		console.info(`⚠️ CI job rerun failed: ${git_gh_helpers.get_error_message_with_stderr(error)}`)
+
+		return []
+	}
+}
+
+const followup_job_rerun = {
+	rerun_failed_jobs,
+}
+
+export { followup_job_rerun, COMPLETE_ATTEMPTS, REGISTER_ATTEMPTS }