@joshuafolkken/kit 1.1514.0 → 1.2114.0

package/scripts/followup/git-pr-followup.ts

@@ -1,7 +1,9 @@
 import { git_gh_command } from '#scripts/gh/git-gh-command'
 import { git_pr_ai_review, type TelegramContext } from '#scripts/gh/git-pr-ai-review'
+import type { PrStateSnapshot } from '#scripts/gh/git-pr-checks'
 import { git_pr_coderabbit } from '#scripts/gh/git-pr-coderabbit'
 import { git_pr_managed_config } from '#scripts/gh/git-pr-managed-config'
+import type { WaitForGreen } from '#scripts/gh/git-pr-merge-checked'
 import { github_issue_url } from '#scripts/gh/github-issue-url'
 import { git_closes_keyword } from '#scripts/git/git-closes-keyword'
 import type { GitNotifyConfig } from '#scripts/notify/git-notify'
@@ -11,6 +13,9 @@ import {
 	type TelegramTaskType,
 } from '#scripts/notify/telegram-notify'
 import { followup_checks_event } from './followup-checks-event'
+import { followup_ci_path } from './followup-ci-path'
+import { followup_coderabbit_retrigger, type Settled } from './followup-coderabbit-retrigger'
+import { followup_coderabbit_wait } from './followup-coderabbit-wait'
 import { git_followup_merged } from './git-followup-merged'
 import { git_followup_pending } from './git-followup-pending'
 import { git_followup_stages, type StageLog } from './git-followup-stages'
@@ -120,6 +125,13 @@ interface RunContext {
 	context: TelegramContext
 }
 
+type CompletionSend = () => Promise<void>
+
+interface PreparedNotify {
+	is_recorded: boolean
+	send: CompletionSend
+}
+
 // **The four reads in front of the check wait are issued together**. Not one
 // of them is a wait on anything: the `closes #N` check reads the pull request body, and the
 // notification needs the repository name, the issue title and the pull request URL — four requests
@@ -162,31 +174,33 @@ async function read_run_context(input: {
 // The pending count replaces the project version line: children no longer
 // bump, so the local `package.json` names the previous release rather than what this run ships.
 // A count that could not be read contributes no line at all, rather than a zero nobody measured.
-async function notify_completion(
+//
+// **The body is read here, the send is returned** (#5080): the send goes out beside the merge, but the
+// pending count is read off `origin`'s default branch, and a fetch that landed after the merge would
+// count this pull request under a line that says it is not counted.
+async function completion_sender(
 	context: TelegramContext,
 	skip_notes: ReadonlyArray<string>,
 	is_merge_pending: boolean,
-): Promise<void> {
+): Promise<CompletionSend> {
 	const pending_line = await git_followup_pending.pending_release_line({ is_merge_pending })
 	const lines = pending_line === undefined ? skip_notes : [pending_line, ...skip_notes]
-	const body = lines.join('\n')
+	const input = build_telegram_input({ task_type: 'completion', context, body: lines.join('\n') })
 
-	// **The tolerant send**: this runs on the way to the merge, so a Telegram
-	// gateway timeout must not leave a reviewed, green pull request unmerged. The failure is reported
-	// under `❗` and the run carries on.
+	// **The tolerant send**: this runs beside the merge, so a Telegram gateway timeout must not leave a
+	// reviewed, green pull request unmerged. The failure is reported under `❗` and the run carries on.
 	//
 	// **No recovery line**, deliberately, in the same sense `git_followup_cleanup`'s epic-close step
 	// has none: a completion notification is never re-sent by hand — `pnpm josh notify --task-type
 	// completion` is prohibited because it populates no PR link — and re-running `followup` after the
 	// merge is not a re-send either. Naming either would send the reader somewhere useless.
-	await telegram_notify.send_or_report(
-		build_telegram_input({
-			task_type: 'completion',
-			context,
-			body,
-		}),
-		undefined,
-	)
+	return async function send_completion(): Promise<void> {
+		await telegram_notify.send_or_report(input, undefined)
+	}
+}
+
+async function send_nothing(): Promise<void> {
+	// A completion already on record is not sent again.
 }
 
 // The two comment scans, marked separately from the check wait because they answer a different
@@ -226,50 +240,97 @@ async function run_comment_scans(
 // different destinations: every note reaches the completion notification, and this one alone also
 // reaches the completion report posted to the Issue. Returning it twice is
 // what lets `run_stages` hand each destination what belongs to it without re-deriving the answer.
+//
+// **The snapshot travels on to the merge**, which names its head, and the same wait is what the merge
+// goes back to when that head moved before it (#4036).
+interface ReviewChecks {
+	notes: Array<string>
+	managed: Array<string>
+	snapshot?: PrStateSnapshot
+}
+
+function wait_for_green_of(
+	input: FollowupInput,
+	context: TelegramContext,
+	issue_number: string | undefined,
+	log: StageLog,
+): WaitForGreen {
+	return async function wait_for_green(): Promise<PrStateSnapshot> {
+		return await run_checks({
+			branch_name: input.branch_name,
+			is_skip_watch: input.is_skip_watch,
+			observe: followup_coderabbit_wait.all(
+				followup_checks_event.observer_of(issue_number, context.pr_url),
+				followup_coderabbit_wait.observer_of(log.coderabbit),
+				followup_ci_path.observer_of(log.ci_path),
+			),
+		})
+	}
+}
+
+// **The `checks-wait` lap ends at the first green**, so the `ci-wait` row holds the checks alone; a
+// rate-limited CodeRabbit retriggered after it is a lap of its own (joshuafolkken/kit#5289).
+async function wait_for_review_green(
+	input: FollowupInput,
+	context: TelegramContext,
+	issue_number: string | undefined,
+	log: StageLog,
+): Promise<Settled<PrStateSnapshot>> {
+	const wait_for_green = wait_for_green_of(input, context, issue_number, log)
+	const checked = await wait_for_green()
+
+	lap(log, STAGE.checks_wait)
+	const settled = await followup_coderabbit_retrigger.settle(
+		checked,
+		wait_for_green,
+		input.branch_name,
+		log.coderabbit,
+	)
+	if (settled.is_retriggered) lap(log, STAGE.coderabbit_retrigger)
+
+	return settled
+}
+
 async function run_review_checks(
 	input: FollowupInput,
 	context: TelegramContext,
 	log: StageLog,
 	issue_number: string | undefined,
-): Promise<{ notes: Array<string>; managed: Array<string> }> {
+): Promise<ReviewChecks> {
 	if (input.is_merged === true) return { notes: [], managed: [] }
 
+	git_followup_stages.hold_issue(log, issue_number)
 	const managed = await git_pr_managed_config.handle_managed_config_changes({
 		should_merge: input.should_merge,
 	})
-	const snapshot = await run_checks({
-		branch_name: input.branch_name,
-		is_skip_watch: input.is_skip_watch,
-		observe: followup_checks_event.observer_of(issue_number, context.pr_url),
-	})
-
-	lap(log, STAGE.checks_wait)
-	const check_notes = read_coderabbit_skip_notes(snapshot)
+	const { snapshot, notes } = await wait_for_review_green(input, context, issue_number, log)
+	const check_notes = [...notes, ...read_coderabbit_skip_notes(snapshot)]
 
 	log_skip_notes(check_notes)
 	const scan_notes = await run_comment_scans(input, context, log)
 
-	return { notes: [...managed, ...check_notes, ...scan_notes], managed }
+	return { notes: [...managed, ...check_notes, ...scan_notes], managed, snapshot }
 }
 
 // **A tail re-run after a merge does not report twice**: where the issue
 // already carries this pull request's completion report, the Telegram is skipped here and the comment
-// in the wrapup. Answers whether it was recorded, so the wrapup skips on the same read.
-async function notify_unless_recorded(
+// in the wrapup. Answers whether it was recorded, so the wrapup skips on the same read, and the send
+// the wrapup issues beside the merge.
+async function prepare_notify(
 	input: FollowupInput,
 	issue_number: string | undefined,
 	context: TelegramContext,
 	checks: { notes: Array<string> },
-): Promise<boolean> {
+): Promise<PreparedNotify> {
 	const is_recorded = await git_followup_merged.is_completion_recorded({
 		merged_at: input.is_merged === true ? input.merged_at : undefined,
 		issue_number,
 		pr_url: context.pr_url,
 	})
 
-	if (!is_recorded) await notify_completion(context, checks.notes, input.should_merge)
+	if (is_recorded) return { is_recorded, send: send_nothing }
 
-	return is_recorded
+	return { is_recorded, send: await completion_sender(context, checks.notes, input.should_merge) }
 }
 
 // **Answers with the issue number the run actually used**, which is the one
@@ -282,9 +343,8 @@ async function run_stages(input: FollowupInput, log: StageLog): Promise<string |
 
 	lap(log, STAGE.closes_and_context)
 	const checks = await run_review_checks(input, context, log, issue_number)
-	const is_completion_recorded = await notify_unless_recorded(input, issue_number, context, checks)
+	const notify = await prepare_notify(input, issue_number, context, checks)
 
-	lap(log, STAGE.telegram)
 	await git_pr_followup_wrapup.run_wrapup(
 		{
 			branch_name: input.branch_name,
@@ -295,7 +355,10 @@ async function run_stages(input: FollowupInput, log: StageLog): Promise<string |
 			should_merge: input.should_merge,
 			managed_notes: checks.managed,
 			is_merged: input.is_merged,
-			is_completion_recorded,
+			is_completion_recorded: notify.is_recorded,
+			send_completion: notify.send,
+			checked: checks.snapshot,
+			wait_for_green: wait_for_green_of(input, context, issue_number, log),
 		},
 		log,
 	)
@@ -319,7 +382,7 @@ async function run(input: FollowupInput): Promise<string | undefined> {
 		throw error
 	} finally {
 		git_followup_stages.print_stages(log)
-		await git_followup_stages.record_ci_wait(log)
+		await git_followup_stages.record_waits(log)
 	}
 }
 

package/scripts/fresh-machine/fresh-machine-cli.ts

@@ -24,7 +24,7 @@ const USAGE =
 	'Usage: josh fresh:run <pattern> | --spec <file.json> [--package-version <version>] [--log-dir <dir>] [--rebuild] | --list'
 // The scratch directory kit's distributed `.gitignore` already ignores, so a log is never committed
 // from kit or from a consumer toolkit.
-const DEFAULT_LOG_DIRECTORY = '.claude/tmp/fresh-machine'
+const DEFAULT_LOG_DIRECTORY = '.josh/tmp/fresh-machine'
 const NO_GITHUB_TOKEN =
 	'This scenario uses GitHub and `gh auth token` printed no token. Run `gh auth login` on this machine first.'
 const NO_CLAUDE_TOKEN = `This scenario uses Claude Code and neither the macOS keychain nor ${claude_token.VARIABLE} holds a token. ${claude_token.SETUP_HINT}`

package/scripts/fresh-machine/fresh-machine-run.ts

@@ -4,13 +4,14 @@ import {
 	mkdirSync,
 	mkdtempSync,
 	readdirSync,
-	readFileSync,
 	rmSync,
 	writeFileSync,
 } from 'node:fs'
 import os from 'node:os'
 import path from 'node:path'
 import { PROJECT_ROOT } from '#scripts/init/init-paths'
+import { package_manifest } from '#scripts/lib/package-manifest'
+import { npm_registry } from '#scripts/version/npm-registry'
 import { execa, type Options } from 'execa'
 import { z } from 'zod'
 import { fresh_machine_docker, type ContainerRun } from './fresh-machine-docker'
@@ -133,19 +134,18 @@ async function pack(pack_directory: string, write: Write): Promise<string> {
 }
 
 function package_name(): string {
-	const manifest = readFileSync(path.join(PROJECT_ROOT, 'package.json'), 'utf8')
-
-	return MANIFEST_SCHEMA.parse(JSON.parse(manifest)).name
+	return package_manifest.read_manifest(PROJECT_ROOT, MANIFEST_SCHEMA).name
 }
 
 // The published version is always written out in the log and the install command, so a run can be
 // repeated against the same release; without `--package-version` it is the registry's latest.
-async function published_version(requested: string | undefined): Promise<string> {
+function published_version(requested: string | undefined): string {
 	if (requested !== undefined) return requested
 
-	const { stdout } = await execa(PNPM, ['view', package_name(), 'version'], { cwd: PROJECT_ROOT })
+	const answer = npm_registry.view(package_name(), 'version')
+	if (answer.exit_code !== 0) throw new Error('the registry did not answer the latest version')
 
-	return stdout.trim()
+	return answer.stdout.trim()
 }
 
 async function container_run(
@@ -163,7 +163,7 @@ async function container_run(
 		seed_directory: staged.seed_directory,
 		pack_directory: staged.pack_directory,
 		tarball: is_tarball ? await pack(staged.pack_directory, write) : undefined,
-		package_version: is_tarball ? undefined : await published_version(request.package_version),
+		package_version: is_tarball ? undefined : published_version(request.package_version),
 		secret_names: Object.keys(request.secrets),
 	}
 }

package/scripts/fresh-machine/fresh-machine-script.ts

@@ -1,3 +1,4 @@
+import { shell_quote } from '#scripts/lib/shell-quote'
 import type { Scenario } from './fresh-machine-spec'
 
 // The script one container runs. Every command is printed before it runs and its exit status after,
@@ -8,28 +9,23 @@ const SEED_MOUNT = '/seed'
 const PACK_MOUNT = '/pack'
 const COPY_SEED = `cp -R ${SEED_MOUNT}/. .`
 const PROBES_HEADING = '# probes — recorded, never failing the run'
-const ESCAPED_QUOTE = String.raw`'\''`
 const PREAMBLE = [
 	String.raw`step() { printf '\n$ %s\n' "$1"; eval "$1"; local code=$?; printf '[exit %s]\n' "$code"; return "$code"; }`,
 	'mkdir -p "$HOME/$RUN_NAME" && cd "$HOME/$RUN_NAME" || exit 1',
 ]
 
-function quote(command: string): string {
-	const escaped = command.split("'").join(ESCAPED_QUOTE)
-
-	return `'${escaped}'`
-}
+const { single_quoted } = shell_quote
 
 function required(command: string): string {
-	return `step ${quote(command)} || exit $?`
+	return `step ${single_quoted(command)} || exit $?`
 }
 
 function probe_lines(probes: ReadonlyArray<string>): Array<string> {
 	if (probes.length === 0) return []
 
 	return [
-		String.raw`printf '\n%s\n' ${quote(PROBES_HEADING)}`,
-		...probes.map((probe) => `step ${quote(probe)}`),
+		String.raw`printf '\n%s\n' ${single_quoted(PROBES_HEADING)}`,
+		...probes.map((probe) => `step ${single_quoted(probe)}`),
 	]
 }
 

package/scripts/fresh-machine/specs/kit-test-backlogrun-python.json

@@ -27,8 +27,8 @@
 		"tail -n 40 \"$(sed -n 's/^backlogrun: started.*; log //p' \"$HOME/backlogrun.out\")\" | cut -c1-400",
 		"grep -c 'requires approval' \"$(sed -n 's/^backlogrun: started.*; log //p' \"$HOME/backlogrun.out\")\"",
 		"grep -c 'run:carry --begin' \"$(sed -n 's/^backlogrun: started.*; log //p' \"$HOME/backlogrun.out\")\"",
-		"tail -n 40 /tmp/josh-lane-dispatch-*.log | cut -c1-400",
-		"cat /tmp/josh-lane-dispatch-*.log | grep -c 'requires approval'",
+		"tail -n 40 \"$(git rev-parse --path-format=absolute --git-common-dir)\"/josh/run-state/josh-lane-dispatch-*.log | cut -c1-400",
+		"cat \"$(git rev-parse --path-format=absolute --git-common-dir)\"/josh/run-state/josh-lane-dispatch-*.log | grep -c 'requires approval'",
 		"gh pr list --state all"
 	]
 }

package/scripts/fresh-machine/specs/kit-test-backlogrun.json

@@ -12,7 +12,7 @@
 		"printf '{ \"private\": true }\\n' > package.json",
 		"cp \"$PACKAGE_TARBALL\" . && pnpm add -D --allow-build=esbuild --allow-build=unrs-resolver \"./$(basename \"$PACKAGE_TARBALL\")\"",
 		"pnpm exec josh start --yes --github",
-		"gh label list --limit 100 --json name --jq '.[].name' > \"$HOME/labels.txt\"; grep -xE 'auto-ok|needs-decision|needs-human-review|already-done|review-round2-skipped|bug' \"$HOME/labels.txt\"; test \"$(grep -cxE 'auto-ok|needs-decision|needs-human-review|already-done|review-round2-skipped|bug' \"$HOME/labels.txt\")\" -eq 6",
+		"gh label list --limit 100 --json name --jq '.[].name' > \"$HOME/labels.txt\"; grep -xE 'auto-ok|needs-decision|needs-human-review|already-done|bug' \"$HOME/labels.txt\"; test \"$(grep -cxE 'auto-ok|needs-decision|needs-human-review|already-done|bug' \"$HOME/labels.txt\")\" -eq 5",
 		"curl -fsSL https://claude.ai/install.sh | bash && export PATH=\"$HOME/.local/bin:$PATH\" && claude --version",
 		"gh issue create --title 'Add a heading to index.html' --body 'Add an h2 under the h1.' --label auto-ok --label run:lane",
 		"timeout 60 pnpm josh backlogrun > \"$HOME/backlogrun.out\" 2>&1; cat \"$HOME/backlogrun.out\"; grep -q '^backlogrun: started' \"$HOME/backlogrun.out\"",
@@ -24,8 +24,8 @@
 	"probes": [
 		"tail -n 40 \"$(sed -n 's/^backlogrun: started.*; log //p' \"$HOME/backlogrun.out\")\" | cut -c1-400",
 		"grep -c 'requires approval' \"$(sed -n 's/^backlogrun: started.*; log //p' \"$HOME/backlogrun.out\")\"",
-		"tail -n 40 /tmp/josh-lane-dispatch-*.log | cut -c1-400",
-		"cat /tmp/josh-lane-dispatch-*.log | grep -c 'requires approval'",
+		"tail -n 40 \"$(git rev-parse --path-format=absolute --git-common-dir)\"/josh/run-state/josh-lane-dispatch-*.log | cut -c1-400",
+		"cat \"$(git rev-parse --path-format=absolute --git-common-dir)\"/josh/run-state/josh-lane-dispatch-*.log | grep -c 'requires approval'",
 		"gh pr list --state all"
 	]
 }

package/scripts/gate/busy-average.ts

@@ -0,0 +1,90 @@
+import type { MachineReading } from './machine-capacity'
+
+// **The busy cores a waiting claim admits against are the mean of its last few seconds of readings**
+// (joshuafolkken/kit#4238). One reading covers a 250 ms window, and a CPU near saturation swings across
+// the threshold from one window to the next, so a claim judged on each reading alone would be admitted
+// on a dip into load that is still there. A claim's readings are its own: each poll of one wait adds
+// one, and its first poll judges on the one it has.
+//
+// **What is averaged is the load outside the ledger, each sample against the ledger of its own
+// moment.** Averaging the raw busy cores and subtracting the ledger of now would count a claim released
+// inside the window as external load: its cores would linger in the mean after leaving the ledger.
+
+// Long enough to hold about ten polls of a waiting claim, short enough that a CPU freed a moment ago is
+// admitted against within seconds.
+const AVERAGE_WINDOW_MS = 3000
+
+interface BusySample {
+	at: number
+	external_cores: number
+}
+
+// The samples still inside the window once `reading` taken at `now`, against a ledger holding
+// `ledger_cores`, is added. An unread reading adds nothing: where the machine cannot be read, nothing
+// is averaged.
+function with_reading(
+	samples: ReadonlyArray<BusySample>,
+	reading: MachineReading,
+	ledger_cores: number,
+	now: number,
+): Array<BusySample> {
+	const kept = samples.filter((sample) => now - sample.at < AVERAGE_WINDOW_MS)
+	const { busy_cores } = reading
+
+	if (busy_cores === undefined) return kept
+
+	return [...kept, { at: now, external_cores: Math.max(0, busy_cores - ledger_cores) }]
+}
+
+// `reading` with its busy cores replaced by the mean load outside the ledger on top of the ledger's
+// `ledger_cores` now, or unchanged when none were read.
+function averaged(
+	reading: MachineReading,
+	samples: ReadonlyArray<BusySample>,
+	ledger_cores: number,
+): MachineReading {
+	if (reading.busy_cores === undefined || samples.length === 0) return reading
+
+	let total = 0
+
+	for (const sample of samples) total += sample.external_cores
+
+	return { ...reading, busy_cores: ledger_cores + total / samples.length }
+}
+
+// **A wait that does not ease the load gains nothing by going on** (joshuafolkken/kit#4982). On a
+// saturated machine the load never falls, and a claim that waits out the cap only starts at minimum
+// width two minutes late. A wait keeps the lowest mean busy count it has read and when it last fell; a
+// fall under a whole core is noise, not easing.
+const EASE_CORES = 1
+
+interface LoadEase {
+	level: number
+	eased_at: number
+}
+
+// `ease` once the mean `busy_cores` read at `now` is added. An unread machine changes nothing, so a wait
+// that cannot read the load is never judged unrelieved.
+function with_ease(
+	ease: LoadEase | undefined,
+	busy_cores: number | undefined,
+	now: number,
+): LoadEase | undefined {
+	if (busy_cores === undefined) return ease
+
+	if (ease === undefined || busy_cores <= ease.level - EASE_CORES) {
+		return { level: busy_cores, eased_at: now }
+	}
+
+	return ease
+}
+
+// Whether the load has not eased for `window_ms` of the wait.
+function is_unrelieved(ease: LoadEase | undefined, now: number, window_ms: number): boolean {
+	return ease !== undefined && now - ease.eased_at >= window_ms
+}
+
+const busy_average = { AVERAGE_WINDOW_MS, averaged, is_unrelieved, with_ease, with_reading }
+
+export type { BusySample, LoadEase }
+export { busy_average }

package/scripts/gate/check-cpu.ts

@@ -0,0 +1,60 @@
+import {
+	run_board_usage_rusage,
+	type RusageReader,
+} from '#scripts/run/board/run-board-usage-rusage'
+
+// **The CPU a check's process tree spent, read off the gate's own reaped-children time**
+// (joshuafolkken/kit#4362). Each check is a `pnpm josh <target>` child that waits for its tool before it
+// exits, so once the gate has reaped it the whole tree's CPU sits in the gate's `proc_pid_rusage`
+// child times (`run-board-usage-rusage.ts`, shared rather than read twice). The reader folds the
+// gate's own time in, so `process.cpuUsage()` takes it back out, and each finished check takes the
+// growth since the previous one finished.
+//
+// **A reading is an attribution, not an isolation**: any other child the gate reaped in between — a
+// `ps` sample, a git call — lands on the next check that finishes, and two checks reaped in one tick
+// split their CPU unevenly. Both are small beside a check's tools. Off macOS, or without `node:ffi`,
+// there is no reader and every check records none.
+
+const US_PER_MS = 1000
+
+// The CPU milliseconds spent by the children reaped since the previous call, `undefined` where the
+// gate's own usage cannot be read.
+type CheckCpu = () => number | undefined
+
+function own_cpu_ms(): number {
+	const usage = process.cpuUsage()
+
+	return (usage.user + usage.system) / US_PER_MS
+}
+
+function reaped_cpu_ms(read: RusageReader | undefined, pid: number): number | undefined {
+	const reading = read?.(pid)
+
+	return reading === undefined ? undefined : reading.cpu_ms - own_cpu_ms()
+}
+
+// One gate's reader, its baseline taken now — before any check is reaped.
+function check_cpu_reader(
+	read: RusageReader | undefined = run_board_usage_rusage.rusage_reader(
+		process.platform,
+		run_board_usage_rusage.load_builtin,
+	),
+	pid: number = process.pid,
+): CheckCpu {
+	const previous = { reaped_ms: reaped_cpu_ms(read, pid) }
+
+	return function since_previous(): number | undefined {
+		const reaped_ms = reaped_cpu_ms(read, pid)
+		const before = previous.reaped_ms
+
+		previous.reaped_ms = reaped_ms
+		if (reaped_ms === undefined || before === undefined) return undefined
+
+		return Math.max(0, reaped_ms - before)
+	}
+}
+
+const check_cpu = { check_cpu_reader }
+
+export type { CheckCpu }
+export { check_cpu }