@joshuafolkken/kit 1.3000.0 → 1.3041.0

package/scripts/observations/observation-ledger-archive.ts

@@ -0,0 +1,180 @@
+import { existsSync, readFileSync } from 'node:fs'
+import { appendFile, readdir, rm } from 'node:fs/promises'
+import path from 'node:path'
+import { git_gh_issue_alias_graphql } from '#scripts/gh/git-gh-issue-alias-graphql'
+import { error_text } from '#scripts/lib/error-message'
+import { z } from 'zod'
+import {
+	CARRIED_ROUNDS_SUFFIX,
+	LEDGER_FILE_EXTENSION,
+	OBSERVATION_LEDGER_DIRECTORY,
+} from './observation-ledger'
+import { observation_ledger_home } from './observation-ledger-home'
+
+// The ledger's retention: archive, never expire (`docs/maintainers/observation-ledger-rationale.md` →
+// "Retention policy", joshuafolkken/kit#7336). A closed issue's `<issue>.md` and
+// `<issue>-comment-rounds.md` move, verbatim and in name order, into `archive-YYYY-MM.md` for the month
+// the issue closed — the same directory, so `observation_ledger_home.read` still counts every line.
+//
+// **Only `observations:flush` runs it, on the default branch**: parallel lanes writing one archive
+// file would conflict. An issue GitHub did not answer as closed keeps its file until a later flush.
+//
+// **The carried mark survives the move**: the carry treats an issue whose rounds file exists as
+// carried, so each archived rounds file's issue is listed in `archived-comment-rounds.txt`, which the
+// carry reads too. It is not a `.md` file, so no reader takes it for ledger lines.
+
+const ARCHIVE_PREFIX = 'archive-'
+const ARCHIVED_ROUNDS_FILE = 'archived-comment-rounds.txt'
+const ISSUE_FILE_PATTERN = new RegExp(
+	String.raw`^(?<issue>\d+)(?<rounds>${CARRIED_ROUNDS_SUFFIX})?\.md$`,
+	'u',
+)
+const CLOSED_FIELDS = 'state closedAt'
+const CLOSED_STATE = 'CLOSED'
+const MONTH_END = 7
+const NONE = 0
+const ARCHIVED_LINE = "closed issues' ledger file(s) moved into the monthly archive files."
+const ARCHIVE_FAILED_LINE =
+	"Could not archive the closed issues' ledger files; the next flush retries:"
+
+const closed_schema = z.object({ state: z.string(), closedAt: z.string().nullish() })
+
+interface IssueFile {
+	name: string
+	issue: number
+	is_rounds: boolean
+}
+
+function issue_file_of(name: string): IssueFile | undefined {
+	const groups = ISSUE_FILE_PATTERN.exec(name)?.groups
+
+	if (groups?.['issue'] === undefined) return undefined
+
+	return { name, issue: Number(groups['issue']), is_rounds: groups['rounds'] !== undefined }
+}
+
+function issue_files(names: ReadonlyArray<string>): Array<IssueFile> {
+	return names
+		.toSorted((left, right) => left.localeCompare(right))
+		.flatMap((name) => issue_file_of(name) ?? [])
+}
+
+function archive_name(closed_at: string): string {
+	return `${ARCHIVE_PREFIX}${closed_at.slice(0, MONTH_END)}${LEDGER_FILE_EXTENSION}`
+}
+
+// The archive file of each closed issue; an open issue, or one GitHub did not answer, is absent.
+async function closed_archives(issues: ReadonlyArray<number>): Promise<Map<number, string>> {
+	const read = await git_gh_issue_alias_graphql.read_aliased(issues, CLOSED_FIELDS, closed_schema)
+
+	return new Map(
+		[...read.answers].flatMap(([issue, answer]) =>
+			answer.state === CLOSED_STATE && typeof answer.closedAt === 'string'
+				? [[issue, archive_name(answer.closedAt)] as const]
+				: [],
+		),
+	)
+}
+
+function by_archive(
+	files: ReadonlyArray<IssueFile>,
+	archives: ReadonlyMap<number, string>,
+): Map<string, Array<IssueFile>> {
+	const groups = new Map<string, Array<IssueFile>>()
+
+	for (const file of files) {
+		const target = archives.get(file.issue)
+
+		if (target !== undefined) groups.set(target, [...(groups.get(target) ?? []), file])
+	}
+
+	return groups
+}
+
+// Written before the sources are removed: a stop between the two leaves a duplicate in the working
+// tree, which `git checkout` undoes, rather than a lost line.
+async function move(
+	directory: string,
+	target: string,
+	files: ReadonlyArray<IssueFile>,
+): Promise<void> {
+	const contents = await Promise.all(
+		files.map(
+			async (file) =>
+				await observation_ledger_home.read_file(path.join(directory, file.name), true),
+		),
+	)
+
+	await appendFile(path.join(directory, target), contents.join(''), 'utf8')
+	await Promise.all(
+		files.map(async (file) => {
+			await rm(path.join(directory, file.name))
+		}),
+	)
+}
+
+function archived_rounds_path(root: string): string {
+	return path.join(root, OBSERVATION_LEDGER_DIRECTORY, ARCHIVED_ROUNDS_FILE)
+}
+
+async function record_rounds(root: string, files: ReadonlyArray<IssueFile>): Promise<void> {
+	const issues = files.filter((file) => file.is_rounds).map((file) => String(file.issue))
+
+	if (issues.length > NONE) {
+		await appendFile(archived_rounds_path(root), `${issues.join('\n')}\n`, 'utf8')
+	}
+}
+
+// The issues whose carried rounds file was archived — carried, so the carry never reads them again.
+function archived_rounds(root: string): ReadonlySet<number> {
+	const target = archived_rounds_path(root)
+
+	if (!existsSync(target)) return new Set()
+
+	return new Set(readFileSync(target, 'utf8').split('\n').filter(Boolean).map(Number))
+}
+
+// Moves every closed issue's ledger files into its month's archive, and answers how many it moved.
+async function archive(root: string): Promise<number> {
+	const directory = path.join(root, OBSERVATION_LEDGER_DIRECTORY)
+
+	if (!existsSync(directory)) return NONE
+
+	const files = issue_files(await readdir(directory))
+	const groups = by_archive(files, await closed_archives(files.map((file) => file.issue)))
+	const moved = [...groups.values()].flat()
+
+	await Promise.all(
+		[...groups].map(async ([name, group]) => {
+			await move(directory, name, group)
+		}),
+	)
+	await record_rounds(root, moved)
+
+	return moved.length
+}
+
+// **An archive that fails never fails the flush**: the files stay where they are and the next flush
+// moves them, so `0` is returned.
+async function archive_reported(root: string): Promise<number> {
+	try {
+		const count = await archive(root)
+
+		if (count > NONE) console.info(`Archived ${String(count)} ${ARCHIVED_LINE}`)
+
+		return count
+	} catch (error) {
+		console.warn(`${ARCHIVE_FAILED_LINE} ${error_text.message_of(error)}`)
+
+		return NONE
+	}
+}
+
+const observation_ledger_archive = {
+	ARCHIVE_FAILED_LINE,
+	archive,
+	archive_reported,
+	archived_rounds,
+}
+
+export { observation_ledger_archive }

package/scripts/observations/observation-ledger-home.ts

@@ -131,6 +131,7 @@ const observation_ledger_home = {
 	ledger_date,
 	ledger_root,
 	read,
+	read_file,
 	writer_path,
 }
 

package/scripts/observations/observation-ledger.ts

@@ -25,6 +25,10 @@ const LEDGER_DIRECTORIES: ReadonlyArray<string> = [
 
 const LEDGER_FILE_EXTENSION = '.md'
 
+// The suffix of a closed issue's carried comment rounds, `<issue>-comment-rounds.md` — named here
+// because both the carry that writes it and the archive that moves it read it.
+const CARRIED_ROUNDS_SUFFIX = '-comment-rounds'
+
 // What the single-file ledgers held is moved here (`observation-ledger-migrate.ts`), so no line is
 // lost to the split.
 const LEGACY_LEDGER_FILE = `legacy${LEDGER_FILE_EXTENSION}`
@@ -135,6 +139,7 @@ const observation_ledger = {
 }
 
 export {
+	CARRIED_ROUNDS_SUFFIX,
 	LEDGER_DIRECTORIES,
 	LEDGER_EXCLUSION_PATHSPEC,
 	LEDGER_FILE_EXTENSION,

package/scripts/observations/observations-flush-transaction.ts

@@ -0,0 +1,358 @@
+import { randomUUID } from 'node:crypto'
+import {
+	cpSync,
+	existsSync,
+	lstatSync,
+	mkdirSync,
+	readFileSync,
+	renameSync,
+	rmSync,
+	writeFileSync,
+} from 'node:fs'
+import path from 'node:path'
+import { git_command } from '#scripts/git/git-command'
+import { git_spawn } from '#scripts/git/git-spawn'
+import { git_spawn_sync } from '#scripts/git/git-spawn-sync'
+import { git_worktree } from '#scripts/git/git-worktree'
+import { repository_lock } from '#scripts/git/repository-lock'
+import { lane_install } from '#scripts/lane/lane-install'
+import { error_text } from '#scripts/lib/error-message'
+import { poll } from '#scripts/lib/poll'
+import { SUITE_TIMEOUT_MS } from '#scripts/lib/timeouts'
+import { z } from 'zod'
+import { observation_ledger, OBSERVATION_LEDGER_DIRECTORY } from './observation-ledger'
+
+const LOCK_WAIT_MS = 30_000
+const COMMIT_MESSAGE = 'Record observation ledger entries'
+const ATTEMPTS = 5
+const INTERVAL_MS = 250
+const snapshot_schema = z.object({
+	name: z.string(),
+	content: z.string().optional(),
+	base: z.string().optional(),
+})
+type SnapshotFile = z.infer<typeof snapshot_schema>
+const state_schema = z.object({
+	branch: z.string(),
+	base: z.string(),
+	tree: z.string(),
+	archived: z.number().optional(),
+	landed: z.boolean().optional(),
+	initialized: z.boolean().optional(),
+	committed: z.boolean().optional(),
+	installed: z.boolean().optional(),
+	index: z.string(),
+	files: z.array(snapshot_schema),
+	reconciled: z.array(snapshot_schema).optional(),
+})
+type State = z.infer<typeof state_schema>
+
+interface Steps {
+	prepare: () => Promise<number>
+	publish: (branch: string, archived: number) => Promise<void>
+}
+
+function lock_failure(error: unknown): undefined {
+	if (!/index\.lock.*(?:File exists|exists)/su.test(error_text.message_of(error))) throw error
+}
+
+async function retry(work: () => Promise<unknown>): Promise<void> {
+	const state: { failure?: unknown } = {}
+	const is_complete = await poll.poll_for(
+		async () => {
+			try {
+				await work()
+				delete state.failure
+			} catch (error) {
+				lock_failure(error)
+				state.failure = error
+			}
+
+			return state.failure === undefined ? true : undefined
+		},
+		{ attempts: ATTEMPTS, interval_ms: INTERVAL_MS },
+	)
+	if (is_complete === undefined) throw state.failure
+}
+
+function read(root: string, name: string): string | undefined {
+	const target = path.join(root, name)
+	if (!existsSync(target)) return undefined
+	if (!lstatSync(target).isFile()) throw new Error(`Not a regular ledger file: ${target}`)
+
+	return readFileSync(target, 'utf8')
+}
+
+function write(root: string, name: string, content: string | undefined): void {
+	const target = path.join(root, name)
+
+	if (content === undefined) {
+		rmSync(target, { force: true })
+
+		return
+	}
+
+	mkdirSync(path.dirname(target), { recursive: true })
+	writeFileSync(target, content, 'utf8')
+}
+
+async function capture(root: string): Promise<Array<SnapshotFile>> {
+	const output = await git_spawn.read([
+		'-C',
+		root,
+		'ls-files',
+		'-z',
+		'--cached',
+		'--others',
+		'--exclude-standard',
+	])
+	const names = new Set(
+		output.split('\0').filter((name) => observation_ledger.is_ledger_path(name)),
+	)
+
+	return [...names].map((name) => ({
+		name,
+		content: read(root, name),
+	}))
+}
+
+function apply(root: string, files: ReadonlyArray<SnapshotFile>): void {
+	for (const file of files) write(root, file.name, file.content)
+}
+
+function reconciled_content(file: SnapshotFile, suffix: string): string | undefined {
+	if (suffix === '' && file.base === undefined) return undefined
+
+	return (file.base ?? '') + suffix
+}
+
+function reconciled_file(root: string, file: SnapshotFile): SnapshotFile {
+	const current = read(root, file.name) ?? ''
+	const original = file.content ?? ''
+
+	if (!current.startsWith(original)) {
+		throw new Error(`Ledger changed beyond an append: ${file.name}; recovery retained`)
+	}
+
+	return { ...file, content: reconciled_content(file, current.slice(original.length)) }
+}
+
+function starts_with(current = '', expected = ''): boolean {
+	return current.startsWith(expected)
+}
+
+function restore_file(root: string, file: SnapshotFile, before: SnapshotFile | undefined): void {
+	const current = read(root, file.name)
+	if (current === file.content) return
+
+	if (before !== undefined && starts_with(current, before.content)) {
+		write(root, file.name, reconciled_file(root, before).content)
+
+		return
+	}
+
+	if (!starts_with(current, file.content)) throw new Error(`Recovery conflict: ${file.name}`)
+}
+
+function restore(
+	root: string,
+	before: ReadonlyArray<SnapshotFile>,
+	after: ReadonlyArray<SnapshotFile>,
+): void {
+	for (const file of after) {
+		restore_file(
+			root,
+			file,
+			before.find((entry) => entry.name === file.name),
+		)
+	}
+}
+
+function save(target: string, state: State): void {
+	writeFileSync(`${target}.tmp`, JSON.stringify(state), 'utf8')
+	renameSync(`${target}.tmp`, target)
+}
+
+async function location(root: string): Promise<string> {
+	const own = await git_spawn.read(['-C', root, 'rev-parse', '--absolute-git-dir'])
+	const directory = path.join(own, 'josh', 'observation-flush')
+
+	mkdirSync(directory, { recursive: true })
+
+	return path.join(directory, 'state.json')
+}
+
+async function create(root: string, branch: string, target: string): Promise<State> {
+	const existing = git_spawn_sync.read(
+		['show-ref', '--verify', `refs/heads/${branch}`],
+		git_spawn_sync.located(root),
+	)
+	const state = {
+		branch: existing === undefined ? branch : `${branch}-${randomUUID()}`,
+		base: await git_command.head_commit(),
+		index: await git_spawn.read(['-C', root, 'diff', '--cached', '--binary']),
+		tree: `${path.dirname(target)}/tree`,
+		files: await capture(root),
+	}
+
+	save(target, state)
+
+	return state
+}
+
+async function initialize(root: string, state: State, target: string): Promise<void> {
+	if (state.initialized === true) return
+	const existing = git_spawn_sync.read(
+		['show-ref', '--verify', `refs/heads/${state.branch}`],
+		git_spawn_sync.located(root),
+	)
+
+	if (!existsSync(state.tree)) {
+		await git_worktree.worktree_add(
+			state.tree,
+			state.branch,
+			existing === undefined ? state.base : undefined,
+		)
+	}
+
+	Object.assign(state, {
+		files: state.files.map((file) => ({ ...file, base: read(state.tree, file.name) })),
+		initialized: true,
+	})
+	save(target, state)
+}
+
+async function load(root: string, branch: string, target: string): Promise<State> {
+	const state = existsSync(target)
+		? state_schema.parse(JSON.parse(readFileSync(target, 'utf8')))
+		: await create(root, branch, target)
+
+	await initialize(root, state, target)
+
+	return state
+}
+
+async function pending(root: string): Promise<boolean> {
+	return existsSync(await location(root))
+}
+
+async function reset_preparation(state: State, target: string): Promise<void> {
+	const ledger = path.join(state.tree, OBSERVATION_LEDGER_DIRECTORY)
+
+	if (existsSync(ledger)) {
+		cpSync(ledger, `${path.dirname(target)}/attempt-${String(Date.now())}`, { recursive: true })
+	}
+
+	await git_spawn.read(['reset', '--hard', state.base])
+	await git_spawn.read(['clean', '-fd', '--', OBSERVATION_LEDGER_DIRECTORY])
+	apply(state.tree, state.files)
+}
+
+async function stage(): Promise<void> {
+	const status = await git_command.status()
+	const unstaged = status
+		.split('\n')
+		.filter((line) => line[1] !== ' ')
+		.join('\n')
+	const names = observation_ledger.ledger_paths(unstaged)
+
+	for (const name of names) {
+		// eslint-disable-next-line no-await-in-loop -- each git add exclusively owns this index
+		await retry(async () => await git_spawn.read(['add', '--', name]))
+	}
+}
+
+async function prepare(state: State, target: string, steps: Steps): Promise<void> {
+	if ((await git_command.head_commit()) !== state.base) return
+
+	if (state.archived === undefined) {
+		await reset_preparation(state, target)
+		Object.assign(state, { archived: await steps.prepare() })
+		save(target, state)
+	}
+
+	await stage()
+	if (!observation_ledger.has_pending_append(await git_command.status())) return
+	await retry(async () => await git_spawn.read(['commit', '-m', COMMIT_MESSAGE], SUITE_TIMEOUT_MS))
+}
+
+async function publish(state: State, target: string, steps: Steps): Promise<void> {
+	const is_committed = (await git_command.head_commit()) !== state.base
+
+	Object.assign(state, { committed: is_committed })
+	save(target, state)
+	if (is_committed) await steps.publish(state.branch, state.archived ?? 0)
+
+	Object.assign(state, { landed: true })
+	save(target, state)
+}
+
+async function install(state: State, target: string): Promise<void> {
+	if (state.installed === true) return
+	const result = await lane_install.install_dependencies(state.tree)
+
+	if (!result.is_installed) {
+		throw new Error(`Flush worktree dependency installation failed: ${result.output}`)
+	}
+
+	Object.assign(state, { installed: true })
+	save(target, state)
+}
+
+async function inside(state: State, target: string, steps: Steps): Promise<void> {
+	if (state.landed === true) return
+	const previous = process.cwd()
+
+	try {
+		process.chdir(state.tree)
+		await install(state, target)
+		await prepare(state, target, steps)
+		await publish(state, target, steps)
+	} finally {
+		process.chdir(previous)
+	}
+}
+
+async function finish(root: string, state: State, target: string): Promise<void> {
+	const index = await git_spawn.read(['-C', root, 'diff', '--cached', '--binary'])
+	if (index !== state.index) throw new Error('Source index changed during flush; recovery retained')
+	state.reconciled ??= state.files.map((file) => reconciled_file(root, file))
+	save(target, state)
+	restore(root, state.files, state.reconciled)
+	if (existsSync(state.tree)) await git_worktree.worktree_remove(state.tree)
+	renameSync(target, `${target}.completed-${path.basename(state.branch)}`)
+}
+
+async function transact(root: string, branch: string, steps: Steps): Promise<string> {
+	const target = await location(root)
+
+	try {
+		const state = await load(root, branch, target)
+
+		await inside(state, target, steps)
+		await finish(root, state, target)
+
+		return state.committed === true ? state.branch : ''
+	} catch (error) {
+		throw new Error(
+			`${error_text.message_of(error)} — observations recovery: ${target}; rerun observations:flush; do not remove the recovery tree`,
+			{ cause: error },
+		)
+	}
+}
+
+async function run(root: string, branch: string, steps: Steps): Promise<string> {
+	const lock = repository_lock.lock_path('josh-observation-flush-', root)
+	const result = await repository_lock.with_lock(
+		async () => await transact(root, branch, steps),
+		lock,
+		LOCK_WAIT_MS,
+	)
+	if (result === undefined) throw new Error('Observation flush is already running; retry later')
+
+	return result
+}
+
+const observations_flush_transaction = { pending, run }
+
+export { observations_flush_transaction }