@joshuafolkken/kit 1.1514.0 → 1.2114.0

package/scripts/issue/defect-report-cli.ts

@@ -0,0 +1,128 @@
+import { FULL_PAGE_QUERY, git_gh_api_path } from '#scripts/gh/git-gh-api-path'
+import { git_gh_command } from '#scripts/gh/git-gh-command'
+import { git_gh_exec } from '#scripts/gh/git-gh-exec'
+import { parse_json } from '#scripts/git/parse-json-array'
+import { bounded_pool } from '#scripts/lib/bounded-pool'
+import { z } from 'zod'
+import { defect_report, type OriginPr, type ReportIssue } from './defect-report'
+import { defect_report_format } from './defect-report-format'
+import { session_cite } from './session-cite'
+
+const SUCCESS_EXIT_CODE = 0
+const FAILURE_EXIT_CODE = 1
+const PR_CONCURRENCY = 4
+const USAGE = 'Usage: pnpm josh defect:report'
+const UNREADABLE_MESSAGE =
+	'Could not read the bug issues from GitHub, so no report is printed — check `gh auth status`.'
+
+const issue_schema = z.object({
+	number: z.number(),
+	created_at: z.string(),
+	body: z.string().nullish(),
+	pull_request: z.unknown().optional(),
+})
+const pull_schema = z.object({ merged_at: z.string().nullish() })
+const file_schema = z.object({
+	filename: z.string(),
+	status: z.string(),
+	additions: z.number(),
+	deletions: z.number(),
+})
+
+// Every issue labelled `bug`, open or closed — the label `issue:file` derives from the declaration.
+// The issues listing carries pull requests too, so those are dropped.
+async function read_issues(repo: string): Promise<ReadonlyArray<ReportIssue> | undefined> {
+	const raw = await git_gh_exec.exec_gh_api({
+		path: `repos/${repo}/issues?labels=bug&state=all&per_page=100`,
+		should_paginate: true,
+	})
+	const issues = parse_json.parse_json_object_or_undefined(raw, z.array(issue_schema))
+
+	return issues
+		?.filter((issue) => issue.pull_request === undefined)
+		.map((issue) => ({
+			number: issue.number,
+			created_at: issue.created_at,
+			body: issue.body ?? '',
+		}))
+}
+
+async function read_pr(repo: string, pr: number): Promise<OriginPr | undefined> {
+	const path = git_gh_api_path.pull_api_path(String(pr), repo)
+	const [pull_raw, files_raw] = await Promise.all([
+		git_gh_exec.exec_gh_api({ path }),
+		git_gh_exec.exec_gh_api({ path: `${path}/files${FULL_PAGE_QUERY}`, should_paginate: true }),
+	])
+	const pull = parse_json.parse_json_object_or_undefined(pull_raw, pull_schema)
+	const files = parse_json.parse_json_object_or_undefined(files_raw, z.array(file_schema))
+
+	if (pull === undefined || files === undefined) return undefined
+
+	return {
+		pr,
+		merged_at: pull.merged_at ?? '',
+		kind: defect_report.change_kind(files),
+		areas: defect_report.areas(files),
+	}
+}
+
+// A pull request that cannot be read contributes nothing, and the report counts it as unread.
+async function read_pr_or_none(repo: string, pr: number): Promise<ReadonlyArray<OriginPr>> {
+	try {
+		const origin = await read_pr(repo, pr)
+
+		return origin === undefined ? [] : [origin]
+	} catch {
+		return []
+	}
+}
+
+async function read_prs(repo: string, prs: ReadonlyArray<number>): Promise<Map<number, OriginPr>> {
+	const read = await bounded_pool.bounded_map(
+		prs,
+		PR_CONCURRENCY,
+		async (pr) => await read_pr_or_none(repo, pr),
+	)
+
+	return new Map(read.flat().map((origin) => [origin.pr, origin]))
+}
+
+function unreadable(): number {
+	console.error(UNREADABLE_MESSAGE)
+
+	return FAILURE_EXIT_CODE
+}
+
+async function report(repo: string): Promise<number> {
+	const issues = await read_issues(repo)
+
+	if (issues === undefined) return unreadable()
+
+	const prs = await read_prs(repo, defect_report.origin_prs(issues))
+
+	const aggregated = defect_report.aggregate(issues, prs)
+
+	console.info(session_cite.text(defect_report_format.format(aggregated)))
+
+	return SUCCESS_EXIT_CODE
+}
+
+async function run(argv: ReadonlyArray<string>): Promise<number> {
+	if (argv.length > 0) {
+		console.error(USAGE)
+
+		return FAILURE_EXIT_CODE
+	}
+
+	try {
+		const repo = await git_gh_command.repo_get_name_with_owner()
+
+		return repo === undefined ? unreadable() : await report(repo)
+	} catch {
+		return unreadable()
+	}
+}
+
+const defect_report_cli = { run }
+
+export { defect_report_cli }

package/scripts/issue/defect-report-format.ts

@@ -0,0 +1,56 @@
+import { defect_report, type DefectReport } from './defect-report'
+import { issue_cite } from './issue-cite'
+
+// The printed `defect:report`: each grouping as a table of counts, largest first.
+
+const INDENT = '  '
+const NONE = `${INDENT}(none)`
+
+function sorted_rows<K>(counts: ReadonlyMap<K, number>, label: (key: K) => string): Array<string> {
+	return [...counts]
+		.toSorted((left, right) => right[1] - left[1])
+		.map(([key, count]) => `${INDENT}${label(key)}  ${String(count)}`)
+}
+
+function section<K>(
+	title: string,
+	counts: ReadonlyMap<K, number>,
+	label: (key: K) => string,
+): Array<string> {
+	const rows = sorted_rows(counts, label)
+
+	return [title, ...(rows.length === 0 ? [NONE] : rows)]
+}
+
+function pr_label(pr: number): string {
+	return issue_cite.plain(pr)
+}
+
+function summary(report: DefectReport): string {
+	let traced = 0
+
+	for (const count of report.by_pr.values()) traced += count
+	const parts = [
+		`traced to a pull request: ${String(traced)}`,
+		`不明: ${String(report.unknown)}`,
+		`no origin line: ${String(report.undeclared)}`,
+	]
+
+	return `defects by origin — ${parts.join(' · ')}`
+}
+
+function format(report: DefectReport): string {
+	const week = `pull requests that added a mechanism — defects filed within ${String(defect_report.WEEK_DAYS)} days of the merge:`
+
+	return [
+		summary(report),
+		...section('by pull request:', report.by_pr, pr_label),
+		...section('by change kind (add / fix / remove):', report.by_kind, String),
+		...section('by area:', report.by_area, String),
+		...section(week, report.added_within_week, pr_label),
+	].join('\n')
+}
+
+const defect_report_format = { format }
+
+export { defect_report_format }

package/scripts/issue/defect-report-main.ts

@@ -0,0 +1,11 @@
+#!/usr/bin/env tsx
+import { fileURLToPath } from 'node:url'
+import { defect_report_cli } from './defect-report-cli'
+
+// The `josh defect:report` entry, kept apart from `defect-report-cli.ts` so a bundle carrying that
+// module never fires this guard (joshuafolkken/kit#4186).
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+	const ARGV_OFFSET = 2
+
+	process.exitCode = await defect_report_cli.run(process.argv.slice(ARGV_OFFSET))
+}

package/scripts/issue/defect-report.ts

@@ -0,0 +1,160 @@
+import { run_record_path } from '#scripts/josh/run-record-path'
+import { defect_origin_line } from './defect-origin-line'
+
+// Defects grouped by the pull request each one's origin line names: how many per pull request, per
+// change kind, and per area of the tree it touched, plus how many each pull request that added a
+// mechanism drew within a week of merging. Read from the bodies and the pull requests' files only,
+// so nothing here asks for a judgement.
+
+const WEEK_DAYS = 7
+const MS_PER_DAY = 86_400_000
+const WEEK_MS = WEEK_DAYS * MS_PER_DAY
+const SCRIPTS_ROOT = 'scripts'
+// `scripts/<dir>` — the two leading segments of a path under `scripts/`.
+const SCRIPT_AREA_DEPTH = 2
+const ROOT_AREA = '(root)'
+const UNREAD_AREA = '(pull request not read)'
+const TEST_FILE = /\.(?:test|e2e)\.ts$|\.snap$/u
+
+type ChangeKind = 'add' | 'fix' | 'remove'
+
+interface ReportIssue {
+	number: number
+	created_at: string
+	body: string
+}
+
+interface PrFile {
+	filename: string
+	status: string
+	additions: number
+	deletions: number
+}
+
+interface OriginPr {
+	pr: number
+	merged_at: string
+	kind: ChangeKind
+	areas: ReadonlyArray<string>
+}
+
+interface Attributed {
+	issue: ReportIssue
+	pr: number
+}
+
+interface DefectReport {
+	by_pr: ReadonlyMap<number, number>
+	by_kind: ReadonlyMap<string, number>
+	by_area: ReadonlyMap<string, number>
+	unknown: number
+	undeclared: number
+	added_within_week: ReadonlyMap<number, number>
+}
+
+// The run's own records, which nearly every merge adds, say nothing about what the pull request
+// changed.
+function changed(files: ReadonlyArray<PrFile>): ReadonlyArray<PrFile> {
+	return files.filter((file) => !run_record_path.is_run_record(file.filename))
+}
+
+// A pull request that brought in a new non-test file added a mechanism; one that took out more than it
+// put in removed; anything else changed what was there.
+function change_kind(files: ReadonlyArray<PrFile>): ChangeKind {
+	const source = changed(files).filter((file) => !TEST_FILE.test(file.filename))
+
+	if (source.some((file) => file.status === 'added')) return 'add'
+
+	const added = source.reduce((sum, file) => sum + file.additions, 0)
+	const removed = source.reduce((sum, file) => sum + file.deletions, 0)
+
+	return removed > added ? 'remove' : 'fix'
+}
+
+// `scripts/<dir>` for a script, the top-level directory otherwise, and one bucket for root files.
+function area_of(filename: string): string {
+	const segments = filename.split('/')
+
+	if (segments.length === 1) return ROOT_AREA
+
+	const is_script = segments[0] === SCRIPTS_ROOT && segments.length > SCRIPT_AREA_DEPTH
+	const depth = is_script ? SCRIPT_AREA_DEPTH : 1
+
+	return segments.slice(0, depth).join('/')
+}
+
+function areas(files: ReadonlyArray<PrFile>): ReadonlyArray<string> {
+	return [...new Set(changed(files).map((file) => area_of(file.filename)))]
+}
+
+function increment<K>(counts: Map<K, number>, key: K): void {
+	counts.set(key, (counts.get(key) ?? 0) + 1)
+}
+
+function count_by<K>(
+	items: ReadonlyArray<Attributed>,
+	keys: (item: Attributed) => ReadonlyArray<K>,
+): Map<K, number> {
+	const counts = new Map<K, number>()
+
+	for (const item of items) for (const key of keys(item)) increment(counts, key)
+
+	return counts
+}
+
+function within_week(issue: ReportIssue, origin: OriginPr): boolean {
+	const elapsed = Date.parse(issue.created_at) - Date.parse(origin.merged_at)
+
+	return elapsed >= 0 && elapsed <= WEEK_MS
+}
+
+function attribute(issue: ReportIssue): Attributed | undefined {
+	const origin = defect_origin_line.parse(issue.body)
+
+	return origin?.kind === 'pr' ? { issue, pr: origin.pr } : undefined
+}
+
+function attributed(issues: ReadonlyArray<ReportIssue>): ReadonlyArray<Attributed> {
+	return issues.map((issue) => attribute(issue)).filter((item) => item !== undefined)
+}
+
+// Every pull request an origin line names, once — what the report has to read.
+function origin_prs(issues: ReadonlyArray<ReportIssue>): ReadonlyArray<number> {
+	return [...new Set(attributed(issues).map(({ pr }) => pr))]
+}
+
+function soon_after_adding(
+	items: ReadonlyArray<Attributed>,
+	prs: ReadonlyMap<number, OriginPr>,
+): Map<number, number> {
+	return count_by(items, ({ issue, pr }) => {
+		const origin = prs.get(pr)
+
+		return origin?.kind === 'add' && within_week(issue, origin) ? [pr] : []
+	})
+}
+
+function count_origins(issues: ReadonlyArray<ReportIssue>, kind: 'unknown' | undefined): number {
+	return issues.filter((issue) => defect_origin_line.parse(issue.body)?.kind === kind).length
+}
+
+function aggregate(
+	issues: ReadonlyArray<ReportIssue>,
+	prs: ReadonlyMap<number, OriginPr>,
+): DefectReport {
+	const items = attributed(issues)
+
+	return {
+		by_pr: count_by(items, ({ pr }) => [pr]),
+		by_kind: count_by(items, ({ pr }) => [prs.get(pr)?.kind ?? UNREAD_AREA]),
+		by_area: count_by(items, ({ pr }) => prs.get(pr)?.areas ?? [UNREAD_AREA]),
+		unknown: count_origins(issues, 'unknown'),
+		undeclared: count_origins(issues, undefined),
+		added_within_week: soon_after_adding(items, prs),
+	}
+}
+
+const defect_report = { aggregate, area_of, areas, change_kind, origin_prs, WEEK_DAYS }
+
+export type { ChangeKind, DefectReport, OriginPr, PrFile, ReportIssue }
+export { defect_report }

package/scripts/issue/effect-audit-cli.ts

@@ -0,0 +1,178 @@
+#!/usr/bin/env tsx
+import { writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { git_gh_command } from '#scripts/gh/git-gh-command'
+import { bounded_pool } from '#scripts/lib/bounded-pool'
+import { cli_flags } from '#scripts/lib/cli-flags'
+import { defect_rate } from './defect-rate'
+import { effect_audit, type AuditRow, type FollowUp } from './effect-audit'
+import { issue_sample_prime } from './issue-sample-prime'
+import { issue_search, type SearchIssue } from './issue-search'
+import { session_cite } from './session-cite'
+
+// `josh effect:audit [--days <N>] [--draft <N>]` — every run-time-shortening issue closed in the window,
+// one line each: its expected effect, its follow-up measurements and their verdict (`effect-audit.ts`).
+// `--draft <N>` writes the retirement candidate for an issue whose measurement fell short and prints the
+// `issue:file` line that files it once its placeholders are filled.
+
+const ARGV_OFFSET = 2
+const FAILURE_EXIT_CODE = 1
+const DEFAULT_DAYS = 30
+const PULL_CONCURRENCY = 4
+const USAGE = 'Usage: josh effect:audit [--days <N>] [--draft <issue-number>]'
+const OPTIONS = { days: { type: 'string' }, draft: { type: 'string' } } as const
+
+interface AuditRequest {
+	days: number
+	draft: number | undefined
+}
+
+interface Audit {
+	rows: ReadonlyArray<AuditRow>
+	since: string
+	is_capped: boolean
+	unattributed: number
+}
+
+interface CitedFollowUp {
+	issue: SearchIssue
+	pr: number
+}
+
+function positive(text: string | undefined): number | undefined {
+	const value = Number(text)
+
+	return Number.isSafeInteger(value) && value > 0 ? value : undefined
+}
+
+function with_draft(days: number, text: string | undefined): AuditRequest | undefined {
+	if (text === undefined) return { days, draft: undefined }
+	const draft = positive(text)
+
+	return draft === undefined ? undefined : { days, draft }
+}
+
+function parse_request(argv: ReadonlyArray<string>): AuditRequest | undefined {
+	const parsed = cli_flags.values_of(argv, OPTIONS)
+	if (parsed === undefined) return undefined
+	const days = positive(parsed.days ?? String(DEFAULT_DAYS))
+
+	return days === undefined ? undefined : with_draft(days, parsed.draft)
+}
+
+// The issue the cited pull request closes; an unreadable pull request leaves the follow-up unattributed,
+// and the listing counts it.
+async function measured_issue(pr: number): Promise<number | undefined> {
+	try {
+		const pull = await issue_sample_prime.read_source(pr)
+
+		return pull.closes
+	} catch {
+		return undefined
+	}
+}
+
+// Each cited pull request is read once, a few at a time: the follow-up search spans every date.
+async function closes_by_pull(
+	prs: ReadonlyArray<number>,
+): Promise<ReadonlyMap<number, number | undefined>> {
+	const unique = [...new Set(prs)]
+	const closes = await bounded_pool.bounded_map(unique, PULL_CONCURRENCY, measured_issue)
+
+	return new Map(unique.map((pr, index) => [pr, closes[index]]))
+}
+
+// A follow-up citing the issue form names what it measures itself; only a cited pull request is read.
+async function follow_ups_of(candidates: ReadonlyArray<SearchIssue>): Promise<Array<FollowUp>> {
+	const cited = candidates
+		.map((issue) => ({ issue, pr: effect_audit.cited_pull(issue) }))
+		.filter((entry): entry is CitedFollowUp => entry.pr !== undefined)
+	const closes = await closes_by_pull(cited.map(({ pr }) => pr))
+	const by_issue = candidates
+		.map((issue) => ({ issue, measures: effect_audit.cited_issue(issue) }))
+		.filter(({ measures }) => measures !== undefined)
+
+	return [...cited.map(({ issue, pr }) => ({ issue, measures: closes.get(pr) })), ...by_issue]
+}
+
+async function read_audit(repo: string, days: number, now_ms: number): Promise<Audit | undefined> {
+	const since = defect_rate.window_start(now_ms, days)
+	const [origins, follow_ups] = await Promise.all([
+		issue_search.search(effect_audit.origin_query(repo, since)),
+		issue_search.search(effect_audit.follow_up_query(repo)),
+	])
+
+	if (origins === undefined || follow_ups === undefined) return undefined
+
+	const linked = await follow_ups_of(follow_ups.issues)
+	const rows = effect_audit.rows(origins.issues, linked)
+	const unattributed = linked.filter((follow_up) => follow_up.measures === undefined).length
+
+	return { rows, since, is_capped: origins.is_capped || follow_ups.is_capped, unattributed }
+}
+
+function print_audit(audit: Audit): number {
+	console.info(effect_audit.summary(audit.rows, audit.since))
+	for (const row of audit.rows) console.info(session_cite.text(effect_audit.format_row(row)))
+	if (audit.is_capped) console.info('(capped: the search served part of the window only)')
+
+	if (audit.unattributed > 0) {
+		console.info(
+			`(${String(audit.unattributed)} follow-ups measure no issue: their pull request could not be read or closes none)`,
+		)
+	}
+
+	return 0
+}
+
+function print_draft(rows: ReadonlyArray<AuditRow>, issue: number): number {
+	const row = rows.find((candidate) => candidate.origin.number === issue)
+
+	if (row?.result !== 'unmet') {
+		console.error(
+			`${session_cite.issue(issue)} is not an issue of the window whose measurement fell short`,
+		)
+
+		return FAILURE_EXIT_CODE
+	}
+
+	const file = path.join(tmpdir(), `josh-effect-retire-${String(issue)}.md`)
+
+	writeFileSync(file, effect_audit.draft_body(row))
+	console.info(`Fill in the <...> lines of ${file}, then file it:`)
+	console.info(
+		`pnpm josh issue:file "${effect_audit.draft_title(row)}" --body-file ${file} --label run:lane`,
+	)
+
+	return 0
+}
+
+async function audit_of(request: AuditRequest, now_ms: number): Promise<Audit | undefined> {
+	const repo = await git_gh_command.repo_get_name_with_owner()
+
+	return repo === undefined ? undefined : await read_audit(repo, request.days, now_ms)
+}
+
+async function run(argv: ReadonlyArray<string>, now_ms: number = Date.now()): Promise<number> {
+	const request = parse_request(argv)
+	if (request === undefined) return (console.error(USAGE), FAILURE_EXIT_CODE)
+	const audit = await audit_of(request, now_ms)
+
+	if (audit === undefined) {
+		return (console.error('effect:audit: the issue search could not be read'), FAILURE_EXIT_CODE)
+	}
+
+	return request.draft === undefined ? print_audit(audit) : print_draft(audit.rows, request.draft)
+}
+
+async function main(argv: ReadonlyArray<string>): Promise<void> {
+	process.exitCode = await run(argv)
+}
+
+const effect_audit_cli = { run }
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) await main(process.argv.slice(ARGV_OFFSET))
+
+export { effect_audit_cli }