@joshuafolkken/kit 1.872.0

package/scripts/document/document-reachability.ts

@@ -0,0 +1,131 @@
+// Where each agent-read document sits on the execution path, derived from `read:set` — never
+// hand-listed (joshuafolkken/kit#2257).
+//
+// **The per-document byte ceiling could not tell an entry read from a reference document.** A
+// document a run reads on every entry costs it every session; one nothing routes to costs it nothing.
+// The per-document ceiling held both to the same rule, so a reference document a human browses was
+// budgeted as tightly as the workflow text a run actually reads. This splits the corpus so the
+// primary budget (`entry-read-budget.ts`) can be the total each entry reads, and the per-document
+// ceiling falls back to the documents no entry reaches.
+//
+// **The split is derived, not transcribed.** `read:set` already computes, per entry, the files it
+// reads and the point-of-use documents it reaches — `covered` is that union, plus the resident
+// `CLAUDE.md` every entry loads. A document `read:set` never enumerates is `unreached`, and a
+// hand-written table of which is which would be the clone `CLAUDE.md` prohibits, drifting the first
+// time a trigger table row moved. `CLAUDE.md` is resident by construction rather than by the graph,
+// because it is loaded on every request whatever the entry — the misclassification joshuafolkken/kit#2257
+// singled out (a bare reachability walk drops it to "off-path", which is wrong).
+//
+// **The corpus is passed in, never imported here.** This module sits on the runtime path
+// (`lint-related` → `document-byte-check` → here), so it may not import `ai-document-fixture` — a test
+// fixture the published package excludes (`skill-meta.ts`). `covered_documents` needs no corpus at all
+// (it is resident ∪ point-of-use, both from `read:set`); the classification helpers that do enumerate
+// the whole corpus take it as an argument, and their only callers are tests, which own the fixture.
+
+import { entry_read_set } from './entry-read-set'
+
+// The always-resident rule document. It is not an entry read — every request carries it — so it is
+// named here rather than derived from a trigger table, and the entry-total budget adds it to every
+// entry's total so its growth is held (joshuafolkken/kit#2257).
+const RESIDENT_BASE: ReadonlyArray<string> = ['CLAUDE.md']
+const WORKFLOW_DIRECTORY = '.claude/skills/workflow-commands'
+
+type Reachability = 'resident' | 'point-of-use' | 'unreached'
+
+// The trigger table names the workflow documents by base name (`fullrun.md`, `SKILL.md`); they all
+// live in the workflow-commands skill, so the corpus path is that directory plus the name.
+function workflow_path(name: string): string {
+	return `${WORKFLOW_DIRECTORY}/${name}`
+}
+
+// Every base name `read:set` reads at an entry — the files across all trigger-table entries and the
+// referenced sections those files point out to. Point-of-use names are handled on their own below.
+function names_of(set: ReturnType<typeof entry_read_set.read_set>): Array<string> {
+	return [...set.files, ...set.sections.map((reference) => reference.file)]
+}
+
+function entry_read_names(root: string): Set<string> {
+	const names = entry_read_set
+		.entries(root)
+		.flatMap((entry) => names_of(entry_read_set.read_set(root, entry)))
+
+	return new Set(names)
+}
+
+function resident_documents(root: string): Set<string> {
+	return new Set([
+		...RESIDENT_BASE,
+		...[...entry_read_names(root)].map((name) => workflow_path(name)),
+	])
+}
+
+function point_of_use_documents(): Set<string> {
+	const per_entry = [...entry_read_set.POINT_OF_USE_BY_ENTRY.values()].flatMap((files) => [
+		...files,
+	])
+
+	return new Set(
+		[...entry_read_set.POINT_OF_USE_FILES, ...per_entry].map((name) => workflow_path(name)),
+	)
+}
+
+function label_of(
+	relative_path: string,
+	resident: ReadonlySet<string>,
+	point_of_use: ReadonlySet<string>,
+): Reachability {
+	if (resident.has(relative_path)) return 'resident'
+
+	return point_of_use.has(relative_path) ? 'point-of-use' : 'unreached'
+}
+
+// Every document in the given corpus mapped to where it sits on the execution path. The corpus is
+// passed in (the caller owns `agent_read_documents`) so this runtime module carries no fixture import.
+function classify(
+	corpus: ReadonlyArray<string>,
+	root: string = process.cwd(),
+): Map<string, Reachability> {
+	const resident = resident_documents(root)
+	const point_of_use = point_of_use_documents()
+
+	return new Map(corpus.map((one) => [one, label_of(one, resident, point_of_use)]))
+}
+
+function documents_labelled(
+	label: Reachability,
+	corpus: ReadonlyArray<string>,
+	root: string = process.cwd(),
+): Array<string> {
+	return [...classify(corpus, root)]
+		.filter(([, value]) => value === label)
+		.map(([one]) => one)
+		.toSorted((left, right) => left.localeCompare(right))
+}
+
+// The documents an entry reads — resident plus point-of-use. The entry-total budget governs these;
+// the per-document ceiling falls back to the unreached rest, so neither holds a document the other
+// does (joshuafolkken/kit#2257). Both sides come from `read:set`, so no corpus is needed — which is
+// what keeps the runtime caller (`document-byte-check`) free of the fixture the corpus lives in.
+function covered_documents(root: string = process.cwd()): Array<string> {
+	return [...new Set([...resident_documents(root), ...point_of_use_documents()])].toSorted(
+		(left, right) => left.localeCompare(right),
+	)
+}
+
+function unreached_documents(
+	corpus: ReadonlyArray<string>,
+	root: string = process.cwd(),
+): Array<string> {
+	return documents_labelled('unreached', corpus, root)
+}
+
+const document_reachability = {
+	RESIDENT_BASE,
+	classify,
+	covered_documents,
+	documents_labelled,
+	unreached_documents,
+}
+
+export type { Reachability }
+export { document_reachability }

package/scripts/document/document-read-cli.ts

@@ -0,0 +1,82 @@
+#!/usr/bin/env tsx
+import { fileURLToPath } from 'node:url'
+import { bash_output_cap_reader } from './bash-output-cap'
+import { document_section } from './document-section'
+import { document_section_cli } from './document-section-cli'
+
+// `josh doc:read <file>` — a Bash-cap-safe read path for a whole document (joshuafolkken/kit#2188).
+//
+// **A `cat` of a document larger than the Bash output cap is a truncated preview, then a second read.**
+// `read:set` measured one entry paying for the same five files twice this way (joshuafolkken/kit#1797):
+// the shell hands back a middle-truncated preview, so the file is read again with the `Read` tool. This
+// command is the path that avoids it — it never emits an over-cap document through the shell at all.
+//
+// **Under the cap it prints the document; over it, it prints a directive and no content.** A document
+// that fits is delivered here in one call. One that does not is *not* printed — a truncated preview is
+// the failure this exists to refuse — so what comes back is one line naming the file and the `Read`
+// tool, which reads it once. `doc:section` remains the way to fetch a single heading; this is for the
+// whole file when a heading will not do.
+
+const ARGV_OFFSET = 2
+const FAILURE_EXIT_CODE = 1
+const FILE_ARGUMENT = 0
+const USAGE = 'Usage: josh doc:read <file.md>'
+const OVER_CAP_PREFIX = 'Over the Bash cap'
+const READ_TOOL_DIRECTIVE = 'read it with the Read tool, not cat:'
+
+function byte_length(text: string): number {
+	return Buffer.byteLength(text, 'utf8')
+}
+
+// The one line that stands in for an over-cap document. It names both figures so the reason is legible
+// and the path so the `Read` tool has its argument, and it carries no content of the file.
+function over_cap_directive(file_path: string, bytes: number, cap: number): string {
+	const figures = `(${bytes.toLocaleString('en-US')} B > ${cap.toLocaleString('en-US')} B)`
+
+	return `${OVER_CAP_PREFIX} ${figures} — ${READ_TOOL_DIRECTIVE} ${file_path}`
+}
+
+function print_document(file_path: string, text: string, cap: number): number {
+	if (byte_length(text) > cap) console.info(over_cap_directive(file_path, byte_length(text), cap))
+	else console.info(text)
+
+	return 0
+}
+
+function run(argv: ReadonlyArray<string>, root: string = process.cwd()): number {
+	const name = argv[FILE_ARGUMENT]
+
+	if (name === undefined) {
+		console.error(USAGE)
+
+		return FAILURE_EXIT_CODE
+	}
+
+	const file_path = document_section_cli.resolve_document(name, root)
+	const text = document_section.read_optional(file_path)
+
+	if (text === undefined) {
+		console.error(`Cannot read ${file_path}`)
+
+		return FAILURE_EXIT_CODE
+	}
+
+	return print_document(file_path, text, bash_output_cap_reader.bash_output_cap(root))
+}
+
+function main(argv: ReadonlyArray<string>): void {
+	process.exitCode = run(argv)
+}
+
+const document_read_cli = {
+	OVER_CAP_PREFIX,
+	READ_TOOL_DIRECTIVE,
+	USAGE,
+	main,
+	over_cap_directive,
+	run,
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) main(process.argv.slice(ARGV_OFFSET))
+
+export { document_read_cli }

package/scripts/document/document-scan.ts

@@ -0,0 +1,108 @@
+// Pulling machine-checkable references out of the distributed documents, so a few structural scans
+// can replace the ~140 per-phrase marker suites the documents used to carry (joshuafolkken/kit#1923).
+// Every function here is a pure string extractor: it takes document text and returns the references
+// it names, which is what lets the scans that call it be checked on a synthetic string with a
+// deliberately broken reference in it.
+
+// A command reference, only inside a code span. `` `pnpm josh gate` `` and `` `josh review:brief` ``
+// are the copy-and-paste form; the leading backtick keeps prose like "josh will decide" out, which
+// would otherwise read "will" as a command name.
+const COMMAND_PATTERN = /`(?:pnpm )?josh ([a-z][a-z0-9:-]+)/gu
+
+// A label the documents operate on — applied (`labels[]=…`), removed (`…/labels/<name>`) or created
+// (`labels -f name=…`). Prose that merely names a label is not scanned; a mistyped name only breaks
+// where a command carries it.
+const LABEL_PATTERN = /(?:labels\[\]=|\/labels\/|labels -f name=)([a-z][a-z0-9:_-]*)/gu
+
+// A section reference: a markdown file in a code span, an arrow, and the cited heading in straight or
+// Japanese quotes — `` `backlogrun-progress.md` → "The hand-off" ``.
+const SECTION_PATTERN = /`([\w./-]+\.md)`\s*→\s*["「]([^"」]+)["」]/gu
+
+// A relative markdown link target, with an optional `#anchor`. External `https://…` links carry no
+// `.md`, so the pattern never matches them.
+const LINK_PATTERN = /\]\(([\w./-]+\.md)(?:#[\w-]+)?\)/gu
+
+// Filenames that appear in the documents as illustrations of the reference form itself, never as real
+// targets — the doc-section explanation cites `` `path.md` → "Heading" `` and `` `X.md` → "Heading" ``.
+const PLACEHOLDER_FILES: ReadonlySet<string> = new Set(['x.md', 'foo.md', 'file.md', 'path.md'])
+
+function matches(text: string, pattern: RegExp): Array<RegExpMatchArray> {
+	return [...text.matchAll(pattern)]
+}
+
+// A trailing colon is a placeholder for the sub-command that follows — `` `josh epic:` `` stands for
+// `epic:audit`, `epic:next` and the rest — so it names no command of its own.
+function command_references(text: string): Array<string> {
+	return matches(text, COMMAND_PATTERN)
+		.map((match) => match[1] ?? '')
+		.filter((name) => !name.endsWith(':'))
+}
+
+// A `<placeholder>` inside a command, and `depth:<n>` after the char class stops at `<`, both leave a
+// token that is not a real label. Real labels never end in a colon and never carry angle brackets.
+function is_real_label(token: string): boolean {
+	return token.length > 0 && !token.endsWith(':') && !token.includes('<')
+}
+
+function label_references(text: string): Array<string> {
+	return matches(text, LABEL_PATTERN)
+		.map((match) => match[1] ?? '')
+		.filter((token) => is_real_label(token))
+}
+
+function base_name(path: string): string {
+	return path.split('/').at(-1) ?? path
+}
+
+function is_placeholder_file(name: string): boolean {
+	return name.includes('<') || PLACEHOLDER_FILES.has(base_name(name).toLowerCase())
+}
+
+interface SectionReference {
+	file: string
+	heading: string
+}
+
+function section_references(text: string): Array<SectionReference> {
+	return matches(text, SECTION_PATTERN)
+		.map(([, file, heading]) => ({ file: file ?? '', heading: heading ?? '' }))
+		.filter((reference) => !is_placeholder_file(reference.file))
+}
+
+function link_targets(text: string): Array<string> {
+	return matches(text, LINK_PATTERN)
+		.map((match) => match[1] ?? '')
+		.filter((file) => !is_placeholder_file(file))
+}
+
+// A cited heading and the heading it names are compared with backticks dropped and whitespace
+// collapsed, because a reference wraps at the column the prose wraps at — sometimes across a
+// blockquote `> ` continuation — while the heading it points at is one unbroken line.
+function normalize_reference(heading: string): string {
+	return heading
+		.replaceAll('`', '')
+		.split(/\s+/u)
+		.filter((word) => word !== '>')
+		.join(' ')
+}
+
+// The two forms a heading can be cited by: with its leading number (`2b. Delegating…`) and without
+// (`Delegating…`), since a document's own numbering is a prefix a reference may or may not carry.
+function reference_forms(title: string): Array<string> {
+	const normalized = normalize_reference(title)
+
+	return [normalized, normalized.replace(/^\d+[a-z]?\. /u, '')]
+}
+
+const document_scan = {
+	command_references,
+	is_placeholder_file,
+	label_references,
+	link_targets,
+	normalize_reference,
+	reference_forms,
+	section_references,
+}
+
+export type { SectionReference }
+export { document_scan }

package/scripts/document/document-section-cli.ts

@@ -0,0 +1,101 @@
+#!/usr/bin/env tsx
+import { existsSync } from 'node:fs'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { document_section } from './document-section'
+import { entry_read_set } from './entry-read-set'
+
+// `josh doc:section` — print one section of a markdown document (joshuafolkken/kit#1776).
+//
+// **It exists because the documents already cite each other by section and the reader could only
+// fetch files.** `` `backlogrun-progress.md` → "The hand-off" `` names 249 lines; opening
+// `backlogrun-progress.md` costs far more. Every entry point pays that difference at the moment it
+// starts, before anything has been implemented.
+//
+// **What it is not is a summary.** The section is printed verbatim, headings and all, so a run that
+// reads a section has read the same words a run that opened the file would have. Nothing is deferred
+// and nothing is condensed — the extent of the read changes, not its content, which is why this is
+// not the "read it later" scheme joshuafolkken/kit#1344 and joshuafolkken/kit#1460 each measured
+// never firing.
+//
+// **A heading that does not resolve is a refusal, not an empty answer.** A pointer whose target was
+// renamed would otherwise print nothing and read as a section with nothing in it — the one outcome
+// that is worse than reading the whole file.
+
+const ARGV_OFFSET = 2
+const FAILURE_EXIT_CODE = 1
+const FILE_ARGUMENT = 0
+const HEADING_ARGUMENT = 1
+const NO_CANDIDATE = 0
+const USAGE = 'Usage: josh doc:section <file.md> "<heading>"'
+
+// A bare `backlogrun.md` means the workflow skill's own copy, because that is how every reference in
+// those documents spells it; anything that resolves as a path is taken as one.
+//
+// **Every branch resolves against `root`, including the first.** Left relative to the process's own
+// directory, a bare name that happened to exist beside the caller would beat the document under
+// `root` and print a section of the wrong file — silently, which is the failure this command exists
+// to refuse (joshuafolkken/kit#1776 review round 2).
+function resolve_document(name: string, root: string = process.cwd()): string {
+	const named = path.resolve(root, name)
+
+	if (existsSync(named)) return named
+
+	const in_skill = entry_read_set.document_path(root, name)
+
+	return existsSync(in_skill) ? in_skill : named
+}
+
+function report_missing(markdown: string, heading: string): number {
+	const near = document_section.candidates(markdown, heading)
+	const listed = near.length > NO_CANDIDATE ? near : document_section.titles(markdown)
+
+	console.error(`No single section matches ${JSON.stringify(heading)}. Headings it could mean:`)
+	for (const title of listed) console.error(`  ${title}`)
+
+	return FAILURE_EXIT_CODE
+}
+
+function print_section(markdown: string, heading: string): number {
+	const found = document_section.section(markdown, heading)
+
+	if (found === undefined) return report_missing(markdown, heading)
+
+	console.info(found.text)
+
+	return 0
+}
+
+function read_named(name: string, root: string): string | undefined {
+	const file_path = resolve_document(name, root)
+	const markdown = document_section.read_optional(file_path)
+
+	if (markdown === undefined) console.error(`Cannot read ${file_path}`)
+
+	return markdown
+}
+
+function run(argv: ReadonlyArray<string>, root: string = process.cwd()): number {
+	const name = argv[FILE_ARGUMENT]
+	const heading = argv[HEADING_ARGUMENT]
+
+	if (name === undefined || heading === undefined) {
+		console.error(USAGE)
+
+		return FAILURE_EXIT_CODE
+	}
+
+	const markdown = read_named(name, root)
+
+	return markdown === undefined ? FAILURE_EXIT_CODE : print_section(markdown, heading)
+}
+
+function main(argv: ReadonlyArray<string>): void {
+	process.exitCode = run(argv)
+}
+
+const document_section_cli = { FAILURE_EXIT_CODE, USAGE, main, resolve_document, run }
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) main(process.argv.slice(ARGV_OFFSET))
+
+export { document_section_cli }

package/scripts/document/document-section.ts

@@ -0,0 +1,156 @@
+// Reading one **section** of a markdown document, because a section is the unit these documents
+// cross-reference each other in (joshuafolkken/kit#1776).
+//
+// **The reference form was already a section reference, and only the reader was missing.** Every
+// workflow document points at another one as `` `backlogrun-progress.md` → "The hand-off" `` — a file
+// *and* a heading. A reader with no way to fetch a heading has one move left, which is to open the
+// file, so a pointer at 249 lines was costing 2,121. `backlogrun.md` and its point-of-use phase
+// documents are opened for many such references and are among the largest items an entry reads.
+//
+// **Nothing here defers a read.** The section is fetched in the same turn, in the main line, by the
+// run that has to obey it — what changes is the extent of the fetch, never whether it happens. That
+// is the distinction joshuafolkken/kit#1344 and joshuafolkken/kit#1460 were written after: a rule
+// moved to "read it later" is a rule that measurably never fires, and this moves nothing to later.
+
+import { readFileSync } from 'node:fs'
+
+// `#` to `######`, one space, then the title. Written with a literal space rather than `\s+` and a
+// lazy title so it cannot backtrack: `sonarjs/super-linear-regex` rejects the `\s+…+?\s*$` shape,
+// and the trim below does the same job in linear time.
+const HEADING_PATTERN = /^(?<hashes>#{1,6}) (?<title>.*)$/u
+const FENCE = '```'
+const SINGLE_MATCH = 1
+
+interface Heading {
+	level: number
+	title: string
+	// Zero-based index into the document's lines, so a slice needs no arithmetic.
+	line: number
+}
+
+interface Section {
+	title: string
+	level: number
+	first_line: number
+	last_line: number
+	text: string
+}
+
+function to_heading(line: string, index: number): Heading | undefined {
+	const groups = HEADING_PATTERN.exec(line)?.groups
+
+	if (groups === undefined) return undefined
+
+	// Destructured rather than read as properties: `noPropertyAccessFromIndexSignature` refuses
+	// `groups.title`. The bracket form was refused too, until joshuafolkken/kit#1783 switched off the
+	// `dot-notation` fix that rewrote it back — so this is a choice now rather than the only spelling.
+	const { hashes = '', title = '' } = groups
+
+	return { level: hashes.length, title: title.trim(), line: index }
+}
+
+function push_heading(found: Array<Heading>, line: string, index: number): void {
+	const heading = to_heading(line, index)
+
+	if (heading !== undefined) found.push(heading)
+}
+
+// **Fenced blocks are skipped, and that is not defensive tidying.** These documents quote shell, and
+// a shell comment at the start of a line is a `#` in column one. Counted as a heading it would end
+// the section above it early, which is a silent wrong answer rather than a visible failure.
+function headings(markdown: string): Array<Heading> {
+	const found: Array<Heading> = []
+	let is_inside_fence = false
+
+	for (const [index, line] of markdown.split('\n').entries()) {
+		if (line.startsWith(FENCE)) is_inside_fence = !is_inside_fence
+		else if (!is_inside_fence) push_heading(found, line, index)
+	}
+
+	return found
+}
+
+function titles(markdown: string): Array<string> {
+	return headings(markdown).map((heading) => heading.title)
+}
+
+// **Prefix, because a reference cites the name and the heading carries the name plus its gloss.**
+// `` → "The hand-off" `` points at `## The hand-off — one session does not have to run the whole
+// epic`, and `` → "The check is asked at every merge" `` at a heading that continues with a comma.
+// Requiring the whole heading would make every such reference unresolvable, and citing the whole
+// heading in the prose would put the gloss into every pointer.
+function is_match(heading: Heading, wanted: string): boolean {
+	return heading.title.startsWith(wanted)
+}
+
+function prefixed(found: ReadonlyArray<Heading>, wanted: string): Array<Heading> {
+	return found.filter((heading) => is_match(heading, wanted))
+}
+
+// **Exact wins, and an ambiguous prefix is refused rather than ranked.** Two headings that both
+// start with the cited text is a document whose reference is genuinely ambiguous; picking the first
+// would hand back a section nobody named, which is a failure a whole-file read does not have.
+function select(found: ReadonlyArray<Heading>, wanted: string): Heading | undefined {
+	const exact = found.find((heading) => heading.title === wanted)
+
+	if (exact !== undefined) return exact
+
+	const near = prefixed(found, wanted)
+
+	return near.length === SINGLE_MATCH ? near[0] : undefined
+}
+
+// A section ends where the next heading of the same or higher rank begins, so a `##` section carries
+// its `###` subsections with it — which is what a pointer at a `##` heading means.
+function last_line_of(found: ReadonlyArray<Heading>, chosen: Heading, total_lines: number): number {
+	const index = found.indexOf(chosen)
+	const next = found.slice(index + SINGLE_MATCH).find((heading) => heading.level <= chosen.level)
+
+	return (next === undefined ? total_lines : next.line) - SINGLE_MATCH
+}
+
+function section(markdown: string, wanted: string): Section | undefined {
+	const lines = markdown.split('\n')
+	const found = headings(markdown)
+	const chosen = select(found, wanted)
+
+	if (chosen === undefined) return undefined
+
+	const last_line = last_line_of(found, chosen, lines.length)
+
+	return {
+		title: chosen.title,
+		level: chosen.level,
+		first_line: chosen.line,
+		last_line,
+		text: lines.slice(chosen.line, last_line + SINGLE_MATCH).join('\n'),
+	}
+}
+
+// Ambiguity and absence are different answers, and the caller reports them differently: one names
+// the candidates, the other lists every heading the file has.
+function candidates(markdown: string, wanted: string): Array<string> {
+	return prefixed(headings(markdown), wanted).map((heading) => heading.title)
+}
+
+function read_optional(file_path: string): string | undefined {
+	try {
+		return readFileSync(file_path, 'utf8')
+	} catch {
+		return undefined
+	}
+}
+
+const document_section = {
+	FENCE,
+	HEADING_PATTERN,
+	candidates,
+	headings,
+	read_optional,
+	section,
+	select,
+	titles,
+}
+
+export type { Heading, Section }
+export { document_section }