@mailwoman/corpus 10.0.0 → 10.1.0

package/lib/tools/golden/relabel-street/directory.ts

@@ -6,11 +6,17 @@
  */
 
 import { pathExists, readLocalBuffer, readLocalTextFile } from "@mailwoman/core/fs/readers"
-import { makeDirectories, writeLocalFile, writeLocalJSONFile, writeLocalTextFile } from "@mailwoman/core/fs/writers"
+import {
+	makeDirectories,
+	writeLocalFile,
+	writeLocalJSONFile,
+	writeLocalTextFile,
+	writeLocalJSONLFile,
+} from "@mailwoman/core/fs/writers"
 import { sha256File } from "@mailwoman/core/hash"
-import { parseJSONStrict, tryParsingJSON } from "@mailwoman/core/json"
+import { parseJSONStrict, tryParsingJSON, stringifyJSON } from "@mailwoman/core/json"
 import { isPresent } from "@mailwoman/core/objects"
-import { basename, join } from "path-ts"
+import { basename, PathBuilder, type PathBuilderLike } from "path-ts"
 import { TextSpliterator } from "spliterator"
 import { Globerator } from "spliterator/node/fs"
 
@@ -50,15 +56,22 @@ export interface RelabelGoldenOptions {
 	 */
 	input: string
 	/**
-	 * Output golden version dir. Created; never overwritten in place.
+	 * Output golden version dir.
+	 *
+	 * Created.
+	 * Never overwritten in place.
 	 */
 	output: string
 	/**
-	 * Review-deck JSONL path. Default `<output>/REVIEW-DECK.jsonl`.
+	 * Review-deck jsonl path.
+	 *
+	 * Default `<output>/review-deck.jsonl`.
 	 */
 	deck?: string
 	/**
-	 * Parent version label recorded in the manifest. Default: the input dir's basename.
+	 * Parent version label recorded in the manifest.
+	 *
+	 * Default: the input dir's basename.
 	 */
 	parentLabel?: string
 	/**
@@ -66,7 +79,9 @@ export interface RelabelGoldenOptions {
 	 */
 	commit?: string
 	/**
-	 * Passed through to {@linkcode relabelGoldenStreetRow}. Default true.
+	 * Passed through to {@linkcode relabelGoldenStreetRow}.
+	 *
+	 * Default true.
 	 */
 	splitPrefix?: boolean
 }
@@ -100,8 +115,9 @@ const EMPTY_COUNTS = (): GoldenRelabelCounts => ({
 })
 
 /**
- * Classes that are LEFT FOLDED but still belong in the deck, because the operator asked to see them by name: a street
- * that is entirely one suffix word, and a bare post-directional tail.
+ * Classes that remain folded and still belong in the deck for operator review.
+ *
+ * They include a street made entirely from one suffix word and a bare post-directional tail.
  */
 const DECK_WORTHY_UNCHANGED: ReadonlySet<GoldenRelabelClass> = new Set([
 	"suffix-only-street",
@@ -110,27 +126,33 @@ const DECK_WORTHY_UNCHANGED: ReadonlySet<GoldenRelabelClass> = new Set([
 ])
 
 /**
- * Relabel every `.jsonl` in a golden version dir, writing a new version dir plus a review deck and a MANIFEST that
- * records the convention, the parent, and the counts. Non-JSONL siblings (README, split manifests) are copied forward
- * so the new version is self-contained; nested split dirs (`dev/`, `test/`) are relabelled recursively.
+ * Relabel every `.jsonl` file in a golden-version directory.
+ *
+ * Write a new version directory with a review deck and manifest.
+ * The manifest records the convention, parent and counts.
+ *
+ * Non-jsonl siblings (readme, split manifests) are copied forward so the new version is self-contained.
+ * Nested split dirs (`dev/`, `test/`) are relabelled recursively.
  */
 export async function relabelGoldenDirectory(
 	options: RelabelGoldenOptions,
 	report: (line: string) => void = console.log
 ): Promise<RelabelGoldenReport> {
 	const { input, output } = options
-	const deckPath = options.deck ?? join(output, "REVIEW-DECK.jsonl")
-	await makeDirectories(output)
+	const outputDir = PathBuilder.from(output)
+	// A string, because the report records it and its markdown sibling path uses suffix replacement.
+	const deckPath = options.deck ?? outputDir("REVIEW-DECK.jsonl").toString()
+	await makeDirectories(outputDir)
 
 	const deck: GoldenRelabelDeckEntry[] = []
 	const files: RelabelGoldenReport["files"] = {}
 
-	const walk = async (dirIn: string, dirOut: string, prefix: string): Promise<void> => {
+	const walk = async (dirIn: PathBuilder, dirOut: PathBuilder, prefix: string): Promise<void> => {
 		await makeDirectories(dirOut)
 
 		for await (const name of Globerator.from("*", { cwd: dirIn, withFileTypes: true, onlyFiles: false })) {
-			const from = join(dirIn, name.name)
-			const to = join(dirOut, name.name)
+			const from = dirIn(name.name)
+			const to = dirOut(name.name)
 
 			if (name.isDirectory()) {
 				await walk(from, to, `${prefix}${name.name}/`)
@@ -139,7 +161,8 @@ export async function relabelGoldenDirectory(
 			}
 
 			if (!name.name.endsWith(".jsonl")) {
-				// MANIFEST is rewritten below; everything else (README, SPLIT-MANIFEST) rides forward.
+				// manifest is rewritten below.
+				// Everything else (readme, split-manifest) moves forward.
 				if (name.name !== "MANIFEST.json") {
 					await writeLocalFile(await readLocalBuffer(from), to)
 				}
@@ -155,11 +178,9 @@ export async function relabelGoldenDirectory(
 			let lineNumber = 0
 
 			for (const line of TextSpliterator.from(await readLocalTextFile(from))) {
-				if (!line.trim()) continue
-
 				lineNumber++
-				// A corrupt answer-key line must STOP the relabel, not silently drop a row — a golden file
-				// short by one row is a floor threshold against a different denominator.
+				// A corrupt answer-key line must stop the relabel rather than silently drop a row.
+				// A golden file short by one row is a floor threshold against a different denominator.
 				const row = parseJSONStrict<GoldenStreetRow>(line)
 				const result = relabelGoldenStreetRow(row, { splitPrefix: options.splitPrefix ?? true })
 
@@ -189,10 +210,10 @@ export async function relabelGoldenDirectory(
 					})
 				}
 
-				out.push(JSON.stringify(result.row))
+				out.push(stringifyJSON(result.row))
 			}
 
-			await writeLocalTextFile(out.join("\n") + "\n", to)
+			await writeLocalTextFile(out, to)
 			files[`${prefix}${name.name}`] = { entries: lineNumber, changed, flagged, prefixSplit, counts }
 
 			report(
@@ -202,9 +223,9 @@ export async function relabelGoldenDirectory(
 	}
 
 	report(`relabel ${input} → ${output}`)
-	await walk(input, output, "")
+	await walk(PathBuilder.from(input), outputDir, "")
 
-	await writeLocalTextFile(deck.map((entry) => JSON.stringify(entry)).join("\n") + "\n", deckPath)
+	await writeLocalJSONLFile(deck, deckPath)
 	await writeLocalFile(renderDeckMarkdown(deck, basename(input), basename(output)), deckPath.replace(/\.jsonl$/, ".md"))
 
 	const manifestFiles: Record<
@@ -215,7 +236,7 @@ export async function relabelGoldenDirectory(
 	for (const [name, stats] of Object.entries(files)) {
 		manifestFiles[name] = {
 			entries: stats.entries,
-			sha256: await sha256File(join(output, name)),
+			sha256: await sha256File(outputDir(name)),
 			changed: stats.changed,
 			flagged: stats.flagged,
 			prefix_split: stats.prefixSplit,
@@ -257,22 +278,25 @@ export async function relabelGoldenDirectory(
 		review_deck: basename(deckPath),
 	}
 
-	await writeLocalJSONFile(manifest, join(output, "MANIFEST.json"))
+	await writeLocalJSONFile(manifest, outputDir("MANIFEST.json"))
 	report(`✓ ${totalChanged} rows split, ${totalFlagged} flagged — deck at ${deckPath}`)
 
 	return { files, deckPath, outputDir: output, totalChanged, totalFlagged }
 }
 
 /**
- * Render the operator-facing half of the review deck: the flagged rows first (those are the ones asking for a ruling),
- * then the classes the tool LEFT FOLDED by name, then a sample of the ordinary corrections. The JSONL sibling carries
- * every row; this file is the one a human reads.
+ * Render the operator-facing half of the review deck: the flagged rows first
+ * (those are the ones asking for a ruling), then the classes the tool left folded by name,
+ * then a sample of the ordinary corrections.
+ *
+ * The jsonl sibling contains every row.
+ * This file is the one a human reads.
  */
 function renderDeckMarkdown(deck: GoldenRelabelDeckEntry[], parent: string, version: string): string {
 	const span = (components: Record<string, string>): string =>
 		[components.street_prefix, components.street, components.street_suffix]
 			.filter(isPresent)
-			.map((s) => JSON.stringify(s))
+			.map((s) => stringifyJSON(s))
 			.join(" + ")
 
 	// The split dirs are copies of the same rows — dedupe the deck to the top-level files for reading.
@@ -285,7 +309,7 @@ function renderDeckMarkdown(deck: GoldenRelabelDeckEntry[], parent: string, vers
 		entries
 			.map(
 				(entry) =>
-					`| ${entry.file}:${entry.line} | ${span(entry.before)} | ${span(entry.after)} | ${entry.flags.map((f) => f.kind).join(", ") || "—"} | ${JSON.stringify(entry.raw)} |`
+					`| ${entry.file}:${entry.line} | ${span(entry.before)} | ${span(entry.after)} | ${entry.flags.map((f) => f.kind).join(", ") || "—"} | ${stringifyJSON(entry.raw)} |`
 			)
 			.join("\n")
 
@@ -323,10 +347,13 @@ export function isLeftFolded(rowClass: GoldenRelabelClass): boolean {
 }
 
 /**
- * True when a golden dir declares the US-split convention — i.e. it is safe to grade it with an UNFOLDED scorer.
+ * True when a golden dir declares the US-split convention — i.e. it is safe
+ * to grade it with an unfolded scorer.
  */
-export async function goldenDeclaresSplitStreets(dir: string): Promise<boolean> {
-	for (const candidate of [join(dir, "MANIFEST.json"), join(dir, "..", "MANIFEST.json")]) {
+export async function goldenDeclaresSplitStreets(dir: PathBuilderLike): Promise<boolean> {
+	const root = PathBuilder.from(dir)
+
+	for (const candidate of [root("MANIFEST.json"), root.dirname()("MANIFEST.json")]) {
 		if (!(await pathExists(candidate))) continue
 
 		const manifest = tryParsingJSON<{ convention?: { street_convention?: Record<string, string> } }>(

package/lib/tools/golden/relabel-street/row.ts

@@ -3,57 +3,7 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Golden-set street-suffix relabel — v0.1.2 → v0.1.3.
- *
- *   ## Why this exists
- *
- *   The golden answer key and the training corpus disagreed about ONE thing, and the v9.0.0
- *   promotion eval read the disagreement as a model regression (`us.street` 87.4 vs a floor of
- *   87.8). The corpus SPLITS a US street into `street` + `street_suffix` — TIGER's adapter
- *   decomposes at `corpus/src/adapters/tiger/street-decompose.ts`, the `street-affix` slice recipe
- *   teaches it from USPS Pub-28, and `ComponentTag` carries `street_suffix` as a first-class tag.
- *   The golden set FOLDED it: 2,216 US rows carry a `street`, and exactly 2 of them label a
- *   `street_suffix`. Operator ruling, 2026-08-06: **the split is canonical**; the golden is the
- *   stale side. This tool moves the answer key onto the corpus convention.
- *
- *   ## The instrument
- *
- *   `matchTrailingSuffix` from `@mailwoman/codex/us` — the USPS Pub-28 Appendix C table, which is
- *   also what the corpus slice recipe splits on. The table is NOT re-implemented here, and the
- *   libpostal dictionary TIGER reads is deliberately not used: measured on this golden set the two
- *   disagree on 51 US rows, and the disagreements run in the codex table's favour (libpostal's
- *   `directionals.txt` lists `center|c`, so TIGER reads the `C` of "C STREET" as a directional
- *   prefix and then emits no suffix at all).
- *
- *   ## What it changes, and what it refuses to
- *
- *   Applied only to rows whose `country` is `US`. Three branches, mirroring the SHAPE of TIGER's
- *   `decomposeStreet` on codex tables:
- *
- *   - **street type** — the last whitespace-separated word is a Pub-28 suffix, and something is left
- *       over: "Main St" → `street: "Main"`, `street_suffix: "St"` (1,559 rows).
- *   - **street type + post-directional** — the last word is a directional AND the one before it is a
- *       Pub-28 suffix: "Pennsylvania Avenue NW" → `street: "Pennsylvania"`,
- *       `street_suffix: "Avenue NW"` (347 rows). The post-directional joins the suffix rather than
- *       becoming a tag of its own, because that is what the corpus adapter emits; there is no
- *       `street_postfix` tag to move it to.
- *   - **everything else is left folded** and reported. In particular a BARE post-directional tail
- *       ("Seymour East", "BROADWAY N" — 16 rows) is NOT split: a directional is not a Pub-28 suffix,
- *       and the observed rows in that class are unit-contaminated ("1ST AVE SW BOX E", where the
- *       trailing "E" is a box letter).
- *
- *   FR rows are untouched, deliberately and permanently as far as this tool is concerned. French
- *   street typology puts the type FIRST ("Rue de la Paix") and the golden labels only 7 of 665 FR
- *   street rows with a `street_prefix`; whether FR should split at all is a different question with
- *   a different table behind it, and nothing here should be read as having answered it.
- *
- *   ## Surface bytes
- *
- *   The split is a break at a whitespace run in the ORIGINAL string — no trimming, no case
- *   normalization, no re-joining of tokens. `street + gap + street_suffix` reconstructs the input
- *   byte-for-byte, so the whitespace between them belongs to neither span (the same shape the corpus
- *   adapter's spans have). The tool asserts this per row and refuses to write a file if it ever
- *   fails.
+ *   Splits the street span of US golden rows into `street_prefix`, `street` and `street_suffix` with the USPS Pub-28 table.
  */
 
 import {
@@ -62,13 +12,12 @@ import {
 	type USStreetSuffix,
 	NAME_PRONE_US_SUFFIXES,
 } from "@mailwoman/codex/us"
+import { stringifyJSON } from "@mailwoman/core/json"
 import { escapeRegExp } from "@mailwoman/core/strings/regexp"
 
-// ── Types ──────────────────────────────────────────────────────────────────
-
 /**
- * A golden-set row, as stored one-per-line in `us.jsonl` / `fr.jsonl` / `adversarial.jsonl`. Only the fields this tool
- * reads are modeled; every other key rides through untouched.
+ * The golden row fields that the relabeler reads.
+ * Other fields are copied unchanged.
  */
 export interface GoldenStreetRow {
 	raw: string
@@ -80,7 +29,7 @@ export interface GoldenStreetRow {
 }
 
 /**
- * What the tool decided about one row. Every value except the two `split-*` classes means "left folded".
+ * The relabel outcome class for one row.
  */
 export type GoldenRelabelClass =
 	| "split-suffix"
@@ -96,8 +45,8 @@ export type GoldenRelabelClass =
 	| "untrimmed-street"
 
 /**
- * A review trigger on a row the tool DID change. A flag is never an adjudication — it marks the row for the operator's
- * deck, and the split is applied either way.
+ * A review note attached to a changed row.
+ * The relabel decision ignores it.
  */
 export interface GoldenRelabelFlag {
 	kind: "name-prone-suffix" | "venue-context" | "remainder-is-affix"
@@ -105,39 +54,24 @@ export interface GoldenRelabelFlag {
 }
 
 /**
- * Row-level relabel outcome.
+ * The relabel result for one row.
  */
 export interface GoldenRelabelResult {
 	/**
-	 * The row to write. Identical object reference when nothing changed.
+	 * The row to write.
+	 * This is the input object itself when no field changed.
 	 */
 	row: GoldenStreetRow
 	changed: boolean
 	rowClass: GoldenRelabelClass
 	flags: GoldenRelabelFlag[]
-	/**
-	 * A leading directional was lifted into `street_prefix` on this row.
-	 */
 	prefixSplit: boolean
 	/**
-	 * The street span as it stood in the parent version — recorded for the review deck.
+	 * The original street span of a changed row, for review.
 	 */
 	beforeStreet?: string
 }
 
-// ── The name-prone suffix set ──────────────────────────────────────────────
-
-/**
- * Pub-28 canonicals that are also ordinary head nouns of PROPER NAMES — "Lincoln Park", "Boston Common", "Willow
- * Brook". A split on one of these is still applied (the table is the table), but the row lands in the review deck
- * because the trailing word may belong to the name rather than to the street type.
- *
- * Chosen against the surfaces this golden set actually carries (park 6, green 5, hill 7, heights 3, hollow 3, brook 3,
- * pass 3 — the whole flagged class is 60 rows of 1,906), not from the whole 200-entry table: flagging every possible
- * name-head would mark a third of the corrections and stop being a review artifact.
- */
-// ── Byte-exact tail split ──────────────────────────────────────────────────
-
 interface TailSplit {
 	head: string
 	gap: string
@@ -145,9 +79,7 @@ interface TailSplit {
 }
 
 /**
- * Split `s` at its LAST whitespace run, returning the three pieces verbatim. Null when there is no interior whitespace,
- * when the head would be empty, or when `s` carries leading/trailing whitespace (a golden row is stored trimmed; an
- * untrimmed one is reported rather than silently normalized).
+ * Splits `s` at its last whitespace run, or returns `null` for an empty, untrimmed, or single-token string.
  */
 function splitLastWord(s: string): TailSplit | null {
 	if (s !== s.trim() || !s) return null
@@ -159,8 +91,7 @@ function splitLastWord(s: string): TailSplit | null {
 }
 
 /**
- * Split `s` at its FIRST whitespace run — the leading-directional counterpart of {@link splitLastWord}. `head` is the
- * first word, `tail` the rest, both verbatim.
+ * Splits `s` at its first whitespace run, with the same `null` cases as {@link splitLastWord}.
  */
 function splitFirstWord(s: string): TailSplit | null {
 	if (s !== s.trim() || !s) return null
@@ -171,11 +102,8 @@ function splitFirstWord(s: string): TailSplit | null {
 	return { head: match[1]!, gap: match[2]!, tail: match[3]! }
 }
 
-// ── Row-level relabel ──────────────────────────────────────────────────────
-
 /**
- * Rebuild `components` with `street_suffix` inserted immediately after `street`, so the written row reads in address
- * order rather than with the new tag appended at the end.
+ * Rebuilds `components` with the street spans in place of `street`, so the keys stay in address order.
  */
 function withStreetSpans(
 	components: Record<string, string>,
@@ -210,19 +138,19 @@ function withStreetSpans(
  */
 export interface RelabelStreetRowOptions {
 	/**
-	 * Also lift a folded LEADING directional out into `street_prefix`. Default true.
+	 * Whether to also move a leading directional into `street_prefix`.
 	 *
-	 * On by default because the fold applies both ways and the answer key has to be corrected on both, or the correction
-	 * is not a correction: 207 of the 1,682 split dev rows (12.3%) still opened with a directional after the suffix move
-	 * — "N Desmet Avenue" would have graded `street: "N Desmet"` against a model that says `street_prefix: "N", street:
-	 * "Desmet"`. Turn it OFF only to measure what the prefix fold alone costs.
+	 * Defaults to `true`; without it `N Desmet Avenue` grades as `street: "N Desmet"`
+	 * against a model that emits a prefix.
 	 */
 	splitPrefix?: boolean
 }
 
 /**
- * Decide, and apply, the US street-span split for ONE golden row. Pure: never mutates its argument, and returns the
- * same object reference when the row is left alone.
+ * Splits the street span of one US golden row without mutating its argument,
+ * returning it when no field changes.
+ *
+ * @throws When the split spans do not rebuild the original street byte for byte.
  */
 export function relabelGoldenStreetRow(
 	row: GoldenStreetRow,
@@ -259,8 +187,8 @@ export function relabelGoldenStreetRow(
 	if (!split) {
 		rowClass = matchTrailingSuffix(street) ? "suffix-only-street" : "single-token"
 	} else if (isStreetDirectionalToken(split.tail)) {
-		// Street type + post-directional ("Pennsylvania Avenue NW"). The corpus adapter emits the pair as
-		// ONE suffix span, and there is no post-directional tag to move it to.
+		// A street type and a post-directional, as in "Pennsylvania Avenue NW",
+		// form one suffix span because no post-directional tag exists.
 		const inner = splitLastWord(split.head)
 		const typeMatch = inner ? matchTrailingSuffix(inner.tail) : null
 
@@ -287,8 +215,6 @@ export function relabelGoldenStreetRow(
 		}
 	}
 
-	// Leading directional → street_prefix, on whatever name survived the suffix move. Independent of the
-	// suffix branch, because "N Main" is as folded as "N Main St" is.
 	let prefix: string | undefined
 	let prefixGap = ""
 
@@ -308,12 +234,10 @@ export function relabelGoldenStreetRow(
 		rowClass = "split-prefix-only"
 	}
 
-	// The invariant this tool exists to keep: the spans plus the whitespace between them ARE the original
-	// span, byte for byte. Anything else means a token was rewritten.
 	const rebuilt = `${prefix ? prefix + prefixGap : ""}${name}${suffix ? suffixGap + suffix : ""}`
 
 	if (rebuilt !== street) {
-		throw new Error(`golden-relabel: span reconstruction failed for ${JSON.stringify(street)}`)
+		throw new Error(`golden-relabel: span reconstruction failed for ${stringifyJSON(street)}`)
 	}
 
 	if (canonical && NAME_PRONE_US_SUFFIXES.has(canonical)) {
@@ -328,18 +252,16 @@ export function relabelGoldenStreetRow(
 	if (suffix && venue && new RegExp(`(^|\\W)${escapeRegExp(suffix)}(\\W|$)`, "i").test(venue)) {
 		flags.push({
 			kind: "venue-context",
-			detail: `venue ${JSON.stringify(venue)} also carries ${JSON.stringify(suffix)}`,
+			detail: `venue ${stringifyJSON(venue)} also carries ${stringifyJSON(suffix)}`,
 		})
 	}
 
-	// Narrow on purpose: a name that happens to be a Pub-28 canonical is NOT interesting ("Mountain Rd",
-	// "Valley Dr", "Mills Ln" are ordinary streets, and flagging them buried the deck — 108 rows of noise
-	// on the first run). A name that is a bare DIRECTIONAL is: "East Rd" leaves `street: "East"`, which is
-	// a direction, not a name.
+	// Only a bare directional remainder is flagged, as in "East Rd"; a Pub-28 suffix
+	// word like "Valley Dr" is an ordinary street.
 	if (isStreetDirectionalToken(name)) {
 		flags.push({
 			kind: "remainder-is-affix",
-			detail: `street would become the bare directional ${JSON.stringify(name)}`,
+			detail: `street would become the bare directional ${stringifyJSON(name)}`,
 		})
 	}