@mailwoman/corpus 8.0.0 → 8.2.0

package/src/shard-recipes/unit.ts

@@ -31,15 +31,32 @@ import { alignRow } from "../align.ts"
 import type { CanonicalRow } from "../types.ts"
 import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
 
-/** A cached OpenAddresses extract: the zip, the CSV member, and the implied (file-level) region. */
+/**
+ * A cached OpenAddresses extract: the zip, the CSV member, and the implied (file-level) region.
+ */
+
+/**
+ * Longest OpenAddresses unit id reused verbatim. Longer values are building codes or free text, so a synthetic id is
+ * substituted instead.
+ */
+const MAX_REAL_UNIT_ID_LENGTH = 6
+
+/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
+   cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
+   output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
+   would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
+   extracted as named constants above. */
+
 interface UnitSource {
 	zip: string
 	csv: string
 	region: string
 }
 
-// OA REGION is empty for US per-state extracts — the region is implied by the file. Train sources are
-// every NON-Vermont state cached; eval is Vermont only (the corpus holdout).
+/**
+ * OA REGION is empty for US per-state extracts — the region is implied by the file. Train sources are every NON-Vermont
+ * state cached; eval is Vermont only (the corpus holdout).
+ */
 const TRAIN_SOURCES: readonly UnitSource[] = [
 	{ zip: "/tmp/oa-cache/us__ca__berkeley.zip", csv: "us/ca/berkeley.csv", region: "CA" },
 	{ zip: "/tmp/oa-cache/us__ca__marin.zip", csv: "us/ca/marin.csv", region: "CA" },
@@ -49,11 +66,14 @@ const TRAIN_SOURCES: readonly UnitSource[] = [
 	{ zip: "/tmp/oa-cache/us__mt__statewide.zip", csv: "us/mt/statewide.csv", region: "MT" },
 	{ zip: "/tmp/oa-cache/us__sd__statewide.zip", csv: "us/sd/statewide.csv", region: "SD" },
 ]
+
 const EVAL_SOURCE: UnitSource = { zip: "/tmp/oa-cache/us__vt__statewide.zip", csv: "us/vt/statewide.csv", region: "VT" }
 
-// USPS Pub-28 C2 designators that take a secondary identifier ("Apt 4B"). Weighted toward the common
-// ones the v0-parity arena failed on (Apt/Ste/Unit/Fl/Rm). Standalone designators (Basement, Lobby,
-// Penthouse) are emitted occasionally with no id.
+/**
+ * USPS Pub-28 C2 designators that take a secondary identifier ("Apt 4B"). Weighted toward the common ones the v0-parity
+ * arena failed on (Apt/Ste/Unit/Fl/Rm). Standalone designators (Basement, Lobby, Penthouse) are emitted occasionally
+ * with no id.
+ */
 const ID_DESIGNATORS: readonly USUnitDesignator[] = [
 	"APARTMENT",
 	"SUITE",
@@ -65,6 +85,7 @@ const ID_DESIGNATORS: readonly USUnitDesignator[] = [
 	"SPACE",
 	"LOT",
 ]
+
 const STANDALONE_DESIGNATORS: readonly USUnitDesignator[] = [
 	"BASEMENT",
 	"LOBBY",
@@ -74,10 +95,16 @@ const STANDALONE_DESIGNATORS: readonly USUnitDesignator[] = [
 	"UPPER",
 	"LOWER",
 ]
-const ID_WEIGHT = 0.85 // 85% id-bearing designators, 15% standalone
+
+/**
+ * 85% id-bearing designators, 15% standalone.
+ */
+const ID_WEIGHT = 0.85
 const SYNTH_IDS: readonly string[] = ["4B", "200", "12", "3", "A", "101", "5", "2A", "310", "B", "7", "1500", "404"]
 
-/** A real US tuple read out of a cached OA zip (number/street/city/postcode + the bare OA unit id). */
+/**
+ * A real US tuple read out of a cached OA zip (number/street/city/postcode + the bare OA unit id).
+ */
 interface UnitTuple {
 	house_number: string
 	street: string
@@ -87,7 +114,9 @@ interface UnitTuple {
 	oaUnit: string
 }
 
-/** Minimal RFC-4180-ish splitter (handles quoted fields). */
+/**
+ * Minimal RFC-4180-ish splitter (handles quoted fields).
+ */
 function splitCSV(line: string): string[] {
 	const out: string[] = []
 	let cur = ""
@@ -100,6 +129,7 @@ function splitCSV(line: string): string[] {
 			if (c === '"') {
 				if (line[i + 1] === '"') {
 					cur += '"'
+
 					i++
 				} else {
 					inQ = false
@@ -116,12 +146,15 @@ function splitCSV(line: string): string[] {
 			cur += c
 		}
 	}
+
 	out.push(cur)
 
 	return out
 }
 
-/** Stream real US tuples (number/street/city/postcode + the bare OA unit id) out of a cached OA zip. */
+/**
+ * Stream real US tuples (number/street/city/postcode + the bare OA unit id) out of a cached OA zip.
+ */
 function readTuples(source: UnitSource): UnitTuple[] {
 	const r = spawnSync("unzip", ["-p", source.zip, source.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
 
@@ -130,16 +163,19 @@ function readTuples(source: UnitSource): UnitTuple[] {
 
 		return []
 	}
+
 	const lines = r.stdout.toString("utf8").split(/\r?\n/)
 
 	if (lines.length < 2) return []
 	const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
 	const idx = (name: string): number => header.indexOf(name)
+
 	const iNum = idx("number"),
 		iStreet = idx("street"),
 		iUnit = idx("unit"),
 		iCity = idx("city"),
 		iPost = idx("postcode")
+
 	const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
 	const tuples: UnitTuple[] = []
 	const seen = new Set<string>()
@@ -156,6 +192,7 @@ function readTuples(source: UnitSource): UnitTuple[] {
 
 		if (seen.has(key)) continue
 		seen.add(key)
+
 		tuples.push({
 			house_number,
 			street,
@@ -169,10 +206,14 @@ function readTuples(source: UnitSource): UnitTuple[] {
 	return tuples
 }
 
-/** Title-case a canonical/abbrev designator ("APARTMENT" → "Apartment", "APT" → "Apt"). */
+/**
+ * Title-case a canonical/abbrev designator ("APARTMENT" → "Apartment", "APT" → "Apt").
+ */
 const title = (s: string): string => s.charAt(0).toUpperCase() + s.slice(1).toLowerCase()
 
-/** Build an injected unit string ("Apt 4B"), varying canonical vs approved-abbrev form per row. */
+/**
+ * Build an injected unit string ("Apt 4B"), varying canonical vs approved-abbrev form per row.
+ */
 function makeUnit(random: () => number, oaUnit: string): string {
 	const standalone = random() >= ID_WEIGHT
 	const pool = standalone ? STANDALONE_DESIGNATORS : ID_DESIGNATORS
@@ -181,12 +222,16 @@ function makeUnit(random: () => number, oaUnit: string): string {
 	const designator = random() < 0.5 ? title(canonical) : title(US_UNIT_DESIGNATOR_PREFERRED_ABBR[canonical])
 
 	if (standalone) return designator
-	const id = oaUnit && oaUnit.length <= 6 ? oaUnit : SYNTH_IDS[Math.floor(random() * SYNTH_IDS.length)]!
+
+	const id =
+		oaUnit && oaUnit.length <= MAX_REAL_UNIT_ID_LENGTH ? oaUnit : SYNTH_IDS[Math.floor(random() * SYNTH_IDS.length)]!
 
 	return `${designator} ${id}`
 }
 
-/** Synthetic recipient/venue prefixes — the "JOHN DOE, ACME INC, ..." arena pattern. */
+/**
+ * Synthetic recipient/venue prefixes — the "JOHN DOE, ACME INC, ..." arena pattern.
+ */
 const VENUES: readonly string[] = [
 	"John Doe",
 	"Jane Smith",
@@ -200,7 +245,9 @@ const VENUES: readonly string[] = [
 	"Riverside Clinic",
 ]
 
-/** Address tail: "City, ST 12345" (or no postcode). */
+/**
+ * Address tail: "City, ST 12345" (or no postcode).
+ */
 const tail = (loc: string, reg: string, pc: string): string => (pc ? `${loc}, ${reg} ${pc}` : `${loc}, ${reg}`)
 
 /**
@@ -218,7 +265,9 @@ function renderUnit(
 		loc = base.locality,
 		reg = base.region,
 		pc = base.postcode
+
 	const road = `${hn} ${street}`
+
 	const full: Partial<Record<ComponentTag, string>> = {
 		house_number: hn,
 		street,
@@ -227,6 +276,7 @@ function renderUnit(
 		region: reg,
 		...(pc ? { postcode: pc } : {}),
 	}
+
 	const r = random()
 
 	if (r < 0.34) return { fmt: "full-after", raw: `${road} ${unit}, ${tail(loc, reg, pc)}`, components: full }
@@ -241,6 +291,10 @@ function renderUnit(
 	return { fmt: "venue", raw: `${v}, ${road} ${unit}, ${tail(loc, reg, pc)}`, components: { venue: v, ...full } }
 }
 
+/**
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
 export const unitRecipe: ShardRecipe = {
 	name: "unit",
 	description: "US secondary-unit rows (#451): real OA skeletons + injected USPS Pub-28 C2 unit designators",
@@ -258,6 +312,7 @@ export const unitRecipe: ShardRecipe = {
 
 		for (const s of sources) {
 			const t = readTuples(s)
+
 			console.error(`  ${s.csv}: ${t.length} unique tuples`)
 
 			for (const x of t) {
@@ -265,7 +320,7 @@ export const unitRecipe: ShardRecipe = {
 			}
 		}
 
-		if (pool.length === 0) {
+		if (!pool.length) {
 			throw new Error("No US tuples found — are the cached OA zips present in /tmp/oa-cache?")
 		}
 
@@ -282,14 +337,18 @@ export const unitRecipe: ShardRecipe = {
 			// The unit must survive verbatim in raw, else alignment can't label it.
 			if (!raw.includes(unit)) {
 				skipped++
+
 				continue
 			}
 
 			if (opts.golden) {
 				write(JSON.stringify({ raw, components, country: "US" }) + "\n")
+
 				emitted++
+
 				continue
 			}
+
 			const canonical: CanonicalRow = {
 				raw,
 				components,
@@ -300,13 +359,17 @@ export const unitRecipe: ShardRecipe = {
 				corpus_version: "0.4.0",
 				license: "OpenAddresses US (non-VT) skeletons + injected USPS Pub-28 C2 unit designators",
 			}
+
 			const aligned = alignRow(canonical)
 
 			if (aligned.kind !== "labeled" || !aligned.row) {
 				skipped++
+
 				continue
 			}
+
 			write(JSON.stringify({ ...aligned.row, synth_method: "unit", synth_base_id: null }) + "\n")
+
 			emitted++
 		}
 

package/src/split.ts

@@ -46,16 +46,24 @@ export interface SplitOptions {
 	holdouts?: Record<string, readonly string[]>
 }
 
-/** Output manifest: source_id lists per split. */
+/**
+ * Output manifest: source_id lists per split.
+ */
 export interface SplitManifest {
 	train: string[]
 	val: string[]
 	test: string[]
-	/** Echoes the holdouts used, so the manifest is self-describing. */
+	/**
+	 * Echoes the holdouts used, so the manifest is self-describing.
+	 */
 	holdouts: Record<string, readonly string[]>
-	/** Corpus version stamped onto the manifest. Read from the first row. */
+	/**
+	 * Corpus version stamped onto the manifest. Read from the first row.
+	 */
 	corpus_version: string
-	/** Counts for quick sanity checks. */
+	/**
+	 * Counts for quick sanity checks.
+	 */
 	counts: { train: number; val: number; test: number; total: number }
 }
 
@@ -117,6 +125,7 @@ export function splitRows(rows: Iterable<SplitInputRow>, opts: SplitOptions = {}
 		if (!corpus_version && row.corpus_version) {
 			corpus_version = row.corpus_version
 		}
+
 		const split = splitForRow(row, holdouts)
 
 		if (split === "train") {
@@ -140,7 +149,9 @@ export function splitRows(rows: Iterable<SplitInputRow>, opts: SplitOptions = {}
 	}
 }
 
-/** Lightweight deterministic 0..(n-1) bucket from a string id. */
+/**
+ * Lightweight deterministic 0..(n-1) bucket from a string id.
+ */
 export function hashBucket(id: string, n: number): number {
 	const digest = createHash("sha256").update(id).digest()
 	// Read 4 bytes as uint32 to avoid bigint overhead.
@@ -160,18 +171,22 @@ export async function writeSplitManifests(manifest: SplitManifest, outputDir: st
 	await mkdir(outputDir, { recursive: true })
 
 	for (const name of ["train", "val", "test"] as const) {
-		const sorted = [...manifest[name]].sort()
+		const sorted = [...manifest[name]].toSorted()
 		await writeFile(join(outputDir, `${name}.txt`), sorted.join("\n") + (sorted.length ? "\n" : ""), "utf8")
 	}
+
 	const summary = {
 		corpus_version: manifest.corpus_version,
 		holdouts: manifest.holdouts,
 		counts: manifest.counts,
 	}
+
 	await writeFile(join(outputDir, "SPLIT_MANIFEST.json"), `${JSON.stringify(summary, null, 2)}\n`, "utf8")
 }
 
-/** Type re-export for callers that want to ingest LabeledRow specifically. */
+/**
+ * Type re-export for callers that want to ingest LabeledRow specifically.
+ */
 export type SplitInputLabeledRow = Pick<LabeledRow, "source_id" | "country" | "corpus_version" | "components">
 
 /**
@@ -199,11 +214,13 @@ export async function writeSplitManifestsFromLabeledFiles(opts: {
 	}
 
 	const total = opts.counts.train + opts.counts.val + opts.counts.test
+
 	const summary = {
 		corpus_version: opts.corpusVersion,
 		holdouts,
 		counts: { ...opts.counts, total },
 	}
+
 	await writeFile(join(opts.outputDir, "SPLIT_MANIFEST.json"), `${JSON.stringify(summary, null, 2)}\n`, "utf8")
 
 	return summary.counts
@@ -216,6 +233,7 @@ export async function writeSplitManifestsFromLabeledFiles(opts: {
 async function streamSortedSourceIds(labeledJsonlPath: string, outPath: string): Promise<void> {
 	const unsortedPath = `${outPath}.unsorted`
 	const out = createWriteStream(unsortedPath, { encoding: "utf8" })
+
 	const outClosed = new Promise<void>((resolve, reject) => {
 		out.on("close", () => resolve())
 		out.on("error", reject)
@@ -233,12 +251,14 @@ async function streamSortedSourceIds(labeledJsonlPath: string, outPath: string):
 	} finally {
 		out.end()
 	}
+
 	await outClosed
 
 	await new Promise<void>((resolve, reject) => {
 		// LC_ALL=C: byte-sort, locale-independent → deterministic across hosts.
 		const proc = spawn("sort", [unsortedPath, "-o", outPath], { env: childEnv({ LC_ALL: "C" }) })
 		proc.on("error", reject)
+
 		proc.on("exit", (code) => {
 			if (code === 0) {
 				resolve()
@@ -247,5 +267,6 @@ async function streamSortedSourceIds(labeledJsonlPath: string, outPath: string):
 			}
 		})
 	})
+
 	await unlink(unsortedPath).catch(() => {})
 }

package/src/synthesize-anchor-absorption.ts

@@ -32,6 +32,12 @@
 
 import type { ComponentTag } from "@mailwoman/core/types"
 
+/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
+   cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
+   output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
+   would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
+   extracted as named constants above. */
+
 export interface AnchorAbsorptionBaseTuple {
 	locality: string
 	region: string
@@ -78,8 +84,10 @@ function houseNum(random: () => number, realZips: ReadonlyArray<string>): string
 	return String(1 + Math.floor(random() * 9999))
 }
 
-// Curated, provenance-light reference vocab (real US street/city/state + DE). Surface forms must appear
-// in `raw` for alignRow; these are plain ASCII tokens that align cleanly.
+/**
+ * Curated, provenance-light reference vocab (real US street/city/state + DE). Surface forms must appear in `raw` for
+ * alignRow; these are plain ASCII tokens that align cleanly.
+ */
 const STREET_NAMES = [
 	"Main",
 	"Oak",
@@ -100,7 +108,9 @@ const STREET_NAMES = [
 	"Westpark",
 	"Crescent Meadow",
 ]
+
 const STREET_TYPES = ["St", "Ave", "Rd", "Dr", "Ln", "Blvd", "Ct", "Way", "Road", "Drive"]
+
 const US_TUPLES: ReadonlyArray<AnchorAbsorptionBaseTuple> = [
 	{ locality: "Springfield", region: "IL", postcode: "62701" },
 	{ locality: "Portland", region: "OR", postcode: "97215" },
@@ -111,10 +121,16 @@ const US_TUPLES: ReadonlyArray<AnchorAbsorptionBaseTuple> = [
 	{ locality: "Rochester", region: "NY", postcode: "14606" },
 	{ locality: "Sacramento", region: "CA", postcode: "95823" },
 ]
-// US rural states where the leading-postcode "{ZIP} {Street}, {STATE}" form (no trailing ZIP) is real —
-// the VT/rural format the #723 override broke and Probe A0 eroded.
+
+/**
+ * US rural states where the leading-postcode "{ZIP} {Street}, {STATE}" form (no trailing ZIP) is real — the VT/rural
+ * format the #723 override broke and Probe A0 eroded.
+ */
 const RURAL_REGIONS = ["VT", "ND", "SD", "NH", "ME", "MT", "WY"]
-// DE leading-postcode tuples: "{postcode} {locality}, {street} {house}".
+
+/**
+ * DE leading-postcode tuples: "{postcode} {locality}, {street} {house}".
+ */
 const DE_TUPLES = [
 	{ postcode: "10115", locality: "Berlin", street: "Hauptstraße" },
 	{ postcode: "80331", locality: "München", street: "Sendlinger Straße" },
@@ -122,11 +138,16 @@ const DE_TUPLES = [
 	{ postcode: "50667", locality: "Köln", street: "Hohe Straße" },
 	{ postcode: "01067", locality: "Dresden", street: "Prager Straße" },
 ]
+
 const HOUSE_NUMS = ["5", "12", "27", "100", "212", "1450", "8"]
-// Fake 5-digit strings that are NOT real US ZIPs (so the anchor lookup MISSES them) — for anchor-fp.
+/**
+ * Fake 5-digit strings that are NOT real US ZIPs (so the anchor lookup MISSES them) — for anchor-fp.
+ */
 const FAKE_ZIPS = ["00000", "99998", "99997", "00001", "99996"]
 
-/** Build one anchor-absorption counter-augmentation row. */
+/**
+ * Build one anchor-absorption counter-augmentation row.
+ */
 export function synthesizeAnchorAbsorptionRow(
 	opts: AnchorAbsorptionSynthesisOpts = {}
 ): SynthesizedAnchorAbsorptionRow {
@@ -199,6 +220,7 @@ export function synthesizeAnchorAbsorptionRow(
 		if (random() < 0.5) {
 			return { raw: `${zip} ${street}`, components: { house_number: zip, street }, locale: "en-US", template }
 		}
+
 		const t = pick(US_TUPLES, random)
 
 		return {
@@ -228,6 +250,7 @@ export function synthesizeAnchorAbsorptionRow(
 			template,
 		}
 	}
+
 	// standard: normal house number + trailing postcode → house_number (baseline, keeps the common case).
 	const hn = houseNum(random, realZips)
 	const t = pick(US_TUPLES, random)
@@ -252,12 +275,16 @@ export function synthesizeAnchorAbsorptionRow(
 // had NO counter-slice for the common locality-bearing house# case. A3 ADDS h-no-trailing-locality (15%)
 // to teach the LOCALITY discriminator (present -> house#, absent + 5-digit -> postcode = p-us-rural) and
 // trims p-us-rural 16->13. Goal: house_number recovers WITHOUT re-eroding postcode/SLICE-H. CASE-P = 26%.
+/**
+ * Every anchor-absorption template, in one list for the shard runner to sample from. Each covers a way a venue or
+ * landmark name can swallow the street token that follows it.
+ */
 export const ALL_TEMPLATES: ReadonlyArray<AnchorAbsorptionTemplate> = [
-	...Array<AnchorAbsorptionTemplate>(25).fill("h-adversarial"),
-	...Array<AnchorAbsorptionTemplate>(15).fill("h-no-trailing-locality"),
-	...Array<AnchorAbsorptionTemplate>(13).fill("p-us-rural"),
-	...Array<AnchorAbsorptionTemplate>(13).fill("p-de"),
-	...Array<AnchorAbsorptionTemplate>(8).fill("anchor-fp"),
-	...Array<AnchorAbsorptionTemplate>(14).fill("locale-ambig"),
-	...Array<AnchorAbsorptionTemplate>(12).fill("standard"),
+	...new Array<AnchorAbsorptionTemplate>(25).fill("h-adversarial"),
+	...new Array<AnchorAbsorptionTemplate>(15).fill("h-no-trailing-locality"),
+	...new Array<AnchorAbsorptionTemplate>(13).fill("p-us-rural"),
+	...new Array<AnchorAbsorptionTemplate>(13).fill("p-de"),
+	...new Array<AnchorAbsorptionTemplate>(8).fill("anchor-fp"),
+	...new Array<AnchorAbsorptionTemplate>(14).fill("locale-ambig"),
+	...new Array<AnchorAbsorptionTemplate>(12).fill("standard"),
 ]