@mailwoman/corpus 10.5.0 → 10.7.0

package/lib/us/adapters/usgov/nad/adapter.ts

@@ -26,7 +26,10 @@ export const USGOV_NAD_ADAPTER_ID = "usgov-nad"
  */
 export const USGOV_NAD_DEFAULT_LICENSE = "Public Domain"
 
-interface NADRecord {
+/**
+ * One NAD address point as the FeatureServer pages publish it, limited to the fields a reader here uses.
+ */
+export interface NADRecord {
 	OBJECTID?: number
 	UUID?: string | null
 	AddNum_Pre?: string | null
@@ -94,7 +97,7 @@ function nonEmpty(...values: Array<string | null | undefined>): string | null {
  * The shared `composeHouseNumber` in `#adapters/street-line` joins exactly two columns with
  * one separator, so it fits a publisher that splits the number in two rather than in four.
  */
-function nadHouseNumber(r: NADRecord): string | null {
+export function nadHouseNumber(r: NADRecord): string | null {
 	const full = (r.AddNo_Full ?? "").toString().trim()
 
 	if (full) return full
@@ -107,16 +110,24 @@ function nadHouseNumber(r: NADRecord): string | null {
 	return [pre, num, suf].filter(isPresent).join(" ").trim() || null
 }
 
-interface DecomposedNADStreet {
+/**
+ * A NAD street split into the prefix, name and suffix its structured columns state,
+ * with the whole street beside them.
+ */
+export interface DecomposedNADStreet {
 	prefix?: string
 	street?: string
 	suffix?: string
 	full: string
 }
 
-// Structured street fields take precedence.
-// If they are empty, `StNam_Full` supplies the whole street.
-function decomposeNADStreet(r: NADRecord): DecomposedNADStreet | null {
+/**
+ * The street a NAD record states.
+ *
+ * Structured street fields take precedence.
+ * If they are empty, `StNam_Full` supplies the whole street.
+ */
+export function decomposeNADStreet(r: NADRecord): DecomposedNADStreet | null {
 	const name = (r.St_Name ?? "").toString().trim()
 
 	if (name) {
@@ -139,9 +150,27 @@ function decomposeNADStreet(r: NADRecord): DecomposedNADStreet | null {
 	return null
 }
 
-// The postal city comes first because it is the name that people write on mail.
-function composeLocality(r: NADRecord): string | null {
-	return nonEmpty(r.Post_City, r.Inc_Muni, r.Census_Plc, r.Uninc_Comm)
+/**
+ * The NAD domain values that fill a place-name column without naming a place, lower-cased.
+ */
+const NAD_PLACEHOLDER_PLACE_NAMES: ReadonlySet<string> = new Set(["not stated", "unincorporated", "0"])
+
+/**
+ * The locality a NAD record states.
+ *
+ * The postal city comes first because it is the name that people write on mail.
+ * NAD fills an unknown `Post_City` with the domain value `Not stated`
+ * (99,998 rows of a 100,000-row Massachusetts sample and 11,713 of a Texas one), and an address
+ * outside any municipality has `Inc_Muni` `Unincorporated` (the same 11,713 Texas rows).
+ *
+ * Another 23 rows of that Texas sample read `0`.
+ * Each of these values counts as absent, and the next column supplies the place name.
+ */
+export function composeNADLocality(r: NADRecord): string | null {
+	const stated = (value: string | null | undefined) =>
+		NAD_PLACEHOLDER_PLACE_NAMES.has(value?.trim().toLowerCase() ?? "") ? null : value
+
+	return nonEmpty(stated(r.Post_City), stated(r.Inc_Muni), stated(r.Census_Plc), stated(r.Uninc_Comm))
 }
 
 function composePostcode(r: NADRecord): string | null {
@@ -202,7 +231,7 @@ export function createUsgovNADAdapter(): CorpusAdapter {
 
 					if (!US_STATES_SET.has(state)) continue
 
-					const locality = composeLocality(record)
+					const locality = composeNADLocality(record)
 
 					if (!locality) continue
 

package/lib/us/recipes/skeleton-sources.ts

@@ -0,0 +1,156 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The US address skeletons the OA-shaped recipes read: two elected OpenAddresses extracts and one
+ *   National Address Database sample per chosen state, with Vermont held out for the golden sets.
+ *
+ *   `unit`, `street-affix`, `suffix-boundary`, `po-box-cedex` and `country-balanced` read these files
+ *   through the same zipped-CSV reader, so a NAD sample is written in OpenAddresses' column layout by
+ *   `packages/mailwoman/tools/dev-tools/corpus/nad-skeletons.run.ts`.
+ */
+
+import { dataRootPath } from "@mailwoman/core/data-root"
+import type { PathBuilderLike } from "path-ts"
+
+/**
+ * The register a skeleton file's rows were published in.
+ *
+ * It prefixes the `base_source_id` a recipe derives from the file.
+ */
+export const USSkeletonRegister = {
+	OpenAddresses: "openaddresses",
+	NationalAddressDatabase: "usgov-nad",
+} as const
+
+export type USSkeletonRegister = (typeof USSkeletonRegister)[keyof typeof USSkeletonRegister]
+
+/**
+ * One cached US skeleton file: the zip, its CSV member and the region every row of it lies in.
+ */
+export interface USSkeletonSource {
+	zip: PathBuilderLike
+	csv: string
+	region: string
+	register: USSkeletonRegister
+}
+
+/**
+ * The data-root directory holding the NAD samples and ZIP-to-city tables the NAD skeleton tool writes.
+ */
+export const NAD_SKELETON_DIRECTORY = "nad-skeletons"
+
+/**
+ * The states sampled from NAD, chosen for address shapes the OpenAddresses files lack
+ * rather than for row count.
+ *
+ * Distinct tuples of those six files carry a unit on 0.6% of rows, and none has
+ * a hyphenated or direction-prefixed house number.
+ *
+ * Each comment states the shape measured over the usable rows of the FeatureServer
+ * pages fetched on 2026-05-18.
+ * `nad-skeletons.run.ts` prints the table for every state.
+ *
+ * Vermont is absent because it is the eval holdout.
+ * No state's rows match a rural-route street pattern, so NAD adds no rural-route form.
+ */
+export const NAD_SKELETON_STATES = [
+	// Chicago's city grid, which `us/il/cook` supplied before it was withdrawn: 8.9% unit, 1,268 ZIPs.
+	"IL",
+	// 36.6% unit, the most of any state with over a million rows, and 154 street types.
+	"MA",
+	// The numbered grid: 43.1% numeric street names and 42.6% post-directionals, as in `500 E`.
+	"UT",
+	// Direction-prefixed rural grid house numbers such as `S9878` on 1.3% of rows, and lettered county roads.
+	"WI",
+	// Hyphenated Queens house numbers such as `244-32` on 5.3% of rows, and 15.8% unit.
+	"NY",
+	// Farm to Market, Ranch Road and County Road prefix types on 5.4% of rows, and 161 street types.
+	"TX",
+] as const
+
+/**
+ * The zip and CSV member the NAD skeleton tool writes for one state.
+ */
+export function nadSkeletonFile(region: string): { zip: PathBuilderLike; csv: string } {
+	const state = region.toLowerCase()
+
+	return {
+		zip: dataRootPath(NAD_SKELETON_DIRECTORY, `us__${state}__nad.zip`),
+		csv: `us/${state}/nad.csv`,
+	}
+}
+
+/**
+ * The NAD skeleton file for every state in {@link NAD_SKELETON_STATES}.
+ */
+export const NAD_SKELETON_SOURCES: readonly USSkeletonSource[] = NAD_SKELETON_STATES.map((region) => ({
+	...nadSkeletonFile(region),
+	region,
+	register: USSkeletonRegister.NationalAddressDatabase,
+}))
+
+/**
+ * The cached OpenAddresses US extracts used for training.
+ *
+ * `us/il/cook` is absent because Cook County's terms restrict redistribution.
+ * NAD's Illinois sample stands in for its city grid.
+ */
+export const OA_SKELETON_TRAIN_SOURCES: readonly USSkeletonSource[] = (
+	[
+		["us__ca__berkeley.zip", "us/ca/berkeley.csv", "CA"],
+		["us__ca__marin.zip", "us/ca/marin.csv", "CA"],
+		["us__dc__statewide.zip", "us/dc/statewide.csv", "DC"],
+		["us__ia__statewide.zip", "us/ia/statewide.csv", "IA"],
+		["us__mt__statewide.zip", "us/mt/statewide.csv", "MT"],
+		["us__sd__statewide.zip", "us/sd/statewide.csv", "SD"],
+	] as const
+).map(([zip, csv, region]) => ({
+	zip: dataRootPath("oa-cache", zip),
+	csv,
+	region,
+	register: USSkeletonRegister.OpenAddresses,
+}))
+
+/**
+ * Every US training skeleton: the OpenAddresses extracts and the NAD samples.
+ */
+export const US_SKELETON_TRAIN_SOURCES: readonly USSkeletonSource[] = [
+	...OA_SKELETON_TRAIN_SOURCES,
+	...NAD_SKELETON_SOURCES,
+]
+
+/**
+ * Vermont, the corpus holdout — `--golden` reads this and no other US source.
+ */
+export const US_SKELETON_EVAL_SOURCE: USSkeletonSource = {
+	zip: dataRootPath("oa-cache", "us__vt__statewide.zip"),
+	csv: "us/vt/statewide.csv",
+	region: "VT",
+	register: USSkeletonRegister.OpenAddresses,
+}
+
+/**
+ * A county whose NAD rows give the ZIP-to-city table the `intersection` recipe renders city tails from.
+ */
+export interface NADZipCityCounty {
+	region: string
+	county: string
+}
+
+/**
+ * The counties of the `intersection` recipe's TIGER training crossings, Cook IL and Morris NJ.
+ */
+export const NAD_ZIP_CITY_COUNTIES: readonly NADZipCityCounty[] = [
+	{ region: "IL", county: "Cook" },
+	{ region: "NJ", county: "Morris" },
+]
+
+/**
+ * The ZIP-to-city CSV the NAD skeleton tool writes for one county.
+ */
+export function nadZipCityPath(entry: NADZipCityCounty): PathBuilderLike {
+	const county = entry.county.toLowerCase().replaceAll(/[^a-z0-9]+/g, "-")
+
+	return dataRootPath(NAD_SKELETON_DIRECTORY, `us__${entry.region.toLowerCase()}__${county}__zip-city.csv`)
+}

package/lib/us/tools/nad-skeletons.ts

@@ -0,0 +1,395 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Samples National Address Database rows into OpenAddresses-layout CSV skeletons, measures the address
+ *   shapes each state's rows carry, and tallies county ZIP-to-city tables.
+ *
+ *   The OA-skeleton recipes read a zipped CSV with OpenAddresses' columns, so a NAD sample written in that
+ *   layout reaches them through the same reader as the OpenAddresses extracts they also read.
+ */
+
+import { matchLeadingDesignator } from "@mailwoman/codex/us"
+import { pathExists } from "@mailwoman/core/fs/readers/stat"
+import { writeLocalBuffer, writeLocalJSONFile, writeLocalTextFile } from "@mailwoman/core/fs/writers"
+import { sha256Hex } from "@mailwoman/core/hash"
+import { tryParsingJSON } from "@mailwoman/core/json"
+import { mulberry32 } from "@mailwoman/core/random"
+import ADMZip from "adm-zip"
+import { resolvePathBuilder, type PathBuilderLike } from "path-ts"
+import { TextSpliterator } from "spliterator"
+import { Globerator } from "spliterator/node/fs"
+
+import { composeNADLocality, decomposeNADStreet, nadHouseNumber, type NADRecord } from "#us/adapters/usgov/nad/adapter"
+import { nadSkeletonFile, nadZipCityPath, type NADZipCityCounty } from "#us/recipes/skeleton-sources"
+
+/**
+ * The OpenAddresses columns a skeleton CSV carries, in OpenAddresses' order.
+ */
+export const NAD_SKELETON_COLUMNS = [
+	"NUMBER",
+	"STREET",
+	"UNIT",
+	"CITY",
+	"DISTRICT",
+	"REGION",
+	"POSTCODE",
+	"ID",
+] as const
+
+/**
+ * One NAD row projected onto {@link NAD_SKELETON_COLUMNS}.
+ */
+export type NADSkeletonRow = Record<(typeof NAD_SKELETON_COLUMNS)[number], string>
+
+const RURAL_ROUTE = /^(RR|R R|RURAL ROUTE|HC|HCR|HIGHWAY CONTRACT)\b/i
+const text = (value: unknown): string => (value == null ? "" : String(value).trim())
+
+/**
+ * The unit identifier of a NAD row without its designator, as OpenAddresses' `UNIT` column mostly states it.
+ *
+ * NAD writes `APT 6`, `UNIT D` or a bare `200`; the recipes add their own designator in
+ * front of the identifier, so the designator NAD wrote is dropped rather than doubled.
+ */
+export function nadUnitIdentifier(record: NADRecord): string {
+	const unit = text(record.Unit) || text(record.Room)
+
+	if (!unit) return ""
+
+	const designator = matchLeadingDesignator(unit)
+	const identifier = designator ? unit.slice(unit.indexOf(designator.matched) + designator.matched.length) : unit
+
+	return identifier.replace(/^\s*#\s*/, "").trim()
+}
+
+/**
+ * Projects one NAD record onto the OpenAddresses columns, or returns null
+ * when it lacks a house number, street, locality, state or five-digit ZIP.
+ *
+ * A street whose name NAD publishes in capitals is written wholly in capitals,
+ * because NAD spells its structured types in title case (`East MAIN Street`)
+ * and a recipe would otherwise teach that mixed case.
+ */
+export function nadSkeletonRow(record: NADRecord): NADSkeletonRow | null {
+	const number = nadHouseNumber(record)
+	const street = decomposeNADStreet(record)
+	const city = composeNADLocality(record)
+	const region = text(record.State).toUpperCase()
+	const postcode = text(record.Zip_Code)
+
+	if (!number || !street?.full || !city || !region || !/^\d{5}$/.test(postcode)) return null
+
+	const name = street.street ?? street.full
+	const capitalized = /[A-Z]/.test(name) && name === name.toUpperCase()
+
+	return {
+		NUMBER: number,
+		STREET: capitalized ? street.full.toUpperCase() : street.full,
+		UNIT: nadUnitIdentifier(record),
+		CITY: city,
+		DISTRICT: text(record.County),
+		REGION: region,
+		POSTCODE: postcode,
+		ID: text(record.UUID) || text(record.OBJECTID),
+	}
+}
+
+function csvCell(value: string): string {
+	return /[",\r\n]/.test(value) ? `"${value.replaceAll('"', '""')}"` : value
+}
+
+/**
+ * The address-shape measures for one state's usable NAD rows.
+ */
+export interface NADStateShape {
+	region: string
+	/**
+	 * Rows with a house number, street, locality and five-digit ZIP.
+	 */
+	usable: number
+	unitShare: number
+	/**
+	 * Distinct `St_PosTyp` and `St_PreTyp` values.
+	 */
+	streetTypes: number
+	zips: number
+	/**
+	 * Share of rows whose street opens with a type, such as `County Road 12` or `Farm to Market 1960`.
+	 */
+	prefixTypeShare: number
+	numericNameShare: number
+	postDirectionalShare: number
+	ruralRouteShare: number
+	hyphenatedNumberShare: number
+	/**
+	 * Share of house numbers that open with a direction letter, as Wisconsin's `E2755` does.
+	 */
+	gridNumberShare: number
+}
+
+interface ShapeTally {
+	usable: number
+	unit: number
+	types: Set<string>
+	zips: Set<string>
+	prefixType: number
+	numericName: number
+	postDirectional: number
+	ruralRoute: number
+	hyphenated: number
+	grid: number
+}
+
+const newTally = (): ShapeTally => ({
+	usable: 0,
+	unit: 0,
+	types: new Set(),
+	zips: new Set(),
+	prefixType: 0,
+	numericName: 0,
+	postDirectional: 0,
+	ruralRoute: 0,
+	hyphenated: 0,
+	grid: 0,
+})
+
+function tallyShape(tally: ShapeTally, record: NADRecord, row: NADSkeletonRow): void {
+	tally.usable++
+
+	if (row.UNIT) {
+		tally.unit++
+	}
+
+	const postType = text(record.St_PosTyp).toUpperCase()
+	const preType = text(record.St_PreTyp).toUpperCase()
+
+	if (postType) {
+		tally.types.add(postType)
+	}
+
+	if (preType) {
+		tally.types.add(preType)
+
+		tally.prefixType++
+	}
+
+	tally.zips.add(row.POSTCODE)
+
+	if (/^\d+$/.test(text(record.St_Name))) {
+		tally.numericName++
+	}
+
+	if (text(record.St_PosDir)) {
+		tally.postDirectional++
+	}
+
+	if (RURAL_ROUTE.test(row.STREET)) {
+		tally.ruralRoute++
+	}
+
+	if (/^\d+-\d+/.test(row.NUMBER)) {
+		tally.hyphenated++
+	}
+
+	if (/^[NSEW]\d/i.test(row.NUMBER)) {
+		tally.grid++
+	}
+}
+
+function shapeOf(region: string, t: ShapeTally): NADStateShape {
+	const share = (n: number) => (t.usable ? n / t.usable : 0)
+
+	return {
+		region,
+		usable: t.usable,
+		unitShare: share(t.unit),
+		streetTypes: t.types.size,
+		zips: t.zips.size,
+		prefixTypeShare: share(t.prefixType),
+		numericNameShare: share(t.numericName),
+		postDirectionalShare: share(t.postDirectional),
+		ruralRouteShare: share(t.ruralRoute),
+		hyphenatedNumberShare: share(t.hyphenated),
+		gridNumberShare: share(t.grid),
+	}
+}
+
+export interface BuildNADSkeletonsOptions {
+	/**
+	 * The directory of FeatureServer NDJSON pages the NAD fetcher writes.
+	 */
+	inputDirectory: PathBuilderLike
+	/**
+	 * The states to sample, as USPS abbreviations.
+	 */
+	states: readonly string[]
+	/**
+	 * The most rows any one state's sample keeps.
+	 */
+	cap: number
+	seed: number
+	zipCityCounties: readonly NADZipCityCounty[]
+	/**
+	 * Called after each page with the count read so far.
+	 */
+	onPage?: (pagesRead: number, pages: number) => void
+}
+
+export interface NADSkeletonSummary {
+	pages: number
+	cap: number
+	seed: number
+	shapes: NADStateShape[]
+	samples: Array<{ region: string; rows: number; zip: string; csv: string; csvSHA256: string }>
+	zipCity: Array<{ region: string; county: string; zips: number; path: string; sha256: string }>
+}
+
+/**
+ * Reads every NAD page once, measures every state's address shapes,
+ * keeps a uniform reservoir sample of at most `cap` rows for each requested state,
+ * and tallies the majority city of each ZIP in each requested county.
+ *
+ * The sample is written in reservoir order rather than page order, so a reader that
+ * takes the first N rows still draws across the whole state.
+ * It refuses to replace a file that already exists.
+ */
+export async function buildNADSkeletons(options: BuildNADSkeletonsOptions): Promise<NADSkeletonSummary> {
+	const { inputDirectory, cap, seed } = options
+	const states = new Set(options.states.map((s) => s.toUpperCase()))
+
+	const outputs = [
+		...[...states].map((region) => nadSkeletonFile(region).zip),
+		...options.zipCityCounties.map(nadZipCityPath),
+	]
+
+	for (const output of outputs) {
+		if (await pathExists(output)) throw new Error(`${output} already exists; a skeleton artifact is never replaced`)
+	}
+
+	const random = mulberry32(seed)
+	const tallies = new Map<string, ShapeTally>()
+	const reservoirs = new Map<string, { seen: number; rows: NADSkeletonRow[] }>()
+	const zipCity = new Map<string, Map<string, Map<string, number>>>()
+	const countyKey = (region: string, county: string) => `${region}|${county.toLowerCase()}`
+	const wantedCounties = new Set(options.zipCityCounties.map((c) => countyKey(c.region, c.county)))
+
+	const pages = await Globerator.files("ndjson", { cwd: inputDirectory, absolute: false, recursive: false }).toSorted()
+
+	let pagesRead = 0
+
+	for (const page of pages) {
+		for await (const line of TextSpliterator.fromAsync(resolvePathBuilder(inputDirectory, page))) {
+			if (!line) continue
+			const record = tryParsingJSON<NADRecord>(line)
+
+			if (!record) continue
+			const row = nadSkeletonRow(record)
+
+			if (!row) continue
+
+			let tally = tallies.get(row.REGION)
+
+			if (!tally) {
+				tallies.set(row.REGION, (tally = newTally()))
+			}
+
+			tallyShape(tally, record, row)
+
+			if (states.has(row.REGION)) {
+				let reservoir = reservoirs.get(row.REGION)
+
+				if (!reservoir) {
+					reservoirs.set(row.REGION, (reservoir = { seen: 0, rows: [] }))
+				}
+
+				reservoir.seen++
+
+				if (reservoir.rows.length < cap) {
+					reservoir.rows.push(row)
+				} else {
+					const slot = Math.floor(random() * reservoir.seen)
+
+					if (slot < cap) {
+						reservoir.rows[slot] = row
+					}
+				}
+			}
+
+			const key = countyKey(row.REGION, row.DISTRICT)
+
+			if (wantedCounties.has(key)) {
+				let byZip = zipCity.get(key)
+
+				if (!byZip) {
+					zipCity.set(key, (byZip = new Map()))
+				}
+
+				let byCity = byZip.get(row.POSTCODE)
+
+				if (!byCity) {
+					byZip.set(row.POSTCODE, (byCity = new Map()))
+				}
+
+				byCity.set(row.CITY, (byCity.get(row.CITY) ?? 0) + 1)
+			}
+		}
+
+		options.onPage?.(++pagesRead, pages.length)
+	}
+
+	const summary: NADSkeletonSummary = {
+		pages: pages.length,
+		cap,
+		seed,
+		shapes: [...tallies].map(([region, t]) => shapeOf(region, t)).toSorted((a, b) => b.usable - a.usable),
+		samples: [],
+		zipCity: [],
+	}
+
+	for (const region of states) {
+		const reservoir = reservoirs.get(region)
+
+		if (!reservoir?.rows.length) throw new Error(`NAD holds no usable row for ${region}`)
+		const { zip, csv } = nadSkeletonFile(region)
+
+		const body = [
+			NAD_SKELETON_COLUMNS.join(","),
+			...reservoir.rows.map((row) => NAD_SKELETON_COLUMNS.map((column) => csvCell(row[column])).join(",")),
+			"",
+		].join("\n")
+
+		const archive = new ADMZip()
+
+		archive.addFile(csv, Buffer.from(body, "utf8"))
+		await writeLocalBuffer(archive.toBuffer(), zip)
+		summary.samples.push({ region, rows: reservoir.rows.length, zip: String(zip), csv, csvSHA256: sha256Hex(body) })
+	}
+
+	for (const county of options.zipCityCounties) {
+		const byZip = zipCity.get(countyKey(county.region, county.county))
+
+		if (!byZip?.size) throw new Error(`NAD holds no usable row for ${county.county} County, ${county.region}`)
+		const lines = ["ZIP,CITY,ROWS"]
+
+		for (const [zip, byCity] of [...byZip].toSorted(([a], [b]) => a.localeCompare(b))) {
+			const [city, rows] = [...byCity].toSorted((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))[0]!
+
+			lines.push([zip, csvCell(city), String(rows)].join(","))
+		}
+
+		const body = `${lines.join("\n")}\n`
+		const path = nadZipCityPath(county)
+
+		await writeLocalTextFile(body, path)
+		summary.zipCity.push({ ...county, zips: byZip.size, path: String(path), sha256: sha256Hex(body) })
+	}
+
+	return summary
+}
+
+/**
+ * Writes the summary beside the skeletons it describes.
+ */
+export function writeNADSkeletonSummary(summary: NADSkeletonSummary, path: PathBuilderLike): Promise<void> {
+	return writeLocalJSONFile(summary, path)
+}