@mailwoman/corpus 8.6.0 → 9.0.0

package/src/adapters/state-tx-notaries/adapter.ts

@@ -15,11 +15,9 @@
  *   License: stamped `"Public Domain"` per Texas state government open-data terms.
  */
 
-import { createReadStream } from "node:fs"
+import { CSVSpliterator } from "spliterator"
 
-import { parse as csvParse } from "csv-parse"
-
-import { stableSourceID } from "../../adapter.ts"
+import { splitStreetLine, stableSourceID } from "../../adapter.ts"
 import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
 import { reconcileComponents } from "../../format.ts"
 import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
@@ -35,8 +33,6 @@ export const STATE_TX_NOTARIES_ADAPTER_ID = "state-tx-notaries"
  */
 export const STATE_TX_NOTARIES_DEFAULT_LICENSE = "Public Domain"
 
-const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
-
 /**
  * Match trailing "CITY, ST ZIP" or "CITY, ST" at the end of an address line.
  */
@@ -49,17 +45,6 @@ interface TxNotaryRow {
 	Address: string
 }
 
-function splitAddress(address: string): { house_number?: string; street: string } | null {
-	const trimmed = address.trim()
-
-	if (!trimmed) return null
-	const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
-
-	if (m) return { house_number: m[1], street: m[2]!.trim() }
-
-	return { street: trimmed }
-}
-
 export function createStateTxNotariesAdapter(): CorpusAdapter {
 	return {
 		id: STATE_TX_NOTARIES_ADAPTER_ID,
@@ -72,99 +57,90 @@ export function createStateTxNotariesAdapter(): CorpusAdapter {
 				throw new Error(`state-tx-notaries adapter: only US supported, got country=${opts.country}`)
 			}
 
-			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
-
-			const parser = stream.pipe(
-				csvParse({
-					columns: true,
-					skip_empty_lines: true,
-					relax_quotes: true,
-					relax_column_count: true,
-				})
-			)
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				mode: "object",
+				normalizeKeys: false,
+				enableQuoteHandling: true,
+			})
 
 			let emitted = 0
 
-			try {
-				for await (const record of parser as AsyncIterable<TxNotaryRow>) {
-					if (opts.signal?.aborted) break
-
-					if (opts.limit !== undefined && emitted >= opts.limit) break
+			for await (const record of rows as AsyncIterable<TxNotaryRow>) {
+				if (opts.signal?.aborted) break
 
-					const rawAddress = (record.Address ?? "").trim()
+				if (opts.limit !== undefined && emitted >= opts.limit) break
 
-					if (!rawAddress) continue
+				const rawAddress = (record.Address ?? "").trim()
 
-					const firstName = (record["First Name"] ?? "").trim()
-					const lastName = (record["Last Name"] ?? "").trim()
-					const notaryID = (record["Notary ID"] ?? "").trim()
+				if (!rawAddress) continue
 
-					// Parse embedded city/state/zip from the trailing portion of the address.
-					// Addresses look like: "1215 MCMILLAN DR\nCEDAR HILL, TX 75104"
-					const addrSingleLine = rawAddress.replaceAll("\n", ", ")
-					const cszMatch = CITY_STATE_ZIP_SUFFIX.exec(addrSingleLine)
+				const firstName = (record["First Name"] ?? "").trim()
+				const lastName = (record["Last Name"] ?? "").trim()
+				const notaryID = (record["Notary ID"] ?? "").trim()
 
-					if (!cszMatch) continue
+				// Parse embedded city/state/zip from the trailing portion of the address.
+				// Addresses look like: "1215 MCMILLAN DR\nCEDAR HILL, TX 75104"
+				const addrSingleLine = rawAddress.replaceAll("\n", ", ")
+				const cszMatch = CITY_STATE_ZIP_SUFFIX.exec(addrSingleLine)
 
-					const city = (cszMatch[1] ?? "").trim()
-					const stateAbbr = (cszMatch[2] ?? "").trim()
-					const zip = (cszMatch[3] ?? "").trim()
+				if (!cszMatch) continue
 
-					if (!city || !stateAbbr) continue
+				const city = (cszMatch[1] ?? "").trim()
+				const stateAbbr = (cszMatch[2] ?? "").trim()
+				const zip = (cszMatch[3] ?? "").trim()
 
-					const state = lookupStateAbbreviation(stateAbbr)
+				if (!city || !stateAbbr) continue
 
-					if (!state) continue
+				const state = lookupStateAbbreviation(stateAbbr)
 
-					// Extract the street portion (everything before the city/state/zip)
-					const streetPortion = addrSingleLine.slice(0, cszMatch.index).replace(/,\s*$/, "").trim()
+				if (!state) continue
 
-					if (!streetPortion) continue
+				// Extract the street portion (everything before the city/state/zip)
+				const streetPortion = addrSingleLine.slice(0, cszMatch.index).replace(/,\s*$/, "").trim()
 
-					const split = splitAddress(streetPortion)
+				if (!streetPortion) continue
 
-					if (!split) continue
+				const split = splitStreetLine(streetPortion)
 
-					const venue = [firstName, lastName].filter(Boolean).join(" ") || undefined
+				if (!split) continue
 
-					const components: CanonicalRow["components"] = {
-						...(venue ? { venue } : {}),
-						...(split.house_number ? { house_number: split.house_number } : {}),
-						street: split.street,
-						locality: city,
-						region: state.abbreviation,
-						...(zip ? { postcode: zip } : {}),
-					}
+				const venue = [firstName, lastName].filter(Boolean).join(" ") || undefined
 
-					const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
+				const components: CanonicalRow["components"] = {
+					...(venue ? { venue } : {}),
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					...(zip ? { postcode: zip } : {}),
+				}
 
-					const raw = [venue, streetPart, [city, [stateAbbr, zip].filter(Boolean).join(" ")].filter(Boolean).join(", ")]
-						.filter(Boolean)
-						.join(", ")
+				const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
 
-					const aligned = reconcileComponents(components, raw)
+				const raw = [venue, streetPart, [city, [stateAbbr, zip].filter(Boolean).join(" ")].filter(Boolean).join(", ")]
+					.filter(Boolean)
+					.join(", ")
 
-					if (Object.keys(aligned).length <= 2) continue
+				const aligned = reconcileComponents(components, raw)
 
-					const sourceID = notaryID
-						? `${STATE_TX_NOTARIES_ADAPTER_ID}-${notaryID}`
-						: stableSourceID(STATE_TX_NOTARIES_ADAPTER_ID, aligned)
+				if (Object.keys(aligned).length <= 2) continue
 
-					yield {
-						raw,
-						components: aligned,
-						country: "US",
-						locale: "en-US",
-						source: STATE_TX_NOTARIES_ADAPTER_ID,
-						source_id: sourceID,
-						corpus_version: "",
-						license: STATE_TX_NOTARIES_DEFAULT_LICENSE,
-					}
+				const sourceID = notaryID
+					? `${STATE_TX_NOTARIES_ADAPTER_ID}-${notaryID}`
+					: stableSourceID(STATE_TX_NOTARIES_ADAPTER_ID, aligned)
 
-					emitted++
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: STATE_TX_NOTARIES_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: STATE_TX_NOTARIES_DEFAULT_LICENSE,
 				}
-			} finally {
-				stream.destroy()
+
+				emitted++
 			}
 		},
 	}

package/src/adapters/synth-po-box/adapter.ts

@@ -20,6 +20,8 @@
  *       carries a `street` field.
  */
 
+import { tryParsingJSON } from "@mailwoman/core/objects"
+import { makeLcg } from "@mailwoman/core/utils"
 import { TextSpliterator } from "spliterator"
 
 import { stableSourceID } from "../../adapter.ts"
@@ -73,16 +75,6 @@ export interface SynthPoBoxAdapterOptions {
 	militaryRatio?: number
 }
 
-function makeRandom(seed: number): () => number {
-	let s = seed
-
-	return () => {
-		s = (s * 1_664_525 + 1_013_904_223) % 4_294_967_296
-
-		return s / 4_294_967_296
-	}
-}
-
 export function createSynthPoBoxAdapter(opts: SynthPoBoxAdapterOptions = {}): CorpusAdapter {
 	const variantsPerInput = opts.variantsPerInput ?? 1
 	const pmbRatio = opts.pmbRatio ?? 0.15
@@ -95,10 +87,10 @@ export function createSynthPoBoxAdapter(opts: SynthPoBoxAdapterOptions = {}): Co
 			"Synthetic PO box / PMB / Apartado / Boîte Postale rows. Consumes JSONL of (locality, region, postcode, country) tuples and emits locale-appropriate PO box variants.",
 
 		async *rows(options: AdapterOptions): AsyncIterable<CanonicalRow> {
-			const random = makeRandom(opts.seed ?? Date.now())
+			const random = makeLcg(opts.seed ?? Date.now())
 
-			// TextSpliterator streams string lines; the per-line try/catch below keeps this reader
-			// tolerant of malformed rows (skipped++), so TextSpliterator + explicit JSON.parse — not
+			// TextSpliterator streams string lines; the per-line tryParsingJSON below keeps this reader
+			// tolerant of malformed rows (skipped++), so TextSpliterator + a non-throwing parse — not
 			// JSONSpliterator, which would throw on the first bad line.
 			const lines = TextSpliterator.fromAsync(options.inputPath)
 
@@ -115,11 +107,9 @@ export function createSynthPoBoxAdapter(opts: SynthPoBoxAdapterOptions = {}): Co
 
 				if (!trimmed) continue
 
-				let input: PoBoxInputRow
+				const input = tryParsingJSON<PoBoxInputRow>(trimmed)
 
-				try {
-					input = JSON.parse(trimmed) as PoBoxInputRow
-				} catch {
+				if (input === null) {
 					skipped++
 
 					continue

package/src/adapters/tiger/street-decompose.ts

@@ -17,45 +17,35 @@
  */
 
 import { readFileSync } from "node:fs"
-import { resolve } from "node:path"
 
 import { resourceDictionaryPath } from "@mailwoman/core/utils"
+import { TextSpliterator } from "spliterator"
 
 function loadDictionary(filename: string): Set<string> {
-	// Resolve via the @mailwoman/core data directory.
-	const candidates = [
-		resourceDictionaryPath("libpostal", "en", filename),
-		resourceDictionaryPath("libpostal", "en", filename),
-		resolve(process.cwd(), "core/data/libpostal/dictionaries/en", filename),
-	]
-
-	for (const path of candidates) {
-		try {
-			const text = readFileSync(path, "utf8")
-			const set = new Set<string>()
-
-			for (const line of text.split("\n")) {
-				const trimmed = line.trim()
-
-				if (!trimmed || trimmed.startsWith("#")) continue
-
-				// libpostal format: canonical|abbr|abbr|... — index all forms
-				for (const form of trimmed.split("|")) {
-					const f = form.trim().toLowerCase()
-
-					if (f) {
-						set.add(f)
-					}
-				}
-			}
+	// `resourceDictionaryPath` already resolves both layouts — `core/data/...` from source and from the
+	// packaged `out/` tree. The candidate list this replaced named it TWICE and then guessed a third path
+	// off `process.cwd()`, and swallowed every error while probing, so a corrupt dictionary reported as a
+	// missing one.
+	const text = readFileSync(resourceDictionaryPath("libpostal", "en", filename), "utf8")
+	const set = new Set<string>()
+
+	// The largest libpostal dictionary is 8.4 KB, and this runs once per process at module load.
+	for (const line of TextSpliterator.from(text)) {
+		const trimmed = line.trim()
+
+		if (!trimmed || trimmed.startsWith("#")) continue
 
-			return set
-		} catch {
-			// try next candidate
+		// libpostal format: canonical|abbr|abbr|... — index all forms
+		for (const form of trimmed.split("|")) {
+			const f = form.trim().toLowerCase()
+
+			if (f) {
+				set.add(f)
+			}
 		}
 	}
 
-	throw new Error(`Could not load libpostal dictionary: ${filename}`)
+	return set
 }
 
 const DIRECTIONALS = loadDictionary("directionals.txt")

package/src/adapters/usgov-hrsa-fqhc/adapter.ts

@@ -8,9 +8,9 @@
  *   Federally Qualified Health Centers (FQHCs) are HRSA-funded community health programs that
  *   self-report site addresses to the HRSA Data Warehouse. The published CSV (`data.hrsa.gov`)
  *   carries the site name, the postal-formatted street address, and the locality/region/postcode
- *   quad. Phase 1.6 §1.2 (#22) selects this source for its adversarial-value-per-row: every
- *   facility name is a human-typed venue string and the addresses pass through enough hands to
- *   accumulate the abbreviation drift + suite designator chaos that pure gazetteer data does not.
+ *   quad. The source earns its place on adversarial-value-per-row (#22): every facility name is a
+ *   human-typed venue string and the addresses pass through enough hands to accumulate the
+ *   abbreviation drift + suite designator chaos that pure gazetteer data does not.
  *
  *   The adapter consumes a CSV file the operator pre-downloads. The HRSA data is published as a
  *   single national CSV (~10K rows), small enough that the operator can re-fetch on every corpus
@@ -29,11 +29,9 @@
  *   terms.
  */
 
-import { createReadStream } from "node:fs"
+import { CSVSpliterator } from "spliterator"
 
-import { parse as csvParse } from "csv-parse"
-
-import { stableSourceID } from "../../adapter.ts"
+import { splitStreetLine, stableSourceID } from "../../adapter.ts"
 import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
 import { reconcileComponents } from "../../format.ts"
 import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
@@ -70,23 +68,11 @@ interface HrsaSiteRow {
  * Split a "123 Main St Suite 4" surface form into `(house_number, street)`. The regex tolerates one trailing letter on
  * the number (`"123A Main St"`) and a hyphenated form (`"40-12 Bell Blvd"`); anything else falls back to street-only.
  *
- * Suite / Apt / Unit designators stay on `street` for Phase 1 — Mailwoman's `unit` component exists but the
- * address-formatter does not have a clean slot for it, and HRSA addresses do not separate the suite into its own
- * column. Leaving the surface form intact in `street` preserves the adversarial training signal (the model learns that
- * a trailing "Suite 4" is part of the road line in this distribution).
+ * Suite / Apt / Unit designators stay on `street` — Mailwoman's `unit` component exists but the address-formatter does
+ * not have a clean slot for it, and HRSA addresses do not separate the suite into its own column. Leaving the surface
+ * form intact in `street` preserves the adversarial training signal (the model learns that a trailing "Suite 4" is part
+ * of the road line in this distribution).
  */
-const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
-
-function splitAddress(address: string): { house_number?: string; street: string } | null {
-	const trimmed = address.trim()
-
-	if (!trimmed) return null
-	const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
-
-	if (m) return { house_number: m[1], street: m[2]!.trim() }
-
-	return { street: trimmed }
-}
 
 /**
  * Compose the raw envelope-style address line. Format:
@@ -122,78 +108,69 @@ export function createUsgovHrsaFqhcAdapter(): CorpusAdapter {
 				throw new Error(`usgov-hrsa-fqhc adapter: only US supported, got country=${opts.country}`)
 			}
 
-			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
-
-			const parser = stream.pipe(
-				csvParse({
-					columns: true,
-					skip_empty_lines: true,
-					relax_quotes: true,
-					relax_column_count: true,
-				})
-			)
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				mode: "object",
+				normalizeKeys: false,
+				enableQuoteHandling: true,
+			})
 
 			let emitted = 0
 
-			try {
-				for await (const record of parser as AsyncIterable<HrsaSiteRow>) {
-					if (opts.signal?.aborted) break
-
-					if (opts.limit !== undefined && emitted >= opts.limit) break
-
-					const venue = (record["Site Name"] ?? "").trim()
-					const split = splitAddress(record["Site Address"] ?? "")
-					const city = (record["Site City"] ?? "").trim()
-					const stateAbbr = (record["Site State Abbreviation"] ?? "").trim()
-					const postcode = (record["Site Postal Code"] ?? "").trim()
-
-					if (!venue || !split || !city || !postcode) continue
-					const state = lookupStateAbbreviation(stateAbbr)
-
-					if (!state) continue
-
-					// Insertion order matters here. `venue` first so alignment claims its span
-					// (which may contain a token like "Buffalo") before `locality` runs its
-					// search — the kryptonite case `Buffalo Health Clinic, Buffalo NY`
-					// otherwise mis-labels the venue's "Buffalo" as locality.
-					const components: CanonicalRow["components"] = {
-						venue,
-						...(split.house_number ? { house_number: split.house_number } : {}),
-						street: split.street,
-						locality: city,
-						region: state.abbreviation,
-						postcode,
-					}
-
-					const raw = composeRaw(venue, split.house_number, split.street, city, state.abbreviation, postcode)
+			for await (const record of rows as AsyncIterable<HrsaSiteRow>) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const venue = (record["Site Name"] ?? "").trim()
+				const split = splitStreetLine(record["Site Address"] ?? "")
+				const city = (record["Site City"] ?? "").trim()
+				const stateAbbr = (record["Site State Abbreviation"] ?? "").trim()
+				const postcode = (record["Site Postal Code"] ?? "").trim()
+
+				if (!venue || !split || !city || !postcode) continue
+				const state = lookupStateAbbreviation(stateAbbr)
+
+				if (!state) continue
+
+				// Insertion order matters here. `venue` first so alignment claims its span
+				// (which may contain a token like "Buffalo") before `locality` runs its
+				// search — the kryptonite case `Buffalo Health Clinic, Buffalo NY`
+				// otherwise mis-labels the venue's "Buffalo" as locality.
+				const components: CanonicalRow["components"] = {
+					venue,
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode,
+				}
 
-					if (!raw) continue
+				const raw = composeRaw(venue, split.house_number, split.street, city, state.abbreviation, postcode)
 
-					const aligned = reconcileComponents(components, raw)
+				if (!raw) continue
 
-					if (!Object.keys(aligned).length) continue
+				const aligned = reconcileComponents(components, raw)
 
-					const siteID = (record["Site ID"] ?? "").trim()
+				if (!Object.keys(aligned).length) continue
 
-					const sourceID = siteID
-						? `${USGOV_HRSA_FQHC_ADAPTER_ID}-${siteID}`
-						: stableSourceID(USGOV_HRSA_FQHC_ADAPTER_ID, aligned)
+				const siteID = (record["Site ID"] ?? "").trim()
 
-					yield {
-						raw,
-						components: aligned,
-						country: "US",
-						locale: "en-US",
-						source: USGOV_HRSA_FQHC_ADAPTER_ID,
-						source_id: sourceID,
-						corpus_version: "",
-						license: USGOV_HRSA_FQHC_DEFAULT_LICENSE,
-					}
+				const sourceID = siteID
+					? `${USGOV_HRSA_FQHC_ADAPTER_ID}-${siteID}`
+					: stableSourceID(USGOV_HRSA_FQHC_ADAPTER_ID, aligned)
 
-					emitted++
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: USGOV_HRSA_FQHC_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: USGOV_HRSA_FQHC_DEFAULT_LICENSE,
 				}
-			} finally {
-				stream.destroy()
+
+				emitted++
 			}
 		},
 	}