@mailwoman/corpus 8.6.0 → 9.1.0

package/src/adapters/usgov-imls-pls/adapter.ts

@@ -18,14 +18,11 @@
  *   License: stamped `"Public Domain"` per IMLS federal government distribution terms.
  */
 
-import { createReadStream } from "node:fs"
-
-import { parse as csvParse } from "csv-parse"
-
-import { stableSourceID } from "../../adapter.ts"
-import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
-import { reconcileComponents } from "../../format.ts"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+import { splitStreetLine, stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import { lookupStateAbbreviation } from "@mailwoman/corpus/codex/us-fips-state"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
+import { reconcileComponents } from "@mailwoman/formatter"
+import { CSVSpliterator } from "spliterator"
 
 /**
  * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
@@ -38,8 +35,6 @@ export const USGOV_IMLS_PLS_ADAPTER_ID = "usgov-imls-pls"
  */
 export const USGOV_IMLS_PLS_DEFAULT_LICENSE = "Public Domain"
 
-const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
-
 interface ImlsOutletRow {
 	LIBNAME: string
 	ADDRESS: string
@@ -50,17 +45,6 @@ interface ImlsOutletRow {
 	FSCSKEY: string
 }
 
-function splitAddress(address: string): { house_number?: string; street: string } | null {
-	const trimmed = address.trim()
-
-	if (!trimmed) return null
-	const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
-
-	if (m) return { house_number: m[1], street: m[2]!.trim() }
-
-	return { street: trimmed }
-}
-
 export function createUsgovImlsPlsAdapter(): CorpusAdapter {
 	return {
 		id: USGOV_IMLS_PLS_ADAPTER_ID,
@@ -72,90 +56,77 @@ export function createUsgovImlsPlsAdapter(): CorpusAdapter {
 				throw new Error(`usgov-imls-pls adapter: only US supported, got country=${opts.country}`)
 			}
 
-			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
-
-			const parser = stream.pipe(
-				csvParse({
-					columns: true,
-					skip_empty_lines: true,
-					relax_quotes: true,
-					relax_column_count: true,
-				})
-			)
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				mode: "object",
+				normalizeKeys: false,
+				enableQuoteHandling: true,
+			})
 
 			let emitted = 0
 
-			try {
-				for await (const record of parser as AsyncIterable<ImlsOutletRow>) {
-					if (opts.signal?.aborted) break
+			for await (const record of rows as AsyncIterable<ImlsOutletRow>) {
+				if (opts.signal?.aborted) break
 
-					if (opts.limit !== undefined && emitted >= opts.limit) break
+				if (opts.limit !== undefined && emitted >= opts.limit) break
 
-					const libName = (record.LIBNAME ?? "").trim()
-					const address = (record.ADDRESS ?? "").trim()
-					const city = (record.CITY ?? "").trim()
-					const zip = (record.ZIP ?? "").trim()
-					const stateAbbr = (record.STABR ?? "").trim()
-					const county = (record.CNTY ?? "").trim()
+				const libName = (record.LIBNAME ?? "").trim()
+				const address = (record.ADDRESS ?? "").trim()
+				const city = (record.CITY ?? "").trim()
+				const zip = (record.ZIP ?? "").trim()
+				const stateAbbr = (record.STABR ?? "").trim()
+				const county = (record.CNTY ?? "").trim()
 
-					if (!libName || !city || !zip) continue
+				if (!libName || !city || !zip) continue
 
-					const state = lookupStateAbbreviation(stateAbbr)
+				const state = lookupStateAbbreviation(stateAbbr)
 
-					if (!state) continue
+				if (!state) continue
 
-					const split = splitAddress(address)
+				const split = splitStreetLine(address)
 
-					if (!split) continue
+				if (!split) continue
 
-					const components: CanonicalRow["components"] = {
-						venue: libName,
-						...(split.house_number ? { house_number: split.house_number } : {}),
-						street: split.street,
-						locality: city,
-						region: state.abbreviation,
-						postcode: zip,
-						// #552: no subregion — US postal addresses don't surface the county, so emitting
-						// subregion creates a phantom component with no raw-span to align to, quarantining
-						// ~21% of rows. The county is still available in the source CSV; it just isn't
-						// a postal-surface component here.
-					}
-
-					const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
+				const components: CanonicalRow["components"] = {
+					venue: libName,
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode: zip,
+					// #552: no subregion — US postal addresses don't surface the county, so emitting
+					// subregion creates a phantom component with no raw-span to align to, quarantining
+					// ~21% of rows. The county is still available in the source CSV; it just isn't
+					// a postal-surface component here.
+				}
 
-					const raw = [
-						libName,
-						streetPart,
-						[city, [stateAbbr, zip].filter(Boolean).join(" ")].filter(Boolean).join(", "),
-					]
-						.filter(Boolean)
-						.join(", ")
+				const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
 
-					const aligned = reconcileComponents(components, raw)
+				const raw = [libName, streetPart, [city, [stateAbbr, zip].filter(Boolean).join(" ")].filter(Boolean).join(", ")]
+					.filter(Boolean)
+					.join(", ")
 
-					if (Object.keys(aligned).length <= 2) continue
+				const aligned = reconcileComponents(components, raw)
 
-					const fscsKey = (record.FSCSKEY ?? "").trim()
+				if (Object.keys(aligned).length <= 2) continue
 
-					const sourceID = fscsKey
-						? `${USGOV_IMLS_PLS_ADAPTER_ID}-${fscsKey}`
-						: stableSourceID(USGOV_IMLS_PLS_ADAPTER_ID, aligned)
+				const fscsKey = (record.FSCSKEY ?? "").trim()
 
-					yield {
-						raw,
-						components: aligned,
-						country: "US",
-						locale: "en-US",
-						source: USGOV_IMLS_PLS_ADAPTER_ID,
-						source_id: sourceID,
-						corpus_version: "",
-						license: USGOV_IMLS_PLS_DEFAULT_LICENSE,
-					}
+				const sourceID = fscsKey
+					? `${USGOV_IMLS_PLS_ADAPTER_ID}-${fscsKey}`
+					: stableSourceID(USGOV_IMLS_PLS_ADAPTER_ID, aligned)
 
-					emitted++
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: USGOV_IMLS_PLS_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: USGOV_IMLS_PLS_DEFAULT_LICENSE,
 				}
-			} finally {
-				stream.destroy()
+
+				emitted++
 			}
 		},
 	}

package/src/adapters/usgov-irs-bmf/adapter.ts

@@ -19,13 +19,10 @@
  *   federal).
  */
 
-import { createReadStream } from "node:fs"
-
-import { parse as csvParse } from "csv-parse"
-
-import { stableSourceID } from "../../adapter.ts"
-import { reconcileComponents } from "../../format.ts"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+import { splitStreetLine, stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
+import { reconcileComponents } from "@mailwoman/formatter"
+import { CSVSpliterator } from "spliterator"
 
 /**
  * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
@@ -38,7 +35,6 @@ export const USGOV_IRS_BMF_ADAPTER_ID = "usgov-irs-bmf"
  */
 export const USGOV_IRS_BMF_DEFAULT_LICENSE = "Public Domain"
 
-const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
 /**
  * PO box in its many written forms: "PO BOX 12", "P.O. BOX 12", "P O BOX 12", "POB 12", "BOX 12".
  */
@@ -55,18 +51,17 @@ interface IrsBmfRow {
 
 /**
  * Classify the street line into a `po_box` or a `{house_number?, street}` split.
+ *
+ * BMF mixes street addresses and PO boxes in one `STREET` column, so the PO-box shapes have to be claimed BEFORE the
+ * shared house-number split runs — otherwise `splitStreetLine` would hand back `"PO Box 1234"` as a plain street, which
+ * is correct for every other US adapter and wrong here.
  */
-function splitStreetLine(street: string): { po_box: string } | { house_number?: string; street: string } | null {
+function splitStreetLineOrPOBox(street: string): { po_box: string } | { house_number?: string; street: string } | null {
 	const trimmed = street.trim()
 
-	if (!trimmed) return null
-
 	if (PO_BOX.test(trimmed)) return { po_box: trimmed }
-	const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
-
-	if (m) return { house_number: m[1], street: m[2]!.trim() }
 
-	return { street: trimmed }
+	return splitStreetLine(trimmed)
 }
 
 function composeRaw(
@@ -93,74 +88,68 @@ export function createUsgovIrsBmfAdapter(): CorpusAdapter {
 				throw new Error(`usgov-irs-bmf adapter: only US supported, got country=${opts.country}`)
 			}
 
-			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
-
-			const parser = stream.pipe(
-				csvParse({ columns: true, skip_empty_lines: true, relax_quotes: true, relax_column_count: true, trim: true })
-			)
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				mode: "object",
+				normalizeKeys: false,
+				enableQuoteHandling: true,
+			})
 
 			let emitted = 0
 
-			try {
-				for await (const record of parser as AsyncIterable<IrsBmfRow>) {
-					if (opts.signal?.aborted) break
-
-					if (opts.limit !== undefined && emitted >= opts.limit) break
+			for await (const record of rows as AsyncIterable<IrsBmfRow>) {
+				if (opts.signal?.aborted) break
 
-					const ein = (record.EIN ?? "").trim()
-					const venue = (record.NAME ?? "").trim() || undefined
-					const street = (record.STREET ?? "").trim()
-					const city = (record.CITY ?? "").trim()
-					const state = (record.STATE ?? "").trim()
-					const zipRaw = (record.ZIP ?? "").trim()
+				if (opts.limit !== undefined && emitted >= opts.limit) break
 
-					if (!city || !zipRaw) continue
-					const postcode = zipRaw.split("-")[0]!.trim() // 5-digit; drop the optional +4
+				const ein = (record.EIN ?? "").trim()
+				const venue = (record.NAME ?? "").trim() || undefined
+				const street = (record.STREET ?? "").trim()
+				const city = (record.CITY ?? "").trim()
+				const state = (record.STATE ?? "").trim()
+				const zipRaw = (record.ZIP ?? "").trim()
 
-					const split = splitStreetLine(street)
+				if (!city || !zipRaw) continue
+				const postcode = zipRaw.split("-")[0]!.trim() // 5-digit; drop the optional +4
 
-					if (!split) continue
+				const split = splitStreetLineOrPOBox(street)
 
-					const streetPart =
-						"po_box" in split ? split.po_box : [split.house_number, split.street].filter(Boolean).join(" ")
+				if (!split) continue
 
-					const components: CanonicalRow["components"] = {
-						...(venue ? { venue } : {}),
-						...("po_box" in split
-							? { po_box: split.po_box }
-							: { ...(split.house_number ? { house_number: split.house_number } : {}), street: split.street }),
-						locality: city,
-						...(state ? { region: state } : {}),
-						postcode,
-					}
+				const streetPart =
+					"po_box" in split ? split.po_box : [split.house_number, split.street].filter(Boolean).join(" ")
 
-					const raw = composeRaw(venue, streetPart, city, state, postcode)
+				const components: CanonicalRow["components"] = {
+					...(venue ? { venue } : {}),
+					...("po_box" in split
+						? { po_box: split.po_box }
+						: { ...(split.house_number ? { house_number: split.house_number } : {}), street: split.street }),
+					locality: city,
+					...(state ? { region: state } : {}),
+					postcode,
+				}
 
-					if (!raw) continue
+				const raw = composeRaw(venue, streetPart, city, state, postcode)
 
-					const aligned = reconcileComponents(components, raw)
+				if (!raw) continue
 
-					if (Object.keys(aligned).length <= 2) continue
+				const aligned = reconcileComponents(components, raw)
 
-					const sourceID = ein
-						? `${USGOV_IRS_BMF_ADAPTER_ID}-${ein}`
-						: stableSourceID(USGOV_IRS_BMF_ADAPTER_ID, aligned)
+				if (Object.keys(aligned).length <= 2) continue
 
-					yield {
-						raw,
-						components: aligned,
-						country: "US",
-						locale: "en-US",
-						source: USGOV_IRS_BMF_ADAPTER_ID,
-						source_id: sourceID,
-						corpus_version: "",
-						license: USGOV_IRS_BMF_DEFAULT_LICENSE,
-					}
+				const sourceID = ein ? `${USGOV_IRS_BMF_ADAPTER_ID}-${ein}` : stableSourceID(USGOV_IRS_BMF_ADAPTER_ID, aligned)
 
-					emitted++
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: USGOV_IRS_BMF_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: USGOV_IRS_BMF_DEFAULT_LICENSE,
 				}
-			} finally {
-				stream.destroy()
+
+				emitted++
 			}
 		},
 	}

package/src/adapters/usgov-nad/adapter.ts

@@ -33,11 +33,11 @@
 import { readdir } from "node:fs/promises"
 import { join } from "node:path"
 
+import { tryParsingJSON } from "@mailwoman/core/objects"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
+import { reconcileComponents } from "@mailwoman/formatter"
 import { TextSpliterator } from "spliterator"
 
-import { reconcileComponents } from "../../format.ts"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
-
 /**
  * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
  * it came from.
@@ -257,8 +257,8 @@ export function createUsgovNADAdapter(): CorpusAdapter {
 			let emitted = 0
 			outer: for (const shard of shards) {
 				if (opts.signal?.aborted) break
-				// TextSpliterator streams string lines; the per-line try/catch below keeps the reader
-				// tolerant of malformed rows (skip silently), so TextSpliterator + explicit JSON.parse —
+				// TextSpliterator streams string lines; the per-line tryParsingJSON below keeps the reader
+				// tolerant of malformed rows (skip silently), so TextSpliterator + a non-throwing parse —
 				// not JSONSpliterator, which would throw. The path string lets the lib own + dispose each
 				// shard's file handle, including on the `break outer` early exit.
 				const lines = TextSpliterator.fromAsync(join(opts.inputPath, shard))
@@ -270,13 +270,9 @@ export function createUsgovNADAdapter(): CorpusAdapter {
 
 					if (!line) continue
 
-					let record: NADRecord
+					const record = tryParsingJSON<NADRecord>(line)
 
-					try {
-						record = JSON.parse(line) as NADRecord
-					} catch {
-						continue // malformed line — skip silently
-					}
+					if (record === null) continue // malformed line — skip silently
 
 					const state = (record.State ?? "").toString().trim().toUpperCase()
 

package/src/adapters/usgov-nppes/adapter.ts

@@ -21,14 +21,11 @@
  *   License: stamped `"Public Domain"` per CMS's federal government distribution terms.
  */
 
-import { createReadStream } from "node:fs"
-
-import { parse as csvParse } from "csv-parse"
-
-import { stableSourceID } from "../../adapter.ts"
-import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
-import { reconcileComponents } from "../../format.ts"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+import { splitStreetLine, stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import { lookupStateAbbreviation } from "@mailwoman/corpus/codex/us-fips-state"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
+import { reconcileComponents } from "@mailwoman/formatter"
+import { CSVSpliterator } from "spliterator"
 
 /**
  * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
@@ -41,8 +38,6 @@ export const USGOV_NPPES_ADAPTER_ID = "usgov-nppes"
  */
 export const USGOV_NPPES_DEFAULT_LICENSE = "Public Domain"
 
-const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
-
 interface NPPESRow {
 	NPI: string
 	"Entity Type Code": string
@@ -56,17 +51,6 @@ interface NPPESRow {
 	"Provider Business Practice Location Address Postal Code": string
 }
 
-function splitAddress(address: string): { house_number?: string; street: string } | null {
-	const trimmed = address.trim()
-
-	if (!trimmed) return null
-	const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
-
-	if (m) return { house_number: m[1], street: m[2]!.trim() }
-
-	return { street: trimmed }
-}
-
 function composeRaw(
 	venue: string | undefined,
 	house: string | undefined,
@@ -93,84 +77,75 @@ export function createUsgovNPPESAdapter(): CorpusAdapter {
 				throw new Error(`usgov-nppes adapter: only US supported, got country=${opts.country}`)
 			}
 
-			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
-
-			const parser = stream.pipe(
-				csvParse({
-					columns: true,
-					skip_empty_lines: true,
-					relax_quotes: true,
-					relax_column_count: true,
-				})
-			)
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				mode: "object",
+				normalizeKeys: false,
+				enableQuoteHandling: true,
+			})
 
 			let emitted = 0
 
-			try {
-				for await (const record of parser as AsyncIterable<NPPESRow>) {
-					if (opts.signal?.aborted) break
+			for await (const record of rows as AsyncIterable<NPPESRow>) {
+				if (opts.signal?.aborted) break
 
-					if (opts.limit !== undefined && emitted >= opts.limit) break
+				if (opts.limit !== undefined && emitted >= opts.limit) break
 
-					const npi = (record.NPI ?? "").trim()
-					const entityType = (record["Entity Type Code"] ?? "").trim()
-					const orgName = (record["Provider Organization Name (Legal Business Name)"] ?? "").trim()
-					const lastName = (record["Provider Last Name (Legal Name)"] ?? "").trim()
-					const firstName = (record["Provider First Name"] ?? "").trim()
+				const npi = (record.NPI ?? "").trim()
+				const entityType = (record["Entity Type Code"] ?? "").trim()
+				const orgName = (record["Provider Organization Name (Legal Business Name)"] ?? "").trim()
+				const lastName = (record["Provider Last Name (Legal Name)"] ?? "").trim()
+				const firstName = (record["Provider First Name"] ?? "").trim()
 
-					const address1 = (record["Provider First Line Business Practice Location Address"] ?? "").trim()
-					const address2 = (record["Provider Second Line Business Practice Location Address"] ?? "").trim()
-					const city = (record["Provider Business Practice Location Address City Name"] ?? "").trim()
-					const stateRaw = (record["Provider Business Practice Location Address State Name"] ?? "").trim()
-					const postcode = (record["Provider Business Practice Location Address Postal Code"] ?? "").trim()
+				const address1 = (record["Provider First Line Business Practice Location Address"] ?? "").trim()
+				const address2 = (record["Provider Second Line Business Practice Location Address"] ?? "").trim()
+				const city = (record["Provider Business Practice Location Address City Name"] ?? "").trim()
+				const stateRaw = (record["Provider Business Practice Location Address State Name"] ?? "").trim()
+				const postcode = (record["Provider Business Practice Location Address Postal Code"] ?? "").trim()
 
-					if (!city || !postcode) continue
+				if (!city || !postcode) continue
 
-					const state = lookupStateAbbreviation(stateRaw)
+				const state = lookupStateAbbreviation(stateRaw)
 
-					if (!state) continue
+				if (!state) continue
 
-					const fullStreet = [address1, address2].filter(Boolean).join(" ")
-					const split = splitAddress(fullStreet)
+				const fullStreet = [address1, address2].filter(Boolean).join(" ")
+				const split = splitStreetLine(fullStreet)
 
-					if (!split) continue
+				if (!split) continue
 
-					const venue = orgName || [firstName, lastName].filter(Boolean).join(" ") || undefined
+				const venue = orgName || [firstName, lastName].filter(Boolean).join(" ") || undefined
 
-					const components: CanonicalRow["components"] = {
-						...(venue ? { venue } : {}),
-						...(split.house_number ? { house_number: split.house_number } : {}),
-						street: split.street,
-						locality: city,
-						region: state.abbreviation,
-						postcode,
-					}
-
-					const raw = composeRaw(venue, split.house_number, split.street, city, state.abbreviation, postcode)
+				const components: CanonicalRow["components"] = {
+					...(venue ? { venue } : {}),
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode,
+				}
 
-					if (!raw) continue
+				const raw = composeRaw(venue, split.house_number, split.street, city, state.abbreviation, postcode)
 
-					const aligned = reconcileComponents(components, raw)
+				if (!raw) continue
 
-					if (Object.keys(aligned).length <= 2) continue
+				const aligned = reconcileComponents(components, raw)
 
-					const sourceID = npi ? `${USGOV_NPPES_ADAPTER_ID}-${npi}` : stableSourceID(USGOV_NPPES_ADAPTER_ID, aligned)
+				if (Object.keys(aligned).length <= 2) continue
 
-					yield {
-						raw,
-						components: aligned,
-						country: "US",
-						locale: "en-US",
-						source: USGOV_NPPES_ADAPTER_ID,
-						source_id: sourceID,
-						corpus_version: "",
-						license: USGOV_NPPES_DEFAULT_LICENSE,
-					}
+				const sourceID = npi ? `${USGOV_NPPES_ADAPTER_ID}-${npi}` : stableSourceID(USGOV_NPPES_ADAPTER_ID, aligned)
 
-					emitted++
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: USGOV_NPPES_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: USGOV_NPPES_DEFAULT_LICENSE,
 				}
-			} finally {
-				stream.destroy()
+
+				emitted++
 			}
 		},
 	}