@mailwoman/corpus 9.4.0 → 10.0.0

package/lib/us/adapters/usgov/irs-bmf/adapter.ts

@@ -0,0 +1,145 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `usgov-irs-bmf`: IRS Exempt Organizations Business Master File (EO BMF) CSV consumer.
+ *
+ *   The EO BMF is the IRS's authoritative registry of US tax-exempt organizations (charities,
+ *   churches, foundations, ...), published as per-region CSVs at
+ *   `https://www.irs.gov/charities-non-profits/exempt-organizations-business-master-file-extract-eo-bmf`
+ *   (`eo1.csv`..`eo4.csv`, `eo_pr.csv`, `eo_xx.csv`). Each row carries an organization NAME plus
+ *   its mailing address. It complements `usgov-nppes` with a DIFFERENT venue population
+ *   (non-profits vs healthcare providers) and, notably, a high share of PO-box addresses — useful
+ *   `po_box`-tag signal (a tag with historically low recall).
+ *
+ *   Output: one row per record with a usable city + postcode. NAME → `venue`; the street line becomes
+ *   `po_box` when it's a PO-box, else `house_number` + `street`; CITY/STATE/ZIP fill the locality
+ *   line. STATE is already a USPS abbreviation in the source. License: `"Public Domain"` (US
+ *   federal).
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address-format"
+import { CSVSpliterator } from "spliterator"
+
+import { splitStreetLine, stableSourceID } from "#adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const USGOV_IRS_BMF_ADAPTER_ID = "usgov-irs-bmf"
+/**
+ * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
+ * than having to look them up.
+ */
+export const USGOV_IRS_BMF_DEFAULT_LICENSE = "Public Domain"
+
+/**
+ * PO box in its many written forms: "PO BOX 12", "P.O. BOX 12", "P O BOX 12", "POB 12", "BOX 12".
+ */
+const PO_BOX = /^\s*(?:P\.?\s?O\.?\s*BOX|POB|BOX)\s+\w/i
+
+interface IRSBMFRow {
+	EIN: string
+	NAME: string
+	STREET: string
+	CITY: string
+	STATE: string
+	ZIP: string
+}
+
+/**
+ * Classify the street line into a `po_box` or a `{house_number?, street}` split.
+ *
+ * BMF mixes street addresses and PO boxes in one `STREET` column, so the PO-box shapes have to be claimed BEFORE the
+ * shared house-number split runs — otherwise `splitStreetLine` would hand back `"PO Box 1234"` as a plain street, which
+ * is correct for every other US adapter and wrong here.
+ */
+function splitStreetLineOrPOBox(street: string): { po_box: string } | { house_number?: string; street: string } | null {
+	const trimmed = street.trim()
+
+	if (PO_BOX.test(trimmed)) return { po_box: trimmed }
+
+	return splitStreetLine(trimmed)
+}
+
+export function createUSGovIRSBMFAdapter(): CorpusAdapter {
+	return {
+		id: USGOV_IRS_BMF_ADAPTER_ID,
+		defaultLicense: USGOV_IRS_BMF_DEFAULT_LICENSE,
+		description:
+			"IRS Exempt Organizations Business Master File — US non-profit venue+address (public-domain), with strong PO-box coverage.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`usgov-irs-bmf adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				normalizeKeys: false,
+			})
+
+			let emitted = 0
+
+			for await (const record of rows as AsyncIterable<IRSBMFRow>) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const ein = (record.EIN ?? "").trim()
+				const venue = (record.NAME ?? "").trim() || undefined
+				const street = (record.STREET ?? "").trim()
+				const city = (record.CITY ?? "").trim()
+				const state = (record.STATE ?? "").trim()
+				const zipRaw = (record.ZIP ?? "").trim()
+
+				if (!city || !zipRaw) continue
+				const postcode = zipRaw.split("-")[0]!.trim() // 5-digit; drop the optional +4
+
+				const split = splitStreetLineOrPOBox(street)
+
+				if (!split) continue
+
+				const components: CanonicalRow["components"] = {
+					...(venue ? { venue } : {}),
+					...("po_box" in split
+						? { po_box: split.po_box }
+						: { ...(split.house_number ? { house_number: split.house_number } : {}), street: split.street }),
+					locality: city,
+					...(state ? { region: state } : {}),
+					postcode,
+				}
+
+				const rendered = formatAddressRow(components, "US", { singleLine: true })
+
+				if (!rendered) continue
+
+				const { raw, components: aligned } = rendered
+
+				if (Object.keys(aligned).length <= 2) continue
+
+				const sourceID = ein ? `${USGOV_IRS_BMF_ADAPTER_ID}-${ein}` : stableSourceID(USGOV_IRS_BMF_ADAPTER_ID, aligned)
+
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: USGOV_IRS_BMF_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: USGOV_IRS_BMF_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const USGovIRSBMFAdapter = createUSGovIRSBMFAdapter()

package/lib/us/adapters/usgov/nad/adapter.ts

@@ -0,0 +1,324 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `usgov-nad`: US DOT National Address Database — ~97M structured address-point records.
+ *
+ *   The single largest US address source available — federal aggregation of state + local 911-grade
+ *   address points (every addressable location). Compared to TIGER ADDRFEAT (~20M segment-level, no
+ *   city/locality) and NPPES (~7M provider-centric venues), NAD covers the entire residential +
+ *   commercial address space with full structured components.
+ *
+ *   The adapter consumes NDJSON slices produced by `fetch-nad.ts`'s featureserver mode (operator
+ *   pre-downloads via `mailwoman corpus fetch nad`). Each slice is per-OID-range
+ *   `oids_<start>-<end>.ndjson` with a sibling `.manifest.json`. Adapter iterates every `.ndjson`
+ *   in the input directory, skipping the `quarantined-bash-bug/` subdir (legacy of the bash-
+ *   fetcher's silent-page-failure bug).
+ *
+ *   Field mapping (NAD v9 → CanonicalRow components):
+ *
+ *   - House_number: `AddNo_Full` (pre-composed); falls back to AddNum_Pre + Add_Number + AddNum_Suf
+ *   - Street: `StNam_Full` (pre-composed); falls back to St_PreDir + St_PreTyp + St_Name + St_PosTyp
+ *
+ *       - St_PosDir + St_PosMod composition
+ *   - Locality: `Post_City` > `Inc_Muni` > `Census_Plc` > `Uninc_Comm` (first non-empty)
+ *   - Region: `State` (2-char USPS code, including territories: PR, GU, VI, AS, MP)
+ *   - Postcode: `Zip_Code` + `Plus_4` (joined as `XXXXX-NNNN` when both present)
+ *   - Venue: `LandmkName` (typically a park, school, hospital, named facility — when present)
+ *
+ *   License: stamped `"Public Domain"` per 17 U.S.C. § 105 (US federal works).
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address-format"
+import { tryParsingJSON } from "@mailwoman/core/json"
+import { isPresent } from "@mailwoman/core/objects"
+import { join } from "path-ts"
+import { TextSpliterator } from "spliterator"
+import { Globerator } from "spliterator/node/fs"
+
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const USGOV_NAD_ADAPTER_ID = "usgov-nad"
+/**
+ * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
+ * than having to look them up.
+ */
+export const USGOV_NAD_DEFAULT_LICENSE = "Public Domain"
+
+interface NADRecord {
+	OBJECTID?: number
+	UUID?: string | null
+	// House number
+	AddNum_Pre?: string | null
+	Add_Number?: number | string | null
+	AddNum_Suf?: string | null
+	AddNo_Full?: string | null
+	// Street parts
+	St_PreMod?: string | null
+	St_PreDir?: string | null
+	St_PreTyp?: string | null
+	St_PreSep?: string | null
+	St_Name?: string | null
+	St_PosTyp?: string | null
+	St_PosDir?: string | null
+	St_PosMod?: string | null
+	StNam_Full?: string | null
+	// Sub-address (carried as part of street for now; Phase 1 has no unit/floor labels)
+	Building?: string | null
+	Floor?: string | null
+	Unit?: string | null
+	Room?: string | null
+	Seat?: string | null
+	Addtl_Loc?: string | null
+	SubAddress?: string | null
+	// Landmark / venue
+	LandmkName?: string | null
+	// Locality alternates (we prefer Post_City for what a human would type)
+	County?: string | null
+	Inc_Muni?: string | null
+	Post_City?: string | null
+	Census_Plc?: string | null
+	Uninc_Comm?: string | null
+	Nbrhd_Comm?: string | null
+	NatAmArea?: string | null
+	NatAmSub?: string | null
+	Urbnztn_PR?: string | null
+	PlaceOther?: string | null
+	PlaceNmTyp?: string | null
+	// State + ZIP
+	State?: string | null
+	Zip_Code?: string | null
+	Plus_4?: string | null
+}
+
+const US_STATES_SET = new Set([
+	"AL",
+	"AK",
+	"AZ",
+	"AR",
+	"CA",
+	"CO",
+	"CT",
+	"DE",
+	"DC",
+	"FL",
+	"GA",
+	"HI",
+	"ID",
+	"IL",
+	"IN",
+	"IA",
+	"KS",
+	"KY",
+	"LA",
+	"ME",
+	"MD",
+	"MA",
+	"MI",
+	"MN",
+	"MS",
+	"MO",
+	"MT",
+	"NE",
+	"NV",
+	"NH",
+	"NJ",
+	"NM",
+	"NY",
+	"NC",
+	"ND",
+	"OH",
+	"OK",
+	"OR",
+	"PA",
+	"RI",
+	"SC",
+	"SD",
+	"TN",
+	"TX",
+	"UT",
+	"VT",
+	"VA",
+	"WA",
+	"WV",
+	"WI",
+	"WY",
+	// Territories that ship in NAD
+	"PR",
+	"GU",
+	"VI",
+	"AS",
+	"MP",
+])
+
+function nonEmpty(...values: Array<string | null | undefined>): string | undefined {
+	for (const v of values) {
+		const trimmed = (v ?? "").toString().trim()
+
+		if (trimmed) return trimmed
+	}
+
+	return undefined
+}
+
+function composeHouseNumber(r: NADRecord): string | undefined {
+	const full = (r.AddNo_Full ?? "").toString().trim()
+
+	if (full) return full
+	const num = r.Add_Number == null ? "" : String(r.Add_Number).trim()
+
+	if (!num) return undefined
+	const pre = (r.AddNum_Pre ?? "").toString().trim()
+	const suf = (r.AddNum_Suf ?? "").toString().trim()
+
+	return [pre, num, suf].filter(isPresent).join(" ").trim() || undefined
+}
+
+interface DecomposedNADStreet {
+	prefix?: string
+	street?: string
+	suffix?: string
+	full: string
+}
+
+function decomposeNADStreet(r: NADRecord): DecomposedNADStreet | undefined {
+	const name = (r.St_Name ?? "").toString().trim()
+
+	if (name) {
+		const preDir = (r.St_PreDir ?? "").toString().trim()
+		const preTyp = (r.St_PreTyp ?? "").toString().trim()
+		const preSep = (r.St_PreSep ?? "").toString().trim()
+		const posTyp = (r.St_PosTyp ?? "").toString().trim()
+		const posDir = (r.St_PosDir ?? "").toString().trim()
+		const prefix = [preDir, preTyp, preSep].filter(isPresent).join(" ") || undefined
+		const suffix = [posTyp, posDir].filter(isPresent).join(" ") || undefined
+		const full = [prefix, name, suffix].filter(isPresent).join(" ")
+
+		return { prefix, street: name, suffix, full }
+	}
+
+	const full = (r.StNam_Full ?? "").toString().trim()
+
+	if (full) return { full, street: full }
+
+	return undefined
+}
+
+function composeLocality(r: NADRecord): string | undefined {
+	return nonEmpty(r.Post_City, r.Inc_Muni, r.Census_Plc, r.Uninc_Comm)
+}
+
+function composePostcode(r: NADRecord): string | undefined {
+	const zip = (r.Zip_Code ?? "").toString().trim()
+
+	if (!zip) return undefined
+	const plus4 = (r.Plus_4 ?? "").toString().trim()
+
+	return plus4 ? `${zip}-${plus4}` : zip
+}
+
+export function createUsgovNADAdapter(): CorpusAdapter {
+	return {
+		id: USGOV_NAD_ADAPTER_ID,
+		defaultLicense: USGOV_NAD_DEFAULT_LICENSE,
+		description:
+			"US DOT National Address Database — ~97M structured US address points (911-grade). Single largest US source.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`usgov-nad adapter: only US supported, got country=${opts.country}`)
+			}
+
+			// inputPath is a directory of NDJSON slices (per fetch-nad.ts featureserver output).
+			// Single-file inputs (e.g. a bulk-extracted CSV) are not currently supported — the
+			// featureserver slice pattern is the primary distribution.
+			const slices = await Globerator.files("ndjson", { cwd: opts.inputPath, absolute: false }).toSorted()
+
+			let emitted = 0
+			outer: for (const slice of slices) {
+				if (opts.signal?.aborted) break
+				// TextSpliterator streams string lines; the per-line tryParsingJSON below keeps the reader
+				// tolerant of malformed rows (skip silently), so TextSpliterator + a non-throwing parse —
+				// not JSONSpliterator, which would throw. The path string lets the lib own + dispose each
+				// slice's file handle, including on the `break outer` early exit.
+				const lines = TextSpliterator.fromAsync(join(opts.inputPath, slice))
+
+				for await (const line of lines) {
+					if (opts.signal?.aborted) break outer
+
+					if (opts.limit !== undefined && emitted >= opts.limit) break outer
+
+					if (!line) continue
+
+					const record = tryParsingJSON<NADRecord>(line)
+
+					if (record === null) continue // malformed line — skip silently
+
+					const state = (record.State ?? "").toString().trim().toUpperCase()
+
+					if (!US_STATES_SET.has(state)) continue
+
+					const locality = composeLocality(record)
+
+					if (!locality) continue
+
+					const postcode = composePostcode(record)
+
+					if (!postcode) continue
+
+					const decomposed = decomposeNADStreet(record)
+					const houseNumber = composeHouseNumber(record)
+					const venue = nonEmpty(record.LandmkName)
+					const unit = nonEmpty(record.Unit, record.Building, record.Floor, record.Room)
+
+					const components: CanonicalRow["components"] = {
+						...(venue ? { venue } : {}),
+						...(houseNumber ? { house_number: houseNumber } : {}),
+						...(decomposed?.prefix ? { street_prefix: decomposed.prefix } : {}),
+						...(decomposed?.street ? { street: decomposed.street } : {}),
+						...(decomposed?.suffix ? { street_suffix: decomposed.suffix } : {}),
+						...(unit ? { unit } : {}),
+						locality,
+						region: state,
+						postcode,
+					}
+
+					const rendered = formatAddressRow(components, "US", { singleLine: true })
+
+					if (!rendered) continue
+
+					const { raw, components: aligned } = rendered
+
+					if (Object.keys(aligned).length <= 2) continue
+
+					const sourceID = record.UUID
+						? `${USGOV_NAD_ADAPTER_ID}-${record.UUID}`
+						: `${USGOV_NAD_ADAPTER_ID}-${record.OBJECTID ?? `${slice}:${emitted}`}`
+
+					yield {
+						raw,
+						components: aligned,
+						country: "US",
+						locale: "en-US",
+						source: USGOV_NAD_ADAPTER_ID,
+						source_id: sourceID,
+						corpus_version: "",
+						license: USGOV_NAD_DEFAULT_LICENSE,
+					}
+
+					emitted++
+				}
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const usgovNADAdapter = createUsgovNADAdapter()

package/lib/us/adapters/usgov/nppes/adapter.ts

@@ -0,0 +1,143 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `usgov-nppes`: CMS National Plan and Provider Enumeration System (NPI registry) CSV consumer.
+ *
+ *   NPPES is the authoritative US healthcare provider registry, published monthly by CMS. Each row
+ *   carries a provider's business practice location address together with their legal business name
+ *   or individual name. At ~7M rows it is the single largest venue+address signal source
+ *   available.
+ *
+ *   The adapter consumes the monthly full-replacement CSV (operator pre-downloads via
+ *   `fetch-nppes.ts`). Column names match the canonical NPPES "Full Replacement Monthly NPI File"
+ *   header published at `https://download.cms.gov/nppes/NPI_Files.html`.
+ *
+ *   Output: one row per CSV record where the practice location address is populated. Organization
+ *   rows carry `venue` from the legal business name; individual rows compose `attention` from
+ *   last+first name. Address quad goes on `(house_number, street, locality, region, postcode)`.
+ *
+ *   License: stamped `"Public Domain"` per CMS's federal government distribution terms.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address-format"
+import { isPresent } from "@mailwoman/core/objects"
+import { formatPersonName } from "@mailwoman/record/name"
+import { CSVSpliterator } from "spliterator"
+
+import { splitStreetLine, stableSourceID } from "#adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+import { lookupStateAbbreviation } from "#us/fips-state"
+
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const USGOV_NPPES_ADAPTER_ID = "usgov-nppes"
+/**
+ * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
+ * than having to look them up.
+ */
+export const USGOV_NPPES_DEFAULT_LICENSE = "Public Domain"
+
+interface NPPESRow {
+	NPI: string
+	"Entity Type Code": string
+	"Provider Organization Name (Legal Business Name)": string
+	"Provider Last Name (Legal Name)": string
+	"Provider First Name": string
+	"Provider First Line Business Practice Location Address": string
+	"Provider Second Line Business Practice Location Address": string
+	"Provider Business Practice Location Address City Name": string
+	"Provider Business Practice Location Address State Name": string
+	"Provider Business Practice Location Address Postcode": string
+}
+
+export function createUsgovNPPESAdapter(): CorpusAdapter {
+	return {
+		id: USGOV_NPPES_ADAPTER_ID,
+		defaultLicense: USGOV_NPPES_DEFAULT_LICENSE,
+		description:
+			"CMS National Plan and Provider Enumeration System — 7M provider practice locations (public-domain). Venue+address co-occurrence at scale.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`usgov-nppes adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				normalizeKeys: false,
+			})
+
+			let emitted = 0
+
+			for await (const record of rows as AsyncIterable<NPPESRow>) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const npi = (record.NPI ?? "").trim()
+				const orgName = (record["Provider Organization Name (Legal Business Name)"] ?? "").trim()
+				const lastName = (record["Provider Last Name (Legal Name)"] ?? "").trim()
+				const firstName = (record["Provider First Name"] ?? "").trim()
+
+				const address1 = (record["Provider First Line Business Practice Location Address"] ?? "").trim()
+				const address2 = (record["Provider Second Line Business Practice Location Address"] ?? "").trim()
+				const city = (record["Provider Business Practice Location Address City Name"] ?? "").trim()
+				const stateRaw = (record["Provider Business Practice Location Address State Name"] ?? "").trim()
+				const postcode = (record["Provider Business Practice Location Address Postcode"] ?? "").trim()
+
+				if (!city || !postcode) continue
+
+				const state = lookupStateAbbreviation(stateRaw)
+
+				if (!state) continue
+
+				const fullStreet = [address1, address2].filter(isPresent).join(" ")
+				const split = splitStreetLine(fullStreet)
+
+				if (!split) continue
+
+				const venue = orgName || formatPersonName({ given: firstName, family: lastName }, "short") || undefined
+
+				const components: CanonicalRow["components"] = {
+					...(venue ? { venue } : {}),
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode,
+				}
+
+				const rendered = formatAddressRow(components, "US", { singleLine: true })
+
+				if (!rendered) continue
+
+				const { raw, components: aligned } = rendered
+
+				if (Object.keys(aligned).length <= 2) continue
+
+				const sourceID = npi ? `${USGOV_NPPES_ADAPTER_ID}-${npi}` : stableSourceID(USGOV_NPPES_ADAPTER_ID, aligned)
+
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: USGOV_NPPES_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: USGOV_NPPES_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const usgovNPPESAdapter = createUsgovNPPESAdapter()