@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/us/adapters/usgov/imls-pls/adapter.ts

@@ -0,0 +1,140 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `usgov-imls-pls`: IMLS Public Libraries Survey outlet CSV consumer.
+ *
+ *   The Institute of Museum and Library Services publishes an annual Public Libraries Survey with one
+ *   row per library outlet (~17K rows). Each row includes the library name, street address, city,
+ *   ZIP, county and geocoordinates.
+ *
+ *   The adapter consumes the outlet CSV the operator pre-downloads via `fetch-imls-pls.ts`. Column
+ *   names match the IMLS PLS outlet file header.
+ *
+ *   Output: one row per outlet with `venue` (library name), `(house_number, street, locality,
+ *   subregion, postcode)`, and lat/lon preserved in `source_id` stability.
+ *
+ *   License: stamped `"Public Domain"` per IMLS federal government distribution terms.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+import { CSVSpliterator } from "spliterator"
+
+import { stableSourceID } from "#adapters/source-id"
+import { splitStreetLine } from "#adapters/street-line"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
+import { lookupStateAbbreviation } from "#us/fips-state"
+
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ */
+export const USGOV_IMLS_PLS_ADAPTER_ID = "usgov-imls-pls"
+/**
+ * License assigned by this source (Public Domain), attached to each row so downstream
+ * consumers inherit the terms rather than having to look them up.
+ */
+export const USGOV_IMLS_PLS_DEFAULT_LICENSE = "Public Domain"
+
+interface ImlsOutletRow {
+	LIBNAME: string
+	ADDRESS: string
+	CITY: string
+	ZIP: string
+	STABR: string
+	CNTY: string
+	FSCSKEY: string
+}
+
+export function createUSGovIMLSPLSAdapter(): CorpusAdapter {
+	return {
+		id: USGOV_IMLS_PLS_ADAPTER_ID,
+		defaultLicense: USGOV_IMLS_PLS_DEFAULT_LICENSE,
+		addressRole: AddressRole.Facility,
+		register: SourceRegister.IMLSPublicLibraries,
+		surface: SurfaceOrigin.Rendered,
+		description: "IMLS Public Libraries Survey — ~17K library outlets with venue+address (public-domain).",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`usgov-imls-pls adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				normalizeKeys: false,
+			})
+
+			let emitted = 0
+
+			for await (const record of rows as AsyncIterable<ImlsOutletRow>) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const libName = record.LIBNAME ?? ""
+				const address = record.ADDRESS ?? ""
+				const city = record.CITY ?? ""
+				const zip = record.ZIP ?? ""
+				const stateAbbr = record.STABR ?? ""
+
+				if (!libName || !city || !zip) continue
+
+				const state = lookupStateAbbreviation(stateAbbr)
+
+				if (!state) continue
+
+				const split = splitStreetLine(address)
+
+				if (!split) continue
+
+				const components: CanonicalRow["components"] = {
+					venue: libName,
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode: zip,
+					// No subregion: US postal addresses don't surface the county, so emitting subregion
+					// creates a phantom component with no raw-span to align to, quarantining ~21% of rows.
+					// The county is still available in the source CSV.
+					// It is not a postal-surface component here.
+				}
+
+				const rendered = formatAddressRow(components, "US", { singleLine: true })
+
+				if (!rendered) continue
+
+				const { raw, components: aligned } = rendered
+
+				if (Object.keys(aligned).length <= 2) continue
+
+				const fscsKey = record.FSCSKEY ?? ""
+
+				const sourceID = fscsKey
+					? `${USGOV_IMLS_PLS_ADAPTER_ID}-${fscsKey}`
+					: stableSourceID(USGOV_IMLS_PLS_ADAPTER_ID, aligned)
+
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: USGOV_IMLS_PLS_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: USGOV_IMLS_PLS_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const usgovImlsPlsAdapter = createUSGovIMLSPLSAdapter()

package/lib/us/adapters/usgov/irs-bmf/adapter.ts

@@ -0,0 +1,154 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `usgov-irs-bmf`: IRS Exempt Organizations Business Master File (EO BMF) CSV consumer.
+ *
+ *   The EO BMF is the IRS's authoritative registry of US tax-exempt organizations (charities,
+ *   churches, foundations, ...), published as per-region CSVs at
+ *   `https://www.irs.gov/charities-non-profits/exempt-organizations-business-master-file-extract-eo-bmf`
+ *   (`eo1.csv`..`eo4.csv`, `eo_pr.csv`, `eo_xx.csv`). Each row includes an organization name plus
+ *   its mailing address. It complements `usgov-nppes` with a different venue population
+ *   (non-profits vs healthcare providers) and, notably, a high share of PO-box addresses — useful
+ *   `po_box`-tag signal (a tag with historically low recall).
+ *
+ *   Output: one row per record with a usable city + postcode. name → `venue`; the street line becomes
+ *   `po_box` when it's a PO-box, else `house_number` + `street`; city/state/ZIP fill the locality
+ *   line. state is already a USPS abbreviation in the source. License: `"Public Domain"` (US
+ *   federal).
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+import { CSVSpliterator } from "spliterator"
+
+import { stableSourceID } from "#adapters/source-id"
+import { splitStreetLine } from "#adapters/street-line"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
+
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ */
+export const USGOV_IRS_BMF_ADAPTER_ID = "usgov-irs-bmf"
+/**
+ * License assigned by this source (Public Domain), attached to each row so downstream
+ * consumers inherit the terms rather than having to look them up.
+ */
+export const USGOV_IRS_BMF_DEFAULT_LICENSE = "Public Domain"
+
+/**
+ * PO box in its many written forms: "PO BOX 12", "P.O.
+ *
+ * BOX 12", "P O BOX 12", "POB 12", "BOX 12".
+ */
+const PO_BOX = /^\s*(?:P\.?\s?O\.?\s*BOX|POB|BOX)\s+\w/i
+
+interface IRSBMFRow {
+	EIN: string
+	NAME: string
+	STREET: string
+	CITY: string
+	STATE: string
+	ZIP: string
+}
+
+/**
+ * Classify the street line into a `po_box` or a `{house_number?, street}` split.
+ *
+ * BMF mixes street addresses and PO boxes in one `street` column, so the PO-box shapes
+ * have to be claimed before the shared house-number split runs.
+ * Otherwise `splitStreetLine` would hand back `"PO Box 1234"` as a plain street.
+ * which is correct for every other US adapter and wrong here.
+ */
+function splitStreetLineOrPOBox(street: string): { po_box: string } | { house_number?: string; street: string } | null {
+	const trimmed = street.trim()
+
+	if (PO_BOX.test(trimmed)) return { po_box: trimmed }
+
+	return splitStreetLine(trimmed)
+}
+
+export function createUSGovIRSBMFAdapter(): CorpusAdapter {
+	return {
+		id: USGOV_IRS_BMF_ADAPTER_ID,
+		defaultLicense: USGOV_IRS_BMF_DEFAULT_LICENSE,
+		addressRole: AddressRole.Mailing,
+		register: SourceRegister.IRSBusinessMasterFile,
+		surface: SurfaceOrigin.Rendered,
+		description:
+			"IRS Exempt Organizations Business Master File — US non-profit venue+address (public-domain), with strong PO-box coverage.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`usgov-irs-bmf adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				normalizeKeys: false,
+			})
+
+			let emitted = 0
+
+			for await (const record of rows as AsyncIterable<IRSBMFRow>) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const ein = record.EIN ?? ""
+				const venue = (record.NAME ?? "") || undefined
+				const street = record.STREET ?? ""
+				const city = record.CITY ?? ""
+				const state = record.STATE ?? ""
+				const zipRaw = record.ZIP ?? ""
+
+				if (!city || !zipRaw) continue
+				const postcode = zipRaw.split("-")[0]!.trim() // 5-digit. drop the optional +4
+
+				const split = splitStreetLineOrPOBox(street)
+
+				if (!split) continue
+
+				const components: CanonicalRow["components"] = {
+					...(venue ? { venue } : {}),
+					...("po_box" in split
+						? { po_box: split.po_box }
+						: { ...(split.house_number ? { house_number: split.house_number } : {}), street: split.street }),
+					locality: city,
+					...(state ? { region: state } : {}),
+					postcode,
+				}
+
+				const rendered = formatAddressRow(components, "US", { singleLine: true })
+
+				if (!rendered) continue
+
+				const { raw, components: aligned } = rendered
+
+				if (Object.keys(aligned).length <= 2) continue
+
+				const sourceID = ein ? `${USGOV_IRS_BMF_ADAPTER_ID}-${ein}` : stableSourceID(USGOV_IRS_BMF_ADAPTER_ID, aligned)
+
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: USGOV_IRS_BMF_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: USGOV_IRS_BMF_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const USGovIRSBMFAdapter = createUSGovIRSBMFAdapter()

package/lib/us/adapters/usgov/nad/adapter.ts

@@ -0,0 +1,263 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Reads the US DOT National Address Database from a directory of NDJSON files written by the NAD fetcher.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+import { US_STATE_ABBREVIATIONS } from "@mailwoman/codex/us/state"
+import { tryParsingJSON } from "@mailwoman/core/json"
+import { isPresent } from "@mailwoman/core/objects"
+import { resolvePathBuilder } from "path-ts"
+import { TextSpliterator } from "spliterator"
+import { Globerator } from "spliterator/node/fs"
+
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
+
+/**
+ * The adapter id stamped on every emitted row.
+ */
+export const USGOV_NAD_ADAPTER_ID = "usgov-nad"
+/**
+ * The license label on every emitted row, because US federal works are public domain under 17 U.S.C. § 105.
+ */
+export const USGOV_NAD_DEFAULT_LICENSE = "Public Domain"
+
+interface NADRecord {
+	OBJECTID?: number
+	UUID?: string | null
+	AddNum_Pre?: string | null
+	Add_Number?: number | string | null
+	AddNum_Suf?: string | null
+	AddNo_Full?: string | null
+	St_PreMod?: string | null
+	St_PreDir?: string | null
+	St_PreTyp?: string | null
+	St_PreSep?: string | null
+	St_Name?: string | null
+	St_PosTyp?: string | null
+	St_PosDir?: string | null
+	St_PosMod?: string | null
+	StNam_Full?: string | null
+	Building?: string | null
+	Floor?: string | null
+	Unit?: string | null
+	Room?: string | null
+	Seat?: string | null
+	Addtl_Loc?: string | null
+	SubAddress?: string | null
+	LandmkName?: string | null
+	County?: string | null
+	Inc_Muni?: string | null
+	Post_City?: string | null
+	Census_Plc?: string | null
+	Uninc_Comm?: string | null
+	Nbrhd_Comm?: string | null
+	NatAmArea?: string | null
+	NatAmSub?: string | null
+	Urbnztn_PR?: string | null
+	PlaceOther?: string | null
+	PlaceNmTyp?: string | null
+	State?: string | null
+	Zip_Code?: string | null
+	Plus_4?: string | null
+}
+
+/**
+ * The USPS abbreviations a NAD row's `State` must match, taken from codex rather than restated.
+ *
+ * `US_STATE_ABBREVIATIONS` holds the 50 states, the District of Columbia and the five primary territories.
+ * That is the set NAD publishes under, so this adapter keeps no list of its own.
+ *
+ * The two had identical membership when they were reconciled, at 56 entries each.
+ */
+const US_STATES_SET: ReadonlySet<string> = new Set(US_STATE_ABBREVIATIONS)
+
+function nonEmpty(...values: Array<string | null | undefined>): string | undefined {
+	for (const v of values) {
+		const trimmed = (v ?? "").toString().trim()
+
+		if (trimmed) return trimmed
+	}
+
+	return undefined
+}
+
+/**
+ * The house number a NAD record states, read from its own four columns.
+ *
+ * NAD publishes `AddNo_Full` already assembled and the parts beside it, so this prefers the
+ * assembled column and falls back to joining `AddNum_Pre`, `Add_Number` and `AddNum_Suf`.
+ * The shared `composeHouseNumber` in `#adapters/street-line` joins exactly two columns with
+ * one separator, so it fits a publisher that splits the number in two rather than in four.
+ */
+function nadHouseNumber(r: NADRecord): string | undefined {
+	const full = (r.AddNo_Full ?? "").toString().trim()
+
+	if (full) return full
+	const num = r.Add_Number == null ? "" : String(r.Add_Number).trim()
+
+	if (!num) return undefined
+	const pre = (r.AddNum_Pre ?? "").toString().trim()
+	const suf = (r.AddNum_Suf ?? "").toString().trim()
+
+	return [pre, num, suf].filter(isPresent).join(" ").trim() || undefined
+}
+
+interface DecomposedNADStreet {
+	prefix?: string
+	street?: string
+	suffix?: string
+	full: string
+}
+
+// Structured street fields take precedence.
+// If they are empty, `StNam_Full` supplies the whole street.
+function decomposeNADStreet(r: NADRecord): DecomposedNADStreet | undefined {
+	const name = (r.St_Name ?? "").toString().trim()
+
+	if (name) {
+		const preDir = (r.St_PreDir ?? "").toString().trim()
+		const preTyp = (r.St_PreTyp ?? "").toString().trim()
+		const preSep = (r.St_PreSep ?? "").toString().trim()
+		const posTyp = (r.St_PosTyp ?? "").toString().trim()
+		const posDir = (r.St_PosDir ?? "").toString().trim()
+		const prefix = [preDir, preTyp, preSep].filter(isPresent).join(" ") || undefined
+		const suffix = [posTyp, posDir].filter(isPresent).join(" ") || undefined
+		const full = [prefix, name, suffix].filter(isPresent).join(" ")
+
+		return { prefix, street: name, suffix, full }
+	}
+
+	const full = (r.StNam_Full ?? "").toString().trim()
+
+	if (full) return { full, street: full }
+
+	return undefined
+}
+
+// The postal city comes first because it is the name that people write on mail.
+function composeLocality(r: NADRecord): string | undefined {
+	return nonEmpty(r.Post_City, r.Inc_Muni, r.Census_Plc, r.Uninc_Comm)
+}
+
+function composePostcode(r: NADRecord): string | undefined {
+	const zip = (r.Zip_Code ?? "").toString().trim()
+
+	if (!zip) return undefined
+	const plus4 = (r.Plus_4 ?? "").toString().trim()
+
+	return plus4 ? `${zip}-${plus4}` : zip
+}
+
+/**
+ * Creates the NAD adapter.
+ *
+ * The adapter skips records without a US state code, locality, or ZIP code.
+ * It also skips rows whose rendering keeps two or fewer components.
+ */
+export function createUsgovNADAdapter(): CorpusAdapter {
+	return {
+		id: USGOV_NAD_ADAPTER_ID,
+		defaultLicense: USGOV_NAD_DEFAULT_LICENSE,
+		addressRole: AddressRole.Premise,
+		register: SourceRegister.NationalAddressDatabase,
+		surface: SurfaceOrigin.Rendered,
+		description:
+			"US DOT National Address Database — ~97M structured US address points (911-grade). Single largest US source.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`usgov-nad adapter: only US supported, got country=${opts.country}`)
+			}
+
+			// The input must be a directory of NDJSON files.
+			const files = await Globerator.files("ndjson", {
+				cwd: opts.inputPath,
+				absolute: false,
+				recursive: false,
+			}).toSorted()
+
+			let emitted = 0
+			outer: for (const file of files) {
+				if (opts.signal?.aborted) break
+				// The spliterator closes the file when the loop exits early.
+				const lines = TextSpliterator.fromAsync(resolvePathBuilder(opts.inputPath, file))
+
+				for await (const line of lines) {
+					if (opts.signal?.aborted) break outer
+
+					if (opts.limit !== undefined && emitted >= opts.limit) break outer
+
+					if (!line) continue
+
+					const record = tryParsingJSON<NADRecord>(line)
+
+					if (record === null) continue
+
+					const state = (record.State ?? "").toString().trim().toUpperCase()
+
+					if (!US_STATES_SET.has(state)) continue
+
+					const locality = composeLocality(record)
+
+					if (!locality) continue
+
+					const postcode = composePostcode(record)
+
+					if (!postcode) continue
+
+					const decomposed = decomposeNADStreet(record)
+					const houseNumber = nadHouseNumber(record)
+					const venue = nonEmpty(record.LandmkName)
+					const unit = nonEmpty(record.Unit, record.Building, record.Floor, record.Room)
+
+					const components: CanonicalRow["components"] = {
+						...(venue ? { venue } : {}),
+						...(houseNumber ? { house_number: houseNumber } : {}),
+						...(decomposed?.prefix ? { street_prefix: decomposed.prefix } : {}),
+						...(decomposed?.street ? { street: decomposed.street } : {}),
+						...(decomposed?.suffix ? { street_suffix: decomposed.suffix } : {}),
+						...(unit ? { unit } : {}),
+						locality,
+						region: state,
+						postcode,
+					}
+
+					const rendered = formatAddressRow(components, "US", { singleLine: true })
+
+					if (!rendered) continue
+
+					const { raw, components: aligned } = rendered
+
+					if (Object.keys(aligned).length <= 2) continue
+
+					const sourceID = record.UUID
+						? `${USGOV_NAD_ADAPTER_ID}-${record.UUID}`
+						: `${USGOV_NAD_ADAPTER_ID}-${record.OBJECTID ?? `${file}:${emitted}`}`
+
+					yield {
+						raw,
+						components: aligned,
+						country: "US",
+						locale: "en-US",
+						source: USGOV_NAD_ADAPTER_ID,
+						source_id: sourceID,
+						corpus_version: "",
+						license: USGOV_NAD_DEFAULT_LICENSE,
+					}
+
+					emitted++
+				}
+			}
+		},
+	}
+}
+
+/**
+ * The NAD adapter instance that the corpus builder registers.
+ */
+export const usgovNADAdapter = createUsgovNADAdapter()