@mailwoman/corpus 7.2.0 → 7.2.1

package/src/adapters/usgov-nad/adapter.ts

@@ -0,0 +1,338 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `usgov-nad`: US DOT National Address Database — ~97M structured address-point records.
+ *
+ *   The single largest US address source available — federal aggregation of state + local 911-grade
+ *   address points (every addressable location). Compared to TIGER ADDRFEAT (~20M segment-level, no
+ *   city/locality) and NPPES (~7M provider-centric venues), NAD covers the entire residential +
+ *   commercial address space with full structured components.
+ *
+ *   The adapter consumes NDJSON shards produced by `fetch-nad.ts`'s featureserver mode (operator
+ *   pre-downloads via `mailwoman corpus fetch nad`). Each shard is per-OID-range
+ *   `oids_<start>-<end>.ndjson` with a sibling `.manifest.json`. Adapter iterates every `.ndjson`
+ *   in the input directory, skipping the `quarantined-bash-bug/` subdir (legacy of the bash-
+ *   fetcher's silent-page-failure bug).
+ *
+ *   Field mapping (NAD v9 → CanonicalRow components):
+ *
+ *   - House_number: `AddNo_Full` (pre-composed); falls back to AddNum_Pre + Add_Number + AddNum_Suf
+ *   - Street: `StNam_Full` (pre-composed); falls back to St_PreDir + St_PreTyp + St_Name + St_PosTyp
+ *
+ *       - St_PosDir + St_PosMod composition
+ *   - Locality: `Post_City` > `Inc_Muni` > `Census_Plc` > `Uninc_Comm` (first non-empty)
+ *   - Region: `State` (2-char USPS code, including territories: PR, GU, VI, AS, MP)
+ *   - Postcode: `Zip_Code` + `Plus_4` (joined as `XXXXX-NNNN` when both present)
+ *   - Venue: `LandmkName` (typically a park, school, hospital, named facility — when present)
+ *
+ *   License: stamped `"Public Domain"` per 17 U.S.C. § 105 (US federal works).
+ */
+
+import { readdir } from "node:fs/promises"
+import { join } from "node:path"
+
+import { TextSpliterator } from "spliterator"
+
+import { reconcileComponents } from "../../format.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+
+export const USGOV_NAD_ADAPTER_ID = "usgov-nad"
+export const USGOV_NAD_DEFAULT_LICENSE = "Public Domain"
+
+interface NADRecord {
+	OBJECTID?: number
+	UUID?: string | null
+	// House number
+	AddNum_Pre?: string | null
+	Add_Number?: number | string | null
+	AddNum_Suf?: string | null
+	AddNo_Full?: string | null
+	// Street parts
+	St_PreMod?: string | null
+	St_PreDir?: string | null
+	St_PreTyp?: string | null
+	St_PreSep?: string | null
+	St_Name?: string | null
+	St_PosTyp?: string | null
+	St_PosDir?: string | null
+	St_PosMod?: string | null
+	StNam_Full?: string | null
+	// Sub-address (carried as part of street for now; Phase 1 has no unit/floor labels)
+	Building?: string | null
+	Floor?: string | null
+	Unit?: string | null
+	Room?: string | null
+	Seat?: string | null
+	Addtl_Loc?: string | null
+	SubAddress?: string | null
+	// Landmark / venue
+	LandmkName?: string | null
+	// Locality alternates (we prefer Post_City for what a human would type)
+	County?: string | null
+	Inc_Muni?: string | null
+	Post_City?: string | null
+	Census_Plc?: string | null
+	Uninc_Comm?: string | null
+	Nbrhd_Comm?: string | null
+	NatAmArea?: string | null
+	NatAmSub?: string | null
+	Urbnztn_PR?: string | null
+	PlaceOther?: string | null
+	PlaceNmTyp?: string | null
+	// State + ZIP
+	State?: string | null
+	Zip_Code?: string | null
+	Plus_4?: string | null
+}
+
+const US_STATES_SET = new Set([
+	"AL",
+	"AK",
+	"AZ",
+	"AR",
+	"CA",
+	"CO",
+	"CT",
+	"DE",
+	"DC",
+	"FL",
+	"GA",
+	"HI",
+	"ID",
+	"IL",
+	"IN",
+	"IA",
+	"KS",
+	"KY",
+	"LA",
+	"ME",
+	"MD",
+	"MA",
+	"MI",
+	"MN",
+	"MS",
+	"MO",
+	"MT",
+	"NE",
+	"NV",
+	"NH",
+	"NJ",
+	"NM",
+	"NY",
+	"NC",
+	"ND",
+	"OH",
+	"OK",
+	"OR",
+	"PA",
+	"RI",
+	"SC",
+	"SD",
+	"TN",
+	"TX",
+	"UT",
+	"VT",
+	"VA",
+	"WA",
+	"WV",
+	"WI",
+	"WY",
+	// Territories that ship in NAD
+	"PR",
+	"GU",
+	"VI",
+	"AS",
+	"MP",
+])
+
+function nonEmpty(...values: Array<string | null | undefined>): string | undefined {
+	for (const v of values) {
+		const trimmed = (v ?? "").toString().trim()
+
+		if (trimmed) return trimmed
+	}
+
+	return undefined
+}
+
+function composeHouseNumber(r: NADRecord): string | undefined {
+	const full = (r.AddNo_Full ?? "").toString().trim()
+
+	if (full) return full
+	const num = r.Add_Number == null ? "" : String(r.Add_Number).trim()
+
+	if (!num) return undefined
+	const pre = (r.AddNum_Pre ?? "").toString().trim()
+	const suf = (r.AddNum_Suf ?? "").toString().trim()
+
+	return [pre, num, suf].filter(Boolean).join(" ").trim() || undefined
+}
+
+interface DecomposedNADStreet {
+	prefix?: string
+	street?: string
+	suffix?: string
+	full: string
+}
+
+function decomposeNADStreet(r: NADRecord): DecomposedNADStreet | undefined {
+	const name = (r.St_Name ?? "").toString().trim()
+
+	if (name) {
+		const preDir = (r.St_PreDir ?? "").toString().trim()
+		const preTyp = (r.St_PreTyp ?? "").toString().trim()
+		const preSep = (r.St_PreSep ?? "").toString().trim()
+		const posTyp = (r.St_PosTyp ?? "").toString().trim()
+		const posDir = (r.St_PosDir ?? "").toString().trim()
+		const prefix = [preDir, preTyp, preSep].filter(Boolean).join(" ") || undefined
+		const suffix = [posTyp, posDir].filter(Boolean).join(" ") || undefined
+		const full = [prefix, name, suffix].filter(Boolean).join(" ")
+
+		return { prefix, street: name, suffix, full }
+	}
+	const full = (r.StNam_Full ?? "").toString().trim()
+
+	if (full) return { full, street: full }
+
+	return undefined
+}
+
+function composeLocality(r: NADRecord): string | undefined {
+	return nonEmpty(r.Post_City, r.Inc_Muni, r.Census_Plc, r.Uninc_Comm)
+}
+
+function composePostcode(r: NADRecord): string | undefined {
+	const zip = (r.Zip_Code ?? "").toString().trim()
+
+	if (!zip) return undefined
+	const plus4 = (r.Plus_4 ?? "").toString().trim()
+
+	return plus4 ? `${zip}-${plus4}` : zip
+}
+
+function composeRaw(parts: {
+	venue?: string
+	houseNumber?: string
+	street?: string
+	unit?: string
+	locality: string
+	region: string
+	postcode: string
+}): string {
+	const streetLine = [parts.houseNumber, parts.street, parts.unit].filter(Boolean).join(" ").trim()
+	const tail = `${parts.locality}, ${parts.region} ${parts.postcode}`
+
+	return [parts.venue, streetLine || undefined, tail].filter(Boolean).join(", ")
+}
+
+export function createUsgovNADAdapter(): CorpusAdapter {
+	return {
+		id: USGOV_NAD_ADAPTER_ID,
+		defaultLicense: USGOV_NAD_DEFAULT_LICENSE,
+		description:
+			"US DOT National Address Database — ~97M structured US address points (911-grade). Single largest US source.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`usgov-nad adapter: only US supported, got country=${opts.country}`)
+			}
+
+			// inputPath is a directory of NDJSON shards (per fetch-nad.ts featureserver output).
+			// Single-file inputs (e.g. a bulk-extracted CSV) are not currently supported — the
+			// featureserver shard pattern is the primary distribution.
+			const entries = await readdir(opts.inputPath)
+			const shards = entries.filter((n) => n.endsWith(".ndjson")).sort()
+
+			let emitted = 0
+			outer: for (const shard of shards) {
+				if (opts.signal?.aborted) break
+				// TextSpliterator streams string lines; the per-line try/catch below keeps the reader
+				// tolerant of malformed rows (skip silently), so TextSpliterator + explicit JSON.parse —
+				// not JSONSpliterator, which would throw. The path string lets the lib own + dispose each
+				// shard's file handle, including on the `break outer` early exit.
+				const lines = TextSpliterator.fromAsync(join(opts.inputPath, shard))
+
+				for await (const line of lines) {
+					if (opts.signal?.aborted) break outer
+
+					if (opts.limit !== undefined && emitted >= opts.limit) break outer
+
+					if (!line) continue
+
+					let record: NADRecord
+
+					try {
+						record = JSON.parse(line) as NADRecord
+					} catch {
+						continue // malformed line — skip silently
+					}
+
+					const state = (record.State ?? "").toString().trim().toUpperCase()
+
+					if (!US_STATES_SET.has(state)) continue
+
+					const locality = composeLocality(record)
+
+					if (!locality) continue
+
+					const postcode = composePostcode(record)
+
+					if (!postcode) continue
+
+					const decomposed = decomposeNADStreet(record)
+					const houseNumber = composeHouseNumber(record)
+					const venue = nonEmpty(record.LandmkName)
+					const unit = nonEmpty(record.Unit, record.Building, record.Floor, record.Room)
+
+					const components: CanonicalRow["components"] = {
+						...(venue ? { venue } : {}),
+						...(houseNumber ? { house_number: houseNumber } : {}),
+						...(decomposed?.prefix ? { street_prefix: decomposed.prefix } : {}),
+						...(decomposed?.street ? { street: decomposed.street } : {}),
+						...(decomposed?.suffix ? { street_suffix: decomposed.suffix } : {}),
+						...(unit ? { unit } : {}),
+						locality,
+						region: state,
+						postcode,
+					}
+
+					const raw = composeRaw({
+						venue,
+						houseNumber,
+						street: decomposed?.full,
+						unit,
+						locality,
+						region: state,
+						postcode,
+					})
+
+					if (!raw) continue
+
+					const aligned = reconcileComponents(components, raw)
+
+					if (Object.keys(aligned).length <= 2) continue
+
+					const sourceID = record.UUID
+						? `${USGOV_NAD_ADAPTER_ID}-${record.UUID}`
+						: `${USGOV_NAD_ADAPTER_ID}-${record.OBJECTID ?? `${shard}:${emitted}`}`
+
+					yield {
+						raw,
+						components: aligned,
+						country: "US",
+						locale: "en-US",
+						source: USGOV_NAD_ADAPTER_ID,
+						source_id: sourceID,
+						corpus_version: "",
+						license: USGOV_NAD_DEFAULT_LICENSE,
+					}
+					emitted++
+				}
+			}
+		},
+	}
+}
+
+export const usgovNADAdapter = createUsgovNADAdapter()

package/src/adapters/usgov-nppes/adapter.ts

@@ -0,0 +1,169 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `usgov-nppes`: CMS National Plan and Provider Enumeration System (NPI registry) CSV consumer.
+ *
+ *   NPPES is the authoritative US healthcare provider registry, published monthly by CMS. Each row
+ *   carries a provider's business practice location address together with their legal business name
+ *   or individual name. At ~7M rows it is the single largest venue+address signal source
+ *   available.
+ *
+ *   The adapter consumes the monthly full-replacement CSV (operator pre-downloads via
+ *   `fetch-nppes.ts`). Column names match the canonical NPPES "Full Replacement Monthly NPI File"
+ *   header published at `https://download.cms.gov/nppes/NPI_Files.html`.
+ *
+ *   Output: one row per CSV record where the practice location address is populated. Organization
+ *   rows carry `venue` from the legal business name; individual rows compose `attention` from
+ *   last+first name. Address quad goes on `(house_number, street, locality, region, postcode)`.
+ *
+ *   License: stamped `"Public Domain"` per CMS's federal government distribution terms.
+ */
+
+import { createReadStream } from "node:fs"
+
+import { parse as csvParse } from "csv-parse"
+
+import { stableSourceID } from "../../adapter.ts"
+import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
+import { reconcileComponents } from "../../format.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+
+export const USGOV_NPPES_ADAPTER_ID = "usgov-nppes"
+export const USGOV_NPPES_DEFAULT_LICENSE = "Public Domain"
+
+const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
+
+interface NPPESRow {
+	NPI: string
+	"Entity Type Code": string
+	"Provider Organization Name (Legal Business Name)": string
+	"Provider Last Name (Legal Name)": string
+	"Provider First Name": string
+	"Provider First Line Business Practice Location Address": string
+	"Provider Second Line Business Practice Location Address": string
+	"Provider Business Practice Location Address City Name": string
+	"Provider Business Practice Location Address State Name": string
+	"Provider Business Practice Location Address Postal Code": string
+}
+
+function splitAddress(address: string): { house_number?: string; street: string } | null {
+	const trimmed = address.trim()
+
+	if (!trimmed) return null
+	const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
+
+	if (m) return { house_number: m[1], street: m[2]!.trim() }
+
+	return { street: trimmed }
+}
+
+function composeRaw(
+	venue: string | undefined,
+	house: string | undefined,
+	street: string,
+	city: string,
+	state: string,
+	postcode: string
+): string {
+	const streetPart = [house, street].filter(Boolean).join(" ").trim()
+	const cityPart = [city.trim(), [state, postcode].filter(Boolean).join(" ").trim()].filter(Boolean).join(", ")
+
+	return [venue, streetPart, cityPart].filter(Boolean).join(", ")
+}
+
+export function createUsgovNPPESAdapter(): CorpusAdapter {
+	return {
+		id: USGOV_NPPES_ADAPTER_ID,
+		defaultLicense: USGOV_NPPES_DEFAULT_LICENSE,
+		description:
+			"CMS National Plan and Provider Enumeration System — 7M provider practice locations (public-domain). Venue+address co-occurrence at scale.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`usgov-nppes adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
+			const parser = stream.pipe(
+				csvParse({
+					columns: true,
+					skip_empty_lines: true,
+					relax_quotes: true,
+					relax_column_count: true,
+				})
+			)
+
+			let emitted = 0
+
+			try {
+				for await (const record of parser as AsyncIterable<NPPESRow>) {
+					if (opts.signal?.aborted) break
+
+					if (opts.limit !== undefined && emitted >= opts.limit) break
+
+					const npi = (record.NPI ?? "").trim()
+					const entityType = (record["Entity Type Code"] ?? "").trim()
+					const orgName = (record["Provider Organization Name (Legal Business Name)"] ?? "").trim()
+					const lastName = (record["Provider Last Name (Legal Name)"] ?? "").trim()
+					const firstName = (record["Provider First Name"] ?? "").trim()
+
+					const address1 = (record["Provider First Line Business Practice Location Address"] ?? "").trim()
+					const address2 = (record["Provider Second Line Business Practice Location Address"] ?? "").trim()
+					const city = (record["Provider Business Practice Location Address City Name"] ?? "").trim()
+					const stateRaw = (record["Provider Business Practice Location Address State Name"] ?? "").trim()
+					const postcode = (record["Provider Business Practice Location Address Postal Code"] ?? "").trim()
+
+					if (!city || !postcode) continue
+
+					const state = lookupStateAbbreviation(stateRaw)
+
+					if (!state) continue
+
+					const fullStreet = [address1, address2].filter(Boolean).join(" ")
+					const split = splitAddress(fullStreet)
+
+					if (!split) continue
+
+					const venue = orgName || [firstName, lastName].filter(Boolean).join(" ") || undefined
+
+					const components: CanonicalRow["components"] = {
+						...(venue ? { venue } : {}),
+						...(split.house_number ? { house_number: split.house_number } : {}),
+						street: split.street,
+						locality: city,
+						region: state.abbreviation,
+						postcode,
+					}
+
+					const raw = composeRaw(venue, split.house_number, split.street, city, state.abbreviation, postcode)
+
+					if (!raw) continue
+
+					const aligned = reconcileComponents(components, raw)
+
+					if (Object.keys(aligned).length <= 2) continue
+
+					const sourceID = npi ? `${USGOV_NPPES_ADAPTER_ID}-${npi}` : stableSourceID(USGOV_NPPES_ADAPTER_ID, aligned)
+
+					yield {
+						raw,
+						components: aligned,
+						country: "US",
+						locale: "en-US",
+						source: USGOV_NPPES_ADAPTER_ID,
+						source_id: sourceID,
+						corpus_version: "",
+						license: USGOV_NPPES_DEFAULT_LICENSE,
+					}
+					emitted++
+				}
+			} finally {
+				stream.destroy()
+			}
+		},
+	}
+}
+
+export const usgovNPPESAdapter = createUsgovNPPESAdapter()