@mailwoman/corpus 9.1.0 → 9.3.0

package/src/adapters/usgov-irs-bmf/adapter.ts

@@ -1,161 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `usgov-irs-bmf`: IRS Exempt Organizations Business Master File (EO BMF) CSV consumer.
- *
- *   The EO BMF is the IRS's authoritative registry of US tax-exempt organizations (charities,
- *   churches, foundations, ...), published as per-region CSVs at
- *   `https://www.irs.gov/charities-non-profits/exempt-organizations-business-master-file-extract-eo-bmf`
- *   (`eo1.csv`..`eo4.csv`, `eo_pr.csv`, `eo_xx.csv`). Each row carries an organization NAME plus
- *   its mailing address. It complements `usgov-nppes` with a DIFFERENT venue population
- *   (non-profits vs healthcare providers) and, notably, a high share of PO-box addresses — useful
- *   `po_box`-tag signal (a tag with historically low recall).
- *
- *   Output: one row per record with a usable city + postcode. NAME → `venue`; the street line becomes
- *   `po_box` when it's a PO-box, else `house_number` + `street`; CITY/STATE/ZIP fill the locality
- *   line. STATE is already a USPS abbreviation in the source. License: `"Public Domain"` (US
- *   federal).
- */
-
-import { splitStreetLine, stableSourceID } from "@mailwoman/corpus/adapters/utils"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
-import { reconcileComponents } from "@mailwoman/formatter"
-import { CSVSpliterator } from "spliterator"
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const USGOV_IRS_BMF_ADAPTER_ID = "usgov-irs-bmf"
-/**
- * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
- * than having to look them up.
- */
-export const USGOV_IRS_BMF_DEFAULT_LICENSE = "Public Domain"
-
-/**
- * PO box in its many written forms: "PO BOX 12", "P.O. BOX 12", "P O BOX 12", "POB 12", "BOX 12".
- */
-const PO_BOX = /^\s*(?:P\.?\s?O\.?\s*BOX|POB|BOX)\s+\w/i
-
-interface IrsBmfRow {
-	EIN: string
-	NAME: string
-	STREET: string
-	CITY: string
-	STATE: string
-	ZIP: string
-}
-
-/**
- * Classify the street line into a `po_box` or a `{house_number?, street}` split.
- *
- * BMF mixes street addresses and PO boxes in one `STREET` column, so the PO-box shapes have to be claimed BEFORE the
- * shared house-number split runs — otherwise `splitStreetLine` would hand back `"PO Box 1234"` as a plain street, which
- * is correct for every other US adapter and wrong here.
- */
-function splitStreetLineOrPOBox(street: string): { po_box: string } | { house_number?: string; street: string } | null {
-	const trimmed = street.trim()
-
-	if (PO_BOX.test(trimmed)) return { po_box: trimmed }
-
-	return splitStreetLine(trimmed)
-}
-
-function composeRaw(
-	venue: string | undefined,
-	streetPart: string,
-	city: string,
-	state: string,
-	postcode: string
-): string {
-	const cityPart = [city.trim(), [state, postcode].filter(Boolean).join(" ").trim()].filter(Boolean).join(", ")
-
-	return [venue, streetPart, cityPart].filter(Boolean).join(", ")
-}
-
-export function createUsgovIrsBmfAdapter(): CorpusAdapter {
-	return {
-		id: USGOV_IRS_BMF_ADAPTER_ID,
-		defaultLicense: USGOV_IRS_BMF_DEFAULT_LICENSE,
-		description:
-			"IRS Exempt Organizations Business Master File — US non-profit venue+address (public-domain), with strong PO-box coverage.",
-
-		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			if (opts.country && opts.country !== "US") {
-				throw new Error(`usgov-irs-bmf adapter: only US supported, got country=${opts.country}`)
-			}
-
-			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
-				mode: "object",
-				normalizeKeys: false,
-				enableQuoteHandling: true,
-			})
-
-			let emitted = 0
-
-			for await (const record of rows as AsyncIterable<IrsBmfRow>) {
-				if (opts.signal?.aborted) break
-
-				if (opts.limit !== undefined && emitted >= opts.limit) break
-
-				const ein = (record.EIN ?? "").trim()
-				const venue = (record.NAME ?? "").trim() || undefined
-				const street = (record.STREET ?? "").trim()
-				const city = (record.CITY ?? "").trim()
-				const state = (record.STATE ?? "").trim()
-				const zipRaw = (record.ZIP ?? "").trim()
-
-				if (!city || !zipRaw) continue
-				const postcode = zipRaw.split("-")[0]!.trim() // 5-digit; drop the optional +4
-
-				const split = splitStreetLineOrPOBox(street)
-
-				if (!split) continue
-
-				const streetPart =
-					"po_box" in split ? split.po_box : [split.house_number, split.street].filter(Boolean).join(" ")
-
-				const components: CanonicalRow["components"] = {
-					...(venue ? { venue } : {}),
-					...("po_box" in split
-						? { po_box: split.po_box }
-						: { ...(split.house_number ? { house_number: split.house_number } : {}), street: split.street }),
-					locality: city,
-					...(state ? { region: state } : {}),
-					postcode,
-				}
-
-				const raw = composeRaw(venue, streetPart, city, state, postcode)
-
-				if (!raw) continue
-
-				const aligned = reconcileComponents(components, raw)
-
-				if (Object.keys(aligned).length <= 2) continue
-
-				const sourceID = ein ? `${USGOV_IRS_BMF_ADAPTER_ID}-${ein}` : stableSourceID(USGOV_IRS_BMF_ADAPTER_ID, aligned)
-
-				yield {
-					raw,
-					components: aligned,
-					country: "US",
-					locale: "en-US",
-					source: USGOV_IRS_BMF_ADAPTER_ID,
-					source_id: sourceID,
-					corpus_version: "",
-					license: USGOV_IRS_BMF_DEFAULT_LICENSE,
-				}
-
-				emitted++
-			}
-		},
-	}
-}
-
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const usgovIrsBmfAdapter = createUsgovIrsBmfAdapter()

package/src/adapters/usgov-nad/adapter.ts

@@ -1,347 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `usgov-nad`: US DOT National Address Database — ~97M structured address-point records.
- *
- *   The single largest US address source available — federal aggregation of state + local 911-grade
- *   address points (every addressable location). Compared to TIGER ADDRFEAT (~20M segment-level, no
- *   city/locality) and NPPES (~7M provider-centric venues), NAD covers the entire residential +
- *   commercial address space with full structured components.
- *
- *   The adapter consumes NDJSON shards produced by `fetch-nad.ts`'s featureserver mode (operator
- *   pre-downloads via `mailwoman corpus fetch nad`). Each shard is per-OID-range
- *   `oids_<start>-<end>.ndjson` with a sibling `.manifest.json`. Adapter iterates every `.ndjson`
- *   in the input directory, skipping the `quarantined-bash-bug/` subdir (legacy of the bash-
- *   fetcher's silent-page-failure bug).
- *
- *   Field mapping (NAD v9 → CanonicalRow components):
- *
- *   - House_number: `AddNo_Full` (pre-composed); falls back to AddNum_Pre + Add_Number + AddNum_Suf
- *   - Street: `StNam_Full` (pre-composed); falls back to St_PreDir + St_PreTyp + St_Name + St_PosTyp
- *
- *       - St_PosDir + St_PosMod composition
- *   - Locality: `Post_City` > `Inc_Muni` > `Census_Plc` > `Uninc_Comm` (first non-empty)
- *   - Region: `State` (2-char USPS code, including territories: PR, GU, VI, AS, MP)
- *   - Postcode: `Zip_Code` + `Plus_4` (joined as `XXXXX-NNNN` when both present)
- *   - Venue: `LandmkName` (typically a park, school, hospital, named facility — when present)
- *
- *   License: stamped `"Public Domain"` per 17 U.S.C. § 105 (US federal works).
- */
-
-import { readdir } from "node:fs/promises"
-import { join } from "node:path"
-
-import { tryParsingJSON } from "@mailwoman/core/objects"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
-import { reconcileComponents } from "@mailwoman/formatter"
-import { TextSpliterator } from "spliterator"
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const USGOV_NAD_ADAPTER_ID = "usgov-nad"
-/**
- * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
- * than having to look them up.
- */
-export const USGOV_NAD_DEFAULT_LICENSE = "Public Domain"
-
-interface NADRecord {
-	OBJECTID?: number
-	UUID?: string | null
-	// House number
-	AddNum_Pre?: string | null
-	Add_Number?: number | string | null
-	AddNum_Suf?: string | null
-	AddNo_Full?: string | null
-	// Street parts
-	St_PreMod?: string | null
-	St_PreDir?: string | null
-	St_PreTyp?: string | null
-	St_PreSep?: string | null
-	St_Name?: string | null
-	St_PosTyp?: string | null
-	St_PosDir?: string | null
-	St_PosMod?: string | null
-	StNam_Full?: string | null
-	// Sub-address (carried as part of street for now; Phase 1 has no unit/floor labels)
-	Building?: string | null
-	Floor?: string | null
-	Unit?: string | null
-	Room?: string | null
-	Seat?: string | null
-	Addtl_Loc?: string | null
-	SubAddress?: string | null
-	// Landmark / venue
-	LandmkName?: string | null
-	// Locality alternates (we prefer Post_City for what a human would type)
-	County?: string | null
-	Inc_Muni?: string | null
-	Post_City?: string | null
-	Census_Plc?: string | null
-	Uninc_Comm?: string | null
-	Nbrhd_Comm?: string | null
-	NatAmArea?: string | null
-	NatAmSub?: string | null
-	Urbnztn_PR?: string | null
-	PlaceOther?: string | null
-	PlaceNmTyp?: string | null
-	// State + ZIP
-	State?: string | null
-	Zip_Code?: string | null
-	Plus_4?: string | null
-}
-
-const US_STATES_SET = new Set([
-	"AL",
-	"AK",
-	"AZ",
-	"AR",
-	"CA",
-	"CO",
-	"CT",
-	"DE",
-	"DC",
-	"FL",
-	"GA",
-	"HI",
-	"ID",
-	"IL",
-	"IN",
-	"IA",
-	"KS",
-	"KY",
-	"LA",
-	"ME",
-	"MD",
-	"MA",
-	"MI",
-	"MN",
-	"MS",
-	"MO",
-	"MT",
-	"NE",
-	"NV",
-	"NH",
-	"NJ",
-	"NM",
-	"NY",
-	"NC",
-	"ND",
-	"OH",
-	"OK",
-	"OR",
-	"PA",
-	"RI",
-	"SC",
-	"SD",
-	"TN",
-	"TX",
-	"UT",
-	"VT",
-	"VA",
-	"WA",
-	"WV",
-	"WI",
-	"WY",
-	// Territories that ship in NAD
-	"PR",
-	"GU",
-	"VI",
-	"AS",
-	"MP",
-])
-
-function nonEmpty(...values: Array<string | null | undefined>): string | undefined {
-	for (const v of values) {
-		const trimmed = (v ?? "").toString().trim()
-
-		if (trimmed) return trimmed
-	}
-
-	return undefined
-}
-
-function composeHouseNumber(r: NADRecord): string | undefined {
-	const full = (r.AddNo_Full ?? "").toString().trim()
-
-	if (full) return full
-	const num = r.Add_Number == null ? "" : String(r.Add_Number).trim()
-
-	if (!num) return undefined
-	const pre = (r.AddNum_Pre ?? "").toString().trim()
-	const suf = (r.AddNum_Suf ?? "").toString().trim()
-
-	return [pre, num, suf].filter(Boolean).join(" ").trim() || undefined
-}
-
-interface DecomposedNADStreet {
-	prefix?: string
-	street?: string
-	suffix?: string
-	full: string
-}
-
-function decomposeNADStreet(r: NADRecord): DecomposedNADStreet | undefined {
-	const name = (r.St_Name ?? "").toString().trim()
-
-	if (name) {
-		const preDir = (r.St_PreDir ?? "").toString().trim()
-		const preTyp = (r.St_PreTyp ?? "").toString().trim()
-		const preSep = (r.St_PreSep ?? "").toString().trim()
-		const posTyp = (r.St_PosTyp ?? "").toString().trim()
-		const posDir = (r.St_PosDir ?? "").toString().trim()
-		const prefix = [preDir, preTyp, preSep].filter(Boolean).join(" ") || undefined
-		const suffix = [posTyp, posDir].filter(Boolean).join(" ") || undefined
-		const full = [prefix, name, suffix].filter(Boolean).join(" ")
-
-		return { prefix, street: name, suffix, full }
-	}
-
-	const full = (r.StNam_Full ?? "").toString().trim()
-
-	if (full) return { full, street: full }
-
-	return undefined
-}
-
-function composeLocality(r: NADRecord): string | undefined {
-	return nonEmpty(r.Post_City, r.Inc_Muni, r.Census_Plc, r.Uninc_Comm)
-}
-
-function composePostcode(r: NADRecord): string | undefined {
-	const zip = (r.Zip_Code ?? "").toString().trim()
-
-	if (!zip) return undefined
-	const plus4 = (r.Plus_4 ?? "").toString().trim()
-
-	return plus4 ? `${zip}-${plus4}` : zip
-}
-
-function composeRaw(parts: {
-	venue?: string
-	houseNumber?: string
-	street?: string
-	unit?: string
-	locality: string
-	region: string
-	postcode: string
-}): string {
-	const streetLine = [parts.houseNumber, parts.street, parts.unit].filter(Boolean).join(" ").trim()
-	const tail = `${parts.locality}, ${parts.region} ${parts.postcode}`
-
-	return [parts.venue, streetLine || undefined, tail].filter(Boolean).join(", ")
-}
-
-export function createUsgovNADAdapter(): CorpusAdapter {
-	return {
-		id: USGOV_NAD_ADAPTER_ID,
-		defaultLicense: USGOV_NAD_DEFAULT_LICENSE,
-		description:
-			"US DOT National Address Database — ~97M structured US address points (911-grade). Single largest US source.",
-
-		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			if (opts.country && opts.country !== "US") {
-				throw new Error(`usgov-nad adapter: only US supported, got country=${opts.country}`)
-			}
-
-			// inputPath is a directory of NDJSON shards (per fetch-nad.ts featureserver output).
-			// Single-file inputs (e.g. a bulk-extracted CSV) are not currently supported — the
-			// featureserver shard pattern is the primary distribution.
-			const entries = await readdir(opts.inputPath)
-			const shards = entries.filter((n) => n.endsWith(".ndjson")).toSorted()
-
-			let emitted = 0
-			outer: for (const shard of shards) {
-				if (opts.signal?.aborted) break
-				// TextSpliterator streams string lines; the per-line tryParsingJSON below keeps the reader
-				// tolerant of malformed rows (skip silently), so TextSpliterator + a non-throwing parse —
-				// not JSONSpliterator, which would throw. The path string lets the lib own + dispose each
-				// shard's file handle, including on the `break outer` early exit.
-				const lines = TextSpliterator.fromAsync(join(opts.inputPath, shard))
-
-				for await (const line of lines) {
-					if (opts.signal?.aborted) break outer
-
-					if (opts.limit !== undefined && emitted >= opts.limit) break outer
-
-					if (!line) continue
-
-					const record = tryParsingJSON<NADRecord>(line)
-
-					if (record === null) continue // malformed line — skip silently
-
-					const state = (record.State ?? "").toString().trim().toUpperCase()
-
-					if (!US_STATES_SET.has(state)) continue
-
-					const locality = composeLocality(record)
-
-					if (!locality) continue
-
-					const postcode = composePostcode(record)
-
-					if (!postcode) continue
-
-					const decomposed = decomposeNADStreet(record)
-					const houseNumber = composeHouseNumber(record)
-					const venue = nonEmpty(record.LandmkName)
-					const unit = nonEmpty(record.Unit, record.Building, record.Floor, record.Room)
-
-					const components: CanonicalRow["components"] = {
-						...(venue ? { venue } : {}),
-						...(houseNumber ? { house_number: houseNumber } : {}),
-						...(decomposed?.prefix ? { street_prefix: decomposed.prefix } : {}),
-						...(decomposed?.street ? { street: decomposed.street } : {}),
-						...(decomposed?.suffix ? { street_suffix: decomposed.suffix } : {}),
-						...(unit ? { unit } : {}),
-						locality,
-						region: state,
-						postcode,
-					}
-
-					const raw = composeRaw({
-						venue,
-						houseNumber,
-						street: decomposed?.full,
-						unit,
-						locality,
-						region: state,
-						postcode,
-					})
-
-					if (!raw) continue
-
-					const aligned = reconcileComponents(components, raw)
-
-					if (Object.keys(aligned).length <= 2) continue
-
-					const sourceID = record.UUID
-						? `${USGOV_NAD_ADAPTER_ID}-${record.UUID}`
-						: `${USGOV_NAD_ADAPTER_ID}-${record.OBJECTID ?? `${shard}:${emitted}`}`
-
-					yield {
-						raw,
-						components: aligned,
-						country: "US",
-						locale: "en-US",
-						source: USGOV_NAD_ADAPTER_ID,
-						source_id: sourceID,
-						corpus_version: "",
-						license: USGOV_NAD_DEFAULT_LICENSE,
-					}
-
-					emitted++
-				}
-			}
-		},
-	}
-}
-
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const usgovNADAdapter = createUsgovNADAdapter()

package/src/adapters/usgov-nppes/adapter.ts

@@ -1,157 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `usgov-nppes`: CMS National Plan and Provider Enumeration System (NPI registry) CSV consumer.
- *
- *   NPPES is the authoritative US healthcare provider registry, published monthly by CMS. Each row
- *   carries a provider's business practice location address together with their legal business name
- *   or individual name. At ~7M rows it is the single largest venue+address signal source
- *   available.
- *
- *   The adapter consumes the monthly full-replacement CSV (operator pre-downloads via
- *   `fetch-nppes.ts`). Column names match the canonical NPPES "Full Replacement Monthly NPI File"
- *   header published at `https://download.cms.gov/nppes/NPI_Files.html`.
- *
- *   Output: one row per CSV record where the practice location address is populated. Organization
- *   rows carry `venue` from the legal business name; individual rows compose `attention` from
- *   last+first name. Address quad goes on `(house_number, street, locality, region, postcode)`.
- *
- *   License: stamped `"Public Domain"` per CMS's federal government distribution terms.
- */
-
-import { splitStreetLine, stableSourceID } from "@mailwoman/corpus/adapters/utils"
-import { lookupStateAbbreviation } from "@mailwoman/corpus/codex/us-fips-state"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
-import { reconcileComponents } from "@mailwoman/formatter"
-import { CSVSpliterator } from "spliterator"
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const USGOV_NPPES_ADAPTER_ID = "usgov-nppes"
-/**
- * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
- * than having to look them up.
- */
-export const USGOV_NPPES_DEFAULT_LICENSE = "Public Domain"
-
-interface NPPESRow {
-	NPI: string
-	"Entity Type Code": string
-	"Provider Organization Name (Legal Business Name)": string
-	"Provider Last Name (Legal Name)": string
-	"Provider First Name": string
-	"Provider First Line Business Practice Location Address": string
-	"Provider Second Line Business Practice Location Address": string
-	"Provider Business Practice Location Address City Name": string
-	"Provider Business Practice Location Address State Name": string
-	"Provider Business Practice Location Address Postal Code": string
-}
-
-function composeRaw(
-	venue: string | undefined,
-	house: string | undefined,
-	street: string,
-	city: string,
-	state: string,
-	postcode: string
-): string {
-	const streetPart = [house, street].filter(Boolean).join(" ").trim()
-	const cityPart = [city.trim(), [state, postcode].filter(Boolean).join(" ").trim()].filter(Boolean).join(", ")
-
-	return [venue, streetPart, cityPart].filter(Boolean).join(", ")
-}
-
-export function createUsgovNPPESAdapter(): CorpusAdapter {
-	return {
-		id: USGOV_NPPES_ADAPTER_ID,
-		defaultLicense: USGOV_NPPES_DEFAULT_LICENSE,
-		description:
-			"CMS National Plan and Provider Enumeration System — 7M provider practice locations (public-domain). Venue+address co-occurrence at scale.",
-
-		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			if (opts.country && opts.country !== "US") {
-				throw new Error(`usgov-nppes adapter: only US supported, got country=${opts.country}`)
-			}
-
-			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
-				mode: "object",
-				normalizeKeys: false,
-				enableQuoteHandling: true,
-			})
-
-			let emitted = 0
-
-			for await (const record of rows as AsyncIterable<NPPESRow>) {
-				if (opts.signal?.aborted) break
-
-				if (opts.limit !== undefined && emitted >= opts.limit) break
-
-				const npi = (record.NPI ?? "").trim()
-				const entityType = (record["Entity Type Code"] ?? "").trim()
-				const orgName = (record["Provider Organization Name (Legal Business Name)"] ?? "").trim()
-				const lastName = (record["Provider Last Name (Legal Name)"] ?? "").trim()
-				const firstName = (record["Provider First Name"] ?? "").trim()
-
-				const address1 = (record["Provider First Line Business Practice Location Address"] ?? "").trim()
-				const address2 = (record["Provider Second Line Business Practice Location Address"] ?? "").trim()
-				const city = (record["Provider Business Practice Location Address City Name"] ?? "").trim()
-				const stateRaw = (record["Provider Business Practice Location Address State Name"] ?? "").trim()
-				const postcode = (record["Provider Business Practice Location Address Postal Code"] ?? "").trim()
-
-				if (!city || !postcode) continue
-
-				const state = lookupStateAbbreviation(stateRaw)
-
-				if (!state) continue
-
-				const fullStreet = [address1, address2].filter(Boolean).join(" ")
-				const split = splitStreetLine(fullStreet)
-
-				if (!split) continue
-
-				const venue = orgName || [firstName, lastName].filter(Boolean).join(" ") || undefined
-
-				const components: CanonicalRow["components"] = {
-					...(venue ? { venue } : {}),
-					...(split.house_number ? { house_number: split.house_number } : {}),
-					street: split.street,
-					locality: city,
-					region: state.abbreviation,
-					postcode,
-				}
-
-				const raw = composeRaw(venue, split.house_number, split.street, city, state.abbreviation, postcode)
-
-				if (!raw) continue
-
-				const aligned = reconcileComponents(components, raw)
-
-				if (Object.keys(aligned).length <= 2) continue
-
-				const sourceID = npi ? `${USGOV_NPPES_ADAPTER_ID}-${npi}` : stableSourceID(USGOV_NPPES_ADAPTER_ID, aligned)
-
-				yield {
-					raw,
-					components: aligned,
-					country: "US",
-					locale: "en-US",
-					source: USGOV_NPPES_ADAPTER_ID,
-					source_id: sourceID,
-					corpus_version: "",
-					license: USGOV_NPPES_DEFAULT_LICENSE,
-				}
-
-				emitted++
-			}
-		},
-	}
-}
-
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const usgovNPPESAdapter = createUsgovNPPESAdapter()