@mailwoman/corpus 9.2.0 → 9.4.0

package/lib/adapters/tiger/street-decompose.ts

@@ -0,0 +1,83 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Decompose a US street name into Stage 3 components: street_prefix, street, street_suffix.
+ *
+ *   Sources directionals and street types from the curated libpostal/en dictionaries
+ *   (`core/data/libpostal/dictionaries/en/{directionals,street_types}.txt`). These are the same
+ *   dictionaries the runtime classifiers (StreetPrefixClassifier, StreetSuffixClassifier) use, so
+ *   corpus labels and runtime classifications agree on the vocabulary.
+ *
+ *   Examples: "N Main St" → { prefix: "N", street: "Main", suffix: "St" } "Pennsylvania Avenue NW" →
+ *   { prefix: null, street: "Pennsylvania", suffix: "Avenue NW" } "Salmon St" → { prefix: null,
+ *   street: "Salmon", suffix: "St" } "SE Hawthorne Blvd" → { prefix: "SE", street: "Hawthorne",
+ *   suffix: "Blvd" }
+ */
+
+import { loadLibpostalDictionary } from "#adapters/utils"
+
+const [DIRECTIONALS, STREET_TYPES] = await Promise.all([
+	loadLibpostalDictionary("en", "directionals.txt"),
+	loadLibpostalDictionary("en", "street_types.txt"),
+])
+
+export interface DecomposedStreet {
+	prefix: string | null
+	street: string
+	suffix: string | null
+}
+
+/**
+ * Decompose a US street name into prefix/name/suffix components.
+ *
+ * Conservative — only emits prefix/suffix when there's a clear directional or street-type keyword. Returns the original
+ * as `street` if nothing matches.
+ */
+export function decomposeStreet(fullname: string): DecomposedStreet {
+	const trimmed = fullname.trim()
+
+	if (!trimmed) return { prefix: null, street: "", suffix: null }
+
+	const tokens = trimmed.split(/\s+/)
+
+	if (tokens.length === 1) return { prefix: null, street: trimmed, suffix: null }
+
+	const norm = (s: string) => s.toLowerCase().replace(/\.$/, "")
+
+	let prefix: string | null = null
+	let suffix: string | null = null
+	let startIdx = 0
+	let endIdx = tokens.length
+
+	// Leading directional prefix
+	if (DIRECTIONALS.has(norm(tokens[0]!)) && tokens.length >= 2) {
+		prefix = tokens[0]!
+		startIdx = 1
+	}
+
+	// Trailing post-directional combined with street type (e.g. "Pennsylvania Ave NW")
+	const last = norm(tokens[endIdx - 1]!)
+	const secondLast = endIdx >= 2 ? norm(tokens[endIdx - 2]!) : ""
+
+	if (DIRECTIONALS.has(last) && STREET_TYPES.has(secondLast)) {
+		suffix = tokens.slice(endIdx - 2, endIdx).join(" ")
+		endIdx -= 2
+	} else if (STREET_TYPES.has(last) && endIdx - startIdx >= 2) {
+		suffix = tokens[endIdx - 1]!
+		endIdx -= 1
+	} else if (DIRECTIONALS.has(last) && endIdx - startIdx >= 2) {
+		// Post-directional without type
+		suffix = tokens[endIdx - 1]!
+		endIdx -= 1
+	}
+
+	const street = tokens.slice(startIdx, endIdx).join(" ").trim()
+
+	if (!street) {
+		return { prefix: null, street: trimmed, suffix: null }
+	}
+
+	return { prefix, street, suffix }
+}

package/lib/adapters/usgov-hrsa-fqhc/adapter.ts

@@ -0,0 +1,167 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `usgov-hrsa-fqhc`: HRSA "Health Center Service Delivery Site Locations" CSV consumer.
+ *
+ *   Federally Qualified Health Centers (FQHCs) are HRSA-funded community health programs that
+ *   self-report site addresses to the HRSA Data Warehouse. The published CSV (`data.hrsa.gov`)
+ *   carries the site name, the postal-formatted street address, and the locality/region/postcode
+ *   quad. The source earns its place on adversarial-value-per-row (#22): every facility name is a
+ *   human-typed venue string and the addresses pass through enough hands to accumulate the
+ *   abbreviation drift + suite designator chaos that pure gazetteer data does not.
+ *
+ *   The adapter consumes a CSV file the operator pre-downloads. The HRSA data is published as a
+ *   single national CSV (~10K rows), small enough that the operator can re-fetch on every corpus
+ *   rebuild without an intermediate SQLite step. Column names below match the HRSA Data Warehouse's
+ *   "Health Center Service Delivery Site" public dataset. Operators substituting the
+ *   closely-related "Site Address" or "Health Center" public extracts may need to remap columns;
+ *   the README documents the expected set.
+ *
+ *   Output: one row per CSV record, with `venue` component carrying the site name and the address
+ *   quad on `(house_number, street, locality, region, postcode)`. Component order is critical:
+ *   `venue` is inserted FIRST so alignment claims its surface span before `locality` searches for
+ *   its own (the kryptonite case "Buffalo Health Clinic, …, Buffalo, NY" relies on `venue`
+ *   consuming the first "Buffalo" so locality lands on the second).
+ *
+ *   License: stamped `"Public Domain"` per the HRSA Data Warehouse's federal government distribution
+ *   terms.
+ */
+
+import { reconcileComponents } from "@mailwoman/formatter"
+import { CSVSpliterator } from "spliterator"
+
+import { composeRaw, splitStreetLine, stableSourceID } from "#adapters/utils"
+import { lookupStateAbbreviation } from "#codex/us-fips-state"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const USGOV_HRSA_FQHC_ADAPTER_ID = "usgov-hrsa-fqhc"
+/**
+ * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
+ * than having to look them up.
+ */
+export const USGOV_HRSA_FQHC_DEFAULT_LICENSE = "Public Domain"
+
+/**
+ * Subset of HRSA "Health Center Service Delivery Site Locations" CSV columns consulted by the adapter. Column names
+ * match the canonical HRSA Data Warehouse export header. Operators substituting a closely-related extract should rename
+ * columns to match; the README has the mapping cheatsheet.
+ */
+interface HRSASiteRow {
+	"Site Name": string
+	"Site Address": string
+	"Site City": string
+	"Site State Abbreviation": string
+	"Site Postal Code": string
+	/**
+	 * Optional. Falls back to `stableSourceID` derived from components when missing.
+	 */
+	"Site ID"?: string
+}
+
+/**
+ * Split a "123 Main St Suite 4" surface form into `(house_number, street)`. The regex tolerates one trailing letter on
+ * the number (`"123A Main St"`) and a hyphenated form (`"40-12 Bell Blvd"`); anything else falls back to street-only.
+ *
+ * Suite / Apt / Unit designators stay on `street` — Mailwoman's `unit` component exists but the address-formatter does
+ * not have a clean slot for it, and HRSA addresses do not separate the suite into its own column. Leaving the surface
+ * form intact in `street` preserves the adversarial training signal (the model learns that a trailing "Suite 4" is part
+ * of the road line in this distribution).
+ */
+
+export function createUSGovHRSAFQHCAdapter(): CorpusAdapter {
+	return {
+		id: USGOV_HRSA_FQHC_ADAPTER_ID,
+		defaultLicense: USGOV_HRSA_FQHC_DEFAULT_LICENSE,
+		description:
+			"HRSA Federally Qualified Health Center site locations (public-domain). Adversarial source: venue + address co-occurrence, hand-entered.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`usgov-hrsa-fqhc adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				mode: "object",
+				normalizeKeys: false,
+				enableQuoteHandling: true,
+			})
+
+			let emitted = 0
+
+			for await (const record of rows as AsyncIterable<HRSASiteRow>) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const venue = (record["Site Name"] ?? "").trim()
+				const split = splitStreetLine(record["Site Address"] ?? "")
+				const city = (record["Site City"] ?? "").trim()
+				const stateAbbr = (record["Site State Abbreviation"] ?? "").trim()
+				const postcode = (record["Site Postal Code"] ?? "").trim()
+
+				if (!venue || !split || !city || !postcode) continue
+				const state = lookupStateAbbreviation(stateAbbr)
+
+				if (!state) continue
+
+				// Insertion order matters here. `venue` first so alignment claims its span
+				// (which may contain a token like "Buffalo") before `locality` runs its
+				// search — the kryptonite case `Buffalo Health Clinic, Buffalo NY`
+				// otherwise mis-labels the venue's "Buffalo" as locality.
+				const components: CanonicalRow["components"] = {
+					venue,
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode,
+				}
+
+				const raw = composeRaw({
+					venue,
+					houseNumber: split.house_number,
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode,
+				})
+
+				if (!raw) continue
+
+				const aligned = reconcileComponents(components, raw)
+
+				if (!Object.keys(aligned).length) continue
+
+				const siteID = (record["Site ID"] ?? "").trim()
+
+				const sourceID = siteID
+					? `${USGOV_HRSA_FQHC_ADAPTER_ID}-${siteID}`
+					: stableSourceID(USGOV_HRSA_FQHC_ADAPTER_ID, aligned)
+
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: USGOV_HRSA_FQHC_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: USGOV_HRSA_FQHC_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const usgovHrsaFqhcAdapter = createUSGovHRSAFQHCAdapter()

package/lib/adapters/usgov-imls-pls/adapter.ts

@@ -0,0 +1,143 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `usgov-imls-pls`: IMLS Public Libraries Survey outlet CSV consumer.
+ *
+ *   The Institute of Museum and Library Services publishes an annual Public Libraries Survey with one
+ *   row per library outlet (~17K rows). Each row carries the library name, street address, city,
+ *   ZIP, county, and geocoordinates.
+ *
+ *   The adapter consumes the outlet CSV the operator pre-downloads via `fetch-imls-pls.ts`. Column
+ *   names match the IMLS PLS outlet file header.
+ *
+ *   Output: one row per outlet with `venue` (library name), `(house_number, street, locality,
+ *   subregion, postcode)`, and lat/lon preserved in `source_id` stability.
+ *
+ *   License: stamped `"Public Domain"` per IMLS federal government distribution terms.
+ */
+
+import { isPresent } from "@mailwoman/core/objects"
+import { reconcileComponents } from "@mailwoman/formatter"
+import { CSVSpliterator } from "spliterator"
+
+import { splitStreetLine, stableSourceID } from "#adapters/utils"
+import { lookupStateAbbreviation } from "#codex/us-fips-state"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const USGOV_IMLS_PLS_ADAPTER_ID = "usgov-imls-pls"
+/**
+ * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
+ * than having to look them up.
+ */
+export const USGOV_IMLS_PLS_DEFAULT_LICENSE = "Public Domain"
+
+interface ImlsOutletRow {
+	LIBNAME: string
+	ADDRESS: string
+	CITY: string
+	ZIP: string
+	STABR: string
+	CNTY: string
+	FSCSKEY: string
+}
+
+export function createUSGovIMLSPLSAdapter(): CorpusAdapter {
+	return {
+		id: USGOV_IMLS_PLS_ADAPTER_ID,
+		defaultLicense: USGOV_IMLS_PLS_DEFAULT_LICENSE,
+		description: "IMLS Public Libraries Survey — ~17K library outlets with venue+address (public-domain).",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`usgov-imls-pls adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				mode: "object",
+				normalizeKeys: false,
+				enableQuoteHandling: true,
+			})
+
+			let emitted = 0
+
+			for await (const record of rows as AsyncIterable<ImlsOutletRow>) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const libName = (record.LIBNAME ?? "").trim()
+				const address = (record.ADDRESS ?? "").trim()
+				const city = (record.CITY ?? "").trim()
+				const zip = (record.ZIP ?? "").trim()
+				const stateAbbr = (record.STABR ?? "").trim()
+
+				if (!libName || !city || !zip) continue
+
+				const state = lookupStateAbbreviation(stateAbbr)
+
+				if (!state) continue
+
+				const split = splitStreetLine(address)
+
+				if (!split) continue
+
+				const components: CanonicalRow["components"] = {
+					venue: libName,
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode: zip,
+					// #552: no subregion — US postal addresses don't surface the county, so emitting
+					// subregion creates a phantom component with no raw-span to align to, quarantining
+					// ~21% of rows. The county is still available in the source CSV; it just isn't
+					// a postal-surface component here.
+				}
+
+				const streetPart = [split.house_number, split.street].filter(isPresent).join(" ").trim()
+
+				const raw = [
+					libName,
+					streetPart,
+					[city, [stateAbbr, zip].filter(isPresent).join(" ")].filter(isPresent).join(", "),
+				]
+					.filter(isPresent)
+					.join(", ")
+
+				const aligned = reconcileComponents(components, raw)
+
+				if (Object.keys(aligned).length <= 2) continue
+
+				const fscsKey = (record.FSCSKEY ?? "").trim()
+
+				const sourceID = fscsKey
+					? `${USGOV_IMLS_PLS_ADAPTER_ID}-${fscsKey}`
+					: stableSourceID(USGOV_IMLS_PLS_ADAPTER_ID, aligned)
+
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: USGOV_IMLS_PLS_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: USGOV_IMLS_PLS_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const usgovImlsPlsAdapter = createUSGovIMLSPLSAdapter()

package/lib/adapters/usgov-irs-bmf/adapter.ts

@@ -0,0 +1,151 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `usgov-irs-bmf`: IRS Exempt Organizations Business Master File (EO BMF) CSV consumer.
+ *
+ *   The EO BMF is the IRS's authoritative registry of US tax-exempt organizations (charities,
+ *   churches, foundations, ...), published as per-region CSVs at
+ *   `https://www.irs.gov/charities-non-profits/exempt-organizations-business-master-file-extract-eo-bmf`
+ *   (`eo1.csv`..`eo4.csv`, `eo_pr.csv`, `eo_xx.csv`). Each row carries an organization NAME plus
+ *   its mailing address. It complements `usgov-nppes` with a DIFFERENT venue population
+ *   (non-profits vs healthcare providers) and, notably, a high share of PO-box addresses — useful
+ *   `po_box`-tag signal (a tag with historically low recall).
+ *
+ *   Output: one row per record with a usable city + postcode. NAME → `venue`; the street line becomes
+ *   `po_box` when it's a PO-box, else `house_number` + `street`; CITY/STATE/ZIP fill the locality
+ *   line. STATE is already a USPS abbreviation in the source. License: `"Public Domain"` (US
+ *   federal).
+ */
+
+import { isPresent } from "@mailwoman/core/objects"
+import { reconcileComponents } from "@mailwoman/formatter"
+import { CSVSpliterator } from "spliterator"
+
+import { composeRaw, splitStreetLine, stableSourceID } from "#adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const USGOV_IRS_BMF_ADAPTER_ID = "usgov-irs-bmf"
+/**
+ * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
+ * than having to look them up.
+ */
+export const USGOV_IRS_BMF_DEFAULT_LICENSE = "Public Domain"
+
+/**
+ * PO box in its many written forms: "PO BOX 12", "P.O. BOX 12", "P O BOX 12", "POB 12", "BOX 12".
+ */
+const PO_BOX = /^\s*(?:P\.?\s?O\.?\s*BOX|POB|BOX)\s+\w/i
+
+interface IRSBMFRow {
+	EIN: string
+	NAME: string
+	STREET: string
+	CITY: string
+	STATE: string
+	ZIP: string
+}
+
+/**
+ * Classify the street line into a `po_box` or a `{house_number?, street}` split.
+ *
+ * BMF mixes street addresses and PO boxes in one `STREET` column, so the PO-box shapes have to be claimed BEFORE the
+ * shared house-number split runs — otherwise `splitStreetLine` would hand back `"PO Box 1234"` as a plain street, which
+ * is correct for every other US adapter and wrong here.
+ */
+function splitStreetLineOrPOBox(street: string): { po_box: string } | { house_number?: string; street: string } | null {
+	const trimmed = street.trim()
+
+	if (PO_BOX.test(trimmed)) return { po_box: trimmed }
+
+	return splitStreetLine(trimmed)
+}
+
+export function createUSGovIRSBMFAdapter(): CorpusAdapter {
+	return {
+		id: USGOV_IRS_BMF_ADAPTER_ID,
+		defaultLicense: USGOV_IRS_BMF_DEFAULT_LICENSE,
+		description:
+			"IRS Exempt Organizations Business Master File — US non-profit venue+address (public-domain), with strong PO-box coverage.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`usgov-irs-bmf adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				mode: "object",
+				normalizeKeys: false,
+				enableQuoteHandling: true,
+			})
+
+			let emitted = 0
+
+			for await (const record of rows as AsyncIterable<IRSBMFRow>) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const ein = (record.EIN ?? "").trim()
+				const venue = (record.NAME ?? "").trim() || undefined
+				const street = (record.STREET ?? "").trim()
+				const city = (record.CITY ?? "").trim()
+				const state = (record.STATE ?? "").trim()
+				const zipRaw = (record.ZIP ?? "").trim()
+
+				if (!city || !zipRaw) continue
+				const postcode = zipRaw.split("-")[0]!.trim() // 5-digit; drop the optional +4
+
+				const split = splitStreetLineOrPOBox(street)
+
+				if (!split) continue
+
+				const streetPart =
+					"po_box" in split ? split.po_box : [split.house_number, split.street].filter(isPresent).join(" ")
+
+				const components: CanonicalRow["components"] = {
+					...(venue ? { venue } : {}),
+					...("po_box" in split
+						? { po_box: split.po_box }
+						: { ...(split.house_number ? { house_number: split.house_number } : {}), street: split.street }),
+					locality: city,
+					...(state ? { region: state } : {}),
+					postcode,
+				}
+
+				const raw = composeRaw({ venue, streetLine: streetPart, locality: city, region: state, postcode })
+
+				if (!raw) continue
+
+				const aligned = reconcileComponents(components, raw)
+
+				if (Object.keys(aligned).length <= 2) continue
+
+				const sourceID = ein ? `${USGOV_IRS_BMF_ADAPTER_ID}-${ein}` : stableSourceID(USGOV_IRS_BMF_ADAPTER_ID, aligned)
+
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: USGOV_IRS_BMF_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: USGOV_IRS_BMF_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const USGovIRSBMFAdapter = createUSGovIRSBMFAdapter()