@mailwoman/corpus 7.2.0 → 7.3.0

package/src/adapters/tiger/street-decompose.ts

@@ -0,0 +1,120 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Decompose a US street name into Stage 3 components: street_prefix, street, street_suffix.
+ *
+ *   Sources directionals and street types from the curated libpostal/en dictionaries
+ *   (`core/data/libpostal/dictionaries/en/{directionals,street_types}.txt`). These are the same
+ *   dictionaries the runtime classifiers (StreetPrefixClassifier, StreetSuffixClassifier) use, so
+ *   corpus labels and runtime classifications agree on the vocabulary.
+ *
+ *   Examples: "N Main St" → { prefix: "N", street: "Main", suffix: "St" } "Pennsylvania Avenue NW" →
+ *   { prefix: null, street: "Pennsylvania", suffix: "Avenue NW" } "Salmon St" → { prefix: null,
+ *   street: "Salmon", suffix: "St" } "SE Hawthorne Blvd" → { prefix: "SE", street: "Hawthorne",
+ *   suffix: "Blvd" }
+ */
+
+import { readFileSync } from "node:fs"
+import { resolve } from "node:path"
+
+import { resourceDictionaryPath } from "@mailwoman/core/utils"
+
+function loadDictionary(filename: string): Set<string> {
+	// Resolve via the @mailwoman/core data directory.
+	const candidates = [
+		resourceDictionaryPath("libpostal", "en", filename),
+		resourceDictionaryPath("libpostal", "en", filename),
+		resolve(process.cwd(), "core/data/libpostal/dictionaries/en", filename),
+	]
+
+	for (const path of candidates) {
+		try {
+			const text = readFileSync(path, "utf8")
+			const set = new Set<string>()
+
+			for (const line of text.split("\n")) {
+				const trimmed = line.trim()
+
+				if (!trimmed || trimmed.startsWith("#")) continue
+
+				// libpostal format: canonical|abbr|abbr|... — index all forms
+				for (const form of trimmed.split("|")) {
+					const f = form.trim().toLowerCase()
+
+					if (f) {
+						set.add(f)
+					}
+				}
+			}
+
+			return set
+		} catch {
+			// try next candidate
+		}
+	}
+	throw new Error(`Could not load libpostal dictionary: ${filename}`)
+}
+
+const DIRECTIONALS = loadDictionary("directionals.txt")
+const STREET_TYPES = loadDictionary("street_types.txt")
+
+export interface DecomposedStreet {
+	prefix: string | null
+	street: string
+	suffix: string | null
+}
+
+/**
+ * Decompose a US street name into prefix/name/suffix components.
+ *
+ * Conservative — only emits prefix/suffix when there's a clear directional or street-type keyword. Returns the original
+ * as `street` if nothing matches.
+ */
+export function decomposeStreet(fullname: string): DecomposedStreet {
+	const trimmed = fullname.trim()
+
+	if (!trimmed) return { prefix: null, street: "", suffix: null }
+
+	const tokens = trimmed.split(/\s+/)
+
+	if (tokens.length === 1) return { prefix: null, street: trimmed, suffix: null }
+
+	const norm = (s: string) => s.toLowerCase().replace(/\.$/, "")
+
+	let prefix: string | null = null
+	let suffix: string | null = null
+	let startIdx = 0
+	let endIdx = tokens.length
+
+	// Leading directional prefix
+	if (DIRECTIONALS.has(norm(tokens[0]!)) && tokens.length >= 2) {
+		prefix = tokens[0]!
+		startIdx = 1
+	}
+
+	// Trailing post-directional combined with street type (e.g. "Pennsylvania Ave NW")
+	const last = norm(tokens[endIdx - 1]!)
+	const secondLast = endIdx >= 2 ? norm(tokens[endIdx - 2]!) : ""
+
+	if (DIRECTIONALS.has(last) && STREET_TYPES.has(secondLast)) {
+		suffix = tokens.slice(endIdx - 2, endIdx).join(" ")
+		endIdx -= 2
+	} else if (STREET_TYPES.has(last) && endIdx - startIdx >= 2) {
+		suffix = tokens[endIdx - 1]!
+		endIdx -= 1
+	} else if (DIRECTIONALS.has(last) && endIdx - startIdx >= 2) {
+		// Post-directional without type
+		suffix = tokens[endIdx - 1]!
+		endIdx -= 1
+	}
+
+	const street = tokens.slice(startIdx, endIdx).join(" ").trim()
+
+	if (!street) {
+		return { prefix: null, street: trimmed, suffix: null }
+	}
+
+	return { prefix, street, suffix }
+}

package/src/adapters/usgov-hrsa-fqhc/adapter.ts

@@ -0,0 +1,189 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `usgov-hrsa-fqhc`: HRSA "Health Center Service Delivery Site Locations" CSV consumer.
+ *
+ *   Federally Qualified Health Centers (FQHCs) are HRSA-funded community health programs that
+ *   self-report site addresses to the HRSA Data Warehouse. The published CSV (`data.hrsa.gov`)
+ *   carries the site name, the postal-formatted street address, and the locality/region/postcode
+ *   quad. Phase 1.6 §1.2 (#22) selects this source for its adversarial-value-per-row: every
+ *   facility name is a human-typed venue string and the addresses pass through enough hands to
+ *   accumulate the abbreviation drift + suite designator chaos that pure gazetteer data does not.
+ *
+ *   The adapter consumes a CSV file the operator pre-downloads. The HRSA data is published as a
+ *   single national CSV (~10K rows), small enough that the operator can re-fetch on every corpus
+ *   rebuild without an intermediate SQLite step. Column names below match the HRSA Data Warehouse's
+ *   "Health Center Service Delivery Site" public dataset. Operators substituting the
+ *   closely-related "Site Address" or "Health Center" public extracts may need to remap columns;
+ *   the README documents the expected set.
+ *
+ *   Output: one row per CSV record, with `venue` component carrying the site name and the address
+ *   quad on `(house_number, street, locality, region, postcode)`. Component order is critical:
+ *   `venue` is inserted FIRST so alignment claims its surface span before `locality` searches for
+ *   its own (the kryptonite case "Buffalo Health Clinic, …, Buffalo, NY" relies on `venue`
+ *   consuming the first "Buffalo" so locality lands on the second).
+ *
+ *   License: stamped `"Public Domain"` per the HRSA Data Warehouse's federal government distribution
+ *   terms.
+ */
+
+import { createReadStream } from "node:fs"
+
+import { parse as csvParse } from "csv-parse"
+
+import { stableSourceID } from "../../adapter.ts"
+import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
+import { reconcileComponents } from "../../format.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+
+export const USGOV_HRSA_FQHC_ADAPTER_ID = "usgov-hrsa-fqhc"
+export const USGOV_HRSA_FQHC_DEFAULT_LICENSE = "Public Domain"
+
+/**
+ * Subset of HRSA "Health Center Service Delivery Site Locations" CSV columns consulted by the adapter. Column names
+ * match the canonical HRSA Data Warehouse export header. Operators substituting a closely-related extract should rename
+ * columns to match; the README has the mapping cheatsheet.
+ */
+interface HrsaSiteRow {
+	"Site Name": string
+	"Site Address": string
+	"Site City": string
+	"Site State Abbreviation": string
+	"Site Postal Code": string
+	/** Optional. Falls back to `stableSourceID` derived from components when missing. */
+	"Site ID"?: string
+}
+
+/**
+ * Split a "123 Main St Suite 4" surface form into `(house_number, street)`. The regex tolerates one trailing letter on
+ * the number (`"123A Main St"`) and a hyphenated form (`"40-12 Bell Blvd"`); anything else falls back to street-only.
+ *
+ * Suite / Apt / Unit designators stay on `street` for Phase 1 — Mailwoman's `unit` component exists but the
+ * address-formatter does not have a clean slot for it, and HRSA addresses do not separate the suite into its own
+ * column. Leaving the surface form intact in `street` preserves the adversarial training signal (the model learns that
+ * a trailing "Suite 4" is part of the road line in this distribution).
+ */
+const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
+
+function splitAddress(address: string): { house_number?: string; street: string } | null {
+	const trimmed = address.trim()
+
+	if (!trimmed) return null
+	const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
+
+	if (m) return { house_number: m[1], street: m[2]!.trim() }
+
+	return { street: trimmed }
+}
+
+/**
+ * Compose the raw envelope-style address line. Format:
+ *
+ * "<Site Name>, <house> <street>, <city>, <state> <postcode>"
+ *
+ * The site name leads (US conventional addressee-then-address ordering) so a downstream model sees the
+ * venue-prefix-then-address shape that HRSA users actually type into geocoders.
+ */
+function composeRaw(
+	venue: string,
+	house: string | undefined,
+	street: string,
+	city: string,
+	state: string,
+	postcode: string
+): string {
+	const streetPart = [house, street].filter(Boolean).join(" ").trim()
+	const cityPart = [city.trim(), [state, postcode].filter(Boolean).join(" ").trim()].filter(Boolean).join(", ")
+
+	return [venue.trim(), streetPart, cityPart].filter(Boolean).join(", ")
+}
+
+export function createUsgovHrsaFqhcAdapter(): CorpusAdapter {
+	return {
+		id: USGOV_HRSA_FQHC_ADAPTER_ID,
+		defaultLicense: USGOV_HRSA_FQHC_DEFAULT_LICENSE,
+		description:
+			"HRSA Federally Qualified Health Center site locations (public-domain). Adversarial source: venue + address co-occurrence, hand-entered.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`usgov-hrsa-fqhc adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
+			const parser = stream.pipe(
+				csvParse({
+					columns: true,
+					skip_empty_lines: true,
+					relax_quotes: true,
+					relax_column_count: true,
+				})
+			)
+
+			let emitted = 0
+
+			try {
+				for await (const record of parser as AsyncIterable<HrsaSiteRow>) {
+					if (opts.signal?.aborted) break
+
+					if (opts.limit !== undefined && emitted >= opts.limit) break
+
+					const venue = (record["Site Name"] ?? "").trim()
+					const split = splitAddress(record["Site Address"] ?? "")
+					const city = (record["Site City"] ?? "").trim()
+					const stateAbbr = (record["Site State Abbreviation"] ?? "").trim()
+					const postcode = (record["Site Postal Code"] ?? "").trim()
+
+					if (!venue || !split || !city || !postcode) continue
+					const state = lookupStateAbbreviation(stateAbbr)
+
+					if (!state) continue
+
+					// Insertion order matters here. `venue` first so alignment claims its span
+					// (which may contain a token like "Buffalo") before `locality` runs its
+					// search — the kryptonite case `Buffalo Health Clinic, Buffalo NY`
+					// otherwise mis-labels the venue's "Buffalo" as locality.
+					const components: CanonicalRow["components"] = {
+						venue,
+						...(split.house_number ? { house_number: split.house_number } : {}),
+						street: split.street,
+						locality: city,
+						region: state.abbreviation,
+						postcode,
+					}
+
+					const raw = composeRaw(venue, split.house_number, split.street, city, state.abbreviation, postcode)
+
+					if (!raw) continue
+
+					const aligned = reconcileComponents(components, raw)
+
+					if (Object.keys(aligned).length === 0) continue
+
+					const siteID = (record["Site ID"] ?? "").trim()
+					const sourceID = siteID
+						? `${USGOV_HRSA_FQHC_ADAPTER_ID}-${siteID}`
+						: stableSourceID(USGOV_HRSA_FQHC_ADAPTER_ID, aligned)
+
+					yield {
+						raw,
+						components: aligned,
+						country: "US",
+						locale: "en-US",
+						source: USGOV_HRSA_FQHC_ADAPTER_ID,
+						source_id: sourceID,
+						corpus_version: "",
+						license: USGOV_HRSA_FQHC_DEFAULT_LICENSE,
+					}
+					emitted++
+				}
+			} finally {
+				stream.destroy()
+			}
+		},
+	}
+}
+
+export const usgovHrsaFqhcAdapter = createUsgovHrsaFqhcAdapter()

package/src/adapters/usgov-imls-pls/adapter.ts

@@ -0,0 +1,152 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `usgov-imls-pls`: IMLS Public Libraries Survey outlet CSV consumer.
+ *
+ *   The Institute of Museum and Library Services publishes an annual Public Libraries Survey with one
+ *   row per library outlet (~17K rows). Each row carries the library name, street address, city,
+ *   ZIP, county, and geocoordinates.
+ *
+ *   The adapter consumes the outlet CSV the operator pre-downloads via `fetch-imls-pls.ts`. Column
+ *   names match the IMLS PLS outlet file header.
+ *
+ *   Output: one row per outlet with `venue` (library name), `(house_number, street, locality,
+ *   subregion, postcode)`, and lat/lon preserved in `source_id` stability.
+ *
+ *   License: stamped `"Public Domain"` per IMLS federal government distribution terms.
+ */
+
+import { createReadStream } from "node:fs"
+
+import { parse as csvParse } from "csv-parse"
+
+import { stableSourceID } from "../../adapter.ts"
+import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
+import { reconcileComponents } from "../../format.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+
+export const USGOV_IMLS_PLS_ADAPTER_ID = "usgov-imls-pls"
+export const USGOV_IMLS_PLS_DEFAULT_LICENSE = "Public Domain"
+
+const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
+
+interface ImlsOutletRow {
+	LIBNAME: string
+	ADDRESS: string
+	CITY: string
+	ZIP: string
+	STABR: string
+	CNTY: string
+	FSCSKEY: string
+}
+
+function splitAddress(address: string): { house_number?: string; street: string } | null {
+	const trimmed = address.trim()
+
+	if (!trimmed) return null
+	const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
+
+	if (m) return { house_number: m[1], street: m[2]!.trim() }
+
+	return { street: trimmed }
+}
+
+export function createUsgovImlsPlsAdapter(): CorpusAdapter {
+	return {
+		id: USGOV_IMLS_PLS_ADAPTER_ID,
+		defaultLicense: USGOV_IMLS_PLS_DEFAULT_LICENSE,
+		description: "IMLS Public Libraries Survey — ~17K library outlets with venue+address (public-domain).",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`usgov-imls-pls adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
+			const parser = stream.pipe(
+				csvParse({
+					columns: true,
+					skip_empty_lines: true,
+					relax_quotes: true,
+					relax_column_count: true,
+				})
+			)
+
+			let emitted = 0
+
+			try {
+				for await (const record of parser as AsyncIterable<ImlsOutletRow>) {
+					if (opts.signal?.aborted) break
+
+					if (opts.limit !== undefined && emitted >= opts.limit) break
+
+					const libName = (record.LIBNAME ?? "").trim()
+					const address = (record.ADDRESS ?? "").trim()
+					const city = (record.CITY ?? "").trim()
+					const zip = (record.ZIP ?? "").trim()
+					const stateAbbr = (record.STABR ?? "").trim()
+					const county = (record.CNTY ?? "").trim()
+
+					if (!libName || !city || !zip) continue
+
+					const state = lookupStateAbbreviation(stateAbbr)
+
+					if (!state) continue
+
+					const split = splitAddress(address)
+
+					if (!split) continue
+
+					const components: CanonicalRow["components"] = {
+						venue: libName,
+						...(split.house_number ? { house_number: split.house_number } : {}),
+						street: split.street,
+						locality: city,
+						region: state.abbreviation,
+						postcode: zip,
+						// #552: no subregion — US postal addresses don't surface the county, so emitting
+						// subregion creates a phantom component with no raw-span to align to, quarantining
+						// ~21% of rows. The county is still available in the source CSV; it just isn't
+						// a postal-surface component here.
+					}
+
+					const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
+					const raw = [
+						libName,
+						streetPart,
+						[city, [stateAbbr, zip].filter(Boolean).join(" ")].filter(Boolean).join(", "),
+					]
+						.filter(Boolean)
+						.join(", ")
+
+					const aligned = reconcileComponents(components, raw)
+
+					if (Object.keys(aligned).length <= 2) continue
+
+					const fscsKey = (record.FSCSKEY ?? "").trim()
+					const sourceID = fscsKey
+						? `${USGOV_IMLS_PLS_ADAPTER_ID}-${fscsKey}`
+						: stableSourceID(USGOV_IMLS_PLS_ADAPTER_ID, aligned)
+
+					yield {
+						raw,
+						components: aligned,
+						country: "US",
+						locale: "en-US",
+						source: USGOV_IMLS_PLS_ADAPTER_ID,
+						source_id: sourceID,
+						corpus_version: "",
+						license: USGOV_IMLS_PLS_DEFAULT_LICENSE,
+					}
+					emitted++
+				}
+			} finally {
+				stream.destroy()
+			}
+		},
+	}
+}
+
+export const usgovImlsPlsAdapter = createUsgovImlsPlsAdapter()

package/src/adapters/usgov-irs-bmf/adapter.ts

@@ -0,0 +1,155 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `usgov-irs-bmf`: IRS Exempt Organizations Business Master File (EO BMF) CSV consumer.
+ *
+ *   The EO BMF is the IRS's authoritative registry of US tax-exempt organizations (charities,
+ *   churches, foundations, ...), published as per-region CSVs at
+ *   `https://www.irs.gov/charities-non-profits/exempt-organizations-business-master-file-extract-eo-bmf`
+ *   (`eo1.csv`..`eo4.csv`, `eo_pr.csv`, `eo_xx.csv`). Each row carries an organization NAME plus
+ *   its mailing address. It complements `usgov-nppes` with a DIFFERENT venue population
+ *   (non-profits vs healthcare providers) and, notably, a high share of PO-box addresses — useful
+ *   `po_box`-tag signal (a tag with historically low recall).
+ *
+ *   Output: one row per record with a usable city + postcode. NAME → `venue`; the street line becomes
+ *   `po_box` when it's a PO-box, else `house_number` + `street`; CITY/STATE/ZIP fill the locality
+ *   line. STATE is already a USPS abbreviation in the source. License: `"Public Domain"` (US
+ *   federal).
+ */
+
+import { createReadStream } from "node:fs"
+
+import { parse as csvParse } from "csv-parse"
+
+import { stableSourceID } from "../../adapter.ts"
+import { reconcileComponents } from "../../format.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+
+export const USGOV_IRS_BMF_ADAPTER_ID = "usgov-irs-bmf"
+export const USGOV_IRS_BMF_DEFAULT_LICENSE = "Public Domain"
+
+const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
+// PO box in its many written forms: "PO BOX 12", "P.O. BOX 12", "P O BOX 12", "POB 12", "BOX 12".
+const PO_BOX = /^\s*(?:P\.?\s?O\.?\s*BOX|POB|BOX)\s+\w/i
+
+interface IrsBmfRow {
+	EIN: string
+	NAME: string
+	STREET: string
+	CITY: string
+	STATE: string
+	ZIP: string
+}
+
+/** Classify the street line into a `po_box` or a `{house_number?, street}` split. */
+function splitStreetLine(street: string): { po_box: string } | { house_number?: string; street: string } | null {
+	const trimmed = street.trim()
+
+	if (!trimmed) return null
+
+	if (PO_BOX.test(trimmed)) return { po_box: trimmed }
+	const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
+
+	if (m) return { house_number: m[1], street: m[2]!.trim() }
+
+	return { street: trimmed }
+}
+
+function composeRaw(
+	venue: string | undefined,
+	streetPart: string,
+	city: string,
+	state: string,
+	postcode: string
+): string {
+	const cityPart = [city.trim(), [state, postcode].filter(Boolean).join(" ").trim()].filter(Boolean).join(", ")
+
+	return [venue, streetPart, cityPart].filter(Boolean).join(", ")
+}
+
+export function createUsgovIrsBmfAdapter(): CorpusAdapter {
+	return {
+		id: USGOV_IRS_BMF_ADAPTER_ID,
+		defaultLicense: USGOV_IRS_BMF_DEFAULT_LICENSE,
+		description:
+			"IRS Exempt Organizations Business Master File — US non-profit venue+address (public-domain), with strong PO-box coverage.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`usgov-irs-bmf adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
+			const parser = stream.pipe(
+				csvParse({ columns: true, skip_empty_lines: true, relax_quotes: true, relax_column_count: true, trim: true })
+			)
+
+			let emitted = 0
+
+			try {
+				for await (const record of parser as AsyncIterable<IrsBmfRow>) {
+					if (opts.signal?.aborted) break
+
+					if (opts.limit !== undefined && emitted >= opts.limit) break
+
+					const ein = (record.EIN ?? "").trim()
+					const venue = (record.NAME ?? "").trim() || undefined
+					const street = (record.STREET ?? "").trim()
+					const city = (record.CITY ?? "").trim()
+					const state = (record.STATE ?? "").trim()
+					const zipRaw = (record.ZIP ?? "").trim()
+
+					if (!city || !zipRaw) continue
+					const postcode = zipRaw.split("-")[0]!.trim() // 5-digit; drop the optional +4
+
+					const split = splitStreetLine(street)
+
+					if (!split) continue
+
+					const streetPart =
+						"po_box" in split ? split.po_box : [split.house_number, split.street].filter(Boolean).join(" ")
+
+					const components: CanonicalRow["components"] = {
+						...(venue ? { venue } : {}),
+						...("po_box" in split
+							? { po_box: split.po_box }
+							: { ...(split.house_number ? { house_number: split.house_number } : {}), street: split.street }),
+						locality: city,
+						...(state ? { region: state } : {}),
+						postcode,
+					}
+
+					const raw = composeRaw(venue, streetPart, city, state, postcode)
+
+					if (!raw) continue
+
+					const aligned = reconcileComponents(components, raw)
+
+					if (Object.keys(aligned).length <= 2) continue
+
+					const sourceID = ein
+						? `${USGOV_IRS_BMF_ADAPTER_ID}-${ein}`
+						: stableSourceID(USGOV_IRS_BMF_ADAPTER_ID, aligned)
+
+					yield {
+						raw,
+						components: aligned,
+						country: "US",
+						locale: "en-US",
+						source: USGOV_IRS_BMF_ADAPTER_ID,
+						source_id: sourceID,
+						corpus_version: "",
+						license: USGOV_IRS_BMF_DEFAULT_LICENSE,
+					}
+					emitted++
+				}
+			} finally {
+				stream.destroy()
+			}
+		},
+	}
+}
+
+export const usgovIrsBmfAdapter = createUsgovIrsBmfAdapter()