@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/us/adapters/state/tx-notaries/adapter.ts

@@ -0,0 +1,155 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `state-tx-notaries`: Texas Notary Public Commissions CSV consumer.
+ *
+ *   The Texas Secretary of State publishes a registry of commissioned notaries public. Each row
+ *   optionally includes a mailing address in free-form text (often multi-line with embedded
+ *   city/state/zip). Address fill rate is ~5-10%.
+ *
+ *   The adapter parses the embedded `Address` field for city/state/zip using a trailing `"city, ST
+ *   ZIP"` pattern.
+ *
+ *   License: stamped `"Public Domain"` per Texas state government open-data terms.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+import { formatPersonName } from "@mailwoman/record/name"
+import { CSVSpliterator } from "spliterator"
+
+import { stableSourceID } from "#adapters/source-id"
+import { splitStreetLine } from "#adapters/street-line"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
+import { lookupStateAbbreviation } from "#us/fips-state"
+
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ */
+export const STATE_TX_NOTARIES_ADAPTER_ID = "state-tx-notaries"
+/**
+ * License assigned by this source (Public Domain), attached to each row so downstream
+ * consumers inherit the terms rather than having to look them up.
+ */
+export const STATE_TX_NOTARIES_DEFAULT_LICENSE = "Public Domain"
+
+/**
+ * Match trailing "city, ST ZIP" or "city, ST" at the end of an address line.
+ */
+const CITY_STATE_ZIP_SUFFIX = /[,]?\s*([^,]+),\s*([A-Z]{2})\s*(\d{5}(?:-\d{4})?)?\s*$/i
+
+interface TxNotaryRow {
+	"Notary ID": string
+	"First Name": string
+	"Last Name": string
+	Address: string
+}
+
+export function createStateTxNotariesAdapter(): CorpusAdapter {
+	return {
+		id: STATE_TX_NOTARIES_ADAPTER_ID,
+		defaultLicense: STATE_TX_NOTARIES_DEFAULT_LICENSE,
+		addressRole: AddressRole.Mailing,
+		register: SourceRegister.TexasNotaries,
+		surface: SurfaceOrigin.Rendered,
+		description:
+			"Texas Notary Public Commissions — name + mailing address with embedded city/state/zip (public-domain).",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`state-tx-notaries adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				normalizeKeys: false,
+			})
+
+			let emitted = 0
+
+			for await (const record of rows as AsyncIterable<TxNotaryRow>) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const rawAddress = record.Address ?? ""
+
+				if (!rawAddress) continue
+
+				const firstName = record["First Name"] ?? ""
+				const lastName = record["Last Name"] ?? ""
+				const notaryID = record["Notary ID"] ?? ""
+
+				// Parse embedded city/state/zip from the trailing portion of the address.
+				// Addresses look like: "1215 mcmillan DR\nCEDAR hill, TX 75104"
+				const addrSingleLine = rawAddress.replaceAll("\n", ", ")
+				const cszMatch = CITY_STATE_ZIP_SUFFIX.exec(addrSingleLine)
+
+				if (!cszMatch) continue
+
+				const city = (cszMatch[1] ?? "").trim()
+				const stateAbbr = (cszMatch[2] ?? "").trim()
+				const zip = (cszMatch[3] ?? "").trim()
+
+				if (!city || !stateAbbr) continue
+
+				const state = lookupStateAbbreviation(stateAbbr)
+
+				if (!state) continue
+
+				// Extract the street portion (everything before the city/state/zip)
+				const streetPortion = addrSingleLine.slice(0, cszMatch.index).replace(/,\s*$/, "").trim()
+
+				if (!streetPortion) continue
+
+				const split = splitStreetLine(streetPortion)
+
+				if (!split) continue
+
+				const venue = formatPersonName({ given: firstName, family: lastName }, "short") || undefined
+
+				const components: CanonicalRow["components"] = {
+					...(venue ? { venue } : {}),
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					...(zip ? { postcode: zip } : {}),
+				}
+
+				const rendered = formatAddressRow(components, "US", { singleLine: true })
+
+				if (!rendered) continue
+
+				const { raw, components: aligned } = rendered
+
+				if (Object.keys(aligned).length <= 2) continue
+
+				const sourceID = notaryID
+					? `${STATE_TX_NOTARIES_ADAPTER_ID}-${notaryID}`
+					: stableSourceID(STATE_TX_NOTARIES_ADAPTER_ID, aligned)
+
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: STATE_TX_NOTARIES_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: STATE_TX_NOTARIES_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const stateTxNotariesAdapter = createStateTxNotariesAdapter()

package/lib/us/adapters/tiger/adapter.ts

@@ -0,0 +1,225 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * `tiger`: Adapter for US Census TIGER/Line data.
+ *
+ * This reads a prebuilt SQLite database (from TIGER shapefiles) and emits:
+ * - Street rows from `tiger_streets`
+ * - Locality rows from `tiger_places`
+ *
+ * TIGER is public domain, so every emitted row is stamped `"Public Domain"`.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+import { DatabaseClient } from "@mailwoman/sqlite/client"
+import type { TIGERDatabase } from "@mailwoman/tiger/schema"
+
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
+import { decomposeStreet } from "#us/adapters/tiger/street-decompose"
+import { lookupFipsState } from "#us/fips-state"
+
+/**
+ * Registry ID for this adapter.
+ */
+export const TIGER_ADAPTER_ID = "tiger"
+/**
+ * Default license for rows from this source.
+ */
+export const TIGER_DEFAULT_LICENSE = "Public Domain"
+
+/**
+ * Country display name used by `formatAddress`.
+ */
+const US_COUNTRY_DISPLAY = "United States of America"
+
+interface TigerStreetRow {
+	linearid: string
+	fullname: string
+	zipl: string | null
+	zipr: string | null
+	statefp: string
+}
+
+interface TigerPlaceRow {
+	geoid: string
+	name: string
+	statefp: string
+	lsad: string | null
+}
+
+/**
+ * Build one or more street variants from a TIGER street segment.
+ */
+function* streetVariants(row: TigerStreetRow): Iterable<{
+	components: CanonicalRow["components"]
+	variantKey: string
+}> {
+	const fullname = row.fullname.trim()
+
+	if (!fullname) return
+	const state = lookupFipsState(row.statefp)
+
+	if (!state) return
+
+	const zipl = row.zipl?.trim() ?? ""
+	const zipr = row.zipr?.trim() ?? ""
+
+	const decomposed = decomposeStreet(fullname)
+
+	const baseComponents: CanonicalRow["components"] = {
+		region: state.abbreviation,
+		street: decomposed.street,
+	}
+
+	if (decomposed.prefix) {
+		baseComponents.street_prefix = decomposed.prefix
+	}
+
+	if (decomposed.suffix) {
+		baseComponents.street_suffix = decomposed.suffix
+	}
+
+	if (!zipl && !zipr) {
+		yield { components: baseComponents, variantKey: "no-zip" }
+
+		return
+	}
+
+	if (zipl && zipr && zipl === zipr) {
+		yield {
+			components: { ...baseComponents, postcode: zipl },
+			variantKey: `zip-${zipl}`,
+		}
+
+		return
+	}
+
+	if (zipl) {
+		yield { components: { ...baseComponents, postcode: zipl }, variantKey: `zipl-${zipl}` }
+	}
+
+	if (zipr && zipr !== zipl) {
+		yield { components: { ...baseComponents, postcode: zipr }, variantKey: `zipr-${zipr}` }
+	}
+}
+
+/**
+ * Build locality variants for a TIGER place.
+ */
+function* placeVariants(row: TigerPlaceRow): Iterable<{
+	components: CanonicalRow["components"]
+	variantKey: string
+}> {
+	const name = row.name.trim()
+
+	if (!name) return
+	const state = lookupFipsState(row.statefp)
+
+	if (!state) return
+
+	yield {
+		components: { locality: name },
+		variantKey: "locality-only",
+	}
+
+	yield {
+		components: { locality: name, region: state.abbreviation },
+		variantKey: "with-region",
+	}
+
+	yield {
+		components: { locality: name, region: state.abbreviation, country: US_COUNTRY_DISPLAY },
+		variantKey: "with-region-country",
+	}
+}
+
+/**
+ * Build a TIGER adapter.
+ */
+export function createTigerAdapter(): CorpusAdapter {
+	return {
+		id: TIGER_ADAPTER_ID,
+		defaultLicense: TIGER_DEFAULT_LICENSE,
+		addressRole: AddressRole.Premise,
+		register: SourceRegister.CensusTIGER,
+		surface: SurfaceOrigin.Rendered,
+		description:
+			"US Census TIGER/Line streets + places consumer (public-domain); SQLite DB built via `mailwoman tiger fetch`.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`tiger adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const db = new DatabaseClient<TIGERDatabase>(opts.inputPath, { readOnly: true })
+			let emitted = 0
+
+			try {
+				const streetStmt = db.prepare(`SELECT linearid, fullname, zipl, zipr, statefp FROM tiger_streets`)
+				const placeStmt = db.prepare(`SELECT geoid, name, statefp, lsad FROM tiger_places`)
+
+				for (const row of streetStmt.iterate() as IterableIterator<TigerStreetRow>) {
+					if (opts.signal?.aborted) return
+
+					for (const variant of streetVariants(row)) {
+						if (opts.limit !== undefined && emitted >= opts.limit) return
+						const rendered = formatAddressRow(variant.components, "US", { singleLine: true })
+
+						if (!rendered) continue
+
+						const { raw, components: aligned } = rendered
+
+						yield {
+							raw,
+							components: aligned,
+							country: "US",
+							locale: "en-US",
+							source: TIGER_ADAPTER_ID,
+							source_id: `${TIGER_ADAPTER_ID}-st-${row.linearid}-${variant.variantKey}`,
+							corpus_version: "",
+							license: TIGER_DEFAULT_LICENSE,
+						}
+
+						emitted++
+					}
+				}
+
+				for (const row of placeStmt.iterate() as IterableIterator<TigerPlaceRow>) {
+					if (opts.signal?.aborted) return
+
+					for (const variant of placeVariants(row)) {
+						if (opts.limit !== undefined && emitted >= opts.limit) return
+						const rendered = formatAddressRow(variant.components, "US", { singleLine: true })
+
+						if (!rendered) continue
+
+						const { raw, components: aligned } = rendered
+
+						yield {
+							raw,
+							components: aligned,
+							country: "US",
+							locale: "en-US",
+							source: TIGER_ADAPTER_ID,
+							source_id: `${TIGER_ADAPTER_ID}-pl-${row.geoid}-${variant.variantKey}`,
+							corpus_version: "",
+							license: TIGER_DEFAULT_LICENSE,
+						}
+
+						emitted++
+					}
+				}
+			} finally {
+				await db.destroy()
+			}
+		},
+	}
+}
+
+/**
+ * Configured adapter instance.
+ */
+export const tigerAdapter = createTigerAdapter()

package/lib/us/adapters/tiger/street-decompose.ts

@@ -0,0 +1,73 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Decompose a US street name into Stage 3 components (street_prefix, street, street_suffix) from the
+ * curated libpostal/en directionals and street_types dictionaries — the same ones the runtime
+ * StreetPrefixClassifier and StreetSuffixClassifier use, so corpus labels and runtime classifications
+ * agree on the vocabulary.
+ */
+
+import { loadLibpostalDictionary } from "#adapters/libpostal-dictionary"
+
+const [DIRECTIONALS, STREET_TYPES] = await Promise.all([
+	loadLibpostalDictionary("en", "directionals.txt"),
+	loadLibpostalDictionary("en", "street_types.txt"),
+])
+
+export interface DecomposedStreet {
+	prefix: string | null
+	street: string
+	suffix: string | null
+}
+
+/**
+ * Decompose a US street name into prefix/name/suffix components.
+ *
+ * Conservative, emitting a prefix or suffix only for a clear directional or street-type keyword.
+ * It returns the original as `street` when no pattern matches.
+ */
+export function decomposeStreet(fullname: string): DecomposedStreet {
+	const trimmed = fullname.trim()
+
+	if (!trimmed) return { prefix: null, street: "", suffix: null }
+
+	const tokens = trimmed.split(/\s+/)
+
+	if (tokens.length === 1) return { prefix: null, street: trimmed, suffix: null }
+
+	const norm = (s: string) => s.toLowerCase().replace(/\.$/, "")
+
+	let prefix: string | null = null
+	let suffix: string | null = null
+	let startIdx = 0
+	let endIdx = tokens.length
+
+	if (DIRECTIONALS.has(norm(tokens[0]!)) && tokens.length >= 2) {
+		prefix = tokens[0]!
+		startIdx = 1
+	}
+
+	const last = norm(tokens[endIdx - 1]!)
+	const secondLast = endIdx >= 2 ? norm(tokens[endIdx - 2]!) : ""
+
+	if (DIRECTIONALS.has(last) && STREET_TYPES.has(secondLast)) {
+		suffix = tokens.slice(endIdx - 2, endIdx).join(" ")
+		endIdx -= 2
+	} else if (STREET_TYPES.has(last) && endIdx - startIdx >= 2) {
+		suffix = tokens[endIdx - 1]!
+		endIdx -= 1
+	} else if (DIRECTIONALS.has(last) && endIdx - startIdx >= 2) {
+		suffix = tokens[endIdx - 1]!
+		endIdx -= 1
+	}
+
+	const street = tokens.slice(startIdx, endIdx).join(" ").trim()
+
+	if (!street) {
+		return { prefix: null, street: trimmed, suffix: null }
+	}
+
+	return { prefix, street, suffix }
+}

package/lib/us/adapters/usgov/hrsa-fqhc/adapter.ts

@@ -0,0 +1,171 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `usgov-hrsa-fqhc`: HRSA "Health Center Service Delivery Site Locations" CSV consumer.
+ *
+ *   Federally Qualified Health Centers (FQHCs) are HRSA-funded community health programs.
+ *   They self-report site addresses to the HRSA Data Warehouse. The published CSV (`data.hrsa.gov`)
+ *   includes the site name, postal-formatted street address, locality, region and postcode.
+ *   Each facility name is a human-typed venue string. Multiple people have edited each address.
+ *   That editing introduces abbreviation drift and suite designators absent from gazetteer data.
+ *
+ *   The adapter consumes a CSV file the operator pre-downloads. The HRSA data is published as a
+ *   single national CSV (~10K rows), small enough that the operator can re-fetch on every corpus
+ *   rebuild without an intermediate SQLite step. Column names below match the HRSA Data Warehouse's
+ *   "Health Center Service Delivery Site" public dataset. Operators substituting the
+ *   closely-related "Site Address" or "Health Center" public extracts may need to remap columns.
+ *   the readme documents the expected set.
+ *
+ *   Output: one row per CSV record, with `venue` component carrying the site name and the address
+ *   quad on `(house_number, street, locality, region, postcode)`. Component order is critical:
+ *   `venue` is inserted first so alignment claims its surface span before `locality` searches for
+ *   its own (the kryptonite case "Buffalo Health Clinic, …, Buffalo, NY" relies on `venue`
+ *   consuming the first "Buffalo" so locality lands on the second).
+ *
+ *   License: stamped `"Public Domain"` per the HRSA Data Warehouse's federal government distribution
+ *   terms.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+import { CSVSpliterator } from "spliterator"
+
+import { stableSourceID } from "#adapters/source-id"
+import { splitStreetLine } from "#adapters/street-line"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
+import { lookupStateAbbreviation } from "#us/fips-state"
+
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ */
+export const USGOV_HRSA_FQHC_ADAPTER_ID = "usgov-hrsa-fqhc"
+/**
+ * License assigned by this source (Public Domain), attached to each row so downstream
+ * consumers inherit the terms rather than having to look them up.
+ */
+export const USGOV_HRSA_FQHC_DEFAULT_LICENSE = "Public Domain"
+
+/**
+ * Subset of HRSA "Health Center Service Delivery Site Locations" CSV columns consulted by the adapter.
+ *
+ * Column names match the canonical HRSA Data Warehouse export header.
+ * Operators substituting a closely-related extract should rename columns to match.
+ * The readme has the mapping cheatsheet.
+ */
+interface HRSASiteRow {
+	"Site Name": string
+	"Site Address": string
+	"Site City": string
+	"Site State Abbreviation": string
+	"Site Postal Code": string
+	/**
+	 * Optional.
+	 *
+	 * Falls back to `stableSourceID` derived from components when missing.
+	 */
+	"Site ID"?: string
+}
+
+/**
+ * Split a "123 Main St Suite 4" surface form into `(house_number, street)`.
+ *
+ * The regex tolerates one trailing letter on the number (`"123A Main St"`)
+ * and a hyphenated form (`"40-12 Bell Blvd"`); anything else falls back to street-only.
+ *
+ * Suite / Apt / Unit designators stay on `street`.
+ * Mailwoman's `unit` component exists.
+ *
+ * The address formatter has no clean slot for it.
+ * HRSA addresses also keep the suite in the street column.
+ *
+ * The intact surface form in `street` keeps the adversarial training signal
+ * (the model learns that a trailing "Suite 4" is part of the road line in this distribution).
+ */
+
+export function createUSGovHRSAFQHCAdapter(): CorpusAdapter {
+	return {
+		id: USGOV_HRSA_FQHC_ADAPTER_ID,
+		defaultLicense: USGOV_HRSA_FQHC_DEFAULT_LICENSE,
+		addressRole: AddressRole.Facility,
+		register: SourceRegister.HRSAHealthCenters,
+		surface: SurfaceOrigin.Rendered,
+		description:
+			"HRSA Federally Qualified Health Center site locations (public-domain). Adversarial source: venue + address co-occurrence, hand-entered.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`usgov-hrsa-fqhc adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				normalizeKeys: false,
+			})
+
+			let emitted = 0
+
+			for await (const record of rows as AsyncIterable<HRSASiteRow>) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const venue = record["Site Name"] ?? ""
+				const split = splitStreetLine(record["Site Address"] ?? "")
+				const city = record["Site City"] ?? ""
+				const stateAbbr = record["Site State Abbreviation"] ?? ""
+				const postcode = record["Site Postal Code"] ?? ""
+
+				if (!venue || !split || !city || !postcode) continue
+				const state = lookupStateAbbreviation(stateAbbr)
+
+				if (!state) continue
+
+				// Insertion order matters here.
+				// `venue` first so alignment claims its span (which may contain a token like "Buffalo")
+				// before `locality` runs its search — the kryptonite case `Buffalo Health Clinic, Buffalo NY`
+				// otherwise mis-labels the venue's "Buffalo" as locality.
+				const components: CanonicalRow["components"] = {
+					venue,
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode,
+				}
+
+				const rendered = formatAddressRow(components, "US", { singleLine: true })
+
+				if (!rendered) continue
+
+				const { raw, components: aligned } = rendered
+
+				const siteID = record["Site ID"] ?? ""
+
+				const sourceID = siteID
+					? `${USGOV_HRSA_FQHC_ADAPTER_ID}-${siteID}`
+					: stableSourceID(USGOV_HRSA_FQHC_ADAPTER_ID, aligned)
+
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: USGOV_HRSA_FQHC_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: USGOV_HRSA_FQHC_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const usgovHrsaFqhcAdapter = createUSGovHRSAFQHCAdapter()