@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/adapters/tiger/adapter.ts

@@ -1,252 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `tiger`: US Census TIGER/Line consumer adapter.
- *
- *   TIGER/Line is the canonical US street + locality dataset published by the Census Bureau as a
- *   **public-domain** product (no ODbL share-alike concerns for US-only corpora). Coverage extends
- *   to every named street segment + every incorporated place + CDP across the 50 states + DC + the
- *   five primary territories — substantially better US street-name coverage than OSM, especially in
- *   rural areas.
- *
- *   Following the `wof-admin` / `wof-postalcode` pattern, this adapter consumes a SQLite database the
- *   operator pre-builds from the raw TIGER shapefiles (see the README for the schema and a
- *   suggested `ogr2ogr` pipeline). The mailwoman side does not parse Shapefile binary directly —
- *   keeping the adapter narrow lets the operator pick their own ingestion tool (ogr2ogr / shp2pgsql
- *   / a custom Python script / etc.) without forcing a heavy native dep into `@mailwoman/corpus`.
- *
- *   Two row classes are emitted:
- *
- *   - **Street-level** (`tiger_streets`): one row per segment, optionally with up to two postcode
- *       variants if `zipl` / `zipr` differ. Components: `{ street, region, postcode? }`. Streets
- *       without a recognized state FIPS are dropped — there's no useful row without `region`.
- *   - **Locality-level** (`tiger_places`): up to three variants per place: locality-only,
- *       locality-with-region, locality-with-region-country (mirrors `wof-admin`'s fan-out for
- *       consistency).
- *   - `packages/corpus/lib/codex/us-fips-state.ts` — the FIPS → `{abbreviation, name}` lookup table
- *       (originally `tiger/state.ts`, AGPL-3.0 → AGPL-3.0). The full isp-nexus TIGER module ships a
- *       TypeORM-backed service layer; mailwoman only needs the lookup data so we don't carry the
- *       service layer over.
- *
- *   License: stamped `"Public Domain"` per Census Bureau guidance on TIGER/Line. No per-row override
- *   needed — every row in TIGER is the same license.
- */
-
-import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
-import { DatabaseClient } from "@mailwoman/sqlite/client"
-import type { TIGERDatabase } from "@mailwoman/tiger/sdk/schema"
-
-import { decomposeStreet } from "#adapters/tiger/street-decompose"
-import { lookupFipsState } from "#codex/us-fips-state"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const TIGER_ADAPTER_ID = "tiger"
-/**
- * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
- * than having to look them up.
- */
-export const TIGER_DEFAULT_LICENSE = "Public Domain"
-
-/**
- * The country surface form used in `formatAddress` for US. Matches the canonical OpenCage US template output so
- * reconciliation doesn't strip it when the row carries `country` explicitly.
- */
-const US_COUNTRY_DISPLAY = "United States of America"
-
-interface TigerStreetRow {
-	linearid: string
-	fullname: string
-	zipl: string | null
-	zipr: string | null
-	statefp: string
-}
-
-interface TigerPlaceRow {
-	geoid: string
-	name: string
-	statefp: string
-	lsad: string | null
-}
-
-/**
- * Yield one or more `CanonicalRow`s per street segment. Postcode variants:
- *
- * - No ZIP set → one row, street + region.
- * - `zipl === zipr` → one row, street + region + postcode.
- * - `zipl !== zipr` → two rows (one per side's ZIP).
- */
-function* streetVariants(row: TigerStreetRow): Iterable<{
-	components: CanonicalRow["components"]
-	variantKey: string
-}> {
-	const fullname = row.fullname.trim()
-
-	if (!fullname) return
-	const state = lookupFipsState(row.statefp)
-
-	if (!state) return
-
-	const zipl = row.zipl?.trim() ?? ""
-	const zipr = row.zipr?.trim() ?? ""
-
-	const decomposed = decomposeStreet(fullname)
-
-	const baseComponents: CanonicalRow["components"] = {
-		region: state.abbreviation,
-		street: decomposed.street,
-	}
-
-	if (decomposed.prefix) {
-		baseComponents.street_prefix = decomposed.prefix
-	}
-
-	if (decomposed.suffix) {
-		baseComponents.street_suffix = decomposed.suffix
-	}
-
-	if (!zipl && !zipr) {
-		yield { components: baseComponents, variantKey: "no-zip" }
-
-		return
-	}
-
-	if (zipl && zipr && zipl === zipr) {
-		yield {
-			components: { ...baseComponents, postcode: zipl },
-			variantKey: `zip-${zipl}`,
-		}
-
-		return
-	}
-
-	if (zipl) {
-		yield { components: { ...baseComponents, postcode: zipl }, variantKey: `zipl-${zipl}` }
-	}
-
-	if (zipr && zipr !== zipl) {
-		yield { components: { ...baseComponents, postcode: zipr }, variantKey: `zipr-${zipr}` }
-	}
-}
-
-/**
- * Three locality-level variants, mirroring `wof-admin`'s fan-out.
- */
-function* placeVariants(row: TigerPlaceRow): Iterable<{
-	components: CanonicalRow["components"]
-	variantKey: string
-}> {
-	const name = row.name.trim()
-
-	if (!name) return
-	const state = lookupFipsState(row.statefp)
-
-	if (!state) return
-
-	yield {
-		components: { locality: name },
-		variantKey: "locality-only",
-	}
-
-	yield {
-		components: { locality: name, region: state.abbreviation },
-		variantKey: "with-region",
-	}
-
-	yield {
-		components: { locality: name, region: state.abbreviation, country: US_COUNTRY_DISPLAY },
-		variantKey: "with-region-country",
-	}
-}
-
-/**
- * Build a TIGER adapter. Pure factory so multiple instances can be created in tests.
- */
-export function createTigerAdapter(): CorpusAdapter {
-	return {
-		id: TIGER_ADAPTER_ID,
-		defaultLicense: TIGER_DEFAULT_LICENSE,
-		description:
-			"US Census TIGER/Line streets + places consumer (public-domain); SQLite DB built via `mailwoman tiger fetch`.",
-
-		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			if (opts.country && opts.country !== "US") {
-				throw new Error(`tiger adapter: only US supported, got country=${opts.country}`)
-			}
-
-			const db = new DatabaseClient<TIGERDatabase>(opts.inputPath, { readOnly: true })
-			let emitted = 0
-
-			try {
-				const streetStmt = db.prepare(`SELECT linearid, fullname, zipl, zipr, statefp FROM tiger_streets`)
-				const placeStmt = db.prepare(`SELECT geoid, name, statefp, lsad FROM tiger_places`)
-
-				for (const row of streetStmt.iterate() as IterableIterator<TigerStreetRow>) {
-					if (opts.signal?.aborted) return
-
-					for (const variant of streetVariants(row)) {
-						if (opts.limit !== undefined && emitted >= opts.limit) return
-						const raw = formatAddress(variant.components, "US", { separator: ", " })
-
-						if (!raw) continue
-						const aligned = reconcileComponents(variant.components, raw)
-
-						if (!Object.keys(aligned).length) continue
-
-						yield {
-							raw,
-							components: aligned,
-							country: "US",
-							locale: "en-US",
-							source: TIGER_ADAPTER_ID,
-							source_id: `${TIGER_ADAPTER_ID}-st-${row.linearid}-${variant.variantKey}`,
-							corpus_version: "",
-							license: TIGER_DEFAULT_LICENSE,
-						}
-
-						emitted++
-					}
-				}
-
-				for (const row of placeStmt.iterate() as IterableIterator<TigerPlaceRow>) {
-					if (opts.signal?.aborted) return
-
-					for (const variant of placeVariants(row)) {
-						if (opts.limit !== undefined && emitted >= opts.limit) return
-						const raw = formatAddress(variant.components, "US", { separator: ", " })
-
-						if (!raw) continue
-						const aligned = reconcileComponents(variant.components, raw)
-
-						if (!Object.keys(aligned).length) continue
-
-						yield {
-							raw,
-							components: aligned,
-							country: "US",
-							locale: "en-US",
-							source: TIGER_ADAPTER_ID,
-							source_id: `${TIGER_ADAPTER_ID}-pl-${row.geoid}-${variant.variantKey}`,
-							corpus_version: "",
-							license: TIGER_DEFAULT_LICENSE,
-						}
-
-						emitted++
-					}
-				}
-			} finally {
-				await db.destroy()
-			}
-		},
-	}
-}
-
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const tigerAdapter = createTigerAdapter()

package/lib/adapters/tiger/street-decompose.ts

@@ -1,83 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Decompose a US street name into Stage 3 components: street_prefix, street, street_suffix.
- *
- *   Sources directionals and street types from the curated libpostal/en dictionaries
- *   (`core/data/libpostal/dictionaries/en/{directionals,street_types}.txt`). These are the same
- *   dictionaries the runtime classifiers (StreetPrefixClassifier, StreetSuffixClassifier) use, so
- *   corpus labels and runtime classifications agree on the vocabulary.
- *
- *   Examples: "N Main St" → { prefix: "N", street: "Main", suffix: "St" } "Pennsylvania Avenue NW" →
- *   { prefix: null, street: "Pennsylvania", suffix: "Avenue NW" } "Salmon St" → { prefix: null,
- *   street: "Salmon", suffix: "St" } "SE Hawthorne Blvd" → { prefix: "SE", street: "Hawthorne",
- *   suffix: "Blvd" }
- */
-
-import { loadLibpostalDictionary } from "#adapters/utils"
-
-const [DIRECTIONALS, STREET_TYPES] = await Promise.all([
-	loadLibpostalDictionary("en", "directionals.txt"),
-	loadLibpostalDictionary("en", "street_types.txt"),
-])
-
-export interface DecomposedStreet {
-	prefix: string | null
-	street: string
-	suffix: string | null
-}
-
-/**
- * Decompose a US street name into prefix/name/suffix components.
- *
- * Conservative — only emits prefix/suffix when there's a clear directional or street-type keyword. Returns the original
- * as `street` if nothing matches.
- */
-export function decomposeStreet(fullname: string): DecomposedStreet {
-	const trimmed = fullname.trim()
-
-	if (!trimmed) return { prefix: null, street: "", suffix: null }
-
-	const tokens = trimmed.split(/\s+/)
-
-	if (tokens.length === 1) return { prefix: null, street: trimmed, suffix: null }
-
-	const norm = (s: string) => s.toLowerCase().replace(/\.$/, "")
-
-	let prefix: string | null = null
-	let suffix: string | null = null
-	let startIdx = 0
-	let endIdx = tokens.length
-
-	// Leading directional prefix
-	if (DIRECTIONALS.has(norm(tokens[0]!)) && tokens.length >= 2) {
-		prefix = tokens[0]!
-		startIdx = 1
-	}
-
-	// Trailing post-directional combined with street type (e.g. "Pennsylvania Ave NW")
-	const last = norm(tokens[endIdx - 1]!)
-	const secondLast = endIdx >= 2 ? norm(tokens[endIdx - 2]!) : ""
-
-	if (DIRECTIONALS.has(last) && STREET_TYPES.has(secondLast)) {
-		suffix = tokens.slice(endIdx - 2, endIdx).join(" ")
-		endIdx -= 2
-	} else if (STREET_TYPES.has(last) && endIdx - startIdx >= 2) {
-		suffix = tokens[endIdx - 1]!
-		endIdx -= 1
-	} else if (DIRECTIONALS.has(last) && endIdx - startIdx >= 2) {
-		// Post-directional without type
-		suffix = tokens[endIdx - 1]!
-		endIdx -= 1
-	}
-
-	const street = tokens.slice(startIdx, endIdx).join(" ").trim()
-
-	if (!street) {
-		return { prefix: null, street: trimmed, suffix: null }
-	}
-
-	return { prefix, street, suffix }
-}

package/lib/adapters/usgov-hrsa-fqhc/adapter.ts

@@ -1,167 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `usgov-hrsa-fqhc`: HRSA "Health Center Service Delivery Site Locations" CSV consumer.
- *
- *   Federally Qualified Health Centers (FQHCs) are HRSA-funded community health programs that
- *   self-report site addresses to the HRSA Data Warehouse. The published CSV (`data.hrsa.gov`)
- *   carries the site name, the postal-formatted street address, and the locality/region/postcode
- *   quad. The source earns its place on adversarial-value-per-row (#22): every facility name is a
- *   human-typed venue string and the addresses pass through enough hands to accumulate the
- *   abbreviation drift + suite designator chaos that pure gazetteer data does not.
- *
- *   The adapter consumes a CSV file the operator pre-downloads. The HRSA data is published as a
- *   single national CSV (~10K rows), small enough that the operator can re-fetch on every corpus
- *   rebuild without an intermediate SQLite step. Column names below match the HRSA Data Warehouse's
- *   "Health Center Service Delivery Site" public dataset. Operators substituting the
- *   closely-related "Site Address" or "Health Center" public extracts may need to remap columns;
- *   the README documents the expected set.
- *
- *   Output: one row per CSV record, with `venue` component carrying the site name and the address
- *   quad on `(house_number, street, locality, region, postcode)`. Component order is critical:
- *   `venue` is inserted FIRST so alignment claims its surface span before `locality` searches for
- *   its own (the kryptonite case "Buffalo Health Clinic, …, Buffalo, NY" relies on `venue`
- *   consuming the first "Buffalo" so locality lands on the second).
- *
- *   License: stamped `"Public Domain"` per the HRSA Data Warehouse's federal government distribution
- *   terms.
- */
-
-import { reconcileComponents } from "@mailwoman/formatter"
-import { CSVSpliterator } from "spliterator"
-
-import { composeRaw, splitStreetLine, stableSourceID } from "#adapters/utils"
-import { lookupStateAbbreviation } from "#codex/us-fips-state"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const USGOV_HRSA_FQHC_ADAPTER_ID = "usgov-hrsa-fqhc"
-/**
- * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
- * than having to look them up.
- */
-export const USGOV_HRSA_FQHC_DEFAULT_LICENSE = "Public Domain"
-
-/**
- * Subset of HRSA "Health Center Service Delivery Site Locations" CSV columns consulted by the adapter. Column names
- * match the canonical HRSA Data Warehouse export header. Operators substituting a closely-related extract should rename
- * columns to match; the README has the mapping cheatsheet.
- */
-interface HRSASiteRow {
-	"Site Name": string
-	"Site Address": string
-	"Site City": string
-	"Site State Abbreviation": string
-	"Site Postal Code": string
-	/**
-	 * Optional. Falls back to `stableSourceID` derived from components when missing.
-	 */
-	"Site ID"?: string
-}
-
-/**
- * Split a "123 Main St Suite 4" surface form into `(house_number, street)`. The regex tolerates one trailing letter on
- * the number (`"123A Main St"`) and a hyphenated form (`"40-12 Bell Blvd"`); anything else falls back to street-only.
- *
- * Suite / Apt / Unit designators stay on `street` — Mailwoman's `unit` component exists but the address-formatter does
- * not have a clean slot for it, and HRSA addresses do not separate the suite into its own column. Leaving the surface
- * form intact in `street` preserves the adversarial training signal (the model learns that a trailing "Suite 4" is part
- * of the road line in this distribution).
- */
-
-export function createUSGovHRSAFQHCAdapter(): CorpusAdapter {
-	return {
-		id: USGOV_HRSA_FQHC_ADAPTER_ID,
-		defaultLicense: USGOV_HRSA_FQHC_DEFAULT_LICENSE,
-		description:
-			"HRSA Federally Qualified Health Center site locations (public-domain). Adversarial source: venue + address co-occurrence, hand-entered.",
-
-		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			if (opts.country && opts.country !== "US") {
-				throw new Error(`usgov-hrsa-fqhc adapter: only US supported, got country=${opts.country}`)
-			}
-
-			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
-				mode: "object",
-				normalizeKeys: false,
-				enableQuoteHandling: true,
-			})
-
-			let emitted = 0
-
-			for await (const record of rows as AsyncIterable<HRSASiteRow>) {
-				if (opts.signal?.aborted) break
-
-				if (opts.limit !== undefined && emitted >= opts.limit) break
-
-				const venue = (record["Site Name"] ?? "").trim()
-				const split = splitStreetLine(record["Site Address"] ?? "")
-				const city = (record["Site City"] ?? "").trim()
-				const stateAbbr = (record["Site State Abbreviation"] ?? "").trim()
-				const postcode = (record["Site Postal Code"] ?? "").trim()
-
-				if (!venue || !split || !city || !postcode) continue
-				const state = lookupStateAbbreviation(stateAbbr)
-
-				if (!state) continue
-
-				// Insertion order matters here. `venue` first so alignment claims its span
-				// (which may contain a token like "Buffalo") before `locality` runs its
-				// search — the kryptonite case `Buffalo Health Clinic, Buffalo NY`
-				// otherwise mis-labels the venue's "Buffalo" as locality.
-				const components: CanonicalRow["components"] = {
-					venue,
-					...(split.house_number ? { house_number: split.house_number } : {}),
-					street: split.street,
-					locality: city,
-					region: state.abbreviation,
-					postcode,
-				}
-
-				const raw = composeRaw({
-					venue,
-					houseNumber: split.house_number,
-					street: split.street,
-					locality: city,
-					region: state.abbreviation,
-					postcode,
-				})
-
-				if (!raw) continue
-
-				const aligned = reconcileComponents(components, raw)
-
-				if (!Object.keys(aligned).length) continue
-
-				const siteID = (record["Site ID"] ?? "").trim()
-
-				const sourceID = siteID
-					? `${USGOV_HRSA_FQHC_ADAPTER_ID}-${siteID}`
-					: stableSourceID(USGOV_HRSA_FQHC_ADAPTER_ID, aligned)
-
-				yield {
-					raw,
-					components: aligned,
-					country: "US",
-					locale: "en-US",
-					source: USGOV_HRSA_FQHC_ADAPTER_ID,
-					source_id: sourceID,
-					corpus_version: "",
-					license: USGOV_HRSA_FQHC_DEFAULT_LICENSE,
-				}
-
-				emitted++
-			}
-		},
-	}
-}
-
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const usgovHrsaFqhcAdapter = createUSGovHRSAFQHCAdapter()

package/lib/adapters/usgov-imls-pls/adapter.ts

@@ -1,143 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `usgov-imls-pls`: IMLS Public Libraries Survey outlet CSV consumer.
- *
- *   The Institute of Museum and Library Services publishes an annual Public Libraries Survey with one
- *   row per library outlet (~17K rows). Each row carries the library name, street address, city,
- *   ZIP, county, and geocoordinates.
- *
- *   The adapter consumes the outlet CSV the operator pre-downloads via `fetch-imls-pls.ts`. Column
- *   names match the IMLS PLS outlet file header.
- *
- *   Output: one row per outlet with `venue` (library name), `(house_number, street, locality,
- *   subregion, postcode)`, and lat/lon preserved in `source_id` stability.
- *
- *   License: stamped `"Public Domain"` per IMLS federal government distribution terms.
- */
-
-import { isPresent } from "@mailwoman/core/objects"
-import { reconcileComponents } from "@mailwoman/formatter"
-import { CSVSpliterator } from "spliterator"
-
-import { splitStreetLine, stableSourceID } from "#adapters/utils"
-import { lookupStateAbbreviation } from "#codex/us-fips-state"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const USGOV_IMLS_PLS_ADAPTER_ID = "usgov-imls-pls"
-/**
- * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
- * than having to look them up.
- */
-export const USGOV_IMLS_PLS_DEFAULT_LICENSE = "Public Domain"
-
-interface ImlsOutletRow {
-	LIBNAME: string
-	ADDRESS: string
-	CITY: string
-	ZIP: string
-	STABR: string
-	CNTY: string
-	FSCSKEY: string
-}
-
-export function createUSGovIMLSPLSAdapter(): CorpusAdapter {
-	return {
-		id: USGOV_IMLS_PLS_ADAPTER_ID,
-		defaultLicense: USGOV_IMLS_PLS_DEFAULT_LICENSE,
-		description: "IMLS Public Libraries Survey — ~17K library outlets with venue+address (public-domain).",
-
-		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			if (opts.country && opts.country !== "US") {
-				throw new Error(`usgov-imls-pls adapter: only US supported, got country=${opts.country}`)
-			}
-
-			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
-				mode: "object",
-				normalizeKeys: false,
-				enableQuoteHandling: true,
-			})
-
-			let emitted = 0
-
-			for await (const record of rows as AsyncIterable<ImlsOutletRow>) {
-				if (opts.signal?.aborted) break
-
-				if (opts.limit !== undefined && emitted >= opts.limit) break
-
-				const libName = (record.LIBNAME ?? "").trim()
-				const address = (record.ADDRESS ?? "").trim()
-				const city = (record.CITY ?? "").trim()
-				const zip = (record.ZIP ?? "").trim()
-				const stateAbbr = (record.STABR ?? "").trim()
-
-				if (!libName || !city || !zip) continue
-
-				const state = lookupStateAbbreviation(stateAbbr)
-
-				if (!state) continue
-
-				const split = splitStreetLine(address)
-
-				if (!split) continue
-
-				const components: CanonicalRow["components"] = {
-					venue: libName,
-					...(split.house_number ? { house_number: split.house_number } : {}),
-					street: split.street,
-					locality: city,
-					region: state.abbreviation,
-					postcode: zip,
-					// #552: no subregion — US postal addresses don't surface the county, so emitting
-					// subregion creates a phantom component with no raw-span to align to, quarantining
-					// ~21% of rows. The county is still available in the source CSV; it just isn't
-					// a postal-surface component here.
-				}
-
-				const streetPart = [split.house_number, split.street].filter(isPresent).join(" ").trim()
-
-				const raw = [
-					libName,
-					streetPart,
-					[city, [stateAbbr, zip].filter(isPresent).join(" ")].filter(isPresent).join(", "),
-				]
-					.filter(isPresent)
-					.join(", ")
-
-				const aligned = reconcileComponents(components, raw)
-
-				if (Object.keys(aligned).length <= 2) continue
-
-				const fscsKey = (record.FSCSKEY ?? "").trim()
-
-				const sourceID = fscsKey
-					? `${USGOV_IMLS_PLS_ADAPTER_ID}-${fscsKey}`
-					: stableSourceID(USGOV_IMLS_PLS_ADAPTER_ID, aligned)
-
-				yield {
-					raw,
-					components: aligned,
-					country: "US",
-					locale: "en-US",
-					source: USGOV_IMLS_PLS_ADAPTER_ID,
-					source_id: sourceID,
-					corpus_version: "",
-					license: USGOV_IMLS_PLS_DEFAULT_LICENSE,
-				}
-
-				emitted++
-			}
-		},
-	}
-}
-
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const usgovImlsPlsAdapter = createUSGovIMLSPLSAdapter()