@mailwoman/corpus 9.2.0 → 9.3.0

package/lib/adapters/index.ts

@@ -0,0 +1,153 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Adapter registry bootstrap.
+ *
+ *   Importing this module registers every built-in adapter with `defaultAdapterRegistry`. The CLI
+ *   (`commands/corpus/list.tsx`, `commands/corpus/run.tsx`) imports it once at startup.
+ *
+ *   Adapters under construction live in their own subdirectories (`./wof-admin-json/`, `./ban/`, ...)
+ *   and are added to the `BUILTIN_ADAPTERS` list here as they come online. Tests that need a
+ *   pristine registry should construct their own `InMemoryAdapterRegistry` instead of mutating the
+ *   default.
+ *
+ *   The WOF adapters export their canonical ids — `wof-admin` and `wof-postalcode` — so existing
+ *   `mailwoman corpus build` callsites do not need to change despite the Phase 1.5.1 SQLite →
+ *   JSON-bundle pivot (`./wof-admin-json/` and `./wof-postalcode-json/` directories hold the
+ *   implementations; the registered ids are unchanged).
+ */
+
+import { banAdapter } from "#adapters/ban/adapter"
+import { fccBdcAdapter } from "#adapters/fcc-bdc/adapter"
+import { geonamesPostalAdapter } from "#adapters/geonames-postal/adapter"
+import { geonamesAdapter } from "#adapters/geonames/adapter"
+import { gnafAdapter } from "#adapters/gnaf/adapter"
+import { openaddressesAdapter } from "#adapters/openaddresses/adapter"
+import { overtureAdapter } from "#adapters/overture/adapter"
+import { stateHiSchoolsAdapter } from "#adapters/state-hi-schools/adapter"
+import { stateIaContractorsAdapter } from "#adapters/state-ia-contractors/adapter"
+import { stateNyNotariesAdapter } from "#adapters/state-ny-notaries/adapter"
+import { stateTxNotariesAdapter } from "#adapters/state-tx-notaries/adapter"
+import { tigerAdapter } from "#adapters/tiger/adapter"
+import { usgovHrsaFqhcAdapter } from "#adapters/usgov-hrsa-fqhc/adapter"
+import { usgovImlsPlsAdapter } from "#adapters/usgov-imls-pls/adapter"
+import { USGovIRSBMFAdapter } from "#adapters/usgov-irs-bmf/adapter"
+import { usgovNADAdapter } from "#adapters/usgov-nad/adapter"
+import { usgovNPPESAdapter } from "#adapters/usgov-nppes/adapter"
+import { defaultAdapterRegistry } from "#adapters/utils"
+import { wofAdminAdapter } from "#adapters/wof-admin-json/adapter"
+import { wofPostalcodeAdapter } from "#adapters/wof-postalcode-json/adapter"
+import type { CorpusAdapter } from "#types"
+
+/**
+ * Built-in adapters. Order is significant: `corpus build` iterates this list to drive every adapter in turn.
+ * Coarse-first (admin → postcode), then street-level (BAN FR, TIGER US, OpenAddresses global), then adversarial-source
+ * (FCC BDC US, HRSA FQHC US).
+ *
+ * The `usgov-samhsa-treatment-locator` adapter is intentionally absent from this list — the SAMHSA Open Data Foundry
+ * bulk CSV the adapter was written against is no longer publicly distributed (see issue #33, 2026-05-17 investigation).
+ * The factory + named export remain available so the adapter can be hand-registered if an operator obtains a compatible
+ * CSV (FOIA, partner channel, upstream restoration). Re-add it here once a stable public source returns.
+ */
+export const BUILTIN_ADAPTERS: readonly CorpusAdapter[] = [
+	wofAdminAdapter,
+	wofPostalcodeAdapter,
+	geonamesAdapter,
+	geonamesPostalAdapter,
+	banAdapter,
+	tigerAdapter,
+	openaddressesAdapter,
+	overtureAdapter,
+	gnafAdapter,
+	fccBdcAdapter,
+	usgovHrsaFqhcAdapter,
+	usgovNPPESAdapter,
+	usgovNADAdapter,
+	usgovImlsPlsAdapter,
+	USGovIRSBMFAdapter,
+	stateIaContractorsAdapter,
+	stateTxNotariesAdapter,
+	stateNyNotariesAdapter,
+	stateHiSchoolsAdapter,
+]
+
+for (const adapter of BUILTIN_ADAPTERS) {
+	if (!defaultAdapterRegistry.get(adapter.id)) {
+		defaultAdapterRegistry.register(adapter)
+	}
+}
+
+export { BAN_ADAPTER_ID, banAdapter } from "#adapters/ban/adapter"
+export { FCC_BDC_ADAPTER_ID, FCC_BDC_DEFAULT_LICENSE, fccBdcAdapter } from "#adapters/fcc-bdc/adapter"
+
+export {
+	GEONAMES_POSTAL_ADAPTER_ID,
+	GEONAMES_POSTAL_DEFAULT_LICENSE,
+	geonamesPostalAdapter,
+} from "#adapters/geonames-postal/adapter"
+
+export { GEONAMES_ADAPTER_ID, GEONAMES_DEFAULT_LICENSE, geonamesAdapter } from "#adapters/geonames/adapter"
+
+export {
+	OPENADDRESSES_ADAPTER_ID,
+	OPENADDRESSES_DEFAULT_LICENSE,
+	openaddressesAdapter,
+} from "#adapters/openaddresses/adapter"
+
+export {
+	STATE_HI_SCHOOLS_ADAPTER_ID,
+	STATE_HI_SCHOOLS_DEFAULT_LICENSE,
+	stateHiSchoolsAdapter,
+} from "#adapters/state-hi-schools/adapter"
+
+export {
+	STATE_IA_CONTRACTORS_ADAPTER_ID,
+	STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
+	stateIaContractorsAdapter,
+} from "#adapters/state-ia-contractors/adapter"
+
+export {
+	STATE_NY_NOTARIES_ADAPTER_ID,
+	STATE_NY_NOTARIES_DEFAULT_LICENSE,
+	stateNyNotariesAdapter,
+} from "#adapters/state-ny-notaries/adapter"
+
+export {
+	STATE_TX_NOTARIES_ADAPTER_ID,
+	STATE_TX_NOTARIES_DEFAULT_LICENSE,
+	stateTxNotariesAdapter,
+} from "#adapters/state-tx-notaries/adapter"
+
+export { TIGER_ADAPTER_ID, TIGER_DEFAULT_LICENSE, tigerAdapter } from "#adapters/tiger/adapter"
+
+export {
+	USGOV_HRSA_FQHC_ADAPTER_ID,
+	USGOV_HRSA_FQHC_DEFAULT_LICENSE,
+	usgovHrsaFqhcAdapter,
+} from "#adapters/usgov-hrsa-fqhc/adapter"
+
+export {
+	USGOV_IMLS_PLS_ADAPTER_ID,
+	USGOV_IMLS_PLS_DEFAULT_LICENSE,
+	usgovImlsPlsAdapter,
+} from "#adapters/usgov-imls-pls/adapter"
+
+export {
+	USGOV_IRS_BMF_ADAPTER_ID,
+	USGOV_IRS_BMF_DEFAULT_LICENSE,
+	USGovIRSBMFAdapter,
+} from "#adapters/usgov-irs-bmf/adapter"
+
+export { USGOV_NAD_ADAPTER_ID, USGOV_NAD_DEFAULT_LICENSE, usgovNADAdapter } from "#adapters/usgov-nad/adapter"
+export { USGOV_NPPES_ADAPTER_ID, USGOV_NPPES_DEFAULT_LICENSE, usgovNPPESAdapter } from "#adapters/usgov-nppes/adapter"
+
+export {
+	USGOV_SAMHSA_ADAPTER_ID,
+	USGOV_SAMHSA_DEFAULT_LICENSE,
+	usgovSamhsaTreatmentLocatorAdapter,
+} from "#adapters/usgov-samhsa-treatment-locator/adapter"
+
+export { WOF_ADMIN_ADAPTER_ID, wofAdminAdapter } from "#adapters/wof-admin-json/adapter"
+export { WOF_POSTALCODE_ADAPTER_ID, wofPostalcodeAdapter } from "#adapters/wof-postalcode-json/adapter"

package/lib/adapters/openaddresses/adapter.ts

@@ -0,0 +1,251 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `openaddresses`: Line-delimited GeoJSON adapter for openaddresses.io exports.
+ *
+ *   OpenAddresses publishes country-partitioned address dumps as either CSV or line-delimited GeoJSON
+ *   (one `Feature` per line, also called ND-GeoJSON / GeoJSONL). This adapter consumes the
+ *   line-delimited GeoJSON shape — it streams cleanly without holding the file in memory, which
+ *   matters for the multi-gigabyte national dumps (e.g. `us-northeast.geojsonl`, ~20M rows).
+ *
+ *   The collection aggregates **hundreds** of underlying sources with **per-source licenses** (city
+ *   open-data portals, county GIS departments, state DOTs). The adapter therefore prefers the
+ *   per-row `LICENSE` property when present and falls back to the configured `defaultLicense`. The
+ *   propagated license travels with each `CanonicalRow` so downstream code can stratify, exclude,
+ *   or re-attribute by license at training time.
+ *
+ *   Country must be explicit (`opts.country` REQUIRED): OpenAddresses files are organized by country
+ *   but the row-level data doesn't include a country code, so the adapter refuses to run without
+ *   one. This matches how a `mailwoman corpus build` invocation pins each file to a country via the
+ *   inputs JSON.
+ *
+ *   Properties consumed (per the canonical OpenAddresses schema; both UPPERCASE and lowercase
+ *   variants are accepted because legacy dumps used UPPERCASE):
+ *
+ *   | Property | ComponentTag | | ------------- |
+ *   -------------------------------------------------------------- | | `number` | `house_number` |
+ *   | `street` | `street` | | `unit` | `unit` (if non-empty) | | `city` | `locality` | | `region` |
+ *   `region` (state code for US, province for CA, etc.) | | `postcode` | `postcode` | | `LICENSE` |
+ *   per-row `license` override | | `hash` / `id` | `source_id` (prefer `hash`; fall back to `id`;
+ *   then synthesize)|
+ *
+ *   `district` is intentionally NOT mapped — for US data it carries borough or county and would
+ *   inflate alignment quarantine because postal addresses don't include it. Phase 6+ may revisit
+ *   for non-US locales where district names DO appear on the envelope.
+ */
+
+import { tryParsingJSON } from "@mailwoman/core/json"
+import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
+import { TextSpliterator } from "spliterator"
+
+import { stableSourceID } from "#adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+import { SHARE_ALIKE_PATTERN } from "#utils"
+
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const OPENADDRESSES_ADAPTER_ID = "openaddresses"
+/**
+ * License carried by this source (CC-BY-4.0), attached to each row so downstream consumers inherit the terms rather
+ * than having to look them up.
+ */
+export const OPENADDRESSES_DEFAULT_LICENSE = "CC-BY-4.0"
+
+/**
+ * Subset of OpenAddresses Feature properties the adapter inspects. The runtime accepts UPPERCASE or lowercase keys;
+ * this interface documents the canonical lowercase form after normalization.
+ */
+interface OaProperties {
+	hash?: string
+	id?: string
+	number?: string
+	street?: string
+	unit?: string
+	city?: string
+	district?: string
+	region?: string
+	postcode?: string
+	license?: string
+}
+
+/**
+ * Return a lowercase-keyed view of a Feature's properties so case variants both work.
+ */
+function normalizeProperties(raw: unknown): OaProperties {
+	if (!raw || typeof raw !== "object") return {}
+	const out: Record<string, string> = {}
+
+	for (const [k, v] of Object.entries(raw as Record<string, unknown>)) {
+		if (typeof v === "string") {
+			out[k.toLowerCase()] = v
+		} else if (typeof v === "number") {
+			out[k.toLowerCase()] = String(v)
+		}
+	}
+
+	return out as OaProperties
+}
+
+/**
+ * Parse a single ND-GeoJSON line; return null for blanks, comments, or non-Feature shapes.
+ */
+function parseFeatureLine(line: string): OaProperties | null {
+	const trimmed = line.trim()
+
+	if (!trimmed || trimmed.startsWith("#")) return null
+	const parsed = tryParsingJSON(trimmed)
+
+	if (!parsed || typeof parsed !== "object") return null
+	const obj = parsed as { type?: string; properties?: unknown }
+
+	if (obj.type !== "Feature") return null
+
+	return normalizeProperties(obj.properties)
+}
+
+export interface OpenaddressesAdapterOptions {
+	/**
+	 * Per-row license used when a Feature lacks an explicit `LICENSE` property. Defaults to `CC-BY-4.0` — the most common
+	 * license across the OpenAddresses collection. Override per dump via the runner's adapter-options passthrough.
+	 */
+	defaultLicense?: string
+
+	/**
+	 * Per-adapter share-alike drop. Default **true** (include) as of 2026-06-19: exclusion is a deliberate BUILD-level
+	 * act (`buildCorpus({ excludeLicenses })` / `--exclude-share-alike`), NOT a silent adapter default (#26 — "purposely
+	 * exclude, don't opt in to include"). Set false only for an explicit adapter-scoped drop; the build-level
+	 * `--exclude-share-alike` is the normal path.
+	 */
+	allowShareAlike?: boolean
+}
+
+/**
+ * Build an OpenAddresses adapter. The optional `defaultLicense` lets callers stamp a non-default fallback for dumps
+ * known to carry a single license throughout (e.g. a PDDL-only state slice).
+ */
+export function createOpenaddressesAdapter(opts: OpenaddressesAdapterOptions = {}): CorpusAdapter {
+	const defaultLicense = opts.defaultLicense ?? OPENADDRESSES_DEFAULT_LICENSE
+	const allowShareAlike = opts.allowShareAlike ?? true
+
+	return {
+		id: OPENADDRESSES_ADAPTER_ID,
+		defaultLicense,
+		description: "OpenAddresses (global): line-delimited GeoJSON dumps with per-row licenses.",
+
+		async *rows(adapterOpts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (!adapterOpts.country) {
+				throw new Error(
+					"openaddresses adapter: --country is required (OpenAddresses files are country-partitioned but rows lack a country field)"
+				)
+			}
+
+			const country = adapterOpts.country
+
+			// TextSpliterator streams string lines (parseFeatureLine keeps tolerating blank/`#`/
+			// malformed lines by returning null); passing the path string lets the lib own + dispose
+			// the file handle, including on an early `break`.
+			const lines = TextSpliterator.fromAsync(adapterOpts.inputPath)
+
+			let emitted = 0
+			let shareAlikeBlocked = 0
+
+			try {
+				for await (const line of lines) {
+					if (adapterOpts.signal?.aborted) break
+
+					if (adapterOpts.limit !== undefined && emitted >= adapterOpts.limit) break
+
+					const props = parseFeatureLine(line)
+
+					if (!props) continue
+
+					const houseNumber = props.number?.trim() ?? ""
+					const street = props.street?.trim() ?? ""
+					const unit = props.unit?.trim() ?? ""
+					const city = props.city?.trim() ?? ""
+					const region = props.region?.trim() ?? ""
+					const postcode = props.postcode?.trim() ?? ""
+
+					// A row is only useful if it has, at minimum, a street + (postcode OR locality).
+					// Pure point-only rows would land in quarantine anyway.
+					if (!street) continue
+
+					if (!city && !postcode) continue
+
+					const license = (props.license?.trim() || defaultLicense).trim()
+
+					if (!allowShareAlike && SHARE_ALIKE_PATTERN.test(license)) {
+						shareAlikeBlocked++
+
+						continue
+					}
+
+					const components: CanonicalRow["components"] = {}
+
+					if (houseNumber) {
+						components.house_number = houseNumber
+					}
+
+					if (street) {
+						components.street = street
+					}
+
+					if (unit) {
+						components.unit = unit
+					}
+
+					if (city) {
+						components.locality = city
+					}
+
+					if (region) {
+						components.region = region
+					}
+
+					if (postcode) {
+						components.postcode = postcode
+					}
+
+					const raw = formatAddress(components, country, { separator: ", " })
+
+					if (!raw) continue
+
+					const aligned = reconcileComponents(components, raw)
+
+					if (!Object.keys(aligned).length) continue
+
+					const sourceIDSeed = props.hash?.trim() || props.id?.trim()
+
+					const sourceID = sourceIDSeed
+						? `${OPENADDRESSES_ADAPTER_ID}-${sourceIDSeed}`
+						: stableSourceID(OPENADDRESSES_ADAPTER_ID, aligned)
+
+					yield {
+						raw,
+						components: aligned,
+						country,
+						source: OPENADDRESSES_ADAPTER_ID,
+						source_id: sourceID,
+						corpus_version: "",
+						license,
+					}
+
+					emitted++
+				}
+			} finally {
+				if (shareAlikeBlocked > 0) {
+					process.stderr.write(`  openaddresses: ${shareAlikeBlocked} share-alike rows dropped, ${emitted} kept\n`)
+				}
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const openaddressesAdapter = createOpenaddressesAdapter()

package/lib/adapters/overture/adapter.ts

@@ -0,0 +1,162 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `overture`: Overture Maps Addresses adapter (epic #470 — the conditional corpus adapter, realized
+ *   2026-06-20). Overture's global Addresses theme is the single-schema, well-normalized address
+ *   dataset that fixes OpenAddresses' per-country patchiness (OA dropped Spain; OA-DE omits the
+ *   Bundesland) — it even re-hosts the OA Spain data the standalone OA bucket no longer serves.
+ *
+ *   This adapter consumes a per-country LINE-DELIMITED JSON dump of the corpus-relevant fields (`{
+ *   street, number, unit, postcode, locality }`), produced by `scripts/ingest-overture-addresses.ts
+ *   --corpus-jsonl` (which does the DuckDB / S3 heavy lifting and flattens `address_levels` → the
+ *   municipality locality). The split keeps `@mailwoman/corpus` — a RUNTIME dep of the `mailwoman`
+ *   CLI — free of the heavy native `@duckdb/node-api`; the adapter just streams JSONL line-by-line,
+ *   exactly like `openaddresses`.
+ *
+ *   The `street` surface carries the locale's street keyword verbatim (`"CALLE JULAN"`, `"VIA
+ *   ROMA"`). We map it to `street` whole and let the downstream affix-relabel split `street_prefix`
+ *   — the same path every other source rides. This slice exists because the model was
+ *   en-us/fr-trained and never saw non-en/fr street formats (the 2026-06-19 EU parse-blocker
+ *   measured loc-correct ES 21% / IT 59% / NL 64% vs FR/US ~98%).
+ *
+ *   `--country` is REQUIRED (the JSONL is per-country and the rows omit a country field), matching
+ *   `openaddresses`. License is Overture's CDLA-Permissive-2.0 (attribution; not share-alike).
+ *
+ *   | Field | ComponentTag | | --------- | ---------------------------------------------- | |
+ *   `street` | `street` (keyword incl.; affix-relabel splits prefix) | | `number` | `house_number`
+ *   (skipped when "S-N"/"S/N" = sin número) | | `unit` | `unit` (if non-empty) | | `postcode`|
+ *   `postcode` | | `locality`| `locality` (Overture address_levels municipality, or postal_city) |
+ */
+
+import { tryParsingJSON } from "@mailwoman/core/json"
+import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
+import { TextSpliterator } from "spliterator"
+
+import { stableSourceID } from "#adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const OVERTURE_ADAPTER_ID = "overture"
+/**
+ * License carried by this source (CDLA-Permissive-2.0), attached to each row so downstream consumers inherit the terms
+ * rather than having to look them up.
+ */
+export const OVERTURE_DEFAULT_LICENSE = "CDLA-Permissive-2.0"
+
+/**
+ * The flattened per-row shape emitted by `ingest-overture-addresses.ts --corpus-jsonl`.
+ */
+interface OvertureCorpusRow {
+	street?: string
+	number?: string
+	unit?: string
+	postcode?: string
+	locality?: string
+}
+
+function parseLine(line: string): OvertureCorpusRow | null {
+	const t = line.trim()
+
+	if (!t || t.startsWith("#")) return null
+
+	const o = tryParsingJSON(t)
+
+	return o && typeof o === "object" ? (o as OvertureCorpusRow) : null
+}
+
+export function createOvertureAdapter(): CorpusAdapter {
+	return {
+		id: OVERTURE_ADAPTER_ID,
+		defaultLicense: OVERTURE_DEFAULT_LICENSE,
+		description: "Overture Maps Addresses (global): per-country JSONL of street/number/postcode/locality.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (!opts.country) {
+				throw new Error(
+					"overture adapter: --country is required (the Overture JSONL is per-country and rows omit a country field)"
+				)
+			}
+
+			const country = opts.country
+
+			// TextSpliterator streams string lines (parseLine keeps tolerating blank/`#`/malformed
+			// lines by returning null); the path string lets the lib own + dispose the file handle,
+			// including on an early `break`.
+			const lines = TextSpliterator.fromAsync(opts.inputPath)
+
+			let emitted = 0
+
+			for await (const line of lines) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const r = parseLine(line)
+
+				if (!r) continue
+
+				const street = r.street?.trim() ?? ""
+				const number = r.number?.trim() ?? ""
+				const unit = r.unit?.trim() ?? ""
+				const postcode = r.postcode?.trim() ?? ""
+				const locality = r.locality?.trim() ?? ""
+
+				// Only useful with a street + (postcode OR locality); point-only rows quarantine anyway.
+				if (!street) continue
+
+				if (!postcode && !locality) continue
+
+				const components: CanonicalRow["components"] = {}
+
+				// Overture "S-N" / "S/N" = sin número; only keep a real numeric house number.
+				if (/^\d/.test(number)) {
+					components.house_number = number
+				}
+
+				components.street = street
+
+				if (unit) {
+					components.unit = unit
+				}
+
+				if (postcode) {
+					components.postcode = postcode
+				}
+
+				if (locality) {
+					components.locality = locality
+				}
+
+				const raw = formatAddress(components, country, { separator: ", " })
+
+				if (!raw) continue
+
+				const aligned = reconcileComponents(components, raw)
+
+				if (!Object.keys(aligned).length) continue
+
+				yield {
+					raw,
+					components: aligned,
+					country,
+					source: OVERTURE_ADAPTER_ID,
+					source_id: stableSourceID(OVERTURE_ADAPTER_ID, aligned),
+					corpus_version: "",
+					license: OVERTURE_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const overtureAdapter = createOvertureAdapter()