@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/adapters/ban/adapter.ts

@@ -1,190 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `ban`: Base Adresse Nationale CSV adapter (FR street-level).
- *
- *   Input: a CSV dump from `adresse.data.gouv.fr` (semicolon-separated, ~25M rows nationally). The
- *   adapter only reads the small set of columns needed for the corpus:
- *
- *   - `numero` → `house_number`
- *   - `rep` → repetition index ("bis", "ter") appended to house_number
- *   - `nom_voie` → `street` (full road name; includes the prefix "Rue", "Avenue", etc.)
- *   - `code_postal` → `postcode`
- *   - `nom_commune` → `locality`
- *
- *   `region` and `country` are not in BAN. The adapter stamps `country: "FR"` on every row; region is
- *   left for the wof-postalcode + wof-admin cross-reference at corpus build time (a future pass;
- *   for Phase 1 the row's region is simply absent).
- *
- *   License: the official BAN (adresse.data.gouv.fr) is DUAL-licensed — Licence Ouverte 2.0 (Etalab,
- *   attribution-only) OR ODbL (share-alike). We ELECT Licence Ouverte 2.0 (issue #26 Tier B:
- *   allowed for training with attribution; the ODbL option's share-alike obligation would defeat
- *   the proprietary-weights goal). Stamped onto every row as `Licence Ouverte 2.0` — NOT the older
- *   conservative `ODbL-1.0` label, which wrongly read as Tier-C-denied in the corpus license audit.
- *   The model card MUST carry the BAN attribution (Tier B obligation).
- *
- *   The adapter is streaming-aware: `CSVSpliterator.fromAsync` reads the `;`-delimited dump row by
- *   row, so a 25M-row file never sits in memory. Honors `opts.limit` for fixture / smoke runs,
- *   `opts.signal` for cancellation, and `opts.country` for a self-consistency check (errors if
- *   country !== FR).
- */
-
-import { isPresent } from "@mailwoman/core/objects"
-import { reconcileComponents } from "@mailwoman/formatter"
-import { CSVSpliterator } from "spliterator"
-
-import { decomposeFrStreet } from "#adapters/ban/street-decompose"
-import { stableSourceID } from "#adapters/utils"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const BAN_ADAPTER_ID = "ban"
-
-/**
- * Subset of BAN CSV columns the adapter consults. Everything else is ignored; declaring the shape explicitly catches
- * column-name drift early if BAN evolves its schema.
- */
-interface BanRow {
-	id: string
-	numero: string
-	rep: string
-	nom_voie: string
-	code_postal: string
-	nom_commune: string
-}
-
-/**
- * Compose `house_number` from `numero` + `rep`. BAN uses `rep` for repetition indices ("bis", "ter", "quater") that
- * follow the house number. Result: `"10 bis"`, `"45"`, etc.
- */
-function composeHouseNumber(numero: string, rep: string): string {
-	const n = numero.trim()
-	const r = rep.trim()
-
-	if (!n) return ""
-
-	return r ? `${n} ${r}` : n
-}
-
-/**
- * Compose the raw FR-style address line. Two common BAN-derived shapes:
- *
- * "10 bis Avenue des Champs-Élysées, 75008 Paris" "45 Cours Lafayette, 69003 Lyon"
- *
- * FR convention puts postcode on the same line as the locality, comma-separated from the street. The adapter renders
- * that directly rather than relying on OpenCage's template — BAN already gives us the canonical FR strings; the
- * template would round-trip identically.
- */
-function composeRaw(house: string, street: string, postcode: string, locality: string): string {
-	const parts: string[] = []
-	const streetPart = [house, street].filter(isPresent).join(" ").trim()
-
-	if (streetPart) {
-		parts.push(streetPart)
-	}
-
-	const cityPart = [postcode, locality].filter(isPresent).join(" ").trim()
-
-	if (cityPart) {
-		parts.push(cityPart)
-	}
-
-	return parts.join(", ").replaceAll(/\s+/g, " ").trim()
-}
-
-export function createBanAdapter(): CorpusAdapter {
-	return {
-		id: BAN_ADAPTER_ID,
-		defaultLicense: "Licence Ouverte 2.0",
-		description: "Base Adresse Nationale (FR): house-number-level street addresses (~25M rows).",
-
-		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			if (opts.country && opts.country !== "FR") {
-				throw new Error(`ban adapter: only FR supported, got country=${opts.country}`)
-			}
-
-			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
-				mode: "object",
-				normalizeKeys: false,
-				columnDelimiter: ";",
-				enableQuoteHandling: true,
-			})
-
-			let emitted = 0
-
-			for await (const record of rows as AsyncIterable<BanRow>) {
-				if (opts.signal?.aborted) break
-
-				if (opts.limit !== undefined && emitted >= opts.limit) break
-
-				const house = composeHouseNumber(record.numero ?? "", record.rep ?? "")
-				const street = (record.nom_voie ?? "").trim()
-				const postcode = (record.code_postal ?? "").trim()
-				const locality = (record.nom_commune ?? "").trim()
-
-				if (!street || !locality) continue
-
-				if (!house && !postcode) continue
-
-				const decomposed = decomposeFrStreet(street)
-
-				const components: CanonicalRow["components"] = {}
-
-				if (house) {
-					components.house_number = house
-				}
-
-				if (decomposed.prefix) {
-					components.street_prefix = decomposed.prefix
-				}
-
-				if (decomposed.street) {
-					components.street = decomposed.street
-				}
-
-				if (postcode) {
-					components.postcode = postcode
-				}
-
-				if (locality) {
-					components.locality = locality
-				}
-
-				const raw = composeRaw(house, street, postcode, locality)
-
-				if (!raw) continue
-
-				const aligned = reconcileComponents(components, raw)
-
-				if (!Object.keys(aligned).length) continue
-
-				const sourceID = record.id?.trim()
-					? `${BAN_ADAPTER_ID}-${record.id.trim()}`
-					: stableSourceID(BAN_ADAPTER_ID, aligned)
-
-				yield {
-					raw,
-					components: aligned,
-					country: "FR",
-					locale: "fr-FR",
-					source: BAN_ADAPTER_ID,
-					source_id: sourceID,
-					corpus_version: "",
-					license: "Licence Ouverte 2.0",
-				}
-
-				emitted++
-			}
-		},
-	}
-}
-
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const banAdapter = createBanAdapter()

package/lib/adapters/ban/street-decompose.ts

@@ -1,66 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Decompose a French street name into Stage 3 components. French convention puts the street type as
- *   a leading word: "Rue de Rivoli", "Avenue des Champs-Élysées", "Bd Voltaire".
- *
- *   The street type becomes street_prefix in our schema. The remaining tokens form the street name.
- *
- *   Examples: "Rue de Rivoli" → { prefix: "Rue", street: "de Rivoli" } "Avenue des Champs-Élysées" →
- *   { prefix: "Avenue", street: "des Champs-Élysées" } "Boulevard Voltaire" → { prefix:
- *   "Boulevard", street: "Voltaire" }
- *
- *   Sources street types from `core/data/libpostal/dictionaries/fr/street_types.txt`.
- */
-
-import { loadLibpostalDictionary } from "#adapters/utils"
-
-/**
- * Tokens a BAN street needs before a type/article/name decomposition is attempted.
- */
-const MIN_TOKENS_FOR_DECOMPOSE = 3
-
-const STREET_TYPES_FR = await loadLibpostalDictionary("fr", "street_types.txt")
-
-export interface DecomposedFrStreet {
-	prefix: string | null
-	street: string
-}
-
-/**
- * Decompose a French street name into prefix (leading type word) and street name.
- *
- * If the first 1-2 tokens match a known street type (allowing for multi-word like "ancien chemin"), they become the
- * prefix. Returns `{ prefix: null, street: original }` if no match.
- */
-export function decomposeFrStreet(fullname: string): DecomposedFrStreet {
-	const trimmed = fullname.trim()
-
-	if (!trimmed) return { prefix: null, street: "" }
-
-	const tokens = trimmed.split(/\s+/)
-
-	if (tokens.length < 2) return { prefix: null, street: trimmed }
-
-	const norm = (s: string) => s.toLowerCase().replace(/[.,;]$/, "")
-
-	// Try 2-word prefix first (e.g. "ancien chemin")
-	if (tokens.length >= MIN_TOKENS_FOR_DECOMPOSE) {
-		const twoWord = norm(tokens[0]!) + " " + norm(tokens[1]!)
-
-		if (STREET_TYPES_FR.has(twoWord)) {
-			return { prefix: tokens.slice(0, 2).join(" "), street: tokens.slice(2).join(" ") }
-		}
-	}
-
-	// Then try 1-word prefix
-	const first = norm(tokens[0]!)
-
-	if (STREET_TYPES_FR.has(first)) {
-		return { prefix: tokens[0]!, street: tokens.slice(1).join(" ") }
-	}
-
-	return { prefix: null, street: trimmed }
-}

package/lib/adapters/fcc-bdc/adapter.ts

@@ -1,174 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `fcc-bdc`: FCC Broadband Data Collection (BDC) — Fabric-derived location consumer.
- *
- *   The first member of Phase 1.6's "adversarial sources" class. BDC ships the public-domain US
- *   broadband-serviceable-location (BSL) fabric: ~120M addresses keyed by stable `location_id`,
- *   carrying `address_primary` + `city` + `state` + `zip` + `zip_suffix`. Compared to the clean
- *   gazetteer rows from WOF / TIGER / BAN, BDC carries the chaos of address data that has passed
- *   through several layers of human entry + automated geocoding + revision: abbreviation drift,
- *   inconsistent unit designators, "RR" / "HC" / "PSC" rural-route shapes, embedded apartment /
- *   suite numbers that did not survive the address parser cleanly. This is the highest-signal,
- *   hardest-to-normalize address corpus in the federal public-domain catalog.
- *
- *   Following the `tiger` / `wof-admin` pattern, this adapter consumes a SQLite database the operator
- *   pre-builds via the isp-nexus BDC ETL (`/srv/isp-nexus/sync/fcc/bdc/`) or any equivalent
- *   host-side pipeline. The mailwoman side does not download or parse the raw CSV/ZIP distribution
- *   directly — that keeps the adapter narrow and the BDC ingest pluggable.
- *
- *   The SQLite schema is documented in README.md and modeled after `NTIARecord`
- *   (`isp-nexus/fcc/bdc/data-collection.ts`): one row per `location_id`. The adapter splits
- *   `address_primary` into `house_number` (leading numeric prefix, if any) + `street` (everything
- *   after), and combines `zip` + `zip_suffix` into the canonical USPS `postcode` slot.
- *
- *   One CanonicalRow per fabric record. Unlike `tiger` (multiple postcode variants per segment) or
- *   `wof-admin` (multiple hierarchy variants per place), BDC records already represent fully
- *   specified addresses; no fan-out is warranted. Adversarial composition (Phase 1.6 §2.1) is the
- *   mechanism for deriving multiple training rows per BDC record.
- *
- *   License: stamped `"Public Domain"` per the BDC fabric's US federal-government distribution terms.
- *   The CostQuest Fabric source data has its own license; consumers who substitute that path should
- *   re-stamp accordingly.
- */
-
-import type { BDCDatabase } from "@mailwoman/bdc/schema"
-import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
-import { DatabaseClient } from "@mailwoman/sqlite/client"
-
-import { splitStreetLine } from "#adapters/utils"
-import { lookupStateAbbreviation } from "#codex/us-fips-state"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const FCC_BDC_ADAPTER_ID = "fcc-bdc"
-/**
- * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
- * than having to look them up.
- */
-export const FCC_BDC_DEFAULT_LICENSE = "Public Domain"
-
-/**
- * SQLite row shape — one row per BSL `location_id`. Columns mirror NTIARecord.
- */
-interface BdcLocationRow {
-	location_id: number
-	address_primary: string
-	city: string
-	state: string
-	zip: string
-	zip_suffix: string | null
-}
-
-/**
- * Combine `zip` + optional `zip_suffix` into the canonical USPS postcode surface form.
- *
- * NTIARecord doc is ambiguous about whether `zip_suffix` is the 4-digit extension alone or the full ZIP+4 string. This
- * handles both:
- *
- * - Bare 4-digit extension (`zip="94103"`, `zip_suffix="1234"`) → `"94103-1234"`
- * - Already-joined form (`zip_suffix="94103-1234"`) → returned as-is
- * - No suffix → bare `zip`
- *
- * Empty / whitespace-only suffix is treated as missing.
- */
-export function buildPostcode(zip: string, suffix: string | null): string {
-	const z = zip.trim()
-
-	if (!z) return ""
-	const s = suffix?.trim() ?? ""
-
-	if (!s) return z
-
-	if (s.includes("-")) return s
-
-	return `${z}-${s}`
-}
-
-/**
- * Build a BDC adapter. Pure factory so multiple instances can be created in tests.
- */
-export function createFccBdcAdapter(): CorpusAdapter {
-	return {
-		id: FCC_BDC_ADAPTER_ID,
-		defaultLicense: FCC_BDC_DEFAULT_LICENSE,
-		description:
-			"FCC Broadband Data Collection — Fabric-derived BSL addresses (public-domain); SQLite DB the operator builds via the isp-nexus BDC ETL.",
-
-		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			if (opts.country && opts.country !== "US") {
-				throw new Error(`fcc-bdc adapter: only US supported, got country=${opts.country}`)
-			}
-
-			const db = new DatabaseClient<BDCDatabase>(opts.inputPath, { readOnly: true })
-			let emitted = 0
-
-			try {
-				const stmt = db.prepare(
-					`SELECT location_id, address_primary, city, state, zip, zip_suffix
-					 FROM bdc_locations
-					 ORDER BY location_id`
-				)
-
-				for (const row of stmt.iterate() as IterableIterator<BdcLocationRow>) {
-					if (opts.signal?.aborted) return
-
-					if (opts.limit !== undefined && emitted >= opts.limit) return
-
-					const split = splitStreetLine(row.address_primary ?? "")
-
-					if (!split) continue
-					const state = lookupStateAbbreviation(row.state)
-
-					if (!state) continue
-					const locality = row.city?.trim()
-
-					if (!locality) continue
-					const postcode = buildPostcode(row.zip ?? "", row.zip_suffix ?? null)
-
-					if (!postcode) continue
-
-					const components: CanonicalRow["components"] = {
-						...(split.house_number ? { house_number: split.house_number } : {}),
-						street: split.street,
-						locality,
-						region: state.abbreviation,
-						postcode,
-					}
-
-					const raw = formatAddress(components, "US", { separator: ", " })
-
-					if (!raw) continue
-					const aligned = reconcileComponents(components, raw)
-
-					if (!Object.keys(aligned).length) continue
-
-					yield {
-						raw,
-						components: aligned,
-						country: "US",
-						locale: "en-US",
-						source: FCC_BDC_ADAPTER_ID,
-						source_id: `${FCC_BDC_ADAPTER_ID}-${row.location_id}`,
-						corpus_version: "",
-						license: FCC_BDC_DEFAULT_LICENSE,
-					}
-
-					emitted++
-				}
-			} finally {
-				await db.destroy()
-			}
-		},
-	}
-}
-
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const fccBdcAdapter = createFccBdcAdapter()

package/lib/adapters/geonames-postal/adapter.ts

@@ -1,122 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `geonames-postal`: GeoNames postal-code dump consumer (https://www.geonames.org/, CC-BY-4.0).
- *
- *   The GeoNames postal export (`https://download.geonames.org/export/zip/<CC>.zip`) is a clean,
- *   per-country `postcode → place → admin1` table with the place + region NAMES inline (no aux-file
- *   join needed). It broadens the corpus's postcode→locality→region coverage to ~80 countries, well
- *   beyond `wof-postalcode`/the coordinate-first table — forward coverage for the multi-locale
- *   goal.
- *
- *   Input: a per-country postal dump (`<CC>.txt`, 12 tab-separated columns, no header): country,
- *   postcode, place, admin1_name, admin1_code, admin2__, admin3__, lat, lon, accuracy.
- *
- *   Output: per row, postcode-FIRST (international) variants — the common order for the non-US
- *   locales this fills (US postcodes are already covered by TIGER/WOF, which use postcode-LAST):
- *
- *   1. `{ postcode, locality }` → "AD100 Canillo"
- *   2. `{ postcode, locality, region }` → "AD100 Canillo, Canillo" Prefer configuring this adapter for
- *        non-US countries; for US, the postcode-last sources are the right order. License:
- *        `"CC-BY-4.0"` per row (attribute "GeoNames").
- */
-
-import { reconcileComponents } from "@mailwoman/formatter"
-import { TSVSpliterator } from "spliterator"
-
-import { stableSourceID } from "#adapters/utils"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const GEONAMES_POSTAL_ADAPTER_ID = "geonames-postal"
-/**
- * License carried by this source (CC-BY-4.0), attached to each row so downstream consumers inherit the terms rather
- * than having to look them up.
- */
-export const GEONAMES_POSTAL_DEFAULT_LICENSE = "CC-BY-4.0"
-
-/**
- * GeoNames postal-dump columns (0-based): country, postcode, place, admin1_name, admin1_code, admin2_name, ….
- *
- * Shared with `tools/postcode-triples.ts`, which additionally reads `admin2Name` — the CITY for the IN/MX/PT-shaped
- * exports whose `place` column is a street or colonia.
- */
-export const GEONAMES_POSTAL_COLUMNS = { country: 0, postcode: 1, place: 2, admin1Name: 3, admin2Name: 5 } as const
-
-export function createGeonamesPostalAdapter(): CorpusAdapter {
-	return {
-		id: GEONAMES_POSTAL_ADAPTER_ID,
-		defaultLicense: GEONAMES_POSTAL_DEFAULT_LICENSE,
-		description:
-			"GeoNames postcodes (CC-BY-4.0) — multi-locale postcode→locality→region, names inline; international postcode-first order.",
-
-		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			// `header: false` — the GeoNames postal dump is headerless, and the spliterator would
-			// otherwise consume row 1 as column names and lose its first postcode.
-			const rows = TSVSpliterator.fromAsync(opts.inputPath, { header: false })
-
-			let emitted = 0
-
-			for await (const rec of rows as AsyncIterable<string[]>) {
-				if (opts.signal?.aborted) break
-
-				if (opts.limit !== undefined && emitted >= opts.limit) break
-
-				const cc = (rec[GEONAMES_POSTAL_COLUMNS.country] ?? "").trim()
-
-				if (!cc) continue
-
-				if (opts.country && cc !== opts.country) continue
-
-				const postcode = (rec[GEONAMES_POSTAL_COLUMNS.postcode] ?? "").trim()
-				const locality = (rec[GEONAMES_POSTAL_COLUMNS.place] ?? "").trim()
-
-				if (!postcode || !locality) continue
-				const region = (rec[GEONAMES_POSTAL_COLUMNS.admin1Name] ?? "").trim()
-
-				// Postcode-first (international) variants. Skip the region variant when admin1 just
-				// repeats the place (common for city-states / micro-admin) to avoid "X X" noise.
-				const variants: Array<{ slot: string; comp: CanonicalRow["components"]; raw: string }> = [
-					{ slot: "pl", comp: { postcode, locality }, raw: `${postcode} ${locality}` },
-				]
-
-				if (region && region.toLowerCase() !== locality.toLowerCase()) {
-					variants.push({
-						slot: "plr",
-						comp: { postcode, locality, region },
-						raw: `${postcode} ${locality}, ${region}`,
-					})
-				}
-
-				for (const v of variants) {
-					if (opts.limit !== undefined && emitted >= opts.limit) break
-					const aligned = reconcileComponents(v.comp, v.raw)
-
-					if (Object.keys(aligned).length < 2) continue
-
-					yield {
-						raw: v.raw,
-						components: aligned,
-						country: cc,
-						source: GEONAMES_POSTAL_ADAPTER_ID,
-						source_id: `${stableSourceID(GEONAMES_POSTAL_ADAPTER_ID, aligned)}-${v.slot}`,
-						corpus_version: "",
-						license: GEONAMES_POSTAL_DEFAULT_LICENSE,
-					}
-
-					emitted++
-				}
-			}
-		},
-	}
-}
-
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const geonamesPostalAdapter = createGeonamesPostalAdapter()