@mailwoman/corpus 9.1.0 → 9.3.0

package/src/adapters/openaddresses/adapter.ts

@@ -1,250 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `openaddresses`: Line-delimited GeoJSON adapter for openaddresses.io exports.
- *
- *   OpenAddresses publishes country-partitioned address dumps as either CSV or line-delimited GeoJSON
- *   (one `Feature` per line, also called ND-GeoJSON / GeoJSONL). This adapter consumes the
- *   line-delimited GeoJSON shape — it streams cleanly without holding the file in memory, which
- *   matters for the multi-gigabyte national dumps (e.g. `us-northeast.geojsonl`, ~20M rows).
- *
- *   The collection aggregates **hundreds** of underlying sources with **per-source licenses** (city
- *   open-data portals, county GIS departments, state DOTs). The adapter therefore prefers the
- *   per-row `LICENSE` property when present and falls back to the configured `defaultLicense`. The
- *   propagated license travels with each `CanonicalRow` so downstream code can stratify, exclude,
- *   or re-attribute by license at training time.
- *
- *   Country must be explicit (`opts.country` REQUIRED): OpenAddresses files are organized by country
- *   but the row-level data doesn't include a country code, so the adapter refuses to run without
- *   one. This matches how a `mailwoman corpus build` invocation pins each file to a country via the
- *   inputs JSON.
- *
- *   Properties consumed (per the canonical OpenAddresses schema; both UPPERCASE and lowercase
- *   variants are accepted because legacy dumps used UPPERCASE):
- *
- *   | Property | ComponentTag | | ------------- |
- *   -------------------------------------------------------------- | | `number` | `house_number` |
- *   | `street` | `street` | | `unit` | `unit` (if non-empty) | | `city` | `locality` | | `region` |
- *   `region` (state code for US, province for CA, etc.) | | `postcode` | `postcode` | | `LICENSE` |
- *   per-row `license` override | | `hash` / `id` | `source_id` (prefer `hash`; fall back to `id`;
- *   then synthesize)|
- *
- *   `district` is intentionally NOT mapped — for US data it carries borough or county and would
- *   inflate alignment quarantine because postal addresses don't include it. Phase 6+ may revisit
- *   for non-US locales where district names DO appear on the envelope.
- */
-
-import { tryParsingJSON } from "@mailwoman/core/objects"
-import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
-import { SHARE_ALIKE_PATTERN } from "@mailwoman/corpus/utils"
-import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
-import { TextSpliterator } from "spliterator"
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const OPENADDRESSES_ADAPTER_ID = "openaddresses"
-/**
- * License carried by this source (CC-BY-4.0), attached to each row so downstream consumers inherit the terms rather
- * than having to look them up.
- */
-export const OPENADDRESSES_DEFAULT_LICENSE = "CC-BY-4.0"
-
-/**
- * Subset of OpenAddresses Feature properties the adapter inspects. The runtime accepts UPPERCASE or lowercase keys;
- * this interface documents the canonical lowercase form after normalization.
- */
-interface OaProperties {
-	hash?: string
-	id?: string
-	number?: string
-	street?: string
-	unit?: string
-	city?: string
-	district?: string
-	region?: string
-	postcode?: string
-	license?: string
-}
-
-/**
- * Return a lowercase-keyed view of a Feature's properties so case variants both work.
- */
-function normalizeProperties(raw: unknown): OaProperties {
-	if (!raw || typeof raw !== "object") return {}
-	const out: Record<string, string> = {}
-
-	for (const [k, v] of Object.entries(raw as Record<string, unknown>)) {
-		if (typeof v === "string") {
-			out[k.toLowerCase()] = v
-		} else if (typeof v === "number") {
-			out[k.toLowerCase()] = String(v)
-		}
-	}
-
-	return out as OaProperties
-}
-
-/**
- * Parse a single ND-GeoJSON line; return null for blanks, comments, or non-Feature shapes.
- */
-function parseFeatureLine(line: string): OaProperties | null {
-	const trimmed = line.trim()
-
-	if (!trimmed || trimmed.startsWith("#")) return null
-	const parsed = tryParsingJSON(trimmed)
-
-	if (!parsed || typeof parsed !== "object") return null
-	const obj = parsed as { type?: string; properties?: unknown }
-
-	if (obj.type !== "Feature") return null
-
-	return normalizeProperties(obj.properties)
-}
-
-export interface OpenaddressesAdapterOptions {
-	/**
-	 * Per-row license used when a Feature lacks an explicit `LICENSE` property. Defaults to `CC-BY-4.0` — the most common
-	 * license across the OpenAddresses collection. Override per dump via the runner's adapter-options passthrough.
-	 */
-	defaultLicense?: string
-
-	/**
-	 * Per-adapter share-alike drop. Default **true** (include) as of 2026-06-19: exclusion is a deliberate BUILD-level
-	 * act (`buildCorpus({ excludeLicenses })` / `--exclude-share-alike`), NOT a silent adapter default (#26 — "purposely
-	 * exclude, don't opt in to include"). Set false only for an explicit adapter-scoped drop; the build-level
-	 * `--exclude-share-alike` is the normal path.
-	 */
-	allowShareAlike?: boolean
-}
-
-/**
- * Build an OpenAddresses adapter. The optional `defaultLicense` lets callers stamp a non-default fallback for dumps
- * known to carry a single license throughout (e.g. a PDDL-only state slice).
- */
-export function createOpenaddressesAdapter(opts: OpenaddressesAdapterOptions = {}): CorpusAdapter {
-	const defaultLicense = opts.defaultLicense ?? OPENADDRESSES_DEFAULT_LICENSE
-	const allowShareAlike = opts.allowShareAlike ?? true
-
-	return {
-		id: OPENADDRESSES_ADAPTER_ID,
-		defaultLicense,
-		description: "OpenAddresses (global): line-delimited GeoJSON dumps with per-row licenses.",
-
-		async *rows(adapterOpts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			if (!adapterOpts.country) {
-				throw new Error(
-					"openaddresses adapter: --country is required (OpenAddresses files are country-partitioned but rows lack a country field)"
-				)
-			}
-
-			const country = adapterOpts.country
-
-			// TextSpliterator streams string lines (parseFeatureLine keeps tolerating blank/`#`/
-			// malformed lines by returning null); passing the path string lets the lib own + dispose
-			// the file handle, including on an early `break`.
-			const lines = TextSpliterator.fromAsync(adapterOpts.inputPath)
-
-			let emitted = 0
-			let shareAlikeBlocked = 0
-
-			try {
-				for await (const line of lines) {
-					if (adapterOpts.signal?.aborted) break
-
-					if (adapterOpts.limit !== undefined && emitted >= adapterOpts.limit) break
-
-					const props = parseFeatureLine(line)
-
-					if (!props) continue
-
-					const houseNumber = props.number?.trim() ?? ""
-					const street = props.street?.trim() ?? ""
-					const unit = props.unit?.trim() ?? ""
-					const city = props.city?.trim() ?? ""
-					const region = props.region?.trim() ?? ""
-					const postcode = props.postcode?.trim() ?? ""
-
-					// A row is only useful if it has, at minimum, a street + (postcode OR locality).
-					// Pure point-only rows would land in quarantine anyway.
-					if (!street) continue
-
-					if (!city && !postcode) continue
-
-					const license = (props.license?.trim() || defaultLicense).trim()
-
-					if (!allowShareAlike && SHARE_ALIKE_PATTERN.test(license)) {
-						shareAlikeBlocked++
-
-						continue
-					}
-
-					const components: CanonicalRow["components"] = {}
-
-					if (houseNumber) {
-						components.house_number = houseNumber
-					}
-
-					if (street) {
-						components.street = street
-					}
-
-					if (unit) {
-						components.unit = unit
-					}
-
-					if (city) {
-						components.locality = city
-					}
-
-					if (region) {
-						components.region = region
-					}
-
-					if (postcode) {
-						components.postcode = postcode
-					}
-
-					const raw = formatAddress(components, country, { separator: ", " })
-
-					if (!raw) continue
-
-					const aligned = reconcileComponents(components, raw)
-
-					if (!Object.keys(aligned).length) continue
-
-					const sourceIDSeed = props.hash?.trim() || props.id?.trim()
-
-					const sourceID = sourceIDSeed
-						? `${OPENADDRESSES_ADAPTER_ID}-${sourceIDSeed}`
-						: stableSourceID(OPENADDRESSES_ADAPTER_ID, aligned)
-
-					yield {
-						raw,
-						components: aligned,
-						country,
-						source: OPENADDRESSES_ADAPTER_ID,
-						source_id: sourceID,
-						corpus_version: "",
-						license,
-					}
-
-					emitted++
-				}
-			} finally {
-				if (shareAlikeBlocked > 0) {
-					process.stderr.write(`  openaddresses: ${shareAlikeBlocked} share-alike rows dropped, ${emitted} kept\n`)
-				}
-			}
-		},
-	}
-}
-
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const openaddressesAdapter = createOpenaddressesAdapter()

package/src/adapters/overture/adapter.ts

@@ -1,161 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `overture`: Overture Maps Addresses adapter (epic #470 — the gated corpus adapter, realized
- *   2026-06-20). Overture's global Addresses theme is the single-schema, well-normalized address
- *   dataset that fixes OpenAddresses' per-country patchiness (OA dropped Spain; OA-DE omits the
- *   Bundesland) — it even re-hosts the OA Spain data the standalone OA bucket no longer serves.
- *
- *   This adapter consumes a per-country LINE-DELIMITED JSON dump of the corpus-relevant fields (`{
- *   street, number, unit, postcode, locality }`), produced by `scripts/ingest-overture-addresses.ts
- *   --corpus-jsonl` (which does the DuckDB / S3 heavy lifting and flattens `address_levels` → the
- *   municipality locality). The split keeps `@mailwoman/corpus` — a RUNTIME dep of the `mailwoman`
- *   CLI — free of the heavy native `@duckdb/node-api`; the adapter just streams JSONL line-by-line,
- *   exactly like `openaddresses`.
- *
- *   The `street` surface carries the locale's street keyword verbatim (`"CALLE JULAN"`, `"VIA
- *   ROMA"`). We map it to `street` whole and let the downstream affix-relabel split `street_prefix`
- *   — the same path every other source rides. This shard exists because the model was
- *   en-us/fr-trained and never saw non-en/fr street formats (the 2026-06-19 EU parse-blocker
- *   measured loc-correct ES 21% / IT 59% / NL 64% vs FR/US ~98%).
- *
- *   `--country` is REQUIRED (the JSONL is per-country and the rows omit a country field), matching
- *   `openaddresses`. License is Overture's CDLA-Permissive-2.0 (attribution; not share-alike).
- *
- *   | Field | ComponentTag | | --------- | ---------------------------------------------- | |
- *   `street` | `street` (keyword incl.; affix-relabel splits prefix) | | `number` | `house_number`
- *   (skipped when "S-N"/"S/N" = sin número) | | `unit` | `unit` (if non-empty) | | `postcode`|
- *   `postcode` | | `locality`| `locality` (Overture address_levels municipality, or postal_city) |
- */
-
-import { tryParsingJSON } from "@mailwoman/core/objects"
-import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
-import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
-import { TextSpliterator } from "spliterator"
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const OVERTURE_ADAPTER_ID = "overture"
-/**
- * License carried by this source (CDLA-Permissive-2.0), attached to each row so downstream consumers inherit the terms
- * rather than having to look them up.
- */
-export const OVERTURE_DEFAULT_LICENSE = "CDLA-Permissive-2.0"
-
-/**
- * The flattened per-row shape emitted by `ingest-overture-addresses.ts --corpus-jsonl`.
- */
-interface OvertureCorpusRow {
-	street?: string
-	number?: string
-	unit?: string
-	postcode?: string
-	locality?: string
-}
-
-function parseLine(line: string): OvertureCorpusRow | null {
-	const t = line.trim()
-
-	if (!t || t.startsWith("#")) return null
-
-	const o = tryParsingJSON(t)
-
-	return o && typeof o === "object" ? (o as OvertureCorpusRow) : null
-}
-
-export function createOvertureAdapter(): CorpusAdapter {
-	return {
-		id: OVERTURE_ADAPTER_ID,
-		defaultLicense: OVERTURE_DEFAULT_LICENSE,
-		description: "Overture Maps Addresses (global): per-country JSONL of street/number/postcode/locality.",
-
-		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			if (!opts.country) {
-				throw new Error(
-					"overture adapter: --country is required (the Overture JSONL is per-country and rows omit a country field)"
-				)
-			}
-
-			const country = opts.country
-
-			// TextSpliterator streams string lines (parseLine keeps tolerating blank/`#`/malformed
-			// lines by returning null); the path string lets the lib own + dispose the file handle,
-			// including on an early `break`.
-			const lines = TextSpliterator.fromAsync(opts.inputPath)
-
-			let emitted = 0
-
-			for await (const line of lines) {
-				if (opts.signal?.aborted) break
-
-				if (opts.limit !== undefined && emitted >= opts.limit) break
-
-				const r = parseLine(line)
-
-				if (!r) continue
-
-				const street = r.street?.trim() ?? ""
-				const number = r.number?.trim() ?? ""
-				const unit = r.unit?.trim() ?? ""
-				const postcode = r.postcode?.trim() ?? ""
-				const locality = r.locality?.trim() ?? ""
-
-				// Only useful with a street + (postcode OR locality); point-only rows quarantine anyway.
-				if (!street) continue
-
-				if (!postcode && !locality) continue
-
-				const components: CanonicalRow["components"] = {}
-
-				// Overture "S-N" / "S/N" = sin número; only keep a real numeric house number.
-				if (/^\d/.test(number)) {
-					components.house_number = number
-				}
-
-				components.street = street
-
-				if (unit) {
-					components.unit = unit
-				}
-
-				if (postcode) {
-					components.postcode = postcode
-				}
-
-				if (locality) {
-					components.locality = locality
-				}
-
-				const raw = formatAddress(components, country, { separator: ", " })
-
-				if (!raw) continue
-
-				const aligned = reconcileComponents(components, raw)
-
-				if (!Object.keys(aligned).length) continue
-
-				yield {
-					raw,
-					components: aligned,
-					country,
-					source: OVERTURE_ADAPTER_ID,
-					source_id: stableSourceID(OVERTURE_ADAPTER_ID, aligned),
-					corpus_version: "",
-					license: OVERTURE_DEFAULT_LICENSE,
-				}
-
-				emitted++
-			}
-		},
-	}
-}
-
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const overtureAdapter = createOvertureAdapter()

package/src/adapters/state-hi-schools/adapter.ts

@@ -1,162 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `state-hi-schools`: Hawaii DOE public + charter schools CSV consumer.
- *
- *   The Hawaii State Department of Education publishes a directory of all HIDOE schools and public
- *   charter schools (PCS) as an XLSX workbook (`SchoolList.xlsx`) with two sheets: `HIDOE` (~258
- *   rows) and `PCS` (~38 rows). Total ~296 rows statewide. Each row carries a school name,
- *   single-line street address, city, ZIP, a numeric `code`, and HI-specific administrative columns
- *   (complex, complex_area, district, island, charter).
- *
- *   The adapter consumes a flat CSV the operator pre-builds via `fetch-state-hi-schools.ts`, which
- *   concatenates both sheets under one shared header. Column names match the workbook header
- *   verbatim (lower-snake-case: `code`, `name`, `address`, `city`, `zip`, ...).
- *
- *   Address parsing notes: Hawaii's residential numbering is hyphenated on Oahu (`47-470 Hui Aeko
- *   Place`), Kauai (`2-4035 Kaumualii Hwy`), and elsewhere. The shared HOUSE_NUMBER_PREFIX regex
- *   covers this via its optional `(?:-\d+)?` group.
- *
- *   The `island` and `district` columns are HIDOE administrative labels (Honolulu, Central, Leeward,
- *   Windward, Hilo, Hawaii, Maui, Kauai) — they are NOT US counties and intentionally are not
- *   surfaced as `subregion`.
- *
- *   Output: one row per school with `venue` (school name), `(house_number?, street, locality,
- *   region=HI, postcode)`, and a stable `source_id` derived from the school `code`.
- *
- *   License: stamped `"Public Domain"` per Hawaii state government open-data terms.
- */
-
-import { splitStreetLine, stableSourceID } from "@mailwoman/corpus/adapters/utils"
-import { lookupStateAbbreviation } from "@mailwoman/corpus/codex/us-fips-state"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
-import { reconcileComponents } from "@mailwoman/formatter"
-import { CSVSpliterator } from "spliterator"
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const STATE_HI_SCHOOLS_ADAPTER_ID = "state-hi-schools"
-/**
- * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
- * than having to look them up.
- */
-export const STATE_HI_SCHOOLS_DEFAULT_LICENSE = "Public Domain"
-
-const HI_STATE_ABBR = "HI"
-
-interface HiSchoolRow {
-	code: string
-	name: string
-	address: string
-	city: string
-	zip: string
-}
-
-function normalizeZip(raw: string): string {
-	const trimmed = raw.trim()
-
-	if (!trimmed) return ""
-
-	// XLSX → CSV conversion may emit numeric ZIPs without leading zeros. HI ZIPs all begin
-	// with 96, so a 4-digit value indicates a leading-zero stripped during numeric coercion
-	// (defensive — has not been observed in the published file as of 2026-05).
-	if (/^\d{4}$/.test(trimmed)) return `0${trimmed}`
-
-	return trimmed
-}
-
-export function createStateHiSchoolsAdapter(): CorpusAdapter {
-	return {
-		id: STATE_HI_SCHOOLS_ADAPTER_ID,
-		defaultLicense: STATE_HI_SCHOOLS_DEFAULT_LICENSE,
-		description: "Hawaii DOE School Directory — ~300 K-12 public + charter schools with venue+address (public-domain).",
-
-		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			if (opts.country && opts.country !== "US") {
-				throw new Error(`state-hi-schools adapter: only US supported, got country=${opts.country}`)
-			}
-
-			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
-				mode: "object",
-				normalizeKeys: false,
-				enableQuoteHandling: true,
-			})
-
-			const state = lookupStateAbbreviation(HI_STATE_ABBR)
-
-			if (!state) {
-				throw new Error(`state-hi-schools adapter: HI not found in state codex (corpus bug)`)
-			}
-
-			let emitted = 0
-
-			for await (const record of rows as AsyncIterable<HiSchoolRow>) {
-				if (opts.signal?.aborted) break
-
-				if (opts.limit !== undefined && emitted >= opts.limit) break
-
-				const name = (record.name ?? "").trim()
-				const address = (record.address ?? "").trim()
-				const city = (record.city ?? "").trim()
-				const zip = normalizeZip(record.zip ?? "")
-
-				if (!name || !address || !city || !zip) continue
-
-				const split = splitStreetLine(address)
-
-				if (!split) continue
-
-				const components: CanonicalRow["components"] = {
-					venue: name,
-					...(split.house_number ? { house_number: split.house_number } : {}),
-					street: split.street,
-					locality: city,
-					region: state.abbreviation,
-					postcode: zip,
-				}
-
-				const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
-
-				const raw = [
-					name,
-					streetPart,
-					[city, [HI_STATE_ABBR, zip].filter(Boolean).join(" ")].filter(Boolean).join(", "),
-				]
-					.filter(Boolean)
-					.join(", ")
-
-				const aligned = reconcileComponents(components, raw)
-
-				if (Object.keys(aligned).length <= 2) continue
-
-				const code = (record.code ?? "").toString().trim()
-
-				const sourceID = code
-					? `${STATE_HI_SCHOOLS_ADAPTER_ID}-${code}`
-					: stableSourceID(STATE_HI_SCHOOLS_ADAPTER_ID, aligned)
-
-				yield {
-					raw,
-					components: aligned,
-					country: "US",
-					locale: "en-US",
-					source: STATE_HI_SCHOOLS_ADAPTER_ID,
-					source_id: sourceID,
-					corpus_version: "",
-					license: STATE_HI_SCHOOLS_DEFAULT_LICENSE,
-				}
-
-				emitted++
-			}
-		},
-	}
-}
-
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const stateHiSchoolsAdapter = createStateHiSchoolsAdapter()