@mailwoman/corpus 7.2.0 → 7.3.0

package/src/adapters/overture/adapter.ts

@@ -0,0 +1,149 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `overture`: Overture Maps Addresses adapter (epic #470 — the gated corpus adapter, realized
+ *   2026-06-20). Overture's global Addresses theme is the single-schema, well-normalized address
+ *   dataset that fixes OpenAddresses' per-country patchiness (OA dropped Spain; OA-DE omits the
+ *   Bundesland) — it even re-hosts the OA Spain data the standalone OA bucket no longer serves.
+ *
+ *   This adapter consumes a per-country LINE-DELIMITED JSON dump of the corpus-relevant fields (`{
+ *   street, number, unit, postcode, locality }`), produced by `scripts/ingest-overture-addresses.ts
+ *   --corpus-jsonl` (which does the DuckDB / S3 heavy lifting and flattens `address_levels` → the
+ *   municipality locality). The split keeps `@mailwoman/corpus` — a RUNTIME dep of the `mailwoman`
+ *   CLI — free of the heavy native `@duckdb/node-api`; the adapter just streams JSONL line-by-line,
+ *   exactly like `openaddresses`.
+ *
+ *   The `street` surface carries the locale's street keyword verbatim (`"CALLE JULAN"`, `"VIA
+ *   ROMA"`). We map it to `street` whole and let the downstream affix-relabel split `street_prefix`
+ *   — the same path every other source rides. This shard exists because the model was
+ *   en-us/fr-trained and never saw non-en/fr street formats (the 2026-06-19 EU parse-blocker
+ *   measured loc-correct ES 21% / IT 59% / NL 64% vs FR/US ~98%).
+ *
+ *   `--country` is REQUIRED (the JSONL is per-country and the rows omit a country field), matching
+ *   `openaddresses`. License is Overture's CDLA-Permissive-2.0 (attribution; not share-alike).
+ *
+ *   | Field | ComponentTag | | --------- | ---------------------------------------------- | |
+ *   `street` | `street` (keyword incl.; affix-relabel splits prefix) | | `number` | `house_number`
+ *   (skipped when "S-N"/"S/N" = sin número) | | `unit` | `unit` (if non-empty) | | `postcode`|
+ *   `postcode` | | `locality`| `locality` (Overture address_levels municipality, or postal_city) |
+ */
+
+import { TextSpliterator } from "spliterator"
+
+import { stableSourceID } from "../../adapter.ts"
+import { formatAddress, reconcileComponents } from "../../format.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+
+export const OVERTURE_ADAPTER_ID = "overture"
+export const OVERTURE_DEFAULT_LICENSE = "CDLA-Permissive-2.0"
+
+/** The flattened per-row shape emitted by `ingest-overture-addresses.ts --corpus-jsonl`. */
+interface OvertureCorpusRow {
+	street?: string
+	number?: string
+	unit?: string
+	postcode?: string
+	locality?: string
+}
+
+function parseLine(line: string): OvertureCorpusRow | null {
+	const t = line.trim()
+
+	if (!t || t.startsWith("#")) return null
+
+	try {
+		const o = JSON.parse(t)
+
+		return o && typeof o === "object" ? (o as OvertureCorpusRow) : null
+	} catch {
+		return null
+	}
+}
+
+export function createOvertureAdapter(): CorpusAdapter {
+	return {
+		id: OVERTURE_ADAPTER_ID,
+		defaultLicense: OVERTURE_DEFAULT_LICENSE,
+		description: "Overture Maps Addresses (global): per-country JSONL of street/number/postcode/locality.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (!opts.country) {
+				throw new Error(
+					"overture adapter: --country is required (the Overture JSONL is per-country and rows omit a country field)"
+				)
+			}
+			const country = opts.country
+
+			// TextSpliterator streams string lines (parseLine keeps tolerating blank/`#`/malformed
+			// lines by returning null); the path string lets the lib own + dispose the file handle,
+			// including on an early `break`.
+			const lines = TextSpliterator.fromAsync(opts.inputPath)
+
+			let emitted = 0
+
+			for await (const line of lines) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const r = parseLine(line)
+
+				if (!r) continue
+
+				const street = r.street?.trim() ?? ""
+				const number = r.number?.trim() ?? ""
+				const unit = r.unit?.trim() ?? ""
+				const postcode = r.postcode?.trim() ?? ""
+				const locality = r.locality?.trim() ?? ""
+
+				// Only useful with a street + (postcode OR locality); point-only rows quarantine anyway.
+				if (!street) continue
+
+				if (!postcode && !locality) continue
+
+				const components: CanonicalRow["components"] = {}
+
+				// Overture "S-N" / "S/N" = sin número; only keep a real numeric house number.
+				if (/^\d/.test(number)) {
+					components.house_number = number
+				}
+				components.street = street
+
+				if (unit) {
+					components.unit = unit
+				}
+
+				if (postcode) {
+					components.postcode = postcode
+				}
+
+				if (locality) {
+					components.locality = locality
+				}
+
+				const raw = formatAddress(components, country, { separator: ", " })
+
+				if (!raw) continue
+
+				const aligned = reconcileComponents(components, raw)
+
+				if (Object.keys(aligned).length === 0) continue
+
+				yield {
+					raw,
+					components: aligned,
+					country,
+					source: OVERTURE_ADAPTER_ID,
+					source_id: stableSourceID(OVERTURE_ADAPTER_ID, aligned),
+					corpus_version: "",
+					license: OVERTURE_DEFAULT_LICENSE,
+				}
+				emitted++
+			}
+		},
+	}
+}
+
+export const overtureAdapter = createOvertureAdapter()

package/src/adapters/state-hi-schools/adapter.ts

@@ -0,0 +1,171 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `state-hi-schools`: Hawaii DOE public + charter schools CSV consumer.
+ *
+ *   The Hawaii State Department of Education publishes a directory of all HIDOE schools and public
+ *   charter schools (PCS) as an XLSX workbook (`SchoolList.xlsx`) with two sheets: `HIDOE` (~258
+ *   rows) and `PCS` (~38 rows). Total ~296 rows statewide. Each row carries a school name,
+ *   single-line street address, city, ZIP, a numeric `code`, and HI-specific administrative columns
+ *   (complex, complex_area, district, island, charter).
+ *
+ *   The adapter consumes a flat CSV the operator pre-builds via `fetch-state-hi-schools.ts`, which
+ *   concatenates both sheets under one shared header. Column names match the workbook header
+ *   verbatim (lower-snake-case: `code`, `name`, `address`, `city`, `zip`, ...).
+ *
+ *   Address parsing notes: Hawaii's residential numbering is hyphenated on Oahu (`47-470 Hui Aeko
+ *   Place`), Kauai (`2-4035 Kaumualii Hwy`), and elsewhere. The shared HOUSE_NUMBER_PREFIX regex
+ *   covers this via its optional `(?:-\d+)?` group.
+ *
+ *   The `island` and `district` columns are HIDOE administrative labels (Honolulu, Central, Leeward,
+ *   Windward, Hilo, Hawaii, Maui, Kauai) — they are NOT US counties and intentionally are not
+ *   surfaced as `subregion`.
+ *
+ *   Output: one row per school with `venue` (school name), `(house_number?, street, locality,
+ *   region=HI, postcode)`, and a stable `source_id` derived from the school `code`.
+ *
+ *   License: stamped `"Public Domain"` per Hawaii state government open-data terms.
+ */
+
+import { createReadStream } from "node:fs"
+
+import { parse as csvParse } from "csv-parse"
+
+import { stableSourceID } from "../../adapter.ts"
+import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
+import { reconcileComponents } from "../../format.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+
+export const STATE_HI_SCHOOLS_ADAPTER_ID = "state-hi-schools"
+export const STATE_HI_SCHOOLS_DEFAULT_LICENSE = "Public Domain"
+
+const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
+const HI_STATE_ABBR = "HI"
+
+interface HiSchoolRow {
+	code: string
+	name: string
+	address: string
+	city: string
+	zip: string
+}
+
+function splitAddress(address: string): { house_number?: string; street: string } | null {
+	const trimmed = address.trim()
+
+	if (!trimmed) return null
+	const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
+
+	if (m) return { house_number: m[1], street: m[2]!.trim() }
+
+	return { street: trimmed }
+}
+
+function normalizeZip(raw: string): string {
+	const trimmed = raw.trim()
+
+	if (!trimmed) return ""
+
+	// XLSX → CSV conversion may emit numeric ZIPs without leading zeros. HI ZIPs all begin
+	// with 96, so a 4-digit value indicates a leading-zero stripped during numeric coercion
+	// (defensive — has not been observed in the published file as of 2026-05).
+	if (/^\d{4}$/.test(trimmed)) return `0${trimmed}`
+
+	return trimmed
+}
+
+export function createStateHiSchoolsAdapter(): CorpusAdapter {
+	return {
+		id: STATE_HI_SCHOOLS_ADAPTER_ID,
+		defaultLicense: STATE_HI_SCHOOLS_DEFAULT_LICENSE,
+		description: "Hawaii DOE School Directory — ~300 K-12 public + charter schools with venue+address (public-domain).",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`state-hi-schools adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
+			const parser = stream.pipe(
+				csvParse({
+					columns: true,
+					skip_empty_lines: true,
+					relax_quotes: true,
+					relax_column_count: true,
+				})
+			)
+
+			const state = lookupStateAbbreviation(HI_STATE_ABBR)
+
+			if (!state) {
+				throw new Error(`state-hi-schools adapter: HI not found in state codex (corpus bug)`)
+			}
+
+			let emitted = 0
+
+			try {
+				for await (const record of parser as AsyncIterable<HiSchoolRow>) {
+					if (opts.signal?.aborted) break
+
+					if (opts.limit !== undefined && emitted >= opts.limit) break
+
+					const name = (record.name ?? "").trim()
+					const address = (record.address ?? "").trim()
+					const city = (record.city ?? "").trim()
+					const zip = normalizeZip(record.zip ?? "")
+
+					if (!name || !address || !city || !zip) continue
+
+					const split = splitAddress(address)
+
+					if (!split) continue
+
+					const components: CanonicalRow["components"] = {
+						venue: name,
+						...(split.house_number ? { house_number: split.house_number } : {}),
+						street: split.street,
+						locality: city,
+						region: state.abbreviation,
+						postcode: zip,
+					}
+
+					const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
+					const raw = [
+						name,
+						streetPart,
+						[city, [HI_STATE_ABBR, zip].filter(Boolean).join(" ")].filter(Boolean).join(", "),
+					]
+						.filter(Boolean)
+						.join(", ")
+
+					const aligned = reconcileComponents(components, raw)
+
+					if (Object.keys(aligned).length <= 2) continue
+
+					const code = (record.code ?? "").toString().trim()
+					const sourceID = code
+						? `${STATE_HI_SCHOOLS_ADAPTER_ID}-${code}`
+						: stableSourceID(STATE_HI_SCHOOLS_ADAPTER_ID, aligned)
+
+					yield {
+						raw,
+						components: aligned,
+						country: "US",
+						locale: "en-US",
+						source: STATE_HI_SCHOOLS_ADAPTER_ID,
+						source_id: sourceID,
+						corpus_version: "",
+						license: STATE_HI_SCHOOLS_DEFAULT_LICENSE,
+					}
+					emitted++
+				}
+			} finally {
+				stream.destroy()
+			}
+		},
+	}
+}
+
+export const stateHiSchoolsAdapter = createStateHiSchoolsAdapter()

package/src/adapters/state-ia-contractors/adapter.ts

@@ -0,0 +1,148 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `state-ia-contractors`: Iowa Active Construction Contractor Registrations CSV consumer.
+ *
+ *   Iowa Workforce Development publishes a public registry of active construction contractors. Each
+ *   row carries a business name, street address, city/state/zip, and contact info.
+ *
+ *   The adapter consumes the CSV the operator pre-downloads via `fetch-state-sources.ts`.
+ *
+ *   Output: one row per contractor with `venue` (business name) and address quad `(house_number,
+ *   street, locality, region, postcode)`.
+ *
+ *   License: stamped `"Public Domain"` per Iowa state government open-data terms.
+ */
+
+import { createReadStream } from "node:fs"
+
+import { parse as csvParse } from "csv-parse"
+
+import { stableSourceID } from "../../adapter.ts"
+import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
+import { reconcileComponents } from "../../format.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+
+export const STATE_IA_CONTRACTORS_ADAPTER_ID = "state-ia-contractors"
+export const STATE_IA_CONTRACTORS_DEFAULT_LICENSE = "Public Domain"
+
+const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
+
+interface IaContractorRow {
+	"Registration #": string
+	"Business Name": string
+	"Address 1": string
+	"Address 2": string
+	City: string
+	State: string
+	"Zip Code": string
+	"First Name": string
+	"Last Name": string
+}
+
+function splitAddress(address: string): { house_number?: string; street: string } | null {
+	const trimmed = address.trim()
+
+	if (!trimmed) return null
+	const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
+
+	if (m) return { house_number: m[1], street: m[2]!.trim() }
+
+	return { street: trimmed }
+}
+
+export function createStateIaContractorsAdapter(): CorpusAdapter {
+	return {
+		id: STATE_IA_CONTRACTORS_ADAPTER_ID,
+		defaultLicense: STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
+		description:
+			"Iowa Active Construction Contractor Registrations — business name + full street address (public-domain).",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`state-ia-contractors adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
+			const parser = stream.pipe(
+				csvParse({
+					columns: true,
+					skip_empty_lines: true,
+					relax_quotes: true,
+					relax_column_count: true,
+				})
+			)
+
+			let emitted = 0
+
+			try {
+				for await (const record of parser as AsyncIterable<IaContractorRow>) {
+					if (opts.signal?.aborted) break
+
+					if (opts.limit !== undefined && emitted >= opts.limit) break
+
+					const businessName = (record["Business Name"] ?? "").trim()
+					const address1 = (record["Address 1"] ?? "").trim()
+					const address2 = (record["Address 2"] ?? "").trim()
+					const city = (record.City ?? "").trim()
+					const stateAbbr = (record.State ?? "").trim()
+					const zip = (record["Zip Code"] ?? "").trim()
+
+					if (!city || !zip) continue
+
+					const state = lookupStateAbbreviation(stateAbbr)
+
+					if (!state) continue
+
+					const fullAddress = [address1, address2].filter(Boolean).join(" ")
+					const split = splitAddress(fullAddress)
+
+					if (!split) continue
+
+					const venue = businessName || undefined
+
+					const components: CanonicalRow["components"] = {
+						...(venue ? { venue } : {}),
+						...(split.house_number ? { house_number: split.house_number } : {}),
+						street: split.street,
+						locality: city,
+						region: state.abbreviation,
+						postcode: zip,
+					}
+
+					const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
+					const raw = [venue, streetPart, [city, [stateAbbr, zip].filter(Boolean).join(" ")].filter(Boolean).join(", ")]
+						.filter(Boolean)
+						.join(", ")
+
+					const aligned = reconcileComponents(components, raw)
+
+					if (Object.keys(aligned).length <= 2) continue
+
+					const regNum = (record["Registration #"] ?? "").trim()
+					const sourceID = regNum
+						? `${STATE_IA_CONTRACTORS_ADAPTER_ID}-${regNum}`
+						: stableSourceID(STATE_IA_CONTRACTORS_ADAPTER_ID, aligned)
+
+					yield {
+						raw,
+						components: aligned,
+						country: "US",
+						locale: "en-US",
+						source: STATE_IA_CONTRACTORS_ADAPTER_ID,
+						source_id: sourceID,
+						corpus_version: "",
+						license: STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
+					}
+					emitted++
+				}
+			} finally {
+				stream.destroy()
+			}
+		},
+	}
+}
+
+export const stateIaContractorsAdapter = createStateIaContractorsAdapter()

package/src/adapters/state-ny-notaries/adapter.ts

@@ -0,0 +1,169 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `state-ny-notaries`: New York Commissioned Notaries CSV consumer.
+ *
+ *   The New York Department of State publishes a registry of commissioned notaries public. Each row
+ *   optionally carries a business name and business address (~1-5% fill rate).
+ *
+ *   The adapter consumes the CSV the operator pre-downloads via `fetch-state-sources.ts`. Column
+ *   names match the data.ny.gov export header (note: some columns have leading spaces).
+ *
+ *   License: stamped `"Public Domain"` per New York state government open-data terms.
+ */
+
+import { createReadStream } from "node:fs"
+
+import { parse as csvParse } from "csv-parse"
+
+import { stableSourceID } from "../../adapter.ts"
+import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
+import { reconcileComponents } from "../../format.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+
+export const STATE_NY_NOTARIES_ADAPTER_ID = "state-ny-notaries"
+export const STATE_NY_NOTARIES_DEFAULT_LICENSE = "Public Domain"
+
+const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
+
+interface NyNotaryRow {
+	"Commission Holder Name": string
+	"Commission Number (UID)": string
+	"Business Name (if available)": string
+	"Business Address 1 (if available)": string
+	"Business Address 2 (if available)": string
+	" Business City (if available)": string
+	"Business State (if available)": string
+	"Business Zip (if available)": string
+	"Commissioned County": string
+}
+
+function splitAddress(address: string): { house_number?: string; street: string } | null {
+	const trimmed = address.trim()
+
+	if (!trimmed) return null
+	const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
+
+	if (m) return { house_number: m[1], street: m[2]!.trim() }
+
+	return { street: trimmed }
+}
+
+const RAW_NY_COLUMNS = [
+	"Commission Holder Name",
+	"Commission Number (UID)",
+	"Business Name (if available)",
+	"Business Address 1 (if available)",
+	"Business Address 2 (if available)",
+	" Business City (if available)",
+	"Business State (if available)",
+	"Business Zip (if available)",
+	"Commissioned County",
+] as const
+
+export function createStateNyNotariesAdapter(): CorpusAdapter {
+	return {
+		id: STATE_NY_NOTARIES_ADAPTER_ID,
+		defaultLicense: STATE_NY_NOTARIES_DEFAULT_LICENSE,
+		description: "New York Commissioned Notaries — name + optional business address (public-domain).",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`state-ny-notaries adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
+			const parser = stream.pipe(
+				csvParse({
+					columns: true,
+					skip_empty_lines: true,
+					relax_quotes: true,
+					relax_column_count: true,
+				})
+			)
+
+			let emitted = 0
+
+			try {
+				for await (const rawRecord of parser as AsyncIterable<Record<string, string>>) {
+					if (opts.signal?.aborted) break
+
+					if (opts.limit !== undefined && emitted >= opts.limit) break
+
+					// NY CSV has columns with leading spaces, so we normalize by trimming keys.
+					const record: Record<string, string> = {}
+
+					for (const key of Object.keys(rawRecord)) {
+						record[key.trim()] = rawRecord[key] ?? ""
+					}
+
+					const holderName = (record["Commission Holder Name"] ?? "").trim()
+					const businessName = (record["Business Name (if available)"] ?? "").trim()
+					const address1 = (record["Business Address 1 (if available)"] ?? "").trim()
+					const address2 = (record["Business Address 2 (if available)"] ?? "").trim()
+					const city = (record["Business City (if available)"] ?? "").trim()
+					const stateAbbr = (record["Business State (if available)"] ?? "").trim()
+					const zip = (record["Business Zip (if available)"] ?? "").trim()
+					const county = (record["Commissioned County"] ?? "").trim()
+
+					if (!city || !stateAbbr || !zip) continue
+
+					if (!address1 && !address2) continue
+
+					const state = lookupStateAbbreviation(stateAbbr)
+
+					if (!state) continue
+
+					const fullAddress = [address1, address2].filter(Boolean).join(" ")
+					const split = splitAddress(fullAddress)
+
+					if (!split) continue
+
+					const venue = businessName || holderName || undefined
+
+					const components: CanonicalRow["components"] = {
+						...(venue ? { venue } : {}),
+						...(split.house_number ? { house_number: split.house_number } : {}),
+						street: split.street,
+						locality: city,
+						region: state.abbreviation,
+						postcode: zip,
+						...(county ? { subregion: county } : {}),
+					}
+
+					const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
+					const raw = [venue, streetPart, [city, [stateAbbr, zip].filter(Boolean).join(" ")].filter(Boolean).join(", ")]
+						.filter(Boolean)
+						.join(", ")
+
+					const aligned = reconcileComponents(components, raw)
+
+					if (Object.keys(aligned).length <= 2) continue
+
+					const commNum = (record["Commission Number (UID)"] ?? "").trim()
+					const sourceID = commNum
+						? `${STATE_NY_NOTARIES_ADAPTER_ID}-${commNum}`
+						: stableSourceID(STATE_NY_NOTARIES_ADAPTER_ID, aligned)
+
+					yield {
+						raw,
+						components: aligned,
+						country: "US",
+						locale: "en-US",
+						source: STATE_NY_NOTARIES_ADAPTER_ID,
+						source_id: sourceID,
+						corpus_version: "",
+						license: STATE_NY_NOTARIES_DEFAULT_LICENSE,
+					}
+					emitted++
+				}
+			} finally {
+				stream.destroy()
+			}
+		},
+	}
+}
+
+export const stateNyNotariesAdapter = createStateNyNotariesAdapter()