@mailwoman/corpus 9.4.0 → 10.0.0

package/lib/us/adapters/fcc-bdc/adapter.ts

@@ -0,0 +1,173 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `fcc-bdc`: FCC Broadband Data Collection (BDC) — Fabric-derived location consumer.
+ *
+ *   The first member of Phase 1.6's "adversarial sources" class. BDC ships the public-domain US
+ *   broadband-serviceable-location (BSL) fabric: ~120M addresses keyed by stable `location_id`,
+ *   carrying `address_primary` + `city` + `state` + `zip` + `zip_suffix`. Compared to the clean
+ *   gazetteer rows from WOF / TIGER / BAN, BDC carries the chaos of address data that has passed
+ *   through several layers of human entry + automated geocoding + revision: abbreviation drift,
+ *   inconsistent unit designators, "RR" / "HC" / "PSC" rural-route shapes, embedded apartment /
+ *   suite numbers that did not survive the address parser cleanly. This is the highest-signal,
+ *   hardest-to-normalize address corpus in the federal public-domain catalog.
+ *
+ *   Following the `tiger` / `wof-admin` pattern, this adapter consumes a SQLite database the operator
+ *   pre-builds via the isp-nexus BDC ETL (`/srv/isp-nexus/sync/fcc/bdc/`) or any equivalent
+ *   host-side pipeline. The mailwoman side does not download or parse the raw CSV/ZIP distribution
+ *   directly — that keeps the adapter narrow and the BDC ingest pluggable.
+ *
+ *   The SQLite schema is documented in README.md and modeled after `NTIARecord`
+ *   (`isp-nexus/fcc/bdc/data-collection.ts`): one row per `location_id`. The adapter splits
+ *   `address_primary` into `house_number` (leading numeric prefix, if any) + `street` (everything
+ *   after), and combines `zip` + `zip_suffix` into the canonical USPS `postcode` slot.
+ *
+ *   One CanonicalRow per fabric record. Unlike `tiger` (multiple postcode variants per segment) or
+ *   `wof-admin` (multiple hierarchy variants per place), BDC records already represent fully
+ *   specified addresses; no fan-out is warranted. Adversarial composition (Phase 1.6 §2.1) is the
+ *   mechanism for deriving multiple training rows per BDC record.
+ *
+ *   License: stamped `"Public Domain"` per the BDC fabric's US federal-government distribution terms.
+ *   The CostQuest Fabric source data has its own license; consumers who substitute that path should
+ *   re-stamp accordingly.
+ */
+
+import type { BDCDatabase } from "@mailwoman/bdc/schema"
+import { formatAddressRow } from "@mailwoman/codex/address-format"
+import { DatabaseClient } from "@mailwoman/sqlite/client"
+
+import { splitStreetLine } from "#adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+import { lookupStateAbbreviation } from "#us/fips-state"
+
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const FCC_BDC_ADAPTER_ID = "fcc-bdc"
+/**
+ * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
+ * than having to look them up.
+ */
+export const FCC_BDC_DEFAULT_LICENSE = "Public Domain"
+
+/**
+ * SQLite row shape — one row per BSL `location_id`. Columns mirror NTIARecord.
+ */
+interface BdcLocationRow {
+	location_id: number
+	address_primary: string
+	city: string
+	state: string
+	zip: string
+	zip_suffix: string | null
+}
+
+/**
+ * Combine `zip` + optional `zip_suffix` into the canonical USPS postcode surface form.
+ *
+ * NTIARecord doc is ambiguous about whether `zip_suffix` is the 4-digit extension alone or the full ZIP+4 string. This
+ * handles both:
+ *
+ * - Bare 4-digit extension (`zip="94103"`, `zip_suffix="1234"`) → `"94103-1234"`
+ * - Already-joined form (`zip_suffix="94103-1234"`) → returned as-is
+ * - No suffix → bare `zip`
+ *
+ * Empty / whitespace-only suffix is treated as missing.
+ */
+export function buildPostcode(zip: string, suffix: string | null): string {
+	const z = zip.trim()
+
+	if (!z) return ""
+	const s = suffix?.trim() ?? ""
+
+	if (!s) return z
+
+	if (s.includes("-")) return s
+
+	return `${z}-${s}`
+}
+
+/**
+ * Build a BDC adapter. Pure factory so multiple instances can be created in tests.
+ */
+export function createFccBdcAdapter(): CorpusAdapter {
+	return {
+		id: FCC_BDC_ADAPTER_ID,
+		defaultLicense: FCC_BDC_DEFAULT_LICENSE,
+		description:
+			"FCC Broadband Data Collection — Fabric-derived BSL addresses (public-domain); SQLite DB the operator builds via the isp-nexus BDC ETL.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`fcc-bdc adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const db = new DatabaseClient<BDCDatabase>(opts.inputPath, { readOnly: true })
+			let emitted = 0
+
+			try {
+				const stmt = db.prepare(
+					`SELECT location_id, address_primary, city, state, zip, zip_suffix
+					 FROM bdc_locations
+					 ORDER BY location_id`
+				)
+
+				for (const row of stmt.iterate() as IterableIterator<BdcLocationRow>) {
+					if (opts.signal?.aborted) return
+
+					if (opts.limit !== undefined && emitted >= opts.limit) return
+
+					const split = splitStreetLine(row.address_primary ?? "")
+
+					if (!split) continue
+					const state = lookupStateAbbreviation(row.state)
+
+					if (!state) continue
+					const locality = row.city?.trim()
+
+					if (!locality) continue
+					const postcode = buildPostcode(row.zip ?? "", row.zip_suffix ?? null)
+
+					if (!postcode) continue
+
+					const components: CanonicalRow["components"] = {
+						...(split.house_number ? { house_number: split.house_number } : {}),
+						street: split.street,
+						locality,
+						region: state.abbreviation,
+						postcode,
+					}
+
+					const rendered = formatAddressRow(components, "US", { singleLine: true })
+
+					if (!rendered) continue
+
+					const { raw, components: aligned } = rendered
+
+					yield {
+						raw,
+						components: aligned,
+						country: "US",
+						locale: "en-US",
+						source: FCC_BDC_ADAPTER_ID,
+						source_id: `${FCC_BDC_ADAPTER_ID}-${row.location_id}`,
+						corpus_version: "",
+						license: FCC_BDC_DEFAULT_LICENSE,
+					}
+
+					emitted++
+				}
+			} finally {
+				await db.destroy()
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const fccBdcAdapter = createFccBdcAdapter()

package/lib/us/adapters/state/hi-schools/adapter.ts

@@ -0,0 +1,160 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `state-hi-schools`: Hawaii DOE public + charter school workbook consumer.
+ *
+ *   The Hawaii State Department of Education publishes a directory of all HIDOE schools and public
+ *   charter schools (PCS) as an XLSX workbook (`SchoolList.xlsx`) with two sheets: `HIDOE` (~258
+ *   rows) and `PCS` (~38 rows). Total ~296 rows statewide. Each row carries a school name,
+ *   single-line street address, city, ZIP, a numeric `code`, and HI-specific administrative columns
+ *   (complex, complex_area, district, island, charter).
+ *
+ *   The adapter consumes the original XLSX retained by the fetcher and reads both worksheets directly.
+ *   Legacy flat CSV artifacts remain accepted so existing source caches do not need conversion.
+ *
+ *   Address parsing notes: Hawaii's residential numbering is hyphenated on Oahu (`47-470 Hui Aeko
+ *   Place`), Kauai (`2-4035 Kaumualii Hwy`), and elsewhere. The shared HOUSE_NUMBER_PREFIX regex
+ *   covers this via its optional `(?:-\d+)?` group.
+ *
+ *   The `island` and `district` columns are HIDOE administrative labels (Honolulu, Central, Leeward,
+ *   Windward, Hilo, Hawaii, Maui, Kauai) — they are NOT US counties and intentionally are not
+ *   surfaced as `subregion`.
+ *
+ *   Output: one row per school with `venue` (school name), `(house_number?, street, locality,
+ *   region=HI, postcode)`, and a stable `source_id` derived from the school `code`.
+ *
+ *   License: stamped `"Public Domain"` per Hawaii state government open-data terms.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address-format"
+import { CSVSpliterator, XLSXSpliterator } from "spliterator"
+
+import { splitStreetLine, stableSourceID } from "#adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+import { type HiSchoolRow, schoolCellText, STATE_HI_SCHOOL_SHEETS } from "#us/adapters/state/hi-schools/workbook"
+import { lookupStateAbbreviation } from "#us/fips-state"
+
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const STATE_HI_SCHOOLS_ADAPTER_ID = "state-hi-schools"
+/**
+ * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
+ * than having to look them up.
+ */
+export const STATE_HI_SCHOOLS_DEFAULT_LICENSE = "Public Domain"
+
+const HI_STATE_ABBR = "HI"
+
+function normalizeZip(raw: HiSchoolRow["zip"]): string {
+	const trimmed = schoolCellText(raw)
+
+	if (!trimmed) return ""
+
+	// XLSX → CSV conversion may emit numeric ZIPs without leading zeros. HI ZIPs all begin
+	// with 96, so a 4-digit value indicates a leading-zero stripped during numeric coercion
+	// (defensive — has not been observed in the published file as of 2026-05).
+	if (/^\d{4}$/.test(trimmed)) return `0${trimmed}`
+
+	return trimmed
+}
+
+async function* readSchoolRows(inputPath: string): AsyncGenerator<HiSchoolRow> {
+	if (inputPath.toLowerCase().endsWith(".xlsx")) {
+		for (const sheet of STATE_HI_SCHOOL_SHEETS) {
+			yield* XLSXSpliterator.fromAsync<HiSchoolRow>(inputPath, {
+				sheet,
+				normalizeKeys: false,
+			})
+		}
+
+		return
+	}
+
+	yield* CSVSpliterator.fromAsync<HiSchoolRow>(inputPath, {
+		normalizeKeys: false,
+	})
+}
+
+export function createStateHiSchoolsAdapter(): CorpusAdapter {
+	return {
+		id: STATE_HI_SCHOOLS_ADAPTER_ID,
+		defaultLicense: STATE_HI_SCHOOLS_DEFAULT_LICENSE,
+		description: "Hawaii DOE School Directory — ~300 K-12 public + charter schools with venue+address (public-domain).",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`state-hi-schools adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const state = lookupStateAbbreviation(HI_STATE_ABBR)
+
+			if (!state) {
+				throw new Error(`state-hi-schools adapter: HI not found in state codex (corpus bug)`)
+			}
+
+			let emitted = 0
+
+			for await (const record of readSchoolRows(opts.inputPath)) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const name = schoolCellText(record.name)
+				const address = schoolCellText(record.address)
+				const city = schoolCellText(record.city)
+				const zip = normalizeZip(record.zip)
+
+				if (!name || !address || !city || !zip) continue
+
+				const split = splitStreetLine(address)
+
+				if (!split) continue
+
+				const components: CanonicalRow["components"] = {
+					venue: name,
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode: zip,
+				}
+
+				const rendered = formatAddressRow(components, "US", { singleLine: true })
+
+				if (!rendered) continue
+
+				const { raw, components: aligned } = rendered
+
+				if (Object.keys(aligned).length <= 2) continue
+
+				const code = schoolCellText(record.code)
+
+				const sourceID = code
+					? `${STATE_HI_SCHOOLS_ADAPTER_ID}-${code}`
+					: stableSourceID(STATE_HI_SCHOOLS_ADAPTER_ID, aligned)
+
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: STATE_HI_SCHOOLS_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: STATE_HI_SCHOOLS_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const stateHiSchoolsAdapter = createStateHiSchoolsAdapter()

package/lib/us/adapters/state/ia-contractors/adapter.ts

@@ -0,0 +1,136 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `state-ia-contractors`: Iowa Active Construction Contractor Registrations CSV consumer.
+ *
+ *   Iowa Workforce Development publishes a public registry of active construction contractors. Each
+ *   row carries a business name, street address, city/state/zip, and contact info.
+ *
+ *   The adapter consumes the CSV the operator pre-downloads via `fetch-state-sources.ts`.
+ *
+ *   Output: one row per contractor with `venue` (business name) and address quad `(house_number,
+ *   street, locality, region, postcode)`.
+ *
+ *   License: stamped `"Public Domain"` per Iowa state government open-data terms.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address-format"
+import { isPresent } from "@mailwoman/core/objects"
+import { CSVSpliterator } from "spliterator"
+
+import { splitStreetLine, stableSourceID } from "#adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+import { lookupStateAbbreviation } from "#us/fips-state"
+
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const STATE_IA_CONTRACTORS_ADAPTER_ID = "state-ia-contractors"
+/**
+ * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
+ * than having to look them up.
+ */
+export const STATE_IA_CONTRACTORS_DEFAULT_LICENSE = "Public Domain"
+
+interface IaContractorRow {
+	"Registration #": string
+	"Business Name": string
+	"Address 1": string
+	"Address 2": string
+	City: string
+	State: string
+	"Zip Code": string
+	"First Name": string
+	"Last Name": string
+}
+
+export function createStateIaContractorsAdapter(): CorpusAdapter {
+	return {
+		id: STATE_IA_CONTRACTORS_ADAPTER_ID,
+		defaultLicense: STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
+		description:
+			"Iowa Active Construction Contractor Registrations — business name + full street address (public-domain).",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`state-ia-contractors adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				normalizeKeys: false,
+			})
+
+			let emitted = 0
+
+			for await (const record of rows as AsyncIterable<IaContractorRow>) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const businessName = (record["Business Name"] ?? "").trim()
+				const address1 = (record["Address 1"] ?? "").trim()
+				const address2 = (record["Address 2"] ?? "").trim()
+				const city = (record.City ?? "").trim()
+				const stateAbbr = (record.State ?? "").trim()
+				const zip = (record["Zip Code"] ?? "").trim()
+
+				if (!city || !zip) continue
+
+				const state = lookupStateAbbreviation(stateAbbr)
+
+				if (!state) continue
+
+				const fullAddress = [address1, address2].filter(isPresent).join(" ")
+				const split = splitStreetLine(fullAddress)
+
+				if (!split) continue
+
+				const venue = businessName || undefined
+
+				const components: CanonicalRow["components"] = {
+					...(venue ? { venue } : {}),
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode: zip,
+				}
+
+				const rendered = formatAddressRow(components, "US", { singleLine: true })
+
+				if (!rendered) continue
+
+				const { raw, components: aligned } = rendered
+
+				if (Object.keys(aligned).length <= 2) continue
+
+				const regNum = (record["Registration #"] ?? "").trim()
+
+				const sourceID = regNum
+					? `${STATE_IA_CONTRACTORS_ADAPTER_ID}-${regNum}`
+					: stableSourceID(STATE_IA_CONTRACTORS_ADAPTER_ID, aligned)
+
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: STATE_IA_CONTRACTORS_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const stateIaContractorsAdapter = createStateIaContractorsAdapter()

package/lib/us/adapters/state/ny-notaries/adapter.ts

@@ -0,0 +1,133 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `state-ny-notaries`: New York Commissioned Notaries CSV consumer.
+ *
+ *   The New York Department of State publishes a registry of commissioned notaries public. Each row
+ *   optionally carries a business name and business address (~1-5% fill rate).
+ *
+ *   The adapter consumes the CSV the operator pre-downloads via `fetch-state-sources.ts`. Column
+ *   names match the data.ny.gov export header (note: some columns have leading spaces).
+ *
+ *   License: stamped `"Public Domain"` per New York state government open-data terms.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address-format"
+import { isPresent } from "@mailwoman/core/objects"
+import { CSVSpliterator } from "spliterator"
+
+import { splitStreetLine, stableSourceID } from "#adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+import { lookupStateAbbreviation } from "#us/fips-state"
+
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const STATE_NY_NOTARIES_ADAPTER_ID = "state-ny-notaries"
+/**
+ * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
+ * than having to look them up.
+ */
+export const STATE_NY_NOTARIES_DEFAULT_LICENSE = "Public Domain"
+
+export function createStateNyNotariesAdapter(): CorpusAdapter {
+	return {
+		id: STATE_NY_NOTARIES_ADAPTER_ID,
+		defaultLicense: STATE_NY_NOTARIES_DEFAULT_LICENSE,
+		description: "New York Commissioned Notaries — name + optional business address (public-domain).",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`state-ny-notaries adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				normalizeKeys: false,
+			})
+
+			let emitted = 0
+
+			for await (const rawRecord of rows as AsyncIterable<Record<string, string>>) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				// NY CSV has columns with leading spaces, so we normalize by trimming keys.
+				const record: Record<string, string> = {}
+
+				for (const key of Object.keys(rawRecord)) {
+					record[key.trim()] = rawRecord[key] ?? ""
+				}
+
+				const holderName = (record["Commission Holder Name"] ?? "").trim()
+				const businessName = (record["Business Name (if available)"] ?? "").trim()
+				const address1 = (record["Business Address 1 (if available)"] ?? "").trim()
+				const address2 = (record["Business Address 2 (if available)"] ?? "").trim()
+				const city = (record["Business City (if available)"] ?? "").trim()
+				const stateAbbr = (record["Business State (if available)"] ?? "").trim()
+				const zip = (record["Business Zip (if available)"] ?? "").trim()
+				const county = (record["Commissioned County"] ?? "").trim()
+
+				if (!city || !stateAbbr || !zip) continue
+
+				if (!address1 && !address2) continue
+
+				const state = lookupStateAbbreviation(stateAbbr)
+
+				if (!state) continue
+
+				const fullAddress = [address1, address2].filter(isPresent).join(" ")
+				const split = splitStreetLine(fullAddress)
+
+				if (!split) continue
+
+				const venue = businessName || holderName || undefined
+
+				const components: CanonicalRow["components"] = {
+					...(venue ? { venue } : {}),
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode: zip,
+					...(county ? { subregion: county } : {}),
+				}
+
+				const rendered = formatAddressRow(components, "US", { singleLine: true })
+
+				if (!rendered) continue
+
+				const { raw, components: aligned } = rendered
+
+				if (Object.keys(aligned).length <= 2) continue
+
+				const commNum = (record["Commission Number (UID)"] ?? "").trim()
+
+				const sourceID = commNum
+					? `${STATE_NY_NOTARIES_ADAPTER_ID}-${commNum}`
+					: stableSourceID(STATE_NY_NOTARIES_ADAPTER_ID, aligned)
+
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: STATE_NY_NOTARIES_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: STATE_NY_NOTARIES_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const stateNyNotariesAdapter = createStateNyNotariesAdapter()