@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/us/adapters/fcc-bdc/adapter.ts

@@ -0,0 +1,158 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `fcc-bdc`: FCC Broadband Data Collection (BDC) adapter.
+ *
+ *   Reads BDC Fabric addresses from a prebuilt SQLite database.
+ * 	 Note that Fabric IDs are managed by CostQuest and may be subject to licensing or usage restrictions.
+ *
+ *   Each row is keyed by `location_id` and includes:
+ *   `address_primary`, `city`, `state`, `zip`, and `zip_suffix`.
+ *
+ *   This adapter does not download raw BDC files.
+ *
+ *   It converts each source row into one CanonicalRow by:
+ *   - splitting `address_primary` into `house_number` + `street`
+ *   - combining `zip` + `zip_suffix` into `postcode`
+ *
+ *   Rows are stamped with the source license: `"Public Domain"`.
+ */
+
+import type { BDCDatabase } from "@mailwoman/bdc/schema"
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+import { DatabaseClient } from "@mailwoman/sqlite/client"
+
+import { splitStreetLine } from "#adapters/street-line"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
+import { lookupStateAbbreviation } from "#us/fips-state"
+
+/**
+ * Stable registry id for this adapter.
+ */
+export const FCC_BDC_ADAPTER_ID = "fcc-bdc"
+/**
+ * Default license stamped on emitted rows.
+ */
+export const FCC_BDC_DEFAULT_LICENSE = "Public Domain"
+
+/**
+ * SQLite row shape (one row per `location_id`).
+ */
+interface BdcLocationRow {
+	location_id: number
+	address_primary: string
+	city: string
+	state: string
+	zip: string
+	zip_suffix: string | null
+}
+
+/**
+ * Build USPS postcode from `zip` and optional `zip_suffix`.
+ * Handles both a 4-digit suffix and a full ZIP+4 value.
+ *
+ * - Bare 4-digit extension (`zip="94103"`, `zip_suffix="1234"`) → `"94103-1234"`
+ * - Already-joined form (`zip_suffix="94103-1234"`) → returned as-is
+ * - No suffix → bare `zip`
+ *
+ * Empty/whitespace suffix is treated as missing.
+ */
+export function buildPostcode(zip: string, suffix: string | null): string {
+	const z = zip.trim()
+
+	if (!z) return ""
+	const s = suffix?.trim() ?? ""
+
+	if (!s) return z
+
+	if (s.includes("-")) return s
+
+	return `${z}-${s}`
+}
+
+/**
+ * Build the BDC corpus adapter.
+ */
+export function createFccBdcAdapter(): CorpusAdapter {
+	return {
+		id: FCC_BDC_ADAPTER_ID,
+		defaultLicense: FCC_BDC_DEFAULT_LICENSE,
+		addressRole: AddressRole.Premise,
+		register: SourceRegister.FCCBroadbandData,
+		surface: SurfaceOrigin.Rendered,
+		description: "FCC Broadband Data Collection — Fabric-derived BSL addresses (public-domain).",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`fcc-bdc adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const db = new DatabaseClient<BDCDatabase>(opts.inputPath, { readOnly: true })
+			let emitted = 0
+
+			try {
+				const stmt = db.prepare(
+					`SELECT location_id, address_primary, city, state, zip, zip_suffix
+					 FROM bdc_locations
+					 ORDER BY location_id`
+				)
+
+				for (const row of stmt.iterate() as IterableIterator<BdcLocationRow>) {
+					if (opts.signal?.aborted) return
+
+					if (opts.limit !== undefined && emitted >= opts.limit) return
+
+					const split = splitStreetLine(row.address_primary ?? "")
+
+					if (!split) continue
+					const state = lookupStateAbbreviation(row.state)
+
+					if (!state) continue
+					const locality = row.city?.trim()
+
+					if (!locality) continue
+					const postcode = buildPostcode(row.zip ?? "", row.zip_suffix ?? null)
+
+					if (!postcode) continue
+
+					const components: CanonicalRow["components"] = {
+						...(split.house_number ? { house_number: split.house_number } : {}),
+						street: split.street,
+						locality,
+						region: state.abbreviation,
+						postcode,
+					}
+
+					const rendered = formatAddressRow(components, "US", { singleLine: true })
+
+					if (!rendered) continue
+
+					const { raw, components: aligned } = rendered
+
+					yield {
+						raw,
+						components: aligned,
+						country: "US",
+						locale: "en-US",
+						source: FCC_BDC_ADAPTER_ID,
+						source_id: `${FCC_BDC_ADAPTER_ID}-${row.location_id}`,
+						corpus_version: "",
+						license: FCC_BDC_DEFAULT_LICENSE,
+					}
+
+					emitted++
+				}
+			} finally {
+				await db.destroy()
+			}
+		},
+	}
+}
+
+/**
+ * Default configured adapter instance.
+ */
+export const fccBdcAdapter = createFccBdcAdapter()

package/lib/us/adapters/state/hi-schools/adapter.ts

@@ -0,0 +1,171 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `state-hi-schools`: Hawaii DOE public + charter school workbook consumer.
+ *
+ *   The Hawaii State Department of Education publishes a directory of all hidoe schools and public
+ *   charter schools (PCS) as an xlsx workbook (`SchoolList.xlsx`) with two sheets: `hidoe` (~258
+ *   rows) and `PCS` (~38 rows). Total ~296 rows statewide. Each row includes a school name,
+ *   single-line street address, city, ZIP, a numeric `code`, and HI-specific administrative columns
+ *   (complex, complex_area, district, island, charter).
+ *
+ *   The adapter consumes the original xlsx retained by the fetcher and reads both worksheets directly.
+ *   Legacy flat CSV artifacts remain accepted so existing source caches do not need conversion.
+ *
+ *   Address parsing notes: Hawaii's residential numbering is hyphenated on Oahu (`47-470 Hui Aeko
+ *   Place`), Kauai (`2-4035 Kaumualii Hwy`), and elsewhere. The shared HOUSE_NUMBER_PREFIX regex
+ *   covers this via its optional `(?:-\d+)?` group.
+ *
+ *   The `island` and `district` columns are hidoe administrative labels (Honolulu, Central, Leeward,
+ *   Windward, Hilo, Hawaii, Maui, Kauai) — they are not US counties and intentionally are not
+ *   surfaced as `subregion`.
+ *
+ *   Output: one row per school with `venue` (school name), `(house_number?, street, locality,
+ *   region=HI, postcode)`, and a stable `source_id` derived from the school `code`.
+ *
+ *   License: stamped `"Public Domain"` per Hawaii state government open-data terms.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+import type { PathBuilderLike } from "path-ts"
+import { CSVSpliterator, XLSXSpliterator } from "spliterator"
+
+import { stableSourceID } from "#adapters/source-id"
+import { splitStreetLine } from "#adapters/street-line"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
+import { type HiSchoolRow, schoolCellText, STATE_HI_SCHOOL_SHEETS } from "#us/adapters/state/hi-schools/workbook"
+import { lookupStateAbbreviation } from "#us/fips-state"
+
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ */
+export const STATE_HI_SCHOOLS_ADAPTER_ID = "state-hi-schools"
+/**
+ * License assigned by this source (Public Domain), attached to each row so downstream
+ * consumers inherit the terms rather than having to look them up.
+ */
+export const STATE_HI_SCHOOLS_DEFAULT_LICENSE = "Public Domain"
+
+const HI_STATE_ABBR = "HI"
+
+function normalizeZip(raw: HiSchoolRow["zip"]): string {
+	const trimmed = schoolCellText(raw)
+
+	if (!trimmed) return ""
+
+	// xlsx → CSV conversion may emit numeric ZIPs without leading zeros.
+	// HI ZIPs all begin with 96, so a 4-digit value indicates a leading-zero stripped during
+	// numeric coercion (defensive — has not been observed in the published file as of 2026-05).
+	if (/^\d{4}$/.test(trimmed)) return `0${trimmed}`
+
+	return trimmed
+}
+
+async function* readSchoolRows(source: PathBuilderLike): AsyncGenerator<HiSchoolRow> {
+	// XLSXSpliterator opens a string, URL, or byte source.
+	// A PathBuilder is none of those.
+	const inputPath = source.toString()
+
+	if (inputPath.toLowerCase().endsWith(".xlsx")) {
+		for (const sheet of STATE_HI_SCHOOL_SHEETS) {
+			yield* XLSXSpliterator.fromAsync<HiSchoolRow>(inputPath, {
+				sheet,
+				normalizeKeys: false,
+			})
+		}
+
+		return
+	}
+
+	yield* CSVSpliterator.fromAsync<HiSchoolRow>(inputPath, {
+		normalizeKeys: false,
+	})
+}
+
+export function createStateHiSchoolsAdapter(): CorpusAdapter {
+	return {
+		id: STATE_HI_SCHOOLS_ADAPTER_ID,
+		defaultLicense: STATE_HI_SCHOOLS_DEFAULT_LICENSE,
+		addressRole: AddressRole.Facility,
+		register: SourceRegister.HawaiiSchools,
+		surface: SurfaceOrigin.Rendered,
+		description: "Hawaii DOE School Directory — ~300 K-12 public + charter schools with venue+address (public-domain).",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`state-hi-schools adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const state = lookupStateAbbreviation(HI_STATE_ABBR)
+
+			if (!state) {
+				throw new Error(`state-hi-schools adapter: HI not found in state codex (corpus bug)`)
+			}
+
+			let emitted = 0
+
+			for await (const record of readSchoolRows(opts.inputPath)) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const name = schoolCellText(record.name)
+				const address = schoolCellText(record.address)
+				const city = schoolCellText(record.city)
+				const zip = normalizeZip(record.zip)
+
+				if (!name || !address || !city || !zip) continue
+
+				const split = splitStreetLine(address)
+
+				if (!split) continue
+
+				const components: CanonicalRow["components"] = {
+					venue: name,
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode: zip,
+				}
+
+				const rendered = formatAddressRow(components, "US", { singleLine: true })
+
+				if (!rendered) continue
+
+				const { raw, components: aligned } = rendered
+
+				if (Object.keys(aligned).length <= 2) continue
+
+				const code = schoolCellText(record.code)
+
+				const sourceID = code
+					? `${STATE_HI_SCHOOLS_ADAPTER_ID}-${code}`
+					: stableSourceID(STATE_HI_SCHOOLS_ADAPTER_ID, aligned)
+
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: STATE_HI_SCHOOLS_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: STATE_HI_SCHOOLS_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const stateHiSchoolsAdapter = createStateHiSchoolsAdapter()

package/lib/us/adapters/state/hi-schools/workbook.ts

@@ -0,0 +1,40 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+import type { XLSXCellValue } from "spliterator"
+
+/**
+ * The two worksheets that together form the public-school directory.
+ */
+export const STATE_HI_SCHOOL_SHEETS = ["HIDOE", "PCS"] as const
+
+/**
+ * Columns required to construct a corpus row.
+ *
+ * Fetch validation uses the same interface as the adapter.
+ */
+export const STATE_HI_SCHOOL_REQUIRED_COLUMNS = ["code", "name", "address", "city", "zip"] as const
+
+/**
+ * Workbook row fields consumed by the adapter.
+ *
+ * Xlsx cells are typed.
+ * The legacy CSV path supplies strings.
+ */
+export interface HiSchoolRow {
+	code: XLSXCellValue | undefined
+	name: XLSXCellValue | undefined
+	address: XLSXCellValue | undefined
+	city: XLSXCellValue | undefined
+	zip: XLSXCellValue | undefined
+}
+
+/**
+ * Convert a typed workbook cell or legacy CSV field to the trimmed text used by the corpus row.
+ */
+export function schoolCellText(value: XLSXCellValue | undefined): string {
+	return value === null || value === undefined ? "" : String(value).trim()
+}

package/lib/us/adapters/state/ia-contractors/adapter.ts

@@ -0,0 +1,148 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `state-ia-contractors`: Iowa Active Construction Contractor Registrations CSV consumer.
+ *
+ *   Iowa Workforce Development publishes a public registry of active construction contractors. Each
+ *   row includes a business name, street address, city/state/zip and contact info.
+ *
+ *   The adapter consumes the CSV the operator pre-downloads via `fetch-state-sources.ts`.
+ *
+ *   Output: one row per contractor with `venue` (business name) and address quad `(house_number,
+ *   street, locality, region, postcode)`.
+ *
+ *   The id spells the publisher's own word for the trade. Iowa titles the dataset `IA Active
+ *   Construction Contractor Registrations`, and the root instructions preserve an external name. The
+ *   id is also stored as data: it is the `source` string on every row of a built corpus and the key a
+ *   training config's `source_weights` map addresses, so renaming it strands every corpus built
+ *   before the rename.
+ *
+ *   License: stamped `"Public Domain"` per Iowa state government open-data terms.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+import { isPresent } from "@mailwoman/core/objects"
+import { CSVSpliterator } from "spliterator"
+
+import { stableSourceID } from "#adapters/source-id"
+import { splitStreetLine } from "#adapters/street-line"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
+import { lookupStateAbbreviation } from "#us/fips-state"
+
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ */
+export const STATE_IA_CONTRACTORS_ADAPTER_ID = "state-ia-contractors"
+/**
+ * License assigned by this source (Public Domain), attached to each row so downstream
+ * consumers inherit the terms rather than having to look them up.
+ */
+export const STATE_IA_CONTRACTORS_DEFAULT_LICENSE = "Public Domain"
+
+interface IaContractorRow {
+	"Registration #": string
+	"Business Name": string
+	"Address 1": string
+	"Address 2": string
+	City: string
+	State: string
+	"Zip Code": string
+	"First Name": string
+	"Last Name": string
+}
+
+export function createStateIaContractorsAdapter(): CorpusAdapter {
+	return {
+		id: STATE_IA_CONTRACTORS_ADAPTER_ID,
+		defaultLicense: STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
+		addressRole: AddressRole.RegisteredOffice,
+		register: SourceRegister.IowaContractors,
+		surface: SurfaceOrigin.Rendered,
+		description:
+			"Iowa Active Construction Contractor Registrations — business name + full street address (public-domain).",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`state-ia-contractors adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				normalizeKeys: false,
+			})
+
+			let emitted = 0
+
+			for await (const record of rows as AsyncIterable<IaContractorRow>) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const businessName = record["Business Name"] ?? ""
+				const address1 = record["Address 1"] ?? ""
+				const address2 = record["Address 2"] ?? ""
+				const city = record.City ?? ""
+				const stateAbbr = record.State ?? ""
+				const zip = record["Zip Code"] ?? ""
+
+				if (!city || !zip) continue
+
+				const state = lookupStateAbbreviation(stateAbbr)
+
+				if (!state) continue
+
+				const fullAddress = [address1, address2].filter(isPresent).join(" ")
+				const split = splitStreetLine(fullAddress)
+
+				if (!split) continue
+
+				const venue = businessName || undefined
+
+				const components: CanonicalRow["components"] = {
+					...(venue ? { venue } : {}),
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode: zip,
+				}
+
+				const rendered = formatAddressRow(components, "US", { singleLine: true })
+
+				if (!rendered) continue
+
+				const { raw, components: aligned } = rendered
+
+				if (Object.keys(aligned).length <= 2) continue
+
+				const regNum = record["Registration #"] ?? ""
+
+				const sourceID = regNum
+					? `${STATE_IA_CONTRACTORS_ADAPTER_ID}-${regNum}`
+					: stableSourceID(STATE_IA_CONTRACTORS_ADAPTER_ID, aligned)
+
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: STATE_IA_CONTRACTORS_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const stateIaContractorsAdapter = createStateIaContractorsAdapter()

package/lib/us/adapters/state/ny-notaries/adapter.ts

@@ -0,0 +1,132 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `state-ny-notaries`: New York Commissioned Notaries CSV consumer.
+ *
+ *   The New York Department of State publishes a registry of commissioned notaries public. Each row
+ *   optionally includes a business name and business address (~1-5% fill rate).
+ *
+ *   The adapter consumes the CSV the operator pre-downloads via `fetch-state-sources.ts`. Column
+ *   names match the data.ny.gov export header (note: some columns have leading spaces).
+ *
+ *   License: stamped `"Public Domain"` per New York state government open-data terms.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+import { isPresent } from "@mailwoman/core/objects"
+import { CSVSpliterator } from "spliterator"
+
+import { stableSourceID } from "#adapters/source-id"
+import { splitStreetLine } from "#adapters/street-line"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
+import { lookupStateAbbreviation } from "#us/fips-state"
+
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ */
+export const STATE_NY_NOTARIES_ADAPTER_ID = "state-ny-notaries"
+/**
+ * License assigned by this source (Public Domain), attached to each row so downstream
+ * consumers inherit the terms rather than having to look them up.
+ */
+export const STATE_NY_NOTARIES_DEFAULT_LICENSE = "Public Domain"
+
+export function createStateNyNotariesAdapter(): CorpusAdapter {
+	return {
+		id: STATE_NY_NOTARIES_ADAPTER_ID,
+		defaultLicense: STATE_NY_NOTARIES_DEFAULT_LICENSE,
+		addressRole: AddressRole.Practice,
+		register: SourceRegister.NewYorkNotaries,
+		surface: SurfaceOrigin.Rendered,
+		description: "New York Commissioned Notaries — name + optional business address (public-domain).",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`state-ny-notaries adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				normalizeKeys: false,
+			})
+
+			let emitted = 0
+
+			for await (const record of rows as AsyncIterable<Record<string, string>>) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const holderName = record["Commission Holder Name"] ?? ""
+				const businessName = record["Business Name (if available)"] ?? ""
+				const address1 = record["Business Address 1 (if available)"] ?? ""
+				const address2 = record["Business Address 2 (if available)"] ?? ""
+				const city = record["Business City (if available)"] ?? ""
+				const stateAbbr = record["Business State (if available)"] ?? ""
+				const zip = record["Business Zip (if available)"] ?? ""
+				const county = record["Commissioned County"] ?? ""
+
+				if (!city || !stateAbbr || !zip) continue
+
+				if (!address1 && !address2) continue
+
+				const state = lookupStateAbbreviation(stateAbbr)
+
+				if (!state) continue
+
+				const fullAddress = [address1, address2].filter(isPresent).join(" ")
+				const split = splitStreetLine(fullAddress)
+
+				if (!split) continue
+
+				const venue = businessName || holderName || undefined
+
+				const components: CanonicalRow["components"] = {
+					...(venue ? { venue } : {}),
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode: zip,
+					...(county ? { subregion: county } : {}),
+				}
+
+				const rendered = formatAddressRow(components, "US", { singleLine: true })
+
+				if (!rendered) continue
+
+				const { raw, components: aligned } = rendered
+
+				if (Object.keys(aligned).length <= 2) continue
+
+				const commNum = record["Commission Number (UID)"] ?? ""
+
+				const sourceID = commNum
+					? `${STATE_NY_NOTARIES_ADAPTER_ID}-${commNum}`
+					: stableSourceID(STATE_NY_NOTARIES_ADAPTER_ID, aligned)
+
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: STATE_NY_NOTARIES_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: STATE_NY_NOTARIES_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const stateNyNotariesAdapter = createStateNyNotariesAdapter()