@mailwoman/corpus 8.6.0 → 9.1.0

package/src/adapters/state-hi-schools/adapter.ts

@@ -29,14 +29,11 @@
  *   License: stamped `"Public Domain"` per Hawaii state government open-data terms.
  */
 
-import { createReadStream } from "node:fs"
-
-import { parse as csvParse } from "csv-parse"
-
-import { stableSourceID } from "../../adapter.ts"
-import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
-import { reconcileComponents } from "../../format.ts"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+import { splitStreetLine, stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import { lookupStateAbbreviation } from "@mailwoman/corpus/codex/us-fips-state"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
+import { reconcileComponents } from "@mailwoman/formatter"
+import { CSVSpliterator } from "spliterator"
 
 /**
  * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
@@ -49,7 +46,6 @@ export const STATE_HI_SCHOOLS_ADAPTER_ID = "state-hi-schools"
  */
 export const STATE_HI_SCHOOLS_DEFAULT_LICENSE = "Public Domain"
 
-const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
 const HI_STATE_ABBR = "HI"
 
 interface HiSchoolRow {
@@ -60,17 +56,6 @@ interface HiSchoolRow {
 	zip: string
 }
 
-function splitAddress(address: string): { house_number?: string; street: string } | null {
-	const trimmed = address.trim()
-
-	if (!trimmed) return null
-	const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
-
-	if (m) return { house_number: m[1], street: m[2]!.trim() }
-
-	return { street: trimmed }
-}
-
 function normalizeZip(raw: string): string {
 	const trimmed = raw.trim()
 
@@ -95,16 +80,11 @@ export function createStateHiSchoolsAdapter(): CorpusAdapter {
 				throw new Error(`state-hi-schools adapter: only US supported, got country=${opts.country}`)
 			}
 
-			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
-
-			const parser = stream.pipe(
-				csvParse({
-					columns: true,
-					skip_empty_lines: true,
-					relax_quotes: true,
-					relax_column_count: true,
-				})
-			)
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				mode: "object",
+				normalizeKeys: false,
+				enableQuoteHandling: true,
+			})
 
 			const state = lookupStateAbbreviation(HI_STATE_ABBR)
 
@@ -114,67 +94,63 @@ export function createStateHiSchoolsAdapter(): CorpusAdapter {
 
 			let emitted = 0
 
-			try {
-				for await (const record of parser as AsyncIterable<HiSchoolRow>) {
-					if (opts.signal?.aborted) break
-
-					if (opts.limit !== undefined && emitted >= opts.limit) break
+			for await (const record of rows as AsyncIterable<HiSchoolRow>) {
+				if (opts.signal?.aborted) break
 
-					const name = (record.name ?? "").trim()
-					const address = (record.address ?? "").trim()
-					const city = (record.city ?? "").trim()
-					const zip = normalizeZip(record.zip ?? "")
+				if (opts.limit !== undefined && emitted >= opts.limit) break
 
-					if (!name || !address || !city || !zip) continue
+				const name = (record.name ?? "").trim()
+				const address = (record.address ?? "").trim()
+				const city = (record.city ?? "").trim()
+				const zip = normalizeZip(record.zip ?? "")
 
-					const split = splitAddress(address)
+				if (!name || !address || !city || !zip) continue
 
-					if (!split) continue
+				const split = splitStreetLine(address)
 
-					const components: CanonicalRow["components"] = {
-						venue: name,
-						...(split.house_number ? { house_number: split.house_number } : {}),
-						street: split.street,
-						locality: city,
-						region: state.abbreviation,
-						postcode: zip,
-					}
+				if (!split) continue
 
-					const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
+				const components: CanonicalRow["components"] = {
+					venue: name,
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode: zip,
+				}
 
-					const raw = [
-						name,
-						streetPart,
-						[city, [HI_STATE_ABBR, zip].filter(Boolean).join(" ")].filter(Boolean).join(", "),
-					]
-						.filter(Boolean)
-						.join(", ")
+				const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
 
-					const aligned = reconcileComponents(components, raw)
+				const raw = [
+					name,
+					streetPart,
+					[city, [HI_STATE_ABBR, zip].filter(Boolean).join(" ")].filter(Boolean).join(", "),
+				]
+					.filter(Boolean)
+					.join(", ")
 
-					if (Object.keys(aligned).length <= 2) continue
+				const aligned = reconcileComponents(components, raw)
 
-					const code = (record.code ?? "").toString().trim()
+				if (Object.keys(aligned).length <= 2) continue
 
-					const sourceID = code
-						? `${STATE_HI_SCHOOLS_ADAPTER_ID}-${code}`
-						: stableSourceID(STATE_HI_SCHOOLS_ADAPTER_ID, aligned)
+				const code = (record.code ?? "").toString().trim()
 
-					yield {
-						raw,
-						components: aligned,
-						country: "US",
-						locale: "en-US",
-						source: STATE_HI_SCHOOLS_ADAPTER_ID,
-						source_id: sourceID,
-						corpus_version: "",
-						license: STATE_HI_SCHOOLS_DEFAULT_LICENSE,
-					}
+				const sourceID = code
+					? `${STATE_HI_SCHOOLS_ADAPTER_ID}-${code}`
+					: stableSourceID(STATE_HI_SCHOOLS_ADAPTER_ID, aligned)
 
-					emitted++
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: STATE_HI_SCHOOLS_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: STATE_HI_SCHOOLS_DEFAULT_LICENSE,
 				}
-			} finally {
-				stream.destroy()
+
+				emitted++
 			}
 		},
 	}

package/src/adapters/state-ia-contractors/adapter.ts

@@ -16,14 +16,11 @@
  *   License: stamped `"Public Domain"` per Iowa state government open-data terms.
  */
 
-import { createReadStream } from "node:fs"
-
-import { parse as csvParse } from "csv-parse"
-
-import { stableSourceID } from "../../adapter.ts"
-import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
-import { reconcileComponents } from "../../format.ts"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+import { splitStreetLine, stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import { lookupStateAbbreviation } from "@mailwoman/corpus/codex/us-fips-state"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
+import { reconcileComponents } from "@mailwoman/formatter"
+import { CSVSpliterator } from "spliterator"
 
 /**
  * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
@@ -36,8 +33,6 @@ export const STATE_IA_CONTRACTORS_ADAPTER_ID = "state-ia-contractors"
  */
 export const STATE_IA_CONTRACTORS_DEFAULT_LICENSE = "Public Domain"
 
-const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
-
 interface IaContractorRow {
 	"Registration #": string
 	"Business Name": string
@@ -50,17 +45,6 @@ interface IaContractorRow {
 	"Last Name": string
 }
 
-function splitAddress(address: string): { house_number?: string; street: string } | null {
-	const trimmed = address.trim()
-
-	if (!trimmed) return null
-	const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
-
-	if (m) return { house_number: m[1], street: m[2]!.trim() }
-
-	return { street: trimmed }
-}
-
 export function createStateIaContractorsAdapter(): CorpusAdapter {
 	return {
 		id: STATE_IA_CONTRACTORS_ADAPTER_ID,
@@ -73,85 +57,76 @@ export function createStateIaContractorsAdapter(): CorpusAdapter {
 				throw new Error(`state-ia-contractors adapter: only US supported, got country=${opts.country}`)
 			}
 
-			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
-
-			const parser = stream.pipe(
-				csvParse({
-					columns: true,
-					skip_empty_lines: true,
-					relax_quotes: true,
-					relax_column_count: true,
-				})
-			)
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				mode: "object",
+				normalizeKeys: false,
+				enableQuoteHandling: true,
+			})
 
 			let emitted = 0
 
-			try {
-				for await (const record of parser as AsyncIterable<IaContractorRow>) {
-					if (opts.signal?.aborted) break
+			for await (const record of rows as AsyncIterable<IaContractorRow>) {
+				if (opts.signal?.aborted) break
 
-					if (opts.limit !== undefined && emitted >= opts.limit) break
+				if (opts.limit !== undefined && emitted >= opts.limit) break
 
-					const businessName = (record["Business Name"] ?? "").trim()
-					const address1 = (record["Address 1"] ?? "").trim()
-					const address2 = (record["Address 2"] ?? "").trim()
-					const city = (record.City ?? "").trim()
-					const stateAbbr = (record.State ?? "").trim()
-					const zip = (record["Zip Code"] ?? "").trim()
+				const businessName = (record["Business Name"] ?? "").trim()
+				const address1 = (record["Address 1"] ?? "").trim()
+				const address2 = (record["Address 2"] ?? "").trim()
+				const city = (record.City ?? "").trim()
+				const stateAbbr = (record.State ?? "").trim()
+				const zip = (record["Zip Code"] ?? "").trim()
 
-					if (!city || !zip) continue
+				if (!city || !zip) continue
 
-					const state = lookupStateAbbreviation(stateAbbr)
+				const state = lookupStateAbbreviation(stateAbbr)
 
-					if (!state) continue
+				if (!state) continue
 
-					const fullAddress = [address1, address2].filter(Boolean).join(" ")
-					const split = splitAddress(fullAddress)
+				const fullAddress = [address1, address2].filter(Boolean).join(" ")
+				const split = splitStreetLine(fullAddress)
 
-					if (!split) continue
+				if (!split) continue
 
-					const venue = businessName || undefined
+				const venue = businessName || undefined
 
-					const components: CanonicalRow["components"] = {
-						...(venue ? { venue } : {}),
-						...(split.house_number ? { house_number: split.house_number } : {}),
-						street: split.street,
-						locality: city,
-						region: state.abbreviation,
-						postcode: zip,
-					}
-
-					const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
+				const components: CanonicalRow["components"] = {
+					...(venue ? { venue } : {}),
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode: zip,
+				}
 
-					const raw = [venue, streetPart, [city, [stateAbbr, zip].filter(Boolean).join(" ")].filter(Boolean).join(", ")]
-						.filter(Boolean)
-						.join(", ")
+				const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
 
-					const aligned = reconcileComponents(components, raw)
+				const raw = [venue, streetPart, [city, [stateAbbr, zip].filter(Boolean).join(" ")].filter(Boolean).join(", ")]
+					.filter(Boolean)
+					.join(", ")
 
-					if (Object.keys(aligned).length <= 2) continue
+				const aligned = reconcileComponents(components, raw)
 
-					const regNum = (record["Registration #"] ?? "").trim()
+				if (Object.keys(aligned).length <= 2) continue
 
-					const sourceID = regNum
-						? `${STATE_IA_CONTRACTORS_ADAPTER_ID}-${regNum}`
-						: stableSourceID(STATE_IA_CONTRACTORS_ADAPTER_ID, aligned)
+				const regNum = (record["Registration #"] ?? "").trim()
 
-					yield {
-						raw,
-						components: aligned,
-						country: "US",
-						locale: "en-US",
-						source: STATE_IA_CONTRACTORS_ADAPTER_ID,
-						source_id: sourceID,
-						corpus_version: "",
-						license: STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
-					}
+				const sourceID = regNum
+					? `${STATE_IA_CONTRACTORS_ADAPTER_ID}-${regNum}`
+					: stableSourceID(STATE_IA_CONTRACTORS_ADAPTER_ID, aligned)
 
-					emitted++
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: STATE_IA_CONTRACTORS_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
 				}
-			} finally {
-				stream.destroy()
+
+				emitted++
 			}
 		},
 	}

package/src/adapters/state-ny-notaries/adapter.ts

@@ -14,14 +14,11 @@
  *   License: stamped `"Public Domain"` per New York state government open-data terms.
  */
 
-import { createReadStream } from "node:fs"
-
-import { parse as csvParse } from "csv-parse"
-
-import { stableSourceID } from "../../adapter.ts"
-import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
-import { reconcileComponents } from "../../format.ts"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+import { splitStreetLine, stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import { lookupStateAbbreviation } from "@mailwoman/corpus/codex/us-fips-state"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
+import { reconcileComponents } from "@mailwoman/formatter"
+import { CSVSpliterator } from "spliterator"
 
 /**
  * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
@@ -34,8 +31,6 @@ export const STATE_NY_NOTARIES_ADAPTER_ID = "state-ny-notaries"
  */
 export const STATE_NY_NOTARIES_DEFAULT_LICENSE = "Public Domain"
 
-const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
-
 interface NyNotaryRow {
 	"Commission Holder Name": string
 	"Commission Number (UID)": string
@@ -48,17 +43,6 @@ interface NyNotaryRow {
 	"Commissioned County": string
 }
 
-function splitAddress(address: string): { house_number?: string; street: string } | null {
-	const trimmed = address.trim()
-
-	if (!trimmed) return null
-	const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
-
-	if (m) return { house_number: m[1], street: m[2]!.trim() }
-
-	return { street: trimmed }
-}
-
 const RAW_NY_COLUMNS = [
 	"Commission Holder Name",
 	"Commission Number (UID)",
@@ -82,97 +66,88 @@ export function createStateNyNotariesAdapter(): CorpusAdapter {
 				throw new Error(`state-ny-notaries adapter: only US supported, got country=${opts.country}`)
 			}
 
-			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
-
-			const parser = stream.pipe(
-				csvParse({
-					columns: true,
-					skip_empty_lines: true,
-					relax_quotes: true,
-					relax_column_count: true,
-				})
-			)
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				mode: "object",
+				normalizeKeys: false,
+				enableQuoteHandling: true,
+			})
 
 			let emitted = 0
 
-			try {
-				for await (const rawRecord of parser as AsyncIterable<Record<string, string>>) {
-					if (opts.signal?.aborted) break
-
-					if (opts.limit !== undefined && emitted >= opts.limit) break
+			for await (const rawRecord of rows as AsyncIterable<Record<string, string>>) {
+				if (opts.signal?.aborted) break
 
-					// NY CSV has columns with leading spaces, so we normalize by trimming keys.
-					const record: Record<string, string> = {}
+				if (opts.limit !== undefined && emitted >= opts.limit) break
 
-					for (const key of Object.keys(rawRecord)) {
-						record[key.trim()] = rawRecord[key] ?? ""
-					}
+				// NY CSV has columns with leading spaces, so we normalize by trimming keys.
+				const record: Record<string, string> = {}
 
-					const holderName = (record["Commission Holder Name"] ?? "").trim()
-					const businessName = (record["Business Name (if available)"] ?? "").trim()
-					const address1 = (record["Business Address 1 (if available)"] ?? "").trim()
-					const address2 = (record["Business Address 2 (if available)"] ?? "").trim()
-					const city = (record["Business City (if available)"] ?? "").trim()
-					const stateAbbr = (record["Business State (if available)"] ?? "").trim()
-					const zip = (record["Business Zip (if available)"] ?? "").trim()
-					const county = (record["Commissioned County"] ?? "").trim()
+				for (const key of Object.keys(rawRecord)) {
+					record[key.trim()] = rawRecord[key] ?? ""
+				}
 
-					if (!city || !stateAbbr || !zip) continue
+				const holderName = (record["Commission Holder Name"] ?? "").trim()
+				const businessName = (record["Business Name (if available)"] ?? "").trim()
+				const address1 = (record["Business Address 1 (if available)"] ?? "").trim()
+				const address2 = (record["Business Address 2 (if available)"] ?? "").trim()
+				const city = (record["Business City (if available)"] ?? "").trim()
+				const stateAbbr = (record["Business State (if available)"] ?? "").trim()
+				const zip = (record["Business Zip (if available)"] ?? "").trim()
+				const county = (record["Commissioned County"] ?? "").trim()
 
-					if (!address1 && !address2) continue
+				if (!city || !stateAbbr || !zip) continue
 
-					const state = lookupStateAbbreviation(stateAbbr)
+				if (!address1 && !address2) continue
 
-					if (!state) continue
+				const state = lookupStateAbbreviation(stateAbbr)
 
-					const fullAddress = [address1, address2].filter(Boolean).join(" ")
-					const split = splitAddress(fullAddress)
+				if (!state) continue
 
-					if (!split) continue
+				const fullAddress = [address1, address2].filter(Boolean).join(" ")
+				const split = splitStreetLine(fullAddress)
 
-					const venue = businessName || holderName || undefined
+				if (!split) continue
 
-					const components: CanonicalRow["components"] = {
-						...(venue ? { venue } : {}),
-						...(split.house_number ? { house_number: split.house_number } : {}),
-						street: split.street,
-						locality: city,
-						region: state.abbreviation,
-						postcode: zip,
-						...(county ? { subregion: county } : {}),
-					}
+				const venue = businessName || holderName || undefined
 
-					const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
+				const components: CanonicalRow["components"] = {
+					...(venue ? { venue } : {}),
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode: zip,
+					...(county ? { subregion: county } : {}),
+				}
 
-					const raw = [venue, streetPart, [city, [stateAbbr, zip].filter(Boolean).join(" ")].filter(Boolean).join(", ")]
-						.filter(Boolean)
-						.join(", ")
+				const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
 
-					const aligned = reconcileComponents(components, raw)
+				const raw = [venue, streetPart, [city, [stateAbbr, zip].filter(Boolean).join(" ")].filter(Boolean).join(", ")]
+					.filter(Boolean)
+					.join(", ")
 
-					if (Object.keys(aligned).length <= 2) continue
+				const aligned = reconcileComponents(components, raw)
 
-					const commNum = (record["Commission Number (UID)"] ?? "").trim()
+				if (Object.keys(aligned).length <= 2) continue
 
-					const sourceID = commNum
-						? `${STATE_NY_NOTARIES_ADAPTER_ID}-${commNum}`
-						: stableSourceID(STATE_NY_NOTARIES_ADAPTER_ID, aligned)
+				const commNum = (record["Commission Number (UID)"] ?? "").trim()
 
-					yield {
-						raw,
-						components: aligned,
-						country: "US",
-						locale: "en-US",
-						source: STATE_NY_NOTARIES_ADAPTER_ID,
-						source_id: sourceID,
-						corpus_version: "",
-						license: STATE_NY_NOTARIES_DEFAULT_LICENSE,
-					}
+				const sourceID = commNum
+					? `${STATE_NY_NOTARIES_ADAPTER_ID}-${commNum}`
+					: stableSourceID(STATE_NY_NOTARIES_ADAPTER_ID, aligned)
 
-					emitted++
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: STATE_NY_NOTARIES_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: STATE_NY_NOTARIES_DEFAULT_LICENSE,
 				}
-			} finally {
-				stream.destroy()
+
+				emitted++
 			}
 		},
 	}