@mailwoman/corpus 8.6.0 → 9.1.0

package/src/adapters/geonames/adapter.ts

@@ -28,14 +28,13 @@
  *   License: stamped `"CC-BY-4.0"` per row (GeoNames' terms); provenance is the `geonames-<id>` key.
  */
 
-import { createReadStream, existsSync, readFileSync } from "node:fs"
-import { dirname, join } from "node:path"
+import { existsSync } from "node:fs"
 
-import { parse as csvParse } from "csv-parse"
-
-import { stableSourceID } from "../../adapter.ts"
-import { reconcileComponents } from "../../format.ts"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
+import { reconcileComponents } from "@mailwoman/formatter"
+import { dirname, join } from "path-ts"
+import { TSVSpliterator } from "spliterator"
 
 /**
  * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
@@ -69,16 +68,14 @@ const NON_CURRENT_PPL = new Set(["PPLH", "PPLQ", "PPLW", "PPLCH"])
 /**
  * Load `admin1CodesASCII.txt` → Map("<CC>.<admin1>" → region name). Empty map if absent.
  */
-function loadAdmin1(dir: string): Map<string, string> {
+async function loadAdmin1(dir: string): Promise<Map<string, string>> {
 	const map = new Map<string, string>()
 	const fp = join(dir, "admin1CodesASCII.txt")
 
 	if (!existsSync(fp)) return map
 
-	for (const line of readFileSync(fp, "utf8").split("\n")) {
-		if (!line) continue
-		const cols = line.split("\t")
-
+	// `header: false` — the file is headerless, and the spliterator eats row 1 as a header otherwise.
+	for await (const cols of TSVSpliterator.fromAsync(fp, { header: false })) {
 		if (cols[0] && cols[1]) {
 			map.set(cols[0], cols[1])
 		}
@@ -90,15 +87,16 @@ function loadAdmin1(dir: string): Map<string, string> {
 /**
  * Load `countryInfo.txt` → Map(ISO → country name). Empty map if absent. The file is `#`-commented.
  */
-function loadCountries(dir: string): Map<string, string> {
+async function loadCountries(dir: string): Promise<Map<string, string>> {
 	const map = new Map<string, string>()
 	const fp = join(dir, "countryInfo.txt")
 
 	if (!existsSync(fp)) return map
 
-	for (const line of readFileSync(fp, "utf8").split("\n")) {
-		if (!line || line.startsWith("#")) continue
-		const cols = line.split("\t")
+	// `header: false` — the file's header IS a `#` comment, so it falls out with the other comments
+	// rather than being consumed as column names.
+	for await (const cols of TSVSpliterator.fromAsync(fp, { header: false })) {
+		if (cols[0]?.startsWith("#")) continue
 
 		// ISO(0), ISO3(1), iso-numeric(2), fips(3), Country(4), ...
 		if (cols[0] && cols[4]) {
@@ -118,85 +116,78 @@ export function createGeonamesAdapter(): CorpusAdapter {
 
 		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
 			const dir = dirname(opts.inputPath)
-			const admin1 = loadAdmin1(dir)
-			const countries = loadCountries(dir)
-
-			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
+			const admin1 = await loadAdmin1(dir)
+			const countries = await loadCountries(dir)
 
-			const parser = stream.pipe(
-				csvParse({ delimiter: "\t", quote: false, relax_column_count: true, skip_empty_lines: true })
-			)
+			// `header: false` — the per-country dump is headerless.
+			const rows = TSVSpliterator.fromAsync(opts.inputPath, { header: false })
 
 			let emitted = 0
 
-			try {
-				for await (const rec of parser as AsyncIterable<string[]>) {
-					if (opts.signal?.aborted) break
+			for await (const rec of rows) {
+				if (opts.signal?.aborted) break
 
-					if (opts.limit !== undefined && emitted >= opts.limit) break
+				if (opts.limit !== undefined && emitted >= opts.limit) break
 
-					if (rec[COL.featureClass] !== "P") continue
+				if (rec[COL.featureClass] !== "P") continue
 
-					if (NON_CURRENT_PPL.has(rec[COL.featureCode] ?? "")) continue
+				if (NON_CURRENT_PPL.has(rec[COL.featureCode] ?? "")) continue
 
-					const cc = (rec[COL.country] ?? "").trim()
+				const cc = (rec[COL.country] ?? "").trim()
 
-					if (!cc) continue
+				if (!cc) continue
 
-					if (opts.country && cc !== opts.country) continue
+				if (opts.country && cc !== opts.country) continue
 
-					const locality = (rec[COL.name] ?? "").trim()
+				const locality = (rec[COL.name] ?? "").trim()
 
-					if (!locality) continue
-					const geonameid = (rec[COL.geonameid] ?? "").trim()
-					const region = admin1.get(`${cc}.${(rec[COL.admin1] ?? "").trim()}`)
-					const country = countries.get(cc)
+				if (!locality) continue
+				const geonameid = (rec[COL.geonameid] ?? "").trim()
+				const region = admin1.get(`${cc}.${(rec[COL.admin1] ?? "").trim()}`)
+				const country = countries.get(cc)
 
-					// Two hierarchy variants (domestic + international order) — but only emit the
-					// distinct ones the available names support.
-					const variants: Array<{ slot: string; comp: CanonicalRow["components"]; raw: string }> = []
+				// Two hierarchy variants (domestic + international order) — but only emit the
+				// distinct ones the available names support.
+				const variants: Array<{ slot: string; comp: CanonicalRow["components"]; raw: string }> = []
 
-					if (region) {
-						variants.push({ slot: "lr", comp: { locality, region }, raw: `${locality}, ${region}` })
+				if (region) {
+					variants.push({ slot: "lr", comp: { locality, region }, raw: `${locality}, ${region}` })
 
-						if (country) {
-							variants.push({
-								slot: "lrc",
-								comp: { locality, region, country },
-								raw: `${locality}, ${region}, ${country}`,
-							})
-						}
-					} else if (country) {
-						variants.push({ slot: "lc", comp: { locality, country }, raw: `${locality}, ${country}` })
-					} else {
-						variants.push({ slot: "l", comp: { locality }, raw: locality })
+					if (country) {
+						variants.push({
+							slot: "lrc",
+							comp: { locality, region, country },
+							raw: `${locality}, ${region}, ${country}`,
+						})
 					}
+				} else if (country) {
+					variants.push({ slot: "lc", comp: { locality, country }, raw: `${locality}, ${country}` })
+				} else {
+					variants.push({ slot: "l", comp: { locality }, raw: locality })
+				}
 
-					for (const v of variants) {
-						if (opts.limit !== undefined && emitted >= opts.limit) break
-						const aligned = reconcileComponents(v.comp, v.raw)
-
-						if (!Object.keys(aligned).length) continue
-
-						const sourceID = geonameid
-							? `${GEONAMES_ADAPTER_ID}-${geonameid}-${v.slot}`
-							: stableSourceID(GEONAMES_ADAPTER_ID, aligned)
-
-						yield {
-							raw: v.raw,
-							components: aligned,
-							country: cc,
-							source: GEONAMES_ADAPTER_ID,
-							source_id: sourceID,
-							corpus_version: "",
-							license: GEONAMES_DEFAULT_LICENSE,
-						}
-
-						emitted++
+				for (const v of variants) {
+					if (opts.limit !== undefined && emitted >= opts.limit) break
+					const aligned = reconcileComponents(v.comp, v.raw)
+
+					if (!Object.keys(aligned).length) continue
+
+					const sourceID = geonameid
+						? `${GEONAMES_ADAPTER_ID}-${geonameid}-${v.slot}`
+						: stableSourceID(GEONAMES_ADAPTER_ID, aligned)
+
+					yield {
+						raw: v.raw,
+						components: aligned,
+						country: cc,
+						source: GEONAMES_ADAPTER_ID,
+						source_id: sourceID,
+						corpus_version: "",
+						license: GEONAMES_DEFAULT_LICENSE,
 					}
+
+					emitted++
 				}
-			} finally {
-				stream.destroy()
 			}
 		},
 	}

package/src/adapters/geonames-postal/adapter.ts

@@ -23,13 +23,10 @@
  *        `"CC-BY-4.0"` per row (attribute "GeoNames").
  */
 
-import { createReadStream } from "node:fs"
-
-import { parse as csvParse } from "csv-parse"
-
-import { stableSourceID } from "../../adapter.ts"
-import { reconcileComponents } from "../../format.ts"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
+import { reconcileComponents } from "@mailwoman/formatter"
+import { TSVSpliterator } from "spliterator"
 
 /**
  * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
@@ -55,67 +52,61 @@ export function createGeonamesPostalAdapter(): CorpusAdapter {
 			"GeoNames postal codes (CC-BY-4.0) — multi-locale postcode→locality→region, names inline; international postcode-first order.",
 
 		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
-
-			const parser = stream.pipe(
-				csvParse({ delimiter: "\t", quote: false, relax_column_count: true, skip_empty_lines: true })
-			)
+			// `header: false` — the GeoNames postal dump is headerless, and the spliterator would
+			// otherwise consume row 1 as column names and lose its first postcode.
+			const rows = TSVSpliterator.fromAsync(opts.inputPath, { header: false })
 
 			let emitted = 0
 
-			try {
-				for await (const rec of parser as AsyncIterable<string[]>) {
-					if (opts.signal?.aborted) break
+			for await (const rec of rows as AsyncIterable<string[]>) {
+				if (opts.signal?.aborted) break
 
-					if (opts.limit !== undefined && emitted >= opts.limit) break
-
-					const cc = (rec[COL.country] ?? "").trim()
+				if (opts.limit !== undefined && emitted >= opts.limit) break
 
-					if (!cc) continue
+				const cc = (rec[COL.country] ?? "").trim()
 
-					if (opts.country && cc !== opts.country) continue
+				if (!cc) continue
 
-					const postcode = (rec[COL.postcode] ?? "").trim()
-					const locality = (rec[COL.place] ?? "").trim()
+				if (opts.country && cc !== opts.country) continue
 
-					if (!postcode || !locality) continue
-					const region = (rec[COL.admin1Name] ?? "").trim()
+				const postcode = (rec[COL.postcode] ?? "").trim()
+				const locality = (rec[COL.place] ?? "").trim()
 
-					// Postcode-first (international) variants. Skip the region variant when admin1 just
-					// repeats the place (common for city-states / micro-admin) to avoid "X X" noise.
-					const variants: Array<{ slot: string; comp: CanonicalRow["components"]; raw: string }> = [
-						{ slot: "pl", comp: { postcode, locality }, raw: `${postcode} ${locality}` },
-					]
+				if (!postcode || !locality) continue
+				const region = (rec[COL.admin1Name] ?? "").trim()
 
-					if (region && region.toLowerCase() !== locality.toLowerCase()) {
-						variants.push({
-							slot: "plr",
-							comp: { postcode, locality, region },
-							raw: `${postcode} ${locality}, ${region}`,
-						})
-					}
+				// Postcode-first (international) variants. Skip the region variant when admin1 just
+				// repeats the place (common for city-states / micro-admin) to avoid "X X" noise.
+				const variants: Array<{ slot: string; comp: CanonicalRow["components"]; raw: string }> = [
+					{ slot: "pl", comp: { postcode, locality }, raw: `${postcode} ${locality}` },
+				]
 
-					for (const v of variants) {
-						if (opts.limit !== undefined && emitted >= opts.limit) break
-						const aligned = reconcileComponents(v.comp, v.raw)
-
-						if (Object.keys(aligned).length < 2) continue
-
-						yield {
-							raw: v.raw,
-							components: aligned,
-							country: cc,
-							source: GEONAMES_POSTAL_ADAPTER_ID,
-							source_id: `${stableSourceID(GEONAMES_POSTAL_ADAPTER_ID, aligned)}-${v.slot}`,
-							corpus_version: "",
-							license: GEONAMES_POSTAL_DEFAULT_LICENSE,
-						}
+				if (region && region.toLowerCase() !== locality.toLowerCase()) {
+					variants.push({
+						slot: "plr",
+						comp: { postcode, locality, region },
+						raw: `${postcode} ${locality}, ${region}`,
+					})
+				}
 
-						emitted++
+				for (const v of variants) {
+					if (opts.limit !== undefined && emitted >= opts.limit) break
+					const aligned = reconcileComponents(v.comp, v.raw)
+
+					if (Object.keys(aligned).length < 2) continue
+
+					yield {
+						raw: v.raw,
+						components: aligned,
+						country: cc,
+						source: GEONAMES_POSTAL_ADAPTER_ID,
+						source_id: `${stableSourceID(GEONAMES_POSTAL_ADAPTER_ID, aligned)}-${v.slot}`,
+						corpus_version: "",
+						license: GEONAMES_POSTAL_DEFAULT_LICENSE,
 					}
+
+					emitted++
 				}
-			} finally {
-				stream.destroy()
 			}
 		},
 	}

package/src/adapters/gnaf/adapter.ts

@@ -28,12 +28,12 @@
  *   line). Open G-NAF licence — attribute "Geoscape Australia".
  */
 
+import { tryParsingJSON } from "@mailwoman/core/objects"
+import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
+import { reconcileComponents } from "@mailwoman/formatter"
 import { TextSpliterator } from "spliterator"
 
-import { stableSourceID } from "../../adapter.ts"
-import { reconcileComponents } from "../../format.ts"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
-
 /**
  * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
  * it came from.
@@ -87,7 +87,7 @@ export function createGNAFAdapter(): CorpusAdapter {
 
 			// Input is the assembled component JSONL (one tuple per line). TextSpliterator auto-disposes on
 			// loop completion and on an early `break` (abort / limit), so the old explicit handle teardown is
-			// gone; JSON.parse tolerates a trailing CR on CRLF sources and the `!line.trim()` guard skips blanks.
+			// gone; the parse tolerates a trailing CR on CRLF sources and the `!line.trim()` guard skips blanks.
 			// The render order rotates (i % 3), matching v1.9.1's rerender.
 			for await (const line of TextSpliterator.fromAsync(opts.inputPath)) {
 				if (opts.signal?.aborted) break
@@ -96,13 +96,9 @@ export function createGNAFAdapter(): CorpusAdapter {
 
 				if (!line.trim()) continue
 
-				let t: GNAFTuple
+				const t = tryParsingJSON<GNAFTuple>(line)
 
-				try {
-					t = JSON.parse(line) as GNAFTuple
-				} catch {
-					continue
-				}
+				if (t === null) continue
 
 				if (!t.house_number || !t.street || !t.locality || !t.postcode) continue
 

package/src/adapters/gnaf/assemble.ts

@@ -27,10 +27,11 @@
  */
 
 import { createWriteStream } from "node:fs"
-import { readdir, readFile } from "node:fs/promises"
+import { readdir } from "node:fs/promises"
 import { join } from "node:path"
 
-import { PSVSpliterator } from "spliterator"
+import { tryParsingJSON } from "@mailwoman/core/objects"
+import { PSVSpliterator, TextSpliterator } from "spliterator"
 
 export interface GNAFAssembleOptions {
 	/**
@@ -109,19 +110,14 @@ async function loadMap<V>(paths: string[], keyCol: string, pick: (r: Row) => V):
  */
 async function loadHoldout(path: string): Promise<Set<string>> {
 	const keys = new Set<string>()
-	const text = await readFile(path, "utf8")
 
-	for (const line of text.split("\n")) {
+	for await (const line of TextSpliterator.fromAsync(path)) {
 		if (!line.trim()) continue
 
-		try {
-			const c = (JSON.parse(line) as { components?: Record<string, string> }).components
+		const c = tryParsingJSON<{ components?: Record<string, string> }>(line)?.components
 
-			if (c?.street && c?.locality && c?.postcode) {
-				keys.add(gnafHoldoutKey(c.street, c.locality, c.postcode))
-			}
-		} catch {
-			/* skip malformed */
+		if (c?.street && c?.locality && c?.postcode) {
+			keys.add(gnafHoldoutKey(c.street, c.locality, c.postcode))
 		}
 	}
 

package/src/adapters/index.ts

@@ -19,8 +19,9 @@
  *   implementations; the registered ids are unchanged).
  */
 
-import { defaultAdapterRegistry } from "../adapter.ts"
-import type { CorpusAdapter } from "../types.ts"
+import { defaultAdapterRegistry } from "@mailwoman/corpus/adapters/utils"
+import type { CorpusAdapter } from "@mailwoman/corpus/types"
+
 import { banAdapter } from "./ban/adapter.ts"
 import { fccBdcAdapter } from "./fcc-bdc/adapter.ts"
 import { geonamesPostalAdapter } from "./geonames-postal/adapter.ts"

package/src/adapters/openaddresses/adapter.ts

@@ -36,13 +36,13 @@
  *   for non-US locales where district names DO appear on the envelope.
  */
 
+import { tryParsingJSON } from "@mailwoman/core/objects"
+import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
+import { SHARE_ALIKE_PATTERN } from "@mailwoman/corpus/utils"
+import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
 import { TextSpliterator } from "spliterator"
 
-import { stableSourceID } from "../../adapter.ts"
-import { formatAddress, reconcileComponents } from "../../format.ts"
-import { SHARE_ALIKE_PATTERN } from "../../license.ts"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
-
 /**
  * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
  * it came from.
@@ -96,13 +96,7 @@ function parseFeatureLine(line: string): OaProperties | null {
 	const trimmed = line.trim()
 
 	if (!trimmed || trimmed.startsWith("#")) return null
-	let parsed: unknown
-
-	try {
-		parsed = JSON.parse(trimmed)
-	} catch {
-		return null
-	}
+	const parsed = tryParsingJSON(trimmed)
 
 	if (!parsed || typeof parsed !== "object") return null
 	const obj = parsed as { type?: string; properties?: unknown }

package/src/adapters/overture/adapter.ts

@@ -30,12 +30,12 @@
  *   `postcode` | | `locality`| `locality` (Overture address_levels municipality, or postal_city) |
  */
 
+import { tryParsingJSON } from "@mailwoman/core/objects"
+import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
+import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
 import { TextSpliterator } from "spliterator"
 
-import { stableSourceID } from "../../adapter.ts"
-import { formatAddress, reconcileComponents } from "../../format.ts"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
-
 /**
  * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
  * it came from.
@@ -63,13 +63,9 @@ function parseLine(line: string): OvertureCorpusRow | null {
 
 	if (!t || t.startsWith("#")) return null
 
-	try {
-		const o = JSON.parse(t)
+	const o = tryParsingJSON(t)
 
-		return o && typeof o === "object" ? (o as OvertureCorpusRow) : null
-	} catch {
-		return null
-	}
+	return o && typeof o === "object" ? (o as OvertureCorpusRow) : null
 }
 
 export function createOvertureAdapter(): CorpusAdapter {