@mailwoman/corpus 7.2.0 → 7.3.0

package/src/adapters/state-tx-notaries/adapter.ts

@@ -0,0 +1,160 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `state-tx-notaries`: Texas Notary Public Commissions CSV consumer.
+ *
+ *   The Texas Secretary of State publishes a registry of commissioned notaries public. Each row
+ *   optionally carries a mailing address in free-form text (often multi-line with embedded
+ *   city/state/zip). Address fill rate is ~5-10%.
+ *
+ *   The adapter parses the embedded `Address` field for city/state/zip using a trailing `"CITY, ST
+ *   ZIP"` pattern.
+ *
+ *   License: stamped `"Public Domain"` per Texas state government open-data terms.
+ */
+
+import { createReadStream } from "node:fs"
+
+import { parse as csvParse } from "csv-parse"
+
+import { stableSourceID } from "../../adapter.ts"
+import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
+import { reconcileComponents } from "../../format.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+
+export const STATE_TX_NOTARIES_ADAPTER_ID = "state-tx-notaries"
+export const STATE_TX_NOTARIES_DEFAULT_LICENSE = "Public Domain"
+
+const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
+
+/** Match trailing "CITY, ST ZIP" or "CITY, ST" at the end of an address line. */
+const CITY_STATE_ZIP_SUFFIX = /[,]?\s*([^,]+),\s*([A-Z]{2})\s*(\d{5}(?:-\d{4})?)?\s*$/i
+
+interface TxNotaryRow {
+	"Notary ID": string
+	"First Name": string
+	"Last Name": string
+	Address: string
+}
+
+function splitAddress(address: string): { house_number?: string; street: string } | null {
+	const trimmed = address.trim()
+
+	if (!trimmed) return null
+	const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
+
+	if (m) return { house_number: m[1], street: m[2]!.trim() }
+
+	return { street: trimmed }
+}
+
+export function createStateTxNotariesAdapter(): CorpusAdapter {
+	return {
+		id: STATE_TX_NOTARIES_ADAPTER_ID,
+		defaultLicense: STATE_TX_NOTARIES_DEFAULT_LICENSE,
+		description:
+			"Texas Notary Public Commissions — name + mailing address with embedded city/state/zip (public-domain).",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`state-tx-notaries adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
+			const parser = stream.pipe(
+				csvParse({
+					columns: true,
+					skip_empty_lines: true,
+					relax_quotes: true,
+					relax_column_count: true,
+				})
+			)
+
+			let emitted = 0
+
+			try {
+				for await (const record of parser as AsyncIterable<TxNotaryRow>) {
+					if (opts.signal?.aborted) break
+
+					if (opts.limit !== undefined && emitted >= opts.limit) break
+
+					const rawAddress = (record.Address ?? "").trim()
+
+					if (!rawAddress) continue
+
+					const firstName = (record["First Name"] ?? "").trim()
+					const lastName = (record["Last Name"] ?? "").trim()
+					const notaryID = (record["Notary ID"] ?? "").trim()
+
+					// Parse embedded city/state/zip from the trailing portion of the address.
+					// Addresses look like: "1215 MCMILLAN DR\nCEDAR HILL, TX 75104"
+					const addrSingleLine = rawAddress.replace(/\n/g, ", ")
+					const cszMatch = CITY_STATE_ZIP_SUFFIX.exec(addrSingleLine)
+
+					if (!cszMatch) continue
+
+					const city = (cszMatch[1] ?? "").trim()
+					const stateAbbr = (cszMatch[2] ?? "").trim()
+					const zip = (cszMatch[3] ?? "").trim()
+
+					if (!city || !stateAbbr) continue
+
+					const state = lookupStateAbbreviation(stateAbbr)
+
+					if (!state) continue
+
+					// Extract the street portion (everything before the city/state/zip)
+					const streetPortion = addrSingleLine.slice(0, cszMatch.index).replace(/,\s*$/, "").trim()
+
+					if (!streetPortion) continue
+
+					const split = splitAddress(streetPortion)
+
+					if (!split) continue
+
+					const venue = [firstName, lastName].filter(Boolean).join(" ") || undefined
+
+					const components: CanonicalRow["components"] = {
+						...(venue ? { venue } : {}),
+						...(split.house_number ? { house_number: split.house_number } : {}),
+						street: split.street,
+						locality: city,
+						region: state.abbreviation,
+						...(zip ? { postcode: zip } : {}),
+					}
+
+					const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
+					const raw = [venue, streetPart, [city, [stateAbbr, zip].filter(Boolean).join(" ")].filter(Boolean).join(", ")]
+						.filter(Boolean)
+						.join(", ")
+
+					const aligned = reconcileComponents(components, raw)
+
+					if (Object.keys(aligned).length <= 2) continue
+
+					const sourceID = notaryID
+						? `${STATE_TX_NOTARIES_ADAPTER_ID}-${notaryID}`
+						: stableSourceID(STATE_TX_NOTARIES_ADAPTER_ID, aligned)
+
+					yield {
+						raw,
+						components: aligned,
+						country: "US",
+						locale: "en-US",
+						source: STATE_TX_NOTARIES_ADAPTER_ID,
+						source_id: sourceID,
+						corpus_version: "",
+						license: STATE_TX_NOTARIES_DEFAULT_LICENSE,
+					}
+					emitted++
+				}
+			} finally {
+				stream.destroy()
+			}
+		},
+	}
+}
+
+export const stateTxNotariesAdapter = createStateTxNotariesAdapter()

package/src/adapters/synth-po-box/adapter.ts

@@ -0,0 +1,197 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `synth-po-box`: PO box / PMB / Apartado / BP synthesizer adapter.
+ *
+ *   Consumes a JSONL stream of (locality, region, postcode, country) tuples — typically extracted
+ *   from existing corpus output (TIGER/NAD/BAN/WOF) — and emits synthetic PO box training rows. See
+ *   `../../synthesize-po-box.ts` for the per-locale templates and number-noise logic.
+ *
+ *   Why an adapter and not an augmenter:
+ *
+ *   - Per USPS Pub 28 / DMM 508, a PO box delivery line is mutually exclusive with a street line.
+ *       Synthesizing PO boxes by mutating a street row would teach the model an invalid pattern.
+ *       The clean shape is: read just (locality, region, postcode, country) and produce a fresh
+ *       PO-box-shaped row.
+ *   - Per-DeepSeek (3-turn consult, 2026-05-28): PMB rows that COMBINE a street line with a PMB number
+ *       ARE valid (CMRA addresses). Those are produced when `pmbRatio > 0` AND the input tuple
+ *       carries a `street` field.
+ */
+
+import { TextSpliterator } from "spliterator"
+
+import { stableSourceID } from "../../adapter.ts"
+import {
+	countryToLocale,
+	REGION_OPTIONAL_LOCALES,
+	synthesizeMilitaryPoBoxRow,
+	synthesizePoBoxRow,
+	type PoBoxBaseTuple,
+} from "../../synthesize-po-box.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+
+export const SYNTH_PO_BOX_ADAPTER_ID = "synth-po-box"
+export const SYNTH_PO_BOX_LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"
+
+export interface PoBoxInputRow extends PoBoxBaseTuple {
+	street?: string
+	houseNumber?: string
+}
+
+export interface SynthPoBoxAdapterOptions {
+	/**
+	 * How many PO box variants to emit per input tuple. Each variant picks a different leader (and possibly a different
+	 * number / noise level). Default 1.
+	 */
+	variantsPerInput?: number
+	/**
+	 * Probability (0..1) of emitting a PMB-with-street variant when both the input has a street and the locale supports
+	 * PMB. Default 0.15.
+	 */
+	pmbRatio?: number
+	/**
+	 * Deterministic seed for reproducible synthesis. Default Date.now().
+	 */
+	seed?: number
+	/**
+	 * Probability (0..1), evaluated per input tuple, of ALSO emitting one US military/diplomatic PO-box row
+	 * (`PSC/CMR/Unit <id> Box <box>, APO/FPO/DPO AA/AE/AP <zip>`, #517). These rows are self-contained — they draw no
+	 * field from the input tuple, so military volume scales with the input stream size. Default 0 (off) — the adapter's
+	 * contract is "one row per input"; the corpus build recipe opts in to seed the rare-but-real military class without
+	 * changing the default.
+	 */
+	militaryRatio?: number
+}
+
+function makeRandom(seed: number): () => number {
+	let s = seed
+
+	return () => {
+		s = (s * 1664525 + 1013904223) % 4294967296
+
+		return s / 4294967296
+	}
+}
+
+export function createSynthPoBoxAdapter(opts: SynthPoBoxAdapterOptions = {}): CorpusAdapter {
+	const variantsPerInput = opts.variantsPerInput ?? 1
+	const pmbRatio = opts.pmbRatio ?? 0.15
+	const militaryRatio = opts.militaryRatio ?? 0
+
+	return {
+		id: SYNTH_PO_BOX_ADAPTER_ID,
+		defaultLicense: SYNTH_PO_BOX_LICENSE,
+		description:
+			"Synthetic PO box / PMB / Apartado / Boîte Postale rows. Consumes JSONL of (locality, region, postcode, country) tuples and emits locale-appropriate PO box variants.",
+
+		async *rows(options: AdapterOptions): AsyncIterable<CanonicalRow> {
+			const random = makeRandom(opts.seed ?? Date.now())
+
+			// TextSpliterator streams string lines; the per-line try/catch below keeps this reader
+			// tolerant of malformed rows (skipped++), so TextSpliterator + explicit JSON.parse — not
+			// JSONSpliterator, which would throw on the first bad line.
+			const lines = TextSpliterator.fromAsync(options.inputPath)
+
+			let emitted = 0
+			let skipped = 0
+			let militarySeq = 0
+
+			for await (const line of lines) {
+				if (options.signal?.aborted) break
+
+				if (options.limit !== undefined && emitted >= options.limit) break
+
+				const trimmed = line.trim()
+
+				if (!trimmed) continue
+
+				let input: PoBoxInputRow
+
+				try {
+					input = JSON.parse(trimmed) as PoBoxInputRow
+				} catch {
+					skipped++
+					continue
+				}
+
+				// Region is required EXCEPT for region-less locales (NZ: `Private Bag 12, Auckland 1010`
+				// has no region token, #517). synthesizePoBoxRow handles region absence; the guard just
+				// must not discard those tuples as "missing region".
+				const regionOptional = input.country ? REGION_OPTIONAL_LOCALES.has(countryToLocale(input.country)) : false
+
+				if (!input.locality || !input.postcode || !input.country || (!input.region && !regionOptional)) {
+					skipped++
+					continue
+				}
+
+				if (options.country && options.country !== input.country) continue
+
+				for (let v = 0; v < variantsPerInput; v++) {
+					const synth = synthesizePoBoxRow(input, { random, pmbRatio })
+
+					if (!synth) continue
+
+					// Include `v` in dependent_locality slot to vary the digest across variants;
+					// stableSourceID only accepts ComponentTag keys.
+					const sourceID = stableSourceID(SYNTH_PO_BOX_ADAPTER_ID, {
+						locality: `${input.locality}#${v}`,
+						region: input.region,
+						postcode: input.postcode,
+						country: input.country,
+					})
+
+					yield {
+						raw: synth.raw,
+						components: synth.components,
+						country: input.country,
+						locale: synth.locale,
+						source: SYNTH_PO_BOX_ADAPTER_ID,
+						source_id: sourceID,
+						corpus_version: "",
+						license: SYNTH_PO_BOX_LICENSE,
+					}
+					emitted++
+
+					if (options.limit !== undefined && emitted >= options.limit) break
+				}
+
+				// US military/diplomatic PO-box rows (#517): self-contained — draw nothing from the input
+				// tuple — emitted per input line with probability `militaryRatio` (off by default, so the
+				// default random stream and output are byte-identical). Military volume scales with the
+				// stream rather than the US-tuple count. US-only: suppressed under a non-US country filter
+				// and counted against `limit` like any other row.
+				const militaryAllowed = !options.country || options.country === "US"
+
+				if (
+					militaryRatio > 0 &&
+					militaryAllowed &&
+					(options.limit === undefined || emitted < options.limit) &&
+					random() < militaryRatio
+				) {
+					const mil = synthesizeMilitaryPoBoxRow({ random })
+					const sourceID = stableSourceID(SYNTH_PO_BOX_ADAPTER_ID, {
+						po_box: `${mil.components.po_box}#mil${militarySeq++}`,
+						locality: mil.components.locality!,
+						region: mil.components.region!,
+						postcode: mil.components.postcode!,
+					})
+					yield {
+						raw: mil.raw,
+						components: mil.components,
+						country: "US",
+						locale: mil.locale,
+						source: SYNTH_PO_BOX_ADAPTER_ID,
+						source_id: sourceID,
+						corpus_version: "",
+						license: SYNTH_PO_BOX_LICENSE,
+					}
+					emitted++
+				}
+			}
+		},
+	}
+}
+
+export const synthPoBoxAdapter = createSynthPoBoxAdapter()

package/src/adapters/tiger/adapter.ts

@@ -0,0 +1,232 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `tiger`: US Census TIGER/Line consumer adapter.
+ *
+ *   TIGER/Line is the canonical US street + locality dataset published by the Census Bureau as a
+ *   **public-domain** product (no ODbL share-alike concerns for US-only corpora). Coverage extends
+ *   to every named street segment + every incorporated place + CDP across the 50 states + DC + the
+ *   five primary territories — substantially better US street-name coverage than OSM, especially in
+ *   rural areas.
+ *
+ *   Following the `wof-admin` / `wof-postalcode` pattern, this adapter consumes a SQLite database the
+ *   operator pre-builds from the raw TIGER shapefiles (see the README for the schema and a
+ *   suggested `ogr2ogr` pipeline). The mailwoman side does not parse Shapefile binary directly —
+ *   keeping the adapter narrow lets the operator pick their own ingestion tool (ogr2ogr / shp2pgsql
+ *   / a custom Python script / etc.) without forcing a heavy native dep into `@mailwoman/corpus`.
+ *
+ *   Two row classes are emitted:
+ *
+ *   - **Street-level** (`tiger_streets`): one row per segment, optionally with up to two postcode
+ *       variants if `zipl` / `zipr` differ. Components: `{ street, region, postcode? }`. Streets
+ *       without a recognized state FIPS are dropped — there's no useful row without `region`.
+ *   - **Locality-level** (`tiger_places`): up to three variants per place: locality-only,
+ *       locality-with-region, locality-with-region-country (mirrors `wof-admin`'s fan-out for
+ *       consistency).
+ *   - `packages/corpus/src/codex/us-fips-state.ts` — the FIPS → `{abbreviation, name}` lookup table
+ *       (originally `tiger/state.ts`, AGPL-3.0 → AGPL-3.0). The full isp-nexus TIGER module ships a
+ *       TypeORM-backed service layer; mailwoman only needs the lookup data so we don't carry the
+ *       service layer over.
+ *
+ *   License: stamped `"Public Domain"` per Census Bureau guidance on TIGER/Line. No per-row override
+ *   needed — every row in TIGER is the same license.
+ */
+
+import { DatabaseSync } from "node:sqlite"
+
+import { lookupFipsState } from "../../codex/us-fips-state.ts"
+import { formatAddress, reconcileComponents } from "../../format.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+import { decomposeStreet } from "./street-decompose.ts"
+
+export const TIGER_ADAPTER_ID = "tiger"
+export const TIGER_DEFAULT_LICENSE = "Public Domain"
+
+/**
+ * The country surface form used in `formatAddress` for US. Matches the canonical OpenCage US template output so
+ * reconciliation doesn't strip it when the row carries `country` explicitly.
+ */
+const US_COUNTRY_DISPLAY = "United States of America"
+
+interface TigerStreetRow {
+	linearid: string
+	fullname: string
+	zipl: string | null
+	zipr: string | null
+	statefp: string
+}
+
+interface TigerPlaceRow {
+	geoid: string
+	name: string
+	statefp: string
+	lsad: string | null
+}
+
+/**
+ * Yield one or more `CanonicalRow`s per street segment. Postcode variants:
+ *
+ * - No ZIP set → one row, street + region.
+ * - `zipl === zipr` → one row, street + region + postcode.
+ * - `zipl !== zipr` → two rows (one per side's ZIP).
+ */
+function* streetVariants(row: TigerStreetRow): Iterable<{
+	components: CanonicalRow["components"]
+	variantKey: string
+}> {
+	const fullname = row.fullname.trim()
+
+	if (!fullname) return
+	const state = lookupFipsState(row.statefp)
+
+	if (!state) return
+
+	const zipl = row.zipl?.trim() ?? ""
+	const zipr = row.zipr?.trim() ?? ""
+
+	const decomposed = decomposeStreet(fullname)
+
+	const baseComponents: CanonicalRow["components"] = {
+		region: state.abbreviation,
+		street: decomposed.street,
+	}
+
+	if (decomposed.prefix) {
+		baseComponents.street_prefix = decomposed.prefix
+	}
+
+	if (decomposed.suffix) {
+		baseComponents.street_suffix = decomposed.suffix
+	}
+
+	if (!zipl && !zipr) {
+		yield { components: baseComponents, variantKey: "no-zip" }
+
+		return
+	}
+
+	if (zipl && zipr && zipl === zipr) {
+		yield {
+			components: { ...baseComponents, postcode: zipl },
+			variantKey: `zip-${zipl}`,
+		}
+
+		return
+	}
+
+	if (zipl) {
+		yield { components: { ...baseComponents, postcode: zipl }, variantKey: `zipl-${zipl}` }
+	}
+
+	if (zipr && zipr !== zipl) {
+		yield { components: { ...baseComponents, postcode: zipr }, variantKey: `zipr-${zipr}` }
+	}
+}
+
+/** Three locality-level variants, mirroring `wof-admin`'s fan-out. */
+function* placeVariants(row: TigerPlaceRow): Iterable<{
+	components: CanonicalRow["components"]
+	variantKey: string
+}> {
+	const name = row.name.trim()
+
+	if (!name) return
+	const state = lookupFipsState(row.statefp)
+
+	if (!state) return
+
+	yield {
+		components: { locality: name },
+		variantKey: "locality-only",
+	}
+	yield {
+		components: { locality: name, region: state.abbreviation },
+		variantKey: "with-region",
+	}
+	yield {
+		components: { locality: name, region: state.abbreviation, country: US_COUNTRY_DISPLAY },
+		variantKey: "with-region-country",
+	}
+}
+
+/** Build a TIGER adapter. Pure factory so multiple instances can be created in tests. */
+export function createTigerAdapter(): CorpusAdapter {
+	return {
+		id: TIGER_ADAPTER_ID,
+		defaultLicense: TIGER_DEFAULT_LICENSE,
+		description:
+			"US Census TIGER/Line streets + places consumer (public-domain); SQLite DB built via `mailwoman tiger fetch`.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`tiger adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const db = new DatabaseSync(opts.inputPath, { readOnly: true })
+			let emitted = 0
+
+			try {
+				const streetStmt = db.prepare(`SELECT linearid, fullname, zipl, zipr, statefp FROM tiger_streets`)
+				const placeStmt = db.prepare(`SELECT geoid, name, statefp, lsad FROM tiger_places`)
+
+				for (const row of streetStmt.iterate() as IterableIterator<TigerStreetRow>) {
+					if (opts.signal?.aborted) return
+
+					for (const variant of streetVariants(row)) {
+						if (opts.limit !== undefined && emitted >= opts.limit) return
+						const raw = formatAddress(variant.components, "US", { separator: ", " })
+
+						if (!raw) continue
+						const aligned = reconcileComponents(variant.components, raw)
+
+						if (Object.keys(aligned).length === 0) continue
+
+						yield {
+							raw,
+							components: aligned,
+							country: "US",
+							locale: "en-US",
+							source: TIGER_ADAPTER_ID,
+							source_id: `${TIGER_ADAPTER_ID}-st-${row.linearid}-${variant.variantKey}`,
+							corpus_version: "",
+							license: TIGER_DEFAULT_LICENSE,
+						}
+						emitted++
+					}
+				}
+
+				for (const row of placeStmt.iterate() as IterableIterator<TigerPlaceRow>) {
+					if (opts.signal?.aborted) return
+
+					for (const variant of placeVariants(row)) {
+						if (opts.limit !== undefined && emitted >= opts.limit) return
+						const raw = formatAddress(variant.components, "US", { separator: ", " })
+
+						if (!raw) continue
+						const aligned = reconcileComponents(variant.components, raw)
+
+						if (Object.keys(aligned).length === 0) continue
+
+						yield {
+							raw,
+							components: aligned,
+							country: "US",
+							locale: "en-US",
+							source: TIGER_ADAPTER_ID,
+							source_id: `${TIGER_ADAPTER_ID}-pl-${row.geoid}-${variant.variantKey}`,
+							corpus_version: "",
+							license: TIGER_DEFAULT_LICENSE,
+						}
+						emitted++
+					}
+				}
+			} finally {
+				db.close()
+			}
+		},
+	}
+}
+
+export const tigerAdapter = createTigerAdapter()