@mailwoman/corpus 9.1.0 → 9.3.0

package/lib/adapters/synth-po-box/adapter.ts

@@ -0,0 +1,204 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `synth-po-box`: PO box / PMB / Apartado / BP synthesizer adapter.
+ *
+ *   Consumes a JSONL stream of (locality, region, postcode, country) tuples — typically extracted
+ *   from existing corpus output (TIGER/NAD/BAN/WOF) — and emits synthetic PO box training rows. See
+ *   `@mailwoman/corpus/synthesizers/po-box` for the per-locale templates and number-noise logic.
+ *
+ *   Why an adapter and not an augmenter:
+ *
+ *   - Per USPS Pub 28 / DMM 508, a PO box delivery line is mutually exclusive with a street line.
+ *       Synthesizing PO boxes by mutating a street row would teach the model an invalid pattern.
+ *       The clean shape is: read just (locality, region, postcode, country) and produce a fresh
+ *       PO-box-shaped row.
+ *   - Per-DeepSeek (3-turn consult, 2026-05-28): PMB rows that COMBINE a street line with a PMB number
+ *       ARE valid (CMRA addresses). Those are produced when `pmbRatio > 0` AND the input tuple
+ *       carries a `street` field.
+ */
+
+import { tryParsingJSON } from "@mailwoman/core/json"
+import { makeLcg } from "@mailwoman/core/random"
+import { TextSpliterator } from "spliterator"
+
+import { stableSourceID } from "#adapters/utils"
+import {
+	countryToLocale,
+	REGION_OPTIONAL_LOCALES,
+	synthesizeMilitaryPoBoxRow,
+	synthesizePoBoxRow,
+	type PoBoxBaseTuple,
+} from "#synthesizers/po-box"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const SYNTH_PO_BOX_ADAPTER_ID = "synth-po-box"
+/**
+ * License for the synthetic PO-box rows. The output is generated, but it inherits the terms of the real tuples it is
+ * derived from, so the attribution travels with it.
+ */
+export const SYNTH_PO_BOX_LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"
+
+export interface PoBoxInputRow extends PoBoxBaseTuple {
+	street?: string
+	houseNumber?: string
+}
+
+export interface SynthPoBoxAdapterOptions {
+	/**
+	 * How many PO box variants to emit per input tuple. Each variant picks a different leader (and possibly a different
+	 * number / noise level). Default 1.
+	 */
+	variantsPerInput?: number
+	/**
+	 * Probability (0..1) of emitting a PMB-with-street variant when both the input has a street and the locale supports
+	 * PMB. Default 0.15.
+	 */
+	pmbRatio?: number
+	/**
+	 * Deterministic seed for reproducible synthesis. Default Date.now().
+	 */
+	seed?: number
+	/**
+	 * Probability (0..1), evaluated per input tuple, of ALSO emitting one US military/diplomatic PO-box row
+	 * (`PSC/CMR/Unit <id> Box <box>, APO/FPO/DPO AA/AE/AP <zip>`, #517). These rows are self-contained — they draw no
+	 * field from the input tuple, so military volume scales with the input stream size. Default 0 (off) — the adapter's
+	 * contract is "one row per input"; the corpus build recipe opts in to seed the rare-but-real military class without
+	 * changing the default.
+	 */
+	militaryRatio?: number
+}
+
+export function createSynthPoBoxAdapter(opts: SynthPoBoxAdapterOptions = {}): CorpusAdapter {
+	const variantsPerInput = opts.variantsPerInput ?? 1
+	const pmbRatio = opts.pmbRatio ?? 0.15
+	const militaryRatio = opts.militaryRatio ?? 0
+
+	return {
+		id: SYNTH_PO_BOX_ADAPTER_ID,
+		defaultLicense: SYNTH_PO_BOX_LICENSE,
+		description:
+			"Synthetic PO box / PMB / Apartado / Boîte Postale rows. Consumes JSONL of (locality, region, postcode, country) tuples and emits locale-appropriate PO box variants.",
+
+		async *rows(options: AdapterOptions): AsyncIterable<CanonicalRow> {
+			const random = makeLcg(opts.seed ?? Date.now())
+
+			// TextSpliterator streams string lines; the per-line tryParsingJSON below keeps this reader
+			// tolerant of malformed rows (skipped++), so TextSpliterator + a non-throwing parse — not
+			// JSONSpliterator, which would throw on the first bad line.
+			const lines = TextSpliterator.fromAsync(options.inputPath)
+
+			let emitted = 0
+			let skipped = 0
+			let militarySeq = 0
+
+			for await (const line of lines) {
+				if (options.signal?.aborted) break
+
+				if (options.limit !== undefined && emitted >= options.limit) break
+
+				const trimmed = line.trim()
+
+				if (!trimmed) continue
+
+				const input = tryParsingJSON<PoBoxInputRow>(trimmed)
+
+				if (input === null) {
+					skipped++
+
+					continue
+				}
+
+				// Region is required EXCEPT for region-less locales (NZ: `Private Bag 12, Auckland 1010`
+				// has no region token, #517). synthesizePoBoxRow handles region absence; the guard just
+				// must not discard those tuples as "missing region".
+				const regionOptional = input.country ? REGION_OPTIONAL_LOCALES.has(countryToLocale(input.country)) : false
+
+				if (!input.locality || !input.postcode || !input.country || (!input.region && !regionOptional)) {
+					skipped++
+
+					continue
+				}
+
+				if (options.country && options.country !== input.country) continue
+
+				for (let v = 0; v < variantsPerInput; v++) {
+					const synth = synthesizePoBoxRow(input, { random, pmbRatio })
+
+					if (!synth) continue
+
+					// Include `v` in dependent_locality slot to vary the digest across variants;
+					// stableSourceID only accepts ComponentTag keys.
+					const sourceID = stableSourceID(SYNTH_PO_BOX_ADAPTER_ID, {
+						locality: `${input.locality}#${v}`,
+						region: input.region,
+						postcode: input.postcode,
+						country: input.country,
+					})
+
+					yield {
+						raw: synth.raw,
+						components: synth.components,
+						country: input.country,
+						locale: synth.locale,
+						source: SYNTH_PO_BOX_ADAPTER_ID,
+						source_id: sourceID,
+						corpus_version: "",
+						license: SYNTH_PO_BOX_LICENSE,
+					}
+
+					emitted++
+
+					if (options.limit !== undefined && emitted >= options.limit) break
+				}
+
+				// US military/diplomatic PO-box rows (#517): self-contained — draw nothing from the input
+				// tuple — emitted per input line with probability `militaryRatio` (off by default, so the
+				// default random stream and output are byte-identical). Military volume scales with the
+				// stream rather than the US-tuple count. US-only: suppressed under a non-US country filter
+				// and counted against `limit` like any other row.
+				const militaryAllowed = !options.country || options.country === "US"
+
+				if (
+					militaryRatio > 0 &&
+					militaryAllowed &&
+					(options.limit === undefined || emitted < options.limit) &&
+					random() < militaryRatio
+				) {
+					const mil = synthesizeMilitaryPoBoxRow({ random })
+
+					const sourceID = stableSourceID(SYNTH_PO_BOX_ADAPTER_ID, {
+						po_box: `${mil.components.po_box}#mil${militarySeq++}`,
+						locality: mil.components.locality!,
+						region: mil.components.region!,
+						postcode: mil.components.postcode!,
+					})
+
+					yield {
+						raw: mil.raw,
+						components: mil.components,
+						country: "US",
+						locale: mil.locale,
+						source: SYNTH_PO_BOX_ADAPTER_ID,
+						source_id: sourceID,
+						corpus_version: "",
+						license: SYNTH_PO_BOX_LICENSE,
+					}
+
+					emitted++
+				}
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const synthPoBoxAdapter = createSynthPoBoxAdapter()

package/lib/adapters/tiger/adapter.ts

@@ -0,0 +1,252 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `tiger`: US Census TIGER/Line consumer adapter.
+ *
+ *   TIGER/Line is the canonical US street + locality dataset published by the Census Bureau as a
+ *   **public-domain** product (no ODbL share-alike concerns for US-only corpora). Coverage extends
+ *   to every named street segment + every incorporated place + CDP across the 50 states + DC + the
+ *   five primary territories — substantially better US street-name coverage than OSM, especially in
+ *   rural areas.
+ *
+ *   Following the `wof-admin` / `wof-postalcode` pattern, this adapter consumes a SQLite database the
+ *   operator pre-builds from the raw TIGER shapefiles (see the README for the schema and a
+ *   suggested `ogr2ogr` pipeline). The mailwoman side does not parse Shapefile binary directly —
+ *   keeping the adapter narrow lets the operator pick their own ingestion tool (ogr2ogr / shp2pgsql
+ *   / a custom Python script / etc.) without forcing a heavy native dep into `@mailwoman/corpus`.
+ *
+ *   Two row classes are emitted:
+ *
+ *   - **Street-level** (`tiger_streets`): one row per segment, optionally with up to two postcode
+ *       variants if `zipl` / `zipr` differ. Components: `{ street, region, postcode? }`. Streets
+ *       without a recognized state FIPS are dropped — there's no useful row without `region`.
+ *   - **Locality-level** (`tiger_places`): up to three variants per place: locality-only,
+ *       locality-with-region, locality-with-region-country (mirrors `wof-admin`'s fan-out for
+ *       consistency).
+ *   - `packages/corpus/lib/codex/us-fips-state.ts` — the FIPS → `{abbreviation, name}` lookup table
+ *       (originally `tiger/state.ts`, AGPL-3.0 → AGPL-3.0). The full isp-nexus TIGER module ships a
+ *       TypeORM-backed service layer; mailwoman only needs the lookup data so we don't carry the
+ *       service layer over.
+ *
+ *   License: stamped `"Public Domain"` per Census Bureau guidance on TIGER/Line. No per-row override
+ *   needed — every row in TIGER is the same license.
+ */
+
+import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
+import { DatabaseClient } from "@mailwoman/sqlite/client"
+import type { TIGERDatabase } from "@mailwoman/tiger/sdk/schema"
+
+import { decomposeStreet } from "#adapters/tiger/street-decompose"
+import { lookupFipsState } from "#codex/us-fips-state"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const TIGER_ADAPTER_ID = "tiger"
+/**
+ * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
+ * than having to look them up.
+ */
+export const TIGER_DEFAULT_LICENSE = "Public Domain"
+
+/**
+ * The country surface form used in `formatAddress` for US. Matches the canonical OpenCage US template output so
+ * reconciliation doesn't strip it when the row carries `country` explicitly.
+ */
+const US_COUNTRY_DISPLAY = "United States of America"
+
+interface TigerStreetRow {
+	linearid: string
+	fullname: string
+	zipl: string | null
+	zipr: string | null
+	statefp: string
+}
+
+interface TigerPlaceRow {
+	geoid: string
+	name: string
+	statefp: string
+	lsad: string | null
+}
+
+/**
+ * Yield one or more `CanonicalRow`s per street segment. Postcode variants:
+ *
+ * - No ZIP set → one row, street + region.
+ * - `zipl === zipr` → one row, street + region + postcode.
+ * - `zipl !== zipr` → two rows (one per side's ZIP).
+ */
+function* streetVariants(row: TigerStreetRow): Iterable<{
+	components: CanonicalRow["components"]
+	variantKey: string
+}> {
+	const fullname = row.fullname.trim()
+
+	if (!fullname) return
+	const state = lookupFipsState(row.statefp)
+
+	if (!state) return
+
+	const zipl = row.zipl?.trim() ?? ""
+	const zipr = row.zipr?.trim() ?? ""
+
+	const decomposed = decomposeStreet(fullname)
+
+	const baseComponents: CanonicalRow["components"] = {
+		region: state.abbreviation,
+		street: decomposed.street,
+	}
+
+	if (decomposed.prefix) {
+		baseComponents.street_prefix = decomposed.prefix
+	}
+
+	if (decomposed.suffix) {
+		baseComponents.street_suffix = decomposed.suffix
+	}
+
+	if (!zipl && !zipr) {
+		yield { components: baseComponents, variantKey: "no-zip" }
+
+		return
+	}
+
+	if (zipl && zipr && zipl === zipr) {
+		yield {
+			components: { ...baseComponents, postcode: zipl },
+			variantKey: `zip-${zipl}`,
+		}
+
+		return
+	}
+
+	if (zipl) {
+		yield { components: { ...baseComponents, postcode: zipl }, variantKey: `zipl-${zipl}` }
+	}
+
+	if (zipr && zipr !== zipl) {
+		yield { components: { ...baseComponents, postcode: zipr }, variantKey: `zipr-${zipr}` }
+	}
+}
+
+/**
+ * Three locality-level variants, mirroring `wof-admin`'s fan-out.
+ */
+function* placeVariants(row: TigerPlaceRow): Iterable<{
+	components: CanonicalRow["components"]
+	variantKey: string
+}> {
+	const name = row.name.trim()
+
+	if (!name) return
+	const state = lookupFipsState(row.statefp)
+
+	if (!state) return
+
+	yield {
+		components: { locality: name },
+		variantKey: "locality-only",
+	}
+
+	yield {
+		components: { locality: name, region: state.abbreviation },
+		variantKey: "with-region",
+	}
+
+	yield {
+		components: { locality: name, region: state.abbreviation, country: US_COUNTRY_DISPLAY },
+		variantKey: "with-region-country",
+	}
+}
+
+/**
+ * Build a TIGER adapter. Pure factory so multiple instances can be created in tests.
+ */
+export function createTigerAdapter(): CorpusAdapter {
+	return {
+		id: TIGER_ADAPTER_ID,
+		defaultLicense: TIGER_DEFAULT_LICENSE,
+		description:
+			"US Census TIGER/Line streets + places consumer (public-domain); SQLite DB built via `mailwoman tiger fetch`.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`tiger adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const db = new DatabaseClient<TIGERDatabase>(opts.inputPath, { readOnly: true })
+			let emitted = 0
+
+			try {
+				const streetStmt = db.prepare(`SELECT linearid, fullname, zipl, zipr, statefp FROM tiger_streets`)
+				const placeStmt = db.prepare(`SELECT geoid, name, statefp, lsad FROM tiger_places`)
+
+				for (const row of streetStmt.iterate() as IterableIterator<TigerStreetRow>) {
+					if (opts.signal?.aborted) return
+
+					for (const variant of streetVariants(row)) {
+						if (opts.limit !== undefined && emitted >= opts.limit) return
+						const raw = formatAddress(variant.components, "US", { separator: ", " })
+
+						if (!raw) continue
+						const aligned = reconcileComponents(variant.components, raw)
+
+						if (!Object.keys(aligned).length) continue
+
+						yield {
+							raw,
+							components: aligned,
+							country: "US",
+							locale: "en-US",
+							source: TIGER_ADAPTER_ID,
+							source_id: `${TIGER_ADAPTER_ID}-st-${row.linearid}-${variant.variantKey}`,
+							corpus_version: "",
+							license: TIGER_DEFAULT_LICENSE,
+						}
+
+						emitted++
+					}
+				}
+
+				for (const row of placeStmt.iterate() as IterableIterator<TigerPlaceRow>) {
+					if (opts.signal?.aborted) return
+
+					for (const variant of placeVariants(row)) {
+						if (opts.limit !== undefined && emitted >= opts.limit) return
+						const raw = formatAddress(variant.components, "US", { separator: ", " })
+
+						if (!raw) continue
+						const aligned = reconcileComponents(variant.components, raw)
+
+						if (!Object.keys(aligned).length) continue
+
+						yield {
+							raw,
+							components: aligned,
+							country: "US",
+							locale: "en-US",
+							source: TIGER_ADAPTER_ID,
+							source_id: `${TIGER_ADAPTER_ID}-pl-${row.geoid}-${variant.variantKey}`,
+							corpus_version: "",
+							license: TIGER_DEFAULT_LICENSE,
+						}
+
+						emitted++
+					}
+				}
+			} finally {
+				await db.destroy()
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const tigerAdapter = createTigerAdapter()

package/lib/adapters/tiger/street-decompose.ts

@@ -0,0 +1,83 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Decompose a US street name into Stage 3 components: street_prefix, street, street_suffix.
+ *
+ *   Sources directionals and street types from the curated libpostal/en dictionaries
+ *   (`core/data/libpostal/dictionaries/en/{directionals,street_types}.txt`). These are the same
+ *   dictionaries the runtime classifiers (StreetPrefixClassifier, StreetSuffixClassifier) use, so
+ *   corpus labels and runtime classifications agree on the vocabulary.
+ *
+ *   Examples: "N Main St" → { prefix: "N", street: "Main", suffix: "St" } "Pennsylvania Avenue NW" →
+ *   { prefix: null, street: "Pennsylvania", suffix: "Avenue NW" } "Salmon St" → { prefix: null,
+ *   street: "Salmon", suffix: "St" } "SE Hawthorne Blvd" → { prefix: "SE", street: "Hawthorne",
+ *   suffix: "Blvd" }
+ */
+
+import { loadLibpostalDictionary } from "#adapters/utils"
+
+const [DIRECTIONALS, STREET_TYPES] = await Promise.all([
+	loadLibpostalDictionary("en", "directionals.txt"),
+	loadLibpostalDictionary("en", "street_types.txt"),
+])
+
+export interface DecomposedStreet {
+	prefix: string | null
+	street: string
+	suffix: string | null
+}
+
+/**
+ * Decompose a US street name into prefix/name/suffix components.
+ *
+ * Conservative — only emits prefix/suffix when there's a clear directional or street-type keyword. Returns the original
+ * as `street` if nothing matches.
+ */
+export function decomposeStreet(fullname: string): DecomposedStreet {
+	const trimmed = fullname.trim()
+
+	if (!trimmed) return { prefix: null, street: "", suffix: null }
+
+	const tokens = trimmed.split(/\s+/)
+
+	if (tokens.length === 1) return { prefix: null, street: trimmed, suffix: null }
+
+	const norm = (s: string) => s.toLowerCase().replace(/\.$/, "")
+
+	let prefix: string | null = null
+	let suffix: string | null = null
+	let startIdx = 0
+	let endIdx = tokens.length
+
+	// Leading directional prefix
+	if (DIRECTIONALS.has(norm(tokens[0]!)) && tokens.length >= 2) {
+		prefix = tokens[0]!
+		startIdx = 1
+	}
+
+	// Trailing post-directional combined with street type (e.g. "Pennsylvania Ave NW")
+	const last = norm(tokens[endIdx - 1]!)
+	const secondLast = endIdx >= 2 ? norm(tokens[endIdx - 2]!) : ""
+
+	if (DIRECTIONALS.has(last) && STREET_TYPES.has(secondLast)) {
+		suffix = tokens.slice(endIdx - 2, endIdx).join(" ")
+		endIdx -= 2
+	} else if (STREET_TYPES.has(last) && endIdx - startIdx >= 2) {
+		suffix = tokens[endIdx - 1]!
+		endIdx -= 1
+	} else if (DIRECTIONALS.has(last) && endIdx - startIdx >= 2) {
+		// Post-directional without type
+		suffix = tokens[endIdx - 1]!
+		endIdx -= 1
+	}
+
+	const street = tokens.slice(startIdx, endIdx).join(" ").trim()
+
+	if (!street) {
+		return { prefix: null, street: trimmed, suffix: null }
+	}
+
+	return { prefix, street, suffix }
+}