@mailwoman/corpus 9.2.0 → 9.4.0

package/src/adapters/synth-po-box/adapter.ts

@@ -1,204 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `synth-po-box`: PO box / PMB / Apartado / BP synthesizer adapter.
- *
- *   Consumes a JSONL stream of (locality, region, postcode, country) tuples — typically extracted
- *   from existing corpus output (TIGER/NAD/BAN/WOF) — and emits synthetic PO box training rows. See
- *   `@mailwoman/corpus/synthesizers/po-box` for the per-locale templates and number-noise logic.
- *
- *   Why an adapter and not an augmenter:
- *
- *   - Per USPS Pub 28 / DMM 508, a PO box delivery line is mutually exclusive with a street line.
- *       Synthesizing PO boxes by mutating a street row would teach the model an invalid pattern.
- *       The clean shape is: read just (locality, region, postcode, country) and produce a fresh
- *       PO-box-shaped row.
- *   - Per-DeepSeek (3-turn consult, 2026-05-28): PMB rows that COMBINE a street line with a PMB number
- *       ARE valid (CMRA addresses). Those are produced when `pmbRatio > 0` AND the input tuple
- *       carries a `street` field.
- */
-
-import { tryParsingJSON } from "@mailwoman/core/objects"
-import { makeLcg } from "@mailwoman/core/utils"
-import { TextSpliterator } from "spliterator"
-
-import { stableSourceID } from "#adapters/utils"
-import {
-	countryToLocale,
-	REGION_OPTIONAL_LOCALES,
-	synthesizeMilitaryPoBoxRow,
-	synthesizePoBoxRow,
-	type PoBoxBaseTuple,
-} from "#synthesizers/po-box"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const SYNTH_PO_BOX_ADAPTER_ID = "synth-po-box"
-/**
- * License for the synthetic PO-box rows. The output is generated, but it inherits the terms of the real tuples it is
- * derived from, so the attribution travels with it.
- */
-export const SYNTH_PO_BOX_LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"
-
-export interface PoBoxInputRow extends PoBoxBaseTuple {
-	street?: string
-	houseNumber?: string
-}
-
-export interface SynthPoBoxAdapterOptions {
-	/**
-	 * How many PO box variants to emit per input tuple. Each variant picks a different leader (and possibly a different
-	 * number / noise level). Default 1.
-	 */
-	variantsPerInput?: number
-	/**
-	 * Probability (0..1) of emitting a PMB-with-street variant when both the input has a street and the locale supports
-	 * PMB. Default 0.15.
-	 */
-	pmbRatio?: number
-	/**
-	 * Deterministic seed for reproducible synthesis. Default Date.now().
-	 */
-	seed?: number
-	/**
-	 * Probability (0..1), evaluated per input tuple, of ALSO emitting one US military/diplomatic PO-box row
-	 * (`PSC/CMR/Unit <id> Box <box>, APO/FPO/DPO AA/AE/AP <zip>`, #517). These rows are self-contained — they draw no
-	 * field from the input tuple, so military volume scales with the input stream size. Default 0 (off) — the adapter's
-	 * contract is "one row per input"; the corpus build recipe opts in to seed the rare-but-real military class without
-	 * changing the default.
-	 */
-	militaryRatio?: number
-}
-
-export function createSynthPoBoxAdapter(opts: SynthPoBoxAdapterOptions = {}): CorpusAdapter {
-	const variantsPerInput = opts.variantsPerInput ?? 1
-	const pmbRatio = opts.pmbRatio ?? 0.15
-	const militaryRatio = opts.militaryRatio ?? 0
-
-	return {
-		id: SYNTH_PO_BOX_ADAPTER_ID,
-		defaultLicense: SYNTH_PO_BOX_LICENSE,
-		description:
-			"Synthetic PO box / PMB / Apartado / Boîte Postale rows. Consumes JSONL of (locality, region, postcode, country) tuples and emits locale-appropriate PO box variants.",
-
-		async *rows(options: AdapterOptions): AsyncIterable<CanonicalRow> {
-			const random = makeLcg(opts.seed ?? Date.now())
-
-			// TextSpliterator streams string lines; the per-line tryParsingJSON below keeps this reader
-			// tolerant of malformed rows (skipped++), so TextSpliterator + a non-throwing parse — not
-			// JSONSpliterator, which would throw on the first bad line.
-			const lines = TextSpliterator.fromAsync(options.inputPath)
-
-			let emitted = 0
-			let skipped = 0
-			let militarySeq = 0
-
-			for await (const line of lines) {
-				if (options.signal?.aborted) break
-
-				if (options.limit !== undefined && emitted >= options.limit) break
-
-				const trimmed = line.trim()
-
-				if (!trimmed) continue
-
-				const input = tryParsingJSON<PoBoxInputRow>(trimmed)
-
-				if (input === null) {
-					skipped++
-
-					continue
-				}
-
-				// Region is required EXCEPT for region-less locales (NZ: `Private Bag 12, Auckland 1010`
-				// has no region token, #517). synthesizePoBoxRow handles region absence; the guard just
-				// must not discard those tuples as "missing region".
-				const regionOptional = input.country ? REGION_OPTIONAL_LOCALES.has(countryToLocale(input.country)) : false
-
-				if (!input.locality || !input.postcode || !input.country || (!input.region && !regionOptional)) {
-					skipped++
-
-					continue
-				}
-
-				if (options.country && options.country !== input.country) continue
-
-				for (let v = 0; v < variantsPerInput; v++) {
-					const synth = synthesizePoBoxRow(input, { random, pmbRatio })
-
-					if (!synth) continue
-
-					// Include `v` in dependent_locality slot to vary the digest across variants;
-					// stableSourceID only accepts ComponentTag keys.
-					const sourceID = stableSourceID(SYNTH_PO_BOX_ADAPTER_ID, {
-						locality: `${input.locality}#${v}`,
-						region: input.region,
-						postcode: input.postcode,
-						country: input.country,
-					})
-
-					yield {
-						raw: synth.raw,
-						components: synth.components,
-						country: input.country,
-						locale: synth.locale,
-						source: SYNTH_PO_BOX_ADAPTER_ID,
-						source_id: sourceID,
-						corpus_version: "",
-						license: SYNTH_PO_BOX_LICENSE,
-					}
-
-					emitted++
-
-					if (options.limit !== undefined && emitted >= options.limit) break
-				}
-
-				// US military/diplomatic PO-box rows (#517): self-contained — draw nothing from the input
-				// tuple — emitted per input line with probability `militaryRatio` (off by default, so the
-				// default random stream and output are byte-identical). Military volume scales with the
-				// stream rather than the US-tuple count. US-only: suppressed under a non-US country filter
-				// and counted against `limit` like any other row.
-				const militaryAllowed = !options.country || options.country === "US"
-
-				if (
-					militaryRatio > 0 &&
-					militaryAllowed &&
-					(options.limit === undefined || emitted < options.limit) &&
-					random() < militaryRatio
-				) {
-					const mil = synthesizeMilitaryPoBoxRow({ random })
-
-					const sourceID = stableSourceID(SYNTH_PO_BOX_ADAPTER_ID, {
-						po_box: `${mil.components.po_box}#mil${militarySeq++}`,
-						locality: mil.components.locality!,
-						region: mil.components.region!,
-						postcode: mil.components.postcode!,
-					})
-
-					yield {
-						raw: mil.raw,
-						components: mil.components,
-						country: "US",
-						locale: mil.locale,
-						source: SYNTH_PO_BOX_ADAPTER_ID,
-						source_id: sourceID,
-						corpus_version: "",
-						license: SYNTH_PO_BOX_LICENSE,
-					}
-
-					emitted++
-				}
-			}
-		},
-	}
-}
-
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const synthPoBoxAdapter = createSynthPoBoxAdapter()

package/src/adapters/tiger/adapter.ts

@@ -1,253 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `tiger`: US Census TIGER/Line consumer adapter.
- *
- *   TIGER/Line is the canonical US street + locality dataset published by the Census Bureau as a
- *   **public-domain** product (no ODbL share-alike concerns for US-only corpora). Coverage extends
- *   to every named street segment + every incorporated place + CDP across the 50 states + DC + the
- *   five primary territories — substantially better US street-name coverage than OSM, especially in
- *   rural areas.
- *
- *   Following the `wof-admin` / `wof-postalcode` pattern, this adapter consumes a SQLite database the
- *   operator pre-builds from the raw TIGER shapefiles (see the README for the schema and a
- *   suggested `ogr2ogr` pipeline). The mailwoman side does not parse Shapefile binary directly —
- *   keeping the adapter narrow lets the operator pick their own ingestion tool (ogr2ogr / shp2pgsql
- *   / a custom Python script / etc.) without forcing a heavy native dep into `@mailwoman/corpus`.
- *
- *   Two row classes are emitted:
- *
- *   - **Street-level** (`tiger_streets`): one row per segment, optionally with up to two postcode
- *       variants if `zipl` / `zipr` differ. Components: `{ street, region, postcode? }`. Streets
- *       without a recognized state FIPS are dropped — there's no useful row without `region`.
- *   - **Locality-level** (`tiger_places`): up to three variants per place: locality-only,
- *       locality-with-region, locality-with-region-country (mirrors `wof-admin`'s fan-out for
- *       consistency).
- *   - `packages/corpus/src/codex/us-fips-state.ts` — the FIPS → `{abbreviation, name}` lookup table
- *       (originally `tiger/state.ts`, AGPL-3.0 → AGPL-3.0). The full isp-nexus TIGER module ships a
- *       TypeORM-backed service layer; mailwoman only needs the lookup data so we don't carry the
- *       service layer over.
- *
- *   License: stamped `"Public Domain"` per Census Bureau guidance on TIGER/Line. No per-row override
- *   needed — every row in TIGER is the same license.
- */
-
-import { DatabaseSync } from "node:sqlite"
-
-import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
-
-import { lookupFipsState } from "#codex/us-fips-state"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
-
-import { decomposeStreet } from "./street-decompose.ts"
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const TIGER_ADAPTER_ID = "tiger"
-/**
- * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
- * than having to look them up.
- */
-export const TIGER_DEFAULT_LICENSE = "Public Domain"
-
-/**
- * The country surface form used in `formatAddress` for US. Matches the canonical OpenCage US template output so
- * reconciliation doesn't strip it when the row carries `country` explicitly.
- */
-const US_COUNTRY_DISPLAY = "United States of America"
-
-interface TigerStreetRow {
-	linearid: string
-	fullname: string
-	zipl: string | null
-	zipr: string | null
-	statefp: string
-}
-
-interface TigerPlaceRow {
-	geoid: string
-	name: string
-	statefp: string
-	lsad: string | null
-}
-
-/**
- * Yield one or more `CanonicalRow`s per street segment. Postcode variants:
- *
- * - No ZIP set → one row, street + region.
- * - `zipl === zipr` → one row, street + region + postcode.
- * - `zipl !== zipr` → two rows (one per side's ZIP).
- */
-function* streetVariants(row: TigerStreetRow): Iterable<{
-	components: CanonicalRow["components"]
-	variantKey: string
-}> {
-	const fullname = row.fullname.trim()
-
-	if (!fullname) return
-	const state = lookupFipsState(row.statefp)
-
-	if (!state) return
-
-	const zipl = row.zipl?.trim() ?? ""
-	const zipr = row.zipr?.trim() ?? ""
-
-	const decomposed = decomposeStreet(fullname)
-
-	const baseComponents: CanonicalRow["components"] = {
-		region: state.abbreviation,
-		street: decomposed.street,
-	}
-
-	if (decomposed.prefix) {
-		baseComponents.street_prefix = decomposed.prefix
-	}
-
-	if (decomposed.suffix) {
-		baseComponents.street_suffix = decomposed.suffix
-	}
-
-	if (!zipl && !zipr) {
-		yield { components: baseComponents, variantKey: "no-zip" }
-
-		return
-	}
-
-	if (zipl && zipr && zipl === zipr) {
-		yield {
-			components: { ...baseComponents, postcode: zipl },
-			variantKey: `zip-${zipl}`,
-		}
-
-		return
-	}
-
-	if (zipl) {
-		yield { components: { ...baseComponents, postcode: zipl }, variantKey: `zipl-${zipl}` }
-	}
-
-	if (zipr && zipr !== zipl) {
-		yield { components: { ...baseComponents, postcode: zipr }, variantKey: `zipr-${zipr}` }
-	}
-}
-
-/**
- * Three locality-level variants, mirroring `wof-admin`'s fan-out.
- */
-function* placeVariants(row: TigerPlaceRow): Iterable<{
-	components: CanonicalRow["components"]
-	variantKey: string
-}> {
-	const name = row.name.trim()
-
-	if (!name) return
-	const state = lookupFipsState(row.statefp)
-
-	if (!state) return
-
-	yield {
-		components: { locality: name },
-		variantKey: "locality-only",
-	}
-
-	yield {
-		components: { locality: name, region: state.abbreviation },
-		variantKey: "with-region",
-	}
-
-	yield {
-		components: { locality: name, region: state.abbreviation, country: US_COUNTRY_DISPLAY },
-		variantKey: "with-region-country",
-	}
-}
-
-/**
- * Build a TIGER adapter. Pure factory so multiple instances can be created in tests.
- */
-export function createTigerAdapter(): CorpusAdapter {
-	return {
-		id: TIGER_ADAPTER_ID,
-		defaultLicense: TIGER_DEFAULT_LICENSE,
-		description:
-			"US Census TIGER/Line streets + places consumer (public-domain); SQLite DB built via `mailwoman tiger fetch`.",
-
-		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			if (opts.country && opts.country !== "US") {
-				throw new Error(`tiger adapter: only US supported, got country=${opts.country}`)
-			}
-
-			const db = new DatabaseSync(opts.inputPath, { readOnly: true })
-			let emitted = 0
-
-			try {
-				const streetStmt = db.prepare(`SELECT linearid, fullname, zipl, zipr, statefp FROM tiger_streets`)
-				const placeStmt = db.prepare(`SELECT geoid, name, statefp, lsad FROM tiger_places`)
-
-				for (const row of streetStmt.iterate() as IterableIterator<TigerStreetRow>) {
-					if (opts.signal?.aborted) return
-
-					for (const variant of streetVariants(row)) {
-						if (opts.limit !== undefined && emitted >= opts.limit) return
-						const raw = formatAddress(variant.components, "US", { separator: ", " })
-
-						if (!raw) continue
-						const aligned = reconcileComponents(variant.components, raw)
-
-						if (!Object.keys(aligned).length) continue
-
-						yield {
-							raw,
-							components: aligned,
-							country: "US",
-							locale: "en-US",
-							source: TIGER_ADAPTER_ID,
-							source_id: `${TIGER_ADAPTER_ID}-st-${row.linearid}-${variant.variantKey}`,
-							corpus_version: "",
-							license: TIGER_DEFAULT_LICENSE,
-						}
-
-						emitted++
-					}
-				}
-
-				for (const row of placeStmt.iterate() as IterableIterator<TigerPlaceRow>) {
-					if (opts.signal?.aborted) return
-
-					for (const variant of placeVariants(row)) {
-						if (opts.limit !== undefined && emitted >= opts.limit) return
-						const raw = formatAddress(variant.components, "US", { separator: ", " })
-
-						if (!raw) continue
-						const aligned = reconcileComponents(variant.components, raw)
-
-						if (!Object.keys(aligned).length) continue
-
-						yield {
-							raw,
-							components: aligned,
-							country: "US",
-							locale: "en-US",
-							source: TIGER_ADAPTER_ID,
-							source_id: `${TIGER_ADAPTER_ID}-pl-${row.geoid}-${variant.variantKey}`,
-							corpus_version: "",
-							license: TIGER_DEFAULT_LICENSE,
-						}
-
-						emitted++
-					}
-				}
-			} finally {
-				db.close()
-			}
-		},
-	}
-}
-
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const tigerAdapter = createTigerAdapter()

package/src/adapters/tiger/street-decompose.ts

@@ -1,111 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Decompose a US street name into Stage 3 components: street_prefix, street, street_suffix.
- *
- *   Sources directionals and street types from the curated libpostal/en dictionaries
- *   (`core/data/libpostal/dictionaries/en/{directionals,street_types}.txt`). These are the same
- *   dictionaries the runtime classifiers (StreetPrefixClassifier, StreetSuffixClassifier) use, so
- *   corpus labels and runtime classifications agree on the vocabulary.
- *
- *   Examples: "N Main St" → { prefix: "N", street: "Main", suffix: "St" } "Pennsylvania Avenue NW" →
- *   { prefix: null, street: "Pennsylvania", suffix: "Avenue NW" } "Salmon St" → { prefix: null,
- *   street: "Salmon", suffix: "St" } "SE Hawthorne Blvd" → { prefix: "SE", street: "Hawthorne",
- *   suffix: "Blvd" }
- */
-
-import { readFileSync } from "node:fs"
-
-import { resourceDictionaryPath } from "@mailwoman/core/utils"
-import { TextSpliterator } from "spliterator"
-
-function loadDictionary(filename: string): Set<string> {
-	// `resourceDictionaryPath` already resolves both layouts — `core/data/...` from source and from the
-	// packaged `out/` tree. The candidate list this replaced named it TWICE and then guessed a third path
-	// off `process.cwd()`, and swallowed every error while probing, so a corrupt dictionary reported as a
-	// missing one.
-	const text = readFileSync(resourceDictionaryPath("libpostal", "en", filename), "utf8")
-	const set = new Set<string>()
-
-	// The largest libpostal dictionary is 8.4 KB, and this runs once per process at module load.
-	for (const line of TextSpliterator.from(text)) {
-		const trimmed = line.trim()
-
-		if (!trimmed || trimmed.startsWith("#")) continue
-
-		// libpostal format: canonical|abbr|abbr|... — index all forms
-		for (const form of trimmed.split("|")) {
-			const f = form.trim().toLowerCase()
-
-			if (f) {
-				set.add(f)
-			}
-		}
-	}
-
-	return set
-}
-
-const DIRECTIONALS = loadDictionary("directionals.txt")
-const STREET_TYPES = loadDictionary("street_types.txt")
-
-export interface DecomposedStreet {
-	prefix: string | null
-	street: string
-	suffix: string | null
-}
-
-/**
- * Decompose a US street name into prefix/name/suffix components.
- *
- * Conservative — only emits prefix/suffix when there's a clear directional or street-type keyword. Returns the original
- * as `street` if nothing matches.
- */
-export function decomposeStreet(fullname: string): DecomposedStreet {
-	const trimmed = fullname.trim()
-
-	if (!trimmed) return { prefix: null, street: "", suffix: null }
-
-	const tokens = trimmed.split(/\s+/)
-
-	if (tokens.length === 1) return { prefix: null, street: trimmed, suffix: null }
-
-	const norm = (s: string) => s.toLowerCase().replace(/\.$/, "")
-
-	let prefix: string | null = null
-	let suffix: string | null = null
-	let startIdx = 0
-	let endIdx = tokens.length
-
-	// Leading directional prefix
-	if (DIRECTIONALS.has(norm(tokens[0]!)) && tokens.length >= 2) {
-		prefix = tokens[0]!
-		startIdx = 1
-	}
-
-	// Trailing post-directional combined with street type (e.g. "Pennsylvania Ave NW")
-	const last = norm(tokens[endIdx - 1]!)
-	const secondLast = endIdx >= 2 ? norm(tokens[endIdx - 2]!) : ""
-
-	if (DIRECTIONALS.has(last) && STREET_TYPES.has(secondLast)) {
-		suffix = tokens.slice(endIdx - 2, endIdx).join(" ")
-		endIdx -= 2
-	} else if (STREET_TYPES.has(last) && endIdx - startIdx >= 2) {
-		suffix = tokens[endIdx - 1]!
-		endIdx -= 1
-	} else if (DIRECTIONALS.has(last) && endIdx - startIdx >= 2) {
-		// Post-directional without type
-		suffix = tokens[endIdx - 1]!
-		endIdx -= 1
-	}
-
-	const street = tokens.slice(startIdx, endIdx).join(" ").trim()
-
-	if (!street) {
-		return { prefix: null, street: trimmed, suffix: null }
-	}
-
-	return { prefix, street, suffix }
-}