@mailwoman/corpus 9.2.0 → 9.3.0

package/src/synthesizers/po-box.ts

@@ -1,339 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   PO box / PMB / Apartado / Boîte Postale synthesizer.
- *
- *   Generates BIO-labeled corpus rows where the delivery line is a PO box (mutually exclusive with
- *   street + house_number per USPS Pub 28 / DMM 508). Locale-aware: emits idiomatic forms for
- *   en-US, en-CA, en-GB, en-AU, fr-FR, fr-CA, es-ES, es-MX, es-AR.
- *
- *   Per-DeepSeek design:
- *
- *   - PMB ("Private Mailbox" — at CMRAs like UPS Store) shares the `po_box` tag with USPS PO Box.
- *       Disambiguation is a downstream heuristic (presence of a street line).
- *   - Whole-phrase span ("PO Box 123") not number-only ("123"). Matches existing golden eval.
- *   - 10% of outputs receive number-format noise (commas, dashes, embedded spaces) to harden against
- *       real-world OCR/transcription input.
- *   - PO boxes drop street/house_number/unit/street_prefix/street_suffix from input components.
- *
- *   References:
- *
- *   - USPS Pub 28 §28C2.040 — Private Mailbox formatting
- *   - USPS DMM 508 §4.1.4 / §4.5.4 — PO Box and street-addressed PO Box
- */
-
-import type { CanonicalRow } from "#types"
-
-/**
- * Digits a box number needs before a thousands comma is plausible (`1,234`).
- */
-const MIN_DIGITS_FOR_COMMA_GROUPING = 4
-
-/**
- * Digits a box number needs before a hyphen group is plausible (`12-34`).
- */
-const MIN_DIGITS_FOR_HYPHEN_GROUPING = 3
-
-/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
-   cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
-   output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
-   would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
-   extracted as named constants above. */
-
-export interface PoBoxBaseTuple {
-	locality: string
-	region: string
-	postcode: string
-	country: string
-}
-
-export interface LocaleTemplate {
-	locale: string
-	leaders: ReadonlyArray<string>
-	// Use 'pmb' to render as "STREET, PMB N, CITY ..." instead of replacing the street line.
-	pmb?: ReadonlyArray<string>
-}
-
-/**
- * The per-locale PO-box designator vocabulary (DeepSeek-signed list, see the header). Exported so shard builders
- * (scripts/build-po-box-cedex-shard.mjs) can reuse THIS list as the single source of truth for non-US leaders instead
- * of re-deriving it — the US slice additionally has `@mailwoman/codex/us` `US_PO_BOX_DESIGNATORS`/`isPOBox` as its
- * matcher-side truth.
- */
-export const PO_BOX_LOCALE_TEMPLATES: ReadonlyArray<LocaleTemplate> = [
-	{
-		locale: "en-US",
-		leaders: ["PO Box", "P.O. Box", "P.O.Box", "PO BOX", "POB", "Post Office Box", "Box"],
-		pmb: ["PMB", "#"],
-	},
-	{
-		locale: "en-CA",
-		leaders: ["PO Box", "P.O. Box", "POB", "Post Office Box"],
-		pmb: ["PMB", "#"],
-	},
-	{
-		locale: "en-GB",
-		leaders: ["PO Box", "P.O. Box", "Post Office Box"],
-	},
-	{
-		locale: "en-AU",
-		leaders: ["PO Box", "P.O. Box", "Post Office Box", "GPO Box", "Locked Bag"],
-	},
-	{
-		locale: "en-NZ",
-		leaders: ["PO Box", "P.O. Box", "Post Office Box", "Private Bag", "Private Box"],
-	},
-	{
-		locale: "fr-FR",
-		leaders: ["BP", "B.P.", "Boîte Postale", "BP."],
-	},
-	{
-		locale: "fr-CA",
-		leaders: ["CP", "C.P.", "Case Postale", "BP", "B.P."],
-	},
-	{
-		locale: "es-ES",
-		leaders: ["Apdo.", "Apdo", "Apartado", "Apartado de Correos"],
-	},
-	{
-		locale: "es-MX",
-		leaders: ["Apdo.", "Apartado", "Apartado Postal", "AP"],
-	},
-	{
-		locale: "es-AR",
-		leaders: ["Casilla", "Casilla de Correo", "CC"],
-	},
-]
-
-const LEADERS_BY_LOCALE = new Map<string, LocaleTemplate>(PO_BOX_LOCALE_TEMPLATES.map((t) => [t.locale, t]))
-
-/**
- * Inject number-format noise into a box number string. Returns the noisy variant or the original (10% probability of
- * noise per the design).
- */
-export function maybeNoisifyBoxNumber(num: string, random: () => number): string {
-	if (random() > 0.1) return num
-
-	const variants: Array<(s: string) => string> = [
-		// Thousand-separator comma (real input: "Box 1,234")
-		(s) => (s.length >= MIN_DIGITS_FOR_COMMA_GROUPING ? `${s.slice(0, -3)},${s.slice(-3)}` : s),
-		// Embedded dash (real input: "PMB-200")
-		(s) => (s.length >= MIN_DIGITS_FOR_HYPHEN_GROUPING ? `${s.slice(0, -2)}-${s.slice(-2)}` : s),
-		// Embedded spaces (real input from OCR: "1 2 3 4")
-		(s) => s.split("").join(" "),
-	]
-
-	const f = variants[Math.floor(random() * variants.length)]!
-
-	return f(num)
-}
-
-/**
- * Compose a PO box phrase like "PO Box 123" or "PMB 200".
- *
- * Returns both the phrase and the canonical leader+number so the BIO aligner can mark the entire span as `po_box`.
- */
-export function composePoBoxPhrase(leader: string, number: string): string {
-	return `${leader} ${number}`
-}
-
-export interface SynthesizedPoBoxRow {
-	raw: string
-	components: CanonicalRow["components"]
-	locale: string
-	template: "po-box" | "pmb-with-street" | "military-po-box"
-}
-
-export interface PoBoxSynthesisOpts {
-	/**
-	 * Random function — pass deterministic seed for tests. Default Math.random.
-	 */
-	random?: () => number
-	/**
-	 * Number generator. Default uniform over 1..99999.
-	 */
-	pickNumber?: (random: () => number) => string
-	/**
-	 * PMB probability when locale supports it (and a street is provided in the base tuple).
-	 */
-	pmbRatio?: number
-}
-
-function defaultPickNumber(random: () => number): string {
-	// 70% of real PO boxes are 1-5 digits; long ones exist (USPS allows up to ~6 digits).
-	const r = random()
-
-	if (r < 0.3) return String(1 + Math.floor(random() * 99))
-
-	// 1-99
-	if (r < 0.7) return String(100 + Math.floor(random() * 900))
-
-	// 100-999
-	if (r < 0.95) return String(1000 + Math.floor(random() * 9000))
-
-	// 1000-9999
-	return String(10_000 + Math.floor(random() * 90_000)) // 10000-99999
-}
-
-/**
- * Generate one PO box row for a base (locality, region, postcode, country) tuple. Picks a locale-appropriate leader and
- * number. Optionally generates a PMB variant when the base tuple includes a street.
- */
-export function synthesizePoBoxRow(
-	base: PoBoxBaseTuple & { street?: string; houseNumber?: string },
-	opts: PoBoxSynthesisOpts = {}
-): SynthesizedPoBoxRow | null {
-	const random = opts.random ?? Math.random
-	const pickNumber = opts.pickNumber ?? defaultPickNumber
-	const pmbRatio = opts.pmbRatio ?? 0
-
-	const locale = countryToLocale(base.country)
-	const tpl = LEADERS_BY_LOCALE.get(locale)
-
-	if (!tpl) return null
-
-	const number = maybeNoisifyBoxNumber(pickNumber(random), random)
-	const leader = tpl.leaders[Math.floor(random() * tpl.leaders.length)]!
-	const poBoxPhrase = composePoBoxPhrase(leader, number)
-
-	// PMB variant: requires both a street and a PMB-supporting locale.
-	const wantPmb = base.street && tpl.pmb && random() < pmbRatio
-
-	if (wantPmb) {
-		const pmbLeader = tpl.pmb![Math.floor(random() * tpl.pmb!.length)]!
-		const pmbPhrase = composePoBoxPhrase(pmbLeader, number)
-		const streetLine = base.houseNumber ? `${base.houseNumber} ${base.street}` : base.street!
-		const raw = `${streetLine}, ${pmbPhrase}, ${base.locality}, ${base.region} ${base.postcode}`
-
-		return {
-			raw,
-			components: {
-				...(base.houseNumber ? { house_number: base.houseNumber } : {}),
-				street: base.street!,
-				po_box: pmbPhrase,
-				locality: base.locality,
-				region: base.region,
-				postcode: base.postcode,
-				country: base.country,
-			},
-			locale,
-			template: "pmb-with-street",
-		}
-	}
-
-	// Standard PO box: replaces the street line entirely. Region-optional — NZ (and other region-less
-	// locales) read "Private Bag 12, Auckland 1010" with no region token between locality and postcode.
-	const hasRegion = Boolean(base.region && base.region.trim())
-	const tail = hasRegion ? `${base.locality}, ${base.region} ${base.postcode}` : `${base.locality} ${base.postcode}`
-	const raw = `${poBoxPhrase}, ${tail}`
-
-	return {
-		raw,
-		components: {
-			po_box: poBoxPhrase,
-			locality: base.locality,
-			...(hasRegion ? { region: base.region } : {}),
-			postcode: base.postcode,
-			country: base.country,
-		},
-		locale,
-		template: "po-box",
-	}
-}
-
-/**
- * The US military/diplomatic PO-box class (#517). A distinct shape the leader-based locale templates can't express: a
- * unit line (`PSC <id> Box <box>`, `CMR <id> Box <box>`, `Unit <id> [Box <box>]`) tagged `po_box`, then the post-office
- * code (APO/FPO/DPO) as the locality and the armed-forces region (AA/AE/AP) as the region, with a theatre-specific ZIP.
- * Authoritative reference + citations: `@mailwoman/codex` `codex/us/military-address.ts`; the small constants are
- * inlined here so the generator is self-contained.
- */
-const MIL_UNITS: ReadonlyArray<{ code: string; boxRequired: boolean }> = [
-	{ code: "PSC", boxRequired: true },
-	{ code: "CMR", boxRequired: true },
-	{ code: "Unit", boxRequired: false },
-]
-
-const MIL_PO_CODES = ["APO", "FPO", "DPO"] as const
-
-/**
- * Region → plausible ZIP prefix (AE Europe 09xxx, AP Pacific 962-966xx, AA Americas 340xx).
- */
-const MIL_REGION_ZIP: ReadonlyArray<{ region: string; zip: (r: () => number) => string }> = [
-	{ region: "AE", zip: (r) => `09${String(Math.floor(r() * 1000)).padStart(3, "0")}` },
-	{ region: "AP", zip: (r) => `96${String(200 + Math.floor(r() * 100)).padStart(3, "0")}` },
-	{ region: "AA", zip: (r) => `340${String(Math.floor(r() * 100)).padStart(2, "0")}` },
-]
-
-/**
- * Generate one US military/diplomatic PO-box row (#517). Self-contained — draws no base tuple.
- */
-export function synthesizeMilitaryPoBoxRow(opts: PoBoxSynthesisOpts = {}): SynthesizedPoBoxRow {
-	const random = opts.random ?? Math.random
-	const unit = MIL_UNITS[Math.floor(random() * MIL_UNITS.length)]!
-	const unitID = String(1 + Math.floor(random() * 9999))
-	const { region, zip } = MIL_REGION_ZIP[Math.floor(random() * MIL_REGION_ZIP.length)]!
-	const zipStr = zip(random)
-	const po = MIL_PO_CODES[Math.floor(random() * MIL_PO_CODES.length)]!
-	const hasBox = unit.boxRequired || random() < 0.5
-	const unitLine = hasBox ? `${unit.code} ${unitID} Box ${1 + Math.floor(random() * 9999)}` : `${unit.code} ${unitID}`
-	const raw = `${unitLine}, ${po} ${region} ${zipStr}`
-
-	return {
-		raw,
-		components: {
-			po_box: unitLine,
-			locality: po,
-			region,
-			postcode: zipStr,
-			country: "US",
-		},
-		locale: "en-US",
-		template: "military-po-box",
-	}
-}
-
-/**
- * Map a country code (ISO-3166-1 alpha-2 or alpha-3, or country display name) to the locale code we have a PO box
- * template for.
- */
-export function countryToLocale(country: string): string {
-	const c = country.trim().toUpperCase()
-
-	if (c === "US" || c === "USA" || c === "UNITED STATES") return "en-US"
-
-	if (c === "CA" || c === "CAN" || c === "CANADA") return "en-CA"
-
-	if (c === "GB" || c === "UK" || c === "GBR" || c === "UNITED KINGDOM") return "en-GB"
-
-	if (c === "AU" || c === "AUS" || c === "AUSTRALIA") return "en-AU"
-
-	if (c === "NZ" || c === "NZL" || c === "NEW ZEALAND") return "en-NZ"
-
-	if (c === "FR" || c === "FRA" || c === "FRANCE") return "fr-FR"
-
-	if (c === "ES" || c === "ESP" || c === "SPAIN") return "es-ES"
-
-	if (c === "MX" || c === "MEX" || c === "MEXICO") return "es-MX"
-
-	if (c === "AR" || c === "ARG" || c === "ARGENTINA") return "es-AR"
-
-	return "en-US"
-}
-
-/**
- * All locales we synthesize for. Exposed for tests and for source-weight tuning.
- */
-export function supportedLocales(): ReadonlyArray<string> {
-	return PO_BOX_LOCALE_TEMPLATES.map((t) => t.locale)
-}
-
-/**
- * Locales whose standard PO-box delivery line carries NO region token — the address reads `<po_box>, <locality>
- * <postcode>` with nothing between locality and postcode (#517). NZ is the canonical case (`Private Bag 12, Auckland
- * 1010`). Consumers (e.g. the synth-po-box adapter) use this to avoid discarding region-less input tuples for these
- * locales as "missing region".
- */
-export const REGION_OPTIONAL_LOCALES: ReadonlySet<string> = new Set(["en-NZ"])

package/src/synthesizers/street.ts

@@ -1,274 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Street-decomposition synthesizer for Stage 3 training. Generates address rows where
- *   `street_prefix`, `street`, and `street_suffix` are emitted as separate BIO spans (instead of
- *   monolithic `street`). Mirrors the PO box synthesizer pattern.
- *
- *   Why this exists: TIGER/NAD/BAN adapter changes (committed earlier tonight) emit decomposed
- *   components from raw source data, but the v0.4.0 parquet shards on Modal were built BEFORE those
- *   changes. Rebuilding the full corpus requires downloading raw TIGER/NAD/BAN data and re-running
- *   adapters end-to-end — out of scope for a single night shift. This synthesizer takes (locality,
- *   region, postcode) tuples and produces freshly-decomposed Stage 3 training rows, same shape as
- *   the PO box pipeline.
- *
- *   Note: uses the SAME decomposition logic as TIGER's `decomposeStreet()` so the synthetic
- *   distribution matches what the model would see if/when TIGER shards are rebuilt with the new
- *   adapter.
- */
-
-import { isPresent } from "@mailwoman/core/objects"
-
-import { decomposeStreet } from "#adapters/tiger/street-decompose"
-import type { CanonicalRow } from "#types"
-
-// Hand-curated US street name pool. Real frequency-weighted street names — sampled
-// from US Census TIGER 2024 top-1000 by occurrence count. Keep ~50 entries so the
-// synthesis distribution doesn't overfit to a tiny vocabulary.
-/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
-   cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
-   output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
-   would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
-   extracted as named constants above. */
-
-const STREET_NAMES = [
-	"Main",
-	"Oak",
-	"Maple",
-	"Pine",
-	"Cedar",
-	"Elm",
-	"Washington",
-	"Lincoln",
-	"Jefferson",
-	"Park",
-	"Lake",
-	"Hill",
-	"Spring",
-	"Center",
-	"Church",
-	"Mill",
-	"School",
-	"River",
-	"Highland",
-	"Sunset",
-	"Forest",
-	"Meadow",
-	"Ridge",
-	"Valley",
-	"Lakeview",
-	"Hillcrest",
-	"1st",
-	"2nd",
-	"3rd",
-	"4th",
-	"5th",
-	"6th",
-	"7th",
-	"8th",
-	"9th",
-	"10th",
-	"Adams",
-	"Madison",
-	"Monroe",
-	"Jackson",
-	"Franklin",
-	"Kennedy",
-	"Roosevelt",
-	"Broadway",
-	"Market",
-	"Commerce",
-	"Industrial",
-	"Airport",
-	"Hospital",
-	"Sherman",
-	"Grant",
-	"Lee",
-	"Custer",
-	"Lewis",
-	"Clark",
-] as const
-
-const STREET_SUFFIXES = [
-	"St",
-	"Street",
-	"Ave",
-	"Avenue",
-	"Rd",
-	"Road",
-	"Blvd",
-	"Boulevard",
-	"Dr",
-	"Drive",
-	"Ln",
-	"Lane",
-	"Way",
-	"Ct",
-	"Court",
-	"Pl",
-	"Place",
-	"Pkwy",
-	"Parkway",
-	"Ter",
-	"Terrace",
-	"Cir",
-	"Circle",
-	"Hwy",
-	"Highway",
-] as const
-
-const DIRECTIONAL_PREFIXES = [
-	"",
-	"",
-	"",
-	"",
-	"", // 5 empty entries → 50% no prefix
-	"N",
-	"S",
-	"E",
-	"W",
-	"NE",
-	"NW",
-	"SE",
-	"SW",
-	"North",
-	"South",
-	"East",
-	"West",
-] as const
-
-const TRAILING_DIRECTIONALS = [
-	"",
-	"",
-	"",
-	"",
-	"",
-	"",
-	"",
-	"", // 8 empty → 80% no trailing
-	"N",
-	"S",
-	"E",
-	"W",
-	"NE",
-	"NW",
-	"SE",
-	"SW",
-] as const
-
-export interface StreetBaseTuple {
-	locality: string
-	region: string
-	postcode: string
-	country: string
-}
-
-export interface SynthesizedStreetRow {
-	raw: string
-	components: CanonicalRow["components"]
-	locale: string
-}
-
-export interface StreetSynthesisOpts {
-	random?: () => number
-	/**
-	 * Probability of emitting house_number alongside the street. Default 0.85.
-	 */
-	includeHouseNumberProb?: number
-	/**
-	 * Probability of emitting the street BARE — no `, City, ST ZIP` tail and no region/locality/ postcode components
-	 * (just `street_prefix`/`street`/`street_suffix` + optional `house_number`). Default 0 (preserves the original
-	 * full-address behavior exactly, including the RNG sequence). Set >0 to teach the model that a bare `10th Ave` /
-	 * `Main St` is a STREET, not a locality — the functional-test failure cluster (bare streets mislabeled `locality`),
-	 * the bare-format analogue of the v0.7.x intersection-bare fix.
-	 */
-	bareProb?: number
-}
-
-function pick<T>(arr: ReadonlyArray<T>, random: () => number): T {
-	return arr[Math.floor(random() * arr.length)]!
-}
-
-function randomHouseNumber(random: () => number): string {
-	// US house number distribution: skewed low. 1-99 (30%), 100-999 (40%),
-	// 1000-9999 (25%), 10000+ (5%).
-	const r = random()
-
-	if (r < 0.3) return String(1 + Math.floor(random() * 99))
-
-	if (r < 0.7) return String(100 + Math.floor(random() * 900))
-
-	if (r < 0.95) return String(1000 + Math.floor(random() * 9000))
-
-	return String(10_000 + Math.floor(random() * 89_999))
-}
-
-/**
- * Synthesize a US street address with decomposed Stage 3 components. The street is built from PREFIX + NAME + SUFFIX,
- * then passed through the same `decomposeStreet()` utility the TIGER adapter uses — guarantees the synthetic
- * distribution matches the canonical decomposition logic.
- */
-export function synthesizeStreetRow(
-	base: StreetBaseTuple,
-	opts: StreetSynthesisOpts = {}
-): SynthesizedStreetRow | null {
-	const random = opts.random ?? Math.random
-	const includeHN = opts.includeHouseNumberProb ?? 0.85
-
-	if (base.country !== "US") return null
-
-	const prefix = pick(DIRECTIONAL_PREFIXES, random)
-	const name = pick(STREET_NAMES, random)
-	const suffix = pick(STREET_SUFFIXES, random)
-	const trailing = pick(TRAILING_DIRECTIONALS, random)
-
-	// Build the "full" street string the adapter would receive from TIGER FULLNAME.
-	const parts = [prefix, name, suffix, trailing].filter(isPresent)
-	const fullStreet = parts.join(" ")
-
-	// Pass through the same decomposeStreet TIGER uses — match the training distribution.
-	const decomposed = decomposeStreet(fullStreet)
-
-	// NOTE: country is intentionally omitted. We don't emit "USA" or "US" in the raw
-	// string, and the aligner's fuzzy match (edit distance 2) will spuriously match
-	// "US" against any 2-char token (e.g. a house number "45" is exactly 2 substitutions
-	// from "US"). The PO box synthesizer skips country for the same reason.
-	// Bare mode is guarded by `> 0` so the default (bareProb=0) consumes no RNG and reproduces the
-	// original full-address output byte-for-byte.
-	const bareProb = opts.bareProb ?? 0
-	const bare = bareProb > 0 && random() < bareProb
-
-	const components: CanonicalRow["components"] = bare
-		? {}
-		: {
-				region: base.region,
-				locality: base.locality,
-				postcode: base.postcode,
-			}
-
-	if (decomposed.prefix) {
-		components.street_prefix = decomposed.prefix
-	}
-
-	if (decomposed.street) {
-		components.street = decomposed.street
-	}
-
-	if (decomposed.suffix) {
-		components.street_suffix = decomposed.suffix
-	}
-
-	let raw: string
-
-	if (random() < includeHN) {
-		const hn = randomHouseNumber(random)
-		components.house_number = hn
-		raw = bare ? `${hn} ${fullStreet}` : `${hn} ${fullStreet}, ${base.locality}, ${base.region} ${base.postcode}`
-	} else {
-		raw = bare ? fullStreet : `${fullStreet}, ${base.locality}, ${base.region} ${base.postcode}`
-	}
-
-	return { raw, components, locale: "en-US" }
-}