@mailwoman/corpus 9.1.0 → 9.3.0

package/lib/synthesizers/po-box.ts

@@ -0,0 +1,323 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   PO box / PMB / Apartado / Boîte Postale synthesizer.
+ *
+ *   Generates BIO-labeled corpus rows where the delivery line is a PO box (mutually exclusive with
+ *   street + house_number per USPS Pub 28 / DMM 508). Locale-aware: emits idiomatic forms for
+ *   en-US, en-CA, en-GB, en-AU, fr-FR, fr-CA, es-ES, es-MX, es-AR.
+ *
+ *   Per-DeepSeek design:
+ *
+ *   - PMB ("Private Mailbox" — at CMRAs like UPS Store) shares the `po_box` tag with USPS PO Box.
+ *       Disambiguation is a downstream heuristic (presence of a street line).
+ *   - Whole-phrase span ("PO Box 123") not number-only ("123"). Matches existing golden eval.
+ *   - 10% of outputs receive number-format noise (commas, dashes, embedded spaces) to harden against
+ *       real-world OCR/transcription input.
+ *   - PO boxes drop street/house_number/unit/street_prefix/street_suffix from input components.
+ *
+ *   References:
+ *
+ *   - USPS Pub 28 §28C2.040 — Private Mailbox formatting
+ *   - USPS DMM 508 §4.1.4 / §4.5.4 — PO Box and street-addressed PO Box
+ */
+
+import { countryToLocale as baseCountryToLocale, pick, tieredNumber } from "#synthesizers/utils"
+import type { CanonicalRow } from "#types"
+
+/**
+ * Digits a box number needs before a thousands comma is plausible (`1,234`).
+ */
+const MIN_DIGITS_FOR_COMMA_GROUPING = 4
+
+/**
+ * Digits a box number needs before a hyphen group is plausible (`12-34`).
+ */
+const MIN_DIGITS_FOR_HYPHEN_GROUPING = 3
+
+/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
+   cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
+   output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
+   would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
+   extracted as named constants above. */
+
+export interface PoBoxBaseTuple {
+	locality: string
+	region: string
+	postcode: string
+	country: string
+}
+
+export interface LocaleTemplate {
+	locale: string
+	leaders: ReadonlyArray<string>
+	// Use 'pmb' to render as "STREET, PMB N, CITY ..." instead of replacing the street line.
+	pmb?: ReadonlyArray<string>
+}
+
+/**
+ * The per-locale PO-box designator vocabulary (DeepSeek-signed list, see the header). Exported so slice builders
+ * (scripts/build-po-box-cedex-slice.mjs) can reuse THIS list as the single source of truth for non-US leaders instead
+ * of re-deriving it — the US slice additionally has `@mailwoman/codex/us` `US_PO_BOX_DESIGNATORS`/`isPOBox` as its
+ * matcher-side truth.
+ */
+export const PO_BOX_LOCALE_TEMPLATES: ReadonlyArray<LocaleTemplate> = [
+	{
+		locale: "en-US",
+		leaders: ["PO Box", "P.O. Box", "P.O.Box", "PO BOX", "POB", "Post Office Box", "Box"],
+		pmb: ["PMB", "#"],
+	},
+	{
+		locale: "en-CA",
+		leaders: ["PO Box", "P.O. Box", "POB", "Post Office Box"],
+		pmb: ["PMB", "#"],
+	},
+	{
+		locale: "en-GB",
+		leaders: ["PO Box", "P.O. Box", "Post Office Box"],
+	},
+	{
+		locale: "en-AU",
+		leaders: ["PO Box", "P.O. Box", "Post Office Box", "GPO Box", "Locked Bag"],
+	},
+	{
+		locale: "en-NZ",
+		leaders: ["PO Box", "P.O. Box", "Post Office Box", "Private Bag", "Private Box"],
+	},
+	{
+		locale: "fr-FR",
+		leaders: ["BP", "B.P.", "Boîte Postale", "BP."],
+	},
+	{
+		locale: "fr-CA",
+		leaders: ["CP", "C.P.", "Case Postale", "BP", "B.P."],
+	},
+	{
+		locale: "es-ES",
+		leaders: ["Apdo.", "Apdo", "Apartado", "Apartado de Correos"],
+	},
+	{
+		locale: "es-MX",
+		leaders: ["Apdo.", "Apartado", "Apartado Postal", "AP"],
+	},
+	{
+		locale: "es-AR",
+		leaders: ["Casilla", "Casilla de Correo", "CC"],
+	},
+]
+
+const LEADERS_BY_LOCALE = new Map<string, LocaleTemplate>(PO_BOX_LOCALE_TEMPLATES.map((t) => [t.locale, t]))
+
+/**
+ * Inject number-format noise into a box number string. Returns the noisy variant or the original (10% probability of
+ * noise per the design).
+ */
+export function maybeNoisifyBoxNumber(num: string, random: () => number): string {
+	if (random() > 0.1) return num
+
+	const variants: Array<(s: string) => string> = [
+		// Thousand-separator comma (real input: "Box 1,234")
+		(s) => (s.length >= MIN_DIGITS_FOR_COMMA_GROUPING ? `${s.slice(0, -3)},${s.slice(-3)}` : s),
+		// Embedded dash (real input: "PMB-200")
+		(s) => (s.length >= MIN_DIGITS_FOR_HYPHEN_GROUPING ? `${s.slice(0, -2)}-${s.slice(-2)}` : s),
+		// Embedded spaces (real input from OCR: "1 2 3 4")
+		(s) => s.split("").join(" "),
+	]
+
+	const f = pick(variants, random)
+
+	return f(num)
+}
+
+/**
+ * Compose a PO box phrase like "PO Box 123" or "PMB 200".
+ *
+ * Returns both the phrase and the canonical leader+number so the BIO aligner can mark the entire span as `po_box`.
+ */
+export function composePoBoxPhrase(leader: string, number: string): string {
+	return `${leader} ${number}`
+}
+
+export interface SynthesizedPoBoxRow {
+	raw: string
+	components: CanonicalRow["components"]
+	locale: string
+	template: "po-box" | "pmb-with-street" | "military-po-box"
+}
+
+export interface PoBoxSynthesisOpts {
+	/**
+	 * Random function — pass deterministic seed for tests. Default Math.random.
+	 */
+	random?: () => number
+	/**
+	 * Number generator. Default uniform over 1..99999.
+	 */
+	pickNumber?: (random: () => number) => string
+	/**
+	 * PMB probability when locale supports it (and a street is provided in the base tuple).
+	 */
+	pmbRatio?: number
+}
+
+function defaultPickNumber(random: () => number): string {
+	// 70% of real PO boxes are 1-5 digits; long ones exist (USPS allows up to ~6 digits).
+	// Bands: 1-99, 100-999, 1000-9999, 10000-99999 (span 90_000 — the street generator's 89_999 is its own).
+	return tieredNumber(random, [
+		{ cutoff: 0.3, base: 1, span: 99 },
+		{ cutoff: 0.7, base: 100, span: 900 },
+		{ cutoff: 0.95, base: 1000, span: 9000 },
+		{ base: 10_000, span: 90_000 },
+	])
+}
+
+/**
+ * Generate one PO box row for a base (locality, region, postcode, country) tuple. Picks a locale-appropriate leader and
+ * number. Optionally generates a PMB variant when the base tuple includes a street.
+ */
+export function synthesizePoBoxRow(
+	base: PoBoxBaseTuple & { street?: string; houseNumber?: string },
+	opts: PoBoxSynthesisOpts = {}
+): SynthesizedPoBoxRow | null {
+	const random = opts.random ?? Math.random
+	const pickNumber = opts.pickNumber ?? defaultPickNumber
+	const pmbRatio = opts.pmbRatio ?? 0
+
+	const locale = countryToLocale(base.country)
+	const tpl = LEADERS_BY_LOCALE.get(locale)
+
+	if (!tpl) return null
+
+	const number = maybeNoisifyBoxNumber(pickNumber(random), random)
+	const leader = pick(tpl.leaders, random)
+	const poBoxPhrase = composePoBoxPhrase(leader, number)
+
+	// PMB variant: requires both a street and a PMB-supporting locale.
+	const wantPmb = base.street && tpl.pmb && random() < pmbRatio
+
+	if (wantPmb) {
+		const pmbLeader = pick(tpl.pmb!, random)
+		const pmbPhrase = composePoBoxPhrase(pmbLeader, number)
+		const streetLine = base.houseNumber ? `${base.houseNumber} ${base.street}` : base.street!
+		const raw = `${streetLine}, ${pmbPhrase}, ${base.locality}, ${base.region} ${base.postcode}`
+
+		return {
+			raw,
+			components: {
+				...(base.houseNumber ? { house_number: base.houseNumber } : {}),
+				street: base.street!,
+				po_box: pmbPhrase,
+				locality: base.locality,
+				region: base.region,
+				postcode: base.postcode,
+				country: base.country,
+			},
+			locale,
+			template: "pmb-with-street",
+		}
+	}
+
+	// Standard PO box: replaces the street line entirely. Region-optional — NZ (and other region-less
+	// locales) read "Private Bag 12, Auckland 1010" with no region token between locality and postcode.
+	const hasRegion = Boolean(base.region && base.region.trim())
+	const tail = hasRegion ? `${base.locality}, ${base.region} ${base.postcode}` : `${base.locality} ${base.postcode}`
+	const raw = `${poBoxPhrase}, ${tail}`
+
+	return {
+		raw,
+		components: {
+			po_box: poBoxPhrase,
+			locality: base.locality,
+			...(hasRegion ? { region: base.region } : {}),
+			postcode: base.postcode,
+			country: base.country,
+		},
+		locale,
+		template: "po-box",
+	}
+}
+
+/**
+ * The US military/diplomatic PO-box class (#517). A distinct shape the leader-based locale templates can't express: a
+ * unit line (`PSC <id> Box <box>`, `CMR <id> Box <box>`, `Unit <id> [Box <box>]`) tagged `po_box`, then the post-office
+ * code (APO/FPO/DPO) as the locality and the armed-forces region (AA/AE/AP) as the region, with a theatre-specific ZIP.
+ * Authoritative reference + citations: `@mailwoman/codex` `codex/us/military-address.ts`; the small constants are
+ * inlined here so the generator is self-contained.
+ */
+const MIL_UNITS: ReadonlyArray<{ code: string; boxRequired: boolean }> = [
+	{ code: "PSC", boxRequired: true },
+	{ code: "CMR", boxRequired: true },
+	{ code: "Unit", boxRequired: false },
+]
+
+const MIL_PO_CODES = ["APO", "FPO", "DPO"] as const
+
+/**
+ * Region → plausible ZIP prefix (AE Europe 09xxx, AP Pacific 962-966xx, AA Americas 340xx).
+ */
+const MIL_REGION_ZIP: ReadonlyArray<{ region: string; zip: (r: () => number) => string }> = [
+	{ region: "AE", zip: (r) => `09${String(Math.floor(r() * 1000)).padStart(3, "0")}` },
+	{ region: "AP", zip: (r) => `96${String(200 + Math.floor(r() * 100)).padStart(3, "0")}` },
+	{ region: "AA", zip: (r) => `340${String(Math.floor(r() * 100)).padStart(2, "0")}` },
+]
+
+/**
+ * Generate one US military/diplomatic PO-box row (#517). Self-contained — draws no base tuple.
+ */
+export function synthesizeMilitaryPoBoxRow(opts: PoBoxSynthesisOpts = {}): SynthesizedPoBoxRow {
+	const random = opts.random ?? Math.random
+	const unit = pick(MIL_UNITS, random)
+	const unitID = String(1 + Math.floor(random() * 9999))
+	const { region, zip } = pick(MIL_REGION_ZIP, random)
+	const zipStr = zip(random)
+	const po = pick(MIL_PO_CODES, random)
+	const hasBox = unit.boxRequired || random() < 0.5
+	const unitLine = hasBox ? `${unit.code} ${unitID} Box ${1 + Math.floor(random() * 9999)}` : `${unit.code} ${unitID}`
+	const raw = `${unitLine}, ${po} ${region} ${zipStr}`
+
+	return {
+		raw,
+		components: {
+			po_box: unitLine,
+			locality: po,
+			region,
+			postcode: zipStr,
+			country: "US",
+		},
+		locale: "en-US",
+		template: "military-po-box",
+	}
+}
+
+const PO_BOX_TEMPLATE_LOCALES: ReadonlySet<string> = new Set(PO_BOX_LOCALE_TEMPLATES.map((t) => t.locale))
+
+/**
+ * Map a country code (ISO-3166-1 alpha-2 or alpha-3, or country display name) to the locale code we have a PO box
+ * template for.
+ *
+ * TEMPLATE-SCOPED on top of the shared `#synthesizers/utils` map: a locale the shared map resolves but
+ * {@link PO_BOX_LOCALE_TEMPLATES} does not carry falls back to `en-US`, so `DE` (shared: `de-DE`, no PO-box template)
+ * keeps rendering the en-US box vocabulary here.
+ */
+export function countryToLocale(country: string): string {
+	const locale = baseCountryToLocale(country)
+
+	return PO_BOX_TEMPLATE_LOCALES.has(locale) ? locale : "en-US"
+}
+
+/**
+ * All locales we synthesize for. Exposed for tests and for source-weight tuning.
+ */
+export function supportedLocales(): ReadonlyArray<string> {
+	return PO_BOX_LOCALE_TEMPLATES.map((t) => t.locale)
+}
+
+/**
+ * Locales whose standard PO-box delivery line carries NO region token — the address reads `<po_box>, <locality>
+ * <postcode>` with nothing between locality and postcode (#517). NZ is the canonical case (`Private Bag 12, Auckland
+ * 1010`). Consumers (e.g. the synth-po-box adapter) use this to avoid discarding region-less input tuples for these
+ * locales as "missing region".
+ */
+export const REGION_OPTIONAL_LOCALES: ReadonlySet<string> = new Set(["en-NZ"])

package/lib/synthesizers/street.ts

@@ -0,0 +1,269 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Street-decomposition synthesizer for Stage 3 training. Generates address rows where
+ *   `street_prefix`, `street`, and `street_suffix` are emitted as separate BIO spans (instead of
+ *   monolithic `street`). Mirrors the PO box synthesizer pattern.
+ *
+ *   Why this exists: TIGER/NAD/BAN adapter changes (committed earlier tonight) emit decomposed
+ *   components from raw source data, but the v0.4.0 parquet slices on Modal were built BEFORE those
+ *   changes. Rebuilding the full corpus requires downloading raw TIGER/NAD/BAN data and re-running
+ *   adapters end-to-end — out of scope for a single night shift. This synthesizer takes (locality,
+ *   region, postcode) tuples and produces freshly-decomposed Stage 3 training rows, same shape as
+ *   the PO box pipeline.
+ *
+ *   Note: uses the SAME decomposition logic as TIGER's `decomposeStreet()` so the synthetic
+ *   distribution matches what the model would see if/when TIGER slices are rebuilt with the new
+ *   adapter.
+ */
+
+import { isPresent } from "@mailwoman/core/objects"
+
+import { decomposeStreet } from "#adapters/tiger/street-decompose"
+import { pick, tieredNumber } from "#synthesizers/utils"
+import type { CanonicalRow } from "#types"
+
+// Hand-curated US street name pool. Real frequency-weighted street names — sampled
+// from US Census TIGER 2024 top-1000 by occurrence count. Keep ~50 entries so the
+// synthesis distribution doesn't overfit to a tiny vocabulary.
+/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
+   cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
+   output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
+   would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
+   extracted as named constants above. */
+
+const STREET_NAMES = [
+	"Main",
+	"Oak",
+	"Maple",
+	"Pine",
+	"Cedar",
+	"Elm",
+	"Washington",
+	"Lincoln",
+	"Jefferson",
+	"Park",
+	"Lake",
+	"Hill",
+	"Spring",
+	"Center",
+	"Church",
+	"Mill",
+	"School",
+	"River",
+	"Highland",
+	"Sunset",
+	"Forest",
+	"Meadow",
+	"Ridge",
+	"Valley",
+	"Lakeview",
+	"Hillcrest",
+	"1st",
+	"2nd",
+	"3rd",
+	"4th",
+	"5th",
+	"6th",
+	"7th",
+	"8th",
+	"9th",
+	"10th",
+	"Adams",
+	"Madison",
+	"Monroe",
+	"Jackson",
+	"Franklin",
+	"Kennedy",
+	"Roosevelt",
+	"Broadway",
+	"Market",
+	"Commerce",
+	"Industrial",
+	"Airport",
+	"Hospital",
+	"Sherman",
+	"Grant",
+	"Lee",
+	"Custer",
+	"Lewis",
+	"Clark",
+] as const
+
+const STREET_SUFFIXES = [
+	"St",
+	"Street",
+	"Ave",
+	"Avenue",
+	"Rd",
+	"Road",
+	"Blvd",
+	"Boulevard",
+	"Dr",
+	"Drive",
+	"Ln",
+	"Lane",
+	"Way",
+	"Ct",
+	"Court",
+	"Pl",
+	"Place",
+	"Pkwy",
+	"Parkway",
+	"Ter",
+	"Terrace",
+	"Cir",
+	"Circle",
+	"Hwy",
+	"Highway",
+] as const
+
+const DIRECTIONAL_PREFIXES = [
+	"",
+	"",
+	"",
+	"",
+	"", // 5 empty entries → 50% no prefix
+	"N",
+	"S",
+	"E",
+	"W",
+	"NE",
+	"NW",
+	"SE",
+	"SW",
+	"North",
+	"South",
+	"East",
+	"West",
+] as const
+
+const TRAILING_DIRECTIONALS = [
+	"",
+	"",
+	"",
+	"",
+	"",
+	"",
+	"",
+	"", // 8 empty → 80% no trailing
+	"N",
+	"S",
+	"E",
+	"W",
+	"NE",
+	"NW",
+	"SE",
+	"SW",
+] as const
+
+export interface StreetBaseTuple {
+	locality: string
+	region: string
+	postcode: string
+	country: string
+}
+
+export interface SynthesizedStreetRow {
+	raw: string
+	components: CanonicalRow["components"]
+	locale: string
+}
+
+export interface StreetSynthesisOpts {
+	random?: () => number
+	/**
+	 * Probability of emitting house_number alongside the street. Default 0.85.
+	 */
+	includeHouseNumberProb?: number
+	/**
+	 * Probability of emitting the street BARE — no `, City, ST ZIP` tail and no region/locality/ postcode components
+	 * (just `street_prefix`/`street`/`street_suffix` + optional `house_number`). Default 0 (preserves the original
+	 * full-address behavior exactly, including the RNG sequence). Set >0 to teach the model that a bare `10th Ave` /
+	 * `Main St` is a STREET, not a locality — the functional-test failure cluster (bare streets mislabeled `locality`),
+	 * the bare-format analogue of the v0.7.x intersection-bare fix.
+	 */
+	bareProb?: number
+}
+
+function randomHouseNumber(random: () => number): string {
+	// US house number distribution: skewed low. 1-99 (30%), 100-999 (40%),
+	// 1000-9999 (25%), 10000+ (5%). The last band's 89_999 span is this file's own
+	// (the PO-box generators use 90_000) and is baked into every shipped street slice.
+	return tieredNumber(random, [
+		{ cutoff: 0.3, base: 1, span: 99 },
+		{ cutoff: 0.7, base: 100, span: 900 },
+		{ cutoff: 0.95, base: 1000, span: 9000 },
+		{ base: 10_000, span: 89_999 },
+	])
+}
+
+/**
+ * Synthesize a US street address with decomposed Stage 3 components. The street is built from PREFIX + NAME + SUFFIX,
+ * then passed through the same `decomposeStreet()` utility the TIGER adapter uses — guarantees the synthetic
+ * distribution matches the canonical decomposition logic.
+ */
+export function synthesizeStreetRow(
+	base: StreetBaseTuple,
+	opts: StreetSynthesisOpts = {}
+): SynthesizedStreetRow | null {
+	const random = opts.random ?? Math.random
+	const includeHN = opts.includeHouseNumberProb ?? 0.85
+
+	if (base.country !== "US") return null
+
+	const prefix = pick(DIRECTIONAL_PREFIXES, random)
+	const name = pick(STREET_NAMES, random)
+	const suffix = pick(STREET_SUFFIXES, random)
+	const trailing = pick(TRAILING_DIRECTIONALS, random)
+
+	// Build the "full" street string the adapter would receive from TIGER FULLNAME.
+	const parts = [prefix, name, suffix, trailing].filter(isPresent)
+	const fullStreet = parts.join(" ")
+
+	// Pass through the same decomposeStreet TIGER uses — match the training distribution.
+	const decomposed = decomposeStreet(fullStreet)
+
+	// NOTE: country is intentionally omitted. We don't emit "USA" or "US" in the raw
+	// string, and the aligner's fuzzy match (edit distance 2) will spuriously match
+	// "US" against any 2-char token (e.g. a house number "45" is exactly 2 substitutions
+	// from "US"). The PO box synthesizer skips country for the same reason.
+	// Bare mode is guarded by `> 0` so the default (bareProb=0) consumes no RNG and reproduces the
+	// original full-address output byte-for-byte.
+	const bareProb = opts.bareProb ?? 0
+	const bare = bareProb > 0 && random() < bareProb
+
+	const components: CanonicalRow["components"] = bare
+		? {}
+		: {
+				region: base.region,
+				locality: base.locality,
+				postcode: base.postcode,
+			}
+
+	if (decomposed.prefix) {
+		components.street_prefix = decomposed.prefix
+	}
+
+	if (decomposed.street) {
+		components.street = decomposed.street
+	}
+
+	if (decomposed.suffix) {
+		components.street_suffix = decomposed.suffix
+	}
+
+	let raw: string
+
+	if (random() < includeHN) {
+		const hn = randomHouseNumber(random)
+		components.house_number = hn
+		raw = bare ? `${hn} ${fullStreet}` : `${hn} ${fullStreet}, ${base.locality}, ${base.region} ${base.postcode}`
+	} else {
+		raw = bare ? fullStreet : `${fullStreet}, ${base.locality}, ${base.region} ${base.postcode}`
+	}
+
+	return { raw, components, locale: "en-US" }
+}