@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/synthesizers/po-box.ts

@@ -3,46 +3,29 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   PO box / PMB / Apartado / Boîte Postale synthesizer.
- *
- *   Generates BIO-labeled corpus rows where the delivery line is a PO box (mutually exclusive with
- *   street + house_number per USPS Pub 28 / DMM 508). Locale-aware: emits idiomatic forms for
- *   en-US, en-CA, en-GB, en-AU, fr-FR, fr-CA, es-ES, es-MX, es-AR.
- *
- *   Per-DeepSeek design:
- *
- *   - PMB ("Private Mailbox" — at CMRAs like UPS Store) shares the `po_box` tag with USPS PO Box.
- *       Disambiguation is a downstream heuristic (presence of a street line).
- *   - Whole-phrase span ("PO Box 123") not number-only ("123"). Matches existing golden eval.
- *   - 10% of outputs receive number-format noise (commas, dashes, embedded spaces) to harden against
- *       real-world OCR/transcription input.
- *   - PO boxes drop street/house_number/unit/street_prefix/street_suffix from input components.
- *
- *   References:
- *
- *   - USPS Pub 28 §28C2.040 — Private Mailbox formatting
- *   - USPS DMM 508 §4.1.4 / §4.5.4 — PO Box and street-addressed PO Box
+ *   Generates localized PO-box and private-mailbox rows in which the designator and number form one `po_box` span.
  */
 
-import { countryToLocale as baseCountryToLocale, pick, tieredNumber } from "#synthesizers/utils"
+import { type ComponentDict, formatAddressRow } from "@mailwoman/codex/address/format"
+import { countryCodeForTable } from "@mailwoman/codex/country"
+import { sample } from "@mailwoman/core/random"
+
+import { countryToLocale as baseCountryToLocale, tieredNumber } from "#synthesizers/utils"
 import type { CanonicalRow } from "#types"
 
-/**
- * Digits a box number needs before a thousands comma is plausible (`1,234`).
- */
 const MIN_DIGITS_FOR_COMMA_GROUPING = 4
 
-/**
- * Digits a box number needs before a hyphen group is plausible (`12-34`).
- */
 const MIN_DIGITS_FOR_HYPHEN_GROUPING = 3
 
 /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
-   cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
-   output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
-   would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
-   extracted as named constants above. */
+   cutoffs rather than thresholds. `const r = random()` followed by `r < 0.4` branches defines the
+   output distribution. The branches show each weight from top to bottom. A separate constant for each cutoff
+   would hide the distribution behind identifiers. Genuine thresholds in these files are
+   constants defined above. */
 
+/**
+ * The address tail for one PO-box row.
+ */
 export interface PoBoxBaseTuple {
 	locality: string
 	region: string
@@ -50,18 +33,23 @@ export interface PoBoxBaseTuple {
 	country: string
 }
 
+/**
+ * One locale's PO-box designators.
+ */
 export interface LocaleTemplate {
 	locale: string
 	leaders: ReadonlyArray<string>
-	// Use 'pmb' to render as "STREET, PMB N, CITY ..." instead of replacing the street line.
+	/**
+	 * Private-mailbox designators.
+	 * They render beside the street instead of replacing it.
+	 */
 	pmb?: ReadonlyArray<string>
 }
 
 /**
- * The per-locale PO-box designator vocabulary (DeepSeek-signed list, see the header). Exported so slice builders
- * (scripts/build-po-box-cedex-slice.mjs) can reuse THIS list as the single source of truth for non-US leaders instead
- * of re-deriving it — the US slice additionally has `@mailwoman/codex/us` `US_PO_BOX_DESIGNATORS`/`isPOBox` as its
- * matcher-side truth.
+ * The PO-box designator vocabulary for each supported locale.
+ *
+ * Other recipes, such as the CEDEX recipe, reuse this list.
  */
 export const PO_BOX_LOCALE_TEMPLATES: ReadonlyArray<LocaleTemplate> = [
 	{
@@ -111,35 +99,34 @@ export const PO_BOX_LOCALE_TEMPLATES: ReadonlyArray<LocaleTemplate> = [
 const LEADERS_BY_LOCALE = new Map<string, LocaleTemplate>(PO_BOX_LOCALE_TEMPLATES.map((t) => [t.locale, t]))
 
 /**
- * Inject number-format noise into a box number string. Returns the noisy variant or the original (10% probability of
- * noise per the design).
+ * Returns the box number unchanged 90% of the time.
+ *
+ * Otherwise, the function adds a thousands comma, an inner hyphen, or spaces between the digits.
  */
 export function maybeNoisifyBoxNumber(num: string, random: () => number): string {
 	if (random() > 0.1) return num
 
 	const variants: Array<(s: string) => string> = [
-		// Thousand-separator comma (real input: "Box 1,234")
 		(s) => (s.length >= MIN_DIGITS_FOR_COMMA_GROUPING ? `${s.slice(0, -3)},${s.slice(-3)}` : s),
-		// Embedded dash (real input: "PMB-200")
 		(s) => (s.length >= MIN_DIGITS_FOR_HYPHEN_GROUPING ? `${s.slice(0, -2)}-${s.slice(-2)}` : s),
-		// Embedded spaces (real input from OCR: "1 2 3 4")
 		(s) => s.split("").join(" "),
 	]
 
-	const f = pick(variants, random)
+	const f = sample(variants, random)
 
 	return f(num)
 }
 
 /**
- * Compose a PO box phrase like "PO Box 123" or "PMB 200".
- *
- * Returns both the phrase and the canonical leader+number so the BIO aligner can mark the entire span as `po_box`.
+ * Joins a designator and a number into one `po_box` phrase.
  */
 export function composePoBoxPhrase(leader: string, number: string): string {
 	return `${leader} ${number}`
 }
 
+/**
+ * One synthesized row with its template.
+ */
 export interface SynthesizedPoBoxRow {
 	raw: string
 	components: CanonicalRow["components"]
@@ -147,24 +134,29 @@ export interface SynthesizedPoBoxRow {
 	template: "po-box" | "pmb-with-street" | "military-po-box"
 }
 
+/**
+ * Options for the PO-box synthesizers.
+ */
 export interface PoBoxSynthesisOpts {
 	/**
-	 * Random function — pass deterministic seed for tests. Default Math.random.
+	 * The random source.
+	 * It defaults to `Math.random`.
 	 */
 	random?: () => number
 	/**
-	 * Number generator. Default uniform over 1..99999.
+	 * The box-number generator.
+	 *
+	 * By default, it chooses a number from 1 to 99,999 and weights short numbers more heavily.
 	 */
 	pickNumber?: (random: () => number) => string
 	/**
-	 * PMB probability when locale supports it (and a street is provided in the base tuple).
+	 * The probability of a private-mailbox row when the locale and tuple allow one.
+	 * The default is 0.
 	 */
 	pmbRatio?: number
 }
 
 function defaultPickNumber(random: () => number): string {
-	// 70% of real PO boxes are 1-5 digits; long ones exist (USPS allows up to ~6 digits).
-	// Bands: 1-99, 100-999, 1000-9999, 10000-99999 (span 90_000 — the street generator's 89_999 is its own).
 	return tieredNumber(random, [
 		{ cutoff: 0.3, base: 1, span: 99 },
 		{ cutoff: 0.7, base: 100, span: 900 },
@@ -174,8 +166,10 @@ function defaultPickNumber(random: () => number): string {
 }
 
 /**
- * Generate one PO box row for a base (locality, region, postcode, country) tuple. Picks a locale-appropriate leader and
- * number. Optionally generates a PMB variant when the base tuple includes a street.
+ * Generates a localized PO-box row, or a private-mailbox row when the tuple
+ * has a street and the locale allows it.
+ *
+ * The function returns `null` when the country lacks a template locale, an ISO code or a layout.
  */
 export function synthesizePoBoxRow(
 	base: PoBoxBaseTuple & { street?: string; houseNumber?: string },
@@ -185,67 +179,61 @@ export function synthesizePoBoxRow(
 	const pickNumber = opts.pickNumber ?? defaultPickNumber
 	const pmbRatio = opts.pmbRatio ?? 0
 
-	const locale = countryToLocale(base.country)
+	const locale = poBoxTemplateLocale(base.country)
 	const tpl = LEADERS_BY_LOCALE.get(locale)
 
 	if (!tpl) return null
 
 	const number = maybeNoisifyBoxNumber(pickNumber(random), random)
-	const leader = pick(tpl.leaders, random)
+	const leader = sample(tpl.leaders, random)
 	const poBoxPhrase = composePoBoxPhrase(leader, number)
 
-	// PMB variant: requires both a street and a PMB-supporting locale.
 	const wantPmb = base.street && tpl.pmb && random() < pmbRatio
 
+	const iso2 = countryCodeForTable(base.country)
+
+	if (!iso2) return null
+
+	// The country's layout decides the order and which components appear.
+	const adminTail: ComponentDict = { locality: base.locality, postcode: base.postcode }
+
+	if (base.region?.trim()) {
+		adminTail.region = base.region
+	}
+
 	if (wantPmb) {
-		const pmbLeader = pick(tpl.pmb!, random)
+		const pmbLeader = sample(tpl.pmb!, random)
 		const pmbPhrase = composePoBoxPhrase(pmbLeader, number)
-		const streetLine = base.houseNumber ? `${base.houseNumber} ${base.street}` : base.street!
-		const raw = `${streetLine}, ${pmbPhrase}, ${base.locality}, ${base.region} ${base.postcode}`
+		const dict: ComponentDict = { ...adminTail, street: base.street!, po_box: pmbPhrase }
+
+		if (base.houseNumber) {
+			dict.house_number = base.houseNumber
+		}
+
+		const rendered = formatAddressRow(dict, iso2, { singleLine: true })
+
+		if (!rendered) return null
 
 		return {
-			raw,
-			components: {
-				...(base.houseNumber ? { house_number: base.houseNumber } : {}),
-				street: base.street!,
-				po_box: pmbPhrase,
-				locality: base.locality,
-				region: base.region,
-				postcode: base.postcode,
-				country: base.country,
-			},
+			raw: rendered.raw,
+			components: { ...rendered.components, country: base.country },
 			locale,
 			template: "pmb-with-street",
 		}
 	}
 
-	// Standard PO box: replaces the street line entirely. Region-optional — NZ (and other region-less
-	// locales) read "Private Bag 12, Auckland 1010" with no region token between locality and postcode.
-	const hasRegion = Boolean(base.region && base.region.trim())
-	const tail = hasRegion ? `${base.locality}, ${base.region} ${base.postcode}` : `${base.locality} ${base.postcode}`
-	const raw = `${poBoxPhrase}, ${tail}`
+	const rendered = formatAddressRow({ ...adminTail, po_box: poBoxPhrase }, iso2, { singleLine: true })
+
+	if (!rendered) return null
 
 	return {
-		raw,
-		components: {
-			po_box: poBoxPhrase,
-			locality: base.locality,
-			...(hasRegion ? { region: base.region } : {}),
-			postcode: base.postcode,
-			country: base.country,
-		},
+		raw: rendered.raw,
+		components: { ...rendered.components, country: base.country },
 		locale,
 		template: "po-box",
 	}
 }
 
-/**
- * The US military/diplomatic PO-box class (#517). A distinct shape the leader-based locale templates can't express: a
- * unit line (`PSC <id> Box <box>`, `CMR <id> Box <box>`, `Unit <id> [Box <box>]`) tagged `po_box`, then the post-office
- * code (APO/FPO/DPO) as the locality and the armed-forces region (AA/AE/AP) as the region, with a theatre-specific ZIP.
- * Authoritative reference + citations: `@mailwoman/codex` `codex/us/military-address.ts`; the small constants are
- * inlined here so the generator is self-contained.
- */
 const MIL_UNITS: ReadonlyArray<{ code: string; boxRequired: boolean }> = [
 	{ code: "PSC", boxRequired: true },
 	{ code: "CMR", boxRequired: true },
@@ -254,9 +242,7 @@ const MIL_UNITS: ReadonlyArray<{ code: string; boxRequired: boolean }> = [
 
 const MIL_PO_CODES = ["APO", "FPO", "DPO"] as const
 
-/**
- * Region → plausible ZIP prefix (AE Europe 09xxx, AP Pacific 962-966xx, AA Americas 340xx).
- */
+// Each armed-forces region has its own ZIP range.
 const MIL_REGION_ZIP: ReadonlyArray<{ region: string; zip: (r: () => number) => string }> = [
 	{ region: "AE", zip: (r) => `09${String(Math.floor(r() * 1000)).padStart(3, "0")}` },
 	{ region: "AP", zip: (r) => `96${String(200 + Math.floor(r() * 100)).padStart(3, "0")}` },
@@ -264,15 +250,19 @@ const MIL_REGION_ZIP: ReadonlyArray<{ region: string; zip: (r: () => number) =>
 ]
 
 /**
- * Generate one US military/diplomatic PO-box row (#517). Self-contained — draws no base tuple.
+ * Generates a US military or diplomatic address without a base tuple.
+ *
+ * The unit line is the `po_box`.
+ * APO, FPO, or DPO is the locality.
+ * The armed-forces code is the region.
  */
 export function synthesizeMilitaryPoBoxRow(opts: PoBoxSynthesisOpts = {}): SynthesizedPoBoxRow {
 	const random = opts.random ?? Math.random
-	const unit = pick(MIL_UNITS, random)
+	const unit = sample(MIL_UNITS, random)
 	const unitID = String(1 + Math.floor(random() * 9999))
-	const { region, zip } = pick(MIL_REGION_ZIP, random)
+	const { region, zip } = sample(MIL_REGION_ZIP, random)
 	const zipStr = zip(random)
-	const po = pick(MIL_PO_CODES, random)
+	const po = sample(MIL_PO_CODES, random)
 	const hasBox = unit.boxRequired || random() < 0.5
 	const unitLine = hasBox ? `${unit.code} ${unitID} Box ${1 + Math.floor(random() * 9999)}` : `${unit.code} ${unitID}`
 	const raw = `${unitLine}, ${po} ${region} ${zipStr}`
@@ -294,30 +284,25 @@ export function synthesizeMilitaryPoBoxRow(opts: PoBoxSynthesisOpts = {}): Synth
 const PO_BOX_TEMPLATE_LOCALES: ReadonlySet<string> = new Set(PO_BOX_LOCALE_TEMPLATES.map((t) => t.locale))
 
 /**
- * Map a country code (ISO-3166-1 alpha-2 or alpha-3, or country display name) to the locale code we have a PO box
- * template for.
+ * Returns the locale whose PO-box vocabulary a country uses.
  *
- * TEMPLATE-SCOPED on top of the shared `#synthesizers/utils` map: a locale the shared map resolves but
- * {@link PO_BOX_LOCALE_TEMPLATES} does not carry falls back to `en-US`, so `DE` (shared: `de-DE`, no PO-box template)
- * keeps rendering the en-US box vocabulary here.
+ * Countries without their own template use `en-US` designators.
+ * Their address order still comes from the country's layout.
  */
-export function countryToLocale(country: string): string {
+export function poBoxTemplateLocale(country: string): string {
 	const locale = baseCountryToLocale(country)
 
 	return PO_BOX_TEMPLATE_LOCALES.has(locale) ? locale : "en-US"
 }
 
 /**
- * All locales we synthesize for. Exposed for tests and for source-weight tuning.
+ * Returns the locales that have PO-box templates.
  */
 export function supportedLocales(): ReadonlyArray<string> {
 	return PO_BOX_LOCALE_TEMPLATES.map((t) => t.locale)
 }
 
 /**
- * Locales whose standard PO-box delivery line carries NO region token — the address reads `<po_box>, <locality>
- * <postcode>` with nothing between locality and postcode (#517). NZ is the canonical case (`Private Bag 12, Auckland
- * 1010`). Consumers (e.g. the synth-po-box adapter) use this to avoid discarding region-less input tuples for these
- * locales as "missing region".
+ * Locales whose standard PO-box layout omits the region.
  */
 export const REGION_OPTIONAL_LOCALES: ReadonlySet<string> = new Set(["en-NZ"])

package/lib/synthesizers/street.ts

@@ -7,32 +7,37 @@
  *   `street_prefix`, `street`, and `street_suffix` are emitted as separate BIO spans (instead of
  *   monolithic `street`). Mirrors the PO box synthesizer pattern.
  *
- *   Why this exists: TIGER/NAD/BAN adapter changes (committed earlier tonight) emit decomposed
- *   components from raw source data, but the v0.4.0 parquet slices on Modal were built BEFORE those
- *   changes. Rebuilding the full corpus requires downloading raw TIGER/NAD/BAN data and re-running
+ *   Why this exists: tiger/NAD/BAN adapter changes (committed earlier tonight) emit decomposed
+ *   components from raw source data, but the v0.4.0 parquet files on Modal were built before those
+ *   changes. A full-corpus rebuild requires downloading raw tiger/NAD/BAN data and re-running
  *   adapters end-to-end — out of scope for a single night shift. This synthesizer takes (locality,
  *   region, postcode) tuples and produces freshly-decomposed Stage 3 training rows, same shape as
  *   the PO box pipeline.
  *
- *   Note: uses the SAME decomposition logic as TIGER's `decomposeStreet()` so the synthetic
- *   distribution matches what the model would see if/when TIGER slices are rebuilt with the new
+ *   Note: uses the same decomposition logic as tiger's `decomposeStreet()` so the synthetic
+ *   distribution matches what the model would see if/when the tiger parquet files are rebuilt with the new
  *   adapter.
  */
 
+/* oxlint-disable mailwoman/prefer-home -- the admin tails below are written as US templates because this synthesizer
+   refuses a non-US tuple outright (`if (base.country !== "US") return null`). A layout call would answer the same
+   string for the only country that reaches it. The function would read as though the file served more. */
+
 import { isPresent } from "@mailwoman/core/objects"
+import { sample } from "@mailwoman/core/random"
 
-import { decomposeStreet } from "#adapters/tiger/street-decompose"
-import { pick, tieredNumber } from "#synthesizers/utils"
+import { tieredNumber } from "#synthesizers/utils"
 import type { CanonicalRow } from "#types"
+import { decomposeStreet } from "#us/adapters/tiger/street-decompose"
 
-// Hand-curated US street name pool. Real frequency-weighted street names — sampled
-// from US Census TIGER 2024 top-1000 by occurrence count. Keep ~50 entries so the
-// synthesis distribution doesn't overfit to a tiny vocabulary.
+// Hand-curated US street name pool.
+// Real frequency-weighted street names — sampled from US Census tiger 2024 top-1000 by occurrence count.
+// Keep ~50 entries so the synthesis distribution doesn't overfit to a tiny vocabulary.
 /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
-   cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
-   output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
+   cutoffs rather than thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches is the
+   output distribution. The cascade shows each weight from top to bottom. A separate constant for each cutoff
    would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
-   extracted as named constants above. */
+    extracted as constants above. */
 
 const STREET_NAMES = [
 	"Main",
@@ -175,23 +180,27 @@ export interface SynthesizedStreetRow {
 export interface StreetSynthesisOpts {
 	random?: () => number
 	/**
-	 * Probability of emitting house_number alongside the street. Default 0.85.
+	 * Probability of emitting house_number alongside the street.
+	 *
+	 * Default 0.85.
 	 */
 	includeHouseNumberProb?: number
 	/**
-	 * Probability of emitting the street BARE — no `, City, ST ZIP` tail and no region/locality/ postcode components
-	 * (just `street_prefix`/`street`/`street_suffix` + optional `house_number`). Default 0 (preserves the original
-	 * full-address behavior exactly, including the RNG sequence). Set >0 to teach the model that a bare `10th Ave` /
-	 * `Main St` is a STREET, not a locality — the functional-test failure cluster (bare streets mislabeled `locality`),
+	 * Probability of emitting the street bare — no `, City, ST ZIP` tail and no region/locality/
+	 * postcode components (just `street_prefix`/`street`/`street_suffix` + optional `house_number`).
+	 *
+	 * Default 0 (leaves the original full-address output exactly as it was, including the RNG sequence).
+	 * Set >0 to teach the model that a bare `10th Ave` / `Main St` is a street rather than a
+	 * locality — the functional-test failure cluster (bare streets mislabeled `locality`),
 	 * the bare-format analogue of the v0.7.x intersection-bare fix.
 	 */
 	bareProb?: number
 }
 
 function randomHouseNumber(random: () => number): string {
-	// US house number distribution: skewed low. 1-99 (30%), 100-999 (40%),
-	// 1000-9999 (25%), 10000+ (5%). The last band's 89_999 span is this file's own
-	// (the PO-box generators use 90_000) and is baked into every shipped street slice.
+	// US house number distribution: skewed low. 1-99 (30%), 100-999 (40%), 1000-9999 (25%), 10000+ (5%).
+	// The last band's 89_999 span is this file's own (the PO-box generators use 90_000)
+	// and is baked into every shipped street recipe output.
 	return tieredNumber(random, [
 		{ cutoff: 0.3, base: 1, span: 99 },
 		{ cutoff: 0.7, base: 100, span: 900 },
@@ -201,8 +210,10 @@ function randomHouseNumber(random: () => number): string {
 }
 
 /**
- * Synthesize a US street address with decomposed Stage 3 components. The street is built from PREFIX + NAME + SUFFIX,
- * then passed through the same `decomposeStreet()` utility the TIGER adapter uses — guarantees the synthetic
+ * Synthesize a US street address with decomposed Stage 3 components.
+ *
+ * The street is built from prefix + name + suffix, then passed through the same
+ * `decomposeStreet()` utility the tiger adapter uses — guarantees the synthetic
  * distribution matches the canonical decomposition logic.
  */
 export function synthesizeStreetRow(
@@ -214,24 +225,25 @@ export function synthesizeStreetRow(
 
 	if (base.country !== "US") return null
 
-	const prefix = pick(DIRECTIONAL_PREFIXES, random)
-	const name = pick(STREET_NAMES, random)
-	const suffix = pick(STREET_SUFFIXES, random)
-	const trailing = pick(TRAILING_DIRECTIONALS, random)
+	const prefix = sample(DIRECTIONAL_PREFIXES, random)
+	const name = sample(STREET_NAMES, random)
+	const suffix = sample(STREET_SUFFIXES, random)
+	const trailing = sample(TRAILING_DIRECTIONALS, random)
 
-	// Build the "full" street string the adapter would receive from TIGER FULLNAME.
+	// Build the "full" street string the adapter would receive from tiger fullname.
 	const parts = [prefix, name, suffix, trailing].filter(isPresent)
 	const fullStreet = parts.join(" ")
 
-	// Pass through the same decomposeStreet TIGER uses — match the training distribution.
+	// Pass through the same decomposeStreet tiger uses — match the training distribution.
 	const decomposed = decomposeStreet(fullStreet)
 
-	// NOTE: country is intentionally omitted. We don't emit "USA" or "US" in the raw
-	// string, and the aligner's fuzzy match (edit distance 2) will spuriously match
-	// "US" against any 2-char token (e.g. a house number "45" is exactly 2 substitutions
-	// from "US"). The PO box synthesizer skips country for the same reason.
-	// Bare mode is guarded by `> 0` so the default (bareProb=0) consumes no RNG and reproduces the
-	// original full-address output byte-for-byte.
+	// note: country is intentionally omitted.
+	// The raw string omits "USA" and "US".
+	// The aligner's fuzzy match (edit distance 2) will spuriously match "US" against any
+	// 2-char token (e.g. A house number "45" is exactly 2 substitutions from "US").
+	// The PO box synthesizer skips country for the same reason.
+	// Bare mode is guarded by `> 0` so the default (bareProb=0) consumes no RNG
+	// and reproduces the original full-address output byte-for-byte.
 	const bareProb = opts.bareProb ?? 0
 	const bare = bareProb > 0 && random() < bareProb