@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/surfaces/locale.ts

@@ -0,0 +1,246 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Render real address tuples into locale-specific surface strings.
+ *
+ *   Public address registers supply inputs such as OA, HM Land Registry and CNIG.
+ *   LINZ-derived extracts also supply inputs. The module applies OpenCage templates to
+ *   choose ordering and punctuation by country.
+ *
+ *   Keep the emitted source ids stable: they are persisted in built corpora and referenced by
+ *   training configs. `recipes/sources.ts` records each one under its operation spelling
+ *   (`rendered-de`) and the `synth-*` spelling it retired on 2026-09-26.
+ */
+
+import { formatAddress } from "@mailwoman/codex/address/format"
+
+import type { CanonicalRow } from "#types"
+
+/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
+	weights. We keep them inline so the output distribution is easy to read. */
+
+/**
+ * Real address tuple (for example one OpenAddresses row).
+ *
+ * Street and locality are required.
+ * Other tuple fields are optional.
+ */
+export interface LocaleBaseTuple {
+	house_number?: string
+	street: string
+	locality: string
+	/**
+	 * Sub-locality below locality (for example suburb/district).
+	 * When present, it is rendered between street and locality.
+	 */
+	dependent_locality?: string
+	region?: string
+	postcode?: string
+}
+
+/**
+ * @deprecated Alias — use LocaleBaseTuple.
+ */
+export type GermanBaseTuple = LocaleBaseTuple
+
+export interface RenderedLocaleRow {
+	raw: string
+	components: CanonicalRow["components"]
+	locale: string
+}
+
+/**
+ * @deprecated Alias — use RenderedLocaleRow.
+ */
+export type RenderedGermanRow = RenderedLocaleRow
+
+export interface LocaleRenderOpts {
+	random?: () => number
+	/**
+	 * The order used to render the same components.
+	 *
+	 * `"native"` (default) uses the country's own template (DE → house-after-street, postcode-before-city).
+	 * `"international"` renders house-first and postcode-after-city.
+	 */
+	order?: "native" | "international"
+	/**
+	 * Postcode surface shape.
+	 *
+	 * `"conventional"` (default) canonicalizes to the country's rendered form
+	 * (NL: OA's glued `1011AB` → the spaced `1011 AB`).
+	 * `"as-source"` keeps the source's own surface.
+	 *
+	 * Today, only NL differs between these modes.
+	 */
+	postcodeShape?: "conventional" | "as-source"
+	/**
+	 * How the native-order render joins street and house number.
+	 *
+	 * The OpenCage ES template comma-joins (`Calle Mayor, 12`, the official Spanish convention).
+	 * OA-derived feeds and our ES eval space-join (`calle mayor 12`, the observed form on all 3,000 eval rows).
+	 *
+	 * `"template"` (default) keeps the template's own join.
+	 * `"space"` collapses `<street>, <house_number>` → `<street> <house_number>` after rendering.
+	 * International order ignores this.
+	 */
+	nativeHouseJoin?: "template" | "space"
+	/**
+	 * The string between rendered address lines.
+	 *
+	 * `", "` (default) is the template's own join.
+	 * `" "` renders the comma-free single-line register for dictation or a copy out of a
+	 * one-field form, as in `Neusser Str. 12 Nippes 50733 Köln` for the same components.
+	 *
+	 * Only native order uses this option.
+	 */
+	separator?: ", " | " "
+}
+
+/**
+ * @deprecated Alias — use LocaleRenderOpts.
+ */
+export type GermanRenderOpts = LocaleRenderOpts
+
+/**
+ * ISO-3166 alpha-2 → BCP-47 tag for the emitted rows (primary language per country).
+ */
+const LOCALE_TAG: Record<string, string> = {
+	DE: "de-DE",
+	ES: "es-ES",
+	IT: "it-IT",
+	NL: "nl-NL",
+	GB: "en-GB",
+	FR: "fr-FR",
+	US: "en-US",
+	NZ: "en-NZ",
+}
+
+/**
+ * Canonicalize a postcode to the form the country's template renders,
+ * so the stored component aligns verbatim against `raw`.
+ *
+ * NL is the main case: OA may store `1011AB`, while the NL template emits `1011 AB`.
+ * Other countries pass through unchanged.
+ */
+function normalizePostcode(postcode: string, country: string): string {
+	if (country === "NL") {
+		const m = /^(\d{4})\s*([A-Za-z]{2})$/.exec(postcode)
+
+		if (m) return `${m[1]} ${m[2]!.toUpperCase()}`
+	}
+
+	return postcode
+}
+
+/**
+ * True when `value` appears verbatim and as a standalone token.
+ */
+function tokenPresent(raw: string, value: string): boolean {
+	if (!raw.includes(value)) return false
+	// Reject numeric substring collisions (for example house "2" inside postcode "12623").
+	const i = raw.indexOf(value)
+	const before = raw[i - 1]
+	const after = raw[i + value.length]
+	const isDigit = (c: string | undefined) => c !== undefined && c >= "0" && c <= "9"
+
+	if (/^\d+$/.test(value) && (isDigit(before) || isDigit(after))) return false
+
+	return true
+}
+
+/**
+ * Render one tuple into a locale-ordered `{raw, components}` row via OpenCage templates,
+ * with light variation (house number and postcode may be dropped).
+ *
+ * Returns `null` when the tuple is too thin or a component would not align cleanly.
+ *
+ * Region handling depends on order:
+ * - native omits region for alignment reliability
+ * - international includes region in the tail
+ *
+ * Pass `opts.order: "international"` to render the same components house-first and postcode-after-city
+ * instead (see {@link LocaleRenderOpts.order}), matching common feed-style layouts.
+ */
+export function renderLocaleRow(
+	base: LocaleBaseTuple,
+	country: string,
+	opts: LocaleRenderOpts = {}
+): RenderedLocaleRow | null {
+	const random = opts.random ?? Math.random
+	const order = opts.order ?? "native"
+
+	if (!base.street || !base.locality) return null
+
+	const components: CanonicalRow["components"] = { street: base.street, locality: base.locality }
+
+	// Sub-locality is part of the address body in both orders.
+	// If the template does not surface it verbatim, the row is dropped below.
+	if (base.dependent_locality) {
+		components.dependent_locality = base.dependent_locality
+	}
+
+	// The sampler keeps the house number in about 80% of rows.
+	// The remaining rows use street-only forms.
+	if (base.house_number && random() < 0.8) {
+		components.house_number = base.house_number
+	}
+
+	// ~85% keep postcode.
+	// Canonicalization uses the country's rendered form.
+	// `postcodeShape: "as-source"` is applied after rendering.
+	if (base.postcode && random() < 0.85) {
+		components.postcode = normalizePostcode(base.postcode, country)
+	}
+
+	// International order places region in the tail.
+	// Native order omits it for alignment.
+	if (order === "international" && base.region) {
+		components.region = base.region
+	}
+
+	// Native order uses the country's template.
+	// International order uses the US template.
+	// No random draw here, keeping RNG sequence stable.
+	const renderCountry = order === "international" ? "US" : country
+	const separator = order === "native" ? (opts.separator ?? ", ") : ", "
+	let raw = formatAddress(components, renderCountry, { separator })
+
+	if (!raw) return null
+
+	// Optional native-order rewrite: `<street>, <hn>` -> `<street> <hn>`.
+	if (order === "native" && separator === ", " && opts.nativeHouseJoin === "space" && components.house_number) {
+		raw = raw.replace(
+			`${components.street}, ${components.house_number}`,
+			`${components.street} ${components.house_number}`
+		)
+	}
+
+	// `postcodeShape: "as-source"`: rewrite both raw and component to source form.
+	// Done post-render because templates may normalize the postcode.
+	if (opts.postcodeShape === "as-source" && components.postcode && base.postcode) {
+		const sourceForm = base.postcode.trim()
+
+		if (sourceForm && sourceForm !== components.postcode && raw.includes(components.postcode)) {
+			raw = raw.replace(components.postcode, sourceForm)
+			components.postcode = sourceForm
+		}
+	}
+
+	// The row is dropped if any component is absent from the rendered address.
+	for (const value of Object.values(components)) {
+		if (!value || !tokenPresent(raw, value)) return null
+	}
+
+	return { raw, components, locale: LOCALE_TAG[country] ?? country.toLowerCase() }
+}
+
+/**
+ * German wrapper over {@link renderLocaleRow}.
+ *
+ * Kept for the `german` recipe (`de/recipes/locale.ts`) and tests.
+ */
+export function renderGermanRow(base: LocaleBaseTuple, opts: LocaleRenderOpts = {}): RenderedLocaleRow | null {
+	return renderLocaleRow(base, "DE", opts)
+}

package/lib/synthesizers/anchor-absorption.ts

@@ -3,49 +3,36 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Anchor-absorption counter-augmentation (#220/#723, Probe A1). Teaches the model the CONTEXT-
- *   DEPENDENT leading-5-digit disambiguation that the killed #723 override was faking, AND that the
- *   `anchor_paint_mode=shaped` WHERE fix alone over-corrected on (Probe A0: it flipped the default
- *   to house_number, recovering CASE-H but ERODING CASE-P — postcode F1 99.3→86.5 on
- *   leading-postcode rows like "05764 Finel Hollow Road, VT").
- *
- *   THE DISCRIMINATOR the model must learn (from the CASE-H vs CASE-P contrast, NOT a flipped
- *   default):
- *
- *   - A leading 5-digit WITH a trailing postcode + street context → it is the HOUSE NUMBER (CASE-H)
- *   - A leading 5-digit with NO trailing postcode (US-rural / DE) → it IS the POSTCODE (CASE-P) Both
- *       leading tokens are real ZIPs (so the painted anchor fires on both); only the surrounding
- *       context separates them. The model attends to the trailing token to decide the leading one.
- *
- *   Slice mix (the A0 learning sets a HEAVY CASE-P floor so the default doesn't flip — DeepSeek's
- *   ≥35% CASE-P; here CASE-P total = 35%): H-adversarial 30% US street, leading real-ZIP house# +
- *   TRAILING postcode → house_number P-us-rural 20% US rural, leading postcode, NO trailing →
- *   postcode (the A0-erosion fix) P-de 15% German leading postcode "{pc} {city}, {street} {hn}" →
- *   postcode anchor-fp 10% leading 5-digit that is NOT a real ZIP + trailing postcode →
- *   house_number locale-ambig 15% minimal context; the local token (street-type vs none) decides
- *   standard 10% normal small house# + trailing postcode → house_number (baseline)
- *
- *   Real ZIPs for the leading-5-digit are sampled from the postcode anchor lookup at build time
- *   (passed via opts.realZips) so the shaped anchor fires on them exactly as at inference; fake
- *   ZIPs (anchor-fp) are 5-digit strings deliberately absent from the lookup.
+ *   Generates address pairs in which context decides whether a leading five-digit token is a house number or a postcode.
  */
 
-import type { ComponentTag } from "@mailwoman/core/types"
+/* oxlint-disable mailwoman/prefer-home -- every tuple comes from the hardcoded US `US_TUPLES` list.
+   The admin tails use US templates. Each row puts a bare five-digit ZIP before a street to make the anchor channel
+   fire on the wrong span. This produces a US postcode shape by construction. */
 
-import { pick } from "#synthesizers/utils"
+import type { ComponentTag } from "@mailwoman/codex/component"
+import { sample } from "@mailwoman/core/random"
 
 /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
-   cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
-   output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
+   cutoffs rather than thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches defines the
+   output distribution. The cascade shows each weight from top to bottom. A separate constant for each cutoff
    would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
-   extracted as named constants above. */
+    extracted as constants above. */
 
+/**
+ * A US locality, region and postcode used as an address tail.
+ */
 export interface AnchorAbsorptionBaseTuple {
 	locality: string
 	region: string
 	postcode: string
 }
 
+/**
+ * The row templates.
+ *
+ * Names that start with `h-` produce a house number and names that start with `p-` a postcode.
+ */
 export type AnchorAbsorptionTemplate =
 	| "h-adversarial"
 	| "h-no-trailing-locality"
@@ -55,16 +42,24 @@ export type AnchorAbsorptionTemplate =
 	| "locale-ambig"
 	| "standard"
 
+/**
+ * Options for {@link synthesizeAnchorAbsorptionRow}.
+ */
 export interface AnchorAbsorptionSynthesisOpts {
 	random?: () => number
 	forceTemplate?: AnchorAbsorptionTemplate
 	/**
-	 * Real US ZIPs (in the anchor lookup) to use as the LEADING 5-digit house number — so the painted anchor fires on it,
-	 * the CASE-H/anchor-fp trigger. Builder loads these from pilot-anchor-lookup.json.
+	 * Real US ZIP codes from the postcode-anchor lookup.
+	 *
+	 * The templates place them where the anchor fires on a token that is really a house number.
+	 * The default is the postcodes of the built-in US tuples.
 	 */
 	realZips?: ReadonlyArray<string>
 }
 
+/**
+ * One synthesized row with its template.
+ */
 export interface SynthesizedAnchorAbsorptionRow {
 	raw: string
 	components: Partial<Record<ComponentTag, string>>
@@ -73,19 +68,15 @@ export interface SynthesizedAnchorAbsorptionRow {
 }
 
 /**
- * A realistic US house number: mostly 1–4 digits (the common range), 25% a real 5-digit ZIP (the HARD case — a 5-digit
- * leading number that is still a house number when a locality is present).
+ * Samples a house number.
+ * One quarter of the samples use a real ZIP code.
  */
 function houseNum(random: () => number, realZips: ReadonlyArray<string>): string {
-	if (random() < 0.25) return pick(realZips, random)
+	if (random() < 0.25) return sample(realZips, random)
 
 	return String(1 + Math.floor(random() * 9999))
 }
 
-/**
- * Curated, provenance-light reference vocab (real US street/city/state + DE). Surface forms must appear in `raw` for
- * alignRow; these are plain ASCII tokens that align cleanly.
- */
 const STREET_NAMES = [
 	"Main",
 	"Oak",
@@ -107,6 +98,7 @@ const STREET_NAMES = [
 	"Crescent Meadow",
 ]
 
+// TODO: Derive this from the US codex package exports
 const STREET_TYPES = ["St", "Ave", "Rd", "Dr", "Ln", "Blvd", "Ct", "Way", "Road", "Drive"]
 
 const US_TUPLES: ReadonlyArray<AnchorAbsorptionBaseTuple> = [
@@ -120,15 +112,8 @@ const US_TUPLES: ReadonlyArray<AnchorAbsorptionBaseTuple> = [
 	{ locality: "Sacramento", region: "CA", postcode: "95823" },
 ]
 
-/**
- * US rural states where the leading-postcode "{ZIP} {Street}, {STATE}" form (no trailing ZIP) is real — the VT/rural
- * format the #723 override broke and Probe A0 eroded.
- */
 const RURAL_REGIONS = ["VT", "ND", "SD", "NH", "ME", "MT", "WY"]
 
-/**
- * DE leading-postcode tuples: "{postcode} {locality}, {street} {house}".
- */
 const DE_TUPLES = [
 	{ postcode: "10115", locality: "Berlin", street: "Hauptstraße" },
 	{ postcode: "80331", locality: "München", street: "Sendlinger Straße" },
@@ -138,26 +123,24 @@ const DE_TUPLES = [
 ]
 
 const HOUSE_NUMS = ["5", "12", "27", "100", "212", "1450", "8"]
-/**
- * Fake 5-digit strings that are NOT real US ZIPs (so the anchor lookup MISSES them) — for anchor-fp.
- */
+// These five-digit values must stay absent from the postcode lookup.
 const FAKE_ZIPS = ["00000", "99998", "99997", "00001", "99996"]
 
 /**
- * Build one anchor-absorption counter-augmentation row.
+ * Builds one row from a sampled or forced template.
  */
 export function synthesizeAnchorAbsorptionRow(
 	opts: AnchorAbsorptionSynthesisOpts = {}
 ): SynthesizedAnchorAbsorptionRow {
 	const random = opts.random ?? Math.random
 	const realZips = opts.realZips && opts.realZips.length ? opts.realZips : US_TUPLES.map((t) => t.postcode)
-	const template = opts.forceTemplate ?? pick(ALL_TEMPLATES, random)
-	const street = `${pick(STREET_NAMES, random)} ${pick(STREET_TYPES, random)}`
+	const template = opts.forceTemplate ?? sample(ALL_TEMPLATES, random)
+	const street = `${sample(STREET_NAMES, random)} ${sample(STREET_TYPES, random)}`
 
 	if (template === "h-adversarial") {
-		// US street, leading real-ZIP house number, WITH a trailing postcode → the leading is house_number.
-		const zip = pick(realZips, random)
-		const t = pick(US_TUPLES, random)
+		// The trailing postcode makes the leading real ZIP code a house number.
+		const zip = sample(realZips, random)
+		const t = sample(US_TUPLES, random)
 		const raw = `${zip} ${street}, ${t.locality}, ${t.region} ${t.postcode}`
 
 		return {
@@ -169,9 +152,9 @@ export function synthesizeAnchorAbsorptionRow(
 	}
 
 	if (template === "p-us-rural") {
-		// US rural, leading postcode, NO trailing postcode → the leading IS the postcode (the A0-erosion fix).
-		const zip = pick(realZips, random)
-		const region = pick(RURAL_REGIONS, random)
+		// Without a locality or trailing postcode, the leading value is the postcode.
+		const zip = sample(realZips, random)
+		const region = sample(RURAL_REGIONS, random)
 		const raw = `${zip} ${street}, ${region}`
 
 		return {
@@ -183,9 +166,8 @@ export function synthesizeAnchorAbsorptionRow(
 	}
 
 	if (template === "p-de") {
-		// German leading postcode "{pc} {city}, {street} {hn}" → the leading is the postcode.
-		const d = pick(DE_TUPLES, random)
-		const hn = pick(HOUSE_NUMS, random)
+		const d = sample(DE_TUPLES, random)
+		const hn = sample(HOUSE_NUMS, random)
 		const raw = `${d.postcode} ${d.locality}, ${d.street} ${hn}`
 
 		return {
@@ -197,9 +179,9 @@ export function synthesizeAnchorAbsorptionRow(
 	}
 
 	if (template === "anchor-fp") {
-		// Leading 5-digit that is NOT a real ZIP (anchor MISSES) + trailing postcode → still house_number.
-		const fake = pick(FAKE_ZIPS, random)
-		const t = pick(US_TUPLES, random)
+		// A five-digit value absent from the ZIP lookup is still a house number here.
+		const fake = sample(FAKE_ZIPS, random)
+		const t = sample(US_TUPLES, random)
 		const raw = `${fake} ${street}, ${t.locality}, ${t.region} ${t.postcode}`
 
 		return {
@@ -211,15 +193,15 @@ export function synthesizeAnchorAbsorptionRow(
 	}
 
 	if (template === "locale-ambig") {
-		// Minimal context — the LOCAL token decides. Half: "{realZip} {street}" (street-type → house#);
-		// half: "{realZip} {locality}" (no street, leading postcode → postcode). No trailing, no region.
-		const zip = pick(realZips, random)
+		// A following street makes the value a house number.
+		// A following locality makes it a postcode.
+		const zip = sample(realZips, random)
 
 		if (random() < 0.5) {
 			return { raw: `${zip} ${street}`, components: { house_number: zip, street }, locale: "en-US", template }
 		}
 
-		const t = pick(US_TUPLES, random)
+		const t = sample(US_TUPLES, random)
 
 		return {
 			raw: `${zip} ${t.locality}`,
@@ -230,15 +212,10 @@ export function synthesizeAnchorAbsorptionRow(
 	}
 
 	if (template === "h-no-trailing-locality") {
-		// The A3 fix (#220): the common US format "{house#} {street}, {locality}, {STATE}" with NO trailing
-		// postcode → the leading number is the HOUSE NUMBER. The CONTRAST to p-us-rural (same no-trailing,
-		// state-bearing shape) is the LOCALITY: present here, absent there. The A2 slice lacked this slice,
-		// so p-us-rural's "leading-number + STATE → postcode" rule over-generalized to 98 golden house#
-		// rows ("36 Oxbow Dr, Bradford, VT" → postcode). The house# spans 1-4 digits AND real 5-digit ZIPs
-		// (the hard case: 5-digit + locality is STILL a house number, distinct from p-us-rural's no-locality).
+		// The locality separates this form from the rural postcode-first form.
 		const hn = houseNum(random, realZips)
-		const t = pick(US_TUPLES, random)
-		const region = random() < 0.5 ? pick(RURAL_REGIONS, random) : t.region
+		const t = sample(US_TUPLES, random)
+		const region = random() < 0.5 ? sample(RURAL_REGIONS, random) : t.region
 		const raw = `${hn} ${street}, ${t.locality}, ${region}`
 
 		return {
@@ -249,9 +226,8 @@ export function synthesizeAnchorAbsorptionRow(
 		}
 	}
 
-	// standard: normal house number + trailing postcode → house_number (baseline, keeps the common case).
 	const hn = houseNum(random, realZips)
-	const t = pick(US_TUPLES, random)
+	const t = sample(US_TUPLES, random)
 	const raw = `${hn} ${street}, ${t.locality}, ${t.region} ${t.postcode}`
 
 	return {
@@ -262,20 +238,8 @@ export function synthesizeAnchorAbsorptionRow(
 	}
 }
 
-// Weighted template bag — the slice mix. Expanded to a flat array so `pick` draws at the target
-// frequencies (matches the boundary-stress ALL_TEMPLATES idiom).
-//
-// A3 (#220, after the per-row diagnostic on the A2 probe): A1/A2 both held SLICE-H (100) + postcode
-// (~98) but cost house_number (95.8->92.8), and the A2 mix-rebalance did NOT move it — so it was never a
-// CASE-P-quantity problem. The row-by-row v192-vs-A2 diff (hn-regression-diff.ts) pinned it: 132/132 house#
-// regressions were house#->POSTCODE on "{house#} {street}, {locality}, {STATE}" no-trailing rows — the
-// p-us-rural rule ("leading-number + STATE + no-trailing -> postcode") OVER-GENERALIZED because the slice
-// had NO counter-slice for the common locality-bearing house# case. A3 ADDS h-no-trailing-locality (15%)
-// to teach the LOCALITY discriminator (present -> house#, absent + 5-digit -> postcode = p-us-rural) and
-// trims p-us-rural 16->13. Goal: house_number recovers WITHOUT re-eroding postcode/SLICE-H. CASE-P = 26%.
 /**
- * Every anchor-absorption template, in one list for the slice runner to sample from. Each covers a way a venue or
- * landmark name can swallow the street token that follows it.
+ * The template sampling pool, in which each template's repeat count is its weight.
  */
 export const ALL_TEMPLATES: ReadonlyArray<AnchorAbsorptionTemplate> = [
 	...new Array<AnchorAbsorptionTemplate>(25).fill("h-adversarial"),