@mailwoman/corpus 9.4.0 → 10.0.0

package/lib/fr/recipes/admin-split.ts

@@ -0,0 +1,243 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `fr-admin-split` slice recipe — the FR admin-split coverage slice (night 2026-06-19,
+ *   surpass-v1.5.0). Teaches the model to SPLIT the département out of the locality on
+ *   bare/space/comma-delimited French place rows — the admin-deciding failure class the pre-GPU
+ *   self-validation proved moves the resolved coordinate (collision communes −61%; see
+ *   docs/articles/evals/experiments/2026-06-19-fr-admin-split-prevalidation.md). Ported from
+ *   scripts/build-fr-admin-split-slice.mjs.
+ *
+ *   Failure shapes (the model currently mis-handles all of these):
+ *
+ *   - `Thauron, Creuse` → région dropped to null (the comma+full-name miss)
+ *   - `Montredon, Lozère` → région = "ère" (the diacritic subword split, #727)
+ *   - (AU analog) `CANBERRA ACT` → the space-delimited admin fuse
+ *
+ *   The département is the essential admin unit for FR postal geography and maps to the `region`
+ *   component tag in our schema. We derive it DETERMINISTICALLY from the real postcode via codex
+ *   `departementForCodePostal` (first two digits = département) — salvage-first, no re-derived
+ *   table.
+ *
+ *   Data (`--communes`, opts.communes): REAL BAN (Base Adresse Nationale) commune+postcode+coord
+ *   tuples, one per line, TAB-separated `commune <TAB> postcode <TAB> lon <TAB> lat`. Build the
+ *   input TSV once from the BAN staging CSV (see the legacy script header). Anchor-ON by
+ *   construction: rows carry a REAL postcode token in `raw` + a `postcode` component, so the
+ *   training loader paints the anchor feature onto that span automatically. The trailing-postcode
+ *   anchor REINFORCES the FR split (FR postcode is trailing, unlike German PLZ-leading — the v0.9.2
+ *   scar is positional, not universal).
+ */
+
+import type { ComponentTag } from "@mailwoman/codex/component"
+import { departementForCodePostal } from "@mailwoman/codex/fr"
+import { tempRootPath } from "@mailwoman/core/data-root"
+import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
+import { CSVSpliterator, Delimiters } from "spliterator"
+
+import { stableSourceID } from "#adapters/utils"
+import type { CorpusRecipe } from "#recipes/scaffold"
+import type { CanonicalRow } from "#types"
+import { alignRow } from "#utils"
+
+const DEFAULT_COMMUNES = tempRootPath("reg", "fr-communes.tsv")
+const LICENSE = "BAN (Base Adresse Nationale) commune+postcode tuples, rendered admin-split — see ingest SOURCE"
+
+/**
+ * One distinct commune row from the TSV, with the département derived from its postcode.
+ */
+interface CommuneRow {
+	commune: string
+	postcode: string
+	departement: string
+	lon: string | undefined
+	lat: string | undefined
+}
+
+/**
+ * One rendered admin-split variant.
+ */
+interface AdminSplitVariant {
+	raw: string
+	components: Partial<Record<ComponentTag, string>>
+	order: string
+}
+
+/**
+ * Read the distinct commune TSV (commune, postcode, lon, lat); derive the département name.
+ */
+async function readCommunes(path: string): Promise<CommuneRow[]> {
+	const rows: CommuneRow[] = []
+
+	// The TSV is headerless — every line is a commune tuple — so `header: false` keeps row 1 instead of
+	// spending it on column names. Source is repo-generated (LF); even under CRLF only the trailing `lat`
+	// column carries a CR, and it's consumed via `Number()` (whitespace-trimming), so it stays harmless.
+	for await (const [commune, postcode, lon, lat] of CSVSpliterator.fromAsync(path, {
+		columnDelimiter: Delimiters.Tab,
+		header: false,
+	})) {
+		if (!commune || !postcode) continue
+		const dep = departementForCodePostal(postcode)
+
+		if (!dep) continue // bad/unmappable postcode — skip (CEDEX, etc.)
+		// Substring invariant: a département whose name isn't a clean token (none are) or a commune
+		// containing the département name would confuse alignment — both are vanishingly rare here.
+		rows.push({ commune, postcode, departement: dep.name, lon, lat })
+	}
+
+	return rows
+}
+
+// 10% of communes render upper-case.
+const UPPER_LOCALITY_SHARE = 0.1
+// Layouts: 25% bare-comma, 25% bare-comma-pc, 20% space-pc, 15% canonical-pc-first, 15% commune-pc.
+const BARE_COMMA_CUTOFF = 0.25
+const BARE_COMMA_PC_CUTOFF = 0.5
+const SPACE_PC_CUTOFF = 0.7
+const CANONICAL_PC_FIRST_CUTOFF = 0.85
+// 20% of rows append an explicit "France" + a `country` component.
+const APPEND_COUNTRY_SHARE = 0.2
+
+/**
+ * Render one admin-split variant. The CORE teaching signal: the département, even as a full word after a comma or a
+ * space, is `region` — never folded into `locality`. Variants 1-3 are the failure class; 4-5 are canonical-FR
+ * preservation so the model doesn't over-fire region on every trailing token (and the bare commune still resolves).
+ */
+function render(random: () => number, c: CommuneRow): AdminSplitVariant {
+	const r = random()
+	const loc = random() < UPPER_LOCALITY_SHARE ? c.commune.toUpperCase() : c.commune
+	const dep = c.departement
+	const pc = c.postcode
+	let out: AdminSplitVariant
+
+	if (r < BARE_COMMA_CUTOFF) {
+		// 1. bare comma, NO postcode — the Thauron/#727 shape (anchor off)
+		out = { raw: `${loc}, ${dep}`, components: { locality: loc, region: dep }, order: "bare-comma" }
+	} else if (r < BARE_COMMA_PC_CUTOFF) {
+		// 2. bare comma + postcode — anchor ON
+		out = {
+			raw: `${loc}, ${dep} ${pc}`,
+			components: { locality: loc, region: dep, postcode: pc },
+			order: "bare-comma-pc",
+		}
+	} else if (r < SPACE_PC_CUTOFF) {
+		// 3. space-delimited admin (the AU `CANBERRA ACT` fuse applied to FR) — anchor ON
+		out = { raw: `${loc} ${dep} ${pc}`, components: { locality: loc, region: dep, postcode: pc }, order: "space-pc" }
+	} else if (r < CANONICAL_PC_FIRST_CUTOFF) {
+		// 4. canonical FR postcode-first (NO département) — preservation, anchor ON
+		out = { raw: `${pc} ${loc}`, components: { postcode: pc, locality: loc }, order: "canonical-pc-first" }
+	} else {
+		// 5. commune + postcode (NO département) — preservation, anchor ON
+		out = { raw: `${loc} ${pc}`, components: { locality: loc, postcode: pc }, order: "commune-pc" }
+	}
+
+	// fr.country preservation (the v1.8.0 #728 finding): the v1.8.0 slice's bare rows carried NO country
+	// token, so the model under-emitted country on FR (fr.country −3.5pp). ~20% of rows now append an
+	// explicit "France" + a `country` component — the model relearns to emit country WHEN the token is
+	// present without over-firing it on the (still-majority) country-less rows. Substring invariant holds.
+	if (random() < APPEND_COUNTRY_SHARE) {
+		out = {
+			raw: `${out.raw}, France`,
+			components: { ...out.components, country: "France" },
+			order: `${out.order}+fr`,
+		}
+	}
+
+	return out
+}
+
+/**
+ * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
+export const frAdminSplitRecipe: CorpusRecipe = {
+	name: "fr-admin-split",
+	description: "FR admin-split rows: BAN communes → split département into `region` (+ canonical-FR preservation)",
+	mode: "generate",
+	options: [
+		{ flag: "--communes <tsv>", description: "BAN commune+postcode+coord TSV. Default /tmp/reg/fr-communes.tsv" },
+	],
+	async run(opts, write) {
+		// Legacy build-fr-admin-split-slice.mjs seeded `mulberry32(opts.seed)`.
+		const random = makeMulberry32(opts.seed)
+		const count = opts.count ?? 60_000
+		const source = opts.sourceName ?? "synth-fr-admin-split"
+		const communesPath = opts.communes ?? DEFAULT_COMMUNES
+
+		const pool = await readCommunes(communesPath)
+
+		console.error(`  ${communesPath}: ${pool.length} communes with derived département`)
+
+		if (!pool.length) {
+			throw new Error("No communes — build the TSV from BAN first (see the recipe header).")
+		}
+
+		let emitted = 0
+		let skipped = 0
+		let guard = 0
+		const orderCounts: Record<string, number> = {}
+		const N = pool.length
+
+		while (emitted < count && guard++ < count * 12) {
+			const base = pool[Math.floor(random() * N)]!
+			const { raw, components, order } = render(random, base)
+
+			// Alignment precondition: every component surface appears verbatim in raw.
+			const values = Object.values(components).filter((v): v is string => Boolean(v))
+
+			if (!values.every((v) => raw.includes(v))) {
+				skipped++
+
+				continue
+			}
+
+			if (opts.golden) {
+				// Held-out eval slice for the centroid check — carries the truth coordinate.
+				write(JSON.stringify({ raw, components, country: "FR", lat: Number(base.lat), lon: Number(base.lon) }) + "\n")
+
+				emitted++
+				orderCounts[order] = (orderCounts[order] ?? 0) + 1
+
+				continue
+			}
+
+			const sourceID = stableSourceID(source, {
+				locality: components.locality,
+				region: components.region,
+				postcode: components.postcode,
+			})
+
+			const canonical: CanonicalRow = {
+				raw,
+				components,
+				country: "FR",
+				locale: "fr-FR",
+				source,
+				source_id: sourceID,
+				corpus_version: "0.5.0",
+				license: LICENSE,
+			}
+
+			const aligned = alignRow(canonical)
+
+			if (aligned.kind !== "labeled" || !aligned.row) {
+				skipped++
+
+				continue
+			}
+
+			write(
+				JSON.stringify({ ...aligned.row, synth_method: "fr-admin-split", synth_order: order, synth_base_id: null }) +
+					"\n"
+			)
+
+			emitted++
+			orderCounts[order] = (orderCounts[order] ?? 0) + 1
+		}
+
+		console.error(`  emitted=${emitted} skipped=${skipped} order-mix=${JSON.stringify(orderCounts)}`)
+
+		return { emitted, skipped }
+	},
+}

package/lib/fr/recipes/bare-street.ts

@@ -0,0 +1,171 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `fr-bare-street` slice recipe (#251) — the postcode-anchoring-imbalance change. BAN (and every
+ *   other comprehensive FR source) is postcode-COMPLETE, so the model learned the French
+ *   street→locality boundary as "the token after the 5-digit postcode," never as "comma + city." Strip
+ *   the postcode and it leaks the street's proper-noun tokens into the following locality ("Rue René
+ *   Cassin, Paris" → street="Rue Ren", locality="Cassin"). This recipe mints the MISSING distribution:
+ *   the BARE comma form, NO postcode, real `(street, number, city)` tuples from BAN (Licence Ouverte —
+ *   permissive; the model stays clean of ODbL, unlike the opt-in OSM rooftop slices).
+ *
+ *   Each tuple → `<n> <Rue/Avenue/…> <proper-noun name>, <City>` with the FR prefix split
+ *   ({@link decomposeFrStreet}: "Rue" → street_prefix, the rest → street). Tuples whose street carries
+ *   no recognized FR type word are skipped — the failing class is precisely the prefix-led street.
+ *
+ *   ⚠ Convention loss-mask: this recipe TEACHES FR `street_prefix`. The conventions loss-mask forbids it
+ *   for FR and will `-inf` these gold labels (the v1.6.0 ~7M-loss blow-up). Disable that mask for any
+ *   run including this slice.
+ */
+
+import { FR_VOIE_TYPES } from "@mailwoman/codex/fr"
+import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
+
+import { decomposeFrStreet } from "#fr/adapters/ban/street-decompose"
+import { alignAndWrite, readTuples, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
+
+/**
+ * Canonical voie type (lowercase, accent-kept) → its most common written abbreviation, from the codex table's first
+ * entry. Types with no attested abbreviation stay canonical in the abbreviated form.
+ */
+const FR_VOIE_ABBREV: Record<string, string> = Object.fromEntries(
+	Object.entries(FR_VOIE_TYPES).flatMap(([canonical, abbrevs]) => (abbrevs[0] ? [[canonical, abbrevs[0]]] : []))
+)
+
+/**
+ * The order-cycle slot for the BARE-STREET-ONLY form (`«voie» «name»`, no number, no locality) — the absence
+ * counterweight to the locality-terminated comma-free forms (see the cycle comment).
+ */
+const BARE_STREET_ONLY_FORM = 3
+
+/**
+ * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
+export const frBareStreetRecipe: CorpusRecipe = {
+	name: "fr-bare-street",
+	description:
+		"FR bare street+city, NO postcode (#251): comma / comma-free / abbreviated-voie surfaces over one label set",
+	mode: "tuples",
+	async run(opts, write) {
+		// Seeded for parity with the other recipes; unused beyond reproducibility (the tuples drive the content).
+		makeMulberry32(opts.seed)
+		let read = 0
+		let emitted = 0
+		let skipped = 0
+
+		for await (const t of readTuples(opts.input!)) {
+			read++
+			const fullStreet = String(t.street ?? "").trim()
+			const number = String(t.number ?? "").trim()
+			const locality = String(t.locality ?? "").trim()
+
+			if (!fullStreet || !number || !locality) {
+				skipped++
+
+				continue
+			}
+
+			const { prefix, street } = decomposeFrStreet(fullStreet)
+
+			// A no-prefix nom_voie ("La Ville Mois") is not the prefix-led class the numbered forms
+			// exercise — but as a BARE surface it is exactly the non-voie-led counterweight the v4.5.1
+			// probe showed missing ('Savile Row'-shaped spans still fell to the trailing-locality
+			// prior; the voie-led bare form protected only voie-led spans). Alternate rows emit the
+			// whole span as a bare street; the rest skip as before.
+			if (!prefix || !street) {
+				if (read % 2 === 0 && fullStreet.split(" ").length >= 2) {
+					const bare = {
+						raw: fullStreet,
+						components: { street: fullStreet } as Record<string, string>,
+						country: "FR",
+						locale: "fr-FR",
+						source: "synth-fr-bare-street",
+						source_id: sliceSourceID("synth-fr-bare-street", {
+							street: fullStreet,
+							f: "bare-nonvoie",
+							v: String(read),
+						}),
+						corpus_version: "0.9.4",
+						license:
+							"Synthetic — fr-bare-street; (street, number, city) from BAN (Base Adresse Nationale, Licence Ouverte)",
+					}
+
+					if (alignAndWrite(write, bare, "fr-bare-street")) {
+						emitted++
+					} else {
+						skipped++
+					}
+
+					continue
+				}
+
+				skipped++
+
+				continue
+			}
+
+			// Four surfaces over the same tuple, cycled deterministically. The comma form was the
+			// original change; the COMMA-FREE form is the colloquial register users actually type
+			// ('12 rue de Rome Paris' — the street↔locality boundary with NO delimiter, the fr-fr
+			// panel's named loss); the ABBREVIATED form is the typeahead register the geocoder-tester
+			// FR slice attests at scale; and the BARE-STREET-ONLY form is the absence counterweight —
+			// without it, every delimiter-free surface in the mix ENDS in a locality, the model learns
+			// "trailing span = locality" as categorical, and bare street names across locales flip to
+			// locality wholesale (the v4.5.0 no-promote's measured erosion: 'Calle de Alcalá',
+			// 'Madison Square West', and COMER's fork all fell to that prior). Tags are identical
+			// where present; each component VALUE is the span as written (BIO alignment binds value
+			// to surface); the bare form carries NO number and NO locality because the surface has
+			// neither.
+			const form = read % 4
+			const prefixSurface = form >= 2 ? (FR_VOIE_ABBREV[prefix.toLowerCase()] ?? prefix) : prefix
+
+			const components: Record<string, string> =
+				form === BARE_STREET_ONLY_FORM
+					? { street_prefix: prefix, street }
+					: { house_number: number, street_prefix: prefixSurface, street, locality }
+
+			// When the tuple carries a WOF-attested neighbourhood, the comma slot renders the
+			// THREE-slot middle surface — the dependent-locality counterweight (the v4.5.1 erosion's
+			// untouched half: the two-slot comma-free endings squeezed the middle tag out).
+			const hood = String(t.neighbourhood ?? "").trim()
+
+			if (form === 0 && hood) {
+				components.dependent_locality = hood
+			}
+
+			const raw =
+				form === 0
+					? hood
+						? `${number} ${prefix} ${street}, ${hood}, ${locality}`
+						: `${number} ${prefix} ${street}, ${locality}`
+					: form === BARE_STREET_ONLY_FORM
+						? `${prefix} ${street}`
+						: `${number} ${prefixSurface} ${street} ${locality}`
+
+			const source_id = sliceSourceID("synth-fr-bare-street", { ...components, f: String(form), v: String(read) })
+
+			const canonical = {
+				raw,
+				components,
+				country: "FR",
+				locale: "fr-FR",
+				source: "synth-fr-bare-street",
+				source_id,
+				corpus_version: "0.9.4",
+				license:
+					"Synthetic — fr-bare-street; (street, number, city) from BAN (Base Adresse Nationale, Licence Ouverte)",
+			}
+
+			if (alignAndWrite(write, canonical, "fr-bare-street")) {
+				emitted++
+			} else {
+				skipped++
+			}
+		}
+
+		return { read, emitted, skipped }
+	},
+}