@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/recipes/cz-pcfirst-preposition.ts

@@ -1,112 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `cz-pcfirst-preposition` — the Czech pc-first prepositional-locality slice, fourth orthography
- *   of the #901 leading-name-boundary family. The #897 close-out read all 8 residual CZ rows as one
- *   class: a LEADING postcode mis-assigned as house_number while the multi-word "nad/pod/u X"
- *   locality shatters ("51244 Rokytnice nad Jizerou, Dolní Rokytnice 111" → street
- *   'RokytnicenadJizerou' + house_number '51244'). That leading-5-digit confusion is the
- *   anchor-pollution class whose decode-time OVERRIDE was correctly killed in #723 — this slice is
- *   the model-first fix as DATA: real prepositional localities in the order that breaks, so the
- *   model learns that a leading postcode before a multi-word name is a postcode. pc-first leads the
- *   cycle (the lesson); canonical and city-first keep the polarity balanced (the v1.9.9 lesson).
- */
-
-import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
-
-import { alignAndWrite, readTuples, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
-/**
- * The order-cycle slot for the STREET-LESS form (`«city» «pc», Česko`) — the exact surface of the
- * `cz-full-praha-100-00` board row, whose absence from the street-bearing orders was the v4.5.0 no-promote's measured
- * gap.
- */
-const STREETLESS_ORDER = 3
-
-/**
- * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
- */
-export const czPcFirstPrepositionRecipe: CorpusRecipe = {
-	name: "cz-pcfirst-preposition",
-	description:
-		"CZ pc-first + prepositional locality (#901 family): '«pc» «city nad X», «st» «n»' — the #723 class as data",
-	mode: "tuples",
-	async run(opts, write) {
-		makeMulberry32(opts.seed)
-		let read = 0
-		let emitted = 0
-		let skipped = 0
-
-		for await (const t of readTuples(opts.input!)) {
-			read++
-			const street = String(t.street ?? "").trim()
-			const city = String(t.locality ?? "").trim()
-			const number = String(t.number ?? "").trim()
-			const postcode = String(t.postcode ?? "").trim()
-
-			if (!street || !city || !number || !postcode) {
-				skipped++
-
-				continue
-			}
-
-			const order = read % 4
-			// The OFFICIAL Czech rendering spaces the PSČ as `NNN NN` ('512 44'); OpenAddresses stores it
-			// unspaced ('51244'), and a model trained only on the source form reads the spaced surface as
-			// house_number + garbage (the 'Praha 100 00' mangle). Alternate the two renderings so both
-			// orthographies are attested — the LABEL is the postcode either way.
-			const spaced = read % 2 === 0 && /^\d{5}$/.test(postcode)
-			const postcodeSurface = spaced ? `${postcode.slice(0, 3)} ${postcode.slice(3)}` : postcode
-			let raw: string
-
-			const components: Record<string, string> =
-				order === STREETLESS_ORDER
-					? { locality: city, postcode: postcodeSurface, country: "Česko" }
-					: { street, house_number: number, postcode: postcodeSurface, locality: city }
-
-			if (order === 0) {
-				raw = `${postcodeSurface} ${city}, ${street} ${number}`
-			} else if (order === 1) {
-				raw = `${street} ${number}, ${postcodeSurface} ${city}`
-			} else if (order === 2) {
-				raw = `${city}, ${postcodeSurface}, ${street} ${number}`
-			} else {
-				// order === STREETLESS_ORDER: the street-less form — `«city» «pc», Česko` — the exact surface of the
-				// cz-full-praha-100-00 board row. The v4.5.0 no-promote receipt measured the gap: every
-				// prior order was street-bearing, so the model never saw a spaced PSČ beside a bare
-				// locality and mangled 'Praha 100 00, Czechia' into house_number spans. Street/number
-				// stay OUT of the components for this form (they are not in the surface).
-				raw = `${city} ${postcodeSurface}, Česko`
-			}
-
-			const source_id = sliceSourceID("synth-cz-pcfirst-preposition", {
-				...components,
-				o: String(order),
-				s: spaced ? "1" : "0",
-				v: String(read),
-			})
-
-			const canonical = {
-				raw,
-				components,
-				country: "CZ",
-				locale: "cs-CZ",
-				source: "synth-cz-pcfirst-preposition",
-				source_id,
-				corpus_version: "0.10.0",
-				license:
-					"Synthetic — cz-pcfirst-preposition; (street, number, postcode, city) from OpenAddresses CZ (per-source attribution in the model card)",
-			}
-
-			if (alignAndWrite(write, canonical, "cz-pcfirst-preposition")) {
-				emitted++
-			} else {
-				skipped++
-			}
-		}
-
-		return { read, emitted, skipped }
-	},
-}

package/lib/recipes/fr-admin-split.ts

@@ -1,243 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `fr-admin-split` slice recipe — the FR admin-split coverage slice (night 2026-06-19,
- *   surpass-v1.5.0). Teaches the model to SPLIT the département out of the locality on
- *   bare/space/comma-delimited French place rows — the admin-deciding failure class the pre-GPU
- *   self-validation proved moves the resolved coordinate (collision communes −61%; see
- *   docs/articles/evals/experiments/2026-06-19-fr-admin-split-prevalidation.md). Ported from
- *   scripts/build-fr-admin-split-slice.mjs.
- *
- *   Failure shapes (the model currently mis-handles all of these):
- *
- *   - `Thauron, Creuse` → région dropped to null (the comma+full-name miss)
- *   - `Montredon, Lozère` → région = "ère" (the diacritic subword split, #727)
- *   - (AU analog) `CANBERRA ACT` → the space-delimited admin fuse
- *
- *   The département is the essential admin unit for FR postal geography and maps to the `region`
- *   component tag in our schema. We derive it DETERMINISTICALLY from the real postcode via codex
- *   `departementForCodePostal` (first two digits = département) — salvage-first, no re-derived
- *   table.
- *
- *   Data (`--communes`, opts.communes): REAL BAN (Base Adresse Nationale) commune+postcode+coord
- *   tuples, one per line, TAB-separated `commune <TAB> postcode <TAB> lon <TAB> lat`. Build the
- *   input TSV once from the BAN staging CSV (see the legacy script header). Anchor-ON by
- *   construction: rows carry a REAL postcode token in `raw` + a `postcode` component, so the
- *   training loader paints the anchor feature onto that span automatically. The trailing-postcode
- *   anchor REINFORCES the FR split (FR postcode is trailing, unlike German PLZ-leading — the v0.9.2
- *   scar is positional, not universal).
- */
-
-import { departementForCodePostal } from "@mailwoman/codex/fr"
-import { tempRootPath } from "@mailwoman/core/data-root"
-import type { ComponentTag } from "@mailwoman/core/types"
-import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
-import { CSVSpliterator, Delimiters } from "spliterator"
-
-import { stableSourceID } from "#adapters/utils"
-import type { CorpusRecipe } from "#recipes/scaffold"
-import type { CanonicalRow } from "#types"
-import { alignRow } from "#utils"
-
-const DEFAULT_COMMUNES = tempRootPath("reg", "fr-communes.tsv")
-const LICENSE = "BAN (Base Adresse Nationale) commune+postcode tuples, rendered admin-split — see ingest SOURCE"
-
-/**
- * One distinct commune row from the TSV, with the département derived from its postcode.
- */
-interface CommuneRow {
-	commune: string
-	postcode: string
-	departement: string
-	lon: string | undefined
-	lat: string | undefined
-}
-
-/**
- * One rendered admin-split variant.
- */
-interface AdminSplitVariant {
-	raw: string
-	components: Partial<Record<ComponentTag, string>>
-	order: string
-}
-
-/**
- * Read the distinct commune TSV (commune, postcode, lon, lat); derive the département name.
- */
-async function readCommunes(path: string): Promise<CommuneRow[]> {
-	const rows: CommuneRow[] = []
-
-	// The TSV is headerless — every line is a commune tuple — so `header: false` keeps row 1 instead of
-	// spending it on column names. Source is repo-generated (LF); even under CRLF only the trailing `lat`
-	// column carries a CR, and it's consumed via `Number()` (whitespace-trimming), so it stays harmless.
-	for await (const [commune, postcode, lon, lat] of CSVSpliterator.fromAsync(path, {
-		columnDelimiter: Delimiters.Tab,
-		header: false,
-	})) {
-		if (!commune || !postcode) continue
-		const dep = departementForCodePostal(postcode)
-
-		if (!dep) continue // bad/unmappable postcode — skip (CEDEX, etc.)
-		// Substring invariant: a département whose name isn't a clean token (none are) or a commune
-		// containing the département name would confuse alignment — both are vanishingly rare here.
-		rows.push({ commune, postcode, departement: dep.name, lon, lat })
-	}
-
-	return rows
-}
-
-// 10% of communes render upper-case.
-const UPPER_LOCALITY_SHARE = 0.1
-// Layouts: 25% bare-comma, 25% bare-comma-pc, 20% space-pc, 15% canonical-pc-first, 15% commune-pc.
-const BARE_COMMA_CUTOFF = 0.25
-const BARE_COMMA_PC_CUTOFF = 0.5
-const SPACE_PC_CUTOFF = 0.7
-const CANONICAL_PC_FIRST_CUTOFF = 0.85
-// 20% of rows append an explicit "France" + a `country` component.
-const APPEND_COUNTRY_SHARE = 0.2
-
-/**
- * Render one admin-split variant. The CORE teaching signal: the département, even as a full word after a comma or a
- * space, is `region` — never folded into `locality`. Variants 1-3 are the failure class; 4-5 are canonical-FR
- * preservation so the model doesn't over-fire region on every trailing token (and the bare commune still resolves).
- */
-function render(random: () => number, c: CommuneRow): AdminSplitVariant {
-	const r = random()
-	const loc = random() < UPPER_LOCALITY_SHARE ? c.commune.toUpperCase() : c.commune
-	const dep = c.departement
-	const pc = c.postcode
-	let out: AdminSplitVariant
-
-	if (r < BARE_COMMA_CUTOFF) {
-		// 1. bare comma, NO postcode — the Thauron/#727 shape (anchor off)
-		out = { raw: `${loc}, ${dep}`, components: { locality: loc, region: dep }, order: "bare-comma" }
-	} else if (r < BARE_COMMA_PC_CUTOFF) {
-		// 2. bare comma + postcode — anchor ON
-		out = {
-			raw: `${loc}, ${dep} ${pc}`,
-			components: { locality: loc, region: dep, postcode: pc },
-			order: "bare-comma-pc",
-		}
-	} else if (r < SPACE_PC_CUTOFF) {
-		// 3. space-delimited admin (the AU `CANBERRA ACT` fuse applied to FR) — anchor ON
-		out = { raw: `${loc} ${dep} ${pc}`, components: { locality: loc, region: dep, postcode: pc }, order: "space-pc" }
-	} else if (r < CANONICAL_PC_FIRST_CUTOFF) {
-		// 4. canonical FR postcode-first (NO département) — preservation, anchor ON
-		out = { raw: `${pc} ${loc}`, components: { postcode: pc, locality: loc }, order: "canonical-pc-first" }
-	} else {
-		// 5. commune + postcode (NO département) — preservation, anchor ON
-		out = { raw: `${loc} ${pc}`, components: { locality: loc, postcode: pc }, order: "commune-pc" }
-	}
-
-	// fr.country preservation (the v1.8.0 #728 finding): the v1.8.0 slice's bare rows carried NO country
-	// token, so the model under-emitted country on FR (fr.country −3.5pp). ~20% of rows now append an
-	// explicit "France" + a `country` component — the model relearns to emit country WHEN the token is
-	// present without over-firing it on the (still-majority) country-less rows. Substring invariant holds.
-	if (random() < APPEND_COUNTRY_SHARE) {
-		out = {
-			raw: `${out.raw}, France`,
-			components: { ...out.components, country: "France" },
-			order: `${out.order}+fr`,
-		}
-	}
-
-	return out
-}
-
-/**
- * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
- */
-export const frAdminSplitRecipe: CorpusRecipe = {
-	name: "fr-admin-split",
-	description: "FR admin-split rows: BAN communes → split département into `region` (+ canonical-FR preservation)",
-	mode: "generate",
-	options: [
-		{ flag: "--communes <tsv>", description: "BAN commune+postcode+coord TSV. Default /tmp/reg/fr-communes.tsv" },
-	],
-	async run(opts, write) {
-		// Legacy build-fr-admin-split-slice.mjs seeded `mulberry32(opts.seed)`.
-		const random = makeMulberry32(opts.seed)
-		const count = opts.count ?? 60_000
-		const source = opts.sourceName ?? "synth-fr-admin-split"
-		const communesPath = opts.communes ?? DEFAULT_COMMUNES
-
-		const pool = await readCommunes(communesPath)
-
-		console.error(`  ${communesPath}: ${pool.length} communes with derived département`)
-
-		if (!pool.length) {
-			throw new Error("No communes — build the TSV from BAN first (see the recipe header).")
-		}
-
-		let emitted = 0
-		let skipped = 0
-		let guard = 0
-		const orderCounts: Record<string, number> = {}
-		const N = pool.length
-
-		while (emitted < count && guard++ < count * 12) {
-			const base = pool[Math.floor(random() * N)]!
-			const { raw, components, order } = render(random, base)
-
-			// Alignment precondition: every component surface appears verbatim in raw.
-			const values = Object.values(components).filter((v): v is string => Boolean(v))
-
-			if (!values.every((v) => raw.includes(v))) {
-				skipped++
-
-				continue
-			}
-
-			if (opts.golden) {
-				// Held-out eval slice for the centroid check — carries the truth coordinate.
-				write(JSON.stringify({ raw, components, country: "FR", lat: Number(base.lat), lon: Number(base.lon) }) + "\n")
-
-				emitted++
-				orderCounts[order] = (orderCounts[order] ?? 0) + 1
-
-				continue
-			}
-
-			const sourceID = stableSourceID(source, {
-				locality: components.locality,
-				region: components.region,
-				postcode: components.postcode,
-			})
-
-			const canonical: CanonicalRow = {
-				raw,
-				components,
-				country: "FR",
-				locale: "fr-FR",
-				source,
-				source_id: sourceID,
-				corpus_version: "0.5.0",
-				license: LICENSE,
-			}
-
-			const aligned = alignRow(canonical)
-
-			if (aligned.kind !== "labeled" || !aligned.row) {
-				skipped++
-
-				continue
-			}
-
-			write(
-				JSON.stringify({ ...aligned.row, synth_method: "fr-admin-split", synth_order: order, synth_base_id: null }) +
-					"\n"
-			)
-
-			emitted++
-			orderCounts[order] = (orderCounts[order] ?? 0) + 1
-		}
-
-		console.error(`  emitted=${emitted} skipped=${skipped} order-mix=${JSON.stringify(orderCounts)}`)
-
-		return { emitted, skipped }
-	},
-}

package/lib/recipes/fr-bare-street.ts

@@ -1,171 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `fr-bare-street` slice recipe (#251) — the postcode-anchoring-imbalance change. BAN (and every
- *   other comprehensive FR source) is postcode-COMPLETE, so the model learned the French
- *   street→locality boundary as "the token after the 5-digit postcode," never as "comma + city." Strip
- *   the postcode and it leaks the street's proper-noun tokens into the following locality ("Rue René
- *   Cassin, Paris" → street="Rue Ren", locality="Cassin"). This recipe mints the MISSING distribution:
- *   the BARE comma form, NO postcode, real `(street, number, city)` tuples from BAN (Licence Ouverte —
- *   permissive; the model stays clean of ODbL, unlike the opt-in OSM rooftop slices).
- *
- *   Each tuple → `<n> <Rue/Avenue/…> <proper-noun name>, <City>` with the FR prefix split
- *   ({@link decomposeFrStreet}: "Rue" → street_prefix, the rest → street). Tuples whose street carries
- *   no recognized FR type word are skipped — the failing class is precisely the prefix-led street.
- *
- *   ⚠ Convention loss-mask: this recipe TEACHES FR `street_prefix`. The conventions loss-mask forbids it
- *   for FR and will `-inf` these gold labels (the v1.6.0 ~7M-loss blow-up). Disable that mask for any
- *   run including this slice.
- */
-
-import { FR_VOIE_TYPES } from "@mailwoman/codex/fr"
-import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
-
-import { decomposeFrStreet } from "#adapters/ban/street-decompose"
-import { alignAndWrite, readTuples, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
-
-/**
- * Canonical voie type (lowercase, accent-kept) → its most common written abbreviation, from the codex table's first
- * entry. Types with no attested abbreviation stay canonical in the abbreviated form.
- */
-const FR_VOIE_ABBREV: Record<string, string> = Object.fromEntries(
-	Object.entries(FR_VOIE_TYPES).flatMap(([canonical, abbrevs]) => (abbrevs[0] ? [[canonical, abbrevs[0]]] : []))
-)
-
-/**
- * The order-cycle slot for the BARE-STREET-ONLY form (`«voie» «name»`, no number, no locality) — the absence
- * counterweight to the locality-terminated comma-free forms (see the cycle comment).
- */
-const BARE_STREET_ONLY_FORM = 3
-
-/**
- * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
- */
-export const frBareStreetRecipe: CorpusRecipe = {
-	name: "fr-bare-street",
-	description:
-		"FR bare street+city, NO postcode (#251): comma / comma-free / abbreviated-voie surfaces over one label set",
-	mode: "tuples",
-	async run(opts, write) {
-		// Seeded for parity with the other recipes; unused beyond reproducibility (the tuples drive the content).
-		makeMulberry32(opts.seed)
-		let read = 0
-		let emitted = 0
-		let skipped = 0
-
-		for await (const t of readTuples(opts.input!)) {
-			read++
-			const fullStreet = String(t.street ?? "").trim()
-			const number = String(t.number ?? "").trim()
-			const locality = String(t.locality ?? "").trim()
-
-			if (!fullStreet || !number || !locality) {
-				skipped++
-
-				continue
-			}
-
-			const { prefix, street } = decomposeFrStreet(fullStreet)
-
-			// A no-prefix nom_voie ("La Ville Mois") is not the prefix-led class the numbered forms
-			// exercise — but as a BARE surface it is exactly the non-voie-led counterweight the v4.5.1
-			// probe showed missing ('Savile Row'-shaped spans still fell to the trailing-locality
-			// prior; the voie-led bare form protected only voie-led spans). Alternate rows emit the
-			// whole span as a bare street; the rest skip as before.
-			if (!prefix || !street) {
-				if (read % 2 === 0 && fullStreet.split(" ").length >= 2) {
-					const bare = {
-						raw: fullStreet,
-						components: { street: fullStreet } as Record<string, string>,
-						country: "FR",
-						locale: "fr-FR",
-						source: "synth-fr-bare-street",
-						source_id: sliceSourceID("synth-fr-bare-street", {
-							street: fullStreet,
-							f: "bare-nonvoie",
-							v: String(read),
-						}),
-						corpus_version: "0.9.4",
-						license:
-							"Synthetic — fr-bare-street; (street, number, city) from BAN (Base Adresse Nationale, Licence Ouverte)",
-					}
-
-					if (alignAndWrite(write, bare, "fr-bare-street")) {
-						emitted++
-					} else {
-						skipped++
-					}
-
-					continue
-				}
-
-				skipped++
-
-				continue
-			}
-
-			// Four surfaces over the same tuple, cycled deterministically. The comma form was the
-			// original change; the COMMA-FREE form is the colloquial register users actually type
-			// ('12 rue de Rome Paris' — the street↔locality boundary with NO delimiter, the fr-fr
-			// panel's named loss); the ABBREVIATED form is the typeahead register the geocoder-tester
-			// FR slice attests at scale; and the BARE-STREET-ONLY form is the absence counterweight —
-			// without it, every delimiter-free surface in the mix ENDS in a locality, the model learns
-			// "trailing span = locality" as categorical, and bare street names across locales flip to
-			// locality wholesale (the v4.5.0 no-promote's measured erosion: 'Calle de Alcalá',
-			// 'Madison Square West', and COMER's fork all fell to that prior). Tags are identical
-			// where present; each component VALUE is the span as written (BIO alignment binds value
-			// to surface); the bare form carries NO number and NO locality because the surface has
-			// neither.
-			const form = read % 4
-			const prefixSurface = form >= 2 ? (FR_VOIE_ABBREV[prefix.toLowerCase()] ?? prefix) : prefix
-
-			const components: Record<string, string> =
-				form === BARE_STREET_ONLY_FORM
-					? { street_prefix: prefix, street }
-					: { house_number: number, street_prefix: prefixSurface, street, locality }
-
-			// When the tuple carries a WOF-attested neighbourhood, the comma slot renders the
-			// THREE-slot middle surface — the dependent-locality counterweight (the v4.5.1 erosion's
-			// untouched half: the two-slot comma-free endings squeezed the middle tag out).
-			const hood = String(t.neighbourhood ?? "").trim()
-
-			if (form === 0 && hood) {
-				components.dependent_locality = hood
-			}
-
-			const raw =
-				form === 0
-					? hood
-						? `${number} ${prefix} ${street}, ${hood}, ${locality}`
-						: `${number} ${prefix} ${street}, ${locality}`
-					: form === BARE_STREET_ONLY_FORM
-						? `${prefix} ${street}`
-						: `${number} ${prefixSurface} ${street} ${locality}`
-
-			const source_id = sliceSourceID("synth-fr-bare-street", { ...components, f: String(form), v: String(read) })
-
-			const canonical = {
-				raw,
-				components,
-				country: "FR",
-				locale: "fr-FR",
-				source: "synth-fr-bare-street",
-				source_id,
-				corpus_version: "0.9.4",
-				license:
-					"Synthetic — fr-bare-street; (street, number, city) from BAN (Base Adresse Nationale, Licence Ouverte)",
-			}
-
-			if (alignAndWrite(write, canonical, "fr-bare-street")) {
-				emitted++
-			} else {
-				skipped++
-			}
-		}
-
-		return { read, emitted, skipped }
-	},
-}