@mailwoman/corpus 9.1.0 → 9.3.0

package/src/shard-recipes/fr-lieudit.ts

@@ -1,300 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `fr-lieudit` shard recipe — FR lieu-dit (hamlet/place) `dependent_locality` coverage
- *   (`.superpowers/sdd/deploc-world-survey.md`, FR section, 2026-07-22). Streams every BAN
- *   `adresses-<dept>.csv` dump under `--ban-dir` through `@mailwoman/ban/sdk`'s
- *   `extractBANAddrPoints`, which now surfaces a cleaned `lieuDit` per record (junk/dup filtering
- *   lives in `ban/sdk/extract.ts`'s `cleanLieuDit`, not duplicated here). Only rows carrying a clean
- *   lieu-dit survive into the pool; the existing `ban`/`synth-fr` sources and their emitted rows are
- *   untouched — this recipe reads the SAME raw CSVs but emits under its own source name.
- *
- *   Mapping: lieu-dit → `dependent_locality`, commune → `locality`. Rendered to match the formatter's
- *   FR `place`-slot convention (`fix(formatter): render dependent_locality for neither-slot templates`,
- *   b1edc1b7, verified via a `formatAddress` smoke call): house+street on line 1, the lieu-dit ALONE on
- *   line 2, postcode+commune on line 3 — French postal convention (La Poste's line 5).
- *
- *   ~1.69M clean rows survive the filter nationally (26M total BAN rows, 1.81M raw `nom_ld` fills, ~6.6%
- *   junk/dup). The pool is read in full (small string tuples only — no coordinates needed) and
- *   Fisher-Yates shuffled with the seeded PRNG before slicing to `--count`, rather than sampled WITH
- *   replacement — at a `--count` a sizeable fraction of the pool, with-replacement draws would produce a
- *   large duplicate rate (birthday-paradox math: ~190k expected collisions at count=800k over a 1.69M
- *   pool).
- */
-
-import { readdirSync } from "node:fs"
-import { join } from "node:path"
-
-import { extractBANAddrPoints } from "@mailwoman/ban/sdk"
-import { COUNTRY_SURFACE_FORMS } from "@mailwoman/codex/country"
-import type { ComponentTag } from "@mailwoman/core/types"
-import { dataRootPath } from "@mailwoman/core/utils"
-import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
-import type { CanonicalRow } from "@mailwoman/corpus/types"
-import { alignRow } from "@mailwoman/corpus/utils"
-
-import { decomposeFrStreet } from "../adapters/ban/street-decompose.ts"
-import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
-
-/**
- * Matches the `ban` adapter's Tier-B election for BAN data.
- */
-const DEFAULT_LICENSE = "Licence Ouverte 2.0"
-
-/**
- * One BAN row surviving the lieu-dit filter — the minimal tuple the pool holds.
- */
-interface LieuDitTuple {
-	numero: string
-	rep: string | null
-	street: string
-	postcode: string | null
-	locality: string
-	dependentLocality: string
-}
-
-/**
- * Enumerate `adresses-<dept>.csv[.gz]` files in `banDir`, ONE path per département. Excludes the `merged`/`france`
- * aggregates (they duplicate the per-département rows) and, when both a `.csv` and a `.csv.gz` exist for the same dept
- * (observed on disk for 13/2A/48/69/75 — a stale re-fetch artifact), prefers the uncompressed `.csv` — mirrors
- * `ban/scripts/build-address-point-shard.ts`'s `departementFiles`, which hit and fixed this exact double-count trap
- * first.
- */
-function departementFiles(banDir: string): string[] {
-	const byDept = new Map<string, string>()
-
-	for (const name of readdirSync(banDir).toSorted()) {
-		const m = /^adresses-(.+?)\.csv(\.gz)?$/.exec(name)
-
-		if (!m) continue
-
-		const dept = m[1]!
-
-		if (dept === "merged" || dept === "france") continue
-
-		const existing = byDept.get(dept)
-
-		if (!existing || (existing.endsWith(".gz") && !name.endsWith(".gz"))) {
-			byDept.set(dept, join(banDir, name))
-		}
-	}
-
-	return [...byDept.keys()].toSorted().map((dept) => byDept.get(dept)!)
-}
-
-/**
- * Stream every département file, keeping only rows with a clean `lieuDit` (junk/dup filtering lives in `ban/sdk`).
- */
-async function readLieuDitPool(banDir: string): Promise<LieuDitTuple[]> {
-	const files = departementFiles(banDir)
-
-	if (!files.length) {
-		throw new Error(
-			`No BAN adresses-<dept>.csv files found in ${banDir} — fetch BAN first (\`mailwoman corpus fetch ban\`).`
-		)
-	}
-
-	const pool: LieuDitTuple[] = []
-	let scanned = 0
-
-	for (const path of files) {
-		let deptCount = 0
-
-		for await (const rec of extractBANAddrPoints(path)) {
-			scanned++
-
-			if (!rec.lieuDit || !rec.city) continue
-
-			pool.push({
-				numero: rec.numero,
-				rep: rec.rep,
-				street: rec.street,
-				postcode: rec.postcode,
-				locality: rec.city,
-				dependentLocality: rec.lieuDit,
-			})
-
-			deptCount++
-		}
-
-		console.error(`  ${path}: ${deptCount.toLocaleString()} clean lieu-dit rows`)
-	}
-
-	console.error(
-		`  scanned ${scanned.toLocaleString()} BAN rows across ${files.length} départements → pool=${pool.length.toLocaleString()}`
-	)
-
-	return pool
-}
-
-/**
- * `house_number` = `numero` + folded `rep` ("10 bis"), matching the `ban` adapter's own composition.
- */
-function composeHouseNumber(numero: string, rep: string | null): string {
-	return rep ? `${numero} ${rep}` : numero
-}
-
-/**
- * Render the raw address string: house+street line, the lieu-dit ALONE on its own line, postcode+commune line — the
- * exact shape `formatAddress` produces for FR's `place`-slot mapping (verified via a smoke call before this recipe was
- * written; see the module docstring).
- */
-function composeRaw(
-	house: string,
-	street: string,
-	dependentLocality: string,
-	postcode: string | null,
-	locality: string
-): string {
-	const lines: string[] = []
-	const streetLine = `${house} ${street}`.trim()
-
-	if (streetLine) {
-		lines.push(streetLine)
-	}
-
-	lines.push(dependentLocality)
-	const cityLine = [postcode, locality].filter(Boolean).join(" ").trim()
-
-	if (cityLine) {
-		lines.push(cityLine)
-	}
-
-	return lines.join("\n")
-}
-
-/**
- * Fisher-Yates shuffle, in place, with the recipe's seeded PRNG — reproducible sampling without replacement.
- */
-function shuffleInPlace<T>(arr: T[], random: () => number): void {
-	for (let i = arr.length - 1; i > 0; i--) {
-		const j = Math.floor(random() * (i + 1))
-		;[arr[i], arr[j]] = [arr[j]!, arr[i]!]
-	}
-}
-
-/**
- * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
- */
-export const frLieuditRecipe: ShardRecipe = {
-	name: "fr-lieudit",
-	description: "FR lieu-dit rows: BAN nom_ld → dependent_locality (commune → locality), lieu-dit on its own line",
-	mode: "generate",
-	options: [
-		{
-			flag: "--ban-dir <dir>",
-			description: "BAN adresses-<dept>.csv directory. Default $MAILWOMAN_DATA_ROOT/corpus/sources/ban",
-		},
-		{
-			flag: "--country-fraction <f>",
-			description: "Fraction of rows that append an explicit 'France' surface form + a `country` component. Default 0",
-		},
-	],
-	async run(opts, write) {
-		const random = makeMulberry32(opts.seed)
-		const source = opts.sourceName ?? "synth-fr-lieudit"
-		const count = opts.count ?? 800_000
-		const banDir = opts.banDir ?? dataRootPath("corpus", "sources", "ban")
-		const countryFraction = opts.countryFraction ?? 0
-
-		if (!(countryFraction >= 0 && countryFraction <= 1)) {
-			throw new Error(`--country-fraction must be in [0, 1], got ${countryFraction}`)
-		}
-
-		const pool = await readLieuDitPool(banDir)
-
-		if (!pool.length) {
-			throw new Error(`No clean lieu-dit rows found under ${banDir} — see ban/sdk/extract.ts's cleanLieuDit filter.`)
-		}
-
-		shuffleInPlace(pool, random)
-		const selected = pool.slice(0, Math.min(count, pool.length))
-
-		let emitted = 0
-		let skipped = 0
-		let countryAppended = 0
-
-		for (const t of selected) {
-			const house = composeHouseNumber(t.numero, t.rep)
-			const decomposed = decomposeFrStreet(t.street)
-
-			const components: Partial<Record<ComponentTag, string>> = {
-				house_number: house,
-				dependent_locality: t.dependentLocality,
-				locality: t.locality,
-			}
-
-			if (decomposed.prefix) {
-				components.street_prefix = decomposed.prefix
-			}
-
-			if (decomposed.street) {
-				components.street = decomposed.street
-			}
-
-			if (t.postcode) {
-				components.postcode = t.postcode
-			}
-
-			let raw = composeRaw(house, t.street, t.dependentLocality, t.postcode, t.locality)
-
-			if (!raw) {
-				skipped++
-
-				continue
-			}
-
-			// Country-append (the fr-admin-split #728 pattern, generalized): ~`countryFraction` of the time
-			// append an explicit "France" surface form onto the trailing (postcode+commune) line + a
-			// `country` component — the model relearns to emit country WHEN present without over-firing it
-			// on the (still-majority) country-less rows. `countryFraction <= 0` (the default) never draws
-			// from `random`, so the byte-stream is unaffected when the flag is unset.
-			if (countryFraction > 0 && random() < countryFraction) {
-				const forms = COUNTRY_SURFACE_FORMS.FR
-				const form = forms[Math.floor(random() * forms.length)]!
-				raw = `${raw}, ${form}`
-				components.country = form
-
-				countryAppended++
-			}
-
-			const sourceID = stableSourceID(source, {
-				street: t.street,
-				house_number: house,
-				dependent_locality: t.dependentLocality,
-				locality: t.locality,
-				postcode: t.postcode ?? undefined,
-			})
-
-			const canonical: CanonicalRow = {
-				raw,
-				components,
-				country: "FR",
-				locale: "fr-FR",
-				source,
-				source_id: sourceID,
-				corpus_version: "",
-				license: DEFAULT_LICENSE,
-			}
-
-			const aligned = alignRow(canonical)
-
-			if (aligned.kind !== "labeled" || !aligned.row) {
-				skipped++
-
-				continue
-			}
-
-			write(JSON.stringify({ ...aligned.row, synth_method: source, synth_base_id: null }) + "\n")
-
-			emitted++
-		}
-
-		console.error(`  emitted=${emitted} skipped=${skipped} country-appended=${countryAppended} pool=${pool.length}`)
-
-		return { emitted, skipped }
-	},
-}

package/src/shard-recipes/fr-order.ts

@@ -1,256 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `fr-order` shard recipe — French reversed-order coverage (#560). Reads REAL OpenAddresses FR
- *   tuples (`fr/countrywide.csv` from the cached zip), then for each row picks — by
- *   `--reversed-fraction` (default 0.5) — whether to render in CANONICAL French order
- *   (number-street, postcode-city) or one of four REVERSED / postcode-first variants the
- *   v4.4.0→v0.5.0 regression exposed (the model misses house_number in every reversed one):
- *
- *   - A: "47110 Sainte-Livrade-sur-Lot, 69 Allée du Bugatel" (postcode city, HN street)
- *   - B: "Sainte-Livrade-sur-Lot, 47110, 619 Impasse de la Rose" (city, postcode, HN street)
- *   - C: "Sainte-Livrade-sur-Lot 59 bis Rue des Ecuries 47110" (city HN street postcode — NO commas)
- *   - D: "47110, 6 rue de la république, Sainte-Livrade-sur-Lot" (postcode, HN street, city)
- *
- *   Sub-modes ride alongside order: `bis`/`ter`/`quater` ordinal suffixes in house_number, and
- *   ALL-CAPS locality. `--golden` emits a held-out reversed-order eval slice with a different
- *   seed.
- *
- *   The inline synthesis (the OA-CSV reader, the ordinal/all-caps tables, the canonical + reversed
- *   renderers) is ported faithfully from scripts/build-fr-order-shard.mjs. This is a
- *   `generate`-mode recipe that still reads REAL tuples off disk — `--count` bounds the OUTPUT, not
- *   the input. The passed `random` (the framework LCG) is consumed in the exact call order the
- *   legacy script used.
- *
- *   NOT ported (diagnostic-only, no effect on emitted bytes): the post-run `runSpanCheck` self-check
- *   (it reads the finished file back with a separate PRNG and prints to stderr; the recipe's output
- *   stream is still open during `run`), and the dead `renderReversed` helper (the legacy `main`
- *   inlined the variant logic and never called it).
- */
-
-import type { ComponentTag } from "@mailwoman/core/types"
-import { dataRootPath } from "@mailwoman/core/utils"
-import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
-import type { CanonicalRow } from "@mailwoman/corpus/types"
-import { alignRow } from "@mailwoman/corpus/utils"
-
-import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
-
-/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
-   cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
-   output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
-   would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
-   extracted as named constants above. */
-
-const SOURCE = { zip: dataRootPath("oa-cache", "fr__countrywide.zip"), csv: "fr/countrywide.csv" }
-
-/**
- * Ordinal suffixes used in French house numbers (BAN corpus), to cover the "8 bis" sub-mode.
- */
-const ORDINAL_SUFFIXES: readonly string[] = ["bis", "ter", "quater"]
-/**
- * Probability that a row gets an ordinal suffix injected (matches the golden's ~10-15% rate).
- */
-const ORDINAL_PROB = 0.12
-/**
- * Probability that a locality renders ALL-CAPS (another sub-mode: "SAINTE-LIVRADE-SUR-LOT").
- */
-const ALLCAPS_PROB = 0.1
-
-/**
- * A real FR tuple read out of the cached OA zip.
- */
-interface FrTuple {
-	house_number: string
-	street: string
-	locality: string
-	postcode: string
-}
-
-/**
- * Stream FR tuples out of the cached OA zip. The countrywide extract is GB-scale, so this reads only as far as `limit`
- * distinct tuples — the `break` closes the reader and releases the archive. Only keeps rows with a house_number (the
- * shard's core signal) and a postcode (required for reversed-order rendering to be meaningful).
- */
-async function readTuples(limit: number): Promise<FrTuple[]> {
-	const tuples: FrTuple[] = []
-	const seen = new Set<string>()
-
-	for await (const row of readZippedCSVRecords(SOURCE.zip, SOURCE.csv)) {
-		if (tuples.length >= limit) break
-		const street = row.street ?? ""
-		const locality = row.city ?? ""
-		const house_number = row.number ?? ""
-		const postcode = row.postcode ?? ""
-
-		// Require all four fields: HN is the signal; postcode drives reversed-order variants.
-		if (!street || !locality || !house_number || !postcode) continue
-		const key = `${house_number}|${street}|${locality}|${postcode}`.toLowerCase()
-
-		if (seen.has(key)) continue
-		seen.add(key)
-		tuples.push({ house_number, street, locality, postcode })
-	}
-
-	return tuples
-}
-
-/**
- * Optionally augment a house_number with a French ordinal suffix ("59 bis", "4 ter"). Appended with a space so it forms
- * one multi-token house_number string that alignRow can still locate verbatim.
- */
-function maybeAddOrdinal(random: () => number, house_number: string): string {
-	if (random() >= ORDINAL_PROB) return house_number
-	const suffix = ORDINAL_SUFFIXES[Math.floor(random() * ORDINAL_SUFFIXES.length)]!
-
-	// Vary suffix case: "bis" (lower) vs "BIS" (upper) — a real-world split in the golden.
-	return `${house_number} ${random() < 0.5 ? suffix : suffix.toUpperCase()}`
-}
-
-/**
- * Render a tuple in CANONICAL French order: "9 Rue de la Promenade, 01200 Villes".
- */
-function renderCanonical(
-	hn: string,
-	street: string,
-	postcode: string,
-	locality: string
-): { raw: string; components: Partial<Record<ComponentTag, string>> } {
-	const raw = `${hn} ${street}, ${postcode} ${locality}`
-
-	return { raw, components: { house_number: hn, street, postcode, locality } }
-}
-
-/**
- * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
- */
-export const frOrderRecipe: ShardRecipe = {
-	name: "fr-order",
-	description: "French reversed-order rows (#560): real OA FR tuples rendered canonical + 4 postcode-first variants",
-	mode: "generate",
-	options: [
-		{ flag: "--reversed-fraction <p>", description: "Fraction rendered reversed-order. Default 0.5" },
-		{ flag: "--golden", description: "Emit the held-out reversed-order eval slice" },
-	],
-	async run(opts, write) {
-		if (opts.count == null) throw new Error("fr-order recipe requires --count <N>")
-		const count = opts.count
-		// Legacy build-fr-order-shard.mjs seeded mulberry32 with the raw seed: `const random = mulberry32(opts.seed)`.
-		// (The omitted diagnostic runSpanCheck used a separate mulberry32(opts.seed + 1) — not part of generation.)
-		const random = makeMulberry32(opts.seed)
-		const source = opts.sourceName ?? "synth-fr-order"
-		const reversedFraction = opts.reversedFraction ?? 0.5
-
-		// Over-read from the CSV so the dedup + filter pass can fill `count` rows.
-		const poolLimit = Math.max(count * 8, 40_000)
-		const pool = await readTuples(poolLimit)
-
-		console.error(`  ${SOURCE.csv}: ${pool.length} unique tuples (capped read)`)
-
-		if (!pool.length) {
-			throw new Error(`No FR tuples found — is ${SOURCE.zip} present?`)
-		}
-
-		let emitted = 0
-		let skipped = 0
-		let guard = 0
-		const N = pool.length
-
-		while (emitted < count && guard++ < count * 10) {
-			const base = pool[Math.floor(random() * N)]!
-			const { street, postcode } = base
-			const locality = random() < ALLCAPS_PROB ? base.locality.toUpperCase() : base.locality
-			const house_number = maybeAddOrdinal(random, base.house_number)
-
-			// Pick canonical vs reversed by --reversed-fraction.
-			const isReversed = random() < reversedFraction
-
-			let rendered: { raw: string; components: Partial<Record<ComponentTag, string>> }
-
-			if (isReversed) {
-				const variantRoll = random()
-				let raw: string
-
-				if (variantRoll < 0.25) {
-					// Variant A: postcode+city as a unit, then HN+street
-					raw = `${postcode} ${locality}, ${house_number} ${street}`
-				} else if (variantRoll < 0.5) {
-					// Variant B: city, then postcode, then HN+street (comma-separated, postcode isolated)
-					raw = `${locality}, ${postcode}, ${house_number} ${street}`
-				} else if (variantRoll < 0.75) {
-					// Variant C: no commas — locality HN street postcode (the "run-together" format)
-					raw = `${locality} ${house_number} ${street} ${postcode}`
-				} else {
-					// Variant D: postcode, HN+street, city (reversed top-to-bottom)
-					raw = `${postcode}, ${house_number} ${street}, ${locality}`
-				}
-
-				rendered = { raw, components: { house_number, street, postcode, locality } }
-			} else {
-				rendered = renderCanonical(house_number, street, postcode, locality)
-			}
-
-			const { raw, components } = rendered
-
-			// Safety check: every component must appear verbatim in raw (alignment precondition).
-			const componentValues = Object.values(components).filter(Boolean) as string[]
-
-			if (!componentValues.every((v) => raw.includes(v))) {
-				skipped++
-
-				continue
-			}
-
-			// --golden: emit per-locale-f1 eval rows ({raw, components, country:"FR"}).
-			if (opts.golden) {
-				write(JSON.stringify({ raw, components, country: "FR" }) + "\n")
-
-				emitted++
-
-				continue
-			}
-
-			const sourceID = stableSourceID(source, {
-				street: components.street,
-				house_number: components.house_number,
-				locality: components.locality,
-				postcode: components.postcode,
-			})
-
-			const canonical: CanonicalRow = {
-				raw,
-				components,
-				country: "FR",
-				locale: "fr-FR",
-				source,
-				source_id: sourceID,
-				corpus_version: "0.5.0",
-				license: "OpenAddresses FR countrywide tuples, rendered canonical + reversed-order — see ingest SOURCE",
-			}
-
-			const aligned = alignRow(canonical)
-
-			if (aligned.kind !== "labeled" || !aligned.row) {
-				skipped++
-
-				continue
-			}
-
-			write(
-				JSON.stringify({
-					...aligned.row,
-					synth_method: "fr-order",
-					synth_order: isReversed ? "reversed" : "canonical",
-					synth_base_id: null,
-				}) + "\n"
-			)
-
-			emitted++
-		}
-
-		return { emitted, skipped }
-	},
-}