@mailwoman/corpus 9.3.0 → 10.0.0

package/lib/fr/recipes/order.ts

@@ -0,0 +1,246 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `fr-order` slice recipe — French reversed-order coverage (#560). Reads REAL OpenAddresses FR
+ *   tuples (`fr/countrywide.csv` from the cached zip), then for each row picks — by
+ *   `--reversed-fraction` (default 0.5) — whether to render in CANONICAL French order
+ *   (number-street, postcode-city) or one of four REVERSED / postcode-first variants the
+ *   v4.4.0→v0.5.0 regression exposed (the model misses house_number in every reversed one):
+ *
+ *   - A: "47110 Sainte-Livrade-sur-Lot, 69 Allée du Bugatel" (postcode city, HN street)
+ *   - B: "Sainte-Livrade-sur-Lot, 47110, 619 Impasse de la Rose" (city, postcode, HN street)
+ *   - C: "Sainte-Livrade-sur-Lot 59 bis Rue des Ecuries 47110" (city HN street postcode — NO commas)
+ *   - D: "47110, 6 rue de la république, Sainte-Livrade-sur-Lot" (postcode, HN street, city)
+ *
+ *   Sub-modes ride alongside order: `bis`/`ter`/`quater` ordinal suffixes in house_number, and
+ *   ALL-CAPS locality. `--golden` emits a held-out reversed-order eval slice with a different
+ *   seed.
+ *
+ *   The inline synthesis (the OA-CSV reader, the ordinal/all-caps tables, the canonical + reversed
+ *   renderers) is ported faithfully from scripts/build-fr-order-slice.mjs. This is a
+ *   `generate`-mode recipe that still reads REAL tuples off disk — `--count` bounds the OUTPUT, not
+ *   the input. The passed `random` (the framework LCG) is consumed in the exact call order the
+ *   legacy script used.
+ *
+ *   NOT ported (diagnostic-only, no effect on emitted bytes): the post-run `runSpanCheck` self-check
+ *   (it reads the finished file back with a separate PRNG and prints to stderr; the recipe's output
+ *   stream is still open during `run`), and the dead `renderReversed` helper (the legacy `main`
+ *   inlined the variant logic and never called it).
+ */
+
+import type { ComponentTag } from "@mailwoman/codex/component"
+import { dataRootPath } from "@mailwoman/core/data-root"
+import { isPresent } from "@mailwoman/core/objects"
+import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
+
+import { stableSourceID } from "#adapters/utils"
+import { readOATuples, type CorpusRecipe } from "#recipes/scaffold"
+import { pick } from "#synthesizers/utils"
+import type { CanonicalRow } from "#types"
+import { alignRow } from "#utils"
+
+const SOURCE = { zip: dataRootPath("oa-cache", "fr__countrywide.zip"), csv: "fr/countrywide.csv" }
+
+/**
+ * Ordinal suffixes used in French house numbers (BAN corpus), to cover the "8 bis" sub-mode.
+ */
+const ORDINAL_SUFFIXES: readonly string[] = ["bis", "ter", "quater"]
+/**
+ * Probability that a row gets an ordinal suffix injected (matches the golden's ~10-15% rate).
+ */
+const ORDINAL_PROB = 0.12
+/**
+ * Probability that a locality renders ALL-CAPS (another sub-mode: "SAINTE-LIVRADE-SUR-LOT").
+ */
+const ALLCAPS_PROB = 0.1
+
+/**
+ * A real FR tuple read out of the cached OA zip.
+ */
+interface FrTuple {
+	house_number: string
+	street: string
+	locality: string
+	postcode: string
+}
+
+/**
+ * Stream FR tuples out of the cached OA zip. The countrywide extract is GB-scale, so this reads only as far as `limit`
+ * distinct tuples — the `break` closes the reader and releases the archive. Only keeps rows with a house_number (the
+ * slice's core signal) and a postcode (required for reversed-order rendering to be meaningful; it is also part of this
+ * recipe's dedup key).
+ */
+async function readTuples(limit: number): Promise<FrTuple[]> {
+	return readOATuples(SOURCE, {
+		limit,
+		requirePostcode: true,
+		dedupIncludesPostcode: true,
+		extra: (fields) => fields,
+	})
+}
+
+/**
+ * Optionally augment a house_number with a French ordinal suffix ("59 bis", "4 ter"). Appended with a space so it forms
+ * one multi-token house_number string that alignRow can still locate verbatim.
+ */
+function maybeAddOrdinal(random: () => number, house_number: string): string {
+	if (random() >= ORDINAL_PROB) return house_number
+	const suffix = pick(ORDINAL_SUFFIXES, random)
+
+	// Vary suffix case: "bis" (lower) vs "BIS" (upper) — a real-world split in the golden.
+	return `${house_number} ${random() < 0.5 ? suffix : suffix.toUpperCase()}`
+}
+
+/**
+ * Render a tuple in CANONICAL French order: "9 Rue de la Promenade, 01200 Villes".
+ */
+function renderCanonical(
+	hn: string,
+	street: string,
+	postcode: string,
+	locality: string
+): { raw: string; components: Partial<Record<ComponentTag, string>> } {
+	const raw = `${hn} ${street}, ${postcode} ${locality}`
+
+	return { raw, components: { house_number: hn, street, postcode, locality } }
+}
+
+// Reversed layouts, a quarter each: A postcode+city then HN+street; B city, postcode, HN+street; C run-together;
+// D postcode, HN+street, city.
+const REVERSED_VARIANT_A_CUTOFF = 0.25
+const REVERSED_VARIANT_B_CUTOFF = 0.5
+const REVERSED_VARIANT_C_CUTOFF = 0.75
+
+/**
+ * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
+export const frOrderRecipe: CorpusRecipe = {
+	name: "fr-order",
+	description: "French reversed-order rows (#560): real OA FR tuples rendered canonical + 4 postcode-first variants",
+	mode: "generate",
+	options: [
+		{ flag: "--reversed-fraction <p>", description: "Fraction rendered reversed-order. Default 0.5" },
+		{ flag: "--golden", description: "Emit the held-out reversed-order eval slice" },
+	],
+	async run(opts, write) {
+		if (opts.count == null) throw new Error("fr-order recipe requires --count <N>")
+		const count = opts.count
+		// Legacy build-fr-order-slice.mjs seeded mulberry32 with the raw seed: `const random = mulberry32(opts.seed)`.
+		// (The omitted diagnostic runSpanCheck used a separate mulberry32(opts.seed + 1) — not part of generation.)
+		const random = makeMulberry32(opts.seed)
+		const source = opts.sourceName ?? "synth-fr-order"
+		const reversedFraction = opts.reversedFraction ?? 0.5
+
+		// Over-read from the CSV so the dedup + filter pass can fill `count` rows.
+		const poolLimit = Math.max(count * 8, 40_000)
+		const pool = await readTuples(poolLimit)
+
+		console.error(`  ${SOURCE.csv}: ${pool.length} unique tuples (capped read)`)
+
+		if (!pool.length) {
+			throw new Error(`No FR tuples found — is ${SOURCE.zip} present?`)
+		}
+
+		let emitted = 0
+		let skipped = 0
+		let guard = 0
+		const N = pool.length
+
+		while (emitted < count && guard++ < count * 10) {
+			const base = pool[Math.floor(random() * N)]!
+			const { street, postcode } = base
+			const locality = random() < ALLCAPS_PROB ? base.locality.toUpperCase() : base.locality
+			const house_number = maybeAddOrdinal(random, base.house_number)
+
+			// Pick canonical vs reversed by --reversed-fraction.
+			const isReversed = random() < reversedFraction
+
+			let rendered: { raw: string; components: Partial<Record<ComponentTag, string>> }
+
+			if (isReversed) {
+				const variantRoll = random()
+				let raw: string
+
+				if (variantRoll < REVERSED_VARIANT_A_CUTOFF) {
+					// Variant A: postcode+city as a unit, then HN+street
+					raw = `${postcode} ${locality}, ${house_number} ${street}`
+				} else if (variantRoll < REVERSED_VARIANT_B_CUTOFF) {
+					// Variant B: city, then postcode, then HN+street (comma-separated, postcode isolated)
+					raw = `${locality}, ${postcode}, ${house_number} ${street}`
+				} else if (variantRoll < REVERSED_VARIANT_C_CUTOFF) {
+					// Variant C: no commas — locality HN street postcode (the "run-together" format)
+					raw = `${locality} ${house_number} ${street} ${postcode}`
+				} else {
+					// Variant D: postcode, HN+street, city (reversed top-to-bottom)
+					raw = `${postcode}, ${house_number} ${street}, ${locality}`
+				}
+
+				rendered = { raw, components: { house_number, street, postcode, locality } }
+			} else {
+				rendered = renderCanonical(house_number, street, postcode, locality)
+			}
+
+			const { raw, components } = rendered
+
+			// Safety check: every component must appear verbatim in raw (alignment precondition).
+			const componentValues = Object.values(components).filter(isPresent)
+
+			if (!componentValues.every((v) => raw.includes(v))) {
+				skipped++
+
+				continue
+			}
+
+			// --golden: emit per-locale-f1 eval rows ({raw, components, country:"FR"}).
+			if (opts.golden) {
+				write(JSON.stringify({ raw, components, country: "FR" }) + "\n")
+
+				emitted++
+
+				continue
+			}
+
+			const sourceID = stableSourceID(source, {
+				street: components.street,
+				house_number: components.house_number,
+				locality: components.locality,
+				postcode: components.postcode,
+			})
+
+			const canonical: CanonicalRow = {
+				raw,
+				components,
+				country: "FR",
+				locale: "fr-FR",
+				source,
+				source_id: sourceID,
+				corpus_version: "0.5.0",
+				license: "OpenAddresses FR countrywide tuples, rendered canonical + reversed-order — see ingest SOURCE",
+			}
+
+			const aligned = alignRow(canonical)
+
+			if (aligned.kind !== "labeled" || !aligned.row) {
+				skipped++
+
+				continue
+			}
+
+			write(
+				JSON.stringify({
+					...aligned.row,
+					synth_method: "fr-order",
+					synth_order: isReversed ? "reversed" : "canonical",
+					synth_base_id: null,
+				}) + "\n"
+			)
+
+			emitted++
+		}
+
+		return { emitted, skipped }
+	},
+}

package/lib/fr/tools/fetch/ban.ts

@@ -0,0 +1,289 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the full French BAN (Base Adresse Nationale) — all metropolitan départements (01-95, 2A,
+ *   2B) plus 5 overseas DOM/TOM (971-976 excl. 975).
+ *
+ *   Source: https://adresse.data.gouv.fr/data/ban/adresses/latest/csv/
+ *   Licence: Licence Ouverte 2.0 (attribution required — Tier B).
+ *
+ *   Files already present with matching sha256 are skipped (re-runnable). Downloads `.csv.gz`,
+ *   decompresses to `.csv`, deletes the `.gz` artifact. One shared `MANIFEST.json` at
+ *   `<outRoot>/ban/MANIFEST.json` covers all codes.
+ *
+ *   Invoke via `mailwoman corpus fetch ban --out-root <path>`. Built-in `fetch` with gzip/brotli
+ *   decompression replaces curl; native `node:zlib` gunzip replaces the `gunzip` subprocess; no
+ *   Python.
+ */
+
+import { gunzip } from "@mailwoman/core/fs/compression"
+import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { isFile, readLocalBuffer, tryStat } from "@mailwoman/core/fs/readers"
+import { makeDirectories, removePath, writeLocalFile } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { sleep } from "@mailwoman/core/utils/sleep"
+import { join } from "path-ts"
+
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download/index"
+import { downloadToFile, loadManifestEntries, writeManifest } from "#tools/fetch/download/index"
+
+/**
+ * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
+ * than data.
+ */
+
+const BASE_URL = "https://adresse.data.gouv.fr/data/ban/adresses/latest/csv"
+
+/**
+ * All département codes — metropolitan 01-95 (with 2A/2B for Corsica instead of 20) plus overseas DOM/TOM. Codes do not
+ * change.
+ */
+const DEPT_CODES = [
+	"01",
+	"02",
+	"03",
+	"04",
+	"05",
+	"06",
+	"07",
+	"08",
+	"09",
+	"10",
+	"11",
+	"12",
+	"13",
+	"14",
+	"15",
+	"16",
+	"17",
+	"18",
+	"19",
+	"21",
+	"22",
+	"23",
+	"24",
+	"25",
+	"26",
+	"27",
+	"28",
+	"29",
+	"2A",
+	"2B",
+	"30",
+	"31",
+	"32",
+	"33",
+	"34",
+	"35",
+	"36",
+	"37",
+	"38",
+	"39",
+	"40",
+	"41",
+	"42",
+	"43",
+	"44",
+	"45",
+	"46",
+	"47",
+	"48",
+	"49",
+	"50",
+	"51",
+	"52",
+	"53",
+	"54",
+	"55",
+	"56",
+	"57",
+	"58",
+	"59",
+	"60",
+	"61",
+	"62",
+	"63",
+	"64",
+	"65",
+	"66",
+	"67",
+	"68",
+	"69",
+	"70",
+	"71",
+	"72",
+	"73",
+	"74",
+	"75",
+	"76",
+	"77",
+	"78",
+	"79",
+	"80",
+	"81",
+	"82",
+	"83",
+	"84",
+	"85",
+	"86",
+	"87",
+	"88",
+	"89",
+	"90",
+	"91",
+	"92",
+	"93",
+	"94",
+	"95",
+	"971",
+	"972",
+	"973",
+	"974",
+	"976",
+]
+
+export interface BanManifestEntry {
+	dept_code: string
+	filename: string
+	source_url: string
+	downloaded_at: string
+	sha256: string
+	bytes: number
+}
+
+export type FetchBanOptions = BaseFetchOptions
+
+export async function fetchBan(options: FetchBanOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	const banDir = join(options.outRoot, "ban")
+	const manifestPath = join(banDir, "MANIFEST.json")
+	await makeDirectories(banDir)
+
+	// Load existing entries (code -> entry): skip detection + preservation of untouched codes.
+	const entries = await loadManifestEntries<BanManifestEntry>(manifestPath, (entry) => entry.dept_code)
+
+	let fetched = 0
+	let skipped = 0
+	let failed = 0
+	const failedCodes: string[] = []
+
+	for (const code of DEPT_CODES) {
+		const filename = `adresses-${code}.csv`
+		const gzFile = join(banDir, `${filename}.gz`)
+		const csvFile = join(banDir, filename)
+		const url = `${BASE_URL}/adresses-${code}.csv.gz`
+
+		report?.(`=== dept ${code}`)
+
+		// If the CSV already exists, compare its sha256 against the manifest.
+		if (await isFile(csvFile)) {
+			const existingSha = await sha256File(csvFile)
+			const recordedSha = entries.get(code)?.sha256
+
+			if (recordedSha && existingSha === recordedSha) {
+				report?.(`  → already present + sha matches — skipping`)
+
+				skipped++
+
+				continue
+			}
+
+			report?.(`  → present but sha mismatch or no manifest entry — re-fetching`)
+			await removePath(csvFile)
+		}
+
+		// Download the gzipped CSV.
+		try {
+			await downloadToFile({
+				url,
+				dest: gzFile,
+				timeoutMs: 600_000,
+				headers: { "Accept-Encoding": "gzip, br" },
+				report,
+			})
+		} catch (error) {
+			report?.(`  ✗ download failed: ${url} (${(error as Error).message})`)
+
+			failed++
+			failedCodes.push(code)
+
+			continue
+		}
+
+		// Guard against truncated 404/error pages.
+		const gzSize = (await tryStat(gzFile))?.size ?? 0
+
+		if (gzSize < BYTES_PER_KIB) {
+			report?.(`  ✗ response too small (${gzSize} bytes) — probable 404 / error page`)
+			await removePath(gzFile)
+
+			failed++
+			failedCodes.push(code)
+
+			continue
+		}
+
+		// Decompress in-place; delete the .gz.
+		try {
+			await writeLocalFile(await gunzip(await readLocalBuffer(gzFile)), csvFile)
+		} catch (error) {
+			report?.(`  ✗ decompress failed: ${(error as Error).message}`)
+			await removePath(gzFile)
+
+			failed++
+			failedCodes.push(code)
+
+			continue
+		}
+
+		await removePath(gzFile)
+
+		if (!(await isFile(csvFile))) {
+			report?.(`  ✗ decompressed file not found at ${csvFile}`)
+
+			failed++
+			failedCodes.push(code)
+
+			continue
+		}
+
+		const bytes = (await tryStat(csvFile))?.size ?? 0
+		const sha = await sha256File(csvFile)
+
+		entries.set(code, {
+			dept_code: code,
+			filename,
+			source_url: url,
+			downloaded_at: new Date().toISOString(),
+			sha256: sha,
+			bytes,
+		})
+
+		report?.(`  ✓ ${ByteFormatter.formatIEC(bytes)}  sha256=${sha}`)
+
+		fetched++
+
+		// Be a polite citizen — short pause between requests.
+		await sleep(200)
+	}
+
+	// Write the consolidated MANIFEST.json (entries sorted by dept_code, codepoint order).
+	const sorted = [...entries.values()].toSorted((a, b) =>
+		a.dept_code < b.dept_code ? -1 : a.dept_code > b.dept_code ? 1 : 0
+	)
+
+	await writeManifest(manifestPath, sorted)
+	report?.(`Wrote ${manifestPath} with ${sorted.length} entries.`)
+
+	report?.(`=== summary ===`)
+	report?.(`fetched:  ${fetched}`)
+	report?.(`skipped:  ${skipped} (already present + sha matched)`)
+	report?.(`failed:   ${failed}`)
+
+	if (failedCodes.length) {
+		report?.(`failed codes: ${failedCodes.join(" ")}`)
+	}
+
+	return { fetched, skipped, failed, failedCodes }
+}

package/lib/index.ts

@@ -5,7 +5,7 @@
  */
 
 export * from "#adapters/utils/index"
-export * from "#adapters/gnaf/assemble"
+export * from "#au/adapters/gnaf/assemble"
 export * from "#adapters/index"
 export * from "#build"
 export * from "#runner"