@mailwoman/corpus 9.3.0 → 10.0.0

package/lib/synthesizers/german.ts

@@ -1,260 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   German address synthesizer — multi-locale coverage (night-shift 2026-06-02, DE-1).
- *
- *   The neural model is out-of-distribution on German: it truncates `Straußstraße`→`Strau` (exits at
- *   the ß-piece boundary), absorbs the house number into the street (`Hauptstraße 5` → one span),
- *   and mis-tags the native-order house number as a postcode (`Prenzlauer Allee 36, 10405 Berlin` →
- *   postcode `36`). The cause is ORDER: the model was trained US+FR (house-number-FIRST,
- *   postcode-AFTER-city), and never saw the German convention (house-number-AFTER-street,
- *   postcode-BEFORE-city). DE-0 confirmed the tokenizer round-trips German orthography cleanly, so
- *   this is a coverage gap, not a tokenizer ceiling.
- *
- *   This generator produces the missing signal as a small targeted supplement slice
- *   (synthesis-as-supplement discipline: weight < 0.25, one-and-done). It does NOT synthesize
- *   German street names (German morphology is hard to fake) — it takes REAL German component tuples
- *   (from OpenAddresses Berlin/Saxony) and renders them in idiomatic German order via the OpenCage
- *   `DE` template (`formatAddress(..., "DE")` → `"Straußstraße 27, 12623 Berlin"`). The corpus
- *   aligner turns the row into BIO labels; every emitted component surface form occurs verbatim in
- *   `raw` so alignment lands.
- */
-
-import { formatAddress } from "@mailwoman/formatter"
-
-import type { CanonicalRow } from "#types"
-
-/**
- * A real address tuple (e.g. one OpenAddresses row): street + locality required, rest optional.
- */
-/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
-   cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
-   output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
-   would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
-   extracted as named constants above. */
-
-export interface LocaleBaseTuple {
-	house_number?: string
-	street: string
-	locality: string
-	/**
-	 * A sub-locality that sits BELOW the locality (a suburb / district). NZ is the case that needs it: the OA DISTRICT
-	 * column holds the city (`Auckland`) and CITY holds the suburb (`Birkenhead`), so the real envelope carries both (`31
-	 * Rawene Road, Birkenhead, Auckland`). Rendered between street and locality in both orders when present.
-	 */
-	dependent_locality?: string
-	region?: string
-	postcode?: string
-}
-
-/**
- * @deprecated Alias — use LocaleBaseTuple.
- */
-export type GermanBaseTuple = LocaleBaseTuple
-
-export interface SynthesizedLocaleRow {
-	raw: string
-	components: CanonicalRow["components"]
-	locale: string
-}
-
-/**
- * @deprecated Alias — use SynthesizedLocaleRow.
- */
-export type SynthesizedGermanRow = SynthesizedLocaleRow
-
-export interface LocaleSynthesisOpts {
-	random?: () => number
-	/**
-	 * Rendering order for the SAME components. `"native"` (default) uses the country's own template (DE →
-	 * house-AFTER-street, postcode-BEFORE-city). `"international"` renders house-FIRST, postcode-AFTER-city — the US/GB
-	 * layout that international feeds, US-centric systems, and our own OpenAddresses de-sample impose on non-US
-	 * addresses. Training both teaches the model that a German address can arrive either way, so the eval's US-order
-	 * rendering stops reading as a collapse. See `docs/articles/evals/resolver-geo/2026-06-06-anchor-pilot.md` (the
-	 * order-artifact correction).
-	 */
-	order?: "native" | "international"
-	/**
-	 * Postcode surface shape. `"conventional"` (default) canonicalizes to the country's rendered form (NL: OA's glued
-	 * `1011AB` → the spaced `1011 AB`); `"as-source"` keeps the source's own surface — the form OA (and the OA-derived
-	 * evals) feed, which for NL is 100% glued. Only NL differs today; every other country passes through identically
-	 * either way. Mixing both teaches the two-letter-suffix `1012 LM` shape AND the glued feed shape (#241 — the model
-	 * currently glues the suffix onto the city).
-	 */
-	postcodeShape?: "conventional" | "as-source"
-	/**
-	 * How the NATIVE-order render joins street and house number. The OpenCage ES template comma-joins (`Calle Mayor, 12`
-	 * — the official Spanish convention); OA-derived feeds and our ES eval space-join (`CALLE MAYOR 12`, the observed
-	 * form on all 3,000 eval rows). `"template"` (default) keeps the template's own join; `"space"` collapses `<street>,
-	 * <house_number>` → `<street> <house_number>` after rendering. Countries whose template already space-joins
-	 * (DE/IT/NL) render identically under both. Mixing both stops an ES slice from teaching the comma as THE street→house
-	 * boundary signal (#241 format-diversity audit). International order ignores this (the US template is already
-	 * house-first space-joined).
-	 */
-	nativeHouseJoin?: "template" | "space"
-	/**
-	 * The string between rendered address lines. `", "` (default) is the template's own join. `" "` renders the
-	 * comma-free single-line register — dictation, a copy out of a one-field form — `Neusser Str. 12 Nippes 50733 Köln`
-	 * for the same components. Stage 2 segments the comma form into three and the comma-free form into ONE, and a single
-	 * segment starves the placetype-pair prior, which is how the comma-free form loses `Nippes` (#1946). Only the native
-	 * order reads it; the international layout keeps its own separator.
-	 */
-	separator?: ", " | " "
-}
-
-/**
- * @deprecated Alias — use LocaleSynthesisOpts.
- */
-export type GermanSynthesisOpts = LocaleSynthesisOpts
-
-/**
- * ISO-3166 alpha-2 → BCP-47 tag for the emitted rows (primary language per country).
- */
-const LOCALE_TAG: Record<string, string> = {
-	DE: "de-DE",
-	ES: "es-ES",
-	IT: "it-IT",
-	NL: "nl-NL",
-	GB: "en-GB",
-	FR: "fr-FR",
-	US: "en-US",
-	NZ: "en-NZ",
-}
-
-/**
- * Canonicalize a postcode to the form the country's template renders, so the stored component aligns verbatim against
- * `raw`. NL is the case that needs it: OA stores `1011AB` but the OpenCage NL template emits the conventional spaced
- * `1011 AB` (4 digits + space + 2 letters), which otherwise fails verbatim alignment and drops the row. Other countries
- * pass through unchanged.
- */
-function normalizePostcode(postcode: string, country: string): string {
-	if (country === "NL") {
-		const m = /^(\d{4})\s*([A-Za-z]{2})$/.exec(postcode)
-
-		if (m) return `${m[1]} ${m[2]!.toUpperCase()}`
-	}
-
-	return postcode
-}
-
-/**
- * True when `value` appears verbatim AND as a standalone token (so BIO alignment lands cleanly).
- */
-function tokenPresent(raw: string, value: string): boolean {
-	if (!raw.includes(value)) return false
-	// Reject substring-of-a-larger-number collisions (e.g. house "2" inside postcode "12623").
-	const i = raw.indexOf(value)
-	const before = raw[i - 1]
-	const after = raw[i + value.length]
-	const isDigit = (c: string | undefined) => c !== undefined && c >= "0" && c <= "9"
-
-	if (/^\d+$/.test(value) && (isDigit(before) || isDigit(after))) return false
-
-	return true
-}
-
-/**
- * Render one real tuple into an idiomatic, locale-ordered `{raw, components}` row via the OpenCage `country` template
- * (DE → house-after-street + postcode-before-city; ES/IT the same; GB house-first; NL carries the `1012 LM` postcode),
- * with light variation (drop house number / postcode some of the time). Returns `null` when the tuple is too thin or a
- * component wouldn't align cleanly.
- *
- * Region handling is order-dependent: NATIVE order omits it (the native template absorbs the admin region into the
- * postcode/city line, so it rarely renders verbatim and would break BIO alignment), while INTERNATIONAL order includes
- * it in the tail ("City, Region Postcode" — the US/feed layout the eval uses; v0.9.3 / #327).
- *
- * Pass `opts.order: "international"` to render the same components house-first / postcode-after-city instead (see
- * {@link LocaleSynthesisOpts.order}) — the layout international feeds impose on foreign addresses, and the one a
- * native-order-trained model treats as a "collapse."
- */
-export function synthesizeLocaleRow(
-	base: LocaleBaseTuple,
-	country: string,
-	opts: LocaleSynthesisOpts = {}
-): SynthesizedLocaleRow | null {
-	const random = opts.random ?? Math.random
-	const order = opts.order ?? "native"
-
-	if (!base.street || !base.locality) return null
-
-	const components: CanonicalRow["components"] = { street: base.street, locality: base.locality }
-
-	// Sub-locality (suburb / district) sits between street and locality and renders in BOTH orders — it's part
-	// of the address body, not the admin-region tail that native order drops. NZ needs it (suburb + city both on
-	// the envelope). The tokenPresent check below drops the row if the template didn't surface it verbatim.
-	if (base.dependent_locality) {
-		components.dependent_locality = base.dependent_locality
-	}
-
-	// ~80% keep the house number (the rest are street-only forms, also idiomatic).
-	if (base.house_number && random() < 0.8) {
-		components.house_number = base.house_number
-	}
-
-	// ~85% keep the postcode (canonicalized to the country's rendered form — NL spaces it). The
-	// `postcodeShape: "as-source"` rewrite happens AFTER the render: the OpenCage NL template
-	// normalizes the postcode itself, so a glued input can't survive rendering directly.
-	if (base.postcode && random() < 0.85) {
-		components.postcode = normalizePostcode(base.postcode, country)
-	}
-
-	// International order carries the REGION in the tail ("City, Region Postcode") — the layout real
-	// US/feed renderings (and our OA eval) use. v0.9.2 rendered international order WITHOUT the region,
-	// so the model never learned to segment the tail and mangled it at eval (region absorbed into the
-	// locality / locality dropped); v0.9.3 closes that gap (#327). Native order still drops the region
-	// (the native template absorbs it into the city line, which would break verbatim alignment).
-	if (order === "international" && base.region) {
-		components.region = base.region
-	}
-
-	// Native order uses the address's own country template; international order uses the US template —
-	// house-first, postcode-after-city, with a region slot for the tail. Neither branch consumes a
-	// `random()` draw for the template, so the RNG sequence existing callers/tests depend on is stable.
-	const renderCountry = order === "international" ? "US" : country
-	const separator = order === "native" ? (opts.separator ?? ", ") : ", "
-	let raw = formatAddress(components, renderCountry, { separator })
-
-	if (!raw) return null
-
-	// Native-order space-join (see {@link LocaleSynthesisOpts.nativeHouseJoin}): collapse the template's
-	// `<street>, <hn>` comma to a space — the OA/feed layout. A no-op for templates that already
-	// space-join (the substring isn't present), and skipped when the house number was dropped above.
-	if (order === "native" && separator === ", " && opts.nativeHouseJoin === "space" && components.house_number) {
-		raw = raw.replace(
-			`${components.street}, ${components.house_number}`,
-			`${components.street} ${components.house_number}`
-		)
-	}
-
-	// `postcodeShape: "as-source"` (see {@link LocaleSynthesisOpts.postcodeShape}): rewrite BOTH raw and
-	// the component back to the source's own surface (NL glued `1011AB`). Post-render because the
-	// OpenCage NL template normalizes the postcode to the spaced form no matter what it's given.
-	if (opts.postcodeShape === "as-source" && components.postcode && base.postcode) {
-		const sourceForm = base.postcode.trim()
-
-		if (sourceForm && sourceForm !== components.postcode && raw.includes(components.postcode)) {
-			raw = raw.replace(components.postcode, sourceForm)
-			components.postcode = sourceForm
-		}
-	}
-
-	// Every component must align — drop the row if the template didn't surface one verbatim, or a
-	// numeric component collides with a neighbouring digit run.
-	for (const value of Object.values(components)) {
-		if (!value || !tokenPresent(raw, value)) return null
-	}
-
-	return { raw, components, locale: LOCALE_TAG[country] ?? country.toLowerCase() }
-}
-
-/**
- * German wrapper over {@link synthesizeLocaleRow}. Kept for the build-german-slice caller + tests.
- */
-export function synthesizeGermanRow(
-	base: LocaleBaseTuple,
-	opts: LocaleSynthesisOpts = {}
-): SynthesizedLocaleRow | null {
-	return synthesizeLocaleRow(base, "DE", opts)
-}

package/lib/tools/fetch/ban.ts

@@ -1,289 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the full French BAN (Base Adresse Nationale) — all metropolitan départements (01-95, 2A,
- *   2B) plus 5 overseas DOM/TOM (971-976 excl. 975).
- *
- *   Source: https://adresse.data.gouv.fr/data/ban/adresses/latest/csv/
- *   Licence: Licence Ouverte 2.0 (attribution required — Tier B).
- *
- *   Files already present with matching sha256 are skipped (re-runnable). Downloads `.csv.gz`,
- *   decompresses to `.csv`, deletes the `.gz` artifact. One shared `MANIFEST.json` at
- *   `<outRoot>/ban/MANIFEST.json` covers all codes.
- *
- *   Invoke via `mailwoman corpus fetch ban --out-root <path>`. Built-in `fetch` with gzip/brotli
- *   decompression replaces curl; native `node:zlib` gunzip replaces the `gunzip` subprocess; no
- *   Python.
- */
-
-import { gunzip } from "@mailwoman/core/fs/compression"
-import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters"
-import { isFile, readLocalBuffer, tryStat } from "@mailwoman/core/fs/readers"
-import { makeDirectories, removePath, writeLocalFile } from "@mailwoman/core/fs/writers"
-import { sha256File } from "@mailwoman/core/hash"
-import { sleep } from "@mailwoman/core/utils/sleep"
-import { join } from "path-ts"
-
-import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
-import { downloadToFile, loadManifestEntries, writeManifest } from "#tools/fetch/download"
-
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-
-const BASE_URL = "https://adresse.data.gouv.fr/data/ban/adresses/latest/csv"
-
-/**
- * All département codes — metropolitan 01-95 (with 2A/2B for Corsica instead of 20) plus overseas DOM/TOM. Codes do not
- * change.
- */
-const DEPT_CODES = [
-	"01",
-	"02",
-	"03",
-	"04",
-	"05",
-	"06",
-	"07",
-	"08",
-	"09",
-	"10",
-	"11",
-	"12",
-	"13",
-	"14",
-	"15",
-	"16",
-	"17",
-	"18",
-	"19",
-	"21",
-	"22",
-	"23",
-	"24",
-	"25",
-	"26",
-	"27",
-	"28",
-	"29",
-	"2A",
-	"2B",
-	"30",
-	"31",
-	"32",
-	"33",
-	"34",
-	"35",
-	"36",
-	"37",
-	"38",
-	"39",
-	"40",
-	"41",
-	"42",
-	"43",
-	"44",
-	"45",
-	"46",
-	"47",
-	"48",
-	"49",
-	"50",
-	"51",
-	"52",
-	"53",
-	"54",
-	"55",
-	"56",
-	"57",
-	"58",
-	"59",
-	"60",
-	"61",
-	"62",
-	"63",
-	"64",
-	"65",
-	"66",
-	"67",
-	"68",
-	"69",
-	"70",
-	"71",
-	"72",
-	"73",
-	"74",
-	"75",
-	"76",
-	"77",
-	"78",
-	"79",
-	"80",
-	"81",
-	"82",
-	"83",
-	"84",
-	"85",
-	"86",
-	"87",
-	"88",
-	"89",
-	"90",
-	"91",
-	"92",
-	"93",
-	"94",
-	"95",
-	"971",
-	"972",
-	"973",
-	"974",
-	"976",
-]
-
-export interface BanManifestEntry {
-	dept_code: string
-	filename: string
-	source_url: string
-	downloaded_at: string
-	sha256: string
-	bytes: number
-}
-
-export type FetchBanOptions = BaseFetchOptions
-
-export async function fetchBan(options: FetchBanOptions, report?: (line: string) => void): Promise<FetchSummary> {
-	const banDir = join(options.outRoot, "ban")
-	const manifestPath = join(banDir, "MANIFEST.json")
-	await makeDirectories(banDir)
-
-	// Load existing entries (code -> entry): skip detection + preservation of untouched codes.
-	const entries = await loadManifestEntries<BanManifestEntry>(manifestPath, (entry) => entry.dept_code)
-
-	let fetched = 0
-	let skipped = 0
-	let failed = 0
-	const failedCodes: string[] = []
-
-	for (const code of DEPT_CODES) {
-		const filename = `adresses-${code}.csv`
-		const gzFile = join(banDir, `${filename}.gz`)
-		const csvFile = join(banDir, filename)
-		const url = `${BASE_URL}/adresses-${code}.csv.gz`
-
-		report?.(`=== dept ${code}`)
-
-		// If the CSV already exists, compare its sha256 against the manifest.
-		if (await isFile(csvFile)) {
-			const existingSha = await sha256File(csvFile)
-			const recordedSha = entries.get(code)?.sha256
-
-			if (recordedSha && existingSha === recordedSha) {
-				report?.(`  → already present + sha matches — skipping`)
-
-				skipped++
-
-				continue
-			}
-
-			report?.(`  → present but sha mismatch or no manifest entry — re-fetching`)
-			await removePath(csvFile)
-		}
-
-		// Download the gzipped CSV.
-		try {
-			await downloadToFile({
-				url,
-				dest: gzFile,
-				timeoutMs: 600_000,
-				headers: { "Accept-Encoding": "gzip, br" },
-				report,
-			})
-		} catch (error) {
-			report?.(`  ✗ download failed: ${url} (${(error as Error).message})`)
-
-			failed++
-			failedCodes.push(code)
-
-			continue
-		}
-
-		// Guard against truncated 404/error pages.
-		const gzSize = (await tryStat(gzFile))?.size ?? 0
-
-		if (gzSize < BYTES_PER_KIB) {
-			report?.(`  ✗ response too small (${gzSize} bytes) — probable 404 / error page`)
-			await removePath(gzFile)
-
-			failed++
-			failedCodes.push(code)
-
-			continue
-		}
-
-		// Decompress in-place; delete the .gz.
-		try {
-			await writeLocalFile(await gunzip(await readLocalBuffer(gzFile)), csvFile)
-		} catch (error) {
-			report?.(`  ✗ decompress failed: ${(error as Error).message}`)
-			await removePath(gzFile)
-
-			failed++
-			failedCodes.push(code)
-
-			continue
-		}
-
-		await removePath(gzFile)
-
-		if (!(await isFile(csvFile))) {
-			report?.(`  ✗ decompressed file not found at ${csvFile}`)
-
-			failed++
-			failedCodes.push(code)
-
-			continue
-		}
-
-		const bytes = (await tryStat(csvFile))?.size ?? 0
-		const sha = await sha256File(csvFile)
-
-		entries.set(code, {
-			dept_code: code,
-			filename,
-			source_url: url,
-			downloaded_at: new Date().toISOString(),
-			sha256: sha,
-			bytes,
-		})
-
-		report?.(`  ✓ ${ByteFormatter.formatIEC(bytes)}  sha256=${sha}`)
-
-		fetched++
-
-		// Be a polite citizen — short pause between requests.
-		await sleep(200)
-	}
-
-	// Write the consolidated MANIFEST.json (entries sorted by dept_code, codepoint order).
-	const sorted = [...entries.values()].toSorted((a, b) =>
-		a.dept_code < b.dept_code ? -1 : a.dept_code > b.dept_code ? 1 : 0
-	)
-
-	await writeManifest(manifestPath, sorted)
-	report?.(`Wrote ${manifestPath} with ${sorted.length} entries.`)
-
-	report?.(`=== summary ===`)
-	report?.(`fetched:  ${fetched}`)
-	report?.(`skipped:  ${skipped} (already present + sha matched)`)
-	report?.(`failed:   ${failed}`)
-
-	if (failedCodes.length) {
-		report?.(`failed codes: ${failedCodes.join(" ")}`)
-	}
-
-	return { fetched, skipped, failed, failedCodes }
-}