@mailwoman/corpus 9.4.0 → 10.0.0

package/lib/synthesizers/german.ts

@@ -1,260 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   German address synthesizer — multi-locale coverage (night-shift 2026-06-02, DE-1).
- *
- *   The neural model is out-of-distribution on German: it truncates `Straußstraße`→`Strau` (exits at
- *   the ß-piece boundary), absorbs the house number into the street (`Hauptstraße 5` → one span),
- *   and mis-tags the native-order house number as a postcode (`Prenzlauer Allee 36, 10405 Berlin` →
- *   postcode `36`). The cause is ORDER: the model was trained US+FR (house-number-FIRST,
- *   postcode-AFTER-city), and never saw the German convention (house-number-AFTER-street,
- *   postcode-BEFORE-city). DE-0 confirmed the tokenizer round-trips German orthography cleanly, so
- *   this is a coverage gap, not a tokenizer ceiling.
- *
- *   This generator produces the missing signal as a small targeted supplement slice
- *   (synthesis-as-supplement discipline: weight < 0.25, one-and-done). It does NOT synthesize
- *   German street names (German morphology is hard to fake) — it takes REAL German component tuples
- *   (from OpenAddresses Berlin/Saxony) and renders them in idiomatic German order via the OpenCage
- *   `DE` template (`formatAddress(..., "DE")` → `"Straußstraße 27, 12623 Berlin"`). The corpus
- *   aligner turns the row into BIO labels; every emitted component surface form occurs verbatim in
- *   `raw` so alignment lands.
- */
-
-import { formatAddress } from "@mailwoman/formatter"
-
-import type { CanonicalRow } from "#types"
-
-/**
- * A real address tuple (e.g. one OpenAddresses row): street + locality required, rest optional.
- */
-/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
-   cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
-   output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
-   would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
-   extracted as named constants above. */
-
-export interface LocaleBaseTuple {
-	house_number?: string
-	street: string
-	locality: string
-	/**
-	 * A sub-locality that sits BELOW the locality (a suburb / district). NZ is the case that needs it: the OA DISTRICT
-	 * column holds the city (`Auckland`) and CITY holds the suburb (`Birkenhead`), so the real envelope carries both (`31
-	 * Rawene Road, Birkenhead, Auckland`). Rendered between street and locality in both orders when present.
-	 */
-	dependent_locality?: string
-	region?: string
-	postcode?: string
-}
-
-/**
- * @deprecated Alias — use LocaleBaseTuple.
- */
-export type GermanBaseTuple = LocaleBaseTuple
-
-export interface SynthesizedLocaleRow {
-	raw: string
-	components: CanonicalRow["components"]
-	locale: string
-}
-
-/**
- * @deprecated Alias — use SynthesizedLocaleRow.
- */
-export type SynthesizedGermanRow = SynthesizedLocaleRow
-
-export interface LocaleSynthesisOpts {
-	random?: () => number
-	/**
-	 * Rendering order for the SAME components. `"native"` (default) uses the country's own template (DE →
-	 * house-AFTER-street, postcode-BEFORE-city). `"international"` renders house-FIRST, postcode-AFTER-city — the US/GB
-	 * layout that international feeds, US-centric systems, and our own OpenAddresses de-sample impose on non-US
-	 * addresses. Training both teaches the model that a German address can arrive either way, so the eval's US-order
-	 * rendering stops reading as a collapse. See `docs/articles/evals/resolver-geo/2026-06-06-anchor-pilot.md` (the
-	 * order-artifact correction).
-	 */
-	order?: "native" | "international"
-	/**
-	 * Postcode surface shape. `"conventional"` (default) canonicalizes to the country's rendered form (NL: OA's glued
-	 * `1011AB` → the spaced `1011 AB`); `"as-source"` keeps the source's own surface — the form OA (and the OA-derived
-	 * evals) feed, which for NL is 100% glued. Only NL differs today; every other country passes through identically
-	 * either way. Mixing both teaches the two-letter-suffix `1012 LM` shape AND the glued feed shape (#241 — the model
-	 * currently glues the suffix onto the city).
-	 */
-	postcodeShape?: "conventional" | "as-source"
-	/**
-	 * How the NATIVE-order render joins street and house number. The OpenCage ES template comma-joins (`Calle Mayor, 12`
-	 * — the official Spanish convention); OA-derived feeds and our ES eval space-join (`CALLE MAYOR 12`, the observed
-	 * form on all 3,000 eval rows). `"template"` (default) keeps the template's own join; `"space"` collapses `<street>,
-	 * <house_number>` → `<street> <house_number>` after rendering. Countries whose template already space-joins
-	 * (DE/IT/NL) render identically under both. Mixing both stops an ES slice from teaching the comma as THE street→house
-	 * boundary signal (#241 format-diversity audit). International order ignores this (the US template is already
-	 * house-first space-joined).
-	 */
-	nativeHouseJoin?: "template" | "space"
-	/**
-	 * The string between rendered address lines. `", "` (default) is the template's own join. `" "` renders the
-	 * comma-free single-line register — dictation, a copy out of a one-field form — `Neusser Str. 12 Nippes 50733 Köln`
-	 * for the same components. Stage 2 segments the comma form into three and the comma-free form into ONE, and a single
-	 * segment starves the placetype-pair prior, which is how the comma-free form loses `Nippes` (#1946). Only the native
-	 * order reads it; the international layout keeps its own separator.
-	 */
-	separator?: ", " | " "
-}
-
-/**
- * @deprecated Alias — use LocaleSynthesisOpts.
- */
-export type GermanSynthesisOpts = LocaleSynthesisOpts
-
-/**
- * ISO-3166 alpha-2 → BCP-47 tag for the emitted rows (primary language per country).
- */
-const LOCALE_TAG: Record<string, string> = {
-	DE: "de-DE",
-	ES: "es-ES",
-	IT: "it-IT",
-	NL: "nl-NL",
-	GB: "en-GB",
-	FR: "fr-FR",
-	US: "en-US",
-	NZ: "en-NZ",
-}
-
-/**
- * Canonicalize a postcode to the form the country's template renders, so the stored component aligns verbatim against
- * `raw`. NL is the case that needs it: OA stores `1011AB` but the OpenCage NL template emits the conventional spaced
- * `1011 AB` (4 digits + space + 2 letters), which otherwise fails verbatim alignment and drops the row. Other countries
- * pass through unchanged.
- */
-function normalizePostcode(postcode: string, country: string): string {
-	if (country === "NL") {
-		const m = /^(\d{4})\s*([A-Za-z]{2})$/.exec(postcode)
-
-		if (m) return `${m[1]} ${m[2]!.toUpperCase()}`
-	}
-
-	return postcode
-}
-
-/**
- * True when `value` appears verbatim AND as a standalone token (so BIO alignment lands cleanly).
- */
-function tokenPresent(raw: string, value: string): boolean {
-	if (!raw.includes(value)) return false
-	// Reject substring-of-a-larger-number collisions (e.g. house "2" inside postcode "12623").
-	const i = raw.indexOf(value)
-	const before = raw[i - 1]
-	const after = raw[i + value.length]
-	const isDigit = (c: string | undefined) => c !== undefined && c >= "0" && c <= "9"
-
-	if (/^\d+$/.test(value) && (isDigit(before) || isDigit(after))) return false
-
-	return true
-}
-
-/**
- * Render one real tuple into an idiomatic, locale-ordered `{raw, components}` row via the OpenCage `country` template
- * (DE → house-after-street + postcode-before-city; ES/IT the same; GB house-first; NL carries the `1012 LM` postcode),
- * with light variation (drop house number / postcode some of the time). Returns `null` when the tuple is too thin or a
- * component wouldn't align cleanly.
- *
- * Region handling is order-dependent: NATIVE order omits it (the native template absorbs the admin region into the
- * postcode/city line, so it rarely renders verbatim and would break BIO alignment), while INTERNATIONAL order includes
- * it in the tail ("City, Region Postcode" — the US/feed layout the eval uses; v0.9.3 / #327).
- *
- * Pass `opts.order: "international"` to render the same components house-first / postcode-after-city instead (see
- * {@link LocaleSynthesisOpts.order}) — the layout international feeds impose on foreign addresses, and the one a
- * native-order-trained model treats as a "collapse."
- */
-export function synthesizeLocaleRow(
-	base: LocaleBaseTuple,
-	country: string,
-	opts: LocaleSynthesisOpts = {}
-): SynthesizedLocaleRow | null {
-	const random = opts.random ?? Math.random
-	const order = opts.order ?? "native"
-
-	if (!base.street || !base.locality) return null
-
-	const components: CanonicalRow["components"] = { street: base.street, locality: base.locality }
-
-	// Sub-locality (suburb / district) sits between street and locality and renders in BOTH orders — it's part
-	// of the address body, not the admin-region tail that native order drops. NZ needs it (suburb + city both on
-	// the envelope). The tokenPresent check below drops the row if the template didn't surface it verbatim.
-	if (base.dependent_locality) {
-		components.dependent_locality = base.dependent_locality
-	}
-
-	// ~80% keep the house number (the rest are street-only forms, also idiomatic).
-	if (base.house_number && random() < 0.8) {
-		components.house_number = base.house_number
-	}
-
-	// ~85% keep the postcode (canonicalized to the country's rendered form — NL spaces it). The
-	// `postcodeShape: "as-source"` rewrite happens AFTER the render: the OpenCage NL template
-	// normalizes the postcode itself, so a glued input can't survive rendering directly.
-	if (base.postcode && random() < 0.85) {
-		components.postcode = normalizePostcode(base.postcode, country)
-	}
-
-	// International order carries the REGION in the tail ("City, Region Postcode") — the layout real
-	// US/feed renderings (and our OA eval) use. v0.9.2 rendered international order WITHOUT the region,
-	// so the model never learned to segment the tail and mangled it at eval (region absorbed into the
-	// locality / locality dropped); v0.9.3 closes that gap (#327). Native order still drops the region
-	// (the native template absorbs it into the city line, which would break verbatim alignment).
-	if (order === "international" && base.region) {
-		components.region = base.region
-	}
-
-	// Native order uses the address's own country template; international order uses the US template —
-	// house-first, postcode-after-city, with a region slot for the tail. Neither branch consumes a
-	// `random()` draw for the template, so the RNG sequence existing callers/tests depend on is stable.
-	const renderCountry = order === "international" ? "US" : country
-	const separator = order === "native" ? (opts.separator ?? ", ") : ", "
-	let raw = formatAddress(components, renderCountry, { separator })
-
-	if (!raw) return null
-
-	// Native-order space-join (see {@link LocaleSynthesisOpts.nativeHouseJoin}): collapse the template's
-	// `<street>, <hn>` comma to a space — the OA/feed layout. A no-op for templates that already
-	// space-join (the substring isn't present), and skipped when the house number was dropped above.
-	if (order === "native" && separator === ", " && opts.nativeHouseJoin === "space" && components.house_number) {
-		raw = raw.replace(
-			`${components.street}, ${components.house_number}`,
-			`${components.street} ${components.house_number}`
-		)
-	}
-
-	// `postcodeShape: "as-source"` (see {@link LocaleSynthesisOpts.postcodeShape}): rewrite BOTH raw and
-	// the component back to the source's own surface (NL glued `1011AB`). Post-render because the
-	// OpenCage NL template normalizes the postcode to the spaced form no matter what it's given.
-	if (opts.postcodeShape === "as-source" && components.postcode && base.postcode) {
-		const sourceForm = base.postcode.trim()
-
-		if (sourceForm && sourceForm !== components.postcode && raw.includes(components.postcode)) {
-			raw = raw.replace(components.postcode, sourceForm)
-			components.postcode = sourceForm
-		}
-	}
-
-	// Every component must align — drop the row if the template didn't surface one verbatim, or a
-	// numeric component collides with a neighbouring digit run.
-	for (const value of Object.values(components)) {
-		if (!value || !tokenPresent(raw, value)) return null
-	}
-
-	return { raw, components, locale: LOCALE_TAG[country] ?? country.toLowerCase() }
-}
-
-/**
- * German wrapper over {@link synthesizeLocaleRow}. Kept for the build-german-slice caller + tests.
- */
-export function synthesizeGermanRow(
-	base: LocaleBaseTuple,
-	opts: LocaleSynthesisOpts = {}
-): SynthesizedLocaleRow | null {
-	return synthesizeLocaleRow(base, "DE", opts)
-}

package/lib/tools/fetch/acra-sg.ts

@@ -1,174 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch Singapore's ACRA corporate-entity register from data.gov.sg: 27 monthly CSVs (one per first
- *   letter of the entity name plus "others"), 53 columns, the registered address FIELDED — `Block`,
- *   `Street Name`, `Level No`, `Unit No`, `Building Name`, `Postal Code` — which is the NPPES shape,
- *   so the corpus adapter renders the string and the spans fall out by construction. A Singapore
- *   postcode names one building, so the postcode column is also the join to the OneMap/Overture rows.
- *
- *   License: Singapore Open Data Licence version 1.0 (attribution; commercial use, modification and
- *   adaptation permitted; no share-alike). The prescribed attribution sentence goes in the manifest.
- *
- *   data.gov.sg serves a dataset in two calls: `initiate-download` prepares a signed URL and
- *   `poll-download` answers it once the export is ready.
- *
- *   Invoke via `mailwoman corpus fetch acra-sg --out-root <path>`.
- */
-
-import { BYTES_PER_KIB } from "@mailwoman/core/fs/formatters"
-import { pathExists } from "@mailwoman/core/fs/readers"
-import { makeDirectories } from "@mailwoman/core/fs/writers"
-import { sha256File } from "@mailwoman/core/hash"
-import { sleep } from "@mailwoman/core/utils/sleep"
-import { join } from "path-ts"
-
-import type { BaseFetchOptions, FetchSummary, SourceCollectionManifest, SourceManifest } from "#tools/fetch/download"
-import { loadCollectionFiles, streamBodyToFile, withRetries, writeManifest } from "#tools/fetch/download"
-
-const SLUG = "acra-sg"
-const COLLECTION_URL = "https://api-production.data.gov.sg/v2/public/api/collections/2/metadata"
-const DATASET_API = "https://api-production.data.gov.sg/v2/public/api/datasets"
-const DOWNLOAD_API = "https://api-open.data.gov.sg/v1/public/api/datasets"
-const LICENSE = "Singapore Open Data Licence version 1.0 — https://data.gov.sg/open-data-licence"
-
-const ATTRIBUTION =
-	"Contains information from ACRA Information on Corporate Entities accessed on <date> from data.gov.sg which is made available under the terms of the Singapore Open Data Licence version 1.0 https://data.gov.sg/open-data-licence"
-
-const POLL_INTERVAL_MS = 3000
-const POLL_ATTEMPTS = 40
-
-export type FetchACRASGOptions = BaseFetchOptions
-
-interface CollectionMetadata {
-	data?: { collectionMetadata?: { childDatasets?: string[] } }
-}
-
-interface DatasetMetadata {
-	data?: { name?: string; lastUpdatedAt?: string }
-}
-
-interface PollResponse {
-	data?: { status?: string; url?: string }
-}
-
-async function readJSON<T>(url: string): Promise<T> {
-	const res = await fetch(url, { headers: { accept: "application/json" } })
-
-	if (!res.ok) throw new Error(`acra-sg: ${url} answered HTTP ${res.status}`)
-
-	return (await res.json()) as T
-}
-
-/**
- * Ask the portal to prepare the CSV export, then poll until it names the signed URL.
- */
-async function signedURLFor(datasetID: string): Promise<string | undefined> {
-	await readJSON(`${DOWNLOAD_API}/${datasetID}/initiate-download`)
-
-	for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt++) {
-		await sleep(POLL_INTERVAL_MS)
-		const poll = await readJSON<PollResponse>(`${DOWNLOAD_API}/${datasetID}/poll-download`)
-
-		if (poll.data?.status === "DOWNLOAD_SUCCESS" && poll.data.url) return poll.data.url
-	}
-
-	return undefined
-}
-
-function filenameFor(name: string, datasetID: string): string {
-	const letter = /\('([^']+)'\)/.exec(name)?.[1]
-
-	return `acra-entities-${(letter ?? datasetID).toLowerCase().replaceAll(/[^a-z0-9]+/g, "-")}.csv`
-}
-
-export async function fetchACRASG(options: FetchACRASGOptions, report?: (line: string) => void): Promise<FetchSummary> {
-	const destDir = join(options.outRoot, SLUG)
-	await makeDirectories(destDir)
-	const manifestPath = join(destDir, "MANIFEST.json")
-
-	const collection = await readJSON<CollectionMetadata>(COLLECTION_URL)
-	const datasetIDs = collection.data?.collectionMetadata?.childDatasets ?? []
-	report?.(`=== ${SLUG}: ${datasetIDs.length} datasets in collection 2`)
-
-	const previous = await loadCollectionFiles(manifestPath)
-	const files = new Map<string, SourceManifest>()
-	let fetched = 0
-	let skipped = 0
-	const failedCodes: string[] = []
-
-	for (const datasetID of datasetIDs) {
-		const metadata = await readJSON<DatasetMetadata>(`${DATASET_API}/${datasetID}/metadata`)
-		const name = metadata.data?.name ?? datasetID
-		const filename = filenameFor(name, datasetID)
-		const dest = join(destDir, filename)
-		const before = previous.get(filename)
-
-		if (before && (await pathExists(dest))) {
-			files.set(filename, before)
-
-			skipped++
-
-			continue
-		}
-
-		report?.(`--- ${name} (${datasetID})`)
-		let bytes: number
-
-		try {
-			bytes = await withRetries(
-				async () => {
-					const url = await signedURLFor(datasetID)
-
-					if (!url) throw new Error(`the export never reached DOWNLOAD_SUCCESS for ${datasetID}`)
-					const res = await fetch(url, { signal: AbortSignal.timeout(1_800_000) })
-
-					if (!res.ok) throw new Error(`HTTP ${res.status} on the signed URL for ${datasetID}`)
-
-					return streamBodyToFile(res, dest)
-				},
-				{ report, label: name }
-			)
-		} catch (error) {
-			report?.(`  ✗ ${(error as Error).message}`)
-			failedCodes.push(datasetID)
-
-			continue
-		}
-
-		if (bytes < BYTES_PER_KIB) {
-			report?.(`  ✗ ${bytes} bytes — an error page, not the register`)
-			failedCodes.push(datasetID)
-
-			continue
-		}
-
-		const sha = await sha256File(dest)
-
-		files.set(filename, {
-			source_url: `https://data.gov.sg/datasets/${datasetID}/view`,
-			downloaded_at: new Date().toISOString(),
-			filename,
-			sha256: sha,
-			bytes,
-		})
-
-		fetched++
-		report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`)
-
-		const manifest: SourceCollectionManifest = {
-			source: SLUG,
-			source_url: "https://data.gov.sg/collections/2/view",
-			license: LICENSE,
-			attribution: ATTRIBUTION,
-			downloaded_at: new Date().toISOString(),
-			files: [...files.values()],
-		}
-
-		await writeManifest(manifestPath, manifest)
-	}
-
-	return { fetched, skipped, failed: failedCodes.length, failedCodes }
-}