@mailwoman/corpus 9.4.0 → 10.0.0

package/lib/synthesizers/anchor-absorption.ts

@@ -30,7 +30,7 @@
  *   ZIPs (anchor-fp) are 5-digit strings deliberately absent from the lookup.
  */
 
-import type { ComponentTag } from "@mailwoman/core/types"
+import type { ComponentTag } from "@mailwoman/codex/component"
 
 import { pick } from "#synthesizers/utils"
 

package/lib/synthesizers/locale.ts

@@ -0,0 +1,261 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Locale address synthesizer — first introduced for German coverage (night-shift 2026-06-02,
+ *   DE-1), now shared by country-oriented recipes.
+ *
+ *   The neural model is out-of-distribution on German: it truncates `Straußstraße`→`Strau` (exits at
+ *   the ß-piece boundary), absorbs the house number into the street (`Hauptstraße 5` → one span),
+ *   and mis-tags the native-order house number as a postcode (`Prenzlauer Allee 36, 10405 Berlin` →
+ *   postcode `36`). The cause is ORDER: the model was trained US+FR (house-number-FIRST,
+ *   postcode-AFTER-city), and never saw the German convention (house-number-AFTER-street,
+ *   postcode-BEFORE-city). DE-0 confirmed the tokenizer round-trips German orthography cleanly, so
+ *   this is a coverage gap, not a tokenizer ceiling.
+ *
+ *   The original German generator produced the missing signal as a small targeted supplement slice
+ *   (synthesis-as-supplement discipline: weight < 0.25, one-and-done). It does NOT synthesize
+ *   German street names (German morphology is hard to fake) — it takes REAL German component tuples
+ *   (from OpenAddresses Berlin/Saxony) and renders them in idiomatic German order via the OpenCage
+ *   `DE` template (`formatAddress(..., "DE")` → `"Straußstraße 27, 12623 Berlin"`). The corpus
+ *   aligner turns the row into BIO labels; every emitted component surface form occurs verbatim in
+ *   `raw` so alignment lands. Its locale-neutral API now also serves the international recipe.
+ */
+
+import { formatAddress } from "@mailwoman/codex/address-format"
+
+import type { CanonicalRow } from "#types"
+
+/**
+ * A real address tuple (e.g. one OpenAddresses row): street + locality required, rest optional.
+ */
+/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
+   cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
+   output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
+   would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
+   extracted as named constants above. */
+
+export interface LocaleBaseTuple {
+	house_number?: string
+	street: string
+	locality: string
+	/**
+	 * A sub-locality that sits BELOW the locality (a suburb / district). NZ is the case that needs it: the OA DISTRICT
+	 * column holds the city (`Auckland`) and CITY holds the suburb (`Birkenhead`), so the real envelope carries both (`31
+	 * Rawene Road, Birkenhead, Auckland`). Rendered between street and locality in both orders when present.
+	 */
+	dependent_locality?: string
+	region?: string
+	postcode?: string
+}
+
+/**
+ * @deprecated Alias — use LocaleBaseTuple.
+ */
+export type GermanBaseTuple = LocaleBaseTuple
+
+export interface SynthesizedLocaleRow {
+	raw: string
+	components: CanonicalRow["components"]
+	locale: string
+}
+
+/**
+ * @deprecated Alias — use SynthesizedLocaleRow.
+ */
+export type SynthesizedGermanRow = SynthesizedLocaleRow
+
+export interface LocaleSynthesisOpts {
+	random?: () => number
+	/**
+	 * Rendering order for the SAME components. `"native"` (default) uses the country's own template (DE →
+	 * house-AFTER-street, postcode-BEFORE-city). `"international"` renders house-FIRST, postcode-AFTER-city — the US/GB
+	 * layout that international feeds, US-centric systems, and our own OpenAddresses de-sample impose on non-US
+	 * addresses. Training both teaches the model that a German address can arrive either way, so the eval's US-order
+	 * rendering stops reading as a collapse. See `docs/articles/evals/resolver-geo/2026-06-06-anchor-pilot.md` (the
+	 * order-artifact correction).
+	 */
+	order?: "native" | "international"
+	/**
+	 * Postcode surface shape. `"conventional"` (default) canonicalizes to the country's rendered form (NL: OA's glued
+	 * `1011AB` → the spaced `1011 AB`); `"as-source"` keeps the source's own surface — the form OA (and the OA-derived
+	 * evals) feed, which for NL is 100% glued. Only NL differs today; every other country passes through identically
+	 * either way. Mixing both teaches the two-letter-suffix `1012 LM` shape AND the glued feed shape (#241 — the model
+	 * currently glues the suffix onto the city).
+	 */
+	postcodeShape?: "conventional" | "as-source"
+	/**
+	 * How the NATIVE-order render joins street and house number. The OpenCage ES template comma-joins (`Calle Mayor, 12`
+	 * — the official Spanish convention); OA-derived feeds and our ES eval space-join (`CALLE MAYOR 12`, the observed
+	 * form on all 3,000 eval rows). `"template"` (default) keeps the template's own join; `"space"` collapses `<street>,
+	 * <house_number>` → `<street> <house_number>` after rendering. Countries whose template already space-joins
+	 * (DE/IT/NL) render identically under both. Mixing both stops an ES slice from teaching the comma as THE street→house
+	 * boundary signal (#241 format-diversity audit). International order ignores this (the US template is already
+	 * house-first space-joined).
+	 */
+	nativeHouseJoin?: "template" | "space"
+	/**
+	 * The string between rendered address lines. `", "` (default) is the template's own join. `" "` renders the
+	 * comma-free single-line register — dictation, a copy out of a one-field form — `Neusser Str. 12 Nippes 50733 Köln`
+	 * for the same components. Stage 2 segments the comma form into three and the comma-free form into ONE, and a single
+	 * segment starves the placetype-pair prior, which is how the comma-free form loses `Nippes` (#1946). Only the native
+	 * order reads it; the international layout keeps its own separator.
+	 */
+	separator?: ", " | " "
+}
+
+/**
+ * @deprecated Alias — use LocaleSynthesisOpts.
+ */
+export type GermanSynthesisOpts = LocaleSynthesisOpts
+
+/**
+ * ISO-3166 alpha-2 → BCP-47 tag for the emitted rows (primary language per country).
+ */
+const LOCALE_TAG: Record<string, string> = {
+	DE: "de-DE",
+	ES: "es-ES",
+	IT: "it-IT",
+	NL: "nl-NL",
+	GB: "en-GB",
+	FR: "fr-FR",
+	US: "en-US",
+	NZ: "en-NZ",
+}
+
+/**
+ * Canonicalize a postcode to the form the country's template renders, so the stored component aligns verbatim against
+ * `raw`. NL is the case that needs it: OA stores `1011AB` but the OpenCage NL template emits the conventional spaced
+ * `1011 AB` (4 digits + space + 2 letters), which otherwise fails verbatim alignment and drops the row. Other countries
+ * pass through unchanged.
+ */
+function normalizePostcode(postcode: string, country: string): string {
+	if (country === "NL") {
+		const m = /^(\d{4})\s*([A-Za-z]{2})$/.exec(postcode)
+
+		if (m) return `${m[1]} ${m[2]!.toUpperCase()}`
+	}
+
+	return postcode
+}
+
+/**
+ * True when `value` appears verbatim AND as a standalone token (so BIO alignment lands cleanly).
+ */
+function tokenPresent(raw: string, value: string): boolean {
+	if (!raw.includes(value)) return false
+	// Reject substring-of-a-larger-number collisions (e.g. house "2" inside postcode "12623").
+	const i = raw.indexOf(value)
+	const before = raw[i - 1]
+	const after = raw[i + value.length]
+	const isDigit = (c: string | undefined) => c !== undefined && c >= "0" && c <= "9"
+
+	if (/^\d+$/.test(value) && (isDigit(before) || isDigit(after))) return false
+
+	return true
+}
+
+/**
+ * Render one real tuple into an idiomatic, locale-ordered `{raw, components}` row via the OpenCage `country` template
+ * (DE → house-after-street + postcode-before-city; ES/IT the same; GB house-first; NL carries the `1012 LM` postcode),
+ * with light variation (drop house number / postcode some of the time). Returns `null` when the tuple is too thin or a
+ * component wouldn't align cleanly.
+ *
+ * Region handling is order-dependent: NATIVE order omits it (the native template absorbs the admin region into the
+ * postcode/city line, so it rarely renders verbatim and would break BIO alignment), while INTERNATIONAL order includes
+ * it in the tail ("City, Region Postcode" — the US/feed layout the eval uses; v0.9.3 / #327).
+ *
+ * Pass `opts.order: "international"` to render the same components house-first / postcode-after-city instead (see
+ * {@link LocaleSynthesisOpts.order}) — the layout international feeds impose on foreign addresses, and the one a
+ * native-order-trained model treats as a "collapse."
+ */
+export function synthesizeLocaleRow(
+	base: LocaleBaseTuple,
+	country: string,
+	opts: LocaleSynthesisOpts = {}
+): SynthesizedLocaleRow | null {
+	const random = opts.random ?? Math.random
+	const order = opts.order ?? "native"
+
+	if (!base.street || !base.locality) return null
+
+	const components: CanonicalRow["components"] = { street: base.street, locality: base.locality }
+
+	// Sub-locality (suburb / district) sits between street and locality and renders in BOTH orders — it's part
+	// of the address body, not the admin-region tail that native order drops. NZ needs it (suburb + city both on
+	// the envelope). The tokenPresent check below drops the row if the template didn't surface it verbatim.
+	if (base.dependent_locality) {
+		components.dependent_locality = base.dependent_locality
+	}
+
+	// ~80% keep the house number (the rest are street-only forms, also idiomatic).
+	if (base.house_number && random() < 0.8) {
+		components.house_number = base.house_number
+	}
+
+	// ~85% keep the postcode (canonicalized to the country's rendered form — NL spaces it). The
+	// `postcodeShape: "as-source"` rewrite happens AFTER the render: the OpenCage NL template
+	// normalizes the postcode itself, so a glued input can't survive rendering directly.
+	if (base.postcode && random() < 0.85) {
+		components.postcode = normalizePostcode(base.postcode, country)
+	}
+
+	// International order carries the REGION in the tail ("City, Region Postcode") — the layout real
+	// US/feed renderings (and our OA eval) use. v0.9.2 rendered international order WITHOUT the region,
+	// so the model never learned to segment the tail and mangled it at eval (region absorbed into the
+	// locality / locality dropped); v0.9.3 closes that gap (#327). Native order still drops the region
+	// (the native template absorbs it into the city line, which would break verbatim alignment).
+	if (order === "international" && base.region) {
+		components.region = base.region
+	}
+
+	// Native order uses the address's own country template; international order uses the US template —
+	// house-first, postcode-after-city, with a region slot for the tail. Neither branch consumes a
+	// `random()` draw for the template, so the RNG sequence existing callers/tests depend on is stable.
+	const renderCountry = order === "international" ? "US" : country
+	const separator = order === "native" ? (opts.separator ?? ", ") : ", "
+	let raw = formatAddress(components, renderCountry, { separator })
+
+	if (!raw) return null
+
+	// Native-order space-join (see {@link LocaleSynthesisOpts.nativeHouseJoin}): collapse the template's
+	// `<street>, <hn>` comma to a space — the OA/feed layout. A no-op for templates that already
+	// space-join (the substring isn't present), and skipped when the house number was dropped above.
+	if (order === "native" && separator === ", " && opts.nativeHouseJoin === "space" && components.house_number) {
+		raw = raw.replace(
+			`${components.street}, ${components.house_number}`,
+			`${components.street} ${components.house_number}`
+		)
+	}
+
+	// `postcodeShape: "as-source"` (see {@link LocaleSynthesisOpts.postcodeShape}): rewrite BOTH raw and
+	// the component back to the source's own surface (NL glued `1011AB`). Post-render because the
+	// OpenCage NL template normalizes the postcode to the spaced form no matter what it's given.
+	if (opts.postcodeShape === "as-source" && components.postcode && base.postcode) {
+		const sourceForm = base.postcode.trim()
+
+		if (sourceForm && sourceForm !== components.postcode && raw.includes(components.postcode)) {
+			raw = raw.replace(components.postcode, sourceForm)
+			components.postcode = sourceForm
+		}
+	}
+
+	// Every component must align — drop the row if the template didn't surface one verbatim, or a
+	// numeric component collides with a neighbouring digit run.
+	for (const value of Object.values(components)) {
+		if (!value || !tokenPresent(raw, value)) return null
+	}
+
+	return { raw, components, locale: LOCALE_TAG[country] ?? country.toLowerCase() }
+}
+
+/**
+ * German wrapper over {@link synthesizeLocaleRow}. Kept for the build-german-slice caller + tests.
+ */
+export function synthesizeGermanRow(
+	base: LocaleBaseTuple,
+	opts: LocaleSynthesisOpts = {}
+): SynthesizedLocaleRow | null {
+	return synthesizeLocaleRow(base, "DE", opts)
+}

package/lib/synthesizers/street.ts

@@ -21,9 +21,9 @@
 
 import { isPresent } from "@mailwoman/core/objects"
 
-import { decomposeStreet } from "#adapters/tiger/street-decompose"
 import { pick, tieredNumber } from "#synthesizers/utils"
 import type { CanonicalRow } from "#types"
+import { decomposeStreet } from "#us/adapters/tiger/street-decompose"
 
 // Hand-curated US street name pool. Real frequency-weighted street names — sampled
 // from US Census TIGER 2024 top-1000 by occurrence count. Keep ~50 entries so the

package/lib/synthesizers/utils.ts

@@ -26,6 +26,7 @@
  *   effect is measured; see the note there.
  */
 
+import type { BIOLabel, ComponentTag } from "@mailwoman/codex/component"
 import {
 	US_STREET_SUFFIX_PREFERRED_ABBR,
 	US_UNIT_DESIGNATOR_PREFERRED_ABBR,
@@ -36,7 +37,6 @@ import {
 import { isPresent } from "@mailwoman/core/objects"
 import { mulberry32 } from "@mailwoman/core/random"
 import { escapeRegExp } from "@mailwoman/core/strings/regexp"
-import type { BIOLabel, ComponentTag } from "@mailwoman/core/types"
 import { stripCombiningMarks } from "@mailwoman/normalize/fold"
 
 import type { CanonicalRow, LabeledRow, QuarantinedRow } from "#types"
@@ -158,7 +158,7 @@ export const accentStrip: Augmentation = (row) => {
 	return withAugmentation(row, "accent-strip", stripped, newComponents)
 }
 
-// --- typo injection (#530) -------------------------------------------------
+// The typo-injection helpers below keep generated rows reproducible.
 // The Phase-1 deferral asked for a "seed-aware API" so the corpus stays reproducible. Resolution:
 // seed the PRNG from the row's own `source_id` — deterministic per row, no global state, fits the
 // existing `(row) => CanonicalRow | null` signature unchanged.

package/lib/tools/audit.ts

@@ -15,9 +15,10 @@
  *   Emits warnings to stderr and the audit table to stdout; never throws on an empty corpus.
  */
 
-import { pathExists, readDirectory, readLocalBuffer, readLocalJSONFile } from "@mailwoman/core/fs/readers"
+import { pathExists, readLocalBuffer, readLocalJSONFile } from "@mailwoman/core/fs/readers"
 import { basename, join, type PathBuilderLike } from "path-ts"
 import { TextSpliterator } from "spliterator"
+import { Globerator } from "spliterator/node/fs"
 
 /**
  * Share of a slice one source may hold before the mix is flagged as dominated by it.
@@ -116,7 +117,7 @@ async function scanSlices(corpusDir: PathBuilderLike, sampleCount: number): Prom
 
 		if (!(await pathExists(splitDir))) continue
 
-		const files = (await readDirectory(splitDir)).filter((f) => f.endsWith(".parquet")).toSorted()
+		const files = await Globerator.files("parquet", { cwd: splitDir, absolute: false }).toSorted()
 
 		stats.totalFiles += files.length
 		const sampleEvery = Math.max(1, Math.floor(files.length / sampleCount))

package/lib/tools/corpus-stats.ts

@@ -37,9 +37,9 @@
  */
 
 import { ByteFormatter } from "@mailwoman/core/fs/formatters"
-import { readDirectory, statPath } from "@mailwoman/core/fs/readers"
+import { statPath } from "@mailwoman/core/fs/readers"
 import { writeLocalJSONFile } from "@mailwoman/core/fs/writers"
-import { join } from "path-ts"
+import { Globerator } from "spliterator/node/fs"
 
 import { accumulateCooccurrences, createCooccurrenceStats, streamTokenLabelRows } from "#utils/slice-stats"
 
@@ -55,7 +55,7 @@ async function discoverSlices(slicesArg: string): Promise<string[]> {
 	const stat = await statPath(slicesArg)
 
 	if (stat.isDirectory()) {
-		return (await readDirectory(slicesArg)).filter((f) => f.endsWith(".parquet")).map((f) => join(slicesArg, f))
+		return await Globerator.files("parquet", { cwd: slicesArg }).toArray()
 	}
 
 	if (stat.isFile() && slicesArg.endsWith(".parquet")) return [slicesArg]

package/lib/tools/fetch/download/index.ts

@@ -0,0 +1,10 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Public entry point for corpus download utilities.
+ */
+
+export * from "#tools/fetch/download/manifest"
+export * from "#tools/fetch/download/network"
+export * from "#tools/fetch/download/retry"

package/lib/tools/fetch/download/manifest.ts

@@ -0,0 +1,102 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Reads and writes corpus-source manifests.
+ */
+
+import { pathExists, readLocalTextFile } from "@mailwoman/core/fs/readers"
+import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams"
+import { movePath, writeLocalTextFile } from "@mailwoman/core/fs/writers"
+import { tryParsingJSON } from "@mailwoman/core/json"
+
+import { HTTPStatusError, type SourceManifest } from "#tools/fetch/download/network"
+
+/**
+ * Read a MANIFEST.json; `null` when missing or corrupt (callers re-fetch from scratch).
+ */
+export async function readManifest<T>(path: string): Promise<T | null> {
+	if (!(await pathExists(path))) return null
+
+	// A read failure (e.g. the file vanished after the existsSync probe) maps to null like corrupt
+	// JSON does; tryParsingJSON returns null for the non-string sentinel.
+	const text = await readLocalTextFile(path).catch(() => null)
+
+	return tryParsingJSON<T>(text)
+}
+
+/**
+ * Load manifest entries into a map so untouched keys survive a partial re-fetch.
+ */
+export async function loadManifestEntries<T>(path: string, key: (entry: T) => string): Promise<Map<string, T>> {
+	const entries = new Map<string, T>()
+	const parsed = await readManifest<T[]>(path)
+
+	for (const entry of parsed ?? []) {
+		entries.set(key(entry), entry)
+	}
+
+	return entries
+}
+
+/**
+ * Write a MANIFEST.json in the house shape: pretty-printed, trailing newline.
+ */
+export async function writeManifest(path: string, manifest: unknown): Promise<void> {
+	await writeLocalTextFile(JSON.stringify(manifest, null, 2) + "\n", path)
+}
+
+/**
+ * The sibling `MANIFEST.json` shape for a source that is a COLLECTION of files behind one portal (a monthly register
+ * published per region, per industry, or per first letter). It carries what a trained artifact has to be able to cite
+ * later: the license the portal labels the data with, the attribution wording it requires, and one
+ * {@link SourceManifest} per file.
+ */
+export interface SourceCollectionManifest {
+	source: string
+	source_url: string
+	license: string
+	attribution: string
+	downloaded_at: string
+	files: SourceManifest[]
+}
+
+/**
+ * Pipe a response body to `dest` and answer the byte count. The one primitive the portal fetchers share when the
+ * request is not a bare GET — a session cookie, a CSRF header, or a form POST stands between the listing and the file,
+ * so {@link streamDownload}'s URL-only contract does not fit and each module builds its own `Response` first. Writes a
+ * `.tmp` sibling and renames, so an interrupted transfer never lands at the final path looking complete.
+ */
+export async function streamBodyToFile(res: Response, dest: string): Promise<number> {
+	if (!res.body) throw new HTTPStatusError(res.status, `HTTP ${res.status} with no body — ${res.url}`)
+	let bytes = 0
+
+	const counter = new TransformStream<Uint8Array, Uint8Array>({
+		transform(chunk, controller) {
+			bytes += chunk.byteLength
+			controller.enqueue(chunk)
+		},
+	})
+
+	const tmp = dest + ".tmp"
+	await pipeline(Readable.fromWeb(res.body.pipeThrough(counter)), openWriteStream(tmp))
+	await movePath(tmp, dest)
+
+	return bytes
+}
+
+/**
+ * The per-file entries of a {@link SourceCollectionManifest} keyed by file name, or an empty map when the manifest is
+ * missing or not in the collection shape, so a re-run after an interruption fetches only what is missing. The
+ * single-file modules' `loadManifestEntries` reads a bare array and is not this.
+ */
+export async function loadCollectionFiles(path: string): Promise<Map<string, SourceManifest>> {
+	const parsed = await readManifest<SourceCollectionManifest>(path)
+	const entries = new Map<string, SourceManifest>()
+
+	for (const entry of Array.isArray(parsed?.files) ? parsed.files : []) {
+		entries.set(entry.filename, entry)
+	}
+
+	return entries
+}