@mailwoman/corpus 10.0.0 → 10.1.0

package/lib/synthesizers/locale.ts

@@ -1,261 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Locale address synthesizer — first introduced for German coverage (night-shift 2026-06-02,
- *   DE-1), now shared by country-oriented recipes.
- *
- *   The neural model is out-of-distribution on German: it truncates `Straußstraße`→`Strau` (exits at
- *   the ß-piece boundary), absorbs the house number into the street (`Hauptstraße 5` → one span),
- *   and mis-tags the native-order house number as a postcode (`Prenzlauer Allee 36, 10405 Berlin` →
- *   postcode `36`). The cause is ORDER: the model was trained US+FR (house-number-FIRST,
- *   postcode-AFTER-city), and never saw the German convention (house-number-AFTER-street,
- *   postcode-BEFORE-city). DE-0 confirmed the tokenizer round-trips German orthography cleanly, so
- *   this is a coverage gap, not a tokenizer ceiling.
- *
- *   The original German generator produced the missing signal as a small targeted supplement slice
- *   (synthesis-as-supplement discipline: weight < 0.25, one-and-done). It does NOT synthesize
- *   German street names (German morphology is hard to fake) — it takes REAL German component tuples
- *   (from OpenAddresses Berlin/Saxony) and renders them in idiomatic German order via the OpenCage
- *   `DE` template (`formatAddress(..., "DE")` → `"Straußstraße 27, 12623 Berlin"`). The corpus
- *   aligner turns the row into BIO labels; every emitted component surface form occurs verbatim in
- *   `raw` so alignment lands. Its locale-neutral API now also serves the international recipe.
- */
-
-import { formatAddress } from "@mailwoman/codex/address-format"
-
-import type { CanonicalRow } from "#types"
-
-/**
- * A real address tuple (e.g. one OpenAddresses row): street + locality required, rest optional.
- */
-/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
-   cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
-   output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
-   would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
-   extracted as named constants above. */
-
-export interface LocaleBaseTuple {
-	house_number?: string
-	street: string
-	locality: string
-	/**
-	 * A sub-locality that sits BELOW the locality (a suburb / district). NZ is the case that needs it: the OA DISTRICT
-	 * column holds the city (`Auckland`) and CITY holds the suburb (`Birkenhead`), so the real envelope carries both (`31
-	 * Rawene Road, Birkenhead, Auckland`). Rendered between street and locality in both orders when present.
-	 */
-	dependent_locality?: string
-	region?: string
-	postcode?: string
-}
-
-/**
- * @deprecated Alias — use LocaleBaseTuple.
- */
-export type GermanBaseTuple = LocaleBaseTuple
-
-export interface SynthesizedLocaleRow {
-	raw: string
-	components: CanonicalRow["components"]
-	locale: string
-}
-
-/**
- * @deprecated Alias — use SynthesizedLocaleRow.
- */
-export type SynthesizedGermanRow = SynthesizedLocaleRow
-
-export interface LocaleSynthesisOpts {
-	random?: () => number
-	/**
-	 * Rendering order for the SAME components. `"native"` (default) uses the country's own template (DE →
-	 * house-AFTER-street, postcode-BEFORE-city). `"international"` renders house-FIRST, postcode-AFTER-city — the US/GB
-	 * layout that international feeds, US-centric systems, and our own OpenAddresses de-sample impose on non-US
-	 * addresses. Training both teaches the model that a German address can arrive either way, so the eval's US-order
-	 * rendering stops reading as a collapse. See `docs/articles/evals/resolver-geo/2026-06-06-anchor-pilot.md` (the
-	 * order-artifact correction).
-	 */
-	order?: "native" | "international"
-	/**
-	 * Postcode surface shape. `"conventional"` (default) canonicalizes to the country's rendered form (NL: OA's glued
-	 * `1011AB` → the spaced `1011 AB`); `"as-source"` keeps the source's own surface — the form OA (and the OA-derived
-	 * evals) feed, which for NL is 100% glued. Only NL differs today; every other country passes through identically
-	 * either way. Mixing both teaches the two-letter-suffix `1012 LM` shape AND the glued feed shape (#241 — the model
-	 * currently glues the suffix onto the city).
-	 */
-	postcodeShape?: "conventional" | "as-source"
-	/**
-	 * How the NATIVE-order render joins street and house number. The OpenCage ES template comma-joins (`Calle Mayor, 12`
-	 * — the official Spanish convention); OA-derived feeds and our ES eval space-join (`CALLE MAYOR 12`, the observed
-	 * form on all 3,000 eval rows). `"template"` (default) keeps the template's own join; `"space"` collapses `<street>,
-	 * <house_number>` → `<street> <house_number>` after rendering. Countries whose template already space-joins
-	 * (DE/IT/NL) render identically under both. Mixing both stops an ES slice from teaching the comma as THE street→house
-	 * boundary signal (#241 format-diversity audit). International order ignores this (the US template is already
-	 * house-first space-joined).
-	 */
-	nativeHouseJoin?: "template" | "space"
-	/**
-	 * The string between rendered address lines. `", "` (default) is the template's own join. `" "` renders the
-	 * comma-free single-line register — dictation, a copy out of a one-field form — `Neusser Str. 12 Nippes 50733 Köln`
-	 * for the same components. Stage 2 segments the comma form into three and the comma-free form into ONE, and a single
-	 * segment starves the placetype-pair prior, which is how the comma-free form loses `Nippes` (#1946). Only the native
-	 * order reads it; the international layout keeps its own separator.
-	 */
-	separator?: ", " | " "
-}
-
-/**
- * @deprecated Alias — use LocaleSynthesisOpts.
- */
-export type GermanSynthesisOpts = LocaleSynthesisOpts
-
-/**
- * ISO-3166 alpha-2 → BCP-47 tag for the emitted rows (primary language per country).
- */
-const LOCALE_TAG: Record<string, string> = {
-	DE: "de-DE",
-	ES: "es-ES",
-	IT: "it-IT",
-	NL: "nl-NL",
-	GB: "en-GB",
-	FR: "fr-FR",
-	US: "en-US",
-	NZ: "en-NZ",
-}
-
-/**
- * Canonicalize a postcode to the form the country's template renders, so the stored component aligns verbatim against
- * `raw`. NL is the case that needs it: OA stores `1011AB` but the OpenCage NL template emits the conventional spaced
- * `1011 AB` (4 digits + space + 2 letters), which otherwise fails verbatim alignment and drops the row. Other countries
- * pass through unchanged.
- */
-function normalizePostcode(postcode: string, country: string): string {
-	if (country === "NL") {
-		const m = /^(\d{4})\s*([A-Za-z]{2})$/.exec(postcode)
-
-		if (m) return `${m[1]} ${m[2]!.toUpperCase()}`
-	}
-
-	return postcode
-}
-
-/**
- * True when `value` appears verbatim AND as a standalone token (so BIO alignment lands cleanly).
- */
-function tokenPresent(raw: string, value: string): boolean {
-	if (!raw.includes(value)) return false
-	// Reject substring-of-a-larger-number collisions (e.g. house "2" inside postcode "12623").
-	const i = raw.indexOf(value)
-	const before = raw[i - 1]
-	const after = raw[i + value.length]
-	const isDigit = (c: string | undefined) => c !== undefined && c >= "0" && c <= "9"
-
-	if (/^\d+$/.test(value) && (isDigit(before) || isDigit(after))) return false
-
-	return true
-}
-
-/**
- * Render one real tuple into an idiomatic, locale-ordered `{raw, components}` row via the OpenCage `country` template
- * (DE → house-after-street + postcode-before-city; ES/IT the same; GB house-first; NL carries the `1012 LM` postcode),
- * with light variation (drop house number / postcode some of the time). Returns `null` when the tuple is too thin or a
- * component wouldn't align cleanly.
- *
- * Region handling is order-dependent: NATIVE order omits it (the native template absorbs the admin region into the
- * postcode/city line, so it rarely renders verbatim and would break BIO alignment), while INTERNATIONAL order includes
- * it in the tail ("City, Region Postcode" — the US/feed layout the eval uses; v0.9.3 / #327).
- *
- * Pass `opts.order: "international"` to render the same components house-first / postcode-after-city instead (see
- * {@link LocaleSynthesisOpts.order}) — the layout international feeds impose on foreign addresses, and the one a
- * native-order-trained model treats as a "collapse."
- */
-export function synthesizeLocaleRow(
-	base: LocaleBaseTuple,
-	country: string,
-	opts: LocaleSynthesisOpts = {}
-): SynthesizedLocaleRow | null {
-	const random = opts.random ?? Math.random
-	const order = opts.order ?? "native"
-
-	if (!base.street || !base.locality) return null
-
-	const components: CanonicalRow["components"] = { street: base.street, locality: base.locality }
-
-	// Sub-locality (suburb / district) sits between street and locality and renders in BOTH orders — it's part
-	// of the address body, not the admin-region tail that native order drops. NZ needs it (suburb + city both on
-	// the envelope). The tokenPresent check below drops the row if the template didn't surface it verbatim.
-	if (base.dependent_locality) {
-		components.dependent_locality = base.dependent_locality
-	}
-
-	// ~80% keep the house number (the rest are street-only forms, also idiomatic).
-	if (base.house_number && random() < 0.8) {
-		components.house_number = base.house_number
-	}
-
-	// ~85% keep the postcode (canonicalized to the country's rendered form — NL spaces it). The
-	// `postcodeShape: "as-source"` rewrite happens AFTER the render: the OpenCage NL template
-	// normalizes the postcode itself, so a glued input can't survive rendering directly.
-	if (base.postcode && random() < 0.85) {
-		components.postcode = normalizePostcode(base.postcode, country)
-	}
-
-	// International order carries the REGION in the tail ("City, Region Postcode") — the layout real
-	// US/feed renderings (and our OA eval) use. v0.9.2 rendered international order WITHOUT the region,
-	// so the model never learned to segment the tail and mangled it at eval (region absorbed into the
-	// locality / locality dropped); v0.9.3 closes that gap (#327). Native order still drops the region
-	// (the native template absorbs it into the city line, which would break verbatim alignment).
-	if (order === "international" && base.region) {
-		components.region = base.region
-	}
-
-	// Native order uses the address's own country template; international order uses the US template —
-	// house-first, postcode-after-city, with a region slot for the tail. Neither branch consumes a
-	// `random()` draw for the template, so the RNG sequence existing callers/tests depend on is stable.
-	const renderCountry = order === "international" ? "US" : country
-	const separator = order === "native" ? (opts.separator ?? ", ") : ", "
-	let raw = formatAddress(components, renderCountry, { separator })
-
-	if (!raw) return null
-
-	// Native-order space-join (see {@link LocaleSynthesisOpts.nativeHouseJoin}): collapse the template's
-	// `<street>, <hn>` comma to a space — the OA/feed layout. A no-op for templates that already
-	// space-join (the substring isn't present), and skipped when the house number was dropped above.
-	if (order === "native" && separator === ", " && opts.nativeHouseJoin === "space" && components.house_number) {
-		raw = raw.replace(
-			`${components.street}, ${components.house_number}`,
-			`${components.street} ${components.house_number}`
-		)
-	}
-
-	// `postcodeShape: "as-source"` (see {@link LocaleSynthesisOpts.postcodeShape}): rewrite BOTH raw and
-	// the component back to the source's own surface (NL glued `1011AB`). Post-render because the
-	// OpenCage NL template normalizes the postcode to the spaced form no matter what it's given.
-	if (opts.postcodeShape === "as-source" && components.postcode && base.postcode) {
-		const sourceForm = base.postcode.trim()
-
-		if (sourceForm && sourceForm !== components.postcode && raw.includes(components.postcode)) {
-			raw = raw.replace(components.postcode, sourceForm)
-			components.postcode = sourceForm
-		}
-	}
-
-	// Every component must align — drop the row if the template didn't surface one verbatim, or a
-	// numeric component collides with a neighbouring digit run.
-	for (const value of Object.values(components)) {
-		if (!value || !tokenPresent(raw, value)) return null
-	}
-
-	return { raw, components, locale: LOCALE_TAG[country] ?? country.toLowerCase() }
-}
-
-/**
- * German wrapper over {@link synthesizeLocaleRow}. Kept for the build-german-slice caller + tests.
- */
-export function synthesizeGermanRow(
-	base: LocaleBaseTuple,
-	opts: LocaleSynthesisOpts = {}
-): SynthesizedLocaleRow | null {
-	return synthesizeLocaleRow(base, "DE", opts)
-}

package/lib/test-kit/index.ts

@@ -1,100 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   The corpus adapter-test harness — the sibling of `mailwoman/test-kit/`, and the shape every
- *   `corpus/src/adapters/*\/adapter.test.ts` had grown for itself.
- *
- *   Nine of them opened with the same twenty lines: the `mkdtemp`/`rm`/`tmpdir`/`join` +
- *   `JSONSpliterator` import header, a `let scratch`, a `beforeEach` that mkdtemps
- *   `mailwoman-<something>-`, an `afterEach` that force-removes it swallowing errors, and a
- *   `loadRows()` that reads the run's `canonical.jsonl` back through `JSONSpliterator`. Nothing about
- *   any of that is adapter-specific; only the tmpdir prefix and the adapter id ever differed.
- *
- *   This directory is excluded from the published tarball by `corpus/package.json`'s `files` (the
- *   `!test-kit/**\/*` entry, which predates this file), which is why importing `vitest` here is safe
- *   — `mailwoman/test-kit/index.ts` imports it on the same grounds.
- */
-
-import { temporaryDirectory, type TemporaryDirectory } from "@mailwoman/core/fs/temporary"
-import { join, resolvePath } from "path-ts"
-import { createNewlineWriter, JSONSpliterator } from "spliterator"
-import { afterEach, beforeEach } from "vitest"
-
-import type { CanonicalRow } from "#types"
-
-/**
- * A per-test scratch directory. `path` is only meaningful inside a test body — it is `""` until the `beforeEach` runs.
- */
-export interface ScratchDir {
-	readonly path: string
-}
-
-/**
- * Register a fresh scratch directory for each test in the current suite, removed afterwards.
- *
- * `slug` names the directory (`mailwoman-<slug>-XXXXXX` under the OS temp dir) and exists only to make a stray leftover
- * traceable to the suite that made it. Teardown swallows its own errors: a test that already removed the directory, or
- * a platform that holds a handle open, must not turn a passing assertion into a failing suite.
- */
-export function useScratchDir(slug: string): ScratchDir {
-	const dir = { path: "" }
-	let owned: TemporaryDirectory | undefined
-
-	beforeEach(async () => {
-		owned = await temporaryDirectory(`mailwoman-${slug}-`)
-		dir.path = resolvePath(owned.path)
-	})
-
-	// The directory is owned by the TEST, never by a module-scoped stack. Under `isolate: false` this module is shared
-	// across every corpus adapter suite in a fork, so a stack disposed by the first file's `afterAll` left every later
-	// file calling `use()` on a disposed stack — which is what "Cannot call AsyncDisposableStack.prototype.use on an
-	// already-disposed DisposableStack" was, across a different set of adapter suites on each run.
-	//
-	// Teardown swallows its own errors: a test that already removed the directory, or a platform that holds a handle
-	// open, must not turn a passing assertion into a failing suite.
-	afterEach(async () => {
-		try {
-			await owned?.[Symbol.asyncDispose]()
-		} catch {
-			// See above.
-		}
-
-		owned = undefined
-	})
-
-	return dir
-}
-
-/**
- * Read back the canonical rows a `runAdapter` call wrote — `<outputDir>/<adapterID>/canonical.jsonl`, streamed through
- * `JSONSpliterator` and collected.
- */
-export function readCanonicalRows(outputDir: string, adapterID: string): Promise<CanonicalRow[]> {
-	return Array.fromAsync(JSONSpliterator.fromAsync<CanonicalRow>(join(outputDir, adapterID, "canonical.jsonl")))
-}
-
-/**
- * Write a delimited fixture — a header line plus the given rows — and answer its path.
- *
- * Three adapter suites carried a hand-rolled copy of this, and the copies disagreed about the one thing a reader cannot
- * see: two joined the rows without a trailing newline and the third appended one. `createNewlineWriter` terminates
- * every line it writes, so the file round-trips through `CSVSpliterator` the same way whichever suite produced it, and
- * a caller passes content without a delimiter.
- */
-export async function writeDelimitedFixture(
-	filePath: string,
-	header: string,
-	rows: readonly string[]
-): Promise<string> {
-	await using out = createNewlineWriter(filePath)
-
-	await out.write(header)
-
-	for (const row of rows) {
-		await out.write(row)
-	}
-
-	return filePath
-}

package/lib/tools/align-slice.ts

@@ -1,68 +0,0 @@
-import { openWriteStream } from "@mailwoman/core/fs/streams"
-import { JSONSpliterator } from "spliterator"
-
-/**
- * Re-emit a CANONICAL jsonl ({raw, components, country, source, ...}) as a LABELED jsonl in the CURRENT align format,
- * by running every row through `alignRow` (corpus/src/align.ts).
- *
- * ## Why this exists
- *
- * Most synthetic slices are GENERATED on demand by a `build-*-slice` recipe (parametrized by --count), so re-emitting
- * them in a new label format is just a re-run. A few slices are FIXED corpora with a hand/DeepSeek-authored canonical
- * source that is never regenerated — notably `deepseek-kryptonite` (the adversarial hard-case set) and the
- * `deepseek-translit-*` variants. Their committed parquets carry whatever label format was current when they were first
- * built.
- *
- * When the corpus label format changes (the v0.5.0 char-offset triple, #519), those fixed slices must be RE-ALIGNED,
- * not regenerated — feed the canonical source back through the same `alignRow` the from-source build uses, so the spans
- * land in the new format with zero drift. That is exactly what this does: canonical jsonl in → labeled jsonl out, one
- * `alignRow` per row, quarantine on miss.
- *
- * It is the uniform counterpart to corpus/src/tools/slice-kryptonite.ts (which couples to a base manifest and writes
- * parquet directly). Output goes to jsonl so it joins the SAME jsonl-to-parquet path every other overlay slice uses.
- *
- * Usage: node scripts/align-canonical-slice.ts\
- * --input /path/canonical-kryptonite.jsonl\
- * --output /tmp/kryptonite-labeled.jsonl\
- * --corpus-version 0.5.0
- */
-import { alignRow } from "#utils"
-
-export interface AlignSliceOptions {
-	input: string
-	output: string
-	corpusVersion: string
-}
-
-export async function alignCanonicalSlice(args: AlignSliceOptions): Promise<void> {
-	// Read phase only — the write path stays on createWriteStream.
-	const outStream = openWriteStream(args.output, { encoding: "utf8" })
-	let labeled = 0
-	let quarantined = 0
-	const quarantineReasons: Record<string, number> = {}
-
-	for await (const canonical of JSONSpliterator.fromAsync<Parameters<typeof alignRow>[0]>(args.input)) {
-		// Stamp the target corpus version so the emitted row's provenance matches the run it joins.
-		canonical.corpus_version = args.corpusVersion
-		const result = alignRow(canonical)
-
-		if (result.kind === "labeled") {
-			outStream.write(JSON.stringify(result.row) + "\n")
-
-			labeled++
-		} else {
-			quarantined++
-			const r = result.row.reason ?? "unknown"
-			quarantineReasons[r] = (quarantineReasons[r] ?? 0) + 1
-		}
-	}
-
-	await new Promise<void>((res) => {
-		outStream.end(res)
-	})
-
-	console.error(
-		`align-canonical-slice: ${labeled} labeled, ${quarantined} quarantined → ${args.output}\n` +
-			`  quarantine reasons: ${JSON.stringify(quarantineReasons)}`
-	)
-}

package/lib/tools/fetch/index.ts

@@ -1,174 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   The `mailwoman corpus fetch <source>` family — reproducible bulk-download recovery modules for
- *   the open-data sources the corpus build pipeline consumes. Each module writes the raw download
- *   files **plus** a sibling `MANIFEST.json` capturing the origin URL, fetch timestamp, byte count,
- *   and sha256 so downstream adapters can verify provenance.
- *
- *   The corpus build pipeline itself does NOT call these modules — the existing convention is for
- *   operators to pre-download into an out-root and point adapters at the resulting files. These
- *   modules exist for **reproducibility** (disk-loss recovery, weekly refresh, fresh-environment
- *   bootstrap).
- *
- *   ## Usage
- *
- *   ```sh
- *   # Default: writes under ./data/corpus/sources/ relative to the working directory
- *   mailwoman corpus fetch state-sources
- *   mailwoman corpus fetch hrsa
- *
- *   # Or point at the standard mailwoman data root
- *   mailwoman corpus fetch state-sources --out-root /data/corpus/sources
- *   ```
- *
- *   Each adapter under `corpus/src/adapters/<adapter>/README.md` documents the specific URL its
- *   input was pulled from; these modules mirror those URLs in a single executable place.
- *
- *   ## Coverage
- *
- *   - `ban` — French BAN (Base Adresse Nationale), all départements incl. DOM/TOM. Tier B (Licence
- *       Ouverte 2.0).
- *   - `nad` — US DOT National Address Database (~97M address points, ArcGIS FeatureServer). Tier A
- *       (US PD).
- *   - `geonames-postal` — GeoNames per-country postal exports (~80 countries). Tier B (CC-BY-4.0,
- *       attribute "GeoNames"). The only source in this family carrying `(postcode, locality, region)`
- *       with the names INLINE, which is why it exists: the `parent_id` join route covers exactly five
- *       countries and the nearest-centroid fallback measured under 50% agreement on four of those five.
- *       GeoNames does not publish every country — Venezuela 404s.
- *   - `hrsa` — HRSA Health Center Service Delivery Sites (federal). Tier A (US PD).
- *   - `imls-pls` — IMLS Public Libraries Survey, outlet-level (~17K library branches, FY 2023).
- *       Tier A (US PD).
- *   - `nppes` — NPPES NPI registry, full monthly dissemination (~7M provider venue+address rows).
- *       Tier A (US PD).
- *   - `openaddresses` — OpenAddresses country collections (default: Canada / `ca`). Tier B/C mixed
- *       — per-row filter.
- *   - `ourairports` — OurAirports global airport CSVs (~83K airports + the country/region/runway
- *       joins). Tier A (public domain). The VENUE half of the sub-venue arc (#35); it carries no
- *       interior structure, which comes from `@mailwoman/osm/sdk`'s `extractOSMSubVenues`.
- *   - `wikidata-subvenue` — the multilingual sub-venue DESIGNATOR vocabulary, pulled as the labels
- *       and aliases of eight Wikidata concepts (terminal, gate, concourse, campus, …) plus the
- *       airport-terminal instance labels as attested usage. Tier A (CC0). The only module in this
- *       family built on `APIClient` — see its docstring for why.
- *   - `state-sources` — NY/TX/DE/OR notaries, IA contractors, WA health providers, HI lobbyists.
- *       Tier A (state PD-equivalent).
- *   - `state-hi-schools` — Hawaii DOE school directory (original XLSX; HIDOE + PCS sheets). Tier A
- *       (state PD-equivalent).
- *   - `tiger-full` — US Census TIGER 2024 ADDRFEAT, all US counties. Tier A (US PD).
- *   - `juso-kr` — the Korean road-name address register (주소DB, plus the English road-name DB) from
- *       business.juso.go.kr, the LABEL source for Korean. Tier B ("이용허락범위 제한 없음", attribution).
- *   - `localdata-kr` — Korea's local-government permit registry, one CSV per category, each row carrying
- *       the lot-number AND road-name address of one premises plus a coordinate. The NOISY source for
- *       Korean. Tier B (same label as `juso-kr`).
- *   - `gcis-tw` — Taiwan's company and business registers from the economic ministry's open-data
- *       platform, 148 files by region and industry. The NOISY source for Taiwanese. Tier B (Open
- *       Government Data License v1.0 — the prescribed attribution is a condition of the grant).
- *   - `acra-sg` — Singapore's ACRA corporate-entity register, 27 CSVs with the address FIELDED. The
- *       NOISY source for Singapore. Tier B (Singapore Open Data Licence 1.0).
- *   - `houjin-jp` — the National Tax Agency's corporate-number register, the nationwide Unicode CSV.
- *       The NOISY source for Japanese. Tier A-equivalent (free use, no attribution condition stated).
- *
- *   License tiers per `docs/licensing-strategy.md` (or the playpen knowledge base mirror at
- *   `docs/docs/projects/mailwoman/licensing-strategy.md`). `openaddresses` is a **Tier-mixed**
- *   source: the downloaded collection includes CC0, CC-BY, OGL, and ODbL/CC-BY-SA rows. The per-row
- *   `LICENSE` filter in the `openaddresses` adapter is essential — Tier-C (ODbL, CC-BY-SA) rows are
- *   dropped at ingest by default to protect proprietary-weights training.
- *
- *   ### OpenAddresses authentication (as of 2026-05-18)
- *
- *   `batch.openaddresses.io` now requires a free registered account for bulk downloads (auth check
- *   prevents CDN abuse; data remains openly licensed). `fetchOpenAddresses` reads `OA_BATCH_TOKEN`
- *   from the environment:
- *
- *   ```sh
- *   # One-time: register at https://batch.openaddresses.io/register
- *   # Log in → Profile → "Create Token" → copy token
- *   export OA_BATCH_TOKEN=<your-token>
- *
- *   # Download Canada (~2 GiB compressed, ~7 GiB uncompressed)
- *   mailwoman corpus fetch openaddresses --country ca \
- *     --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
- *
- *   # Or any other OA country code
- *   mailwoman corpus fetch openaddresses --country fr
- *   ```
- *
- *   Without a token the command prints setup instructions and reports the failure.
- *
- *   ## Adding a new source
- *
- *   1. Pick the right module (or create a sibling one if the source is from a meaningfully
- *      different family).
- *   2. Append to the `SOURCES` array: `{ slug, filename, url }`.
- *   3. Confirm the destination URL via `curl -sI -L <url> | head` before committing — state
- *      open-data portals occasionally rotate Socrata view IDs.
- *   4. Run the command against a scratch `--out-root` to verify the download succeeds + the
- *      MANIFEST is well-formed.
- *   5. Add the source's adapter (or extend an existing one) under `corpus/src/adapters/`.
- */
-
-import { fetchBan } from "#fr/tools/fetch/ban"
-import { fetchHoujinJP } from "#jp/tools/fetch/houjin"
-import { fetchJusoKR } from "#kr/tools/fetch/juso"
-import { fetchLocaldataKR } from "#kr/tools/fetch/localdata"
-import { fetchACRASG } from "#sg/tools/fetch/acra"
-import { fetchGeonamesDumps } from "#tools/fetch/geonames/dump"
-import { fetchGeonamesPostal } from "#tools/fetch/geonames/postal"
-import { fetchOpenAddresses } from "#tools/fetch/openaddresses"
-import { fetchOurAirports } from "#tools/fetch/ourairports"
-import { fetchWikidataSubVenue } from "#tools/fetch/wikidata-subvenue"
-import { fetchGCISTW } from "#tw/tools/fetch/gcis"
-import { fetchHRSA } from "#us/tools/fetch/hrsa"
-import { fetchIMLSPLS } from "#us/tools/fetch/imls-pls"
-import { fetchNAD } from "#us/tools/fetch/nad"
-import { fetchNPPES } from "#us/tools/fetch/nppes"
-import { fetchStateHISchools } from "#us/tools/fetch/state/hi-schools"
-import { fetchStateSources } from "#us/tools/fetch/state/sources"
-import { fetchTigerFull } from "#us/tools/fetch/tiger-full"
-
-export * from "#sg/tools/fetch/acra"
-export * from "#fr/tools/fetch/ban"
-export * from "#tw/tools/fetch/gcis"
-export * from "#tools/fetch/geonames/dump"
-export * from "#tools/fetch/geonames/postal"
-export * from "#jp/tools/fetch/houjin"
-export * from "#us/tools/fetch/hrsa"
-export * from "#us/tools/fetch/imls-pls"
-export * from "#kr/tools/fetch/juso"
-export * from "#kr/tools/fetch/localdata"
-export * from "#us/tools/fetch/nad"
-export * from "#us/tools/fetch/nppes"
-export * from "#tools/fetch/openaddresses"
-export * from "#tools/fetch/ourairports"
-export * from "#us/tools/fetch/state/hi-schools"
-export * from "#us/tools/fetch/state/sources"
-export * from "#us/tools/fetch/tiger-full"
-export * from "#tools/fetch/wikidata-subvenue"
-
-/**
- * The fetch-source registry: id → module entry point. Each entry point takes its own options interface.
- */
-export const FETCH_SOURCES = {
-	"acra-sg": fetchACRASG,
-	ban: fetchBan,
-	"gcis-tw": fetchGCISTW,
-	"houjin-jp": fetchHoujinJP,
-	"juso-kr": fetchJusoKR,
-	"localdata-kr": fetchLocaldataKR,
-	nad: fetchNAD,
-	"geonames-dump": fetchGeonamesDumps,
-	"geonames-postal": fetchGeonamesPostal,
-	hrsa: fetchHRSA,
-	"imls-pls": fetchIMLSPLS,
-	nppes: fetchNPPES,
-	openaddresses: fetchOpenAddresses,
-	ourairports: fetchOurAirports,
-	"state-sources": fetchStateSources,
-	"state-hi-schools": fetchStateHISchools,
-	"tiger-full": fetchTigerFull,
-	"wikidata-subvenue": fetchWikidataSubVenue,
-} as const
-
-export type FetchSourceID = keyof typeof FETCH_SOURCES

package/lib/tools/gb-title-case.ts

@@ -1,69 +0,0 @@
-/**
- * Title-case an ALL-CAPS GB place/street string (#690 — all-caps is OOD for the model). PPD ships every field
- * upper-case; the model trains on natural casing. Particles (upon, super, next, …) stay lowercase mid-name, both
- * between words and between hyphen segments. Each apostrophe-separated segment is capitalized on its own (D'Arcy,
- * James'), except a bare trailing possessive 's, which stays lowercase (BISHOP'S → Bishop's).
- */
-const GB_PARTICLES = new Set([
-	"upon",
-	"on",
-	"under",
-	"in",
-	"by",
-	"the",
-	"le",
-	"la",
-	"de",
-	"cum",
-	"next",
-	"with",
-	"over",
-	"at",
-	"super",
-	"sub",
-	"and",
-	"of",
-	"y",
-	"en",
-])
-
-function caseWord(word: string, isFirst: boolean): string {
-	if (!word) return word
-	const lower = word.toLowerCase()
-
-	if (!isFirst && GB_PARTICLES.has(lower)) return lower
-
-	// Capitalize the first letter of each apostrophe-separated segment, except possessive 's.
-	return lower
-		.split("'")
-		.map((segment, idx) => {
-			if (!segment) return segment
-
-			// Don't capitalize a trailing bare possessive 's
-			if (idx > 0 && segment === "s") return segment
-
-			// Capitalize the first letter of this segment
-			return segment[0]!.toUpperCase() + segment.slice(1)
-		})
-		.join("'")
-}
-
-export function titleCaseGB(value: string): string {
-	value = value.trim()
-	let wordIndex = 0
-
-	return value
-		.split(/\s+/)
-		.map((token) => {
-			const cased = token
-				.split("-")
-				.map((seg, segIndex) => caseWord(seg, wordIndex === 0 && segIndex === 0))
-				.join("-")
-
-			wordIndex += 1
-
-			return cased
-		})
-		.join(" ")
-		.trim()
-}