@mailwoman/corpus 9.3.0 → 10.0.0

package/lib/adapters/fcc-bdc/adapter.ts

@@ -1,174 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `fcc-bdc`: FCC Broadband Data Collection (BDC) — Fabric-derived location consumer.
- *
- *   The first member of Phase 1.6's "adversarial sources" class. BDC ships the public-domain US
- *   broadband-serviceable-location (BSL) fabric: ~120M addresses keyed by stable `location_id`,
- *   carrying `address_primary` + `city` + `state` + `zip` + `zip_suffix`. Compared to the clean
- *   gazetteer rows from WOF / TIGER / BAN, BDC carries the chaos of address data that has passed
- *   through several layers of human entry + automated geocoding + revision: abbreviation drift,
- *   inconsistent unit designators, "RR" / "HC" / "PSC" rural-route shapes, embedded apartment /
- *   suite numbers that did not survive the address parser cleanly. This is the highest-signal,
- *   hardest-to-normalize address corpus in the federal public-domain catalog.
- *
- *   Following the `tiger` / `wof-admin` pattern, this adapter consumes a SQLite database the operator
- *   pre-builds via the isp-nexus BDC ETL (`/srv/isp-nexus/sync/fcc/bdc/`) or any equivalent
- *   host-side pipeline. The mailwoman side does not download or parse the raw CSV/ZIP distribution
- *   directly — that keeps the adapter narrow and the BDC ingest pluggable.
- *
- *   The SQLite schema is documented in README.md and modeled after `NTIARecord`
- *   (`isp-nexus/fcc/bdc/data-collection.ts`): one row per `location_id`. The adapter splits
- *   `address_primary` into `house_number` (leading numeric prefix, if any) + `street` (everything
- *   after), and combines `zip` + `zip_suffix` into the canonical USPS `postcode` slot.
- *
- *   One CanonicalRow per fabric record. Unlike `tiger` (multiple postcode variants per segment) or
- *   `wof-admin` (multiple hierarchy variants per place), BDC records already represent fully
- *   specified addresses; no fan-out is warranted. Adversarial composition (Phase 1.6 §2.1) is the
- *   mechanism for deriving multiple training rows per BDC record.
- *
- *   License: stamped `"Public Domain"` per the BDC fabric's US federal-government distribution terms.
- *   The CostQuest Fabric source data has its own license; consumers who substitute that path should
- *   re-stamp accordingly.
- */
-
-import type { BDCDatabase } from "@mailwoman/bdc/schema"
-import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
-import { DatabaseClient } from "@mailwoman/sqlite/client"
-
-import { splitStreetLine } from "#adapters/utils"
-import { lookupStateAbbreviation } from "#codex/us-fips-state"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const FCC_BDC_ADAPTER_ID = "fcc-bdc"
-/**
- * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
- * than having to look them up.
- */
-export const FCC_BDC_DEFAULT_LICENSE = "Public Domain"
-
-/**
- * SQLite row shape — one row per BSL `location_id`. Columns mirror NTIARecord.
- */
-interface BdcLocationRow {
-	location_id: number
-	address_primary: string
-	city: string
-	state: string
-	zip: string
-	zip_suffix: string | null
-}
-
-/**
- * Combine `zip` + optional `zip_suffix` into the canonical USPS postcode surface form.
- *
- * NTIARecord doc is ambiguous about whether `zip_suffix` is the 4-digit extension alone or the full ZIP+4 string. This
- * handles both:
- *
- * - Bare 4-digit extension (`zip="94103"`, `zip_suffix="1234"`) → `"94103-1234"`
- * - Already-joined form (`zip_suffix="94103-1234"`) → returned as-is
- * - No suffix → bare `zip`
- *
- * Empty / whitespace-only suffix is treated as missing.
- */
-export function buildPostcode(zip: string, suffix: string | null): string {
-	const z = zip.trim()
-
-	if (!z) return ""
-	const s = suffix?.trim() ?? ""
-
-	if (!s) return z
-
-	if (s.includes("-")) return s
-
-	return `${z}-${s}`
-}
-
-/**
- * Build a BDC adapter. Pure factory so multiple instances can be created in tests.
- */
-export function createFccBdcAdapter(): CorpusAdapter {
-	return {
-		id: FCC_BDC_ADAPTER_ID,
-		defaultLicense: FCC_BDC_DEFAULT_LICENSE,
-		description:
-			"FCC Broadband Data Collection — Fabric-derived BSL addresses (public-domain); SQLite DB the operator builds via the isp-nexus BDC ETL.",
-
-		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			if (opts.country && opts.country !== "US") {
-				throw new Error(`fcc-bdc adapter: only US supported, got country=${opts.country}`)
-			}
-
-			const db = new DatabaseClient<BDCDatabase>(opts.inputPath, { readOnly: true })
-			let emitted = 0
-
-			try {
-				const stmt = db.prepare(
-					`SELECT location_id, address_primary, city, state, zip, zip_suffix
-					 FROM bdc_locations
-					 ORDER BY location_id`
-				)
-
-				for (const row of stmt.iterate() as IterableIterator<BdcLocationRow>) {
-					if (opts.signal?.aborted) return
-
-					if (opts.limit !== undefined && emitted >= opts.limit) return
-
-					const split = splitStreetLine(row.address_primary ?? "")
-
-					if (!split) continue
-					const state = lookupStateAbbreviation(row.state)
-
-					if (!state) continue
-					const locality = row.city?.trim()
-
-					if (!locality) continue
-					const postcode = buildPostcode(row.zip ?? "", row.zip_suffix ?? null)
-
-					if (!postcode) continue
-
-					const components: CanonicalRow["components"] = {
-						...(split.house_number ? { house_number: split.house_number } : {}),
-						street: split.street,
-						locality,
-						region: state.abbreviation,
-						postcode,
-					}
-
-					const raw = formatAddress(components, "US", { separator: ", " })
-
-					if (!raw) continue
-					const aligned = reconcileComponents(components, raw)
-
-					if (!Object.keys(aligned).length) continue
-
-					yield {
-						raw,
-						components: aligned,
-						country: "US",
-						locale: "en-US",
-						source: FCC_BDC_ADAPTER_ID,
-						source_id: `${FCC_BDC_ADAPTER_ID}-${row.location_id}`,
-						corpus_version: "",
-						license: FCC_BDC_DEFAULT_LICENSE,
-					}
-
-					emitted++
-				}
-			} finally {
-				await db.destroy()
-			}
-		},
-	}
-}
-
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const fccBdcAdapter = createFccBdcAdapter()

package/lib/adapters/geonames-postal/adapter.ts

@@ -1,122 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `geonames-postal`: GeoNames postal-code dump consumer (https://www.geonames.org/, CC-BY-4.0).
- *
- *   The GeoNames postal export (`https://download.geonames.org/export/zip/<CC>.zip`) is a clean,
- *   per-country `postcode → place → admin1` table with the place + region NAMES inline (no aux-file
- *   join needed). It broadens the corpus's postcode→locality→region coverage to ~80 countries, well
- *   beyond `wof-postalcode`/the coordinate-first table — forward coverage for the multi-locale
- *   goal.
- *
- *   Input: a per-country postal dump (`<CC>.txt`, 12 tab-separated columns, no header): country,
- *   postcode, place, admin1_name, admin1_code, admin2__, admin3__, lat, lon, accuracy.
- *
- *   Output: per row, postcode-FIRST (international) variants — the common order for the non-US
- *   locales this fills (US postcodes are already covered by TIGER/WOF, which use postcode-LAST):
- *
- *   1. `{ postcode, locality }` → "AD100 Canillo"
- *   2. `{ postcode, locality, region }` → "AD100 Canillo, Canillo" Prefer configuring this adapter for
- *        non-US countries; for US, the postcode-last sources are the right order. License:
- *        `"CC-BY-4.0"` per row (attribute "GeoNames").
- */
-
-import { reconcileComponents } from "@mailwoman/formatter"
-import { TSVSpliterator } from "spliterator"
-
-import { stableSourceID } from "#adapters/utils"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const GEONAMES_POSTAL_ADAPTER_ID = "geonames-postal"
-/**
- * License carried by this source (CC-BY-4.0), attached to each row so downstream consumers inherit the terms rather
- * than having to look them up.
- */
-export const GEONAMES_POSTAL_DEFAULT_LICENSE = "CC-BY-4.0"
-
-/**
- * GeoNames postal-dump columns (0-based): country, postcode, place, admin1_name, admin1_code, admin2_name, ….
- *
- * Shared with `tools/postcode-triples.ts`, which additionally reads `admin2Name` — the CITY for the IN/MX/PT-shaped
- * exports whose `place` column is a street or colonia.
- */
-export const GEONAMES_POSTAL_COLUMNS = { country: 0, postcode: 1, place: 2, admin1Name: 3, admin2Name: 5 } as const
-
-export function createGeonamesPostalAdapter(): CorpusAdapter {
-	return {
-		id: GEONAMES_POSTAL_ADAPTER_ID,
-		defaultLicense: GEONAMES_POSTAL_DEFAULT_LICENSE,
-		description:
-			"GeoNames postcodes (CC-BY-4.0) — multi-locale postcode→locality→region, names inline; international postcode-first order.",
-
-		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			// `header: false` — the GeoNames postal dump is headerless, and the spliterator would
-			// otherwise consume row 1 as column names and lose its first postcode.
-			const rows = TSVSpliterator.fromAsync(opts.inputPath, { header: false })
-
-			let emitted = 0
-
-			for await (const rec of rows as AsyncIterable<string[]>) {
-				if (opts.signal?.aborted) break
-
-				if (opts.limit !== undefined && emitted >= opts.limit) break
-
-				const cc = (rec[GEONAMES_POSTAL_COLUMNS.country] ?? "").trim()
-
-				if (!cc) continue
-
-				if (opts.country && cc !== opts.country) continue
-
-				const postcode = (rec[GEONAMES_POSTAL_COLUMNS.postcode] ?? "").trim()
-				const locality = (rec[GEONAMES_POSTAL_COLUMNS.place] ?? "").trim()
-
-				if (!postcode || !locality) continue
-				const region = (rec[GEONAMES_POSTAL_COLUMNS.admin1Name] ?? "").trim()
-
-				// Postcode-first (international) variants. Skip the region variant when admin1 just
-				// repeats the place (common for city-states / micro-admin) to avoid "X X" noise.
-				const variants: Array<{ slot: string; comp: CanonicalRow["components"]; raw: string }> = [
-					{ slot: "pl", comp: { postcode, locality }, raw: `${postcode} ${locality}` },
-				]
-
-				if (region && region.toLowerCase() !== locality.toLowerCase()) {
-					variants.push({
-						slot: "plr",
-						comp: { postcode, locality, region },
-						raw: `${postcode} ${locality}, ${region}`,
-					})
-				}
-
-				for (const v of variants) {
-					if (opts.limit !== undefined && emitted >= opts.limit) break
-					const aligned = reconcileComponents(v.comp, v.raw)
-
-					if (Object.keys(aligned).length < 2) continue
-
-					yield {
-						raw: v.raw,
-						components: aligned,
-						country: cc,
-						source: GEONAMES_POSTAL_ADAPTER_ID,
-						source_id: `${stableSourceID(GEONAMES_POSTAL_ADAPTER_ID, aligned)}-${v.slot}`,
-						corpus_version: "",
-						license: GEONAMES_POSTAL_DEFAULT_LICENSE,
-					}
-
-					emitted++
-				}
-			}
-		},
-	}
-}
-
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const geonamesPostalAdapter = createGeonamesPostalAdapter()

package/lib/adapters/gnaf/adapter.ts

@@ -1,149 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   G-NAF (Australia) corpus adapter — the parser-teaching half of #208.
- *
- *   The model mis-parses Australian addresses in their native postcode-first / house-number-last
- *   order: it tags a leading 4-digit postcode as a house number (its US/EU prior) and swaps street
- *   ↔ locality with it. `scripts/eval/au-order-probe.ts` proved this is a word-ORDER coverage gap,
- *   not capability — the same addresses parse perfectly in canonical order (65% → 87% @25km if the
- *   parse were order-robust). EU survives the same eval because its postcodes are
- *   format-distinctive (a hyphenated `26-300` reads as a postcode anywhere); a bare AU `3053` only
- *   disambiguates by position.
- *
- *   So this adapter renders each assembled G-NAF tuple (from {@link ./assemble}) in one of three real
- *   AU layouts — real-AU canonical (number-first, postcode-trailing), postcode-first,
- *   locality-first — ROTATED by row index (`i % 3`), so the locality + postcode each land in every
- *   position across the slice. This is the exact mechanism that fixed #148's v1.9.0 order-overfit
- *   for the 16 EU locales (`scripts/rerender-overture-multiorder.mjs`, v1.9.1 → shipped v4.13.0);
- *   AU was simply never in that train (`country_weights` had no AU, and `data_loader.py` excludes
- *   unlisted countries). Rotating one order per row (rather than emitting all three) keeps this a
- *   clean single-variable extension of the proven recipe + matches its source-mass structure. The
- *   corpus aligner BIO-labels each (every component surface form occurs verbatim in `raw`, so
- *   alignment lands).
- *
- *   Input: the assembled component JSONL (one `{house_number,street,locality,region,postcode}` per
- *   line). Open G-NAF licence — attribute "Geoscape Australia".
- */
-
-import { tryParsingJSON } from "@mailwoman/core/json"
-import { reconcileComponents } from "@mailwoman/formatter"
-import { TextSpliterator } from "spliterator"
-
-import { stableSourceID } from "#adapters/utils"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const GNAF_ADAPTER_ID = "gnaf"
-/**
- * Open G-NAF is freely redistributable with attribution to Geoscape Australia (CC-BY-style).
- */
-export const GNAF_DEFAULT_LICENSE = "CC-BY-4.0"
-
-interface GNAFTuple {
-	house_number: string
-	street: string
-	locality: string
-	region?: string
-	postcode: string
-}
-
-/**
- * The address layouts an AU address actually arrives in. The model already handles postcode-TRAILING (canonical); the
- * two postcode-LEADING forms are the ones it fails, so they carry the change. We keep the canonical form too so the
- * retrain doesn't forget it.
- */
-function renderOrders(c: GNAFTuple): string[] {
-	const region = c.region ? ` ${c.region}` : ""
-
-	return [
-		// real-AU canonical: number-first, street, suburb [state] postcode — "50 Barry Street, Carlton NSW 2000"
-		`${c.house_number} ${c.street}, ${c.locality}${region} ${c.postcode}`,
-		// postcode-first (the dominant failure): "2000 Carlton, Barry Street 50"
-		`${c.postcode} ${c.locality}, ${c.street} ${c.house_number}`,
-		// locality-first: "Carlton, 2000, Barry Street 50"
-		`${c.locality}, ${c.postcode}, ${c.street} ${c.house_number}`,
-	]
-}
-
-/**
- * Build the G-NAF adapter. `inputPath` is the assembled component JSONL (see {@link ./assemble}); it is country-pinned
- * to AU regardless of `opts.country` (G-NAF is Australia-only).
- */
-export function createGNAFAdapter(): CorpusAdapter {
-	return {
-		id: GNAF_ADAPTER_ID,
-		defaultLicense: GNAF_DEFAULT_LICENSE,
-		description:
-			"G-NAF (Australia): assembled address tuples rendered in multiple word orders (canonical / postcode-first / locality-first) — teaches the model AU's postcode-first layout.",
-
-		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			let emitted = 0
-			let idx = 0
-
-			// Input is the assembled component JSONL (one tuple per line). TextSpliterator auto-disposes on
-			// loop completion and on an early `break` (abort / limit), so the old explicit handle teardown is
-			// gone; the parse tolerates a trailing CR on CRLF sources and the `!line.trim()` guard skips blanks.
-			// The render order rotates (i % 3), matching v1.9.1's rerender.
-			for await (const line of TextSpliterator.fromAsync(opts.inputPath)) {
-				if (opts.signal?.aborted) break
-
-				if (opts.limit !== undefined && emitted >= opts.limit) break
-
-				if (!line.trim()) continue
-
-				const t = tryParsingJSON<GNAFTuple>(line)
-
-				if (t === null) continue
-
-				if (!t.house_number || !t.street || !t.locality || !t.postcode) continue
-
-				const orders = renderOrders(t)
-				const order = idx % orders.length
-
-				idx++
-				const raw = orders[order]!
-
-				const components: CanonicalRow["components"] = {
-					house_number: t.house_number,
-					street: t.street,
-					locality: t.locality,
-					postcode: t.postcode,
-				}
-
-				// region (state) rides only the canonical render (order 0); the postcode-leading layouts
-				// omit it (matching the eval's serialization) so it never breaks verbatim alignment.
-				if (order === 0 && t.region) {
-					components.region = t.region
-				}
-
-				const aligned = reconcileComponents(components, raw)
-
-				if (!Object.keys(aligned).length) continue
-
-				yield {
-					raw,
-					components: aligned,
-					country: "AU",
-					locale: "en-AU",
-					source: GNAF_ADAPTER_ID,
-					source_id: `${stableSourceID(GNAF_ADAPTER_ID, aligned)}-o${order}`,
-					corpus_version: "",
-					license: GNAF_DEFAULT_LICENSE,
-				}
-
-				emitted++
-			}
-		},
-	}
-}
-
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const gnafAdapter = createGNAFAdapter()

package/lib/adapters/gnaf/assemble.ts

@@ -1,225 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Assemble a sampled, component-labeled Australian address set from G-NAF (the Geocoded National
- *   Address File — Geoscape Australia, Open G-NAF licence; any derived artifact must attribute
- *   "Geoscape Australia"). G-NAF is a relational PSV distribution (~16.9M addresses);
- *   reconstructing a street address joins three tables — ADDRESS_DETAIL (number, postcode, the
- *   PIDs) → STREET_LOCALITY (street name + type) → LOCALITY (suburb). State is the per-file prefix
- *   (ACT/NSW/…).
- *
- *   Streaming + in-memory join via the house {@link PSVSpliterator} (pipe-separated; `mode: "object"`
- *   keys each row by its header) — NOT raw `read_csv` SQL, which a flat-file join doesn't need and
- *   which the #183–190 cleanup retired. The two lookup tables (STREET_LOCALITY ~765k rows, LOCALITY
- *   ~16k) fit as Maps; ADDRESS_DETAIL is streamed once and reservoir-sampled, so memory stays
- *   bounded (the OOM lesson from the Overture ingest).
- *
- *   No coordinates: the output feeds the PARSER ({@link ../gnaf/adapter}, #208) — teaching the model
- *   AU's postcode-first / house-number-last word order, the gap `scripts/eval/au-order-probe.ts`
- *   pinned (65%→87% if the parse were order-robust). The parser needs the address string +
- *   component labels, not lat/lon.
- *
- *   Output: component tuples as JSONL, consumed by the `gnaf` corpus adapter (which renders them in
- *   multiple orders + the corpus aligner BIO-labels them). An optional held-out eval set is
- *   excluded by (street, locality, postcode) so the training slice never overlaps the benchmark.
- */
-
-import { readDirectory } from "@mailwoman/core/fs/readers"
-import { openWriteStream } from "@mailwoman/core/fs/streams"
-import { tryParsingJSON } from "@mailwoman/core/json"
-import { join } from "path-ts"
-import { PSVSpliterator, TextSpliterator } from "spliterator"
-
-export interface GNAFAssembleOptions {
-	/**
-	 * G-NAF `Standard` directory (holds the per-state `*_psv.psv` tables).
-	 */
-	standardDir: string
-	/**
-	 * Target sample size (uniform reservoir → population-proportional across states).
-	 */
-	sampleSize: number
-	/**
-	 * Output JSONL path.
-	 */
-	out: string
-	/**
-	 * Optional held-out eval JSONL (rows with a `components` field) — its (street,locality,postcode) are excluded.
-	 */
-	holdoutPath?: string
-	/**
-	 * Progress sink (the CLI passes a setter).
-	 */
-	onProgress?: (message: string) => void
-}
-
-export interface GNAFAssembleResult {
-	written: number
-	seen: number
-	heldOut: number
-	byState: Record<string, number>
-}
-
-/**
- * UPPERCASE → Title Case, preserving intra-word apostrophes/hyphens (O'Brien, Coff's Harbour).
- */
-export function titlecase(s: string): string {
-	return s
-		.toLowerCase()
-		.replaceAll(/(^|[\s'\-/])([a-z])/g, (_m, p: string, c: string) => p + c.toUpperCase())
-		.trim()
-}
-
-/**
- * Holdout/dedup key: a street within a locality+postcode (house-number-agnostic — conservative).
- */
-export function gnafHoldoutKey(street: string, locality: string, postcode: string): string {
-	return `${street}|${locality}|${postcode}`.toLowerCase()
-}
-
-type Row = Record<string, string | number | undefined>
-
-async function* psvObjects(path: string): AsyncIterable<Row> {
-	yield* PSVSpliterator.fromAsync(path, { mode: "object", header: true }) as AsyncIterable<Row>
-}
-
-/**
- * Load a small lookup table fully into a Map keyed by `keyCol`.
- */
-async function loadMap<V>(paths: string[], keyCol: string, pick: (r: Row) => V): Promise<Map<string, V>> {
-	const m = new Map<string, V>()
-
-	for (const p of paths) {
-		for await (const r of psvObjects(p)) {
-			const k = r[keyCol]
-
-			if (k != null && k !== "") {
-				m.set(String(k), pick(r))
-			}
-		}
-	}
-
-	return m
-}
-
-/**
- * Build the held-out key set from an eval JSONL whose rows carry a `components` object.
- */
-async function loadHoldout(path: string): Promise<Set<string>> {
-	const keys = new Set<string>()
-
-	for await (const line of TextSpliterator.fromAsync(path)) {
-		if (!line.trim()) continue
-
-		const c = tryParsingJSON<{ components?: Record<string, string> }>(line)?.components
-
-		if (c?.street && c?.locality && c?.postcode) {
-			keys.add(gnafHoldoutKey(c.street, c.locality, c.postcode))
-		}
-	}
-
-	return keys
-}
-
-export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssembleResult> {
-	const progress = opts.onProgress ?? (() => {})
-	const files = await readDirectory(opts.standardDir)
-
-	const pick = (re: RegExp, exclude?: RegExp) =>
-		files.filter((f) => re.test(f) && !(exclude && exclude.test(f))).map((f) => join(opts.standardDir, f))
-
-	// `*_LOCALITY_psv.psv` also globs `*_STREET_LOCALITY_psv.psv` — exclude the latter explicitly.
-	const streetPaths = pick(/_STREET_LOCALITY_psv\.psv$/)
-	const localityPaths = pick(/_LOCALITY_psv\.psv$/, /_STREET_LOCALITY_psv\.psv$/)
-	const addressPaths = pick(/_ADDRESS_DETAIL_psv\.psv$/)
-
-	const holdout = opts.holdoutPath ? await loadHoldout(opts.holdoutPath) : new Set<string>()
-
-	if (opts.holdoutPath) {
-		progress(`held-out eval keys: ${holdout.size}`)
-	}
-
-	progress(`loading STREET_LOCALITY (${streetPaths.length} files) + LOCALITY (${localityPaths.length})…`)
-
-	const streetMap = await loadMap(streetPaths, "STREET_LOCALITY_PID", (r) => ({
-		name: String(r.STREET_NAME ?? ""),
-		type: String(r.STREET_TYPE_CODE ?? ""),
-		suffix: String(r.STREET_SUFFIX_CODE ?? ""),
-	}))
-
-	const localityMap = await loadMap(localityPaths, "LOCALITY_PID", (r) => String(r.LOCALITY_NAME ?? ""))
-	progress(`streets=${streetMap.size.toLocaleString()} localities=${localityMap.size.toLocaleString()}`)
-
-	const reservoir: Array<{ house_number: string; street: string; locality: string; region: string; postcode: string }> =
-		[]
-
-	let seen = 0
-	let heldOut = 0
-
-	for (const p of addressPaths) {
-		const state = (p.match(/\/([A-Z]+)_ADDRESS_DETAIL/) ?? [])[1] ?? ""
-
-		for await (const r of psvObjects(p)) {
-			const numberFirst = String(r.NUMBER_FIRST ?? "")
-
-			if (!numberFirst || r.DATE_RETIRED || !r.POSTCODE) continue
-			const st = streetMap.get(String(r.STREET_LOCALITY_PID ?? ""))
-			const suburbRaw = localityMap.get(String(r.LOCALITY_PID ?? ""))
-
-			if (!st?.name || !suburbRaw) continue
-			const street = `${titlecase(st.name)} ${titlecase(st.type)}${st.suffix ? " " + titlecase(st.suffix) : ""}`.trim()
-			const locality = titlecase(suburbRaw)
-			const postcode = String(r.POSTCODE)
-
-			if (holdout.has(gnafHoldoutKey(street, locality, postcode))) {
-				heldOut++
-
-				continue
-			}
-
-			let house = numberFirst + (r.NUMBER_FIRST_SUFFIX ? String(r.NUMBER_FIRST_SUFFIX) : "")
-
-			if (r.NUMBER_LAST) {
-				house = `${house}-${String(r.NUMBER_LAST)}`
-			}
-
-			if (r.FLAT_NUMBER) {
-				house = `${String(r.FLAT_NUMBER)}/${house}`
-			}
-
-			const tuple = { house_number: house, street, locality, region: state, postcode }
-
-			seen++
-
-			if (reservoir.length < opts.sampleSize) {
-				reservoir.push(tuple)
-			} else {
-				const j = Math.floor(Math.random() * seen)
-
-				if (j < opts.sampleSize) {
-					reservoir[j] = tuple
-				}
-			}
-		}
-
-		progress(`${state}: ${seen.toLocaleString()} valid joinable seen`)
-	}
-
-	const out = openWriteStream(opts.out)
-	const byState: Record<string, number> = {}
-
-	for (const t of reservoir) {
-		out.write(JSON.stringify(t) + "\n")
-		byState[t.region] = (byState[t.region] ?? 0) + 1
-	}
-
-	await new Promise<void>((res) => {
-		out.end(res)
-	})
-
-	progress(`wrote ${reservoir.length.toLocaleString()} tuples → ${opts.out}`)
-
-	return { written: reservoir.length, seen, heldOut, byState }
-}