@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/adapters.ts

@@ -0,0 +1,230 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Adapter registry bootstrap.
+ *
+ *   The module registers every built-in adapter with `defaultAdapterRegistry` when loaded. The CLI
+ *   (`commands/corpus/list.tsx`, `commands/corpus/run.tsx`) imports it once at startup.
+ *
+ *   Each adapter has its own directory, and the publisher decides which parent it sits under. An
+ *   adapter reading one country's publisher lives under that country, at `lib/<iso2>/adapters/<id>/`:
+ *   G-NAF under `au`, Digitaal Vlaanderen under `be`, ČÚZK under `cz`, Klimadatastyrelsen under
+ *   `dk`, Maa- ja Ruumiamet under `ee`, Ryhti under `fi`, BAN, FINESS and the Annuaire de l'éducation under `fr`, ANAC under `it`, the
+ *   Administration du
+ *   cadastre et de la topographie under `lu`, the State Address Register under `lv`, Kartverket's Matrikkelen
+ *   under `no`, EMUiA under `pl`, Romania's school network under `ro`, and the FCC and USGov readers under `us`. A national publisher
+ *   stays there even when its own files cover that state's dependencies, because the publisher is
+ *   what the adapter reads: BAN carries France and ten overseas jurisdictions, Matrikkelen carries
+ *   Norway and Svalbard, and Ryhti's one national file carries Åland beside the Finnish mainland.
+ *
+ *   Flanders is the counter-case worth stating. Its adapter sits under `be` and covers the Flemish
+ *   Region alone, because Brussels and Wallonia publish the INSPIRE theme through their own
+ *   services, so Belgium needs three adapters rather than one.
+ *
+ *   Spain needs four, for the same reason. `es-catastro` reads the Dirección General del Catastro's
+ *   52 provinces, and `es-bizkaia`, `es-gipuzkoa` and `es-navarra` read the foral cadastres, which
+ *   state their own terms and which the national feed links rather than serves.
+ *
+ *   An adapter reading a publisher that is not any one country's lives here, at `lib/adapters/<id>/`.
+ *   That is the aggregators, `./openaddresses/`, `./osm/`, `./overture/`, `./geonames/` and `./wof/`,
+ *   plus `./po-box/`.
+ *
+ *   A new adapter joins the `BUILTIN_ADAPTERS` list below. A test that needs a pristine registry
+ *   constructs its own `InMemoryAdapterRegistry` rather than mutating the default.
+ *
+ *   The WOF adapters keep their canonical ids, `wof-admin` and `wof-postalcode`, so existing
+ *   `mailwoman corpus build` callsites continue to work. `./wof/admin/json/` and
+ *   `./wof/postalcode-json/` hold the implementations.
+ */
+
+import { cordisAdapter } from "#adapters/cordis/adapter"
+import { geonamesAdapter } from "#adapters/geonames/adapter"
+import { geonamesPostalAdapter } from "#adapters/geonames/postal/adapter"
+import { gleifAdapter } from "#adapters/gleif/adapter"
+import { openaddressesAdapter } from "#adapters/openaddresses/adapter"
+import { osmAdapter } from "#adapters/osm/adapter"
+import { overtureAdapter } from "#adapters/overture/adapter"
+import { defaultAdapterRegistry } from "#adapters/registry"
+import { wofAdminAdapter } from "#adapters/wof/admin/json/adapter"
+import { wofPostalcodeAdapter } from "#adapters/wof/postalcode-json/adapter"
+import { gnafAdapter } from "#au/adapters/gnaf/adapter"
+import { brusselsAdapter } from "#be/adapters/brussels/adapter"
+import { vlaanderenAdapter } from "#be/adapters/vlaanderen/adapter"
+import { wallonieAdapter } from "#be/adapters/wallonie/adapter"
+import { czCuzkAdapter } from "#cz/adapters/cuzk/adapter"
+import { dkInspireAdapter } from "#dk/adapters/inspire/adapter"
+import { adsAdapter } from "#ee/adapters/ads/adapter"
+import { esBizkaiaAdapter } from "#es/adapters/bizkaia/adapter"
+import { esCatastroAdapter } from "#es/adapters/catastro/adapter"
+import { esGipuzkoaAdapter } from "#es/adapters/gipuzkoa/adapter"
+import { esNavarraAdapter } from "#es/adapters/navarra/adapter"
+import { ryhtiAdapter } from "#fi/adapters/ryhti/adapter"
+import { annuaireEducationAdapter } from "#fr/adapters/annuaire-education/adapter"
+import { banAdapter } from "#fr/adapters/ban/adapter"
+import { finessAdapter } from "#fr/adapters/finess/adapter"
+import { itANACAdapter } from "#it/adapters/anac/adapter"
+import { bdAdressesAdapter } from "#lu/adapters/bd-adresses/adapter"
+import { varisAdapter } from "#lv/adapters/varis/adapter"
+import { nlKadasterAdapter } from "#nl/adapters/kadaster/adapter"
+import { matrikkelenAdapter } from "#no/adapters/matrikkelen/adapter"
+import { emuiaAdapter } from "#pl/adapters/emuia/adapter"
+import { reteaScolaraAdapter } from "#ro/adapters/retea-scolara/adapter"
+import { skInspireAdapter } from "#sk/adapters/inspire/adapter"
+import type { CorpusAdapter } from "#types"
+import { fccBdcAdapter } from "#us/adapters/fcc-bdc/adapter"
+import { stateHiSchoolsAdapter } from "#us/adapters/state/hi-schools/adapter"
+import { stateIaContractorsAdapter } from "#us/adapters/state/ia-contractors/adapter"
+import { stateNyNotariesAdapter } from "#us/adapters/state/ny-notaries/adapter"
+import { stateTxNotariesAdapter } from "#us/adapters/state/tx-notaries/adapter"
+import { tigerAdapter } from "#us/adapters/tiger/adapter"
+import { usgovHrsaFqhcAdapter } from "#us/adapters/usgov/hrsa-fqhc/adapter"
+import { usgovImlsPlsAdapter } from "#us/adapters/usgov/imls-pls/adapter"
+import { USGovIRSBMFAdapter } from "#us/adapters/usgov/irs-bmf/adapter"
+import { usgovNADAdapter } from "#us/adapters/usgov/nad/adapter"
+import { usgovNPPESAdapter } from "#us/adapters/usgov/nppes/adapter"
+
+/**
+ * Built-in adapters.
+ *
+ * Order is significant, because `corpus build` iterates this list to drive every adapter in turn.
+ * The order runs coarse-first (admin, then postcode), then street-level
+ * (BAN FR, tiger US, OpenAddresses global), then adversarial-source (FCC BDC US, HRSA fqhc US).
+ *
+ * The `usgov-samhsa-treatment-locator` adapter is intentionally absent from this list.
+ * The samhsa Open Data Foundry bulk CSV it was written against is no longer publicly distributed.
+ *
+ * Its factory and export remain available so an operator who obtains a compatible CSV can hand-register it.
+ * Re-add it here once a stable public source returns.
+ */
+export const BUILTIN_ADAPTERS: readonly CorpusAdapter[] = [
+	wofAdminAdapter,
+	wofPostalcodeAdapter,
+	geonamesAdapter,
+	geonamesPostalAdapter,
+	banAdapter,
+	matrikkelenAdapter,
+	ryhtiAdapter,
+	czCuzkAdapter,
+	dkInspireAdapter,
+	adsAdapter,
+	emuiaAdapter,
+	vlaanderenAdapter,
+	wallonieAdapter,
+	brusselsAdapter,
+	nlKadasterAdapter,
+	bdAdressesAdapter,
+	skInspireAdapter,
+	esCatastroAdapter,
+	esBizkaiaAdapter,
+	esGipuzkoaAdapter,
+	esNavarraAdapter,
+	tigerAdapter,
+	openaddressesAdapter,
+	overtureAdapter,
+	osmAdapter,
+	gnafAdapter,
+	fccBdcAdapter,
+	usgovHrsaFqhcAdapter,
+	usgovNPPESAdapter,
+	usgovNADAdapter,
+	usgovImlsPlsAdapter,
+	USGovIRSBMFAdapter,
+	stateIaContractorsAdapter,
+	stateTxNotariesAdapter,
+	stateNyNotariesAdapter,
+	stateHiSchoolsAdapter,
+	itANACAdapter,
+	reteaScolaraAdapter,
+	cordisAdapter,
+	varisAdapter,
+	finessAdapter,
+	annuaireEducationAdapter,
+	gleifAdapter,
+]
+
+for (const adapter of BUILTIN_ADAPTERS) {
+	if (!defaultAdapterRegistry.get(adapter.id)) {
+		defaultAdapterRegistry.register(adapter)
+	}
+}
+
+export { BAN_ADAPTER_ID, banAdapter } from "#fr/adapters/ban/adapter"
+export { MATRIKKELEN_ADAPTER_ID, matrikkelenAdapter } from "#no/adapters/matrikkelen/adapter"
+export { FCC_BDC_ADAPTER_ID, FCC_BDC_DEFAULT_LICENSE, fccBdcAdapter } from "#us/adapters/fcc-bdc/adapter"
+
+export {
+	GEONAMES_POSTAL_ADAPTER_ID,
+	GEONAMES_POSTAL_DEFAULT_LICENSE,
+	geonamesPostalAdapter,
+} from "#adapters/geonames/postal/adapter"
+
+export { GEONAMES_ADAPTER_ID, GEONAMES_DEFAULT_LICENSE, geonamesAdapter } from "#adapters/geonames/adapter"
+
+export {
+	OPENADDRESSES_ADAPTER_ID,
+	OPENADDRESSES_DEFAULT_LICENSE,
+	openaddressesAdapter,
+} from "#adapters/openaddresses/adapter"
+
+export {
+	STATE_HI_SCHOOLS_ADAPTER_ID,
+	STATE_HI_SCHOOLS_DEFAULT_LICENSE,
+	stateHiSchoolsAdapter,
+} from "#us/adapters/state/hi-schools/adapter"
+
+export {
+	STATE_IA_CONTRACTORS_ADAPTER_ID,
+	STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
+	stateIaContractorsAdapter,
+} from "#us/adapters/state/ia-contractors/adapter"
+
+export {
+	STATE_NY_NOTARIES_ADAPTER_ID,
+	STATE_NY_NOTARIES_DEFAULT_LICENSE,
+	stateNyNotariesAdapter,
+} from "#us/adapters/state/ny-notaries/adapter"
+
+export {
+	STATE_TX_NOTARIES_ADAPTER_ID,
+	STATE_TX_NOTARIES_DEFAULT_LICENSE,
+	stateTxNotariesAdapter,
+} from "#us/adapters/state/tx-notaries/adapter"
+
+export { TIGER_ADAPTER_ID, TIGER_DEFAULT_LICENSE, tigerAdapter } from "#us/adapters/tiger/adapter"
+
+export {
+	USGOV_HRSA_FQHC_ADAPTER_ID,
+	USGOV_HRSA_FQHC_DEFAULT_LICENSE,
+	usgovHrsaFqhcAdapter,
+} from "#us/adapters/usgov/hrsa-fqhc/adapter"
+
+export {
+	USGOV_IMLS_PLS_ADAPTER_ID,
+	USGOV_IMLS_PLS_DEFAULT_LICENSE,
+	usgovImlsPlsAdapter,
+} from "#us/adapters/usgov/imls-pls/adapter"
+
+export {
+	USGOV_IRS_BMF_ADAPTER_ID,
+	USGOV_IRS_BMF_DEFAULT_LICENSE,
+	USGovIRSBMFAdapter,
+} from "#us/adapters/usgov/irs-bmf/adapter"
+
+export { USGOV_NAD_ADAPTER_ID, USGOV_NAD_DEFAULT_LICENSE, usgovNADAdapter } from "#us/adapters/usgov/nad/adapter"
+
+export {
+	USGOV_NPPES_ADAPTER_ID,
+	USGOV_NPPES_DEFAULT_LICENSE,
+	usgovNPPESAdapter,
+} from "#us/adapters/usgov/nppes/adapter"
+
+export {
+	USGOV_SAMHSA_ADAPTER_ID,
+	USGOV_SAMHSA_DEFAULT_LICENSE,
+	usgovSamhsaTreatmentLocatorAdapter,
+} from "#us/adapters/usgov/samhsa-treatment-locator/adapter"
+
+export { WOF_ADMIN_ADAPTER_ID, wofAdminAdapter } from "#adapters/wof/admin/json/adapter"
+export { WOF_POSTALCODE_ADAPTER_ID, wofPostalcodeAdapter } from "#adapters/wof/postalcode-json/adapter"

package/lib/au/adapters/gnaf/adapter.ts

@@ -0,0 +1,144 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * G-NAF (Australia) corpus adapter.
+ *
+ * The model mis-parses Australian addresses in their native postcode-first and house-number-last
+ * order, where it tags a leading four-digit postcode as a house number and swaps street against
+ * locality. This adapter renders each assembled G-NAF tuple (see {@link ./assemble}) in three real
+ * AU layouts, rotated by row index, so the locality and postcode each land in every position across
+ * the corpus. Input is the assembled component jsonl, one tuple per line. Open G-NAF license
+ * requires attribution to "Geoscape Australia".
+ */
+
+/* oxlint-disable mailwoman/prefer-home -- this adapter reads Australia's national register and writes AU surfaces
+   only. One of the forms below is a deliberate malformation. The postcode-first order is the dominant failure this
+   source exists to teach against — and a renderer that produces well-formed addresses cannot express it. */
+
+import { componentsPresentIn } from "@mailwoman/codex/address/format"
+import { tryParsingJSON } from "@mailwoman/core/json"
+import { TextSpliterator } from "spliterator"
+
+import { stableSourceID } from "#adapters/source-id"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
+
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ */
+export const GNAF_ADAPTER_ID = "gnaf"
+/**
+ * Open G-NAF is freely redistributable with attribution to Geoscape Australia (CC-BY-style).
+ */
+export const GNAF_DEFAULT_LICENSE = "CC-BY-4.0"
+
+interface GNAFTuple {
+	house_number: string
+	street: string
+	locality: string
+	region?: string
+	postcode: string
+}
+
+/**
+ * The address layouts an AU address actually arrives in.
+ *
+ * The model already handles postcode-trailing (canonical); the two postcode-leading
+ * forms are the ones it fails, so they show the change.
+ * We keep the canonical form too so the retrain doesn't forget it.
+ */
+function renderOrders(c: GNAFTuple): string[] {
+	const region = c.region ? ` ${c.region}` : ""
+
+	return [
+		// real-AU canonical: number-first, street, suburb [state] postcode — "50 Barry Street, Carlton NSW 2000"
+		`${c.house_number} ${c.street}, ${c.locality}${region} ${c.postcode}`,
+		// postcode-first (the dominant failure): "2000 Carlton, Barry Street 50"
+		`${c.postcode} ${c.locality}, ${c.street} ${c.house_number}`,
+		// locality-first: "Carlton, 2000, Barry Street 50"
+		`${c.locality}, ${c.postcode}, ${c.street} ${c.house_number}`,
+	]
+}
+
+/**
+ * Build the G-NAF adapter.
+ *
+ * `inputPath` is the assembled component jsonl (see {@link ./assemble}); it is
+ * country-pinned to AU regardless of `opts.country` (G-NAF is Australia-only).
+ */
+export function createGNAFAdapter(): CorpusAdapter {
+	return {
+		id: GNAF_ADAPTER_ID,
+		defaultLicense: GNAF_DEFAULT_LICENSE,
+		addressRole: AddressRole.Premise,
+		register: SourceRegister.GNAF,
+		surface: SurfaceOrigin.Rendered,
+		description:
+			"G-NAF (Australia): assembled address tuples rendered in multiple word orders (canonical / postcode-first / locality-first) — teaches the model AU's postcode-first layout.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			let emitted = 0
+			let idx = 0
+
+			// TextSpliterator auto-disposes on loop completion and on an early `break` (abort / limit).
+			for await (const line of TextSpliterator.fromAsync(opts.inputPath)) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const t = tryParsingJSON<GNAFTuple>(line)
+
+				if (t === null) continue
+
+				if (!t.house_number || !t.street || !t.locality || !t.postcode) continue
+
+				const orders = renderOrders(t)
+				const order = idx % orders.length
+
+				idx++
+				const raw = orders[order]!
+
+				const components: CanonicalRow["components"] = {
+					house_number: t.house_number,
+					street: t.street,
+					locality: t.locality,
+					postcode: t.postcode,
+				}
+
+				// `region` appears only in the canonical render.
+				// The postcode-leading layouts omit it so verbatim alignment never breaks.
+				if (order === 0 && t.region) {
+					components.region = t.region
+				}
+
+				// `raw` is one of three deliberate word orders, two of which no layout prints.
+				// Containment is therefore checked against the string this adapter built.
+				const aligned = componentsPresentIn(components, raw)
+
+				if (!Object.keys(aligned).length) continue
+
+				yield {
+					raw,
+					components: aligned,
+					country: "AU",
+					locale: "en-AU",
+					source: GNAF_ADAPTER_ID,
+					source_id: `${stableSourceID(GNAF_ADAPTER_ID, aligned)}-o${order}`,
+					corpus_version: "",
+					license: GNAF_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const gnafAdapter = createGNAFAdapter()

package/lib/au/adapters/gnaf/assemble.ts

@@ -0,0 +1,205 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Assemble a sampled, component-labeled Australian address set from G-NAF (the Geocoded National
+ * Address File, Geoscape Australia, Open G-NAF license). Any derived artifact must attribute
+ * "Geoscape Australia". G-NAF is a relational PSV distribution whose street address reconstructs
+ * by joining ADDRESS_DETAIL (number, postcode, the PIDs) to STREET_LOCALITY (street name and type)
+ * and then to locality (suburb). State is the per-file prefix (ACT/NSW/…).
+ *
+ * The two lookup tables load as Maps. ADDRESS_DETAIL streams once and is reservoir-sampled, so
+ * memory stays bounded.
+ *
+ * No coordinates. The output feeds the parser. The parser needs the address string and its component
+ * labels. Output is component tuples as jsonl. An optional held-out eval set is excluded by
+ * (street, locality, postcode) so the training corpus never overlaps the benchmark.
+ */
+
+import { tryParsingJSON, stringifyJSON } from "@mailwoman/core/json"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
+import { titleCase, createNewlineWriter, PSVSpliterator, TextSpliterator } from "spliterator"
+import { Globerator } from "spliterator/node/fs"
+
+export interface GNAFAssembleOptions {
+	/**
+	 * G-NAF `Standard` directory (holds the per-state `*_psv.psv` tables).
+	 */
+	standardDir: PathBuilderLike
+	/**
+	 * Target sample size (uniform reservoir → population-proportional across states).
+	 */
+	sampleSize: number
+	/**
+	 * Output jsonl path.
+	 */
+	out: string
+	/**
+	 * Optional held-out eval jsonl (rows with a `components` field) —
+	 * its (street,locality,postcode) are excluded.
+	 */
+	holdoutPath?: string
+	/**
+	 * Progress sink (the CLI passes a setter).
+	 */
+	onProgress?: (message: string) => void
+}
+
+export interface GNAFAssembleResult {
+	written: number
+	seen: number
+	heldOut: number
+	byState: Record<string, number>
+}
+
+/**
+ * Holdout/dedup key: a street within a locality+postcode (house-number-agnostic — conservative).
+ */
+export function gnafHoldoutKey(street: string, locality: string, postcode: string): string {
+	return `${street}|${locality}|${postcode}`.toLowerCase()
+}
+
+type Row = Record<string, string | number | undefined>
+
+async function* psvObjects(path: string): AsyncIterable<Row> {
+	yield* PSVSpliterator.fromAsync(path) as AsyncIterable<Row>
+}
+
+/**
+ * Load a small lookup table fully into a Map keyed by `keyCol`.
+ */
+async function loadMap<V>(paths: string[], keyCol: string, pick: (r: Row) => V): Promise<Map<string, V>> {
+	const m = new Map<string, V>()
+
+	for (const p of paths) {
+		for await (const r of psvObjects(p)) {
+			const k = r[keyCol]
+
+			if (k != null && k !== "") {
+				m.set(String(k), pick(r))
+			}
+		}
+	}
+
+	return m
+}
+
+/**
+ * Build the held-out key set from an eval jsonl whose rows contain a `components` object.
+ */
+async function loadHoldout(path: string): Promise<Set<string>> {
+	const keys = new Set<string>()
+
+	for await (const line of TextSpliterator.fromAsync(path)) {
+		const c = tryParsingJSON<{ components?: Record<string, string> }>(line)?.components
+
+		if (c?.street && c?.locality && c?.postcode) {
+			keys.add(gnafHoldoutKey(c.street, c.locality, c.postcode))
+		}
+	}
+
+	return keys
+}
+
+export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssembleResult> {
+	const progress = opts.onProgress ?? (() => {})
+	const standardDir = PathBuilder.from(opts.standardDir)
+	const files = await Globerator.from("*", { cwd: standardDir, absolute: false }).toArray()
+
+	// Strings, because the address loop reads the state code out of each path.
+	const pick = (re: RegExp, exclude?: RegExp) =>
+		files.filter((f) => re.test(f) && !(exclude && exclude.test(f))).map((f) => standardDir(f).toString())
+
+	// `*_LOCALITY_psv.psv` also globs `*_STREET_LOCALITY_psv.psv` — exclude the latter explicitly.
+	const streetPaths = pick(/_STREET_LOCALITY_psv\.psv$/)
+	const localityPaths = pick(/_LOCALITY_psv\.psv$/, /_STREET_LOCALITY_psv\.psv$/)
+	const addressPaths = pick(/_ADDRESS_DETAIL_psv\.psv$/)
+
+	const holdout = opts.holdoutPath ? await loadHoldout(opts.holdoutPath) : new Set<string>()
+
+	if (opts.holdoutPath) {
+		progress(`held-out eval keys: ${holdout.size}`)
+	}
+
+	progress(`loading STREET_LOCALITY (${streetPaths.length} files) + LOCALITY (${localityPaths.length})…`)
+
+	const streetMap = await loadMap(streetPaths, "street_locality_pid", (r) => ({
+		name: String(r.street_name ?? ""),
+		type: String(r.street_type_code ?? ""),
+		suffix: String(r.street_suffix_code ?? ""),
+	}))
+
+	const localityMap = await loadMap(localityPaths, "locality_pid", (r) => String(r.locality_name ?? ""))
+	progress(`streets=${streetMap.size.toLocaleString()} localities=${localityMap.size.toLocaleString()}`)
+
+	const reservoir: Array<{ house_number: string; street: string; locality: string; region: string; postcode: string }> =
+		[]
+
+	let seen = 0
+	let heldOut = 0
+
+	for (const p of addressPaths) {
+		const state = (p.match(/\/([A-Z]+)_ADDRESS_DETAIL/) ?? [])[1] ?? ""
+
+		for await (const r of psvObjects(p)) {
+			const numberFirst = String(r.number_first ?? "")
+
+			if (!numberFirst || r.date_retired || !r.postcode) continue
+			const st = streetMap.get(String(r.street_locality_pid ?? ""))
+			const suburbRaw = localityMap.get(String(r.locality_pid ?? ""))
+
+			if (!st?.name || !suburbRaw) continue
+			const street = `${titleCase(st.name)} ${titleCase(st.type)}${st.suffix ? " " + titleCase(st.suffix) : ""}`.trim()
+			const locality = titleCase(suburbRaw)
+			const postcode = String(r.postcode)
+
+			if (holdout.has(gnafHoldoutKey(street, locality, postcode))) {
+				heldOut++
+
+				continue
+			}
+
+			let house = numberFirst + (r.number_first_suffix ? String(r.number_first_suffix) : "")
+
+			if (r.number_last) {
+				house = `${house}-${String(r.number_last)}`
+			}
+
+			if (r.flat_number) {
+				house = `${String(r.flat_number)}/${house}`
+			}
+
+			const tuple = { house_number: house, street, locality, region: state, postcode }
+
+			seen++
+
+			if (reservoir.length < opts.sampleSize) {
+				reservoir.push(tuple)
+			} else {
+				const j = Math.floor(Math.random() * seen)
+
+				if (j < opts.sampleSize) {
+					reservoir[j] = tuple
+				}
+			}
+		}
+
+		progress(`${state}: ${seen.toLocaleString()} valid joinable seen`)
+	}
+
+	const byState: Record<string, number> = {}
+
+	{
+		await using out = createNewlineWriter(opts.out)
+
+		for (const t of reservoir) {
+			await out.write(stringifyJSON(t))
+			byState[t.region] = (byState[t.region] ?? 0) + 1
+		}
+	}
+
+	progress(`wrote ${reservoir.length.toLocaleString()} tuples → ${opts.out}`)
+
+	return { written: reservoir.length, seen, heldOut, byState }
+}