@mailwoman/corpus 7.1.0 → 7.2.1

package/src/adapters/geonames/adapter.ts

@@ -0,0 +1,185 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `geonames`: GeoNames populated-places consumer (https://www.geonames.org/, CC-BY-4.0).
+ *
+ *   GeoNames is a global gazetteer of ~12M features. This adapter ingests the POPULATED PLACES
+ *   (`feature_class = "P"`, excluding historical/abandoned/destroyed variants) from a per-country
+ *   dump file — global locality coverage, including the small towns and villages a coarser admin
+ *   gazetteer (WOF) lacks. It's the cheapest path to broadening the corpus's LOCALE coverage.
+ *
+ *   Input: a per-country tab-separated dump (e.g. `US.txt` from
+ *   `https://download.geonames.org/export/dump/`, 19 columns, no header). Two sibling files in the
+ *   same directory supply human-readable names (downloaded once from the same place):
+ *
+ *   - `admin1CodesASCII.txt` — `<CC>.<admin1_code>` → region name (e.g. `US.VT` → "Vermont").
+ *   - `countryInfo.txt` — ISO alpha-2 → country name (e.g. `US` → "United States"); `#`-commented. If a
+ *       sibling is missing, the corresponding component is simply omitted (graceful degradation).
+ *
+ *   Output: per place, up to two hierarchy variants (mirroring `wof-admin`'s with/without-country
+ *   balance so the model sees both domestic and international order) —
+ *
+ *   1. `{ locality, region }` → "City, Region"
+ *   2. `{ locality, region, country }` → "City, Region, Country" `reconcileComponents` drops any
+ *        component that didn't survive into the rendered `raw`.
+ *
+ *   License: stamped `"CC-BY-4.0"` per row (GeoNames' terms); provenance is the `geonames-<id>` key.
+ */
+
+import { createReadStream, existsSync, readFileSync } from "node:fs"
+import { dirname, join } from "node:path"
+
+import { parse as csvParse } from "csv-parse"
+
+import { stableSourceID } from "../../adapter.ts"
+import { reconcileComponents } from "../../format.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+
+export const GEONAMES_ADAPTER_ID = "geonames"
+export const GEONAMES_DEFAULT_LICENSE = "CC-BY-4.0"
+
+// GeoNames main-table column indices (0-based; see the export README).
+const COL = {
+	geonameid: 0,
+	name: 1,
+	alternatenames: 3,
+	featureClass: 6,
+	featureCode: 7,
+	country: 8,
+	admin1: 10,
+} as const
+
+// Populated-place feature codes that are NOT current real places — skip them.
+const NON_CURRENT_PPL = new Set(["PPLH", "PPLQ", "PPLW", "PPLCH"])
+
+/** Load `admin1CodesASCII.txt` → Map("<CC>.<admin1>" → region name). Empty map if absent. */
+function loadAdmin1(dir: string): Map<string, string> {
+	const map = new Map<string, string>()
+	const fp = join(dir, "admin1CodesASCII.txt")
+
+	if (!existsSync(fp)) return map
+
+	for (const line of readFileSync(fp, "utf8").split("\n")) {
+		if (!line) continue
+		const cols = line.split("\t")
+
+		if (cols[0] && cols[1]) {
+			map.set(cols[0], cols[1])
+		}
+	}
+
+	return map
+}
+
+/** Load `countryInfo.txt` → Map(ISO → country name). Empty map if absent. The file is `#`-commented. */
+function loadCountries(dir: string): Map<string, string> {
+	const map = new Map<string, string>()
+	const fp = join(dir, "countryInfo.txt")
+
+	if (!existsSync(fp)) return map
+
+	for (const line of readFileSync(fp, "utf8").split("\n")) {
+		if (!line || line.startsWith("#")) continue
+		const cols = line.split("\t")
+
+		// ISO(0), ISO3(1), iso-numeric(2), fips(3), Country(4), ...
+		if (cols[0] && cols[4]) {
+			map.set(cols[0], cols[4])
+		}
+	}
+
+	return map
+}
+
+export function createGeonamesAdapter(): CorpusAdapter {
+	return {
+		id: GEONAMES_ADAPTER_ID,
+		defaultLicense: GEONAMES_DEFAULT_LICENSE,
+		description:
+			"GeoNames populated places (CC-BY-4.0) — global locality coverage incl. small towns, with region/country names from the sibling admin1/countryInfo files.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			const dir = dirname(opts.inputPath)
+			const admin1 = loadAdmin1(dir)
+			const countries = loadCountries(dir)
+
+			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
+			const parser = stream.pipe(
+				csvParse({ delimiter: "\t", quote: false, relax_column_count: true, skip_empty_lines: true })
+			)
+
+			let emitted = 0
+
+			try {
+				for await (const rec of parser as AsyncIterable<string[]>) {
+					if (opts.signal?.aborted) break
+
+					if (opts.limit !== undefined && emitted >= opts.limit) break
+
+					if (rec[COL.featureClass] !== "P") continue
+
+					if (NON_CURRENT_PPL.has(rec[COL.featureCode] ?? "")) continue
+
+					const cc = (rec[COL.country] ?? "").trim()
+
+					if (!cc) continue
+
+					if (opts.country && cc !== opts.country) continue
+
+					const locality = (rec[COL.name] ?? "").trim()
+
+					if (!locality) continue
+					const geonameid = (rec[COL.geonameid] ?? "").trim()
+					const region = admin1.get(`${cc}.${(rec[COL.admin1] ?? "").trim()}`)
+					const country = countries.get(cc)
+
+					// Two hierarchy variants (domestic + international order) — but only emit the
+					// distinct ones the available names support.
+					const variants: Array<{ slot: string; comp: CanonicalRow["components"]; raw: string }> = []
+
+					if (region) {
+						variants.push({ slot: "lr", comp: { locality, region }, raw: `${locality}, ${region}` })
+
+						if (country) {
+							variants.push({
+								slot: "lrc",
+								comp: { locality, region, country },
+								raw: `${locality}, ${region}, ${country}`,
+							})
+						}
+					} else if (country) {
+						variants.push({ slot: "lc", comp: { locality, country }, raw: `${locality}, ${country}` })
+					} else {
+						variants.push({ slot: "l", comp: { locality }, raw: locality })
+					}
+
+					for (const v of variants) {
+						if (opts.limit !== undefined && emitted >= opts.limit) break
+						const aligned = reconcileComponents(v.comp, v.raw)
+
+						if (Object.keys(aligned).length === 0) continue
+						const sourceID = geonameid
+							? `${GEONAMES_ADAPTER_ID}-${geonameid}-${v.slot}`
+							: stableSourceID(GEONAMES_ADAPTER_ID, aligned)
+						yield {
+							raw: v.raw,
+							components: aligned,
+							country: cc,
+							source: GEONAMES_ADAPTER_ID,
+							source_id: sourceID,
+							corpus_version: "",
+							license: GEONAMES_DEFAULT_LICENSE,
+						}
+						emitted++
+					}
+				}
+			} finally {
+				stream.destroy()
+			}
+		},
+	}
+}
+
+export const geonamesAdapter = createGeonamesAdapter()

package/src/adapters/geonames-postal/adapter.ts

@@ -0,0 +1,111 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `geonames-postal`: GeoNames postal-code dump consumer (https://www.geonames.org/, CC-BY-4.0).
+ *
+ *   The GeoNames postal export (`https://download.geonames.org/export/zip/<CC>.zip`) is a clean,
+ *   per-country `postcode → place → admin1` table with the place + region NAMES inline (no aux-file
+ *   join needed). It broadens the corpus's postcode→locality→region coverage to ~80 countries, well
+ *   beyond `wof-postalcode`/the coordinate-first table — forward coverage for the multi-locale
+ *   goal.
+ *
+ *   Input: a per-country postal dump (`<CC>.txt`, 12 tab-separated columns, no header): country,
+ *   postcode, place, admin1_name, admin1_code, admin2__, admin3__, lat, lon, accuracy.
+ *
+ *   Output: per row, postcode-FIRST (international) variants — the common order for the non-US
+ *   locales this fills (US postcodes are already covered by TIGER/WOF, which use postcode-LAST):
+ *
+ *   1. `{ postcode, locality }` → "AD100 Canillo"
+ *   2. `{ postcode, locality, region }` → "AD100 Canillo, Canillo" Prefer configuring this adapter for
+ *        non-US countries; for US, the postcode-last sources are the right order. License:
+ *        `"CC-BY-4.0"` per row (attribute "GeoNames").
+ */
+
+import { createReadStream } from "node:fs"
+
+import { parse as csvParse } from "csv-parse"
+
+import { stableSourceID } from "../../adapter.ts"
+import { reconcileComponents } from "../../format.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+
+export const GEONAMES_POSTAL_ADAPTER_ID = "geonames-postal"
+export const GEONAMES_POSTAL_DEFAULT_LICENSE = "CC-BY-4.0"
+
+// GeoNames postal-dump columns (0-based).
+const COL = { country: 0, postcode: 1, place: 2, admin1Name: 3 } as const
+
+export function createGeonamesPostalAdapter(): CorpusAdapter {
+	return {
+		id: GEONAMES_POSTAL_ADAPTER_ID,
+		defaultLicense: GEONAMES_POSTAL_DEFAULT_LICENSE,
+		description:
+			"GeoNames postal codes (CC-BY-4.0) — multi-locale postcode→locality→region, names inline; international postcode-first order.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
+			const parser = stream.pipe(
+				csvParse({ delimiter: "\t", quote: false, relax_column_count: true, skip_empty_lines: true })
+			)
+
+			let emitted = 0
+
+			try {
+				for await (const rec of parser as AsyncIterable<string[]>) {
+					if (opts.signal?.aborted) break
+
+					if (opts.limit !== undefined && emitted >= opts.limit) break
+
+					const cc = (rec[COL.country] ?? "").trim()
+
+					if (!cc) continue
+
+					if (opts.country && cc !== opts.country) continue
+
+					const postcode = (rec[COL.postcode] ?? "").trim()
+					const locality = (rec[COL.place] ?? "").trim()
+
+					if (!postcode || !locality) continue
+					const region = (rec[COL.admin1Name] ?? "").trim()
+
+					// Postcode-first (international) variants. Skip the region variant when admin1 just
+					// repeats the place (common for city-states / micro-admin) to avoid "X X" noise.
+					const variants: Array<{ slot: string; comp: CanonicalRow["components"]; raw: string }> = [
+						{ slot: "pl", comp: { postcode, locality }, raw: `${postcode} ${locality}` },
+					]
+
+					if (region && region.toLowerCase() !== locality.toLowerCase()) {
+						variants.push({
+							slot: "plr",
+							comp: { postcode, locality, region },
+							raw: `${postcode} ${locality}, ${region}`,
+						})
+					}
+
+					for (const v of variants) {
+						if (opts.limit !== undefined && emitted >= opts.limit) break
+						const aligned = reconcileComponents(v.comp, v.raw)
+
+						if (Object.keys(aligned).length < 2) continue
+						yield {
+							raw: v.raw,
+							components: aligned,
+							country: cc,
+							source: GEONAMES_POSTAL_ADAPTER_ID,
+							source_id: `${stableSourceID(GEONAMES_POSTAL_ADAPTER_ID, aligned)}-${v.slot}`,
+							corpus_version: "",
+							license: GEONAMES_POSTAL_DEFAULT_LICENSE,
+						}
+						emitted++
+					}
+				}
+			} finally {
+				stream.destroy()
+			}
+		},
+	}
+}
+
+export const geonamesPostalAdapter = createGeonamesPostalAdapter()

package/src/adapters/gnaf/adapter.ts

@@ -0,0 +1,139 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   G-NAF (Australia) corpus adapter — the parser-teaching half of #208.
+ *
+ *   The model mis-parses Australian addresses in their native postcode-first / house-number-last
+ *   order: it tags a leading 4-digit postcode as a house number (its US/EU prior) and swaps street
+ *   ↔ locality with it. `scripts/eval/au-order-probe.ts` proved this is a word-ORDER coverage gap,
+ *   not capability — the same addresses parse perfectly in canonical order (65% → 87% @25km if the
+ *   parse were order-robust). EU survives the same eval because its postcodes are
+ *   format-distinctive (a hyphenated `26-300` reads as a postcode anywhere); a bare AU `3053` only
+ *   disambiguates by position.
+ *
+ *   So this adapter renders each assembled G-NAF tuple (from {@link ./assemble}) in one of three real
+ *   AU layouts — real-AU canonical (number-first, postcode-trailing), postcode-first,
+ *   locality-first — ROTATED by row index (`i % 3`), so the locality + postcode each land in every
+ *   position across the shard. This is the exact mechanism that fixed #148's v1.9.0 order-overfit
+ *   for the 16 EU locales (`scripts/rerender-overture-multiorder.mjs`, v1.9.1 → shipped v4.13.0);
+ *   AU was simply never in that train (`country_weights` had no AU, and `data_loader.py` excludes
+ *   unlisted countries). Rotating one order per row (rather than emitting all three) keeps this a
+ *   clean single-variable extension of the proven recipe + matches its source-mass structure. The
+ *   corpus aligner BIO-labels each (every component surface form occurs verbatim in `raw`, so
+ *   alignment lands).
+ *
+ *   Input: the assembled component JSONL (one `{house_number,street,locality,region,postcode}` per
+ *   line). Open G-NAF licence — attribute "Geoscape Australia".
+ */
+
+import { TextSpliterator } from "spliterator"
+
+import { stableSourceID } from "../../adapter.ts"
+import { reconcileComponents } from "../../format.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+
+export const GNAF_ADAPTER_ID = "gnaf"
+/** Open G-NAF is freely redistributable with attribution to Geoscape Australia (CC-BY-style). */
+export const GNAF_DEFAULT_LICENSE = "CC-BY-4.0"
+
+interface GNAFTuple {
+	house_number: string
+	street: string
+	locality: string
+	region?: string
+	postcode: string
+}
+
+/**
+ * The address layouts an AU address actually arrives in. The model already handles postcode-TRAILING (canonical); the
+ * two postcode-LEADING forms are the ones it fails, so they carry the lever. We keep the canonical form too so the
+ * retrain doesn't forget it.
+ */
+function renderOrders(c: GNAFTuple): string[] {
+	const region = c.region ? ` ${c.region}` : ""
+
+	return [
+		// real-AU canonical: number-first, street, suburb [state] postcode — "50 Barry Street, Carlton NSW 2000"
+		`${c.house_number} ${c.street}, ${c.locality}${region} ${c.postcode}`,
+		// postcode-first (the dominant failure): "2000 Carlton, Barry Street 50"
+		`${c.postcode} ${c.locality}, ${c.street} ${c.house_number}`,
+		// locality-first: "Carlton, 2000, Barry Street 50"
+		`${c.locality}, ${c.postcode}, ${c.street} ${c.house_number}`,
+	]
+}
+
+/**
+ * Build the G-NAF adapter. `inputPath` is the assembled component JSONL (see {@link ./assemble}); it is country-pinned
+ * to AU regardless of `opts.country` (G-NAF is Australia-only).
+ */
+export function createGNAFAdapter(): CorpusAdapter {
+	return {
+		id: GNAF_ADAPTER_ID,
+		defaultLicense: GNAF_DEFAULT_LICENSE,
+		description:
+			"G-NAF (Australia): assembled address tuples rendered in multiple word orders (canonical / postcode-first / locality-first) — teaches the model AU's postcode-first layout.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			let emitted = 0
+			let idx = 0
+
+			// Input is the assembled component JSONL (one tuple per line). TextSpliterator auto-disposes on
+			// loop completion and on an early `break` (abort / limit), so the old explicit handle teardown is
+			// gone; JSON.parse tolerates a trailing CR on CRLF sources and the `!line.trim()` guard skips blanks.
+			// The render order rotates (i % 3), matching v1.9.1's rerender.
+			for await (const line of TextSpliterator.fromAsync(opts.inputPath)) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				if (!line.trim()) continue
+
+				let t: GNAFTuple
+
+				try {
+					t = JSON.parse(line) as GNAFTuple
+				} catch {
+					continue
+				}
+
+				if (!t.house_number || !t.street || !t.locality || !t.postcode) continue
+
+				const orders = renderOrders(t)
+				const order = idx % orders.length
+				idx++
+				const raw = orders[order]!
+				const components: CanonicalRow["components"] = {
+					house_number: t.house_number,
+					street: t.street,
+					locality: t.locality,
+					postcode: t.postcode,
+				}
+
+				// region (state) rides only the canonical render (order 0); the postcode-leading layouts
+				// omit it (matching the eval's serialization) so it never breaks verbatim alignment.
+				if (order === 0 && t.region) {
+					components.region = t.region
+				}
+
+				const aligned = reconcileComponents(components, raw)
+
+				if (Object.keys(aligned).length === 0) continue
+				yield {
+					raw,
+					components: aligned,
+					country: "AU",
+					locale: "en-AU",
+					source: GNAF_ADAPTER_ID,
+					source_id: `${stableSourceID(GNAF_ADAPTER_ID, aligned)}-o${order}`,
+					corpus_version: "",
+					license: GNAF_DEFAULT_LICENSE,
+				}
+				emitted++
+			}
+		},
+	}
+}
+
+export const gnafAdapter = createGNAFAdapter()

package/src/adapters/gnaf/assemble.ts

@@ -0,0 +1,200 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Assemble a sampled, component-labeled Australian address set from G-NAF (the Geocoded National
+ *   Address File — Geoscape Australia, Open G-NAF licence; any derived artifact must attribute
+ *   "Geoscape Australia"). G-NAF is a relational PSV distribution (~16.9M addresses);
+ *   reconstructing a street address joins three tables — ADDRESS_DETAIL (number, postcode, the
+ *   PIDs) → STREET_LOCALITY (street name + type) → LOCALITY (suburb). State is the per-file prefix
+ *   (ACT/NSW/…).
+ *
+ *   Streaming + in-memory join via the house {@link PSVSpliterator} (pipe-separated; `mode: "object"`
+ *   keys each row by its header) — NOT raw `read_csv` SQL, which a flat-file join doesn't need and
+ *   which the #183–190 cleanup retired. The two lookup tables (STREET_LOCALITY ~765k rows, LOCALITY
+ *   ~16k) fit as Maps; ADDRESS_DETAIL is streamed once and reservoir-sampled, so memory stays
+ *   bounded (the OOM lesson from the Overture ingest).
+ *
+ *   No coordinates: the output feeds the PARSER ({@link ../gnaf/adapter}, #208) — teaching the model
+ *   AU's postcode-first / house-number-last word order, the gap `scripts/eval/au-order-probe.ts`
+ *   pinned (65%→87% if the parse were order-robust). The parser needs the address string +
+ *   component labels, not lat/lon.
+ *
+ *   Output: component tuples as JSONL, consumed by the `gnaf` corpus adapter (which renders them in
+ *   multiple orders + the corpus aligner BIO-labels them). An optional held-out eval set is
+ *   excluded by (street, locality, postcode) so the training shard never overlaps the benchmark.
+ */
+
+import { createWriteStream } from "node:fs"
+import { readdir, readFile } from "node:fs/promises"
+import { join } from "node:path"
+
+import { PSVSpliterator } from "spliterator"
+
+export interface GNAFAssembleOptions {
+	/** G-NAF `Standard` directory (holds the per-state `*_psv.psv` tables). */
+	standardDir: string
+	/** Target sample size (uniform reservoir → population-proportional across states). */
+	sampleSize: number
+	/** Output JSONL path. */
+	out: string
+	/**
+	 * Optional held-out eval JSONL (rows with a `components` field) — its (street,locality,postcode) are excluded.
+	 */
+	holdoutPath?: string
+	/** Progress sink (the CLI passes a setter). */
+	onProgress?: (message: string) => void
+}
+
+export interface GNAFAssembleResult {
+	written: number
+	seen: number
+	heldOut: number
+	byState: Record<string, number>
+}
+
+/** UPPERCASE → Title Case, preserving intra-word apostrophes/hyphens (O'Brien, Coff's Harbour). */
+export function titlecase(s: string): string {
+	return s
+		.toLowerCase()
+		.replace(/(^|[\s'\-/])([a-z])/g, (_m, p: string, c: string) => p + c.toUpperCase())
+		.trim()
+}
+
+/** Holdout/dedup key: a street within a locality+postcode (house-number-agnostic — conservative). */
+export function gnafHoldoutKey(street: string, locality: string, postcode: string): string {
+	return `${street}|${locality}|${postcode}`.toLowerCase()
+}
+
+type Row = Record<string, string | number | undefined>
+async function* psvObjects(path: string): AsyncIterable<Row> {
+	yield* PSVSpliterator.fromAsync(path, { mode: "object", header: true }) as AsyncIterable<Row>
+}
+
+/** Load a small lookup table fully into a Map keyed by `keyCol`. */
+async function loadMap<V>(paths: string[], keyCol: string, pick: (r: Row) => V): Promise<Map<string, V>> {
+	const m = new Map<string, V>()
+
+	for (const p of paths) {
+		for await (const r of psvObjects(p)) {
+			const k = r[keyCol]
+
+			if (k != null && k !== "") {
+				m.set(String(k), pick(r))
+			}
+		}
+	}
+
+	return m
+}
+
+/** Build the held-out key set from an eval JSONL whose rows carry a `components` object. */
+async function loadHoldout(path: string): Promise<Set<string>> {
+	const keys = new Set<string>()
+	const text = await readFile(path, "utf8")
+
+	for (const line of text.split("\n")) {
+		if (!line.trim()) continue
+
+		try {
+			const c = (JSON.parse(line) as { components?: Record<string, string> }).components
+
+			if (c?.street && c?.locality && c?.postcode) {
+				keys.add(gnafHoldoutKey(c.street, c.locality, c.postcode))
+			}
+		} catch {
+			/* skip malformed */
+		}
+	}
+
+	return keys
+}
+
+export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssembleResult> {
+	const progress = opts.onProgress ?? (() => {})
+	const files = await readdir(opts.standardDir)
+	const pick = (re: RegExp, exclude?: RegExp) =>
+		files.filter((f) => re.test(f) && !(exclude && exclude.test(f))).map((f) => join(opts.standardDir, f))
+
+	// `*_LOCALITY_psv.psv` also globs `*_STREET_LOCALITY_psv.psv` — exclude the latter explicitly.
+	const streetPaths = pick(/_STREET_LOCALITY_psv\.psv$/)
+	const localityPaths = pick(/_LOCALITY_psv\.psv$/, /_STREET_LOCALITY_psv\.psv$/)
+	const addressPaths = pick(/_ADDRESS_DETAIL_psv\.psv$/)
+
+	const holdout = opts.holdoutPath ? await loadHoldout(opts.holdoutPath) : new Set<string>()
+
+	if (opts.holdoutPath) {
+		progress(`held-out eval keys: ${holdout.size}`)
+	}
+
+	progress(`loading STREET_LOCALITY (${streetPaths.length} files) + LOCALITY (${localityPaths.length})…`)
+	const streetMap = await loadMap(streetPaths, "STREET_LOCALITY_PID", (r) => ({
+		name: String(r.STREET_NAME ?? ""),
+		type: String(r.STREET_TYPE_CODE ?? ""),
+		suffix: String(r.STREET_SUFFIX_CODE ?? ""),
+	}))
+	const localityMap = await loadMap(localityPaths, "LOCALITY_PID", (r) => String(r.LOCALITY_NAME ?? ""))
+	progress(`streets=${streetMap.size.toLocaleString()} localities=${localityMap.size.toLocaleString()}`)
+
+	const reservoir: Array<{ house_number: string; street: string; locality: string; region: string; postcode: string }> =
+		[]
+	let seen = 0
+	let heldOut = 0
+
+	for (const p of addressPaths) {
+		const state = (p.match(/\/([A-Z]+)_ADDRESS_DETAIL/) ?? [])[1] ?? ""
+
+		for await (const r of psvObjects(p)) {
+			const numberFirst = String(r.NUMBER_FIRST ?? "")
+
+			if (!numberFirst || r.DATE_RETIRED || !r.POSTCODE) continue
+			const st = streetMap.get(String(r.STREET_LOCALITY_PID ?? ""))
+			const suburbRaw = localityMap.get(String(r.LOCALITY_PID ?? ""))
+
+			if (!st?.name || !suburbRaw) continue
+			const street = `${titlecase(st.name)} ${titlecase(st.type)}${st.suffix ? " " + titlecase(st.suffix) : ""}`.trim()
+			const locality = titlecase(suburbRaw)
+			const postcode = String(r.POSTCODE)
+
+			if (holdout.has(gnafHoldoutKey(street, locality, postcode))) {
+				heldOut++
+				continue
+			}
+			let house = numberFirst + (r.NUMBER_FIRST_SUFFIX ? String(r.NUMBER_FIRST_SUFFIX) : "")
+
+			if (r.NUMBER_LAST) {
+				house = `${house}-${String(r.NUMBER_LAST)}`
+			}
+
+			if (r.FLAT_NUMBER) {
+				house = `${String(r.FLAT_NUMBER)}/${house}`
+			}
+			const tuple = { house_number: house, street, locality, region: state, postcode }
+			seen++
+
+			if (reservoir.length < opts.sampleSize) {
+				reservoir.push(tuple)
+			} else {
+				const j = Math.floor(Math.random() * seen)
+
+				if (j < opts.sampleSize) {
+					reservoir[j] = tuple
+				}
+			}
+		}
+		progress(`${state}: ${seen.toLocaleString()} valid joinable seen`)
+	}
+
+	const out = createWriteStream(opts.out)
+	const byState: Record<string, number> = {}
+
+	for (const t of reservoir) {
+		out.write(JSON.stringify(t) + "\n")
+		byState[t.region] = (byState[t.region] ?? 0) + 1
+	}
+	await new Promise<void>((res) => out.end(res))
+	progress(`wrote ${reservoir.length.toLocaleString()} tuples → ${opts.out}`)
+
+	return { written: reservoir.length, seen, heldOut, byState }
+}