@mailwoman/corpus 9.4.0 → 10.0.0

package/lib/au/adapters/gnaf/adapter.ts

@@ -0,0 +1,152 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   G-NAF (Australia) corpus adapter — the parser-teaching half of #208.
+ *
+ *   The model mis-parses Australian addresses in their native postcode-first / house-number-last
+ *   order: it tags a leading 4-digit postcode as a house number (its US/EU prior) and swaps street
+ *   ↔ locality with it. `scripts/eval/au-order-probe.ts` proved this is a word-ORDER coverage gap,
+ *   not capability — the same addresses parse perfectly in canonical order (65% → 87% @25km if the
+ *   parse were order-robust). EU survives the same eval because its postcodes are
+ *   format-distinctive (a hyphenated `26-300` reads as a postcode anywhere); a bare AU `3053` only
+ *   disambiguates by position.
+ *
+ *   So this adapter renders each assembled G-NAF tuple (from {@link ./assemble}) in one of three real
+ *   AU layouts — real-AU canonical (number-first, postcode-trailing), postcode-first,
+ *   locality-first — ROTATED by row index (`i % 3`), so the locality + postcode each land in every
+ *   position across the slice. This is the exact mechanism that fixed #148's v1.9.0 order-overfit
+ *   for the 16 EU locales (`scripts/rerender-overture-multiorder.mjs`, v1.9.1 → shipped v4.13.0);
+ *   AU was simply never in that train (`country_weights` had no AU, and `data_loader.py` excludes
+ *   unlisted countries). Rotating one order per row (rather than emitting all three) keeps this a
+ *   clean single-variable extension of the proven recipe + matches its source-mass structure. The
+ *   corpus aligner BIO-labels each (every component surface form occurs verbatim in `raw`, so
+ *   alignment lands).
+ *
+ *   Input: the assembled component JSONL (one `{house_number,street,locality,region,postcode}` per
+ *   line). Open G-NAF licence — attribute "Geoscape Australia".
+ */
+
+import { componentsPresentIn } from "@mailwoman/codex/address-format"
+import { tryParsingJSON } from "@mailwoman/core/json"
+import { TextSpliterator } from "spliterator"
+
+import { stableSourceID } from "#adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const GNAF_ADAPTER_ID = "gnaf"
+/**
+ * Open G-NAF is freely redistributable with attribution to Geoscape Australia (CC-BY-style).
+ */
+export const GNAF_DEFAULT_LICENSE = "CC-BY-4.0"
+
+interface GNAFTuple {
+	house_number: string
+	street: string
+	locality: string
+	region?: string
+	postcode: string
+}
+
+/**
+ * The address layouts an AU address actually arrives in. The model already handles postcode-TRAILING (canonical); the
+ * two postcode-LEADING forms are the ones it fails, so they carry the change. We keep the canonical form too so the
+ * retrain doesn't forget it.
+ */
+function renderOrders(c: GNAFTuple): string[] {
+	const region = c.region ? ` ${c.region}` : ""
+
+	return [
+		// real-AU canonical: number-first, street, suburb [state] postcode — "50 Barry Street, Carlton NSW 2000"
+		`${c.house_number} ${c.street}, ${c.locality}${region} ${c.postcode}`,
+		// postcode-first (the dominant failure): "2000 Carlton, Barry Street 50"
+		`${c.postcode} ${c.locality}, ${c.street} ${c.house_number}`,
+		// locality-first: "Carlton, 2000, Barry Street 50"
+		`${c.locality}, ${c.postcode}, ${c.street} ${c.house_number}`,
+	]
+}
+
+/**
+ * Build the G-NAF adapter. `inputPath` is the assembled component JSONL (see {@link ./assemble}); it is country-pinned
+ * to AU regardless of `opts.country` (G-NAF is Australia-only).
+ */
+export function createGNAFAdapter(): CorpusAdapter {
+	return {
+		id: GNAF_ADAPTER_ID,
+		defaultLicense: GNAF_DEFAULT_LICENSE,
+		description:
+			"G-NAF (Australia): assembled address tuples rendered in multiple word orders (canonical / postcode-first / locality-first) — teaches the model AU's postcode-first layout.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			let emitted = 0
+			let idx = 0
+
+			// Input is the assembled component JSONL (one tuple per line). TextSpliterator auto-disposes on
+			// loop completion and on an early `break` (abort / limit), so the old explicit handle teardown is
+			// gone; the parse tolerates a trailing CR on CRLF sources and the `!line.trim()` guard skips blanks.
+			// The render order rotates (i % 3), matching v1.9.1's rerender.
+			for await (const line of TextSpliterator.fromAsync(opts.inputPath)) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				if (!line.trim()) continue
+
+				const t = tryParsingJSON<GNAFTuple>(line)
+
+				if (t === null) continue
+
+				if (!t.house_number || !t.street || !t.locality || !t.postcode) continue
+
+				const orders = renderOrders(t)
+				const order = idx % orders.length
+
+				idx++
+				const raw = orders[order]!
+
+				const components: CanonicalRow["components"] = {
+					house_number: t.house_number,
+					street: t.street,
+					locality: t.locality,
+					postcode: t.postcode,
+				}
+
+				// region (state) rides only the canonical render (order 0); the postcode-leading layouts
+				// omit it (matching the eval's serialization) so it never breaks verbatim alignment.
+				if (order === 0 && t.region) {
+					components.region = t.region
+				}
+
+				// `raw` here is one of three DELIBERATE word orders, two of which no layout prints — the postcode-leading
+				// forms this adapter exists to teach. So the question is containment against a string this adapter built,
+				// not what a layout would have printed.
+				const aligned = componentsPresentIn(components, raw)
+
+				if (!Object.keys(aligned).length) continue
+
+				yield {
+					raw,
+					components: aligned,
+					country: "AU",
+					locale: "en-AU",
+					source: GNAF_ADAPTER_ID,
+					source_id: `${stableSourceID(GNAF_ADAPTER_ID, aligned)}-o${order}`,
+					corpus_version: "",
+					license: GNAF_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const gnafAdapter = createGNAFAdapter()

package/lib/au/adapters/gnaf/assemble.ts

@@ -0,0 +1,225 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Assemble a sampled, component-labeled Australian address set from G-NAF (the Geocoded National
+ *   Address File — Geoscape Australia, Open G-NAF licence; any derived artifact must attribute
+ *   "Geoscape Australia"). G-NAF is a relational PSV distribution (~16.9M addresses);
+ *   reconstructing a street address joins three tables — ADDRESS_DETAIL (number, postcode, the
+ *   PIDs) → STREET_LOCALITY (street name + type) → LOCALITY (suburb). State is the per-file prefix
+ *   (ACT/NSW/…).
+ *
+ *   Streaming + in-memory join via the house {@link PSVSpliterator} (pipe-separated; header names key each row) — NOT
+ *   raw `read_csv` SQL, which a flat-file join doesn't need and
+ *   which the #183–190 cleanup retired. The two lookup tables (STREET_LOCALITY ~765k rows, LOCALITY
+ *   ~16k) fit as Maps; ADDRESS_DETAIL is streamed once and reservoir-sampled, so memory stays
+ *   bounded (the OOM lesson from the Overture ingest).
+ *
+ *   No coordinates: the output feeds the PARSER ({@link ../gnaf/adapter}, #208) — teaching the model
+ *   AU's postcode-first / house-number-last word order, the gap `scripts/eval/au-order-probe.ts`
+ *   pinned (65%→87% if the parse were order-robust). The parser needs the address string +
+ *   component labels, not lat/lon.
+ *
+ *   Output: component tuples as JSONL, consumed by the `gnaf` corpus adapter (which renders them in
+ *   multiple orders + the corpus aligner BIO-labels them). An optional held-out eval set is
+ *   excluded by (street, locality, postcode) so the training slice never overlaps the benchmark.
+ */
+
+import { openWriteStream } from "@mailwoman/core/fs/streams"
+import { tryParsingJSON } from "@mailwoman/core/json"
+import { join } from "path-ts"
+import { PSVSpliterator, TextSpliterator } from "spliterator"
+import { Globerator } from "spliterator/node/fs"
+
+export interface GNAFAssembleOptions {
+	/**
+	 * G-NAF `Standard` directory (holds the per-state `*_psv.psv` tables).
+	 */
+	standardDir: string
+	/**
+	 * Target sample size (uniform reservoir → population-proportional across states).
+	 */
+	sampleSize: number
+	/**
+	 * Output JSONL path.
+	 */
+	out: string
+	/**
+	 * Optional held-out eval JSONL (rows with a `components` field) — its (street,locality,postcode) are excluded.
+	 */
+	holdoutPath?: string
+	/**
+	 * Progress sink (the CLI passes a setter).
+	 */
+	onProgress?: (message: string) => void
+}
+
+export interface GNAFAssembleResult {
+	written: number
+	seen: number
+	heldOut: number
+	byState: Record<string, number>
+}
+
+/**
+ * UPPERCASE → Title Case, preserving intra-word apostrophes/hyphens (O'Brien, Coff's Harbour).
+ */
+export function titlecase(s: string): string {
+	return s
+		.toLowerCase()
+		.replaceAll(/(^|[\s'\-/])([a-z])/g, (_m, p: string, c: string) => p + c.toUpperCase())
+		.trim()
+}
+
+/**
+ * Holdout/dedup key: a street within a locality+postcode (house-number-agnostic — conservative).
+ */
+export function gnafHoldoutKey(street: string, locality: string, postcode: string): string {
+	return `${street}|${locality}|${postcode}`.toLowerCase()
+}
+
+type Row = Record<string, string | number | undefined>
+
+async function* psvObjects(path: string): AsyncIterable<Row> {
+	yield* PSVSpliterator.fromAsync(path) as AsyncIterable<Row>
+}
+
+/**
+ * Load a small lookup table fully into a Map keyed by `keyCol`.
+ */
+async function loadMap<V>(paths: string[], keyCol: string, pick: (r: Row) => V): Promise<Map<string, V>> {
+	const m = new Map<string, V>()
+
+	for (const p of paths) {
+		for await (const r of psvObjects(p)) {
+			const k = r[keyCol]
+
+			if (k != null && k !== "") {
+				m.set(String(k), pick(r))
+			}
+		}
+	}
+
+	return m
+}
+
+/**
+ * Build the held-out key set from an eval JSONL whose rows carry a `components` object.
+ */
+async function loadHoldout(path: string): Promise<Set<string>> {
+	const keys = new Set<string>()
+
+	for await (const line of TextSpliterator.fromAsync(path)) {
+		if (!line.trim()) continue
+
+		const c = tryParsingJSON<{ components?: Record<string, string> }>(line)?.components
+
+		if (c?.street && c?.locality && c?.postcode) {
+			keys.add(gnafHoldoutKey(c.street, c.locality, c.postcode))
+		}
+	}
+
+	return keys
+}
+
+export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssembleResult> {
+	const progress = opts.onProgress ?? (() => {})
+	const files = await Globerator.from("*", { cwd: opts.standardDir, absolute: false }).toArray()
+
+	const pick = (re: RegExp, exclude?: RegExp) =>
+		files.filter((f) => re.test(f) && !(exclude && exclude.test(f))).map((f) => join(opts.standardDir, f))
+
+	// `*_LOCALITY_psv.psv` also globs `*_STREET_LOCALITY_psv.psv` — exclude the latter explicitly.
+	const streetPaths = pick(/_STREET_LOCALITY_psv\.psv$/)
+	const localityPaths = pick(/_LOCALITY_psv\.psv$/, /_STREET_LOCALITY_psv\.psv$/)
+	const addressPaths = pick(/_ADDRESS_DETAIL_psv\.psv$/)
+
+	const holdout = opts.holdoutPath ? await loadHoldout(opts.holdoutPath) : new Set<string>()
+
+	if (opts.holdoutPath) {
+		progress(`held-out eval keys: ${holdout.size}`)
+	}
+
+	progress(`loading STREET_LOCALITY (${streetPaths.length} files) + LOCALITY (${localityPaths.length})…`)
+
+	const streetMap = await loadMap(streetPaths, "street_locality_pid", (r) => ({
+		name: String(r.street_name ?? ""),
+		type: String(r.street_type_code ?? ""),
+		suffix: String(r.street_suffix_code ?? ""),
+	}))
+
+	const localityMap = await loadMap(localityPaths, "locality_pid", (r) => String(r.locality_name ?? ""))
+	progress(`streets=${streetMap.size.toLocaleString()} localities=${localityMap.size.toLocaleString()}`)
+
+	const reservoir: Array<{ house_number: string; street: string; locality: string; region: string; postcode: string }> =
+		[]
+
+	let seen = 0
+	let heldOut = 0
+
+	for (const p of addressPaths) {
+		const state = (p.match(/\/([A-Z]+)_ADDRESS_DETAIL/) ?? [])[1] ?? ""
+
+		for await (const r of psvObjects(p)) {
+			const numberFirst = String(r.number_first ?? "")
+
+			if (!numberFirst || r.date_retired || !r.postcode) continue
+			const st = streetMap.get(String(r.street_locality_pid ?? ""))
+			const suburbRaw = localityMap.get(String(r.locality_pid ?? ""))
+
+			if (!st?.name || !suburbRaw) continue
+			const street = `${titlecase(st.name)} ${titlecase(st.type)}${st.suffix ? " " + titlecase(st.suffix) : ""}`.trim()
+			const locality = titlecase(suburbRaw)
+			const postcode = String(r.postcode)
+
+			if (holdout.has(gnafHoldoutKey(street, locality, postcode))) {
+				heldOut++
+
+				continue
+			}
+
+			let house = numberFirst + (r.number_first_suffix ? String(r.number_first_suffix) : "")
+
+			if (r.number_last) {
+				house = `${house}-${String(r.number_last)}`
+			}
+
+			if (r.flat_number) {
+				house = `${String(r.flat_number)}/${house}`
+			}
+
+			const tuple = { house_number: house, street, locality, region: state, postcode }
+
+			seen++
+
+			if (reservoir.length < opts.sampleSize) {
+				reservoir.push(tuple)
+			} else {
+				const j = Math.floor(Math.random() * seen)
+
+				if (j < opts.sampleSize) {
+					reservoir[j] = tuple
+				}
+			}
+		}
+
+		progress(`${state}: ${seen.toLocaleString()} valid joinable seen`)
+	}
+
+	const out = openWriteStream(opts.out)
+	const byState: Record<string, number> = {}
+
+	for (const t of reservoir) {
+		out.write(JSON.stringify(t) + "\n")
+		byState[t.region] = (byState[t.region] ?? 0) + 1
+	}
+
+	await new Promise<void>((res) => {
+		out.end(res)
+	})
+
+	progress(`wrote ${reservoir.length.toLocaleString()} tuples → ${opts.out}`)
+
+	return { written: reservoir.length, seen, heldOut, byState }
+}