@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/tools/postcode-triples.ts

@@ -3,54 +3,24 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Extract `(postcode, locality, region, country)` tuples for the `trailing-region` slice.
+ * Extract `(postcode, locality, region, country)` tuples for the `trailing-region` recipe.
  *
- *   This step used to be a one-off. Its output survived — 17,908 rows at
- *   `$MAILWOMAN_DATA_ROOT/corpus/tuples/trailing-region-structured-tuples.jsonl` — but the code that produced it did
- *   not, so a note describing the join was the only record and it did not match what the databases contain. That is the
- *   reason this file exists: the slice is unbuildable for a new country without it.
- *
- *   ## Two sources, because one route does not reach every country
- *
- *   - **`postalcode-intl.db`** carries a real `parent_id` that resolves in the admin gazetteer. Measured share of rows
- *     with a parent: NL 97.5%, FR 90.7%, DE 66.1%, ES 34.9%, IT 27.4%; of those, 93.8–100% land on a `locality` or
- *     `localadmin`, and the region comes from that place's own ancestry. It is the ONLY slice with this — every
- *     `postalcode-geonames-*` and `postalcode-<cc>-overture.db` row reads `parent_id = 0`.
- *   - **GeoNames postal exports** carry the place and admin1 NAMES in columns 3 and 4, so there is nothing to join.
- *     `mailwoman corpus fetch geonames-postal` puts them on disk.
- *
- *   A nearest-locality-centroid join was measured as the general fallback and REJECTED: scored against the `parent_id`
- *   truth, it agreed NL 81.1% / DE 44.7% / ES 35.1% / FR 29.0% / IT 13.5%. A locality's centroid sits at its middle, so
- *   an edge postcode is routinely nearer a neighbouring town's centroid. Do not reach for it again.
- *
- *   ## The hub cap is PER COUNTRY, because a pooled one is a mixture
- *
- *   A few localities act as catch-all parents — `Schwedt/Oder` claims 9,222 DE postcodes against a DE median of 1. Left
- *   in, a handful of places dominate the slice. But the distribution differs so much by country that one threshold is
- *   not one rule: a p99 pooled across the five countries lands at 522, which keeps 100% of ES and IT, 83% of FR, 49% of
- *   NL and 19% of DE.
- *
- *   So the bound is a QUOTA and not a threshold. A threshold DELETES a locality that exceeds it, which removes exactly
- *   the largest cities — the places a parser most needs to have seen. A quota keeps every locality and bounds how many
- *   of its postcodes ride along, which is the balance the cap was reaching for without the deletion.
- *
- *   ## Placement is data, not a formatting choice
- *
- *   Each tuple is stamped with its country's {@link PostcodePlacement}. The same digits change tag with position, so a
- *   tuple that does not carry its placement teaches whichever convention the recipe happens to default to — see the
- *   recipe's header for the measurement.
+ * Each tuple is stamped with its country's {@link PostcodePlacement}, because the same digits change tag with position.
  */
 
+import { officialLanguagesAlpha3, regionLanguagesAlpha3 } from "@mailwoman/codex/country"
+import { foldName } from "@mailwoman/codex/normalize"
 import { dataRootPath } from "@mailwoman/core/data-root"
-import { pathExists } from "@mailwoman/core/fs/readers"
+import { assertPathExists } from "@mailwoman/core/fs/readers"
+import { wofDatabasePath } from "@mailwoman/resolver-wof-sqlite/paths"
 import type { WOFDatabase } from "@mailwoman/resolver-wof-sqlite/schema"
 import { DatabaseClient } from "@mailwoman/sqlite/client"
-import { join } from "path-ts"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
 import { TSVSpliterator } from "spliterator"
 
-import { GEONAMES_POSTAL_COLUMNS } from "#adapters/geonames-postal/adapter"
+import { GEONAMES_POSTAL_COLUMNS } from "#adapters/geonames/postal/adapter"
+import { escapeSQLString } from "#parquet/duckdb"
 import type { PostcodePlacement } from "#recipes/scaffold"
-import { escapeSQLString } from "#utils/parquet"
 
 /**
  * One extracted tuple, in the shape `readTuples` yields and the recipe consumes.
@@ -58,10 +28,9 @@ import { escapeSQLString } from "#utils/parquet"
 export interface PostcodeTriple {
 	postcode: string
 	/**
-	 * The segment before the locality, when the source has one. A slice whose every row begins with the locality teaches
-	 * that the first named segment IS the locality, and that flips the model's default. Measured on the v4.8.0 candidate,
-	 * which had no such segment — `Ye Three Lords, 27 Minories, London EC3N 1DE` came back `locality: "Ye Three Lords"`
-	 * with the venue and the street both gone, and 11 of its 25 regressions were venue-led rows across seven countries.
+	 * The segment before the locality, when the source has one.
+	 *
+	 * A recipe output whose every row begins with the locality teaches that the first segment is the locality.
 	 */
 	dependentLocality?: string
 	locality: string
@@ -73,21 +42,27 @@ export interface PostcodeTriple {
 }
 
 /**
- * Where a country writes the postcode, and the locale tag its rows carry.
+ * The GeoNames postal column that contains the locality for a country.
  *
- * A country is in this table only when a gauntlet board row ATTESTS its surface. An absent country is not an oversight
- * to be filled in by guessing: extracting it with the wrong placement teaches a convention that country does not use,
- * which is worse than not teaching it at all. `LEADING_POSTCODE_COUNTRIES` in `@mailwoman/neural`'s
- * `placetype-pair-prior.ts` draws the same line for the same reason.
+ * `admin2` is the default.
+ * For the US it is the inverse, because column 3 is the city and admin2 the county,
+ * so taking the default would train counties as cities.
+ */
+export type GeonamesLocalityColumn = "place" | "admin2"
+
+/**
+ * Records where a country writes the postcode.
+ * It also records the locale tag in its rows.
  *
- * AU and ZA are the worked examples of the bar. Both look like obvious additions and neither qualifies: the board's AU
- * rows are bare-city (`Melbourne`, `Sydney, Australia`) and carry no postcode at all, so nothing here says where AU
- * writes it; and ZA's `14 Long St, Green Point, Cape Town, 8001` carries no REGION, which this slice requires — a fact
- * its GeoNames export agrees with, at 100% place and 0% admin1.
+ * A country is in this table only when a gauntlet board row attests its surface, because extracting
+ * an absent country with the wrong placement teaches a convention that country does not use.
  */
-export const POSTCODE_CONVENTIONS: ReadonlyMap<string, { placement: PostcodePlacement; locale: string }> = new Map([
-	// `Rue de l'Église, 3, 29217 Plougonvelin, Bretagne, France` and its siblings — `fr_structured`, `de_structured`,
-	// `es_structured`, `it_structured`, `pt_structured`, `mx_supermanzana`, `nl-op4-p-r-sloterdijk`.
+export const POSTCODE_CONVENTIONS: ReadonlyMap<
+	string,
+	{ placement: PostcodePlacement; locale: string; localityColumn?: GeonamesLocalityColumn }
+> = new Map([
+	// `Rue de l'Église, 3, 29217 Plougonvelin, Bretagne, France` and its siblings in
+	// the structured slices attest this leading placement.
 	["FR", { placement: "leading", locale: "fr-FR" }],
 	["DE", { placement: "leading", locale: "de-DE" }],
 	["ES", { placement: "leading", locale: "es-ES" }],
@@ -95,35 +70,39 @@ export const POSTCODE_CONVENTIONS: ReadonlyMap<string, { placement: PostcodePlac
 	["NL", { placement: "leading", locale: "nl-NL" }],
 	["PT", { placement: "leading", locale: "pt-PT" }],
 	["MX", { placement: "leading", locale: "es-MX" }],
-	// `…, Barcelona 6001, Anzoátegui, Venezuela` — the four `ve_city_postcode_trailing_state` rows. No postcode source
-	// on disk and GeoNames does not publish VE, so this entry currently yields nothing; it is here because the
-	// placement is what makes the absence legible.
+	// No postcode source on disk and GeoNames does not publish VE, so this entry yields no row.
+	// It is here because the placement makes the absence legible.
 	["VE", { placement: "after_locality", locale: "es-VE" }],
-	// `12 MG Road, Indiranagar, Bengaluru, Karnataka 560038, India` — three `in_*` rows, and `AGENTS.md` says the same
-	// ("en-IN is absent BECAUSE the PIN goes last"). The one trailing placement with real data behind it.
+	// `12 MG Road, Indiranagar, Bengaluru, Karnataka 560038, India` — three `in_*` rows,
+	// the one trailing placement with real data behind it.
 	["IN", { placement: "after_region", locale: "en-IN" }],
+	// Attested by the four `us_city_state_postcode` board rows; `localityColumn` keeps it from training counties.
+	["US", { placement: "after_region", locale: "en-US", localityColumn: "place" }],
+	// The two `br_*` rows list locality, region and CEP in that order.
+	// The default `admin2` column is right here, because BR's export writes the
+	// municipality in column 3 and admin2 alike.
+	["BR", { placement: "after_region", locale: "pt-BR" }],
 ])
 
 /**
  * How many postcodes one locality may contribute.
  *
- * A quota rather than a cutoff — see the header. It bounds repetition without deleting a locality: a city with 9,222
- * postcodes contributes this many and stays in the slice.
+ * A quota rather than a cutoff: it bounds repetition without deleting a locality.
  */
 export const DEFAULT_LOCALITY_QUOTA = 24
 
 /**
  * Take at most `quota` tuples per locality, in the order they arrive.
  *
- * Order matters and is the caller's to choose: both readers below walk their source in id / file order, which is stable
- * across runs, so the same quota selects the same rows.
+ * Both readers walk their source in id/file order.
+ * That order is stable across runs, so the same quota selects the same rows.
  */
-export function applyLocalityQuota(
-	triples: readonly PostcodeTriple[],
+export function applyLocalityQuota<T extends { cc: string; locality: string }>(
+	triples: readonly T[],
 	quota: number = DEFAULT_LOCALITY_QUOTA
-): PostcodeTriple[] {
+): T[] {
 	const seen = new Map<string, number>()
-	const kept: PostcodeTriple[] = []
+	const kept: T[] = []
 
 	for (const triple of triples) {
 		const key = `${triple.cc} ${triple.locality}`
@@ -139,62 +118,143 @@ export function applyLocalityQuota(
 }
 
 /**
- * Take at most `budget` tuples per COUNTRY, in the order they arrive.
+ * Take at most `budget` tuples per country, in the order they arrive.
  *
- * A per-locality quota bounds how often one place repeats; it cannot bound a country. IN has 128,152 distinct
- * localities, so even at a quota of ONE it contributes 63,533 rows against 39,790 from the other seven combined — the
- * slice would teach the trailing surface as an Indian fact rather than a general one, and at 103,323 rows it would take
- * 30% of an 8,000-step run's sample budget at three reps per row.
- *
- * Applied AFTER {@link applyLocalityQuota}, so a country's budget is spent on breadth (many localities) rather than on
- * one city's postcode list.
+ * Applied after {@link applyLocalityQuota}, and spent BY region in rounds so a cap
+ * smaller than the source still reaches every region rather than one corner.
+ * `subKey` adds a second round-robin dimension so the budget spreads across `(region, subKey)`.
  */
-export function applyCountryBudget(
-	triples: readonly PostcodeTriple[],
-	budget: number | ReadonlyMap<string, number>
-): PostcodeTriple[] {
-	const spent = new Map<string, number>()
-	const kept: PostcodeTriple[] = []
+export function applyCountryBudget<T extends { cc: string; region?: string }>(
+	triples: readonly T[],
+	budget: number | ReadonlyMap<string, number>,
+	subKey?: (triple: T) => string
+): T[] {
+	const byRegion = new Map<string, T[]>()
+	const order: string[] = []
 
 	for (const triple of triples) {
 		const cap = typeof budget === "number" ? budget : budget.get(triple.cc)
 
 		if (cap === undefined) continue
 
-		const n = spent.get(triple.cc) ?? 0
+		const key = `${triple.cc} ${triple.region ?? ""}${subKey ? ` ${subKey(triple)}` : ""}`
+		const bucket = byRegion.get(key)
+
+		if (bucket) {
+			bucket.push(triple)
+		} else {
+			byRegion.set(key, [triple])
+			order.push(key)
+		}
+	}
 
-		if (n >= cap) continue
+	const spent = new Map<string, number>()
+	const kept: T[] = []
+	const deepest = Math.max(0, ...[...byRegion.values()].map((bucket) => bucket.length))
 
-		spent.set(triple.cc, n + 1)
-		kept.push(triple)
+	for (let round = 0; round < deepest; round++) {
+		for (const key of order) {
+			const triple = byRegion.get(key)![round]
+
+			if (!triple) continue
+
+			const cap = typeof budget === "number" ? budget : budget.get(triple.cc)!
+			const n = spent.get(triple.cc) ?? 0
+
+			if (n >= cap) continue
+
+			spent.set(triple.cc, n + 1)
+			kept.push(triple)
+		}
 	}
 
 	return kept
 }
 
 /**
- * Read triples out of `postalcode-intl.db` by following each postcode's `parent_id` into the admin gazetteer and that
- * place's ancestry to a region.
+ * A place's preferred names in the languages its addresses are written in,
+ * read from the gazetteer's `names` table.
+ */
+interface PreferredNames {
+	/**
+	 * Preferred names in the country's official languages, in the table's order.
+	 */
+	official: readonly string[]
+	/**
+	 * Preferred names in the region's co-official languages.
+	 */
+	coOfficial: readonly string[]
+}
+
+/**
+ * The surfaces a region is written as: its preferred names in the official language(s),
+ * then the region's co-official languages, then the gazetteer's own `spr.name`
+ * (the English exonym), deduplicated with order kept.
+ *
+ * An input that uses only `spr.name` teaches stripped exonyms (`Balearic Islands`, `Cordoba`)
+ * against the forms a user writes.
+ */
+export function regionWrittenForms(sprName: string, names: PreferredNames): string[] {
+	const out: string[] = []
+
+	for (const raw of [...names.official, ...names.coOfficial, ...sprName.split(BILINGUAL_JOINED)]) {
+		const name = raw.trim().replace(PROVINCE_GENERIC, "")
+
+		if (name && !out.includes(name)) {
+			out.push(name)
+		}
+	}
+
+	return out
+}
+
+/**
+ * The separator Who's On First joins a region's two co-official names with.
  *
- * A postcode whose parent does not resolve, or whose parent has no region ancestor, is DROPPED rather than emitted with
- * a blank — the recipe already skips a tuple with no region, and a blank here would hide how much of the source is
- * actually reachable.
+ * Spaces are required on both sides, because a slash with no surrounding spaces appears inside single names.
+ */
+const BILINGUAL_JOINED = / \/ /
+
+/**
+ * The provincial generic Who's On First keeps in a province's Catalan and Asturian
+ * preferred names, dropped so the surface dedupes against the Castilian one.
+ */
+const PROVINCE_GENERIC = /^prov[ií]ncia (?:de |d')/iu
+
+/**
+ * The surface a locality is written as: the official-language preferred name matching
+ * `spr.name` folded, else the first official-language name, else `spr.name`.
+ */
+export function localityWrittenForm(sprName: string, names: PreferredNames): string {
+	const folded = foldName(sprName)
+
+	return names.official.find((name) => foldName(name) === folded) ?? names.official[0] ?? sprName
+}
+
+/**
+ * Read triples out of `postalcode-intl.db` by following each postcode's `parent_id`
+ * into the admin gazetteer and that place's ancestry to a region.
+ *
+ * A postcode whose parent does not resolve, or whose parent has no region ancestor,
+ * is dropped rather than emitted with a blank.
  */
 export async function readTriplesFromParentJoin(
 	countries: readonly string[],
-	options: { postcodeDB?: string; adminDB?: string } = {}
+	options: { postcodeDB?: PathBuilderLike; adminDB?: PathBuilderLike } = {}
 ): Promise<PostcodeTriple[]> {
-	const postcodeDB = options.postcodeDB ?? String(dataRootPath("wof", "postalcode-intl.db"))
-	const adminDB = options.adminDB ?? String(dataRootPath("wof", "admin-global-priority-importance.db"))
+	const postcodeDB = options.postcodeDB ?? wofDatabasePath("postalcode-intl.db")
+	const adminDB = options.adminDB ?? wofDatabasePath("admin-global-priority-importance.db")
 
-	if (!(await pathExists(postcodeDB)) || !(await pathExists(adminDB))) return []
+	await assertPathExists(postcodeDB, `readTriplesFromParentJoin: no postcode database at ${postcodeDB}`)
+	await assertPathExists(adminDB, `readTriplesFromParentJoin: no admin gazetteer at ${adminDB}`)
 
 	using db = new DatabaseClient<WOFDatabase>(adminDB, { readOnly: true })
 
-	db.exec(`ATTACH DATABASE '${escapeSQLString(postcodeDB)}' AS pc`)
+	db.exec(`ATTACH DATABASE '${escapeSQLString(postcodeDB.toString())}' AS pc`)
 
 	const statement = db.prepare(`
-		SELECT p.name AS postcode, a.name AS locality, r.name AS region, c.name AS country, p.country AS cc
+		SELECT p.name AS postcode, a.id AS locality_id, a.name AS locality, r.id AS region_id, r.name AS region,
+		       c.name AS country, p.country AS cc
 		FROM pc.spr p
 		JOIN spr a ON a.id = p.parent_id AND a.placetype IN ('locality', 'localadmin')
 		JOIN ancestors anc ON anc.id = a.id AND anc.ancestor_placetype = 'region'
@@ -205,6 +265,7 @@ export async function readTriplesFromParentJoin(
 		ORDER BY p.id
 	`)
 
+	const surfaces = createSurfaceReader(db)
 	const out: PostcodeTriple[] = []
 
 	for (const cc of countries) {
@@ -212,18 +273,29 @@ export async function readTriplesFromParentJoin(
 
 		if (!convention) continue
 
-		for (const row of statement.all(cc) as Array<Record<string, string>>) {
-			if (!row["postcode"] || !row["locality"] || !row["region"]) continue
-
-			out.push({
-				postcode: row["postcode"],
-				locality: row["locality"],
-				region: row["region"],
-				country: row["country"] ?? "",
-				cc,
-				locale: convention.locale,
-				postcodePlacement: convention.placement,
-			})
+		for (const row of statement.all(cc) as Array<{
+			postcode: string | null
+			locality_id: number
+			locality: string | null
+			region_id: number
+			region: string | null
+			country: string | null
+		}>) {
+			if (!row.postcode || !row.locality || !row.region) continue
+
+			const locality = surfaces.locality(cc, row.locality_id, row.locality)
+
+			for (const region of surfaces.region(cc, row.region_id, row.region)) {
+				out.push({
+					postcode: row.postcode,
+					locality,
+					region,
+					country: row.country ?? "",
+					cc,
+					locale: convention.locale,
+					postcodePlacement: convention.placement,
+				})
+			}
 		}
 	}
 
@@ -231,22 +303,157 @@ export async function readTriplesFromParentJoin(
 }
 
 /**
- * A predicate answering whether a name is a LOCALITY the admin gazetteer knows, for one country.
+ * Reads the surfaces a place is written as against one open admin gazetteer, caching each
+ * place's preferred names by id so a region met once per locality is read once in total.
+ */
+interface SurfaceReader {
+	/**
+	 * Every surface the region is written as, in the order {@link regionWrittenForms} defines.
+	 */
+	region: (cc: string, id: number, sprName: string) => string[]
+	/**
+	 * The one surface the locality is written as.
+	 *
+	 * See {@link localityWrittenForm} for why it is one and not a fan-out.
+	 */
+	locality: (cc: string, id: number, sprName: string) => string
+}
+
+/**
+ * Bind {@link regionWrittenForms} and {@link localityWrittenForm} to an open gazetteer.
  *
- * The parent-join reader gets this for free — its query restricts the parent to `locality`/`localadmin`, so every name
- * it emits is one by construction. The GeoNames reader has no such guarantee, and the gap is large enough to matter:
- * sampling 400 rows per country against the gazetteer, the share of GeoNames place names that name a locality we know
- * is PT 75%, IN 62%, **MX 41%**. The Mexican misses are colonias — `Zona Centro`, `San Fernando INFONAVIT`, `FOVISSSTE
- * 3a Sección` — and a row teaching one of those as `locality` trains the locality/dependent_locality boundary in the
- * wrong direction. Dropping the row instead costs coverage and teaches nothing false, which is the better of the two.
+ * The languages come from the codex instead of the names table's language list.
+ * The table's "preferred" name may use a language not spoken in the region and may refer to the parent.
+ */
+function createSurfaceReader(db: DatabaseClient<WOFDatabase>): SurfaceReader {
+	const preferredStatement = db.prepare(
+		`SELECT language, name FROM names WHERE id = ? AND privateuse = 'preferred' AND language IS NOT NULL ORDER BY language, name`
+	)
+
+	const preferredByID = new Map<number, Map<string, string[]>>()
+
+	const preferredNames = (id: number): Map<string, string[]> => {
+		let byLanguage = preferredByID.get(id)
+
+		if (!byLanguage) {
+			byLanguage = new Map()
+
+			for (const row of preferredStatement.all(id) as Array<{ language: string; name: string }>) {
+				const list = byLanguage.get(row.language) ?? []
+
+				list.push(row.name)
+				byLanguage.set(row.language, list)
+			}
+
+			preferredByID.set(id, byLanguage)
+		}
+
+		return byLanguage
+	}
+
+	const namesIn = (byLanguage: Map<string, string[]>, languages: readonly string[]): string[] =>
+		languages.flatMap((language) => byLanguage.get(language) ?? [])
+
+	return {
+		region(cc, id, sprName) {
+			const officialLanguages = officialLanguagesAlpha3(cc)
+			const official = namesIn(preferredNames(id), officialLanguages)
+			// The co-official table is keyed by the region's name in the first official language.
+			// A region the names table has no such name for is looked up by its `spr.name`,
+			// which for a monolingual country is the same string.
+			const regionLanguages = regionLanguagesAlpha3(cc, official[0] ?? sprName)
+			const coOfficialLanguages = regionLanguages.filter((language) => !officialLanguages.includes(language))
+
+			return regionWrittenForms(sprName, { official, coOfficial: namesIn(preferredNames(id), coOfficialLanguages) })
+		},
+		locality(cc, id, sprName) {
+			const official = namesIn(preferredNames(id), officialLanguagesAlpha3(cc))
+
+			return localityWrittenForm(sprName, { official, coOfficial: [] })
+		},
+	}
+}
+
+/**
+ * One extracted `(locality, region, country)` pair, carrying no postcode.
+ */
+export type AdminPair = Omit<PostcodeTriple, "postcode" | "postcodePlacement">
+
+/**
+ * Read `(locality, region, country)` pairs for a country straight from the
+ * admin gazetteer, with no postcode.
  *
- * Returns a predicate that answers `true` for everything when the gazetteer is not on disk, so a checkout without it
- * builds the same rows it did before rather than silently emitting none.
+ * Produces one pair per region surface, following {@link readTriplesFromParentJoin}.
+ * The reader synthesizes no postcode.
  */
-export async function createKnownLocalityCheck(country: string, adminDB?: string): Promise<(name: string) => boolean> {
-	const path = adminDB ?? String(dataRootPath("wof", "admin-global-priority-importance.db"))
+export async function readPairsFromAdmin(
+	countries: readonly string[],
+	options: { adminDB?: PathBuilderLike; locale?: (cc: string) => string } = {}
+): Promise<AdminPair[]> {
+	const adminDB = options.adminDB ?? wofDatabasePath("admin-global-priority-importance.db")
 
-	if (!(await pathExists(path))) return () => true
+	await assertPathExists(adminDB, `readPairsFromAdmin: no admin gazetteer at ${adminDB}`)
+
+	using db = new DatabaseClient<WOFDatabase>(adminDB, { readOnly: true })
+
+	const statement = db.prepare(`
+		SELECT a.id AS locality_id, a.name AS locality, r.id AS region_id, r.name AS region, c.name AS country
+		FROM spr a
+		JOIN ancestors anc ON anc.id = a.id AND anc.ancestor_placetype = 'region'
+		JOIN spr r ON r.id = anc.ancestor_id
+		JOIN ancestors cnc ON cnc.id = a.id AND cnc.ancestor_placetype = 'country'
+		JOIN spr c ON c.id = cnc.ancestor_id
+		WHERE a.country = ? AND a.placetype IN ('locality', 'localadmin') AND a.is_current != 0 AND a.is_deprecated = 0
+		ORDER BY a.id
+	`)
+
+	const surfaces = createSurfaceReader(db)
+	const out: AdminPair[] = []
+
+	for (const cc of countries) {
+		const locale = options.locale?.(cc) ?? "und"
+
+		for (const row of statement.all(cc) as Array<{
+			locality_id: number
+			locality: string | null
+			region_id: number
+			region: string | null
+			country: string | null
+		}>) {
+			if (!row.locality || !row.region) continue
+
+			const locality = surfaces.locality(cc, row.locality_id, row.locality)
+
+			for (const region of surfaces.region(cc, row.region_id, row.region)) {
+				// A pair whose region repeats its locality adds no signal about the boundary this recipe measures.
+				// The recipe drops that pair, so filtering it here keeps the country budget for rows that remain.
+				if (region === locality) continue
+
+				out.push({ locality, region, country: row.country ?? "", cc, locale })
+			}
+		}
+	}
+
+	return out
+}
+
+/**
+ * A predicate answering whether a name is a locality the admin gazetteer knows, for one country.
+ *
+ * The parent-join reader supplies this field directly.
+ * The GeoNames reader requires this correction because treating a colonia as `locality`
+ * teaches the wrong locality/dependent_locality boundary.
+ *
+ * @throws When the gazetteer is not on disk, because a predicate that accepted every
+ * name would emit unfiltered rows as though the filter had run.
+ */
+export async function createKnownLocalityCheck(
+	country: string,
+	adminDB?: PathBuilderLike
+): Promise<(name: string) => boolean> {
+	const path = adminDB ?? wofDatabasePath("admin-global-priority-importance.db")
+
+	await assertPathExists(path, `createKnownLocalityCheck: no admin gazetteer at ${path}`)
 
 	using db = new DatabaseClient<WOFDatabase>(path, { readOnly: true })
 
@@ -260,78 +467,65 @@ export async function createKnownLocalityCheck(country: string, adminDB?: string
 		}
 	}
 
-	// An empty set means the gazetteer has no localities for this country at all, which is a coverage fact about the
-	// gazetteer rather than a verdict on the source — so check nothing rather than drop everything.
+	// An empty set is a coverage fact about the gazetteer rather than a verdict on the source,
+	// so accept every name rather than dropping everything.
 	if (!names.size) return () => true
 
 	return (name: string) => names.has(name.toLowerCase())
 }
 
 /**
- * Read triples straight out of a GeoNames `<CC>.txt` export — no join, the names are columns 3 and 4.
+ * Read triples straight out of a GeoNames `<CC>.txt` export, with no join.
  *
- * COLUMN 3 IS NOT THE LOCALITY. It is the finest-grained named place for the code, and for `560001` that is `Mahatma
- * Gandhi Road` — a STREET — while the city, `Bengaluru`, is column 5 (admin2). MX is the same shape (`Roma Norte` is a
- * colonia inside `Cuauhtémoc`) and so is PT (`Abrigada` inside `Alenquer`). Reading column 3 as the locality is how the
- * v4.8.0 slice came to teach street names as cities. So `admin2` is the locality, `admin1` the region, and column 3 the
- * DEPENDENT locality — which is also the left context the slice needs.
+ * `admin2` supplies the locality, `admin1` supplies the region and column 3 supplies the dependent locality.
+ * A locality value from column 3 taught `Mahatma Gandhi Road` as a city.
  *
- * WHICH COUNTRIES THIS READER CAN SERVE. It needs `admin2` (the city) and `admin1` (the region), and a country can
- * publish one without the other. Measured 2026-08-23 across the exports on disk:
+ * A country whose export lacks admin1 or admin2 yields zero from this reader.
+ * That result is correct and needs no fallback.
  *
- * | country    | rows      | admin1 | admin2 | usable here                     |
- * | ---------- | --------: | -----: | -----: | ------------------------------- |
- * | PT, MX, IN | 145k–207k | 100%   | 100%   | yes                             |
- * | BD, LK     | 1.3k–1.8k | 100%   | 100%   | yes                             |
- * | PH         | 2,317     | 88%    | 88%    | yes, 88% of rows                |
- * | PK         | 2,563     | 100%   | **0%** | NO — no city column             |
- * | TH         | 903       | 100%   | **1%** | NO — effectively no city column |
- * | ID         | 81,058    | **0%** | 0%     | NO — no region either           |
- * | ZA         | 3,920     | **0%** | —      | NO                              |
- *
- * A country in the NO rows yields zero from this reader, and that is the correct outcome rather than a gap to route
- * around: taking column 3 as the locality is what made the v4.8.0 slice train `Mahatma Gandhi Road` as a city. If one
- * of them is wanted, it needs a city column from somewhere else, not a relaxed mapping.
- *
- * NOT PUBLISHED AT ALL by GeoNames, checked the same day: VE, VN, NP, MM, KH. Those are acquisition questions, and for
- * VE specifically OpenAddresses 404s too — see the arc retrospective.
- *
- * Three source properties a caller cannot see from a row count, all handled here. Hyphen-format countries publish each
- * code TWICE (`3750-000` and `3750000`, exactly 2.00× for PT and PL), so the first surface of a code wins and its twin
- * is dropped. Some countries populate the place but not admin1 — ZA is 100% place, 0% region — which yields nothing
- * this slice can use, so those rows are dropped rather than emitted with a blank region. And the "place name" is often
- * a SUB-locality, which {@link createKnownLocalityCheck} filters.
+ * Hyphen-format countries publish each code twice, so the first surface of a code wins.
  */
 export async function readTriplesFromGeonames(
 	country: string,
-	path: string,
+	path: PathBuilderLike,
 	countryName: string,
 	options: { isKnownLocality?: (name: string) => boolean } = {}
 ): Promise<PostcodeTriple[]> {
 	const convention = POSTCODE_CONVENTIONS.get(country)
 
-	if (!convention || !(await pathExists(path))) return []
+	if (!convention) return []
+
+	await assertPathExists(path, `readTriplesFromGeonames: no GeoNames postal export at ${path}`)
 
 	const isKnownLocality = options.isKnownLocality ?? (await createKnownLocalityCheck(country))
 	const out: PostcodeTriple[] = []
 	const seen = new Set<string>()
 
+	// Which column the city sits in is a property of the country's export —
+	// see {@link GeonamesLocalityColumn}; the other column becomes the dependent locality,
+	// a sub-locality for PT/MX/IN and a county for the US.
+	const localityIsPlace = convention.localityColumn === "place"
+
 	for await (const cells of TSVSpliterator.fromAsync(path, { header: false }) as AsyncIterable<string[]>) {
-		const postcode = (cells[GEONAMES_POSTAL_COLUMNS.postcode] ?? "").trim()
-		const dependentLocality = (cells[GEONAMES_POSTAL_COLUMNS.place] ?? "").trim()
-		const locality = (cells[GEONAMES_POSTAL_COLUMNS.admin2Name] ?? "").trim()
-		const region = (cells[GEONAMES_POSTAL_COLUMNS.admin1Name] ?? "").trim()
+		const postcode = cells[GEONAMES_POSTAL_COLUMNS.postcode] ?? ""
+		const place = cells[GEONAMES_POSTAL_COLUMNS.place] ?? ""
+		const admin2 = cells[GEONAMES_POSTAL_COLUMNS.admin2Name] ?? ""
+		const locality = localityIsPlace ? place : admin2
+		const dependentLocality = localityIsPlace ? "" : place
+		const region = cells[GEONAMES_POSTAL_COLUMNS.admin1Name] ?? ""
 
 		if (!postcode || !locality || !region) continue
 
-		// The check applies to the LOCALITY — admin2 — not to the fine-grained name, which is expected to be a street or
-		// a colonia and is emitted as the dependent locality rather than dropped.
+		// The check applies to the locality.
+		// For PT/MX/IN, the other column may hold a street or colonia.
+		// The reader emits that value as the dependent locality instead of dropping it.
 		if (!isKnownLocality(locality)) continue
 
-		// A dependent locality that merely repeats its parent teaches a doubled segment, not a boundary.
+		// A dependent locality that merely repeats its parent teaches a doubled segment rather than a boundary.
 		const dep = dependentLocality && dependentLocality !== locality ? dependentLocality : ""
 
-		// The bare twin of a punctuated code carries no new fact, and keeping both doubles the country's weight.
+		// The bare twin of a punctuated code adds no new fact.
+		// Both entries would double the country's weight.
 		const key = `${postcode.replaceAll("-", "")} ${locality} ${dep}`
 
 		if (seen.has(key)) continue
@@ -356,8 +550,9 @@ export async function readTriplesFromGeonames(
 /**
  * Resolve a GeoNames export path under the standard fetch out-root.
  */
-export function geonamesPostalPath(country: string, sourcesRoot?: string): string {
-	const root = sourcesRoot ?? String(dataRootPath("corpus", "sources"))
-
-	return join(root, "geonames-postal", `${country.toUpperCase()}.txt`)
+export function geonamesPostalPath(country: string, sourcesRoot?: PathBuilderLike): PathBuilder {
+	return PathBuilder.from(sourcesRoot ?? dataRootPath("corpus", "sources"))(
+		"geonames-postal",
+		`${country.toUpperCase()}.txt`
+	)
 }