@mailwoman/resolver-wof-sqlite 10.0.0 → 10.2.0

package/lib/geonames/aliases.ts

@@ -2,63 +2,26 @@
  * @copyright Sister Software
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
- *
- *   #743/#193 — fold GeoNames bilingual / alt-language place-names into a WOF/unified admin DB as
- *   first-class places. The hard-filter recall gap on bilingual countries (the address says
- *   "Karjaa" but the table holds the Swedish "Karis") is missing alt-LANGUAGE names, not missing
- *   places: the WOF/Overture `names` carried only the primary, so the candidate build's Latin-alias
- *   explode (build-candidate pass 2) had nothing to widen. GeoNames' per-country dump carries the
- *   variants inline (the Karis row's `alternatenames` includes "Karjaa").
- *
- *   For each POPULATED place (feature class `P`) this writes an `spr` row + `names` rows (primary +
- *   Latin alt-names) + population into the SAME tables the WOF/Overture paths use — synthetic ids
- *   based at {@link GEONAMES_ID_BASE} so the three sources never collide. The caller then rebuilds
- *   `place_search` ({@link buildPlaceSearchFTS} with `drop: true`) so the candidate build carries
- *   Karjaa↔Karis. Proven (FI hard-resolve 69.5 → 85.8 %, coverage 74.4 → 94.0 %); duplicating a
- *   place already held under another source is benign — the rows share name_key+coord and the
- *   candidate ranking dedupes by score.
- *
- *   This is the package home so the canonical `build-unified-wof --geonames-countries`, the
- *   pipeline fold (`gazetteer-pipeline/admin/fold-geonames`), AND the `mailwoman gazetteer` commands all share
- *   ONE implementation. GeoNames dump = `download.geonames.org/export/dump/<CC>.zip` → `<CC>.txt`
- *   (TSV).
  */
 
 import { isOfficialLanguage } from "@mailwoman/codex/country"
-import { pathExists } from "@mailwoman/core/fs/readers"
+import { readUnquotedTSV } from "@mailwoman/core/fs/delimited"
+import { pathExists } from "@mailwoman/core/fs/readers/stat"
 import { GEONAMES_ID_BASE, GEONAMES_POSTAL_ID_BASE } from "@mailwoman/core/resolver/synthetic-id-ranges"
 import type { DatabaseClient } from "@mailwoman/sqlite/client"
-import { join, type PathBuilderLike } from "path-ts"
-import { TSVSpliterator } from "spliterator"
+import { type PathBuilderLike, resolvePathBuilder } from "path-ts"
 
 import type { WOFDatabase } from "#schema"
 
-/**
- * The four tables the alias fold writes into the range it owns. The purge and the ingest must agree on this list — a
- * table written but not purged is exactly the #1514 defect.
- */
 const FOLD_OWNED_TABLES = ["spr", "names", "place_population", "ancestors"] as const
 
 /**
- * Clear everything the alias fold owns — `[GEONAMES_ID_BASE, GEONAMES_POSTAL_ID_BASE)` across {@link
- * FOLD_OWNED_TABLES}
- * — so a fold's output is a function of its country list and dumps ALONE, never of what the DB happened to hold first.
- * Returns the rows removed per table.
- *
- * This exists because the synthetic id is a POSITION (`GEONAMES_ID_BASE + n`, counted across the country list in
- * order), not a derivation from the geonameid: change the list, its order, or a dump's contents and every subsequent id
- * shifts. Without the purge, a second fold overwrote the `spr` PREFIX it reached (`INSERT OR REPLACE`) while its
- * `names`/`ancestors` rows merely appended (bare `INSERT`) and `place_population` skipped unpopulated places entirely —
- * so the previous run's names, ancestry and populations stayed bound to ids now describing other places.
+ * Deletes every row the GeoNames alias fold owns, with ids in
+ * `[GEONAMES_ID_BASE, GEONAMES_POSTAL_ID_BASE)`, and returns the count removed per table.
  *
- * Measured on the live 2026-08-05 artifact: a 14-country re-fold over the 161-country fold baked into
- * `admin-global-priority.db` put 522,184 of 2,110,096 name rows in the range (24.7 %) on a place from a different
- * country. Gaborone/BW at id 9000000121151 became Aichegg/AT and kept all 26 of its names; Kinshasa's 16,000,000
- * population landed on a Lithuanian hamlet.
- *
- * The upper bound is NOT the end of the id space. Each later extract owns a range above this one — GeoNames-postal @
- * 9.5e12, NL-PC6 @ 9.6e12, Code-Point @ 9.7e12, NI @ 9.8e12 — and a purge that ran past {@link GEONAMES_POSTAL_ID_BASE}
- * would delete them.
+ * Fold ids are assigned sequentially, so a re-fold without this purge would attach
+ * stale names, ancestors and populations to other places.
+ * The upper bound leaves the postal and postcode id ranges above it untouched.
  */
 export function purgeGeonamesAliasRange<DB>(db: DatabaseClient<DB>): Record<string, number> {
 	const removed: Record<string, number> = {}
@@ -75,59 +38,61 @@ export function purgeGeonamesAliasRange<DB>(db: DatabaseClient<DB>): Record<stri
 }
 
 /**
- * Per-country progress for the ingest — one event per country dump processed (or skipped).
+ * One progress event per country dump that the ingest processed or skipped.
  */
 export interface GeonamesIngestProgress {
 	/**
-	 * ISO 3166-1 alpha-2 code.
+	 * The ISO 3166-1 alpha-2 country code.
 	 */
 	country: string
+
 	/**
-	 * Populated places ingested from this country's dump (0 when skipped).
+	 * The number of populated places ingested from this country's dump, or 0 when it was skipped.
 	 */
 	places: number
+
 	/**
-	 * True when the country's `<CC>.txt` dump was missing — the country is skipped, not fatal.
+	 * Whether the country's `<CC>.txt` dump was missing.
+	 *
+	 * A missing dump skips the country without failing the ingest.
 	 */
 	skipped: boolean
+
+	/**
+	 * The number of alternate names that the alias filter rejected.
+	 *
+	 * It distinguishes a sparse source from aliases that the fold discarded.
+	 */
+	aliasesRefused: number | null
 }
 
 /**
- * One spelling's language attribution, as reconciled across every alternate-names row that carries it.
+ * Formats a GeoNames fold progress event as one log line.
  */
+export function formatGeonamesIngestProgress(event: GeonamesIngestProgress, missingFile?: PathBuilderLike): string {
+	if (event.skipped) {
+		return `${event.country}: ${missingFile ? `${missingFile} missing` : "dump missing"} — download from download.geonames.org/export/dump/${event.country}.zip; skipped`
+	}
+
+	return `${event.country}: ${event.places.toLocaleString()} places, ${(event.aliasesRefused ?? 0).toLocaleString()} alt-names refused`
+}
+
 interface V2Alias {
 	language: string
 	privateuse: string
 	official: number
 }
 
-/**
- * Parse a country's alternateNamesV2 file into `geonameid -> spelling -> attribution`, restricted to the populated
- * places (`P`) in `wanted`.
- */
 async function parseAlternateNamesV2(
-	v2File: string,
+	v2File: PathBuilderLike,
 	cc: string,
 	wanted: ReadonlySet<number>
 ): Promise<Map<number, Map<string, V2Alias>>> {
 	const v2 = new Map<number, Map<string, V2Alias>>()
 
-	// V2 columns (0-indexed): 1 geonameid, 2 isolanguage, 3 name, 4 isPreferredName, 5 isShortName,
-	// 6 isColloquial, 7 isHistoric, 8 from, 9 to.
-	//
-	// Two passes, because historic-ness is a fact about the NAME, not the row: GeoNames splits one
-	// spelling across rows — Malabo carries "Santa Isabel" as (es, unflagged) AND as (no-language,
-	// isHistoric=1, to=1973). Officialness must see the flags from EVERY row for the spelling, or the
-	// colonial-era name sails through on the language-tagged row (the #936 review's Malabo finding).
-	// Do NOT condition on isPreferredName instead — it's sparse annotation, not a signal (Turku's sv "Åbo"
-	// is unflagged; FI has 1,746 flags across the whole dump).
-	//
-	// Both passes STREAM the file rather than sharing one materialized array: NO's V2 dump is 33 MB,
-	// and a second read off the page cache costs less than holding half a million line strings.
-	// `header: false` — the dump is headerless.
 	const historicNames = new Set<string>()
 
-	for await (const f of TSVSpliterator.fromAsync(v2File, { header: false })) {
+	for await (const f of readUnquotedTSV(v2File)) {
 		if (f[6] === "1" || f[7] === "1" || (f[9] ?? "").trim() !== "") {
 			const alt = (f[3] ?? "").trim()
 
@@ -137,13 +102,12 @@ async function parseAlternateNamesV2(
 		}
 	}
 
-	for await (const f of TSVSpliterator.fromAsync(v2File, { header: false })) {
+	for await (const f of readUnquotedTSV(v2File)) {
 		const gid = Number(f[1])
 
 		if (!wanted.has(gid)) continue
 		const lang = f[2] ?? ""
 
-		// ISO 639 codes are 2-3 letters; GeoNames' pseudo-codes (post, link, iata, wkdt, …) are 4+.
 		if (!/^[a-z]{2,3}$/.test(lang)) continue
 		const alt = (f[3] ?? "").trim()
 
@@ -176,12 +140,12 @@ async function parseAlternateNamesV2(
 }
 
 /**
- * Fold the GeoNames `P`-class places (+ their Latin alt-names) for `countries` into `db`'s `spr` / `names` /
- * `place_population` tables. Returns the total places ingested.
+ * Purges the fold's id range, then adds GeoNames `P`-class places and their
+ * alternate names for `countries` to `db`.
+ * It returns the number of places ingested.
  *
- * `onProgress` receives one event per country (default: a stderr line, matching the build scripts' legacy output). The
- * caller MUST rebuild `place_search` afterward (`buildPlaceSearchFTS(db, { drop: true })`) for the new names to reach
- * the candidate build's alias pass.
+ * The caller must then rebuild `place_search` with `buildPlaceSearchFTS(db, { drop: true })`
+ * to make the new names searchable.
  */
 export async function ingestGeonamesAliases(
 	db: DatabaseClient<WOFDatabase>,
@@ -189,27 +153,11 @@ export async function ingestGeonamesAliases(
 	geonamesDir: PathBuilderLike,
 	onProgress?: (event: GeonamesIngestProgress) => void,
 	opts?: {
-		/**
-		 * #267: the countries for which to ALSO fold the GeoNames A-class admin (PCLI country + ADM1 regions) and link each
-		 * locality's `parent_id` + ancestry chain (locality → region → country). PER-COUNTRY because a country that already
-		 * carries WOF admin would double up — pass only the ZERO-COVERAGE gap countries (the coverage-expansion targets),
-		 * never the EU alias set. Without admin, a gap country's localities are orphans (`parent_id=-1`, no ancestors), so
-		 * `parentID` scoping and adminCoherence can't reach them and "Tbilisi, GE" can't resolve.
-		 */
 		adminForCountries?: ReadonlySet<string>
-		/**
-		 * #936: directory of per-country alternateNamesV2 dumps
-		 * (`download.geonames.org/export/dump/alternatenames/<CC>.zip` → `<CC>.txt`). When a country's file is present,
-		 * alias rows gain their language tag, `privateuse` ("preferred" from `isPreferredName`), and the `official` bit
-		 * (language is CLDR-official for the country, colloquial/historic excluded — the rule the #936 risk probe measured
-		 * at 7 new name-exact collisions globally). The main dump's bare `alternatenames` list still decides WHICH rows
-		 * exist; V2 only decorates them. Missing file = the pre-#936 untagged behavior, not an error.
-		 */
+
 		alternateDir?: PathBuilderLike
 	}
 ): Promise<number> {
-	// Latin-only, no bracket/paren noise GeoNames packs into `alternatenames` ("(( Karis Landskommun ))",
-	// airport codes), 2–60 chars, at least one letter (drops bare postcodes/numbers).
 	const LATIN_NAME = /^[\p{Script=Latin}\p{M}\s\-'.]{2,60}$/u
 
 	const clean = (s: string): string | null => {
@@ -218,6 +166,19 @@ export async function ingestGeonamesAliases(
 		return t && LATIN_NAME.test(t) && /\p{L}/u.test(t) ? t : null
 	}
 
+	const NAME_NOISE = /[()[\]{}<>|/\\_@#$%^*+=~`"]/u
+
+	const NAME_MIN_LENGTH = 2
+	const NAME_MAX_LENGTH = 60
+
+	const cleanAlias = (s: string): string | null => {
+		const t = s.trim()
+
+		return t.length >= NAME_MIN_LENGTH && t.length <= NAME_MAX_LENGTH && /\p{L}/u.test(t) && !NAME_NOISE.test(t)
+			? t
+			: null
+	}
+
 	const sprInsert = db.prepare(
 		`INSERT OR REPLACE INTO spr (id, parent_id, name, placetype, country, latitude, longitude, min_latitude, min_longitude, max_latitude, max_longitude, is_current, is_deprecated, is_ceased, is_superseded, is_superseding, lastmodified) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
 	)
@@ -228,23 +189,15 @@ export async function ingestGeonamesAliases(
 
 	const populationInsert = db.prepare(`INSERT OR REPLACE INTO place_population (id, population) VALUES (?, ?)`)
 
-	// #267 admin linkage: ancestor rows (locality→region→country) so parentID scoping + adminCoherence reach
-	// the gap countries. Only used for a country in opts.adminForCountries.
 	const ancestorInsert = db.prepare(
 		`INSERT INTO ancestors (id, ancestor_id, ancestor_placetype, lastmodified) VALUES (?, ?, ?, 0)`
 	)
 
-	const report = (event: GeonamesIngestProgress, missingFile?: string): void => {
+	const report = (event: GeonamesIngestProgress, missingFile?: PathBuilderLike): void => {
 		if (onProgress) {
 			onProgress(event)
-		} else if (event.skipped) {
-			console.error(
-				`  GeoNames ${event.country}: ${missingFile} missing — download from download.geonames.org/export/dump/${event.country}.zip; skipped`
-			)
 		} else {
-			console.error(
-				`  GeoNames ${event.country}: ${event.places.toLocaleString()} populated places (+ Latin alt-names)`
-			)
+			console.error(`  GeoNames ${formatGeonamesIngestProgress(event, missingFile)}`)
 		}
 	}
 
@@ -252,41 +205,29 @@ export async function ingestGeonamesAliases(
 	let total = 0
 	db.exec("BEGIN")
 
-	// #1514: clear the range before writing it. Inside the transaction, so a fold that throws leaves the
-	// DB as it found it rather than half-purged. See purgeGeonamesAliasRange for why the ids demand this.
 	purgeGeonamesAliasRange(db)
 
 	for (const cc of countries) {
-		const file = join(geonamesDir, `${cc}.txt`)
+		const file = resolvePathBuilder(geonamesDir, `${cc}.txt`)
 
 		if (!(await pathExists(file))) {
-			report({ country: cc, places: 0, skipped: true }, file)
+			report({ country: cc, places: 0, skipped: true, aliasesRefused: null }, file)
 
 			continue
 		}
 
 		let nc = 0
-		// #267: add A-class admin + ancestry only for the gap countries this country is in (never the EU set).
+		let refused = 0
+
 		const addAdmin = opts?.adminForCountries?.has(cc) ?? false
-		const v2File = opts?.alternateDir ? join(opts.alternateDir, `${cc}.txt`) : undefined
+		const v2File = opts?.alternateDir ? resolvePathBuilder(opts.alternateDir, `${cc}.txt`) : undefined
 		const readV2 = Boolean(v2File && (await pathExists(v2File)))
 
-		// Survey pass. The dump is STREAMED — NO's is 71 MB and only the caller knows which country
-		// comes next — so what a later pass needs has to be collected here rather than re-scanned off a
-		// materialized array. Two things qualify, and both are small: the P-class geonameid set the V2
-		// decoration is restricted to, and the handful of A-class rows the admin pre-pass writes (one
-		// PCL* + the ADM1s). Everything else is re-read in the emit pass below. Neither is wanted
-		// without V2 tags or admin, and then the pass is skipped rather than read for nothing.
-		//
-		// GeoNames dump columns (0-indexed): 0 geonameid, 1 name, 2 asciiname, 3 alternatenames, 4 lat, 5 lon,
-		// 6 feature_class, 7 feature_code, 10 admin1 code, 14 pop.
-		//
-		// `header: false` — the dump is headerless, so row 1 is a place, not column names.
 		const wanted = new Set<number>()
 		const adminRows: string[][] = []
 
 		if (readV2 || addAdmin) {
-			for await (const f of TSVSpliterator.fromAsync(file, { header: false })) {
+			for await (const f of readUnquotedTSV(file)) {
 				if (f[6] === "P") {
 					if (readV2) {
 						wanted.add(Number(f[0]))
@@ -301,13 +242,8 @@ export async function ingestGeonamesAliases(
 			}
 		}
 
-		// #936: V2 tags for this country's P-class rows — geonameid → exact alias spelling → tag. The V2
-		// dump repeats one spelling under several languages ("Åbo" sv/da/no); the merged tag is official /
-		// preferred if ANY qualifying row is.
 		const v2 = readV2 ? await parseAlternateNamesV2(v2File!, cc, wanted) : undefined
 
-		// #267 admin pre-pass (gap countries): fold the country (PCLI) + regions (ADM1), self+ancestry them, and
-		// build the admin1→region map the localities link through. Point bbox (GeoNames gives a centroid only).
 		let countryID = -1
 		const adminMap = new Map<string, number>()
 
@@ -320,10 +256,7 @@ export async function ingestGeonamesAliases(
 				const lon = Number(f[5]) || 0
 
 				if (f[7]?.startsWith("PCL")) {
-					// Any country-level political entity — PCLI (independent), PCLD (dependent territory),
-					// PCLF (freely associated), PCLS (special administrative region: HK/MO/PS). All are the
-					// country tier; restricting to PCLI left those ~17 territories without a country row.
-					if (countryID >= 0) continue // one country row
+					if (countryID >= 0) continue
 					countryID = id++
 					sprInsert.run(countryID, -1, aname, "country", cc, lat, lon, lat, lon, lat, lon, 1, 0, 0, 0, 0, 0)
 					namesInsert.run(countryID, aname, "country", cc, "", "", 0, 0)
@@ -337,7 +270,6 @@ export async function ingestGeonamesAliases(
 				}
 			}
 
-			// Re-parent regions + ancestor them to the (now-known) country.
 			if (countryID >= 0) {
 				for (const rid of adminMap.values()) {
 					db.prepare("UPDATE spr SET parent_id = ? WHERE id = ?").run(countryID, rid)
@@ -346,9 +278,8 @@ export async function ingestGeonamesAliases(
 			}
 		}
 
-		// Emit pass — a second stream over the same file, off the page cache the survey pass just warmed.
-		for await (const f of TSVSpliterator.fromAsync(file, { header: false })) {
-			if (f[6] !== "P") continue // populated places only
+		for await (const f of readUnquotedTSV(file)) {
+			if (f[6] !== "P") continue
 			const lat = Number(f[4])
 			const lon = Number(f[5])
 
@@ -357,19 +288,13 @@ export async function ingestGeonamesAliases(
 
 			if (!name) continue
 			const nid = id++
-			// #267: link to the locality's region (else country) for gap countries; -1 (orphan) otherwise.
+
 			const regionID = addAdmin ? (adminMap.get(f[10] ?? "") ?? -1) : -1
 			const parentID = regionID >= 0 ? regionID : addAdmin && countryID >= 0 ? countryID : -1
-			// Point bbox — a GeoNames row is a centroid; the candidate's region-bbox disambiguation just
-			// sees it as contained in itself, fine for a locality.
+
 			sprInsert.run(nid, parentID, name, "locality", cc, lat, lon, lat, lon, lat, lon, 1, 0, 0, 0, 0, 0)
 			namesInsert.run(nid, name, "locality", cc, "", "", 0, 0)
 
-			// The SELF row is unconditional (#1514). `populateAncestors` writes one for every spr row it
-			// sees, so a full build's freeze phase produces it either way — but the freeze does not run on
-			// the fold-on-copy path, and since the purge now clears this range the fold has to emit
-			// everything the closure would. Measured on the 161-country fold: 212,993 self rows, exactly the
-			// non-gap countries' localities, which is what the base artifact carried and a fold-on-copy lost.
 			ancestorInsert.run(nid, nid, "locality")
 
 			if (addAdmin) {
@@ -387,7 +312,11 @@ export async function ingestGeonamesAliases(
 			const tags = v2?.get(Number(f[0]))
 
 			for (const raw of [f[2] ?? "", ...(f[3] ? f[3].split(",") : [])]) {
-				const alt = clean(raw)
+				const alt = cleanAlias(raw)
+
+				if (!alt && raw.trim()) {
+					refused++
+				}
 
 				if (alt && !seen.has(alt)) {
 					seen.add(alt)
@@ -406,7 +335,7 @@ export async function ingestGeonamesAliases(
 			nc++
 		}
 
-		report({ country: cc, places: nc, skipped: false })
+		report({ country: cc, places: nc, skipped: false, aliasesRefused: refused })
 		total += nc
 	}