@mailwoman/resolver-wof-sqlite 10.0.0 → 10.2.0

package/out/fts.js

@@ -0,0 +1,188 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+import { tableExists } from "@mailwoman/sqlite/introspection";
+/**
+ * The name of the FTS5 virtual table that indexes WOF place names for `match` queries.
+ */
+export const PLACE_SEARCH_TABLE = "place_search";
+/**
+ * The separator between aliases in the `alt_names` bag.
+ * It stops a phrase query from matching across two aliases.
+ *
+ * FTS5 assigns positions only to tokens.
+ * Spaces and punctuation are not tokens and would leave the aliases adjacent,
+ * but unicode61 indexes the Private Use Area codepoint U+E000 as a token.
+ *
+ * Query sanitizers strip this character, so no user query can match the separator.
+ */
+export const ALIAS_SEPARATOR = "\uE000";
+const ALIAS_SEPARATOR_CODEPOINT = ALIAS_SEPARATOR.codePointAt(0);
+/**
+ * Folds a query or name for exact-tier comparison by lowercasing, trimming
+ * and collapsing internal whitespace.
+ *
+ * Every exact-tier consumer uses this function, so its output compares equal to
+ * the aliases that {@link aliasBagExactMatch} parses.
+ */
+export function foldQueryText(input) {
+    return input.toLowerCase().trim().replaceAll(/\s+/g, " ");
+}
+/**
+ * Returns whether any alias in an `alt_names` bag exactly equals the folded query.
+ *
+ * A bag without {@link ALIAS_SEPARATOR} is a legacy space-joined bag without alias boundaries.
+ * For such a bag, the function checks word-bounded containment only when no candidate matched strictly.
+ *
+ * Containment by itself would promote fragments such as "York" inside "New York City".
+ *
+ * @param altNames The `alt_names` bag from `place_search`, or null when the row has no aliases.
+ * @param normalizedQuery The query folded by {@link foldQueryText}.
+ * @param anyStrictExact Whether any candidate already matched on its canonical name or region abbreviation.
+ * Only legacy bags use it.
+ */
+export function aliasBagExactMatch(altNames, normalizedQuery, anyStrictExact) {
+    if (!altNames || altNames === "" || !normalizedQuery)
+        return false;
+    if (altNames.includes(ALIAS_SEPARATOR)) {
+        return altNames.split(ALIAS_SEPARATOR).some((alias) => foldQueryText(alias) === normalizedQuery);
+    }
+    if (anyStrictExact)
+        return false;
+    return ` ${foldQueryText(altNames)} `.includes(` ${normalizedQuery} `);
+}
+/**
+ * The name of the R*Tree virtual table that indexes WOF place bounding boxes for bbox and proximity lookups.
+ */
+export const PLACE_BBOX_TABLE = "place_bbox";
+/**
+ * The name of the sparse table that stores `wof:population` per place.
+ *
+ * The lookup uses this value to apply the population ranking boost.
+ *
+ * A place without a row gets no boost and no penalty.
+ */
+export const PLACE_POPULATION_TABLE = "place_population";
+/**
+ * The name of the table of per-place `referential` and `encyclopedic` scores,
+ * built by `mailwoman gazetteer importance`.
+ *
+ * The lookup copies `encyclopedic` onto results without ranking by it.
+ * It refuses to read an older table with a single `importance` column.
+ */
+export const PLACE_IMPORTANCE_TABLE = "place_importance";
+/**
+ * Builds the `place_search` FTS5 table and the `place_bbox` R*Tree from a WOF
+ * database's `spr` and `names` tables.
+ *
+ * Without `drop: true`, an existing index is kept.
+ */
+export function buildPlaceSearchFTS(db, opts = {}) {
+    const start = Date.now();
+    const onProgress = opts.onProgress ?? (() => { });
+    onProgress("checking");
+    const ftsExisting = tableExists(db, PLACE_SEARCH_TABLE);
+    const bboxExisting = tableExists(db, PLACE_BBOX_TABLE);
+    let ftsCreated = false;
+    if (ftsExisting && opts.drop) {
+        onProgress("dropping", PLACE_SEARCH_TABLE);
+        db.exec(`DROP TABLE ${PLACE_SEARCH_TABLE}`);
+    }
+    if (!ftsExisting || opts.drop) {
+        onProgress("creating");
+        db.exec(`
+			CREATE VIRTUAL TABLE ${PLACE_SEARCH_TABLE} USING fts5(
+				wof_id UNINDEXED,
+				name,
+				alt_names,
+				tokenize = 'unicode61 remove_diacritics 2'
+			);
+		`);
+        onProgress("populating");
+        db.exec(`
+			INSERT INTO ${PLACE_SEARCH_TABLE} (wof_id, name, alt_names)
+			SELECT
+				spr.id,
+				spr.name,
+				COALESCE((
+					SELECT GROUP_CONCAT(
+						REPLACE(name, char(${ALIAS_SEPARATOR_CODEPOINT}), ' '),
+						' ' || char(${ALIAS_SEPARATOR_CODEPOINT}) || ' '
+					) || ' ' || char(${ALIAS_SEPARATOR_CODEPOINT})
+					FROM names WHERE names.id = spr.id
+				), '')
+			FROM spr
+			WHERE spr.is_current != 0
+				AND spr.is_deprecated = 0
+				AND spr.name IS NOT NULL;
+		`);
+        ftsCreated = true;
+    }
+    const ftsCountRow = db.prepare(`SELECT COUNT(*) AS n FROM ${PLACE_SEARCH_TABLE}`).get();
+    let bboxCreated = false;
+    if (bboxExisting && opts.drop) {
+        onProgress("dropping", PLACE_BBOX_TABLE);
+        db.exec(`DROP TABLE ${PLACE_BBOX_TABLE}`);
+    }
+    if (!bboxExisting || opts.drop) {
+        onProgress("creating-bbox");
+        db.exec(`
+			CREATE VIRTUAL TABLE ${PLACE_BBOX_TABLE} USING rtree(
+				id,
+				min_lat, max_lat,
+				min_lon, max_lon
+			);
+		`);
+        onProgress("populating-bbox");
+        db.exec(`
+			INSERT INTO ${PLACE_BBOX_TABLE} (id, min_lat, max_lat, min_lon, max_lon)
+			SELECT
+				spr.id,
+				spr.min_latitude,
+				spr.max_latitude,
+				spr.min_longitude,
+				spr.max_longitude
+			FROM spr
+			WHERE spr.is_current != 0
+				AND spr.is_deprecated = 0
+				AND spr.min_latitude IS NOT NULL
+				AND spr.max_latitude IS NOT NULL
+				AND spr.min_longitude IS NOT NULL
+				AND spr.max_longitude IS NOT NULL
+				AND NOT (spr.min_latitude = 0 AND spr.max_latitude = 0
+				     AND spr.min_longitude = 0 AND spr.max_longitude = 0);
+		`);
+        bboxCreated = true;
+    }
+    const bboxCountRow = db.prepare(`SELECT COUNT(*) AS n FROM ${PLACE_BBOX_TABLE}`).get();
+    onProgress("done", `${ftsCountRow.n} FTS rows + ${bboxCountRow.n} bbox rows ` +
+        `(${ftsCreated ? "built" : "preexisting"} / ${bboxCreated ? "built" : "preexisting"})`);
+    return {
+        created: ftsCreated,
+        indexedRows: ftsCountRow.n,
+        bboxCreated,
+        bboxIndexedRows: bboxCountRow.n,
+        durationMs: Date.now() - start,
+    };
+}
+/**
+ * Returns whether the connected database has the `place_search` FTS table.
+ */
+export function placeSearchFTSExists(db) {
+    return tableExists(db, PLACE_SEARCH_TABLE);
+}
+/**
+ * Returns whether the connected database has the `place_bbox` R*Tree table.
+ */
+export function placeBboxExists(db) {
+    return tableExists(db, PLACE_BBOX_TABLE);
+}
+/**
+ * Returns whether the connected database has the `place_population` table.
+ */
+export function placePopulationExists(db) {
+    return tableExists(db, PLACE_POPULATION_TABLE);
+}
+//# sourceMappingURL=fts.js.map

package/out/fts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fts.js","sourceRoot":"","sources":["../lib/fts.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAA;AAE7D;;GAEG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,cAAc,CAAA;AAEhD;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,QAAQ,CAAA;AAEvC,MAAM,yBAAyB,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC,CAAW,CAAA;AAE1E;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,KAAa;IAC1C,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;AAC1D,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,kBAAkB,CAAC,QAAuB,EAAE,eAAuB,EAAE,cAAuB;IAC3G,IAAI,CAAC,QAAQ,IAAI,QAAQ,KAAK,EAAE,IAAI,CAAC,eAAe;QAAE,OAAO,KAAK,CAAA;IAElE,IAAI,QAAQ,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;QACxC,OAAO,QAAQ,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,eAAe,CAAC,CAAA;IACjG,CAAC;IAED,IAAI,cAAc;QAAE,OAAO,KAAK,CAAA;IAEhC,OAAO,IAAI,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,eAAe,GAAG,CAAC,CAAA;AACvE,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,YAAY,CAAA;AAE5C;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,kBAAkB,CAAA;AAExD;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,kBAAkB,CAAA;AAuDxD;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAClC,EAAsB,EACtB,IAAI,GAA4B,EAAE;IAElC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IACxB,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;IAEhD,UAAU,CAAC,UAAU,CAAC,CAAA;IACtB,MAAM,WAAW,GAAG,WAAW,CAAC,EAAE,EAAE,kBAAkB,CAAC,CAAA;IACvD,MAAM,YAAY,GAAG,WAAW,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAA;IAEtD,IAAI,UAAU,GAAG,KAAK,CAAA;IAEtB,IAAI,WAAW,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QAC9B,UAAU,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAA;QAC1C,EAAE,CAAC,IAAI,CAAC,cAAc,kBAAkB,EAAE,CAAC,CAAA;IAC5C,CAAC;IAED,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,UAAU,CAAC,UAAU,CAAC,CAAA;QAEtB,EAAE,CAAC,IAAI,CAAC;0BACgB,kBAAkB;;;;;;GAMzC,CAAC,CAAA;QAEF,UAAU,CAAC,YAAY,CAAC,CAAA;QAExB,EAAE,CAAC,IAAI,CAAC;iBACO,kBAAkB;;;;;;2BAMR,yBAAyB;oBAChC,yBAAyB;wBACrB,yBAAyB;;;;;;;GAO9C,CAAC,CAAA;QAEF,UAAU,GAAG,IAAI,CAAA;IAClB,CAAC;IAED,MAAM,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC,6BAA6B,kBAAkB,EAAE,CAAC,CAAC,GAAG,EAAmB,CAAA;IAExG,IAAI,WAAW,GAAG,KAAK,CAAA;IAEvB,IAAI,YAAY,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,UAAU,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAA;QACxC,EAAE,CAAC,IAAI,CAAC,cAAc,gBAAgB,EAAE,CAAC,CAAA;IAC1C,CAAC;IAED,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,UAAU,CAAC,eAAe,CAAC,CAAA;QAE3B,EAAE,CAAC,IAAI,CAAC;0BACgB,gBAAgB;;;;;GAKvC,CAAC,CAAA;QAEF,UAAU,CAAC,iBAAiB,CAAC,CAAA;QAE7B,EAAE,CAAC,IAAI,CAAC;iBACO,gBAAgB;;;;;;;;;;;;;;;;GAgB9B,CAAC,CAAA;QAEF,WAAW,GAAG,IAAI,CAAA;IACnB,CAAC;IAED,MAAM,YAAY,GAAG,EAAE,CAAC,OAAO,CAAC,6BAA6B,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAmB,CAAA;IAEvG,UAAU,CACT,MAAM,EACN,GAAG,WAAW,CAAC,CAAC,eAAe,YAAY,CAAC,CAAC,aAAa;QACzD,IAAI,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,MAAM,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,GAAG,CACvF,CAAA;IAED,OAAO;QACN,OAAO,EAAE,UAAU;QACnB,WAAW,EAAE,WAAW,CAAC,CAAC;QAC1B,WAAW;QACX,eAAe,EAAE,YAAY,CAAC,CAAC;QAC/B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;KAC9B,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAK,EAAsB;IAC9D,OAAO,WAAW,CAAC,EAAE,EAAE,kBAAkB,CAAC,CAAA;AAC3C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAK,EAAsB;IACzD,OAAO,WAAW,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAA;AACzC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAK,EAAsB;IAC/D,OAAO,WAAW,CAAC,EAAE,EAAE,sBAAsB,CAAC,CAAA;AAC/C,CAAC"}

package/out/geonames/aliases.d.ts

@@ -2,94 +2,58 @@
  * @copyright Sister Software
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
- *
- *   #743/#193 — fold GeoNames bilingual / alt-language place-names into a WOF/unified admin DB as
- *   first-class places. The hard-filter recall gap on bilingual countries (the address says
- *   "Karjaa" but the table holds the Swedish "Karis") is missing alt-LANGUAGE names, not missing
- *   places: the WOF/Overture `names` carried only the primary, so the candidate build's Latin-alias
- *   explode (build-candidate pass 2) had nothing to widen. GeoNames' per-country dump carries the
- *   variants inline (the Karis row's `alternatenames` includes "Karjaa").
- *
- *   For each POPULATED place (feature class `P`) this writes an `spr` row + `names` rows (primary +
- *   Latin alt-names) + population into the SAME tables the WOF/Overture paths use — synthetic ids
- *   based at {@link GEONAMES_ID_BASE} so the three sources never collide. The caller then rebuilds
- *   `place_search` ({@link buildPlaceSearchFTS} with `drop: true`) so the candidate build carries
- *   Karjaa↔Karis. Proven (FI hard-resolve 69.5 → 85.8 %, coverage 74.4 → 94.0 %); duplicating a
- *   place already held under another source is benign — the rows share name_key+coord and the
- *   candidate ranking dedupes by score.
- *
- *   This is the package home so the canonical `build-unified-wof --geonames-countries`, the
- *   pipeline fold (`gazetteer-pipeline/admin/fold-geonames`), AND the `mailwoman gazetteer` commands all share
- *   ONE implementation. GeoNames dump = `download.geonames.org/export/dump/<CC>.zip` → `<CC>.txt`
- *   (TSV).
  */
 import type { DatabaseClient } from "@mailwoman/sqlite/client";
 import { type PathBuilderLike } from "path-ts";
 import type { WOFDatabase } from "#schema";
 /**
- * Clear everything the alias fold owns — `[GEONAMES_ID_BASE, GEONAMES_POSTAL_ID_BASE)` across {@link
- * FOLD_OWNED_TABLES}
- * — so a fold's output is a function of its country list and dumps ALONE, never of what the DB happened to hold first.
- * Returns the rows removed per table.
- *
- * This exists because the synthetic id is a POSITION (`GEONAMES_ID_BASE + n`, counted across the country list in
- * order), not a derivation from the geonameid: change the list, its order, or a dump's contents and every subsequent id
- * shifts. Without the purge, a second fold overwrote the `spr` PREFIX it reached (`INSERT OR REPLACE`) while its
- * `names`/`ancestors` rows merely appended (bare `INSERT`) and `place_population` skipped unpopulated places entirely —
- * so the previous run's names, ancestry and populations stayed bound to ids now describing other places.
+ * Deletes every row the GeoNames alias fold owns, with ids in
+ * `[GEONAMES_ID_BASE, GEONAMES_POSTAL_ID_BASE)`, and returns the count removed per table.
  *
- * Measured on the live 2026-08-05 artifact: a 14-country re-fold over the 161-country fold baked into
- * `admin-global-priority.db` put 522,184 of 2,110,096 name rows in the range (24.7 %) on a place from a different
- * country. Gaborone/BW at id 9000000121151 became Aichegg/AT and kept all 26 of its names; Kinshasa's 16,000,000
- * population landed on a Lithuanian hamlet.
- *
- * The upper bound is NOT the end of the id space. Each later extract owns a range above this one — GeoNames-postal @
- * 9.5e12, NL-PC6 @ 9.6e12, Code-Point @ 9.7e12, NI @ 9.8e12 — and a purge that ran past {@link GEONAMES_POSTAL_ID_BASE}
- * would delete them.
+ * Fold ids are assigned sequentially, so a re-fold without this purge would attach
+ * stale names, ancestors and populations to other places.
+ * The upper bound leaves the postal and postcode id ranges above it untouched.
  */
 export declare function purgeGeonamesAliasRange<DB>(db: DatabaseClient<DB>): Record<string, number>;
 /**
- * Per-country progress for the ingest — one event per country dump processed (or skipped).
+ * One progress event per country dump that the ingest processed or skipped.
  */
 export interface GeonamesIngestProgress {
     /**
-     * ISO 3166-1 alpha-2 code.
+     * The ISO 3166-1 alpha-2 country code.
      */
     country: string;
     /**
-     * Populated places ingested from this country's dump (0 when skipped).
+     * The number of populated places ingested from this country's dump, or 0 when it was skipped.
      */
     places: number;
     /**
-     * True when the country's `<CC>.txt` dump was missing — the country is skipped, not fatal.
+     * Whether the country's `<CC>.txt` dump was missing.
+     *
+     * A missing dump skips the country without failing the ingest.
      */
     skipped: boolean;
+    /**
+     * The number of alternate names that the alias filter rejected.
+     *
+     * It distinguishes a sparse source from aliases that the fold discarded.
+     */
+    aliasesRefused: number | null;
 }
 /**
- * Fold the GeoNames `P`-class places (+ their Latin alt-names) for `countries` into `db`'s `spr` / `names` /
- * `place_population` tables. Returns the total places ingested.
+ * Formats a GeoNames fold progress event as one log line.
+ */
+export declare function formatGeonamesIngestProgress(event: GeonamesIngestProgress, missingFile?: PathBuilderLike): string;
+/**
+ * Purges the fold's id range, then adds GeoNames `P`-class places and their
+ * alternate names for `countries` to `db`.
+ * It returns the number of places ingested.
  *
- * `onProgress` receives one event per country (default: a stderr line, matching the build scripts' legacy output). The
- * caller MUST rebuild `place_search` afterward (`buildPlaceSearchFTS(db, { drop: true })`) for the new names to reach
- * the candidate build's alias pass.
+ * The caller must then rebuild `place_search` with `buildPlaceSearchFTS(db, { drop: true })`
+ * to make the new names searchable.
  */
 export declare function ingestGeonamesAliases(db: DatabaseClient<WOFDatabase>, countries: string[], geonamesDir: PathBuilderLike, onProgress?: (event: GeonamesIngestProgress) => void, opts?: {
-    /**
-     * #267: the countries for which to ALSO fold the GeoNames A-class admin (PCLI country + ADM1 regions) and link each
-     * locality's `parent_id` + ancestry chain (locality → region → country). PER-COUNTRY because a country that already
-     * carries WOF admin would double up — pass only the ZERO-COVERAGE gap countries (the coverage-expansion targets),
-     * never the EU alias set. Without admin, a gap country's localities are orphans (`parent_id=-1`, no ancestors), so
-     * `parentID` scoping and adminCoherence can't reach them and "Tbilisi, GE" can't resolve.
-     */
     adminForCountries?: ReadonlySet<string>;
-    /**
-     * #936: directory of per-country alternateNamesV2 dumps
-     * (`download.geonames.org/export/dump/alternatenames/<CC>.zip` → `<CC>.txt`). When a country's file is present,
-     * alias rows gain their language tag, `privateuse` ("preferred" from `isPreferredName`), and the `official` bit
-     * (language is CLDR-official for the country, colloquial/historic excluded — the rule the #936 risk probe measured
-     * at 7 new name-exact collisions globally). The main dump's bare `alternatenames` list still decides WHICH rows
-     * exist; V2 only decorates them. Missing file = the pre-#936 untagged behavior, not an error.
-     */
     alternateDir?: PathBuilderLike;
 }): Promise<number>;
 //# sourceMappingURL=aliases.d.ts.map

package/out/geonames/aliases.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"aliases.d.ts","sourceRoot":"","sources":["../../lib/geonames/aliases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAKH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAC9D,OAAO,EAAQ,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAGpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAQ1C;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,uBAAuB,CAAC,EAAE,EAAE,EAAE,EAAE,cAAc,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAY1F;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,OAAO,EAAE,OAAO,CAAA;CAChB;AAqFD;;;;;;;GAOG;AACH,wBAAsB,qBAAqB,CAC1C,EAAE,EAAE,cAAc,CAAC,WAAW,CAAC,EAC/B,SAAS,EAAE,MAAM,EAAE,EACnB,WAAW,EAAE,eAAe,EAC5B,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,EACpD,IAAI,CAAC,EAAE;IACN;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAA;IACvC;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,eAAe,CAAA;CAC9B,GACC,OAAO,CAAC,MAAM,CAAC,CA8MjB"}
+{"version":3,"file":"aliases.d.ts","sourceRoot":"","sources":["../../lib/geonames/aliases.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAC9D,OAAO,EAAE,KAAK,eAAe,EAAsB,MAAM,SAAS,CAAA;AAElE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAI1C;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CAAC,EAAE,EAAE,EAAE,EAAE,cAAc,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAY1F;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;;;OAIG;IACH,OAAO,EAAE,OAAO,CAAA;IAEhB;;;;OAIG;IACH,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;CAC7B;AAED;;GAEG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,sBAAsB,EAAE,WAAW,CAAC,EAAE,eAAe,GAAG,MAAM,CAMjH;AAgED;;;;;;;GAOG;AACH,wBAAsB,qBAAqB,CAC1C,EAAE,EAAE,cAAc,CAAC,WAAW,CAAC,EAC/B,SAAS,EAAE,MAAM,EAAE,EACnB,WAAW,EAAE,eAAe,EAC5B,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,EACpD,IAAI,CAAC,EAAE;IACN,iBAAiB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAA;IAEvC,YAAY,CAAC,EAAE,eAAe,CAAA;CAC9B,GACC,OAAO,CAAC,MAAM,CAAC,CAyLjB"}

package/out/geonames/aliases.js

@@ -2,57 +2,20 @@
  * @copyright Sister Software
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
- *
- *   #743/#193 — fold GeoNames bilingual / alt-language place-names into a WOF/unified admin DB as
- *   first-class places. The hard-filter recall gap on bilingual countries (the address says
- *   "Karjaa" but the table holds the Swedish "Karis") is missing alt-LANGUAGE names, not missing
- *   places: the WOF/Overture `names` carried only the primary, so the candidate build's Latin-alias
- *   explode (build-candidate pass 2) had nothing to widen. GeoNames' per-country dump carries the
- *   variants inline (the Karis row's `alternatenames` includes "Karjaa").
- *
- *   For each POPULATED place (feature class `P`) this writes an `spr` row + `names` rows (primary +
- *   Latin alt-names) + population into the SAME tables the WOF/Overture paths use — synthetic ids
- *   based at {@link GEONAMES_ID_BASE} so the three sources never collide. The caller then rebuilds
- *   `place_search` ({@link buildPlaceSearchFTS} with `drop: true`) so the candidate build carries
- *   Karjaa↔Karis. Proven (FI hard-resolve 69.5 → 85.8 %, coverage 74.4 → 94.0 %); duplicating a
- *   place already held under another source is benign — the rows share name_key+coord and the
- *   candidate ranking dedupes by score.
- *
- *   This is the package home so the canonical `build-unified-wof --geonames-countries`, the
- *   pipeline fold (`gazetteer-pipeline/admin/fold-geonames`), AND the `mailwoman gazetteer` commands all share
- *   ONE implementation. GeoNames dump = `download.geonames.org/export/dump/<CC>.zip` → `<CC>.txt`
- *   (TSV).
  */
 import { isOfficialLanguage } from "@mailwoman/codex/country";
-import { pathExists } from "@mailwoman/core/fs/readers";
+import { readUnquotedTSV } from "@mailwoman/core/fs/delimited";
+import { pathExists } from "@mailwoman/core/fs/readers/stat";
 import { GEONAMES_ID_BASE, GEONAMES_POSTAL_ID_BASE } from "@mailwoman/core/resolver/synthetic-id-ranges";
-import { join } from "path-ts";
-import { TSVSpliterator } from "spliterator";
-/**
- * The four tables the alias fold writes into the range it owns. The purge and the ingest must agree on this list — a
- * table written but not purged is exactly the #1514 defect.
- */
+import { resolvePathBuilder } from "path-ts";
 const FOLD_OWNED_TABLES = ["spr", "names", "place_population", "ancestors"];
 /**
- * Clear everything the alias fold owns — `[GEONAMES_ID_BASE, GEONAMES_POSTAL_ID_BASE)` across {@link
- * FOLD_OWNED_TABLES}
- * — so a fold's output is a function of its country list and dumps ALONE, never of what the DB happened to hold first.
- * Returns the rows removed per table.
- *
- * This exists because the synthetic id is a POSITION (`GEONAMES_ID_BASE + n`, counted across the country list in
- * order), not a derivation from the geonameid: change the list, its order, or a dump's contents and every subsequent id
- * shifts. Without the purge, a second fold overwrote the `spr` PREFIX it reached (`INSERT OR REPLACE`) while its
- * `names`/`ancestors` rows merely appended (bare `INSERT`) and `place_population` skipped unpopulated places entirely —
- * so the previous run's names, ancestry and populations stayed bound to ids now describing other places.
- *
- * Measured on the live 2026-08-05 artifact: a 14-country re-fold over the 161-country fold baked into
- * `admin-global-priority.db` put 522,184 of 2,110,096 name rows in the range (24.7 %) on a place from a different
- * country. Gaborone/BW at id 9000000121151 became Aichegg/AT and kept all 26 of its names; Kinshasa's 16,000,000
- * population landed on a Lithuanian hamlet.
+ * Deletes every row the GeoNames alias fold owns, with ids in
+ * `[GEONAMES_ID_BASE, GEONAMES_POSTAL_ID_BASE)`, and returns the count removed per table.
  *
- * The upper bound is NOT the end of the id space. Each later extract owns a range above this one — GeoNames-postal @
- * 9.5e12, NL-PC6 @ 9.6e12, Code-Point @ 9.7e12, NI @ 9.8e12 — and a purge that ran past {@link GEONAMES_POSTAL_ID_BASE}
- * would delete them.
+ * Fold ids are assigned sequentially, so a re-fold without this purge would attach
+ * stale names, ancestors and populations to other places.
+ * The upper bound leaves the postal and postcode id ranges above it untouched.
  */
 export function purgeGeonamesAliasRange(db) {
     const removed = {};
@@ -65,26 +28,18 @@ export function purgeGeonamesAliasRange(db) {
     return removed;
 }
 /**
- * Parse a country's alternateNamesV2 file into `geonameid -> spelling -> attribution`, restricted to the populated
- * places (`P`) in `wanted`.
+ * Formats a GeoNames fold progress event as one log line.
  */
+export function formatGeonamesIngestProgress(event, missingFile) {
+    if (event.skipped) {
+        return `${event.country}: ${missingFile ? `${missingFile} missing` : "dump missing"} — download from download.geonames.org/export/dump/${event.country}.zip; skipped`;
+    }
+    return `${event.country}: ${event.places.toLocaleString()} places, ${(event.aliasesRefused ?? 0).toLocaleString()} alt-names refused`;
+}
 async function parseAlternateNamesV2(v2File, cc, wanted) {
     const v2 = new Map();
-    // V2 columns (0-indexed): 1 geonameid, 2 isolanguage, 3 name, 4 isPreferredName, 5 isShortName,
-    // 6 isColloquial, 7 isHistoric, 8 from, 9 to.
-    //
-    // Two passes, because historic-ness is a fact about the NAME, not the row: GeoNames splits one
-    // spelling across rows — Malabo carries "Santa Isabel" as (es, unflagged) AND as (no-language,
-    // isHistoric=1, to=1973). Officialness must see the flags from EVERY row for the spelling, or the
-    // colonial-era name sails through on the language-tagged row (the #936 review's Malabo finding).
-    // Do NOT condition on isPreferredName instead — it's sparse annotation, not a signal (Turku's sv "Åbo"
-    // is unflagged; FI has 1,746 flags across the whole dump).
-    //
-    // Both passes STREAM the file rather than sharing one materialized array: NO's V2 dump is 33 MB,
-    // and a second read off the page cache costs less than holding half a million line strings.
-    // `header: false` — the dump is headerless.
     const historicNames = new Set();
-    for await (const f of TSVSpliterator.fromAsync(v2File, { header: false })) {
+    for await (const f of readUnquotedTSV(v2File)) {
         if (f[6] === "1" || f[7] === "1" || (f[9] ?? "").trim() !== "") {
             const alt = (f[3] ?? "").trim();
             if (alt && wanted.has(Number(f[1]))) {
@@ -92,12 +47,11 @@ async function parseAlternateNamesV2(v2File, cc, wanted) {
             }
         }
     }
-    for await (const f of TSVSpliterator.fromAsync(v2File, { header: false })) {
+    for await (const f of readUnquotedTSV(v2File)) {
         const gid = Number(f[1]);
         if (!wanted.has(gid))
             continue;
         const lang = f[2] ?? "";
-        // ISO 639 codes are 2-3 letters; GeoNames' pseudo-codes (post, link, iata, wkdt, …) are 4+.
         if (!/^[a-z]{2,3}$/.test(lang))
             continue;
         const alt = (f[3] ?? "").trim();
@@ -126,70 +80,59 @@ async function parseAlternateNamesV2(v2File, cc, wanted) {
     return v2;
 }
 /**
- * Fold the GeoNames `P`-class places (+ their Latin alt-names) for `countries` into `db`'s `spr` / `names` /
- * `place_population` tables. Returns the total places ingested.
+ * Purges the fold's id range, then adds GeoNames `P`-class places and their
+ * alternate names for `countries` to `db`.
+ * It returns the number of places ingested.
  *
- * `onProgress` receives one event per country (default: a stderr line, matching the build scripts' legacy output). The
- * caller MUST rebuild `place_search` afterward (`buildPlaceSearchFTS(db, { drop: true })`) for the new names to reach
- * the candidate build's alias pass.
+ * The caller must then rebuild `place_search` with `buildPlaceSearchFTS(db, { drop: true })`
+ * to make the new names searchable.
  */
 export async function ingestGeonamesAliases(db, countries, geonamesDir, onProgress, opts) {
-    // Latin-only, no bracket/paren noise GeoNames packs into `alternatenames` ("(( Karis Landskommun ))",
-    // airport codes), 2–60 chars, at least one letter (drops bare postcodes/numbers).
     const LATIN_NAME = /^[\p{Script=Latin}\p{M}\s\-'.]{2,60}$/u;
     const clean = (s) => {
         const t = s.trim();
         return t && LATIN_NAME.test(t) && /\p{L}/u.test(t) ? t : null;
     };
+    const NAME_NOISE = /[()[\]{}<>|/\\_@#$%^*+=~`"]/u;
+    const NAME_MIN_LENGTH = 2;
+    const NAME_MAX_LENGTH = 60;
+    const cleanAlias = (s) => {
+        const t = s.trim();
+        return t.length >= NAME_MIN_LENGTH && t.length <= NAME_MAX_LENGTH && /\p{L}/u.test(t) && !NAME_NOISE.test(t)
+            ? t
+            : null;
+    };
     const sprInsert = db.prepare(`INSERT OR REPLACE INTO spr (id, parent_id, name, placetype, country, latitude, longitude, min_latitude, min_longitude, max_latitude, max_longitude, is_current, is_deprecated, is_ceased, is_superseded, is_superseding, lastmodified) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
     const namesInsert = db.prepare(`INSERT INTO names (id, name, placetype, country, language, privateuse, official, lastmodified) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
     const populationInsert = db.prepare(`INSERT OR REPLACE INTO place_population (id, population) VALUES (?, ?)`);
-    // #267 admin linkage: ancestor rows (locality→region→country) so parentID scoping + adminCoherence reach
-    // the gap countries. Only used for a country in opts.adminForCountries.
     const ancestorInsert = db.prepare(`INSERT INTO ancestors (id, ancestor_id, ancestor_placetype, lastmodified) VALUES (?, ?, ?, 0)`);
     const report = (event, missingFile) => {
         if (onProgress) {
             onProgress(event);
         }
-        else if (event.skipped) {
-            console.error(`  GeoNames ${event.country}: ${missingFile} missing — download from download.geonames.org/export/dump/${event.country}.zip; skipped`);
-        }
         else {
-            console.error(`  GeoNames ${event.country}: ${event.places.toLocaleString()} populated places (+ Latin alt-names)`);
+            console.error(`  GeoNames ${formatGeonamesIngestProgress(event, missingFile)}`);
         }
     };
     let id = GEONAMES_ID_BASE;
     let total = 0;
     db.exec("BEGIN");
-    // #1514: clear the range before writing it. Inside the transaction, so a fold that throws leaves the
-    // DB as it found it rather than half-purged. See purgeGeonamesAliasRange for why the ids demand this.
     purgeGeonamesAliasRange(db);
     for (const cc of countries) {
-        const file = join(geonamesDir, `${cc}.txt`);
+        const file = resolvePathBuilder(geonamesDir, `${cc}.txt`);
         if (!(await pathExists(file))) {
-            report({ country: cc, places: 0, skipped: true }, file);
+            report({ country: cc, places: 0, skipped: true, aliasesRefused: null }, file);
             continue;
         }
         let nc = 0;
-        // #267: add A-class admin + ancestry only for the gap countries this country is in (never the EU set).
+        let refused = 0;
         const addAdmin = opts?.adminForCountries?.has(cc) ?? false;
-        const v2File = opts?.alternateDir ? join(opts.alternateDir, `${cc}.txt`) : undefined;
+        const v2File = opts?.alternateDir ? resolvePathBuilder(opts.alternateDir, `${cc}.txt`) : undefined;
         const readV2 = Boolean(v2File && (await pathExists(v2File)));
-        // Survey pass. The dump is STREAMED — NO's is 71 MB and only the caller knows which country
-        // comes next — so what a later pass needs has to be collected here rather than re-scanned off a
-        // materialized array. Two things qualify, and both are small: the P-class geonameid set the V2
-        // decoration is restricted to, and the handful of A-class rows the admin pre-pass writes (one
-        // PCL* + the ADM1s). Everything else is re-read in the emit pass below. Neither is wanted
-        // without V2 tags or admin, and then the pass is skipped rather than read for nothing.
-        //
-        // GeoNames dump columns (0-indexed): 0 geonameid, 1 name, 2 asciiname, 3 alternatenames, 4 lat, 5 lon,
-        // 6 feature_class, 7 feature_code, 10 admin1 code, 14 pop.
-        //
-        // `header: false` — the dump is headerless, so row 1 is a place, not column names.
         const wanted = new Set();
         const adminRows = [];
         if (readV2 || addAdmin) {
-            for await (const f of TSVSpliterator.fromAsync(file, { header: false })) {
+            for await (const f of readUnquotedTSV(file)) {
                 if (f[6] === "P") {
                     if (readV2) {
                         wanted.add(Number(f[0]));
@@ -201,12 +144,7 @@ export async function ingestGeonamesAliases(db, countries, geonamesDir, onProgre
                 }
             }
         }
-        // #936: V2 tags for this country's P-class rows — geonameid → exact alias spelling → tag. The V2
-        // dump repeats one spelling under several languages ("Åbo" sv/da/no); the merged tag is official /
-        // preferred if ANY qualifying row is.
         const v2 = readV2 ? await parseAlternateNamesV2(v2File, cc, wanted) : undefined;
-        // #267 admin pre-pass (gap countries): fold the country (PCLI) + regions (ADM1), self+ancestry them, and
-        // build the admin1→region map the localities link through. Point bbox (GeoNames gives a centroid only).
         let countryID = -1;
         const adminMap = new Map();
         if (addAdmin) {
@@ -217,11 +155,8 @@ export async function ingestGeonamesAliases(db, countries, geonamesDir, onProgre
                 const lat = Number(f[4]) || 0;
                 const lon = Number(f[5]) || 0;
                 if (f[7]?.startsWith("PCL")) {
-                    // Any country-level political entity — PCLI (independent), PCLD (dependent territory),
-                    // PCLF (freely associated), PCLS (special administrative region: HK/MO/PS). All are the
-                    // country tier; restricting to PCLI left those ~17 territories without a country row.
                     if (countryID >= 0)
-                        continue; // one country row
+                        continue;
                     countryID = id++;
                     sprInsert.run(countryID, -1, aname, "country", cc, lat, lon, lat, lon, lat, lon, 1, 0, 0, 0, 0, 0);
                     namesInsert.run(countryID, aname, "country", cc, "", "", 0, 0);
@@ -235,7 +170,6 @@ export async function ingestGeonamesAliases(db, countries, geonamesDir, onProgre
                     adminMap.set(f[10], rid);
                 }
             }
-            // Re-parent regions + ancestor them to the (now-known) country.
             if (countryID >= 0) {
                 for (const rid of adminMap.values()) {
                     db.prepare("UPDATE spr SET parent_id = ? WHERE id = ?").run(countryID, rid);
@@ -243,10 +177,9 @@ export async function ingestGeonamesAliases(db, countries, geonamesDir, onProgre
                 }
             }
         }
-        // Emit pass — a second stream over the same file, off the page cache the survey pass just warmed.
-        for await (const f of TSVSpliterator.fromAsync(file, { header: false })) {
+        for await (const f of readUnquotedTSV(file)) {
             if (f[6] !== "P")
-                continue; // populated places only
+                continue;
             const lat = Number(f[4]);
             const lon = Number(f[5]);
             if (!Number.isFinite(lat) || !Number.isFinite(lon))
@@ -255,18 +188,10 @@ export async function ingestGeonamesAliases(db, countries, geonamesDir, onProgre
             if (!name)
                 continue;
             const nid = id++;
-            // #267: link to the locality's region (else country) for gap countries; -1 (orphan) otherwise.
             const regionID = addAdmin ? (adminMap.get(f[10] ?? "") ?? -1) : -1;
             const parentID = regionID >= 0 ? regionID : addAdmin && countryID >= 0 ? countryID : -1;
-            // Point bbox — a GeoNames row is a centroid; the candidate's region-bbox disambiguation just
-            // sees it as contained in itself, fine for a locality.
             sprInsert.run(nid, parentID, name, "locality", cc, lat, lon, lat, lon, lat, lon, 1, 0, 0, 0, 0, 0);
             namesInsert.run(nid, name, "locality", cc, "", "", 0, 0);
-            // The SELF row is unconditional (#1514). `populateAncestors` writes one for every spr row it
-            // sees, so a full build's freeze phase produces it either way — but the freeze does not run on
-            // the fold-on-copy path, and since the purge now clears this range the fold has to emit
-            // everything the closure would. Measured on the 161-country fold: 212,993 self rows, exactly the
-            // non-gap countries' localities, which is what the base artifact carried and a fold-on-copy lost.
             ancestorInsert.run(nid, nid, "locality");
             if (addAdmin) {
                 if (regionID >= 0) {
@@ -279,7 +204,10 @@ export async function ingestGeonamesAliases(db, countries, geonamesDir, onProgre
             const seen = new Set([name]);
             const tags = v2?.get(Number(f[0]));
             for (const raw of [f[2] ?? "", ...(f[3] ? f[3].split(",") : [])]) {
-                const alt = clean(raw);
+                const alt = cleanAlias(raw);
+                if (!alt && raw.trim()) {
+                    refused++;
+                }
                 if (alt && !seen.has(alt)) {
                     seen.add(alt);
                     const tag = tags?.get(alt);
@@ -292,7 +220,7 @@ export async function ingestGeonamesAliases(db, countries, geonamesDir, onProgre
             }
             nc++;
         }
-        report({ country: cc, places: nc, skipped: false });
+        report({ country: cc, places: nc, skipped: false, aliasesRefused: refused });
         total += nc;
     }
     db.exec("COMMIT");