@mailwoman/resolver-wof-sqlite 9.4.0 → 10.1.0

package/out/fts/query.js

@@ -3,9 +3,7 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Query shaping for the FTS5 lookup: placetype normalization and the MATCH-expression sanitizer.
- *   Both turn a caller's loose input into something SQLite's FTS5 parser accepts without throwing —
- *   an unescaped quote or a bare `*` is a syntax error, not an empty result.
+ * Query shaping for the FTS5 lookup: placetype normalization and the match-expression sanitizer.
  */
 export function normalizePlacetypes(p) {
     if (!p)
@@ -13,30 +11,7 @@ export function normalizePlacetypes(p) {
     return Array.isArray(p) ? p : [p];
 }
 /**
- * Make an arbitrary user-typed string safe for FTS5 MATCH.
- *
- * FTS5 has its own query syntax (`"phrase"`, `term1 OR term2`, `prefix*`, NEAR/N, etc.). Letting raw user input through
- * means a user typing `Paris's` or `St. (Petersburg)` causes a syntax error.
- *
- * Per-token rules:
- *
- * - Strip all punctuation except trailing `*` from each whitespace-separated token.
- * - **Trailing `*`** is preserved as FTS5 **prefix syntax** — `627*` becomes the literal `627*` (unquoted). The caller
- *   signaled they want a prefix; respect that.
- * - All other tokens are wrapped in `"..."` as a single-word phrase. Conservative — handles apostrophes, parens, accented
- *   input, etc. safely.
- * - Multiple tokens join with implicit AND.
- *
- * Examples:
- *
- * - `"Paris"` → `"Paris"` (phrase)
- * - `"627*"` → `627*` (prefix)
- * - `"St. (Petersburg)"` → `"St" "Petersburg"` (two phrases, AND-joined)
- * - `"Thiron-Gardais"` → `"Thiron" "Gardais"` (intra-token punctuation SPLITS — #945; fusing to `ThironGardais` matched
- *   nothing because the FTS doc tokenizes the hyphenated name as two terms)
- * - `"110 00"` with `fuseTokens` (postcode-typed) → `"110" "00"` per-token fused — the #920 name law
- * - `"Pari* TX"` → `Pari* "TX"` (mixed prefix + phrase)
- * - `"*"` alone → `""` (no body → drop)
+ * Make an arbitrary user-typed string safe for FTS5 match.
  */
 export function sanitizeFTSQuery(text, opts) {
     const out = [];
@@ -45,8 +20,7 @@ export function sanitizeFTSQuery(text, opts) {
         if (!trimmed)
             continue;
         const hasPrefixStar = trimmed.endsWith("*");
-        // #920 name law (postcode-typed queries ONLY): delete intra-token punctuation and FUSE the
-        // remainder — postal names are stored in this collapsed shape ("SW1A" stays one term).
+        // Fused mode deletes intra-token punctuation because postal names are stored in that collapsed shape.
         if (opts?.fuseTokens) {
             const body = trimmed.replaceAll(/[^\p{L}\p{N}]/gu, "");
             if (!body)
@@ -54,19 +28,14 @@ export function sanitizeFTSQuery(text, opts) {
             out.push(hasPrefixStar ? `${body}*` : `"${body.replaceAll('"', '""')}"`);
             continue;
         }
-        // Everything else SPLITS on intra-token punctuation — the behavior the docstring always
-        // promised ("St. (Petersburg)" → two phrases). The old code DELETED punctuation instead,
-        // fusing "Thiron-Gardais" into the unmatchable single term `ThironGardais` while the FTS
-        // doc holds two terms (#945 — the entire hyphenated-name class missed at the raw lookup;
-        // masked for years because pre-splice tokenizers never emitted hyphen-preserved values).
-        const parts = trimmed.split(/[^\p{L}\p{N}]+/u).filter((part) => part.length > 0);
+        const parts = trimmed.split(/[^\p{L}\p{N}]+/u).filter((part) => part.length);
         if (!parts.length)
             continue;
         for (let i = 0; i < parts.length; i++) {
             const body = parts[i].replaceAll("*", "");
             if (!body)
                 continue;
-            // The caller's trailing `*` applies to the FINAL part ("Thiron-Gard*" → "Thiron" Gard*).
+            // The caller's trailing `*` applies only to the final part.
             out.push(hasPrefixStar && i === parts.length - 1 ? `${body}*` : `"${body.replaceAll('"', '""')}"`);
         }
     }

package/out/fts/query.js.map

@@ -1 +1 @@
-{"version":3,"file":"query.js","sourceRoot":"","sources":["../../lib/fts/query.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,MAAM,UAAU,mBAAmB,CAAC,CAA8B;IACjE,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IAEnB,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;AAClC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,IAA+B;IAC7E,MAAM,GAAG,GAAa,EAAE,CAAA;IAExB,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7D,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAA;QAE/B,IAAI,CAAC,OAAO;YAAE,SAAQ;QACtB,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;QAE3C,2FAA2F;QAC3F,uFAAuF;QACvF,IAAI,IAAI,EAAE,UAAU,EAAE,CAAC;YACtB,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAA;YAEtD,IAAI,CAAC,IAAI;gBAAE,SAAQ;YACnB,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAA;YAExE,SAAQ;QACT,CAAC;QAED,wFAAwF;QACxF,yFAAyF;QACzF,yFAAyF;QACzF,yFAAyF;QACzF,yFAAyF;QACzF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAEhF,IAAI,CAAC,KAAK,CAAC,MAAM;YAAE,SAAQ;QAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;YAE1C,IAAI,CAAC,IAAI;gBAAE,SAAQ;YACnB,yFAAyF;YACzF,GAAG,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAA;QACnG,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AACrB,CAAC"}
+{"version":3,"file":"query.js","sourceRoot":"","sources":["../../lib/fts/query.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,MAAM,UAAU,mBAAmB,CAAC,CAA8B;IACjE,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IAEnB,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;AAClC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,IAA+B;IAC7E,MAAM,GAAG,GAAa,EAAE,CAAA;IAExB,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7D,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAA;QAE/B,IAAI,CAAC,OAAO;YAAE,SAAQ;QACtB,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;QAE3C,sGAAsG;QACtG,IAAI,IAAI,EAAE,UAAU,EAAE,CAAC;YACtB,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAA;YAEtD,IAAI,CAAC,IAAI;gBAAE,SAAQ;YACnB,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAA;YAExE,SAAQ;QACT,CAAC;QAED,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAE5E,IAAI,CAAC,KAAK,CAAC,MAAM;YAAE,SAAQ;QAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;YAE1C,IAAI,CAAC,IAAI;gBAAE,SAAQ;YACnB,4DAA4D;YAC5D,GAAG,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAA;QACnG,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AACrB,CAAC"}

package/out/fts.d.ts

@@ -0,0 +1,126 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+import type { DatabaseClient } from "@mailwoman/sqlite/client";
+/**
+ * The name of the FTS5 virtual table that indexes WOF place names for `match` queries.
+ */
+export declare const PLACE_SEARCH_TABLE = "place_search";
+/**
+ * The separator between aliases in the `alt_names` bag.
+ * It stops a phrase query from matching across two aliases.
+ *
+ * FTS5 assigns positions only to tokens.
+ * Spaces and punctuation are not tokens and would leave the aliases adjacent,
+ * but unicode61 indexes the Private Use Area codepoint U+E000 as a token.
+ *
+ * Query sanitizers strip this character, so no user query can match the separator.
+ */
+export declare const ALIAS_SEPARATOR = "\uE000";
+/**
+ * Folds a query or name for exact-tier comparison by lowercasing, trimming
+ * and collapsing internal whitespace.
+ *
+ * Every exact-tier consumer uses this function, so its output compares equal to
+ * the aliases that {@link aliasBagExactMatch} parses.
+ */
+export declare function foldQueryText(input: string): string;
+/**
+ * Returns whether any alias in an `alt_names` bag exactly equals the folded query.
+ *
+ * A bag without {@link ALIAS_SEPARATOR} is a legacy space-joined bag without alias boundaries.
+ * For such a bag, the function checks word-bounded containment only when no candidate matched strictly.
+ *
+ * Containment by itself would promote fragments such as "York" inside "New York City".
+ *
+ * @param altNames The `alt_names` bag from `place_search`, or null when the row has no aliases.
+ * @param normalizedQuery The query folded by {@link foldQueryText}.
+ * @param anyStrictExact Whether any candidate already matched on its canonical name or region abbreviation.
+ * Only legacy bags use it.
+ */
+export declare function aliasBagExactMatch(altNames: string | null, normalizedQuery: string, anyStrictExact: boolean): boolean;
+/**
+ * The name of the R*Tree virtual table that indexes WOF place bounding boxes for bbox and proximity lookups.
+ */
+export declare const PLACE_BBOX_TABLE = "place_bbox";
+/**
+ * The name of the sparse table that stores `wof:population` per place.
+ *
+ * The lookup uses this value to apply the population ranking boost.
+ *
+ * A place without a row gets no boost and no penalty.
+ */
+export declare const PLACE_POPULATION_TABLE = "place_population";
+/**
+ * The name of the table of per-place `referential` and `encyclopedic` scores,
+ * built by `mailwoman gazetteer importance`.
+ *
+ * The lookup copies `encyclopedic` onto results without ranking by it.
+ * It refuses to read an older table with a single `importance` column.
+ */
+export declare const PLACE_IMPORTANCE_TABLE = "place_importance";
+/**
+ * The indexes a {@link buildPlaceSearchFTS} run built and their row counts.
+ */
+export interface BuildPlaceSearchFTSResult {
+    /**
+     * Whether this call built the FTS5 index.
+     * It is false when an existing index was kept.
+     */
+    created: boolean;
+    /**
+     * The row count of the `place_search` table after the call.
+     */
+    indexedRows: number;
+    /**
+     * Whether this call built the R*Tree bbox index.
+     * It is false when an existing index was kept.
+     */
+    bboxCreated: boolean;
+    /**
+     * The row count of the `place_bbox` R*Tree after the call.
+     */
+    bboxIndexedRows: number;
+    /**
+     * The wall-clock duration of the whole build in milliseconds.
+     */
+    durationMs: number;
+}
+/**
+ * Options for {@link buildPlaceSearchFTS}.
+ */
+export interface BuildPlaceSearchFTSOpts {
+    /**
+     * Whether to drop and rebuild existing `place_search` and `place_bbox` tables,
+     * such as after an `spr` or `names` update.
+     * By default an existing index is kept.
+     */
+    drop?: boolean;
+    /**
+     * Receives each build phase as it begins.
+     * A planet-scale build takes minutes to populate.
+     */
+    onProgress?: (phase: "checking" | "dropping" | "creating" | "populating" | "creating-bbox" | "populating-bbox" | "done", detail?: string) => void;
+}
+/**
+ * Builds the `place_search` FTS5 table and the `place_bbox` R*Tree from a WOF
+ * database's `spr` and `names` tables.
+ *
+ * Without `drop: true`, an existing index is kept.
+ */
+export declare function buildPlaceSearchFTS<DB>(db: DatabaseClient<DB>, opts?: BuildPlaceSearchFTSOpts): BuildPlaceSearchFTSResult;
+/**
+ * Returns whether the connected database has the `place_search` FTS table.
+ */
+export declare function placeSearchFTSExists<DB>(db: DatabaseClient<DB>): boolean;
+/**
+ * Returns whether the connected database has the `place_bbox` R*Tree table.
+ */
+export declare function placeBboxExists<DB>(db: DatabaseClient<DB>): boolean;
+/**
+ * Returns whether the connected database has the `place_population` table.
+ */
+export declare function placePopulationExists<DB>(db: DatabaseClient<DB>): boolean;
+//# sourceMappingURL=fts.d.ts.map

package/out/fts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fts.d.ts","sourceRoot":"","sources":["../lib/fts.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAG9D;;GAEG;AACH,eAAO,MAAM,kBAAkB,iBAAiB,CAAA;AAEhD;;;;;;;;;GASG;AACH,eAAO,MAAM,eAAe,WAAW,CAAA;AAIvC;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEnD;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,GAAG,OAAO,CAUrH;AAED;;GAEG;AACH,eAAO,MAAM,gBAAgB,eAAe,CAAA;AAE5C;;;;;;GAMG;AACH,eAAO,MAAM,sBAAsB,qBAAqB,CAAA;AAExD;;;;;;GAMG;AACH,eAAO,MAAM,sBAAsB,qBAAqB,CAAA;AAExD;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACzC;;;OAGG;IACH,OAAO,EAAE,OAAO,CAAA;IAEhB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IAEnB;;;OAGG;IACH,WAAW,EAAE,OAAO,CAAA;IAEpB;;OAEG;IACH,eAAe,EAAE,MAAM,CAAA;IAEvB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACvC;;;;OAIG;IACH,IAAI,CAAC,EAAE,OAAO,CAAA;IAEd;;;OAGG;IACH,UAAU,CAAC,EAAE,CACZ,KAAK,EAAE,UAAU,GAAG,UAAU,GAAG,UAAU,GAAG,YAAY,GAAG,eAAe,GAAG,iBAAiB,GAAG,MAAM,EACzG,MAAM,CAAC,EAAE,MAAM,KACX,IAAI,CAAA;CACT;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,EAAE,EACrC,EAAE,EAAE,cAAc,CAAC,EAAE,CAAC,EACtB,IAAI,GAAE,uBAA4B,GAChC,yBAAyB,CA6G3B;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,EAAE,EAAE,EAAE,EAAE,cAAc,CAAC,EAAE,CAAC,GAAG,OAAO,CAExE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,EAAE,EAAE,EAAE,EAAE,cAAc,CAAC,EAAE,CAAC,GAAG,OAAO,CAEnE;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,EAAE,EAAE,EAAE,EAAE,cAAc,CAAC,EAAE,CAAC,GAAG,OAAO,CAEzE"}

package/out/fts.js

@@ -0,0 +1,188 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+import { tableExists } from "@mailwoman/sqlite/introspection";
+/**
+ * The name of the FTS5 virtual table that indexes WOF place names for `match` queries.
+ */
+export const PLACE_SEARCH_TABLE = "place_search";
+/**
+ * The separator between aliases in the `alt_names` bag.
+ * It stops a phrase query from matching across two aliases.
+ *
+ * FTS5 assigns positions only to tokens.
+ * Spaces and punctuation are not tokens and would leave the aliases adjacent,
+ * but unicode61 indexes the Private Use Area codepoint U+E000 as a token.
+ *
+ * Query sanitizers strip this character, so no user query can match the separator.
+ */
+export const ALIAS_SEPARATOR = "\uE000";
+const ALIAS_SEPARATOR_CODEPOINT = ALIAS_SEPARATOR.codePointAt(0);
+/**
+ * Folds a query or name for exact-tier comparison by lowercasing, trimming
+ * and collapsing internal whitespace.
+ *
+ * Every exact-tier consumer uses this function, so its output compares equal to
+ * the aliases that {@link aliasBagExactMatch} parses.
+ */
+export function foldQueryText(input) {
+    return input.toLowerCase().trim().replaceAll(/\s+/g, " ");
+}
+/**
+ * Returns whether any alias in an `alt_names` bag exactly equals the folded query.
+ *
+ * A bag without {@link ALIAS_SEPARATOR} is a legacy space-joined bag without alias boundaries.
+ * For such a bag, the function checks word-bounded containment only when no candidate matched strictly.
+ *
+ * Containment by itself would promote fragments such as "York" inside "New York City".
+ *
+ * @param altNames The `alt_names` bag from `place_search`, or null when the row has no aliases.
+ * @param normalizedQuery The query folded by {@link foldQueryText}.
+ * @param anyStrictExact Whether any candidate already matched on its canonical name or region abbreviation.
+ * Only legacy bags use it.
+ */
+export function aliasBagExactMatch(altNames, normalizedQuery, anyStrictExact) {
+    if (altNames === null || altNames === "" || !normalizedQuery)
+        return false;
+    if (altNames.includes(ALIAS_SEPARATOR)) {
+        return altNames.split(ALIAS_SEPARATOR).some((alias) => foldQueryText(alias) === normalizedQuery);
+    }
+    if (anyStrictExact)
+        return false;
+    return ` ${foldQueryText(altNames)} `.includes(` ${normalizedQuery} `);
+}
+/**
+ * The name of the R*Tree virtual table that indexes WOF place bounding boxes for bbox and proximity lookups.
+ */
+export const PLACE_BBOX_TABLE = "place_bbox";
+/**
+ * The name of the sparse table that stores `wof:population` per place.
+ *
+ * The lookup uses this value to apply the population ranking boost.
+ *
+ * A place without a row gets no boost and no penalty.
+ */
+export const PLACE_POPULATION_TABLE = "place_population";
+/**
+ * The name of the table of per-place `referential` and `encyclopedic` scores,
+ * built by `mailwoman gazetteer importance`.
+ *
+ * The lookup copies `encyclopedic` onto results without ranking by it.
+ * It refuses to read an older table with a single `importance` column.
+ */
+export const PLACE_IMPORTANCE_TABLE = "place_importance";
+/**
+ * Builds the `place_search` FTS5 table and the `place_bbox` R*Tree from a WOF
+ * database's `spr` and `names` tables.
+ *
+ * Without `drop: true`, an existing index is kept.
+ */
+export function buildPlaceSearchFTS(db, opts = {}) {
+    const start = Date.now();
+    const onProgress = opts.onProgress ?? (() => { });
+    onProgress("checking");
+    const ftsExisting = tableExists(db, PLACE_SEARCH_TABLE);
+    const bboxExisting = tableExists(db, PLACE_BBOX_TABLE);
+    let ftsCreated = false;
+    if (ftsExisting && opts.drop) {
+        onProgress("dropping", PLACE_SEARCH_TABLE);
+        db.exec(`DROP TABLE ${PLACE_SEARCH_TABLE}`);
+    }
+    if (!ftsExisting || opts.drop) {
+        onProgress("creating");
+        db.exec(`
+			CREATE VIRTUAL TABLE ${PLACE_SEARCH_TABLE} USING fts5(
+				wof_id UNINDEXED,
+				name,
+				alt_names,
+				tokenize = 'unicode61 remove_diacritics 2'
+			);
+		`);
+        onProgress("populating");
+        db.exec(`
+			INSERT INTO ${PLACE_SEARCH_TABLE} (wof_id, name, alt_names)
+			SELECT
+				spr.id,
+				spr.name,
+				COALESCE((
+					SELECT GROUP_CONCAT(
+						REPLACE(name, char(${ALIAS_SEPARATOR_CODEPOINT}), ' '),
+						' ' || char(${ALIAS_SEPARATOR_CODEPOINT}) || ' '
+					) || ' ' || char(${ALIAS_SEPARATOR_CODEPOINT})
+					FROM names WHERE names.id = spr.id
+				), '')
+			FROM spr
+			WHERE spr.is_current != 0
+				AND spr.is_deprecated = 0
+				AND spr.name IS NOT NULL;
+		`);
+        ftsCreated = true;
+    }
+    const ftsCountRow = db.prepare(`SELECT COUNT(*) AS n FROM ${PLACE_SEARCH_TABLE}`).get();
+    let bboxCreated = false;
+    if (bboxExisting && opts.drop) {
+        onProgress("dropping", PLACE_BBOX_TABLE);
+        db.exec(`DROP TABLE ${PLACE_BBOX_TABLE}`);
+    }
+    if (!bboxExisting || opts.drop) {
+        onProgress("creating-bbox");
+        db.exec(`
+			CREATE VIRTUAL TABLE ${PLACE_BBOX_TABLE} USING rtree(
+				id,
+				min_lat, max_lat,
+				min_lon, max_lon
+			);
+		`);
+        onProgress("populating-bbox");
+        db.exec(`
+			INSERT INTO ${PLACE_BBOX_TABLE} (id, min_lat, max_lat, min_lon, max_lon)
+			SELECT
+				spr.id,
+				spr.min_latitude,
+				spr.max_latitude,
+				spr.min_longitude,
+				spr.max_longitude
+			FROM spr
+			WHERE spr.is_current != 0
+				AND spr.is_deprecated = 0
+				AND spr.min_latitude IS NOT NULL
+				AND spr.max_latitude IS NOT NULL
+				AND spr.min_longitude IS NOT NULL
+				AND spr.max_longitude IS NOT NULL
+				AND NOT (spr.min_latitude = 0 AND spr.max_latitude = 0
+				     AND spr.min_longitude = 0 AND spr.max_longitude = 0);
+		`);
+        bboxCreated = true;
+    }
+    const bboxCountRow = db.prepare(`SELECT COUNT(*) AS n FROM ${PLACE_BBOX_TABLE}`).get();
+    onProgress("done", `${ftsCountRow.n} FTS rows + ${bboxCountRow.n} bbox rows ` +
+        `(${ftsCreated ? "built" : "preexisting"} / ${bboxCreated ? "built" : "preexisting"})`);
+    return {
+        created: ftsCreated,
+        indexedRows: ftsCountRow.n,
+        bboxCreated,
+        bboxIndexedRows: bboxCountRow.n,
+        durationMs: Date.now() - start,
+    };
+}
+/**
+ * Returns whether the connected database has the `place_search` FTS table.
+ */
+export function placeSearchFTSExists(db) {
+    return tableExists(db, PLACE_SEARCH_TABLE);
+}
+/**
+ * Returns whether the connected database has the `place_bbox` R*Tree table.
+ */
+export function placeBboxExists(db) {
+    return tableExists(db, PLACE_BBOX_TABLE);
+}
+/**
+ * Returns whether the connected database has the `place_population` table.
+ */
+export function placePopulationExists(db) {
+    return tableExists(db, PLACE_POPULATION_TABLE);
+}
+//# sourceMappingURL=fts.js.map

package/out/fts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fts.js","sourceRoot":"","sources":["../lib/fts.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAA;AAE7D;;GAEG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,cAAc,CAAA;AAEhD;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,QAAQ,CAAA;AAEvC,MAAM,yBAAyB,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC,CAAW,CAAA;AAE1E;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,KAAa;IAC1C,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;AAC1D,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,kBAAkB,CAAC,QAAuB,EAAE,eAAuB,EAAE,cAAuB;IAC3G,IAAI,QAAQ,KAAK,IAAI,IAAI,QAAQ,KAAK,EAAE,IAAI,CAAC,eAAe;QAAE,OAAO,KAAK,CAAA;IAE1E,IAAI,QAAQ,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;QACxC,OAAO,QAAQ,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,eAAe,CAAC,CAAA;IACjG,CAAC;IAED,IAAI,cAAc;QAAE,OAAO,KAAK,CAAA;IAEhC,OAAO,IAAI,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,eAAe,GAAG,CAAC,CAAA;AACvE,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,YAAY,CAAA;AAE5C;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,kBAAkB,CAAA;AAExD;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,kBAAkB,CAAA;AAuDxD;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAClC,EAAsB,EACtB,IAAI,GAA4B,EAAE;IAElC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IACxB,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;IAEhD,UAAU,CAAC,UAAU,CAAC,CAAA;IACtB,MAAM,WAAW,GAAG,WAAW,CAAC,EAAE,EAAE,kBAAkB,CAAC,CAAA;IACvD,MAAM,YAAY,GAAG,WAAW,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAA;IAEtD,IAAI,UAAU,GAAG,KAAK,CAAA;IAEtB,IAAI,WAAW,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QAC9B,UAAU,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAA;QAC1C,EAAE,CAAC,IAAI,CAAC,cAAc,kBAAkB,EAAE,CAAC,CAAA;IAC5C,CAAC;IAED,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,UAAU,CAAC,UAAU,CAAC,CAAA;QAEtB,EAAE,CAAC,IAAI,CAAC;0BACgB,kBAAkB;;;;;;GAMzC,CAAC,CAAA;QAEF,UAAU,CAAC,YAAY,CAAC,CAAA;QAExB,EAAE,CAAC,IAAI,CAAC;iBACO,kBAAkB;;;;;;2BAMR,yBAAyB;oBAChC,yBAAyB;wBACrB,yBAAyB;;;;;;;GAO9C,CAAC,CAAA;QAEF,UAAU,GAAG,IAAI,CAAA;IAClB,CAAC;IAED,MAAM,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC,6BAA6B,kBAAkB,EAAE,CAAC,CAAC,GAAG,EAAmB,CAAA;IAExG,IAAI,WAAW,GAAG,KAAK,CAAA;IAEvB,IAAI,YAAY,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,UAAU,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAA;QACxC,EAAE,CAAC,IAAI,CAAC,cAAc,gBAAgB,EAAE,CAAC,CAAA;IAC1C,CAAC;IAED,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,UAAU,CAAC,eAAe,CAAC,CAAA;QAE3B,EAAE,CAAC,IAAI,CAAC;0BACgB,gBAAgB;;;;;GAKvC,CAAC,CAAA;QAEF,UAAU,CAAC,iBAAiB,CAAC,CAAA;QAE7B,EAAE,CAAC,IAAI,CAAC;iBACO,gBAAgB;;;;;;;;;;;;;;;;GAgB9B,CAAC,CAAA;QAEF,WAAW,GAAG,IAAI,CAAA;IACnB,CAAC;IAED,MAAM,YAAY,GAAG,EAAE,CAAC,OAAO,CAAC,6BAA6B,gBAAgB,EAAE,CAAC,CAAC,GAAG,EAAmB,CAAA;IAEvG,UAAU,CACT,MAAM,EACN,GAAG,WAAW,CAAC,CAAC,eAAe,YAAY,CAAC,CAAC,aAAa;QACzD,IAAI,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,MAAM,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,GAAG,CACvF,CAAA;IAED,OAAO;QACN,OAAO,EAAE,UAAU;QACnB,WAAW,EAAE,WAAW,CAAC,CAAC;QAC1B,WAAW;QACX,eAAe,EAAE,YAAY,CAAC,CAAC;QAC/B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;KAC9B,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAK,EAAsB;IAC9D,OAAO,WAAW,CAAC,EAAE,EAAE,kBAAkB,CAAC,CAAA;AAC3C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAK,EAAsB;IACzD,OAAO,WAAW,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAA;AACzC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAK,EAAsB;IAC/D,OAAO,WAAW,CAAC,EAAE,EAAE,sBAAsB,CAAC,CAAA;AAC/C,CAAC"}

package/out/geonames/aliases.d.ts

@@ -2,99 +2,58 @@
  * @copyright Sister Software
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
- *
- *   #743/#193 — fold GeoNames bilingual / alt-language place-names into a WOF/unified admin DB as
- *   first-class places. The hard-filter recall gap on bilingual countries (the address says
- *   "Karjaa" but the table holds the Swedish "Karis") is missing alt-LANGUAGE names, not missing
- *   places: the WOF/Overture `names` carried only the primary, so the candidate build's Latin-alias
- *   explode (build-candidate pass 2) had nothing to widen. GeoNames' per-country dump carries the
- *   variants inline (the Karis row's `alternatenames` includes "Karjaa").
- *
- *   For each POPULATED place (feature class `P`) this writes an `spr` row + `names` rows (primary +
- *   Latin alt-names) + population into the SAME tables the WOF/Overture paths use — synthetic ids
- *   based at {@link GEONAMES_ID_BASE} so the three sources never collide. The caller then rebuilds
- *   `place_search` ({@link buildPlaceSearchFTS} with `drop: true`) so the candidate build carries
- *   Karjaa↔Karis. Proven (FI hard-resolve 69.5 → 85.8 %, coverage 74.4 → 94.0 %); duplicating a
- *   place already held under another source is benign — the rows share name_key+coord and the
- *   candidate ranking dedupes by score.
- *
- *   This is the package home so the canonical `build-unified-wof --geonames-countries`, the
- *   pipeline fold (`gazetteer-pipeline/admin/fold-geonames`), AND the `mailwoman gazetteer` commands all share
- *   ONE implementation. GeoNames dump = `download.geonames.org/export/dump/<CC>.zip` → `<CC>.txt`
- *   (TSV).
  */
 import type { DatabaseClient } from "@mailwoman/sqlite/client";
 import { type PathBuilderLike } from "path-ts";
 import type { WOFDatabase } from "#schema";
 /**
- * Synthetic id base for GeoNames-sourced rows (#743/#193) — above Overture's 8e12 so the three sources (WOF real ids,
- * Overture, GeoNames) never collide in a combined DB.
- */
-export declare const GEONAMES_ID_BASE = 9000000000000;
-/**
- * Clear everything the alias fold owns — `[GEONAMES_ID_BASE, GEONAMES_POSTAL_ID_BASE)` across {@link
- * FOLD_OWNED_TABLES}
- * — so a fold's output is a function of its country list and dumps ALONE, never of what the DB happened to hold first.
- * Returns the rows removed per table.
- *
- * This exists because the synthetic id is a POSITION (`GEONAMES_ID_BASE + n`, counted across the country list in
- * order), not a derivation from the geonameid: change the list, its order, or a dump's contents and every subsequent id
- * shifts. Without the purge, a second fold overwrote the `spr` PREFIX it reached (`INSERT OR REPLACE`) while its
- * `names`/`ancestors` rows merely appended (bare `INSERT`) and `place_population` skipped unpopulated places entirely —
- * so the previous run's names, ancestry and populations stayed bound to ids now describing other places.
+ * Deletes every row the GeoNames alias fold owns, with ids in
+ * `[GEONAMES_ID_BASE, GEONAMES_POSTAL_ID_BASE)`, and returns the count removed per table.
  *
- * Measured on the live 2026-08-05 artifact: a 14-country re-fold over the 161-country fold baked into
- * `admin-global-priority.db` put 522,184 of 2,110,096 name rows in the range (24.7 %) on a place from a different
- * country. Gaborone/BW at id 9000000121151 became Aichegg/AT and kept all 26 of its names; Kinshasa's 16,000,000
- * population landed on a Lithuanian hamlet.
- *
- * The upper bound is NOT the end of the id space. Each later extract owns a range above this one — GeoNames-postal @
- * 9.5e12, NL-PC6 @ 9.6e12, Code-Point @ 9.7e12, NI @ 9.8e12 — and a purge that ran past {@link GEONAMES_POSTAL_ID_BASE}
- * would delete them.
+ * Fold ids are assigned sequentially, so a re-fold without this purge would attach
+ * stale names, ancestors and populations to other places.
+ * The upper bound leaves the postal and postcode id ranges above it untouched.
  */
 export declare function purgeGeonamesAliasRange<DB>(db: DatabaseClient<DB>): Record<string, number>;
 /**
- * Per-country progress for the ingest — one event per country dump processed (or skipped).
+ * One progress event per country dump that the ingest processed or skipped.
  */
 export interface GeonamesIngestProgress {
     /**
-     * ISO 3166-1 alpha-2 code.
+     * The ISO 3166-1 alpha-2 country code.
      */
     country: string;
     /**
-     * Populated places ingested from this country's dump (0 when skipped).
+     * The number of populated places ingested from this country's dump, or 0 when it was skipped.
      */
     places: number;
     /**
-     * True when the country's `<CC>.txt` dump was missing — the country is skipped, not fatal.
+     * Whether the country's `<CC>.txt` dump was missing.
+     *
+     * A missing dump skips the country without failing the ingest.
      */
     skipped: boolean;
+    /**
+     * The number of alternate names that the alias filter rejected.
+     *
+     * It distinguishes a sparse source from aliases that the fold discarded.
+     */
+    aliasesRefused?: number;
 }
 /**
- * Fold the GeoNames `P`-class places (+ their Latin alt-names) for `countries` into `db`'s `spr` / `names` /
- * `place_population` tables. Returns the total places ingested.
+ * Formats a GeoNames fold progress event as one log line.
+ */
+export declare function formatGeonamesIngestProgress(event: GeonamesIngestProgress, missingFile?: PathBuilderLike): string;
+/**
+ * Purges the fold's id range, then adds GeoNames `P`-class places and their
+ * alternate names for `countries` to `db`.
+ * It returns the number of places ingested.
  *
- * `onProgress` receives one event per country (default: a stderr line, matching the build scripts' legacy output). The
- * caller MUST rebuild `place_search` afterward (`buildPlaceSearchFTS(db, { drop: true })`) for the new names to reach
- * the candidate build's alias pass.
+ * The caller must then rebuild `place_search` with `buildPlaceSearchFTS(db, { drop: true })`
+ * to make the new names searchable.
  */
 export declare function ingestGeonamesAliases(db: DatabaseClient<WOFDatabase>, countries: string[], geonamesDir: PathBuilderLike, onProgress?: (event: GeonamesIngestProgress) => void, opts?: {
-    /**
-     * #267: the countries for which to ALSO fold the GeoNames A-class admin (PCLI country + ADM1 regions) and link each
-     * locality's `parent_id` + ancestry chain (locality → region → country). PER-COUNTRY because a country that already
-     * carries WOF admin would double up — pass only the ZERO-COVERAGE gap countries (the coverage-expansion targets),
-     * never the EU alias set. Without admin, a gap country's localities are orphans (`parent_id=-1`, no ancestors), so
-     * `parentID` scoping and adminCoherence can't reach them and "Tbilisi, GE" can't resolve.
-     */
     adminForCountries?: ReadonlySet<string>;
-    /**
-     * #936: directory of per-country alternateNamesV2 dumps
-     * (`download.geonames.org/export/dump/alternatenames/<CC>.zip` → `<CC>.txt`). When a country's file is present,
-     * alias rows gain their language tag, `privateuse` ("preferred" from `isPreferredName`), and the `official` bit
-     * (language is CLDR-official for the country, colloquial/historic excluded — the rule the #936 risk probe measured
-     * at 7 new name-exact collisions globally). The main dump's bare `alternatenames` list still decides WHICH rows
-     * exist; V2 only decorates them. Missing file = the pre-#936 untagged behavior, not an error.
-     */
     alternateDir?: PathBuilderLike;
 }): Promise<number>;
 //# sourceMappingURL=aliases.d.ts.map

package/out/geonames/aliases.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"aliases.d.ts","sourceRoot":"","sources":["../../lib/geonames/aliases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAIH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAC9D,OAAO,EAAQ,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAIpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAE1C;;;GAGG;AACH,eAAO,MAAM,gBAAgB,gBAAoB,CAAA;AAQjD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,uBAAuB,CAAC,EAAE,EAAE,EAAE,EAAE,cAAc,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAY1F;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,OAAO,EAAE,OAAO,CAAA;CAChB;AAqFD;;;;;;;GAOG;AACH,wBAAsB,qBAAqB,CAC1C,EAAE,EAAE,cAAc,CAAC,WAAW,CAAC,EAC/B,SAAS,EAAE,MAAM,EAAE,EACnB,WAAW,EAAE,eAAe,EAC5B,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,EACpD,IAAI,CAAC,EAAE;IACN;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAA;IACvC;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,eAAe,CAAA;CAC9B,GACC,OAAO,CAAC,MAAM,CAAC,CA8MjB"}
+{"version":3,"file":"aliases.d.ts","sourceRoot":"","sources":["../../lib/geonames/aliases.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAC9D,OAAO,EAAE,KAAK,eAAe,EAAsB,MAAM,SAAS,CAAA;AAElE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAI1C;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CAAC,EAAE,EAAE,EAAE,EAAE,cAAc,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAY1F;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;;;OAIG;IACH,OAAO,EAAE,OAAO,CAAA;IAEhB;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAA;CACvB;AAED;;GAEG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,sBAAsB,EAAE,WAAW,CAAC,EAAE,eAAe,GAAG,MAAM,CAMjH;AAgED;;;;;;;GAOG;AACH,wBAAsB,qBAAqB,CAC1C,EAAE,EAAE,cAAc,CAAC,WAAW,CAAC,EAC/B,SAAS,EAAE,MAAM,EAAE,EACnB,WAAW,EAAE,eAAe,EAC5B,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,EACpD,IAAI,CAAC,EAAE;IACN,iBAAiB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAA;IAEvC,YAAY,CAAC,EAAE,eAAe,CAAA;CAC9B,GACC,OAAO,CAAC,MAAM,CAAC,CAyLjB"}