@mailwoman/resolver-wof-sqlite 9.0.0 → 9.2.0

package/search-fetch.ts

@@ -0,0 +1,256 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The FTS5 candidate fetch behind the fuzzy name match: the schema-qualified `place_search`
+ *   MATCH, its population-ordered companion fetch, and the raw row shape both of them return.
+ *
+ *   Bbox and near-with-radius narrow at the SQL level through SQLite's built-in `rtree`, whose index name and schema
+ *   live in `fts.ts` beside the FTS5 build. That is why this package pulls neither SpatiaLite nor turf: the R*Tree does
+ *   the narrowing, and the passes downstream of it operate on ≤ a few hundred candidates per query rather than the
+ *   whole corpus, so an exact haversine over the survivors is cheap.
+ */
+
+import type { DatabaseSync, SQLInputValue } from "node:sqlite"
+
+import { bboxAround } from "@mailwoman/spatial"
+
+import { PLACE_BBOX_TABLE, PLACE_POPULATION_TABLE } from "./fts.ts"
+import type { RankingWeights } from "./ranking-weights.ts"
+import { allRows } from "./sqlite-utils.ts"
+import type { FindPlaceQuery, WOFPlacetype } from "./types.ts"
+
+/**
+ * Query length at or below which the FTS window is widened. A two- or three-character query is almost always a region
+ * abbreviation, where the exact match can otherwise fall outside the window behind higher-bm25 partial hits — "NY"
+ * losing to "New York".
+ */
+const SHORT_QUERY_MAX_LENGTH = 3
+
+/**
+ * Over-fetch floor for SHORT (≤3-char) queries — region abbreviations like "NY"/"VT". An exact-abbrev holder's BM25 is
+ * poor (long multilingual alt-name document), so the normal `limit * 4` window can drop it before `exactMatchTiering`
+ * promotes it. 200 comfortably covers every same-abbrev region across the 12-country gazetteer (a 2-letter token
+ * matches a few dozen regions at most) while staying a cheap region-placetype fetch. See the `#fuzzyNameMatch`
+ * over-fetch comment.
+ */
+const SHORT_QUERY_OVERFETCH = 200
+
+/**
+ * How many rows the population-ordered companion fetch (#905) adds to the candidate pool. Small on purpose: its only
+ * job is to guarantee the FAMOUS holders of a name enter the pool at all — for "Paris"-class floods the bm25 window is
+ * saturated by thousands of tiny same-name rows and no boost inside the bm25-based ORDER BY can rescue a candidate
+ * whose bm25 is length-poisoned by ~15 points (see the fetch-site comment).
+ */
+const POPULATION_FETCH_LIMIT = 15
+
+export interface RawSearchRow {
+	id: number
+	name: string
+	placetype: string
+	country: string | null
+	parent_id: number | null
+	rank: number // BM25 (lower = better in SQLite); we negate to get higher-is-better
+	lat: number | null
+	lon: number | null
+	min_latitude: number | null
+	max_latitude: number | null
+	min_longitude: number | null
+	max_longitude: number | null
+	population: number | null // from the place_population aux table; null when missing
+	/**
+	 * From `place_importance.encyclopedic` when the shard's table carries the two-score split columns. NULL means the
+	 * place has no Wikipedia article, or the shard predates the split — absence either way, and never 0 (ROAD_TO_V9 §2).
+	 */
+	encyclopedic: number | null
+}
+
+/**
+ * Fetch the raw candidate rows for a name match on one shard: the BM25-ordered window over `place_search` (widened for
+ * short queries), plus the population-ordered companion fetch that keeps the prominent holders of a name pool-complete.
+ * `schemaName` is the routed shard's bare schema name — validated at construction, so it is interpolated directly.
+ */
+export function fetchSearchRows(options: {
+	db: DatabaseSync
+	schemaName: string
+	query: FindPlaceQuery
+	placetypes: WOFPlacetype[] | null
+	ftsQuery: string
+	limit: number
+	hasBboxIndex: ReadonlyMap<string, boolean>
+	hasPopulationIndex: ReadonlyMap<string, boolean>
+	encyclopedicClauses: ReadonlyMap<string, { select: string; join: string }>
+	weights: RankingWeights
+}): RawSearchRow[] {
+	const {
+		db,
+		schemaName: sch,
+		query,
+		placetypes,
+		ftsQuery,
+		limit,
+		hasBboxIndex,
+		hasPopulationIndex,
+		encyclopedicClauses,
+		weights,
+	} = options
+
+	// Over-fetch so post-scoring + exact-match tiering have room to re-rank. SHORT queries (a 2–3-char
+	// region abbreviation like "NY"/"VT") are the danger case the `exactMatchTiering` docstring flags:
+	// the exact-abbrev holder's BM25 is poor (its long multilingual alt-name document tanks the score),
+	// so under the normal `limit * 4` window it drops OUT of the candidate pool BEFORE tiering can
+	// promote it — "NY" then resolves to a token-matching foreign region (Highland, GB) instead of New
+	// York. Widen the window for short queries so the exact match is always present to be tiered.
+	// (Cross-country abbrev collisions — "VT" is BOTH Vermont and Viterbo — still need a country/
+	// postcode signal to disambiguate; this only rescues the window-drop class, not genuine ambiguity.
+	// With a `country` hint every abbrev resolves; bare + no-context lifts 7→10/15 US states.)
+	const ftsLimit =
+		query.text.trim().length <= SHORT_QUERY_MAX_LENGTH ? Math.max(limit * 4, SHORT_QUERY_OVERFETCH) : limit * 4
+
+	// Filter out historical / superseded / deprecated places by default — they live in the same
+	// spr table but should never win a contemporary lookup. `is_current = 0` is the only WOF
+	// value that means "not current"; both `-1` (modern) and `1` (legacy) mean current. See #91.
+	// Note: with schema-qualified FROM the bare `place_search` reference in MATCH resolves to
+	// the FROM table — required by FTS5 parser, see sharding.ts header comment.
+	const where: string[] = ["place_search MATCH ?", "spr.is_current != 0", "spr.is_deprecated = 0"]
+	const params: SQLInputValue[] = [ftsQuery]
+
+	if (placetypes && placetypes.length) {
+		where.push(`spr.placetype IN (${placetypes.map(() => "?").join(", ")})`)
+		params.push(...placetypes)
+	}
+
+	if (query.country) {
+		where.push("spr.country = ?")
+		params.push(query.country)
+	}
+
+	if (query.parentID !== undefined) {
+		where.push(`(spr.parent_id = ? OR spr.id IN (SELECT id FROM ${sch}.ancestors WHERE ancestor_id = ?))`)
+		params.push(query.parentID, query.parentID)
+	}
+
+	// Bbox + near-with-radius are SQL-level filters via the R*Tree. We only emit the JOIN when
+	// the active shard has the R*Tree; missing-but-requested is silently treated as no-bbox-
+	// filter so legacy DBs / shards-without-bbox don't crash.
+	const shardHasBbox = hasBboxIndex.get(sch) === true
+	const useBboxJoin = (query.bbox || query.near?.maxDistanceKm !== undefined) && shardHasBbox
+	let joinClause = `JOIN ${sch}.spr ON spr.id = place_search.wof_id`
+
+	if (useBboxJoin) {
+		joinClause += ` JOIN ${sch}.${PLACE_BBOX_TABLE} bbox ON bbox.id = spr.id`
+		// AABB intersection — both bbox sides must overlap. R*Tree handles this in O(log n).
+		const filterBox = query.bbox || bboxAround(query.near!.lat, query.near!.lon, query.near!.maxDistanceKm!)
+		where.push("bbox.min_lat <= ? AND bbox.max_lat >= ?", "bbox.min_lon <= ? AND bbox.max_lon >= ?")
+		params.push(filterBox.maxLat, filterBox.minLat, filterBox.maxLon, filterBox.minLon)
+	}
+
+	// LEFT JOIN the population aux table when present. Missing-on-this-shard means the SELECT
+	// just doesn't include the population column; the post-scoring loop treats it as 0.
+	const shardHasPopulation = hasPopulationIndex.get(sch) === true
+
+	const populationSelect = shardHasPopulation
+		? `${PLACE_POPULATION_TABLE}.population AS population`
+		: `NULL AS population`
+
+	const populationJoin = shardHasPopulation
+		? `LEFT JOIN ${sch}.${PLACE_POPULATION_TABLE} ON ${PLACE_POPULATION_TABLE}.id = spr.id`
+		: ""
+
+	// The encyclopedic score is CARRIED, never ranked on (ROAD_TO_V9 §2, ratified 2026-08-06) — it
+	// appears in the SELECT and in no ORDER BY, here or in the companion fetch below. Gated on the
+	// split column, so a pre-split shard emits a literal NULL and builds no join at all.
+	const { select: encyclopedicSelect, join: encyclopedicJoin } = encyclopedicClauses.get(sch)!
+
+	// Push the population boost into the ORDER BY when the index is available, so famous places
+	// (whose long alt-name lists hurt BM25) actually make it into the over-fetch window. The TS
+	// post-scoring will still compute the same boost for the final score; this just ensures the
+	// candidate set is right.
+	//
+	// Formula: rank_adjusted = bm25 - populationBoost * min(1.0, log10(1 + pop) / scaleLog10)
+	// Lower rank_adjusted = better (matches SQLite's bm25 convention of "more negative = better").
+	//
+	// #905 — do NOT reach for bm25 column weights here. Measured falsification (2026-07-02): FTS5's
+	// bm25 length normalization is polluted by the row's TOTAL document size, so identical 1-token
+	// `name` docs read −16.0 (empty alt_names) vs −0.43 (2.7 KB alt_names) EVEN with the alt_names
+	// column weighted to zero — no weighting isolates name relevance in this schema. The famous-
+	// holder guarantee lives in the population-ordered companion fetch below instead, and the
+	// exact tier breaks ties by population in the post-scoring sort.
+	const orderByExpr = shardHasPopulation
+		? `(bm25(place_search) - ? * MIN(1.0, COALESCE(log10(1.0 + ${PLACE_POPULATION_TABLE}.population), 0) / ?))`
+		: "bm25(place_search)"
+
+	// Schema-qualified FROM with bare-name MATCH — required syntax for FTS5 on attached schemas.
+	// See sharding.ts header for the gotcha that drove this design.
+	const stmt = db.prepare(`
+		SELECT
+			spr.id AS id,
+			spr.name,
+			spr.placetype,
+			spr.country,
+			spr.parent_id,
+			bm25(place_search) AS rank,
+			spr.latitude AS lat,
+			spr.longitude AS lon,
+			spr.min_latitude, spr.max_latitude, spr.min_longitude, spr.max_longitude,
+			${populationSelect},
+			${encyclopedicSelect}
+		FROM ${sch}.place_search
+		${joinClause}
+		${populationJoin}
+		${encyclopedicJoin}
+		WHERE ${where.join(" AND ")}
+		ORDER BY ${orderByExpr} ASC
+		LIMIT ?
+	`)
+
+	if (shardHasPopulation) {
+		params.push(weights.populationBoost, weights.populationScaleLog10)
+	}
+
+	params.push(ftsLimit)
+
+	const rawRows = allRows<RawSearchRow>(stmt, ...params)
+
+	// #905 companion fetch: the same MATCH, ordered by population alone. For name floods
+	// ("Paris" matches thousands of gap-fill villages) the bm25-based window above cannot admit
+	// the famous holder — its bm25 is length-poisoned by the row's alias bulk (measured ~15 pts,
+	// vs a +4.0 boost cap), so FR Paris never even reaches post-scoring. This fetch makes the
+	// prominent holders of a name pool-complete BY CONSTRUCTION; the exact-tier sort below
+	// decides whether they win. Skipped without a population index (nothing to order by).
+	if (shardHasPopulation) {
+		const popStmt = db.prepare(`
+			SELECT
+				spr.id AS id,
+				spr.name,
+				spr.placetype,
+				spr.country,
+				spr.parent_id,
+				bm25(place_search) AS rank,
+				spr.latitude AS lat,
+				spr.longitude AS lon,
+				spr.min_latitude, spr.max_latitude, spr.min_longitude, spr.max_longitude,
+				${populationSelect},
+				${encyclopedicSelect}
+			FROM ${sch}.place_search
+			${joinClause}
+			${populationJoin}
+			${encyclopedicJoin}
+			WHERE ${where.join(" AND ")}
+			ORDER BY COALESCE(${PLACE_POPULATION_TABLE}.population, 0) DESC
+			LIMIT ?
+		`)
+
+		const popParams = params.slice(0, -3) // drop the two boost params + ftsLimit
+		const seen = new Set(rawRows.map((r) => r.id))
+
+		for (const row of allRows<RawSearchRow>(popStmt, ...popParams, POPULATION_FETCH_LIMIT)) {
+			if (!seen.has(row.id)) {
+				rawRows.push(row)
+			}
+		}
+	}
+
+	return rawRows
+}

package/sharding.ts

@@ -3,7 +3,7 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Multi-shard support for `WOFSqlitePlaceLookup` — opens multiple WOF SQLite distributions on one
+ *   Multi-shard support for `WOFSQLitePlaceLookup` — opens multiple WOF SQLite distributions on one
  *   connection via `ATTACH DATABASE`, and routes queries to the right shard based on placetype.
  *
  *   ## The FTS5 syntax rule that drove this design
@@ -78,7 +78,7 @@ export interface ShardConfig {
 
 /**
  * Resolved post-derivation: paired path + chosen schema name + (possibly empty) placetypes hint. Used internally by
- * `WOFSqlitePlaceLookup` so the routing logic operates on uniform structures.
+ * `WOFSQLitePlaceLookup` so the routing logic operates on uniform structures.
  */
 export interface ResolvedShard {
 	path: string
@@ -200,7 +200,7 @@ export function pickShardForPlacetype(
 		 */
 		country?: string
 		/**
-		 * Per-schema probed country sets (see `WOFSqlitePlaceLookup`'s construction probe).
+		 * Per-schema probed country sets (see `WOFSQLitePlaceLookup`'s construction probe).
 		 */
 		countriesBySchema?: ReadonlyMap<string, ReadonlySet<string>>
 	}

package/sqlite-convention-source.ts

@@ -32,7 +32,7 @@ export class SqliteConventionSource implements ConventionSource {
 	/**
 	 * @param db An open handle to a DB that has the convention asset attached (or is it).
 	 * @param schema The schema name the `address_convention` table lives under (`main` or an ATTACHed shard name —
-	 *   `WOFSqlitePlaceLookup` auto-detects which shard carries the table).
+	 *   `WOFSQLitePlaceLookup` auto-detects which shard carries the table).
 	 */
 	constructor(db: DatabaseSync, schema: string) {
 		this.#db = db

package/sqlite-utils.ts

@@ -6,7 +6,48 @@
  *   Small shared helpers for the SQLite-backed lookups.
  */
 
-import type { DatabaseSync } from "node:sqlite"
+import type { DatabaseSync, SQLInputValue } from "node:sqlite"
+
+import { allRows, getRow } from "@mailwoman/core/utils"
+
+// The row-shape assertion itself lives in `core` so the readers that cannot depend on this package reach the same
+// seam; re-exported here because this module is where this package's readers already look for it.
+export { allRows, getRow } from "@mailwoman/core/utils"
+
+/**
+ * A prepared single-row query whose parameter tuple remains visible to TypeScript. `StatementSync` accepts only the
+ * broad `SQLInputValue[]`, which otherwise erases tagged key types before they reach SQLite.
+ */
+export type PreparedGet<Parameters extends SQLInputValue[], Row> = (...parameters: Parameters) => Row | undefined
+
+/**
+ * Prepare a single-row query while preserving its exact parameter tuple at every call site.
+ */
+export function prepareGet<Parameters extends SQLInputValue[], Row>(
+	db: DatabaseSync,
+	sql: string
+): PreparedGet<Parameters, Row> {
+	const statement = db.prepare(sql)
+
+	return (...parameters) => getRow<Row>(statement, ...parameters)
+}
+
+/**
+ * Multi-row counterpart to {@link PreparedGet}.
+ */
+export type PreparedAll<Parameters extends SQLInputValue[], Row> = (...parameters: Parameters) => Row[]
+
+/**
+ * Prepare a multi-row query while preserving its exact parameter tuple at every call site.
+ */
+export function prepareAll<Parameters extends SQLInputValue[], Row>(
+	db: DatabaseSync,
+	sql: string
+): PreparedAll<Parameters, Row> {
+	const statement = db.prepare(sql)
+
+	return (...parameters) => allRows<Row>(statement, ...parameters)
+}
 
 /**
  * True when `name` is a table in the open database. The street-level lookups use this to degrade gracefully on an
@@ -23,3 +64,24 @@ export function hasTable(db: DatabaseSync, name: string): boolean {
 		return false
 	}
 }
+
+/**
+ * True when `table` exists in the open database AND carries `column`.
+ *
+ * The column-level sibling of {@link hasTable}, and it exists for the same reason one layer down: an artifact built
+ * before a column was added is still a VALID artifact, and a reader that unconditionally names the new column in its
+ * `SELECT` turns "this gazetteer is a build behind" into `no such column` at the first keystroke. Probe once at
+ * construction and shape the query — `table_info` is a PRAGMA, so it must not sit on a per-query path.
+ *
+ * Note the interpolation: PRAGMA does not take bound parameters, so `table` is spliced. Every caller passes a
+ * module-level constant; never pass user input.
+ */
+export function hasColumn(db: DatabaseSync, table: string, column: string): boolean {
+	try {
+		const rows = allRows<{ name: string }>(db.prepare(`PRAGMA table_info(${table})`))
+
+		return rows.some((r) => String(r.name) === column)
+	} catch {
+		return false
+	}
+}

package/street-centroid-schema.ts

@@ -22,6 +22,8 @@
 
 import type { Kysely } from "kysely"
 
+import type { NameKey, StreetKey } from "./street-normalize.ts"
+
 /**
  * One street roll-up. `(street_norm, postcode, locality_base)` is unique. `lat`/`lon` are the UNWEIGHTED mean of the
  * group's member address points (each source row = one point), so a cross-group weighted mean (`SUM(lat*point_count) /
@@ -32,7 +34,7 @@ export interface StreetCentroidTable {
 	/**
 	 * Shared `normalizeStreetForKeyLocale` of the street — the build/query-consistent probe key.
 	 */
-	street_norm: string
+	street_norm: StreetKey
 	/**
 	 * The 5-digit postcode of this group, or null when the source row carried none.
 	 */
@@ -40,7 +42,7 @@ export interface StreetCentroidTable {
 	/**
 	 * Arrondissement-stripped commune (`stripArrondissement(normalizeLocalityForKey(commune))`) — the fallback scope.
 	 */
-	locality_base: string
+	locality_base: NameKey
 	/**
 	 * Weighted-mean centroid latitude of the street's member points.
 	 */
@@ -75,6 +77,10 @@ export interface StreetCentroidTable {
 	 * name-evidence rerank folds the model's street surface with the SAME `foldStreetSurface` used to build this column
 	 * (the fold-parity contract), so it must not drift from `street_norm`'s richer normalizer. Indexed (`idx_sc_name`)
 	 * for a direct seek.
+	 *
+	 * Deliberately UNBRANDED despite the `name_key` column name: `foldStreetSurface` is a different fold from the
+	 * {@link NameKey} one every other `name_key` column carries, and giving it that brand would invite exactly the
+	 * cross-fold probe the brands exist to stop.
 	 */
 	name_key: string
 }

package/street-centroid.ts

@@ -23,10 +23,13 @@ import { DatabaseSync } from "node:sqlite"
 
 import type { StreetCentroidHit, StreetCentroidLookup } from "@mailwoman/resolver"
 
-import { hasTable } from "./sqlite-utils.ts"
+import { hasTable, prepareGet, type PreparedGet } from "./sqlite-utils.ts"
 import {
 	normalizeLocalityForKey,
 	normalizeStreetForKeyLocale,
+	type NameKey,
+	type StreetKey,
+	streetLocaleForSurface,
 	type StreetLocale,
 	stripArrondissement,
 } from "./street-normalize.ts"
@@ -72,8 +75,8 @@ function extentRadiusM(minLat: number, maxLat: number, minLon: number, maxLon: n
 export class StreetCentroidSqliteLookup implements StreetCentroidLookup {
 	readonly #db: DatabaseSync
 	readonly #locale: StreetLocale
-	readonly #byPostcode: ReturnType<DatabaseSync["prepare"]> | undefined
-	readonly #byLocality: ReturnType<DatabaseSync["prepare"]> | undefined
+	readonly #byPostcode: PreparedGet<[postcode: string, street: StreetKey], AggRow> | undefined
+	readonly #byLocality: PreparedGet<[locality: NameKey, street: StreetKey], AggRow> | undefined
 
 	/**
 	 * @param dbPath Shard path.
@@ -87,11 +90,13 @@ export class StreetCentroidSqliteLookup implements StreetCentroidLookup {
 		// Degrade gracefully on an empty/tableless shard (interrupted build, stray 0-byte file): with no
 		// `street_centroid` table this lookup is a no-op miss, not a crash (mirrors the address-point reader).
 		if (hasTable(this.#db, "street_centroid")) {
-			this.#byPostcode = this.#db.prepare(
+			this.#byPostcode = prepareGet(
+				this.#db,
 				`SELECT ${AGG_SELECT} FROM street_centroid WHERE postcode = ? AND street_norm = ?`
 			)
 
-			this.#byLocality = this.#db.prepare(
+			this.#byLocality = prepareGet(
+				this.#db,
 				`SELECT ${AGG_SELECT} FROM street_centroid WHERE locality_base = ? AND street_norm = ?`
 			)
 		}
@@ -99,19 +104,19 @@ export class StreetCentroidSqliteLookup implements StreetCentroidLookup {
 
 	find(query: { street: string; postcode?: string; locality?: string }): StreetCentroidHit | null {
 		if (!this.#byPostcode || !this.#byLocality) return null
-		const streetNorm = normalizeStreetForKeyLocale(query.street, this.#locale)
+		const streetNorm = normalizeStreetForKeyLocale(query.street, streetLocaleForSurface(query.street, this.#locale))
 
 		if (!streetNorm) return null
 
 		let row: AggRow | undefined
 
 		if (query.postcode?.trim()) {
-			row = this.#byPostcode.get(query.postcode.trim(), streetNorm) as AggRow | undefined
+			row = this.#byPostcode(query.postcode.trim(), streetNorm)
 		}
 
 		if ((!row || row.lat == null) && query.locality?.trim()) {
 			const base = stripArrondissement(normalizeLocalityForKey(query.locality))
-			row = this.#byLocality.get(base, streetNorm) as AggRow | undefined
+			row = this.#byLocality(base, streetNorm)
 		}
 
 		if (!row || row.lat == null || row.lon == null) return null

package/street-morphology-fst-builder.ts

@@ -196,10 +196,11 @@ export function buildStreetMorphologyFST(opts: BuildStreetMorphologyFSTOpts): Bu
 			placetype: "street_affix",
 			name: canonical,
 			parentChain: [],
-			// Fixed importance: street affixes are structurally unambiguous (Avenue is almost never
-			// anything but street-typing). The morphology prior caps bias separately; this value
-			// just feeds the cap formula `importance * cap`.
-			importance: 1,
+			// Fixed referential score: street affixes are structurally unambiguous (Avenue is almost
+			// never anything but street-typing). The morphology prior caps bias separately; this value
+			// just feeds the cap formula `referential * cap`. No encyclopedic field — a street affix is
+			// not a place and has no article; absence here is the correct statement.
+			referential: 1,
 			lat: 0,
 			lon: 0,
 		}