@mailwoman/resolver-wof-sqlite 10.0.0 → 10.2.0

package/lib/place-importance-schema.ts

@@ -2,33 +2,6 @@
  * @copyright Sister Software
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
- *
- *   The TWO-SCORE SPLIT (ROAD_TO_V9 §2 R1) — typed schema, table builder, and the two derivations, in
- *   one module so the read contract and the DDL cannot drift.
- *
- *   THE POLICY THIS ENCODES, ratified 2026-08-06: **the importance of a knowledge-base article is not
- *   the probability that this is the place the user means.** The geocoder ranks by REFERENTIAL
- *   likelihood; encyclopedic importance is carried as data and is never the ranking key. So the one
- *   `place_importance.importance` column — which was a Wikipedia score where the concordance join
- *   landed and a population-derived pseudo-score everywhere else — becomes two named columns that can
- *   never be confused for one another:
- *
- *   - {@link PlaceImportanceTable.referential} — population-anchored, ALWAYS derivable, the ranking
- *     backbone. {@link referentialFromPopulation} is the normalization the FST builder's population
- *     fallback has always used; naming it here is the whole change.
- *   - {@link PlaceImportanceTable.encyclopedic} — the fan-out-guarded Wikipedia join
- *     (`importance-fanout.ts`, #1497). NULLABLE, and null means ABSENT, never "an importance of
- *     zero": ~1.5 M of the 1.54 M rows in the 2026-08-05 build have no Wikipedia article at all, and
- *     a consumer that reads a 0 there would be reading a fact nobody recorded.
- *
- *   WHY SAINT-DENIS IS THE TEST. The Seine-Saint-Denis suburb (pop 96,128) carries encyclopedic
- *   0.1173; the Aude hamlet (pop 418) carries 0.5683 — the encyclopedic signal ranks the hamlet 4.8x
- *   ABOVE the place every user means. Referentially the suburb wins by 230x on population. One score
- *   cannot serve both readers, which is why there are two.
- *
- *   THE LEGACY COLUMN STAYS, AND IS DERIVED. `importance` is written by {@link blendImportance} — the
- *   bounded blend the bare-toponym fame consumer (#28) ranks on. It is the CONFLATION, and nothing new
- *   should read it; new code reads the split columns.
  */
 
 import { referentialFromPopulation } from "@mailwoman/core/resolver"
@@ -36,42 +9,49 @@ import { allRows } from "@mailwoman/core/utils"
 import type { DatabaseClient } from "@mailwoman/sqlite/client"
 import type { Kysely } from "kysely"
 
-//#region Schema
-
 /**
- * One row of `place_importance`. Keyed by WOF place id.
+ * Describes one row of the `place_importance` table, keyed by WOF place id.
  */
 export interface PlaceImportanceTable {
 	id: number
+
 	/**
-	 * Population-anchored referential likelihood in [0, 1] — see {@link referentialFromPopulation}. NOT NULL because it is
-	 * always derivable: a place with no population row scores 0, which here genuinely means "no population evidence", the
-	 * same state the ranking has always treated as "no boost, never a penalty".
+	 * The population-anchored referential likelihood in [0, 1], from {@link referentialFromPopulation}.
+	 *
+	 * It is never NULL.
+	 * A place without a population row scores `0`.
+	 *
+	 * That score represents no population evidence, so it earns no boost or penalty.
 	 */
 	referential: number
+
 	/**
-	 * Fan-out-guarded Wikipedia importance in [0, 1], or NULL when this place has no surviving concordance. NULL is
-	 * ABSENCE — never coalesce it to 0 in a consumer, and never rank on it at all.
+	 * The fan-out-guarded Wikipedia importance in [0, 1], or NULL when the place has no surviving concordance.
+	 *
+	 * NULL means absence, so consumers must not coalesce it to 0 or rank on it.
 	 */
 	encyclopedic: number | null
+
 	/**
-	 * DEPRECATED — the pre-split conflation, written by {@link blendImportance} so the bare-toponym fame consumer (#28)
-	 * keeps one cross-bearer scale. New code reads {@link PlaceImportanceTable.referential} (to rank) or
-	 * {@link PlaceImportanceTable.encyclopedic} (to display).
+	 * Deprecated blend of the two scores, written by {@link blendImportance} for
+	 * consumers that still need one scale.
+	 *
+	 * New code ranks on `referential` and displays `encyclopedic`.
 	 */
 	importance: number
 }
 
 /**
- * The `place_importance` slice of a WOF admin database, for `new DatabaseClient<PlaceImportanceDatabase>(...)`.
+ * Types the `place_importance` table of a WOF admin database
+ * for `new DatabaseClient<PlaceImportanceDatabase>(...)`.
  */
 export interface PlaceImportanceDatabase {
 	place_importance: PlaceImportanceTable
 }
 
 /**
- * The columns in declaration order. The builder's INSERT derives its column list from this, so a field added to
- * {@link PlaceImportanceTable} without a matching DDL column is a compile error at the insert site.
+ * Lists the `place_importance` columns in declaration order, each checked
+ * against {@link PlaceImportanceTable}.
  */
 export const PLACE_IMPORTANCE_COLUMNS = [
 	"id",
@@ -81,8 +61,7 @@ export const PLACE_IMPORTANCE_COLUMNS = [
 ] as const satisfies readonly (keyof PlaceImportanceTable)[]
 
 /**
- * Create `place_importance` at the split schema. Drops any existing table first — this is a rebuild-in-place step of
- * `mailwoman gazetteer importance`, never a migration.
+ * Drops and recreates the `place_importance` table with separate referential and encyclopedic columns.
  */
 export async function createPlaceImportanceTable(db: Kysely<PlaceImportanceDatabase>): Promise<void> {
 	await db.schema.dropTable("place_importance").ifExists().execute()
@@ -96,110 +75,55 @@ export async function createPlaceImportanceTable(db: Kysely<PlaceImportanceDatab
 		.execute()
 }
 
-//#endregion
-
-//#region The referential derivation
-
-/**
- * Re-exported from `@mailwoman/core/resolver`, which is where the derivation lives so that `@mailwoman/resolver`
- * (backend-agnostic — it cannot import this package) reads the same number. Re-exported HERE so the schema module stays
- * the one-stop read for the table: the column and the function that fills it are one hop apart.
- */
-
-//#endregion
-
-//#region The legacy blend
-
 /**
- * The most the encyclopedic channel may raise a place's blended importance above its population-anchored referential
- * score.
- *
- * In referential units 0.25 is 3.5 population doublings (the referential curve divides log2 by 14), so an article can
- * promote a place as if it were up to ~11x its recorded population — never more. The bound exists because the two
- * channels' scales CROSS at the article floor: merely having a Wikipedia article scores ~0.25–0.35, which exceeds the
- * referential score of a mid-size town, so an unbounded blend ranks a 136-person village with an article above a
- * 16,026-person town without one.
- *
- * The value is bracketed by two decided contests, measured on the 2026-08-24 staging build:
+ * Caps how far the encyclopedic score may raise a place's blended importance above its referential score.
  *
- * - `> 0.2282`, or bare `Whitby` stops answering Whitby GB (pop 13,130, referential 0.2729, encyclopedic 0.5496) over
- *   Whitby CA (pop 128,377, referential 0.5011) — the #28 design case the fame prior exists to serve.
- * - `< 0.2790`, or bare `Tó`/`To` answers Tó PT (pop 136, referential 0.0131, encyclopedic 0.3375) over Tô BF (pop
- *   16,026, no article) — the `bf-gloss-to-*` board pair.
- *
- * 0.25 sits mid-interval with ~0.02 margin to each bound.
+ * Merely having an article scores about 0.25 to 0.35, above a mid-size town's referential score,
+ * so without the cap a tiny village with an article outranks a large town without one.
  */
 export const ENCYCLOPEDIC_BOOST_CAP = 0.25
 
 /**
- * The legacy `importance` blend — one cross-bearer fame scale for the #28 consumer, derived from the two split
- * channels:
- *
- * - No article → the referential score.
- * - No population evidence (`referential` 0) → the encyclopedic score stands alone: there is nothing to bound the
- *   article's claim against, and a constant cap would demote every famous place WOF records no population for
- *   (meaning-of-zero: referential 0 is "unmeasured", not "tiny").
- * - Both present → the encyclopedic value clamped to at most {@link ENCYCLOPEDIC_BOOST_CAP} above the referential score,
- *   and never below it. The floor half repairs the downward inversion (the Seine-Saint-Denis suburb's weak article
- *   scored 0.1173 and REPLACED its referential 0.4716 under the old `COALESCE`, so a 418-person Aude hamlet outranked
- *   it 4.8x); the cap half repairs the upward one (`Tó`, above).
- *
- * Scale of the clamp on the 2026-08-24 staging build: of 628,202 article-bearing rows, 209,738 sit above the cap and
- * 13,888 sit below their referential floor; the 305,168 article-without-population rows pass through unchanged.
+ * Blends the two channels into the legacy `importance` column: the referential score
+ * when there is no article, only the encyclopedic score when population is unknown
+ * (`referential` 0), and otherwise the encyclopedic score clamped between the
+ * referential score and {@link ENCYCLOPEDIC_BOOST_CAP} above it.
  */
-export function blendImportance(referential: number, encyclopedic: number | null | undefined): number {
-	if (encyclopedic === null || encyclopedic === undefined) return referential
+export function blendImportance(referential: number, encyclopedic: number | null): number {
+	if (encyclopedic === null) return referential
 
 	if (referential <= 0) return encyclopedic
 
 	return Math.max(referential, Math.min(encyclopedic, referential + ENCYCLOPEDIC_BOOST_CAP))
 }
 
-//#endregion
-
-//#region Reading a database that may or may not carry the split
-
 /**
- * Where a reader's two scores came from. Recorded into the FST stamp so an artifact says, in its own provenance,
- * whether its encyclopedic channel is real, reconstructed, or absent.
+ * Source names used by {@link loadImportanceSplit}.
+ * The FST stamp records these values as provenance.
  */
 export const IMPORTANCE_SPLIT_SOURCES = {
-	/**
-	 * `place_importance` carries `referential` + `encyclopedic` — the post-split build.
-	 */
 	splitColumns: "split-columns",
-	/**
-	 * `place_importance` carries only the conflated `importance`; the split was RECONSTRUCTED against `place_population`
-	 * (see {@link splitLegacyImportance}).
-	 */
+
 	legacyReconstructed: "legacy-reconstructed",
-	/**
-	 * No `place_importance` at all — referential from `place_population`, encyclopedic absent for every place.
-	 */
+
 	populationOnly: "population-only",
-	/**
-	 * Neither table. Every score is 0, and 0 here means the database carries no salience evidence whatsoever.
-	 */
+
 	none: "none",
 } as const
 
+/**
+ * Identifies how an {@link ImportanceSplit} was obtained, as one of the
+ * {@link IMPORTANCE_SPLIT_SOURCES} values.
+ */
 export type ImportanceSplitSource = (typeof IMPORTANCE_SPLIT_SOURCES)[keyof typeof IMPORTANCE_SPLIT_SOURCES]
 
 /**
- * The `SELECT` term and `LEFT JOIN` a name lookup needs in order to CARRY `encyclopedic` onto its results, probed
- * against `schemaName`'s `place_importance`.
+ * Returns the `select` term and `left join` that add `encyclopedic` to name-lookup results
+ * for `schemaName`, degrading to `NULL` and no join when the column is absent.
  *
- * THE PROBE IS A COLUMN, NOT A TABLE, and that is the whole reason this lives here rather than beside a caller's other
- * table probes. The pre-split table exists and holds a single conflated `importance` column whose value is a Wikipedia
- * score on some rows and a population proxy on others, with nothing in the row to say which — reading that as
- * encyclopedic would surface the exact confusion ROAD_TO_V9 §2 exists to end.
- *
- * There is deliberately no ORDER BY counterpart, and there should never be one: §2's policy is that this score is
- * carried and never ranked on. Without the column the select degrades to a literal `NULL` and the join is the empty
- * string, so a pre-split extract's query plan is byte-identical to what it was before the split. No shipped gazetteer
- * carries the column yet, so today that degraded form is the only one anything builds.
- *
- * Call it ONCE per extract and cache the result — it runs a `PRAGMA`, and the callers are per-keystroke hot.
+ * It probes for the column rather than the table, because a pre-split table's
+ * `importance` mixes Wikipedia and population scores.
+ * Cache the result per extract, since it runs a `PRAGMA` and lookups are per keystroke.
  */
 export function encyclopedicClauses<DB>(db: DatabaseClient<DB>, schemaName: string): { select: string; join: string } {
 	let present: boolean
@@ -221,47 +145,26 @@ export function encyclopedicClauses<DB>(db: DatabaseClient<DB>, schemaName: stri
 }
 
 /**
- * The tolerance {@link splitLegacyImportance} treats as "this value reproduces the population curve". Eight ULP —
- * comfortably wider than the one-ULP `log2` spread measured between CPython and V8 (see that function's docstring), and
- * ~1e-15 absolute against scores Nominatim publishes to four decimals.
+ * Sets the relative tolerance within which {@link splitLegacyImportance} treats
+ * a value as lying on the population curve.
  */
 export const LEGACY_FALLBACK_EPSILON = 8 * Number.EPSILON
 
 /**
- * Split one row of a LEGACY (pre-split) `place_importance` table back into its two components.
- *
- * WHY THIS IS RECOVERABLE AT ALL. The legacy builder ran in two passes: Wikipedia scores first, then `INSERT OR IGNORE`
- * of `min(1, log2(1+pop/1000)/14)` for every place with a population that Wikipedia had missed. So a legacy value is a
- * fallback row IFF it reproduces {@link referentialFromPopulation} of that place's population — the two passes wrote
- * different arithmetic, and a score from Nominatim's four-decimal TSV landing on the log2 curve is a measure-zero
- * event.
+ * Splits a legacy `place_importance` value into referential and encyclopedic scores,
+ * treating it as a population fallback when it matches {@link referentialFromPopulation}
+ * within {@link LEGACY_FALLBACK_EPSILON}.
  *
- * THE COMPARISON IS ULP-TOLERANT, AND THAT IS NOT DEFENSIVE PADDING. The first version compared for exact bit equality,
- * which is correct — in the runtime that wrote the values. Cross-checking the same rule in CPython returned 166,638
- * encyclopedic rows against Node's 133,096: `math.log2` and V8's `Math.log2` disagree by one ULP on 33,542 of the 1.5 M
- * inputs (worked example: wof 85803233, population 21,299 — stored 0.31992193633838988953, CPython
- * 0.31992193633838994504, delta 5.55e-17). Bit equality would therefore INVENT 33,542 encyclopedic scores for anyone
- * who ported this rule to another runtime, and invented data is the failure mode this whole module exists to end. The
- * tolerance is a few ULP of the score's own magnitude; a genuine Wikipedia value that close to the population curve is
- * not distinguishable from it by any consequence.
- *
- * MEASURED, not reasoned (2026-08-06, `wof/fst-staging-2026-08-05/admin-global-priority-importance.db`): 1,543,753 rows
- * split **1,410,657 fallback / 133,096 encyclopedic**, and the arithmetic closes on itself — 1,410,657 + 108,861
- * (encyclopedic rows that ALSO have a population) = 1,519,518, which is exactly the count of `place_population` rows
- * with `population > 0`, i.e. every row the fallback pass could have written. The remaining 24,235 encyclopedic rows
- * have no population row at all. Under the exact-equality rule, Node found ZERO mismatches within one ULP, so the
- * tolerance changes no classification on this database — it only makes the answer runtime-independent.
- *
- * Referential is NOT read out of the legacy column under any branch — it is always re-derived from population, because
- * a legacy Wikipedia row overwrote whatever population would have said.
+ * The match allows a small tolerance because `log2` differs by an ULP across runtimes.
+ * Bit equality would invent encyclopedic scores.
  */
 export function splitLegacyImportance(
-	legacy: number | undefined,
+	legacy: number | null,
 	population: number | null | undefined
 ): { referential: number; encyclopedic?: number } {
-	const referential = referentialFromPopulation(population)
+	const referential = referentialFromPopulation(population ?? null)
 
-	if (legacy === undefined) return { referential }
+	if (legacy === null) return { referential }
 
 	if (Math.abs(legacy - referential) <= LEGACY_FALLBACK_EPSILON * Math.max(referential, 1)) return { referential }
 
@@ -273,24 +176,23 @@ export function splitLegacyImportance(
  */
 export interface ImportanceSplit {
 	/**
-	 * WOF id → referential likelihood. Sparse: absent means 0 (no population evidence).
+	 * Referential likelihood by WOF ID, where a missing ID means 0 because only positive scores are stored.
 	 */
 	referential: Map<number, number>
+
 	/**
-	 * WOF id → encyclopedic importance. Sparse, and ABSENCE IS ABSENCE — never fill a 0 in.
+	 * Encyclopedic importance by WOF ID, where a missing ID means no score and must not be filled with 0.
 	 */
 	encyclopedic: Map<number, number>
 	source: ImportanceSplitSource
+
 	/**
-	 * Rows the legacy reconstruction attributed to the population fallback (only meaningful under
-	 * {@link IMPORTANCE_SPLIT_SOURCES.legacyReconstructed}).
+	 * The number of legacy rows attributed to the population fallback,
+	 * nonzero only for the `legacy-reconstructed` source.
 	 */
 	legacyFallbackRows: number
 }
 
-/**
- * Does `table` exist in `db`, and if so which of `columns` does it have?
- */
 function tableColumns<DB>(db: DatabaseClient<DB>, table: string): Set<string> {
 	try {
 		const rows = allRows<{ name: string }>(db.prepare(`PRAGMA table_info(${table})`))
@@ -302,11 +204,7 @@ function tableColumns<DB>(db: DatabaseClient<DB>, table: string): Set<string> {
 }
 
 /**
- * Load both scores from a WOF admin database, whatever schema generation it is at.
- *
- * Handles all four states in {@link IMPORTANCE_SPLIT_SOURCES} so callers never branch on schema themselves — the FST
- * builder in particular must read the shipped population-only databases, the read-only 2026-08-05 staging database
- * (legacy conflated column), and post-split builds with one code path.
+ * Loads both importance scores from a WOF admin database, whichever schema generation it uses.
  */
 export function loadImportanceSplit<DB>(db: DatabaseClient<DB>): ImportanceSplit {
 	const referential = new Map<number, number>()
@@ -334,9 +232,6 @@ export function loadImportanceSplit<DB>(db: DatabaseClient<DB>): ImportanceSplit
 	const importanceColumns = tableColumns(db, "place_importance")
 
 	if (importanceColumns.has("referential")) {
-		// Post-split build: the columns ARE the contract. Referential is read verbatim rather than
-		// re-derived, so a build that scored referential differently stays visible instead of being
-		// silently overwritten by this reader's own formula.
 		const rows = allRows<{
 			id: number
 			referential: number
@@ -389,5 +284,3 @@ export function loadImportanceSplit<DB>(db: DatabaseClient<DB>): ImportanceSplit
 		legacyFallbackRows: 0,
 	}
 }
-
-//#endregion