@mailwoman/resolver-wof-sqlite 9.0.0 → 9.2.0

package/fst-autocomplete.ts

@@ -3,20 +3,29 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   FST-based autocomplete. Prefix walk + BFS expansion to collect ranked place suggestions. O(depth
- *   × branching) — the FST IS the autocomplete index.
+ *   FST-based autocomplete — mailwoman vocabulary over `@mailwoman/ancestrie`'s generic algorithm
+ *   (#1728 phase 2). The #587 behavior — prefix walk + BFS expansion, partial-last-token completion,
+ *   per-branch capping, dedupe — lives in ancestrie's `autocomplete`; this module contributes only
+ *   the storage adapter ({@link FSTMatcher} → `AncestrieReaderLike`) and the mapping back to
+ *   mailwoman's suggestion shape (name, placetype, referential/encyclopedic, WOF ids).
  *
- *   Two query shapes are handled (the FST is a trie over normalized WORD tokens):
- *
- *   - COMPLETE tokens ("new york") — `walk` lands on a state; collect its accepting entries + BFS a
- *       couple tokens past it for nearby completions. This is the CLI's "complete a place word"
- *       path.
- *   - A PARTIAL last token ("new yor", "chic") — `walk` fails (there is no "yor" edge, only "york"). So
- *       walk the complete prefix, then complete the partial token by prefix-filtering the
- *       continuation edges (`token.startsWith(partial)`). This is what a char-level typeahead
- *       needs; without it "new yor" returns nothing useful. (#587)
+ *   THE BYTES DO NOT MIGRATE. The shipped artifacts are `FST\0` v1–v5 (`fst-serialize.ts`), not
+ *   ancestrie's `ANCT`: ancestrie entries are id-keyed with one record per id, while an FST place row
+ *   is per-(surface, place) — `crossCountryBranches` is a property of the SURFACE, so the same wofID
+ *   legitimately carries different values under different aliases and cannot be represented id-keyed.
+ *   The matcher, both deserializers, and the serializer therefore stay here; what migrated is the
+ *   ALGORITHM, which is the half that drifts (the #861 share-the-function rule).
  */
 
+import type {
+	AncestrieContinuation,
+	AncestrieMatch,
+	AncestrieReaderLike,
+	AncestrieRecord,
+	AncestrieSuggestion,
+} from "@mailwoman/ancestrie"
+import { autocomplete as ancestrieAutocomplete } from "@mailwoman/ancestrie"
+
 import type { FSTMatcher } from "./fst-matcher.ts"
 import { normalizeTokens } from "./fst-matcher.ts"
 import type { PlaceEntry } from "./fst-types.ts"
@@ -31,7 +40,17 @@ export interface AutocompleteResult {
 export interface AutocompleteSuggestion {
 	name: string
 	placetype: string
-	importance: number
+	/**
+	 * The REFERENTIAL likelihood the suggestion is ranked by (ROAD_TO_V9 §2). Autocomplete answers "which place does the
+	 * user mean", so it ranks referentially like everything else; encyclopedic importance rides along on
+	 * {@link AutocompleteSuggestion.encyclopedic} for display and never enters the order.
+	 */
+	referential: number
+	/**
+	 * Encyclopedic (Wikipedia) importance, when the FST artifact carries one for this place. `undefined` = no article, or
+	 * a pre-v5 binary — never 0.
+	 */
+	encyclopedic?: number
 	wofID: number
 	parentChain: number[]
 	matchDepth: number
@@ -42,152 +61,116 @@ export interface AutocompleteOpts {
 	maxSuggestions?: number
 	maxExpansionDepth?: number
 	/**
-	 * Collapse same-name suggestions to the single highest-importance one. Off by default (the CLI surfaces distinct
+	 * Collapse same-name suggestions to the single highest-referential one. Off by default (the CLI surfaces distinct
 	 * same-name places — New York the city vs the county); a typeahead wants it ON so the dropdown isn't four "New
 	 * London"s. (#587)
 	 */
 	dedupeByName?: boolean
 }
 
-interface BfsItem {
-	stateID: number
-	depth: number
-	tokens: string[]
-}
-
 /**
  * Max accepting entries collected per BFS branch — keeps one dense branch from starving the search.
  */
 const PER_BRANCH = 4
 
 /**
- * The top-`k` entries by importance (descending). Avoids sorting/allocating when `entries` is small.
+ * The top-`k` entries by REFERENTIAL likelihood (descending). Avoids sorting/allocating when `entries` is small — and
+ * that shortcut is part of the observable contract: at or under `k` the INSERTION order is served, which decides
+ * suggestion order among referential ties.
  */
-function topByImportance(entries: readonly PlaceEntry[], k: number): PlaceEntry[] {
+function topByReferential(entries: readonly PlaceEntry[], k: number): PlaceEntry[] {
 	if (entries.length <= k) return [...entries]
 
-	return [...entries].toSorted((a, b) => b.importance - a.importance).slice(0, k)
+	return [...entries].toSorted((a, b) => b.referential - a.referential).slice(0, k)
 }
 
 /**
- * Autocomplete from the current prefix. Returns suggestions ranked importance-descending.
+ * {@link FSTMatcher} presented through ancestrie's storage seam. Records carry the {@link PlaceEntry} itself as the
+ * payload, so the entry that WINS the algorithm's shallowest-depth rule is the entry whose fields the suggestion
+ * reports — a side lookup keyed on id could pick a different surface's row (`crossCountryBranches` differs per
+ * surface).
  */
-export function autocomplete(fst: FSTMatcher, query: string, opts: AutocompleteOpts = {}): AutocompleteResult {
-	const maxSuggestions = opts.maxSuggestions ?? 10
-	const maxExpansionDepth = opts.maxExpansionDepth ?? 2
-	const normalizedTokens = normalizeTokens(query)
-
-	if (!normalizedTokens.length) {
-		return { query, normalizedTokens: [], depth: 0, suggestions: [] }
-	}
-
-	const seen = new Map<number, AutocompleteSuggestion>()
-	const queue: BfsItem[] = []
-	let depth: number
-
-	const match = fst.walk(normalizedTokens)
-
-	if (match) {
-		// COMPLETE-token prefix landed on a state. Seed at the match state (accepting + continuations).
-		depth = match.depth
-
-		for (const entry of fst.accepting(match.stateID)) {
-			addSuggestion(seen, entry, match.depth, [])
-		}
-
-		for (const cont of fst.continuations(match.stateID)) {
-			queue.push({ stateID: cont.targetState, depth: 1, tokens: [cont.token] })
-		}
-	} else {
-		// PARTIAL last token — walk the complete prefix, complete the partial by prefix-filtering edges.
-		const complete = normalizedTokens.slice(0, -1)
-		const partial = normalizedTokens.at(-1)!
-		const prefixState = !complete.length ? 0 : (fst.walk(complete)?.stateID ?? undefined)
+class FSTReader implements AncestrieReaderLike<PlaceEntry> {
+	readonly #fst: FSTMatcher
 
-		if (prefixState === undefined) {
-			return { query, normalizedTokens, depth: 0, suggestions: [] }
-		}
-
-		depth = complete.length
-
-		for (const cont of fst.continuations(prefixState)) {
-			if (!cont.token.startsWith(partial)) continue
+	/**
+	 * Parent chains of the entries this reader has served, id-keyed. A place's chain is identical across its surfaces (it
+	 * is place-row data), so last-write-wins is safe. The algorithm asks {@link FSTReader.ancestorsOf} only for ids it
+	 * just received from {@link FSTReader.entriesAt}, so serving from this memo answers every real call without an
+	 * artifact-wide id index.
+	 */
+	readonly #chains = new Map<number, number[]>()
 
-			// This edge completes the typed partial token — its target is a real match at depth+1.
-			for (const entry of topByImportance(fst.accepting(cont.targetState), PER_BRANCH)) {
-				addSuggestion(seen, entry, complete.length + 1, [cont.token])
-			}
+	constructor(fst: FSTMatcher) {
+		this.#fst = fst
+	}
 
-			// BFS a little past it too (multi-token completions: "new yor" → "New York Mills").
-			queue.push({ stateID: cont.targetState, depth: 1, tokens: [cont.token] })
-		}
+	walk(tokens: readonly string[]): AncestrieMatch | null {
+		return this.#fst.walk([...tokens])
 	}
 
-	// BFS expansion (shared by both paths) — find nearby completions up to maxExpansionDepth. Each
-	// branch contributes only its top PER_BRANCH places: a state like "new london" has dozens of
-	// accepting entries and would otherwise blow the budget before the BFS ever reaches "new york"
-	// (the "new" state has 311 continuations). Per-branch capping keeps the search broad. (#587)
-	while (queue.length && seen.size < maxSuggestions * 4) {
-		const item = queue.shift()!
+	continuations(stateID: number): AncestrieContinuation[] {
+		// Insertion order, verbatim — BFS visit order under the suggestion budget depends on it.
+		return this.#fst.continuations(stateID).map((c) => ({
+			token: c.token,
+			targetState: c.targetState,
+			entryCount: c.acceptingCount,
+		}))
+	}
 
-		if (item.depth > maxExpansionDepth) continue
+	entriesAt(stateID: number, limit?: number): AncestrieRecord<PlaceEntry>[] {
+		const places = this.#fst.accepting(stateID)
+		const selected = limit === undefined ? places : topByReferential(places, limit)
 
-		for (const entry of topByImportance(fst.accepting(item.stateID), PER_BRANCH)) {
-			addSuggestion(seen, entry, depth + item.depth, item.tokens)
-		}
+		return selected.map((entry) => {
+			this.#chains.set(entry.wofID, entry.parentChain)
 
-		if (item.depth < maxExpansionDepth) {
-			for (const cont of fst.continuations(item.stateID)) {
-				queue.push({ stateID: cont.targetState, depth: item.depth + 1, tokens: [...item.tokens, cont.token] })
+			return {
+				id: entry.wofID,
+				rank: entry.referential,
+				parentIDs: entry.parentChain,
+				payload: entry,
 			}
-		}
+		})
 	}
 
-	let suggestions = [...seen.values()].toSorted((a, b) => b.importance - a.importance)
-
-	if (opts.dedupeByName) {
-		suggestions = dedupeByName(suggestions)
+	ancestorsOf(id: number): number[] {
+		return this.#chains.get(id) ?? []
 	}
-
-	return { query, normalizedTokens, depth, suggestions: suggestions.slice(0, maxSuggestions) }
-}
-
-function addSuggestion(
-	seen: Map<number, AutocompleteSuggestion>,
-	entry: PlaceEntry,
-	matchDepth: number,
-	completionTokens: string[]
-): void {
-	const existing = seen.get(entry.wofID)
-
-	if (existing && existing.matchDepth <= matchDepth) return
-
-	seen.set(entry.wofID, {
-		name: entry.name,
-		placetype: entry.placetype,
-		importance: entry.importance,
-		wofID: entry.wofID,
-		parentChain: entry.parentChain,
-		matchDepth,
-		completionTokens: [...completionTokens],
-	})
 }
 
 /**
- * Keep one suggestion per name — the highest-importance. Input is already importance-sorted, so the first occurrence
- * per name wins; order is preserved.
+ * Autocomplete from the current prefix. Returns suggestions ranked referential-descending.
  */
-function dedupeByName(suggestions: AutocompleteSuggestion[]): AutocompleteSuggestion[] {
-	const seenNames = new Set<string>()
-	const out: AutocompleteSuggestion[] = []
+export function autocomplete(fst: FSTMatcher, query: string, opts: AutocompleteOpts = {}): AutocompleteResult {
+	const normalizedTokens = normalizeTokens(query)
 
-	for (const s of suggestions) {
-		const key = s.name.toLowerCase()
+	const result = ancestrieAutocomplete<PlaceEntry>(new FSTReader(fst), normalizedTokens, {
+		...(opts.maxSuggestions === undefined ? {} : { maxSuggestions: opts.maxSuggestions }),
+		...(opts.maxExpansionDepth === undefined ? {} : { maxExpansionDepth: opts.maxExpansionDepth }),
+		perBranchLimit: PER_BRANCH,
+		// The dedupe key is the DISPLAY name, not the token path: two surfaces of one name must still collapse. (#587)
+		...(opts.dedupeByName ? { dedupe: (s: AncestrieSuggestion<PlaceEntry>) => s.payload!.name.toLowerCase() } : {}),
+	})
 
-		if (seenNames.has(key)) continue
-		seenNames.add(key)
-		out.push(s)
+	return {
+		query,
+		normalizedTokens,
+		depth: result.depth,
+		suggestions: result.suggestions.map((s) => {
+			// Every record this adapter serves carries its entry; the assertion documents the invariant.
+			const entry = s.payload!
+
+			return {
+				name: entry.name,
+				placetype: entry.placetype,
+				referential: entry.referential,
+				...(entry.encyclopedic === undefined ? {} : { encyclopedic: entry.encyclopedic }),
+				wofID: s.id,
+				parentChain: s.parentIDs,
+				matchDepth: s.matchDepth,
+				completionTokens: s.completionTokens,
+			}
+		}),
 	}
-
-	return out
 }

package/fst-builder.ts

@@ -17,6 +17,8 @@ import { readWOFSourceIdentity } from "./fst-freshness.ts"
 import type { FSTNode } from "./fst-matcher.ts"
 import { FSTMatcher, normalizeTokens } from "./fst-matcher.ts"
 import type { BuildFSTOpts, BuildFSTResult, FSTProvenance, PlaceEntry, PlacetypeID } from "./fst-types.ts"
+import { loadImportanceSplit } from "./place-importance-schema.ts"
+import { allRows, getRow } from "./sqlite-utils.ts"
 
 const DEFAULT_PLACETYPES: PlacetypeID[] = [
 	"country",
@@ -52,11 +54,6 @@ interface NameRow {
 	privateuse: string
 }
 
-interface PopulationRow {
-	id: number
-	population: number
-}
-
 export function buildFSTFromWOF(opts: BuildFSTOpts): {
 	matcher: FSTMatcher
 	provenance: FSTProvenance
@@ -82,7 +79,7 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
 		   AND placetype IN (${placeholders(placetypes)})`
 	)
 
-	const sprRows = sprStmt.all(...countries, ...placetypes) as unknown as SprRow[]
+	const sprRows = allRows<SprRow>(sprStmt, ...countries, ...placetypes)
 	progress("spr", `Loaded ${sprRows.length} places`)
 
 	// Phase 2: Build a lookup for parent chain resolution.
@@ -107,8 +104,8 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
 		for (let i = 0; i < orphanIDs.length; i += ANCESTOR_CHUNK) {
 			const chunk = orphanIDs.slice(i, i + ANCESTOR_CHUNK)
 
-			const rows = db
-				.prepare(
+			const rows = allRows<{ id: number; ancestor_id: number }>(
+				db.prepare(
 					`SELECT DISTINCT id, ancestor_id FROM ancestors
 					 WHERE id IN (${chunk.map(() => "?").join(",")}) AND ancestor_placetype IN ('country', 'region', 'county')
 					 ORDER BY id, CASE ancestor_placetype
@@ -116,8 +113,9 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
 					   WHEN 'region' THEN 2
 					   WHEN 'country' THEN 3
 					 END`
-				)
-				.all(...chunk) as unknown as Array<{ id: number; ancestor_id: number }>
+				),
+				...chunk
+			)
 
 			for (const row of rows) {
 				let chain = ancestorsByID.get(row.id)
@@ -154,7 +152,7 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
 			let parentRow = sprByID.get(current)
 
 			if (!parentRow) {
-				const fetched = parentStmt.get(current) as unknown as SprRow | undefined
+				const fetched = getRow<SprRow>(parentStmt, current)
 
 				if (!fetched) break
 				parentRow = fetched
@@ -171,45 +169,32 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
 		return chain
 	}
 
-	// Phase 3: Load importance data (Wikipedia-based, falls back to population-scaled).
-	// See docs/articles/concepts/importance-vs-population.md for the two-signal contract.
-	progress("importance", "Loading importance data")
-	const importanceMap = new Map<number, number>()
+	// Phase 3: Load BOTH scores (ROAD_TO_V9 §2 R1, the two-score split).
+	//
+	// Referential is ALWAYS population-anchored and never read out of a legacy `place_importance`
+	// column, because a legacy row that got a Wikipedia score overwrote whatever population would have
+	// said and the two are indistinguishable afterwards. Encyclopedic rides along for consumers and is
+	// never handed to the decoder. `loadImportanceSplit` handles all four schema generations; the
+	// source it reports is stamped into provenance so an artifact says which one it read.
+	progress("importance", "Loading referential + encyclopedic scores")
+	const split = loadImportanceSplit(db)
 
-	try {
-		const impStmt = db.prepare("SELECT id, importance FROM place_importance")
-		const impRows = impStmt.all() as unknown as Array<{ id: number; importance: number }>
-
-		for (const row of impRows) {
-			importanceMap.set(row.id, row.importance)
-		}
-
-		progress("importance", `Loaded ${importanceMap.size} importance scores`)
-	} catch {
-		progress("importance", "No place_importance table — falling back to population")
-
-		try {
-			const popStmt = db.prepare("SELECT id, population FROM place_population")
-			const popRows = popStmt.all() as unknown as PopulationRow[]
-
-			for (const row of popRows) {
-				const normalized = row.population > 0 ? Math.min(1, Math.log2(1 + row.population / 1000) / 14) : 0
-				importanceMap.set(row.id, normalized)
-			}
-		} catch {
-			progress("importance", "No place_population either — using 0 for all")
-		}
-	}
+	progress(
+		"importance",
+		`${split.referential.size} referential, ${split.encyclopedic.size} encyclopedic (source: ${split.source}` +
+			(split.legacyFallbackRows ? `, ${split.legacyFallbackRows} legacy rows attributed to the population pass` : "") +
+			")"
+	)
 
 	// Phase 4: Load names for matching places.
 	progress("names", "Loading name variants")
-	const placeIds = sprRows.map((r) => r.id)
+	const placeIDs = sprRows.map((r) => r.id)
 	const namesByPlace = new Map<number, string[]>()
 
 	const allLanguages = languages.includes("*")
 
-	for (let i = 0; i < placeIds.length; i += 500) {
-		const chunk = placeIds.slice(i, i + 500)
+	for (let i = 0; i < placeIDs.length; i += 500) {
+		const chunk = placeIDs.slice(i, i + 500)
 		const idPlaceholders = chunk.map(() => "?").join(",")
 
 		const nameStmt = allLanguages
@@ -218,9 +203,9 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
 					`SELECT id, name, language, privateuse FROM names WHERE id IN (${idPlaceholders}) AND language IN (${languages.map(() => "?").join(",")})`
 				)
 
-		const nameRows = (allLanguages
-			? nameStmt.all(...chunk)
-			: nameStmt.all(...chunk, ...languages)) as unknown as NameRow[]
+		const nameRows = allLanguages
+			? allRows<NameRow>(nameStmt, ...chunk)
+			: allRows<NameRow>(nameStmt, ...chunk, ...languages)
 
 		for (const row of nameRows) {
 			const existing = namesByPlace.get(row.id) ?? []
@@ -304,12 +289,17 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
 	for (const row of sprRows) {
 		const parentChain = resolveParentChain(row.id)
 
+		const encyclopedic = split.encyclopedic.get(row.id)
+
 		const entry: PlaceEntry = {
 			wofID: row.id,
 			placetype: row.placetype as PlacetypeID,
 			name: row.name,
 			parentChain,
-			importance: importanceMap.get(row.id) ?? 0,
+			referential: split.referential.get(row.id) ?? 0,
+			// Spread rather than assigned: a place with no Wikipedia article must carry NO field, not a
+			// zero. The serializer's per-place presence bit reads `!== undefined`.
+			...(encyclopedic === undefined ? {} : { encyclopedic }),
 			lat: row.latitude,
 			lon: row.longitude,
 		}
@@ -361,7 +351,9 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
 		placeCount: sprRows.length,
 		edgeCount,
 		nameInsertions: insertCount,
-		importanceMatches: importanceMap.size,
+		importanceMatches: split.referential.size,
+		encyclopedicMatches: split.encyclopedic.size,
+		importanceSource: split.source,
 		sourceDB: opts.dbPath,
 		sourceDBMD5: source.md5,
 		sourceDBBytes: source.bytes,

package/fst-deserialize-web.ts

@@ -39,17 +39,44 @@ const VERSION_WITH_METADATA = 3
 
 const HEADER_SIZE = 32
 const EDGE_ENTRY_SIZE = 8
-const PLACE_ENTRY_SIZE = 56
+
+/**
+ * Format version that split the single `importance` float into `referential` + `encyclopedic` (ROAD_TO_V9 §2 R1),
+ * growing the place entry from 56 to 60 bytes. Mirrors `fst-serialize.ts`'s constant of the same name.
+ */
+const VERSION_TWO_SCORE_SPLIT = 5
+
+/**
+ * Place-entry size in bytes at or above {@link VERSION_TWO_SCORE_SPLIT}.
+ */
+const SPLIT_PLACE_ENTRY_SIZE = 60
+
+/**
+ * Place-entry size in bytes below {@link VERSION_TWO_SCORE_SPLIT}.
+ */
+const LEGACY_PLACE_ENTRY_SIZE = 56
+
+/**
+ * Byte offset of the encyclopedic float inside a v5 place entry — immediately after the 8-slot parent chain.
+ */
+const ENCYCLOPEDIC_OFFSET = 56
+
+/**
+ * `placeFlags` bit 0 (byte `pp+7`, v5+): this place carries an encyclopedic score. Per-place because absence is the
+ * common case; see `fst-serialize.ts`.
+ */
+const PLACE_FLAG_HAS_ENCYCLOPEDIC = 1
 /**
  * "FST\0".
  */
 const MAGIC_BYTES = [0x46, 0x53, 0x54, 0x00]
 /**
- * Must track the serializer's VERSION (fst-serialize.ts, currently 4). The v3 provenance + v4 16-byte-state/u32-count
+ * Must track the serializer's VERSION (fst-serialize.ts, currently 5). The v3 provenance + v4 16-byte-state/u32-count
  * layout logic below already matches the Node deserializer; only this gate was left stale at 2, so the browser FST
- * loader rejected every real (v4) artifact.
+ * loader rejected every real (v4) artifact. It was left stale again at 4 by the v5 two-score split until this line
+ * moved with it — the gate is a SEPARATE number from the layout branches, which is exactly why it keeps drifting.
  */
-const MAX_VERSION = 4
+const MAX_VERSION = 5
 
 const PLACETYPE_ORDER: readonly PlacetypeID[] = [
 	"country",
@@ -88,6 +115,7 @@ export function deserializeFSTWeb(input: ArrayBuffer | Uint8Array): FSTMatcher {
 	}
 
 	const isV2 = version >= 2
+	const isSplit = version >= VERSION_TWO_SCORE_SPLIT
 	// flags bit0 (survey #4, mirrors fst-serialize.ts): place rows carry surface-ambiguity data.
 	const hasAmbiguity = (view.getUint16(6, true) & 1) === 1
 
@@ -120,6 +148,8 @@ export function deserializeFSTWeb(input: ArrayBuffer | Uint8Array): FSTMatcher {
 
 	// --- State table ---
 	const stateEntrySize = version >= VERSION_WIDE_STATE_COUNTERS ? WIDE_STATE_ENTRY_SIZE : NARROW_STATE_ENTRY_SIZE
+	// v5 grew the place entry by the encyclopedic float; v4-and-below files are read at the old stride.
+	const placeEntrySize = isSplit ? SPLIT_PLACE_ENTRY_SIZE : LEGACY_PLACE_ENTRY_SIZE
 	const stateTableStart = pos
 	const edgeTableStart = stateTableStart + stateCount * stateEntrySize
 	const placeTableStart = edgeTableStart + edgeCount * EDGE_ENTRY_SIZE
@@ -149,7 +179,7 @@ export function deserializeFSTWeb(input: ArrayBuffer | Uint8Array): FSTMatcher {
 		const places: PlaceEntry[] = new Array(placeCountForState)
 
 		for (let pi = 0; pi < placeCountForState; pi++) {
-			const pp = placeTableStart + (placeStart + pi) * PLACE_ENTRY_SIZE
+			const pp = placeTableStart + (placeStart + pi) * placeEntrySize
 			const chainLen = view.getUint8(pp + 5)
 			const parentChain: number[] = []
 
@@ -157,19 +187,25 @@ export function deserializeFSTWeb(input: ArrayBuffer | Uint8Array): FSTMatcher {
 				parentChain.push(view.getUint32(pp + 24 + ci * 4, true))
 			}
 
-			const rawImportance = isV2
+			// v1 stored a raw population u32 here; v2-v4 the conflated `importance` float; v5 the
+			// referential score. See the Node deserializer for why a v1 value is genuinely referential.
+			const referential = isV2
 				? view.getFloat32(pp + 12, true)
 				: Math.min(1, Math.log2(1 + view.getUint32(pp + 12, true) / 1000) / 14)
 
+			const hasEncyclopedic =
+				isSplit && (view.getUint8(pp + 7) & PLACE_FLAG_HAS_ENCYCLOPEDIC) === PLACE_FLAG_HAS_ENCYCLOPEDIC
+
 			places[pi] = {
 				wofID: view.getUint32(pp, true),
 				placetype: PLACETYPE_ORDER[view.getUint8(pp + 4)] ?? "locality",
 				name: strings[view.getUint32(pp + 8, true)]!,
-				importance: rawImportance,
+				referential,
 				lat: view.getFloat32(pp + 16, true),
 				lon: view.getFloat32(pp + 20, true),
 				parentChain,
 				...(hasAmbiguity ? { crossCountryBranches: view.getUint8(pp + 6) } : {}),
+				...(hasEncyclopedic ? { encyclopedic: view.getFloat32(pp + ENCYCLOPEDIC_OFFSET, true) } : {}),
 			}
 		}
 

package/fst-freshness.ts

@@ -15,7 +15,7 @@
  *   `FSTProvenance` already recorded `sourceDB` — the source's PATH, which is exactly the field that
  *   cannot change when the bytes behind it do. So the stamp gains the source's IDENTITY (md5 + byte
  *   size) and this module compares it. The shape deliberately mirrors
- *   `scripts/weights-overlay-linker.ts`'s `pairIndexStaleReason`: one function returning a reason
+ *   `@mailwoman/resolver-wof-sqlite/weights-overlay-linker`'s `pairIndexStaleReason`: one function returning a reason
  *   string or `undefined`, so a fact added to the stamp cannot be checked by some callers and not
  *   others — which is how three of the four base linkers ended up unable to notice a PIX1 schema bump.
  *
@@ -162,7 +162,7 @@ export function peekFSTStampFields(path: string): FSTStampFields | undefined {
  *
  * The async `md5File` in `@mailwoman/core/utils` is the one to reach for anywhere else. This exists because the FST
  * builder and its whole call chain are synchronous by design (`buildFSTFromWOF` → `buildLocaleFSTs`), and making them
- * async to stamp a checksum would cascade through the Pastel commands and the tests for one hash. It reads in
+ * async to stamp a checksum would cascade through command callers and tests for one hash. It reads in
  * {@link MD5_CHUNK_BYTES} chunks rather than `readFileSync` — the source is a multi-gigabyte database.
  */
 export function md5FileSync(path: string): string {