@mailwoman/resolver-wof-sqlite 8.6.0 → 9.1.0

package/fst-builder.ts

@@ -13,9 +13,11 @@
 
 import { DatabaseSync } from "node:sqlite"
 
+import { readWOFSourceIdentity } from "./fst-freshness.ts"
 import type { FSTNode } from "./fst-matcher.ts"
 import { FSTMatcher, normalizeTokens } from "./fst-matcher.ts"
 import type { BuildFSTOpts, BuildFSTResult, FSTProvenance, PlaceEntry, PlacetypeID } from "./fst-types.ts"
+import { loadImportanceSplit } from "./place-importance-schema.ts"
 
 const DEFAULT_PLACETYPES: PlacetypeID[] = [
 	"country",
@@ -29,6 +31,11 @@ const DEFAULT_PLACETYPES: PlacetypeID[] = [
 
 const DEFAULT_COUNTRIES = ["US"]
 const DEFAULT_LANGUAGES = ["eng", ""]
+/**
+ * Ids per `IN (…)` batch. SQLITE_MAX_VARIABLE_NUMBER defaults to 32,766; 500 matches the name-load batch a few phases
+ * down, so both read paths bind the same shape.
+ */
+const ANCESTOR_CHUNK = 500
 
 interface SprRow {
 	id: number
@@ -46,11 +53,6 @@ interface NameRow {
 	privateuse: string
 }
 
-interface PopulationRow {
-	id: number
-	population: number
-}
-
 export function buildFSTFromWOF(opts: BuildFSTOpts): {
 	matcher: FSTMatcher
 	provenance: FSTProvenance
@@ -89,19 +91,40 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
 	// Also load parent rows that might be outside our placetype filter (e.g., country for region).
 	const parentStmt = db.prepare("SELECT id, name, placetype, parent_id, latitude, longitude FROM spr WHERE id = ?")
 
-	// Fallback: use ancestors table when parent_id is a sentinel (-1, -4, etc.).
-	let ancestorStmt: ReturnType<typeof db.prepare> | null = null
+	// Fallback for a sentinel parent_id (-1, -4, …): the ancestors table. Read in chunked `IN (…)`
+	// batches ONCE — the point-query version fired per orphan row, and on a global build the orphans
+	// run to six figures. Ordering is county → region → country, preserved by the same CASE the
+	// per-row query used, with `id` leading so one pass groups the rows.
+	const ancestorsByID = new Map<number, number[]>()
 
 	try {
-		ancestorStmt = db.prepare(
-			`SELECT DISTINCT ancestor_id FROM ancestors
-			 WHERE id = ? AND ancestor_placetype IN ('country', 'region', 'county')
-			 ORDER BY CASE ancestor_placetype
-			   WHEN 'county' THEN 1
-			   WHEN 'region' THEN 2
-			   WHEN 'country' THEN 3
-			 END`
-		)
+		const orphanIDs = sprRows.filter((row) => row.parent_id <= 0).map((row) => row.id)
+
+		for (let i = 0; i < orphanIDs.length; i += ANCESTOR_CHUNK) {
+			const chunk = orphanIDs.slice(i, i + ANCESTOR_CHUNK)
+
+			const rows = db
+				.prepare(
+					`SELECT DISTINCT id, ancestor_id FROM ancestors
+					 WHERE id IN (${chunk.map(() => "?").join(",")}) AND ancestor_placetype IN ('country', 'region', 'county')
+					 ORDER BY id, CASE ancestor_placetype
+					   WHEN 'county' THEN 1
+					   WHEN 'region' THEN 2
+					   WHEN 'country' THEN 3
+					 END`
+				)
+				.all(...chunk) as unknown as Array<{ id: number; ancestor_id: number }>
+
+			for (const row of rows) {
+				let chain = ancestorsByID.get(row.id)
+
+				if (!chain) {
+					ancestorsByID.set(row.id, (chain = []))
+				}
+
+				chain.push(row.ancestor_id)
+			}
+		}
 	} catch {
 		progress("ancestors", "No ancestors table — sentinel parent_ids will produce empty chains")
 	}
@@ -112,10 +135,8 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
 		if (!row) return []
 
 		// If parent_id is a sentinel (≤ 0), use ancestors table.
-		if (row.parent_id <= 0 && ancestorStmt) {
-			const ancestors = ancestorStmt.all(id) as unknown as Array<{ ancestor_id: number }>
-
-			return ancestors.map((a) => a.ancestor_id).filter((aid) => aid !== id)
+		if (row.parent_id <= 0) {
+			return (ancestorsByID.get(id) ?? []).filter((ancestorID) => ancestorID !== id)
 		}
 
 		// Normal case: walk parent_id chain.
@@ -146,35 +167,22 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
 		return chain
 	}
 
-	// Phase 3: Load importance data (Wikipedia-based, falls back to population-scaled).
-	// See docs/articles/concepts/importance-vs-population.md for the two-signal contract.
-	progress("importance", "Loading importance data")
-	const importanceMap = new Map<number, number>()
-
-	try {
-		const impStmt = db.prepare("SELECT id, importance FROM place_importance")
-		const impRows = impStmt.all() as unknown as Array<{ id: number; importance: number }>
-
-		for (const row of impRows) {
-			importanceMap.set(row.id, row.importance)
-		}
+	// Phase 3: Load BOTH scores (ROAD_TO_V9 §2 R1, the two-score split).
+	//
+	// Referential is ALWAYS population-anchored and never read out of a legacy `place_importance`
+	// column, because a legacy row that got a Wikipedia score overwrote whatever population would have
+	// said and the two are indistinguishable afterwards. Encyclopedic rides along for consumers and is
+	// never handed to the decoder. `loadImportanceSplit` handles all four schema generations; the
+	// source it reports is stamped into provenance so an artifact says which one it read.
+	progress("importance", "Loading referential + encyclopedic scores")
+	const split = loadImportanceSplit(db)
 
-		progress("importance", `Loaded ${importanceMap.size} importance scores`)
-	} catch {
-		progress("importance", "No place_importance table — falling back to population")
-
-		try {
-			const popStmt = db.prepare("SELECT id, population FROM place_population")
-			const popRows = popStmt.all() as unknown as PopulationRow[]
-
-			for (const row of popRows) {
-				const normalized = row.population > 0 ? Math.min(1, Math.log2(1 + row.population / 1000) / 14) : 0
-				importanceMap.set(row.id, normalized)
-			}
-		} catch {
-			progress("importance", "No place_population either — using 0 for all")
-		}
-	}
+	progress(
+		"importance",
+		`${split.referential.size} referential, ${split.encyclopedic.size} encyclopedic (source: ${split.source}` +
+			(split.legacyFallbackRows ? `, ${split.legacyFallbackRows} legacy rows attributed to the population pass` : "") +
+			")"
+	)
 
 	// Phase 4: Load names for matching places.
 	progress("names", "Loading name variants")
@@ -279,12 +287,17 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
 	for (const row of sprRows) {
 		const parentChain = resolveParentChain(row.id)
 
+		const encyclopedic = split.encyclopedic.get(row.id)
+
 		const entry: PlaceEntry = {
 			wofID: row.id,
 			placetype: row.placetype as PlacetypeID,
 			name: row.name,
 			parentChain,
-			importance: importanceMap.get(row.id) ?? 0,
+			referential: split.referential.get(row.id) ?? 0,
+			// Spread rather than assigned: a place with no Wikipedia article must carry NO field, not a
+			// zero. The serializer's per-place presence bit reads `!== undefined`.
+			...(encyclopedic === undefined ? {} : { encyclopedic }),
 			lat: row.latitude,
 			lon: row.longitude,
 		}
@@ -320,6 +333,15 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
 	const edgeCount = nodes.reduce((sum, n) => sum + n.edges.size, 0)
 	const matcher = FSTMatcher.fromNodes(nodes)
 
+	// The build stamp (2026-08-05). `sourceDB` alone was never enough to tell a reader whether this
+	// artifact matches the database at that path — the admin DB is sealed and REPLACED by a rebuild, so
+	// the path is constant across every generation of it. Hashing costs 7.3 s for the 5.27 GB admin DB
+	// and is free whenever the `.md5` sidecar is current, which the admin build already writes.
+	// `sourceIdentity` lets a caller that already knows the digest (or is building from something that
+	// is not a file at all) supply it instead.
+	progress("stamp", `Reading source identity for ${opts.dbPath}`)
+	const source = opts.sourceIdentity ?? readWOFSourceIdentity(opts.dbPath)
+
 	const provenance: FSTProvenance = {
 		builtAt: new Date().toISOString(),
 		countries,
@@ -327,8 +349,12 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
 		placeCount: sprRows.length,
 		edgeCount,
 		nameInsertions: insertCount,
-		importanceMatches: importanceMap.size,
+		importanceMatches: split.referential.size,
+		encyclopedicMatches: split.encyclopedic.size,
+		importanceSource: split.source,
 		sourceDB: opts.dbPath,
+		sourceDBMD5: source.md5,
+		sourceDBBytes: source.bytes,
 		...(excludeSurfaces !== undefined || excludeAllTokensOf !== undefined
 			? { exclusionPolicy: opts.exclusionPolicy ?? "unspecified", excludedInsertions: excludedCount }
 			: {}),

package/fst-deserialize-web.ts

@@ -10,6 +10,8 @@
  *   build-time operation).
  */
 
+import { tryParsingJSON } from "@mailwoman/core/objects"
+
 import type { FSTNode } from "./fst-matcher.ts"
 import { FSTMatcher } from "./fst-matcher.ts"
 import type { FSTProvenance, PlaceEntry, PlacetypeID } from "./fst-types.ts"
@@ -37,17 +39,44 @@ const VERSION_WITH_METADATA = 3
 
 const HEADER_SIZE = 32
 const EDGE_ENTRY_SIZE = 8
-const PLACE_ENTRY_SIZE = 56
+
+/**
+ * Format version that split the single `importance` float into `referential` + `encyclopedic` (ROAD_TO_V9 §2 R1),
+ * growing the place entry from 56 to 60 bytes. Mirrors `fst-serialize.ts`'s constant of the same name.
+ */
+const VERSION_TWO_SCORE_SPLIT = 5
+
+/**
+ * Place-entry size in bytes at or above {@link VERSION_TWO_SCORE_SPLIT}.
+ */
+const SPLIT_PLACE_ENTRY_SIZE = 60
+
+/**
+ * Place-entry size in bytes below {@link VERSION_TWO_SCORE_SPLIT}.
+ */
+const LEGACY_PLACE_ENTRY_SIZE = 56
+
+/**
+ * Byte offset of the encyclopedic float inside a v5 place entry — immediately after the 8-slot parent chain.
+ */
+const ENCYCLOPEDIC_OFFSET = 56
+
+/**
+ * `placeFlags` bit 0 (byte `pp+7`, v5+): this place carries an encyclopedic score. Per-place because absence is the
+ * common case; see `fst-serialize.ts`.
+ */
+const PLACE_FLAG_HAS_ENCYCLOPEDIC = 1
 /**
  * "FST\0".
  */
 const MAGIC_BYTES = [0x46, 0x53, 0x54, 0x00]
 /**
- * Must track the serializer's VERSION (fst-serialize.ts, currently 4). The v3 provenance + v4 16-byte-state/u32-count
+ * Must track the serializer's VERSION (fst-serialize.ts, currently 5). The v3 provenance + v4 16-byte-state/u32-count
  * layout logic below already matches the Node deserializer; only this gate was left stale at 2, so the browser FST
- * loader rejected every real (v4) artifact.
+ * loader rejected every real (v4) artifact. It was left stale again at 4 by the v5 two-score split until this line
+ * moved with it — the gate is a SEPARATE number from the layout branches, which is exactly why it keeps drifting.
  */
-const MAX_VERSION = 4
+const MAX_VERSION = 5
 
 const PLACETYPE_ORDER: readonly PlacetypeID[] = [
 	"country",
@@ -86,6 +115,7 @@ export function deserializeFSTWeb(input: ArrayBuffer | Uint8Array): FSTMatcher {
 	}
 
 	const isV2 = version >= 2
+	const isSplit = version >= VERSION_TWO_SCORE_SPLIT
 	// flags bit0 (survey #4, mirrors fst-serialize.ts): place rows carry surface-ambiguity data.
 	const hasAmbiguity = (view.getUint16(6, true) & 1) === 1
 
@@ -118,6 +148,8 @@ export function deserializeFSTWeb(input: ArrayBuffer | Uint8Array): FSTMatcher {
 
 	// --- State table ---
 	const stateEntrySize = version >= VERSION_WIDE_STATE_COUNTERS ? WIDE_STATE_ENTRY_SIZE : NARROW_STATE_ENTRY_SIZE
+	// v5 grew the place entry by the encyclopedic float; v4-and-below files are read at the old stride.
+	const placeEntrySize = isSplit ? SPLIT_PLACE_ENTRY_SIZE : LEGACY_PLACE_ENTRY_SIZE
 	const stateTableStart = pos
 	const edgeTableStart = stateTableStart + stateCount * stateEntrySize
 	const placeTableStart = edgeTableStart + edgeCount * EDGE_ENTRY_SIZE
@@ -147,7 +179,7 @@ export function deserializeFSTWeb(input: ArrayBuffer | Uint8Array): FSTMatcher {
 		const places: PlaceEntry[] = new Array(placeCountForState)
 
 		for (let pi = 0; pi < placeCountForState; pi++) {
-			const pp = placeTableStart + (placeStart + pi) * PLACE_ENTRY_SIZE
+			const pp = placeTableStart + (placeStart + pi) * placeEntrySize
 			const chainLen = view.getUint8(pp + 5)
 			const parentChain: number[] = []
 
@@ -155,19 +187,25 @@ export function deserializeFSTWeb(input: ArrayBuffer | Uint8Array): FSTMatcher {
 				parentChain.push(view.getUint32(pp + 24 + ci * 4, true))
 			}
 
-			const rawImportance = isV2
+			// v1 stored a raw population u32 here; v2-v4 the conflated `importance` float; v5 the
+			// referential score. See the Node deserializer for why a v1 value is genuinely referential.
+			const referential = isV2
 				? view.getFloat32(pp + 12, true)
 				: Math.min(1, Math.log2(1 + view.getUint32(pp + 12, true) / 1000) / 14)
 
+			const hasEncyclopedic =
+				isSplit && (view.getUint8(pp + 7) & PLACE_FLAG_HAS_ENCYCLOPEDIC) === PLACE_FLAG_HAS_ENCYCLOPEDIC
+
 			places[pi] = {
 				wofID: view.getUint32(pp, true),
 				placetype: PLACETYPE_ORDER[view.getUint8(pp + 4)] ?? "locality",
 				name: strings[view.getUint32(pp + 8, true)]!,
-				importance: rawImportance,
+				referential,
 				lat: view.getFloat32(pp + 16, true),
 				lon: view.getFloat32(pp + 20, true),
 				parentChain,
 				...(hasAmbiguity ? { crossCountryBranches: view.getUint8(pp + 6) } : {}),
+				...(hasEncyclopedic ? { encyclopedic: view.getFloat32(pp + ENCYCLOPEDIC_OFFSET, true) } : {}),
 			}
 		}
 
@@ -194,7 +232,7 @@ export function readFSTProvenanceWeb(input: ArrayBuffer | Uint8Array): FSTProven
 		const jsonLen = view.getUint32(provenanceOffset, true)
 		const jsonStr = decoder.decode(bytes.subarray(provenanceOffset + 4, provenanceOffset + 4 + jsonLen))
 
-		return JSON.parse(jsonStr) as FSTProvenance
+		return tryParsingJSON<FSTProvenance>(jsonStr) ?? undefined
 	} catch {
 		return undefined
 	}

package/fst-freshness.ts

@@ -0,0 +1,333 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Does an `fst-*.bin` still match the gazetteer it was built from?
+ *
+ *   WHY THIS EXISTS. Every FST artifact is a projection of one WOF admin database, and the admin
+ *   database is a sealed readonly artifact that a rebuild REPLACES. Nothing mechanically tied the two
+ *   together: the 2026-08-04 admin swap (4.87M rows, ancestry repaired, macrohood/microhood ingested)
+ *   left `fst-global-priority.bin` at its 2026-05-28 build and the per-locale set at 2026-07-26, and
+ *   the only way to notice was to compare mtimes by hand. The artifacts kept loading, kept answering
+ *   queries, and answered them from a gazetteer that no longer exists.
+ *
+ *   `FSTProvenance` already recorded `sourceDB` — the source's PATH, which is exactly the field that
+ *   cannot change when the bytes behind it do. So the stamp gains the source's IDENTITY (md5 + byte
+ *   size) and this module compares it. The shape deliberately mirrors
+ *   `scripts/weights-overlay-linker.ts`'s `pairIndexStaleReason`: one function returning a reason
+ *   string or `undefined`, so a fact added to the stamp cannot be checked by some callers and not
+ *   others — which is how three of the four base linkers ended up unable to notice a PIX1 schema bump.
+ *
+ *   FORMAT IS PART OF FRESHNESS. The check compares the serializer version too, not just the source
+ *   md5. A guard that checks only the source reads a format-obsolete binary as "current" (the R5
+ *   freshness-guard lesson, format edition), and a file below {@link MIN_STAMPED_FORMAT_VERSION}
+ *   cannot carry a stamp at all — reported as its own reason rather than silently passing.
+ *
+ *   STALE IS A WARNING, NOT A FAULT. A dev tree with an old FST must still run; the artifact is a
+ *   decode-time bias list, not a correctness dependency. Callers print {@link formatFSTStaleWarning}
+ *   and continue.
+ */
+
+import { createHash } from "node:crypto"
+import { closeSync, existsSync, openSync, readFileSync, readSync, statSync, writeFileSync } from "node:fs"
+
+import { tryParsingJSON } from "@mailwoman/core/objects"
+
+import { FST_FORMAT_VERSION } from "./fst-serialize.ts"
+import type { FSTProvenance } from "./fst-types.ts"
+
+/**
+ * Fixed header size in bytes — mirrors `fst-serialize.ts`'s `HEADER_SIZE`. Duplicated rather than exported across
+ * because this module reads the header by SEEK (never buffering the file), and the serializer's constant is private to
+ * its own read/write pair.
+ */
+const HEADER_SIZE = 32
+
+/**
+ * `"FST\0"` little-endian, the four magic bytes every artifact opens with.
+ */
+const MAGIC = 0x00_54_53_46
+
+/**
+ * Byte offset of the u32 trailer offset inside the header (the last of its eight fields).
+ */
+const PROVENANCE_OFFSET_FIELD = 28
+
+/**
+ * First serializer version carrying the trailing provenance block. Below this a file has no place to put a stamp, so
+ * "unstamped" is a statement about the FORMAT, not about the builder.
+ */
+export const MIN_STAMPED_FORMAT_VERSION = 3
+
+/**
+ * Hex characters in an md5 digest.
+ */
+const MD5_HEX_LENGTH = 32
+
+/**
+ * Bytes per read in {@link md5FileSync}. 8 MiB measured at 7.3 s for the 5.27 GB admin DB on the lab playpen — the same
+ * throughput as `md5sum(1)`, so the sidecar below is what makes the check cheap, not the chunk size.
+ */
+const MD5_CHUNK_BYTES = 8 * 1024 * 1024
+
+/**
+ * What an FST was built from, recorded so a later reader can tell whether that thing still exists.
+ *
+ * `bytes` is not redundant with `md5` — it is the field that survives a truncated or half-written source and makes the
+ * mismatch legible in the warning ("5,273,722,880 → 5,372,076,032" names the rebuild; a hex delta does not).
+ */
+export interface FSTSourceIdentity {
+	md5: string
+	bytes: number
+}
+
+/**
+ * The stamp fields a freshness guard reads off an artifact, plus the format version it was written at.
+ *
+ * Every field is optional because every one of them can be legitimately absent on an artifact that predates it, and the
+ * meaning-of-zero rule applies to all of them: absent is a distinct state from "matches", and the reasons below say so
+ * in different words.
+ */
+export interface FSTStampFields {
+	formatVersion: number
+	provenance: FSTProvenance | undefined
+}
+
+/**
+ * What a caller expects the artifact to have been built from.
+ *
+ * `exclusionPolicy` is optional and caller-supplied on purpose. The policy id lives in
+ * `mailwoman/gazetteer-pipeline/fst.ts` (which depends on this package, not the other way round), so only the caller
+ * knows which policy it means — the same split as the pair-index guard, where the format+magnitude half is shared and
+ * the source-md5 half stays with the script that knows its sources.
+ */
+export interface FSTExpectation {
+	source: FSTSourceIdentity
+	/**
+	 * Serializer version to require. Defaults to {@link FST_FORMAT_VERSION} — the version this tree writes — so a caller
+	 * cannot forget the format half of the comparison. Override only to check against a specific older floor.
+	 */
+	formatVersion?: number
+	exclusionPolicy?: string
+}
+
+/**
+ * Read an artifact's stamp WITHOUT deserializing it — a header seek plus the trailer, three reads totalling a few
+ * kilobytes. The distinction matters: `fst-global-priority.bin` is 317 MB and this runs on every `yarn test` via the
+ * weights linkers, so `readFileSync` + `readFSTProvenance` would trade a freshness guard for a slower test suite and
+ * nobody would keep it.
+ *
+ * Returns `undefined` for a file that is absent, too small, or not an FST at all — none of which is this function's
+ * business to diagnose.
+ */
+export function peekFSTStampFields(path: string): FSTStampFields | undefined {
+	if (!existsSync(path)) return undefined
+	const size = statSync(path).size
+
+	if (size < HEADER_SIZE) return undefined
+	const fd = openSync(path, "r")
+
+	try {
+		const header = Buffer.alloc(HEADER_SIZE)
+		readSync(fd, header, 0, HEADER_SIZE, 0)
+
+		if (header.readUInt32LE(0) !== MAGIC) return undefined
+		const formatVersion = header.readUInt16LE(4)
+
+		if (formatVersion < MIN_STAMPED_FORMAT_VERSION) return { formatVersion, provenance: undefined }
+		const trailerStart = header.readUInt32LE(PROVENANCE_OFFSET_FIELD)
+
+		// 0 = "this build wrote no trailer" (the serializer's own encoding); anything past EOF is a
+		// truncated file. Both read as "no stamp", which is what the caller does with them anyway.
+		if (trailerStart === 0 || trailerStart + 4 > size) return { formatVersion, provenance: undefined }
+
+		const lengthBytes = Buffer.alloc(4)
+		readSync(fd, lengthBytes, 0, 4, trailerStart)
+		const jsonLength = lengthBytes.readUInt32LE(0)
+
+		if (jsonLength === 0 || trailerStart + 4 + jsonLength > size) return { formatVersion, provenance: undefined }
+
+		const json = Buffer.alloc(jsonLength)
+		readSync(fd, json, 0, jsonLength, trailerStart + 4)
+
+		return { formatVersion, provenance: tryParsingJSON<FSTProvenance>(json.toString("utf8")) ?? undefined }
+	} finally {
+		closeSync(fd)
+	}
+}
+
+/**
+ * Streaming MD5 of a file, SYNCHRONOUS.
+ *
+ * The async `md5File` in `@mailwoman/core/utils` is the one to reach for anywhere else. This exists because the FST
+ * builder and its whole call chain are synchronous by design (`buildFSTFromWOF` → `buildLocaleFSTs`), and making them
+ * async to stamp a checksum would cascade through the Pastel commands and the tests for one hash. It reads in
+ * {@link MD5_CHUNK_BYTES} chunks rather than `readFileSync` — the source is a multi-gigabyte database.
+ */
+export function md5FileSync(path: string): string {
+	const hash = createHash("md5")
+	const fd = openSync(path, "r")
+
+	try {
+		const chunk = Buffer.alloc(MD5_CHUNK_BYTES)
+
+		for (;;) {
+			const read = readSync(fd, chunk, 0, MD5_CHUNK_BYTES, null)
+
+			if (read <= 0) break
+			hash.update(chunk.subarray(0, read))
+		}
+	} finally {
+		closeSync(fd)
+	}
+
+	return hash.digest("hex")
+}
+
+/**
+ * The source identity an FST build should stamp, or a check should compare against.
+ *
+ * Uses the `.md5` sidecar convention the weights linkers already established on this exact file: `<path>.md5` in
+ * md5sum(1) format (`<hash> <filename>`), trusted only while its mtime is at least the source's. An older sidecar is
+ * recomputed. Without it the admin DB costs 7.3 s per call and the guard would be quietly disabled by whoever noticed
+ * `yarn test` got slower.
+ *
+ * `refreshSidecar` writes the recomputed digest back. Best-effort: a sealed data root or a read-only mount fails the
+ * write and the caller still gets its answer, because refusing to check freshness on a read-only tree would be the
+ * wrong trade.
+ */
+export function readWOFSourceIdentity(path: string, { refreshSidecar = true } = {}): FSTSourceIdentity {
+	const stats = statSync(path)
+	const memoKey = `${path}\0${stats.mtimeMs}\0${stats.size}`
+	const hit = sourceIdentityMemo.get(memoKey)
+
+	if (hit) return hit
+	const sidecarPath = `${path}.md5`
+	let md5: string | undefined
+
+	if (existsSync(sidecarPath)) {
+		const sidecarStats = statSync(sidecarPath)
+
+		if (sidecarStats.mtimeMs >= stats.mtimeMs) {
+			const [hash] = readFileSync(sidecarPath, "utf8").trim().split(/\s+/)
+
+			if (hash && hash.length === MD5_HEX_LENGTH) {
+				md5 = hash
+			}
+		}
+	}
+
+	if (!md5) {
+		md5 = md5FileSync(path)
+
+		if (refreshSidecar) {
+			try {
+				writeFileSync(sidecarPath, `${md5}  ${path.split("/").pop()}\n`)
+			} catch {
+				// Read-only data root — the digest is still correct, it just isn't cached.
+			}
+		}
+	}
+
+	const identity: FSTSourceIdentity = { md5, bytes: stats.size }
+	sourceIdentityMemo.set(memoKey, identity)
+
+	return identity
+}
+
+/**
+ * Memo for {@link readWOFSourceIdentity}, keyed on (path, mtimeMs, size) — NOT on path alone, for the same reason
+ * `computeSurfaceCountryCounts` isn't: the admin DB is a sealed artifact that a rebuild REPLACES, so a path-only memo
+ * would serve a stale digest against a new file for the life of the process.
+ */
+const sourceIdentityMemo = new Map<string, FSTSourceIdentity>()
+
+/**
+ * Why an FST artifact is stale against `expected`, or `undefined` when it still matches.
+ *
+ * The order is deliberate: FORMAT first (a version-obsolete file is stale whatever its source says), then the presence
+ * of a stamp, then the source identity, then the build policy. Each returns prose a reader can act on — the reasons are
+ * printed verbatim into {@link formatFSTStaleWarning}.
+ */
+export function fstStaleReason(fields: FSTStampFields | undefined, expected: FSTExpectation): string | undefined {
+	if (!fields) return "unreadable or not an FST artifact"
+	const requiredFormat = expected.formatVersion ?? FST_FORMAT_VERSION
+
+	if (fields.formatVersion < requiredFormat) {
+		return `format v${fields.formatVersion} → v${requiredFormat}`
+	}
+
+	if (fields.formatVersion < MIN_STAMPED_FORMAT_VERSION) {
+		return `format v${fields.formatVersion} predates the build stamp (needs v${MIN_STAMPED_FORMAT_VERSION}+)`
+	}
+
+	const provenance = fields.provenance
+
+	if (!provenance) return "carries no build stamp"
+
+	if (!provenance.sourceDBMD5) {
+		return `built ${provenance.builtAt} with no source checksum — rebuild to stamp one`
+	}
+
+	if (provenance.sourceDBMD5 !== expected.source.md5) {
+		return `source db ${provenance.sourceDBMD5.slice(0, 8)} → ${expected.source.md5.slice(0, 8)} (built ${provenance.builtAt})`
+	}
+
+	// Reached only when the md5s agree, so a size disagreement means one of the two was recorded
+	// against a different file than it was hashed from. Cheap to check, and it never fires by accident.
+	if (provenance.sourceDBBytes !== undefined && provenance.sourceDBBytes !== expected.source.bytes) {
+		return `source db size ${provenance.sourceDBBytes} → ${expected.source.bytes} at a matching md5 — one of the two is misrecorded`
+	}
+
+	if (expected.exclusionPolicy !== undefined && provenance.exclusionPolicy !== expected.exclusionPolicy) {
+		return `exclusion policy ${provenance.exclusionPolicy ?? "(none)"} → ${expected.exclusionPolicy}`
+	}
+
+	return undefined
+}
+
+/**
+ * The whole check, for a caller that has a path and a source DB and wants a warning string or nothing.
+ *
+ * Returns `undefined` when the artifact is current OR when it is absent — an absent artifact is a different problem
+ * with a different message, and every existing caller already reports it in place.
+ */
+export function fstFreshnessWarning({
+	fstPath,
+	sourceDBPath,
+	formatVersion,
+	exclusionPolicy,
+	rebuildCommand,
+}: {
+	fstPath: string
+	sourceDBPath: string
+	formatVersion?: number
+	exclusionPolicy?: string
+	rebuildCommand: string
+}): string | undefined {
+	if (!existsSync(fstPath) || !existsSync(sourceDBPath)) return undefined
+
+	const reason = fstStaleReason(peekFSTStampFields(fstPath), {
+		source: readWOFSourceIdentity(sourceDBPath),
+		...(formatVersion === undefined ? {} : { formatVersion }),
+		...(exclusionPolicy === undefined ? {} : { exclusionPolicy }),
+	})
+
+	return reason === undefined ? undefined : formatFSTStaleWarning({ fstPath, reason, rebuildCommand })
+}
+
+/**
+ * The one warning format, so `grep -r "FST STALE"` finds every site that can emit one.
+ */
+export function formatFSTStaleWarning({
+	fstPath,
+	reason,
+	rebuildCommand,
+}: {
+	fstPath: string
+	reason: string
+	rebuildCommand: string
+}): string {
+	return `WARNING: FST STALE — ${fstPath}: ${reason}. Rebuild with: ${rebuildCommand}`
+}