@mailwoman/resolver-wof-sqlite 10.0.0 → 10.2.0

package/lib/fst/freshness.ts

@@ -3,74 +3,41 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Does an `fst-*.bin` still match the gazetteer it was built from?
- *
- *   WHY THIS EXISTS. Every FST artifact is a projection of one WOF admin database, and the admin
- *   database is a sealed readonly artifact that a rebuild REPLACES. Nothing mechanically tied the two
- *   together: the 2026-08-04 admin swap (4.87M rows, ancestry repaired, macrohood/microhood ingested)
- *   left `fst-global-priority.bin` at its 2026-05-28 build and the per-locale set at 2026-07-26, and
- *   the only way to notice was to compare mtimes by hand. The artifacts kept loading, kept answering
- *   queries, and answered them from a gazetteer that no longer exists.
- *
- *   `FSTProvenance` already recorded `sourceDB` — the source's PATH, which is exactly the field that
- *   cannot change when the bytes behind it do. So the stamp gains the source's IDENTITY (md5 + byte
- *   size) and this module compares it. The shape deliberately mirrors
- *   `@mailwoman/resolver-wof-sqlite/weights-overlay-linker`'s `pairIndexStaleReason`: one function returning a reason
- *   string or `undefined`, so a fact added to the stamp cannot be checked by some callers and not
- *   others — which is how three of the four base linkers ended up unable to notice a PIX1 schema bump.
- *
- *   FORMAT IS PART OF FRESHNESS. The check compares the serializer version too, not just the source
- *   md5. A guard that checks only the source reads a format-obsolete binary as "current" (the R5
- *   freshness-guard lesson, format edition), and a file below {@link MIN_STAMPED_FORMAT_VERSION}
- *   cannot carry a stamp at all — reported as its own reason rather than silently passing.
- *
- *   STALE IS A WARNING, NOT A FAULT. A dev tree with an old FST must still run; the artifact is a
- *   decode-time bias list, not a correctness dependency. Callers print {@link formatFSTStaleWarning}
- *   and continue.
+ * Does an `fst-*.bin` still match the gazetteer it was built from?
  */
 
-import { pathExists, readFileRange, readLocalTextFile, statPath } from "@mailwoman/core/fs/readers"
+import { readFileRange, readLocalTextFile } from "@mailwoman/core/fs/readers"
+import { pathExists, statPath } from "@mailwoman/core/fs/readers/stat"
 import { writeLocalTextFile } from "@mailwoman/core/fs/writers"
 import { md5File } from "@mailwoman/core/hash"
 import { tryParsingJSON } from "@mailwoman/core/json"
 import { basename, resolvePath, type PathBuilderLike } from "path-ts"
 
-import { FST_FORMAT_VERSION } from "#fst/serialize"
+import { FST_FORMAT_VERSION } from "#fst/format"
 import type { FSTProvenance } from "#fst/types"
 
 /**
- * Fixed header size in bytes — mirrors `fst-serialize.ts`'s `HEADER_SIZE`. Duplicated rather than exported across
- * because this module reads the header by SEEK (never buffering the file), and the serializer's constant is private to
- * its own read/write pair.
+ * Fixed header size in bytes, mirroring `fst-serialize.ts`'s private `HEADER_SIZE`.
  */
 const HEADER_SIZE = 32
 
 /**
- * `"FST\0"` little-endian, the four magic bytes every artifact opens with.
+ * `"FST\0"` little-endian, the four bytes every artifact opens with.
  */
 const MAGIC = 0x00_54_53_46
 
-/**
- * Byte offset of the u32 trailer offset inside the header (the last of its eight fields).
- */
 const PROVENANCE_OFFSET_FIELD = 28
 
 /**
- * First serializer version carrying the trailing provenance block. Below this a file has no place to put a stamp, so
- * "unstamped" is a statement about the FORMAT, not about the builder.
+ * First serializer version carrying the trailing provenance block. below it
+ * a file has no place to put a stamp.
  */
 export const MIN_STAMPED_FORMAT_VERSION = 3
 
-/**
- * Hex characters in an md5 digest.
- */
 const MD5_HEX_LENGTH = 32
 
 /**
  * What an FST was built from, recorded so a later reader can tell whether that thing still exists.
- *
- * `bytes` is not redundant with `md5` — it is the field that survives a truncated or half-written source and makes the
- * mismatch legible in the warning ("5,273,722,880 → 5,372,076,032" names the rebuild; a hex delta does not).
  */
 export interface FSTSourceIdentity {
 	md5: string
@@ -79,85 +46,55 @@ export interface FSTSourceIdentity {
 
 /**
  * The stamp fields a freshness guard reads off an artifact, plus the format version it was written at.
- *
- * Every field is optional because every one of them can be legitimately absent on an artifact that predates it, and the
- * meaning-of-zero rule applies to all of them: absent is a distinct state from "matches", and the reasons below say so
- * in different words.
  */
 export interface FSTStampFields {
 	formatVersion: number
-	provenance: FSTProvenance | undefined
+	provenance: FSTProvenance | null
 }
 
 /**
  * What a caller expects the artifact to have been built from.
- *
- * `exclusionPolicy` is optional and caller-supplied on purpose. The policy id lives in
- * `mailwoman/gazetteer-pipeline/fst.ts` (which depends on this package, not the other way round), so only the caller
- * knows which policy it means — the same split as the pair-index guard, where the format+magnitude half is shared and
- * the source-md5 half stays with the script that knows its sources.
  */
 export interface FSTExpectation {
 	source: FSTSourceIdentity
 	/**
-	 * Serializer version to require. Defaults to {@link FST_FORMAT_VERSION} — the version this tree writes — so a caller
-	 * cannot forget the format half of the comparison. Override only to check against a specific older floor.
+	 * Serializer version to require, defaulting to {@link FST_FORMAT_VERSION}.
 	 */
-	formatVersion?: number
-	exclusionPolicy?: string
+	formatVersion?: number | null
+	exclusionPolicy?: string | null
 }
 
 /**
- * Read an artifact's stamp WITHOUT deserializing it — a header seek plus the trailer, three reads totalling a few
- * kilobytes. The distinction matters: `fst-global-priority.bin` is 317 MB and this runs on every `yarn test` via the
- * weights linkers, so `readFileSync` + `readFSTProvenance` would trade a freshness guard for a slower test suite and
- * nobody would keep it.
- *
- * Returns `undefined` for a file that is absent, too small, or not an FST at all — none of which is this function's
- * business to diagnose.
+ * Read an artifact's stamp without deserializing it.
  */
-export async function peekFSTStampFields(path: string): Promise<FSTStampFields | undefined> {
-	if (!(await pathExists(path))) return undefined
+export async function peekFSTStampFields(path: PathBuilderLike): Promise<FSTStampFields | null> {
+	if (!(await pathExists(path))) return null
 	const size = (await statPath(path)).size
 
-	if (size < HEADER_SIZE) return undefined
+	if (size < HEADER_SIZE) return null
 	const header = await readFileRange(path, 0, HEADER_SIZE)
 
-	if (header.readUInt32LE(0) !== MAGIC) return undefined
+	if (header.readUInt32LE(0) !== MAGIC) return null
 	const formatVersion = header.readUInt16LE(4)
 
-	if (formatVersion < MIN_STAMPED_FORMAT_VERSION) return { formatVersion, provenance: undefined }
+	if (formatVersion < MIN_STAMPED_FORMAT_VERSION) return { formatVersion, provenance: null }
 	const trailerStart = header.readUInt32LE(PROVENANCE_OFFSET_FIELD)
 
-	// 0 = "this build wrote no trailer" (the serializer's own encoding); anything past EOF is a
-	// truncated file. Both read as "no stamp", which is what the caller does with them anyway.
-	if (trailerStart === 0 || trailerStart + 4 > size) return { formatVersion, provenance: undefined }
+	// 0 means this build wrote no trailer and past-EOF means truncated.
+	// Both read as no stamp.
+	if (trailerStart === 0 || trailerStart + 4 > size) return { formatVersion, provenance: null }
 
 	const jsonLength = (await readFileRange(path, trailerStart, 4)).readUInt32LE(0)
 
-	if (jsonLength === 0 || trailerStart + 4 + jsonLength > size) return { formatVersion, provenance: undefined }
+	if (jsonLength === 0 || trailerStart + 4 + jsonLength > size) return { formatVersion, provenance: null }
 
 	const json = await readFileRange(path, trailerStart + 4, jsonLength)
 
-	return { formatVersion, provenance: tryParsingJSON<FSTProvenance>(json.toString("utf8")) ?? undefined }
+	return { formatVersion, provenance: tryParsingJSON<FSTProvenance>(json.toString("utf8")) ?? null }
 }
 
-/**
- * Re-exported from `@mailwoman/core/hash`. This package's own test imports it from here, and the sidecar convention
- * below is what it is for.
- */
-
 /**
  * The source identity an FST build should stamp, or a check should compare against.
- *
- * Uses the `.md5` sidecar convention the weights linkers already established on this exact file: `<path>.md5` in
- * md5sum(1) format (`<hash> <filename>`), trusted only while its mtime is at least the source's. An older sidecar is
- * recomputed. Without it the admin DB costs 7.3 s per call and the guard would be quietly disabled by whoever noticed
- * `yarn test` got slower.
- *
- * `refreshSidecar` writes the recomputed digest back. Best-effort: a sealed data root or a read-only mount fails the
- * write and the caller still gets its answer, because refusing to check freshness on a read-only tree would be the
- * wrong trade.
  */
 export async function readWOFSourceIdentity(
 	source: PathBuilderLike,
@@ -170,7 +107,7 @@ export async function readWOFSourceIdentity(
 
 	if (hit) return hit
 	const sidecarPath = `${path}.md5`
-	let md5: string | undefined
+	let md5: string | null = null
 
 	if (await pathExists(sidecarPath)) {
 		const sidecarStats = await statPath(sidecarPath)
@@ -191,7 +128,7 @@ export async function readWOFSourceIdentity(
 			try {
 				await writeLocalTextFile(`${md5}  ${basename(path)}\n`, sidecarPath)
 			} catch {
-				// Read-only data root — the digest is still correct, it just isn't cached.
+				// Read-only data root: the digest is still correct, only the cache write failed.
 			}
 		}
 	}
@@ -202,21 +139,12 @@ export async function readWOFSourceIdentity(
 	return identity
 }
 
-/**
- * Memo for {@link readWOFSourceIdentity}, keyed on (path, mtimeMs, size) — NOT on path alone, for the same reason
- * `computeSurfaceCountryCounts` isn't: the admin DB is a sealed artifact that a rebuild REPLACES, so a path-only memo
- * would serve a stale digest against a new file for the life of the process.
- */
 const sourceIdentityMemo = new Map<string, FSTSourceIdentity>()
 
 /**
- * Why an FST artifact is stale against `expected`, or `undefined` when it still matches.
- *
- * The order is deliberate: FORMAT first (a version-obsolete file is stale whatever its source says), then the presence
- * of a stamp, then the source identity, then the build policy. Each returns prose a reader can act on — the reasons are
- * printed verbatim into {@link formatFSTStaleWarning}.
+ * Why an FST artifact is stale against `expected`, or `null` when it still matches.
  */
-export function fstStaleReason(fields: FSTStampFields | undefined, expected: FSTExpectation): string | undefined {
+export function fstStaleReason(fields: FSTStampFields | null, expected: FSTExpectation): string | null {
 	if (!fields) return "unreadable or not an FST artifact"
 	const requiredFormat = expected.formatVersion ?? FST_FORMAT_VERSION
 
@@ -240,24 +168,19 @@ export function fstStaleReason(fields: FSTStampFields | undefined, expected: FST
 		return `source db ${provenance.sourceDBMD5.slice(0, 8)} → ${expected.source.md5.slice(0, 8)} (built ${provenance.builtAt})`
 	}
 
-	// Reached only when the md5s agree, so a size disagreement means one of the two was recorded
-	// against a different file than it was hashed from. Cheap to check, and it never fires by accident.
-	if (provenance.sourceDBBytes !== undefined && provenance.sourceDBBytes !== expected.source.bytes) {
+	if (provenance.sourceDBBytes != null && provenance.sourceDBBytes !== expected.source.bytes) {
 		return `source db size ${provenance.sourceDBBytes} → ${expected.source.bytes} at a matching md5 — one of the two is misrecorded`
 	}
 
-	if (expected.exclusionPolicy !== undefined && provenance.exclusionPolicy !== expected.exclusionPolicy) {
+	if (expected.exclusionPolicy && provenance.exclusionPolicy !== expected.exclusionPolicy) {
 		return `exclusion policy ${provenance.exclusionPolicy ?? "(none)"} → ${expected.exclusionPolicy}`
 	}
 
-	return undefined
+	return null
 }
 
 /**
- * The whole check, for a caller that has a path and a source DB and wants a warning string or nothing.
- *
- * Returns `undefined` when the artifact is current OR when it is absent — an absent artifact is a different problem
- * with a different message, and every existing caller already reports it in place.
+ * The whole check, for a caller with a path and a source DB that wants a warning string or none.
  */
 export async function fstFreshnessWarning({
 	fstPath,
@@ -266,34 +189,34 @@ export async function fstFreshnessWarning({
 	exclusionPolicy,
 	rebuildCommand,
 }: {
-	fstPath: string
-	sourceDBPath: string
-	formatVersion?: number
-	exclusionPolicy?: string
+	fstPath: PathBuilderLike
+	sourceDBPath: PathBuilderLike
+	formatVersion?: number | null
+	exclusionPolicy?: string | null
 	rebuildCommand: string
-}): Promise<string | undefined> {
-	if (!(await pathExists(fstPath)) || !(await pathExists(sourceDBPath))) return undefined
+}): Promise<string | null> {
+	if (!(await pathExists(fstPath)) || !(await pathExists(sourceDBPath))) return null
 
 	const reason = fstStaleReason(await peekFSTStampFields(fstPath), {
 		source: await readWOFSourceIdentity(sourceDBPath),
-		...(formatVersion === undefined ? {} : { formatVersion }),
-		...(exclusionPolicy === undefined ? {} : { exclusionPolicy }),
+		formatVersion: formatVersion ?? null,
+		exclusionPolicy: exclusionPolicy ?? null,
 	})
 
-	return reason === undefined ? undefined : formatFSTStaleWarning({ fstPath, reason, rebuildCommand })
+	return reason ? formatFSTStaleWarning({ fstPath, reason, rebuildCommand }) : null
 }
 
 /**
- * The one warning format, so `grep -r "FST STALE"` finds every site that can emit one.
+ * The one warning format, so `grep -r "FST stale"` finds every site that emits one.
  */
 export function formatFSTStaleWarning({
 	fstPath,
 	reason,
 	rebuildCommand,
 }: {
-	fstPath: string
+	fstPath: PathBuilderLike
 	reason: string
 	rebuildCommand: string
 }): string {
-	return `WARNING: FST STALE — ${fstPath}: ${reason}. Rebuild with: ${rebuildCommand}`
+	return `WARNING: FST STALE — ${fstPath.toString()}: ${reason}. Rebuild with: ${rebuildCommand}`
 }

package/lib/fst/matcher.ts

@@ -8,7 +8,9 @@
  *   arrays at accepting states.
  */
 
-import type { FSTContinuation, FSTMatchResult, FSTQueryResult, PlaceEntry } from "#fst/types"
+import type { AncestrieMatch } from "@mailwoman/ancestrie"
+
+import type { FSTContinuation, FSTQueryResult, PlaceEntry } from "#fst/types"
 
 interface FSTNode {
 	edges: Map<string, number>
@@ -36,7 +38,7 @@ export class FSTMatcher {
 		return count
 	}
 
-	walk(tokens: string[]): FSTMatchResult | null {
+	walk(tokens: string[]): AncestrieMatch | null {
 		let stateID = 0
 
 		for (const token of tokens) {
@@ -54,7 +56,7 @@ export class FSTMatcher {
 		return { stateID, accepted: node.places.length > 0, depth: tokens.length }
 	}
 
-	walkFrom(prev: FSTMatchResult, token: string): FSTMatchResult | null {
+	walkFrom(prev: AncestrieMatch, token: string): AncestrieMatch | null {
 		const node = this.nodes[prev.stateID]
 
 		if (!node) return null
@@ -143,7 +145,7 @@ export class FSTMatcher {
 }
 
 /**
- * Normalize text into FST tokens: lowercase, NFKC, strip punctuation, split on whitespace.
+ * Normalize text into FST tokens: lowercase, nfkc, strip punctuation, split on whitespace.
  */
 export function normalizeTokens(text: string): string[] {
 	return text
@@ -151,7 +153,7 @@ export function normalizeTokens(text: string): string[] {
 		.toLowerCase()
 		.replaceAll(/[\p{P}\p{S}]/gu, "")
 		.split(/\s+/)
-		.filter((t) => t.length > 0)
+		.filter((t) => t.length)
 }
 
 export type { FSTNode }

package/lib/fst/serialize.ts

@@ -2,38 +2,27 @@
  * @copyright Sister Software
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
+ * @file Binary serialization for the FST gazetteer.
  *
- *   Binary serialization for the FST gazetteer. Format:
+ *   All integers are little-endian. The writer emits the current version. The reader accepts every earlier version.
  *
- *   HEADER (32 bytes) magic [u8; 4] "FST\0" version u16 1 flags u16 0 (reserved) stateCount u32
- *   edgeCount u32 total edges across all states placeCount u32 total place entries across all
- *   states stringCount u32 unique strings in string table stringBytes u32 total bytes of string
- *   data _reserved u32
+ *   - Header, 32 bytes: magic `FST\0`, version u16, flags u16 (bit 0 means `crossCountryBranches` is present),
+ *     The header stores five u32 counts (stateCount, edgeCount, placeCount, stringCount, stringBytes).
+ *     Its provenance offset is u32. The value 0 means the trailer is absent.
+ *   - The string table contains `stringCount + 1` u32 offsets, with the final offset as a sentinel, followed by the UTF-8 data.
+ *   - State table, 16 bytes per state: edgeStart u32, placeStart u32, edgeCount u32, placeCount u32. Before version
+ *     4 the entry is 12 bytes with u16 counts.
+ *   - Edge table, 8 bytes per edge: stringIdx u32, targetState u32.
+ *   - Place table, 60 bytes per place: wofID u32, placetypeIdx u8, chainLen u8, crossCountryBranches u8,
+ *     placeFlags u8 (bit 0 means `encyclopedic` is present), nameIdx u32, referential f32, lat f32, lon f32.
+ *     Each entry then stores eight u32 parent ids. Version 5 added encyclopedic f32. Earlier entries contain 56 bytes.
+ *     Those entries have no encyclopedic score.
+ *   - Provenance trailer: a u32 length followed by JSON.
  *
- *   STRING TABLE offsets [u32; stringCount + 1] byte offset into data (last = sentinel) data [u8;
- *   stringBytes] concatenated UTF-8
- *
- *   STATE TABLE [stateCount × 12 bytes] edgeStart u32 index into edge table placeStart u32 index into
- *   place table edgeCount u16 placeCount u16
- *
- *   EDGE TABLE [edgeCount × 8 bytes] stringIdx u32 index into string table targetState u32
- *
- *   PLACE TABLE [placeCount × 60 bytes at V5, 56 below] wofID u32 placetypeIdx u8 index into
- *   PLACETYPE_ORDER chainLen u8 0..8 crossCountryBranches u8 (header flags bit0 enables the read)
- *   placeFlags u8 (V5; bit0 = encyclopedic present) nameIdx u32 index into string table referential f32
- *   population-anchored likelihood [0,1] — was the conflated `importance` (V2–V4), was population u32
- *   (V1) lat f32 lon f32 chain [u32; 8] parent chain (unused slots = 0) encyclopedic f32 (V5 only;
- *   read only when placeFlags bit0 is set)
- *
- *   THE V5 BUMP IS THE TWO-SCORE SPLIT (ROAD_TO_V9 §2 R1). Through V4 one float carried whichever score
- *   the source database held, and nothing in the bytes said which — so a reader could not tell a
- *   population proxy from a Wikipedia score. V5 names the ranking score `referential` and gives the
- *   encyclopedic signal its own slot plus a PER-PLACE presence bit, because most places have no
- *   Wikipedia article and a 0 there would be a fact nobody recorded. `fst-freshness.ts` reports every
- *   V4-and-below artifact as format-stale for exactly this reason: its single float is unattributable.
+ *   Presence bits keep a missing value distinct from a stored zero.
  */
 
-import { tryParsingJSON } from "@mailwoman/core/json"
+import { tryParsingJSON, stringifyJSON } from "@mailwoman/core/json"
 
 import {
 	EDGE_ENTRY_SIZE,
@@ -55,16 +44,12 @@ import type { FSTNode } from "#fst/matcher"
 import { FSTMatcher } from "#fst/matcher"
 import type { FSTProvenance, PlaceEntry } from "#fst/types"
 
-export { FST_FORMAT_VERSION } from "#fst/format"
-
-/**
- * File magic as a Buffer, for the Node-side prefix comparison.
- */
 const MAGIC = Buffer.from(FST_MAGIC_BYTES)
 
 /**
- * Longest ancestry chain stored per place. Deeper hierarchies are truncated at the leaf end, since the specific end of
- * the chain is what disambiguates and the country end is recoverable anyway.
+ * The longest parent chain stored per place.
+ *
+ * The writer keeps the first eight valid parent ids and drops the rest.
  */
 const MAX_CHAIN_LEN = 8
 
@@ -74,10 +59,12 @@ for (let i = 0; i < PLACETYPE_ORDER.length; i++) {
 	placetypeToIdx.set(PLACETYPE_ORDER[i]!, i)
 }
 
+/**
+ * Serializes a matcher, with optional provenance, into the current FST binary format.
+ */
 export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): Buffer {
 	const nodes = matcher.toNodes() as FSTNode[]
 
-	// Intern state-edge strings before serializing their table.
 	const stringMap = new Map<string, number>()
 	const strings: string[] = []
 
@@ -106,7 +93,6 @@ export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): B
 	const encodedStrings = strings.map((s) => Buffer.from(s, "utf8"))
 	const stringBytes = encodedStrings.reduce((sum, b) => sum + b.length, 0)
 
-	// Count each serialized record class.
 	let totalEdges = 0
 	let totalPlaces = 0
 
@@ -115,23 +101,20 @@ export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): B
 		totalPlaces += node.places.length
 	}
 
-	// Allocate the complete binary buffer.
 	const stringTableSize = (strings.length + 1) * 4 + stringBytes
 	const stateTableSize = nodes.length * WIDE_STATE_ENTRY_SIZE
 	const edgeTableSize = totalEdges * EDGE_ENTRY_SIZE
 	const placeTableSize = totalPlaces * SPLIT_PLACE_ENTRY_SIZE
-	const provenanceJson = provenance ? Buffer.from(JSON.stringify(provenance), "utf8") : null
+	const provenanceJson = provenance ? Buffer.from(stringifyJSON(provenance), "utf8") : null
 	const provenanceSize = provenanceJson ? 4 + provenanceJson.length : 0
 	const binarySize = HEADER_SIZE + stringTableSize + stateTableSize + edgeTableSize + placeTableSize
 	const totalSize = binarySize + provenanceSize
 	const buf = Buffer.alloc(totalSize)
 	let pos = 0
 
-	// Read the versioned binary header.
-	// flags bit0 (survey #4, 2026-07-27): place rows carry surface-ambiguity data in the former _pad
-	// byte (pp+6 = crossCountryBranches u8, pp+7 reserved). Presence-signaled here so VERSION stays
-	// put: pre-ambiguity artifacts read flags=0 → readers expose `undefined`, never a fake 0.
-	const hasAmbiguity = nodes.some((n) => n.places.some((p) => p.crossCountryBranches !== undefined))
+	// Flag bit 0 marks that place rows include `crossCountryBranches`.
+	// Readers of a file without it report `undefined`.
+	const hasAmbiguity = nodes.some((n) => n.places.some((p) => p.crossCountryBranches != null))
 	MAGIC.copy(buf, pos)
 	pos += 4
 	buf.writeUInt16LE(FST_FORMAT_VERSION, pos)
@@ -151,7 +134,6 @@ export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): B
 	buf.writeUInt32LE(provenanceJson ? binarySize : 0, pos)
 	pos += 4
 
-	// Read the interned string table.
 	let strOffset = 0
 
 	for (const encoded of encodedStrings) {
@@ -160,17 +142,15 @@ export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): B
 		strOffset += encoded.length
 	}
 
+	// The final offset is the sentinel that ends the last string.
 	buf.writeUInt32LE(strOffset, pos)
 	pos += 4
 
-	// sentinel
-
 	for (const encoded of encodedStrings) {
 		encoded.copy(buf, pos)
 		pos += encoded.length
 	}
 
-	// Read state, edge, and place tables.
 	const stateTableStart = pos
 	const edgeTableStart = stateTableStart + stateTableSize
 	const placeTableStart = edgeTableStart + edgeTableSize
@@ -197,17 +177,16 @@ export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): B
 
 		for (const place of node.places) {
 			const pp = placeTableStart + placeIdx * SPLIT_PLACE_ENTRY_SIZE
-			// Filter out WOF sentinel parent IDs (negative values like -1, -4).
+			// WOF uses negative parent ids such as -1 as sentinels.
 			const validChain = place.parentChain.filter((id) => id > 0)
 			const chainLen = Math.min(validChain.length, MAX_CHAIN_LEN)
 			buf.writeUInt32LE(place.wofID, pp)
 			buf.writeUInt8(placetypeToIdx.get(place.placetype) ?? 0, pp + 4)
 			buf.writeUInt8(chainLen, pp + 5)
-			// Former _pad: byte 0 = crossCountryBranches (header flags bit0 enables the read), byte 1 = v5 placeFlags.
 			buf.writeUInt8(hasAmbiguity ? Math.min(place.crossCountryBranches ?? 0, 255) : 0, pp + 6)
-			// An absent encyclopedic score writes flag 0 AND a 0.0 float. The float is unread in that
-			// state, so absence can never surface as a score — the meaning-of-zero rule, in bytes.
-			const hasEncyclopedic = place.encyclopedic !== undefined
+			// An absent encyclopedic score writes flag 0 and a 0.0 float.
+			// Readers skip the float when the flag is 0.
+			const hasEncyclopedic = place.encyclopedic != null
 			buf.writeUInt8(hasEncyclopedic ? PLACE_FLAG_HAS_ENCYCLOPEDIC : 0, pp + 7)
 			buf.writeUInt32LE(intern(place.name), pp + 8)
 			buf.writeFloatLE(place.referential, pp + 12)
@@ -218,7 +197,7 @@ export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): B
 				buf.writeUInt32LE(ci < chainLen ? validChain[ci]! : 0, pp + 24 + ci * 4)
 			}
 
-			buf.writeFloatLE(hasEncyclopedic ? place.encyclopedic! : 0, pp + ENCYCLOPEDIC_OFFSET)
+			buf.writeFloatLE(place.encyclopedic ?? 0, pp + ENCYCLOPEDIC_OFFSET)
 
 			placeIdx++
 		}
@@ -233,8 +212,12 @@ export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): B
 	return buf
 }
 
+/**
+ * Deserializes an FST binary of any supported version into a matcher.
+ *
+ * @throws When the buffer is too small, has the wrong magic, or has an unsupported version.
+ */
 export function deserializeFST(buf: Buffer): FSTMatcher {
-	// Verify the versioned binary header.
 	if (buf.length < HEADER_SIZE) throw new Error("FST buffer too small for header")
 
 	if (!buf.subarray(0, 4).equals(MAGIC)) throw new Error("FST magic mismatch")
@@ -246,7 +229,6 @@ export function deserializeFST(buf: Buffer): FSTMatcher {
 
 	const isV2 = version >= 2
 	const isSplit = version >= VERSION_TWO_SCORE_SPLIT
-	// flags bit0 (survey #4): place rows carry surface-ambiguity data in the former _pad byte.
 	const hasAmbiguity = (buf.readUInt16LE(6) & 1) === 1
 
 	const stateCount = buf.readUInt32LE(8)
@@ -257,7 +239,6 @@ export function deserializeFST(buf: Buffer): FSTMatcher {
 
 	let pos = HEADER_SIZE
 
-	// Verify string-table offsets and contents.
 	const strOffsets = new Uint32Array(stringCount + 1)
 
 	for (let i = 0; i <= stringCount; i++) {
@@ -276,9 +257,7 @@ export function deserializeFST(buf: Buffer): FSTMatcher {
 
 	pos += stringBytes
 
-	// Verify state-table offsets and transitions.
 	const stateEntrySize = version >= VERSION_WIDE_STATE_COUNTERS ? WIDE_STATE_ENTRY_SIZE : NARROW_STATE_ENTRY_SIZE
-	// v5 grew the place entry by the encyclopedic float; v4-and-below files are read at the old stride.
 	const placeEntrySize = version >= VERSION_TWO_SCORE_SPLIT ? SPLIT_PLACE_ENTRY_SIZE : LEGACY_PLACE_ENTRY_SIZE
 	const stateTableStart = pos
 	const edgeTableStart = stateTableStart + stateCount * stateEntrySize
@@ -317,16 +296,14 @@ export function deserializeFST(buf: Buffer): FSTMatcher {
 				parentChain.push(buf.readUInt32LE(pp + 24 + ci * 4))
 			}
 
-			// v1 stored a raw population u32 here; v2–v4 the conflated `importance` float; v5 the
-			// referential score. A v1 file's population is mapped through the SAME curve
-			// `referentialFromPopulation` uses, so its value is genuinely referential — the only
-			// generation of this format for which that can be said without reading the source database.
+			// Version 1 stored a raw population u32 here.
+			// This version maps it through the `referentialFromPopulation` curve.
+			// Later versions store an f32 score.
 			const referential = isV2
 				? buf.readFloatLE(pp + 12)
 				: Math.min(1, Math.log2(1 + buf.readUInt32LE(pp + 12) / 1000) / 14)
 
-			// Per-place presence bit (v5+). A v4-and-below file has no encyclopedic channel at all, so
-			// the field stays undefined rather than reading the reserved byte as a flag.
+			// Files before version 5 have no encyclopedic score, so their reserved byte is not read as a flag.
 			const hasEncyclopedic =
 				isSplit && (buf.readUInt8(pp + 7) & PLACE_FLAG_HAS_ENCYCLOPEDIC) === PLACE_FLAG_HAS_ENCYCLOPEDIC
 
@@ -338,9 +315,8 @@ export function deserializeFST(buf: Buffer): FSTMatcher {
 				lat: buf.readFloatLE(pp + 16),
 				lon: buf.readFloatLE(pp + 20),
 				parentChain,
-				// Header flags bit0 enables the read (survey #4): pre-ambiguity artifacts expose undefined.
-				...(hasAmbiguity ? { crossCountryBranches: buf.readUInt8(pp + 6) } : {}),
-				...(hasEncyclopedic ? { encyclopedic: buf.readFloatLE(pp + ENCYCLOPEDIC_OFFSET) } : {}),
+				crossCountryBranches: hasAmbiguity ? buf.readUInt8(pp + 6) : null,
+				encyclopedic: hasEncyclopedic ? buf.readFloatLE(pp + ENCYCLOPEDIC_OFFSET) : null,
 			}
 		}
 
@@ -350,23 +326,27 @@ export function deserializeFST(buf: Buffer): FSTMatcher {
 	return FSTMatcher.fromNodes(nodes)
 }
 
-export function readFSTProvenance(buf: Buffer): FSTProvenance | undefined {
-	if (buf.length < HEADER_SIZE) return undefined
+/**
+ * Reads the provenance trailer from an FST binary, or returns `undefined`
+ * when the file has none or it cannot be parsed.
+ */
+export function readFSTProvenance(buf: Buffer): FSTProvenance | null {
+	if (buf.length < HEADER_SIZE) return null
 
-	if (!buf.subarray(0, 4).equals(MAGIC)) return undefined
+	if (!buf.subarray(0, 4).equals(MAGIC)) return null
 	const version = buf.readUInt16LE(4)
 
-	if (version < VERSION_WITH_METADATA) return undefined
+	if (version < VERSION_WITH_METADATA) return null
 	const provenanceOffset = buf.readUInt32LE(28)
 
-	if (provenanceOffset === 0 || provenanceOffset >= buf.length) return undefined
+	if (provenanceOffset === 0 || provenanceOffset >= buf.length) return null
 
 	try {
 		const jsonLen = buf.readUInt32LE(provenanceOffset)
 		const jsonStr = buf.toString("utf8", provenanceOffset + 4, provenanceOffset + 4 + jsonLen)
 
-		return tryParsingJSON<FSTProvenance>(jsonStr) ?? undefined
+		return tryParsingJSON<FSTProvenance>(jsonStr) ?? null
 	} catch {
-		return undefined
+		return null
 	}
 }