@mailwoman/resolver-wof-sqlite 7.2.0 → 7.3.0

package/fst-builder.ts

@@ -0,0 +1,291 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build an FST (finite-state transducer) from a WOF SQLite database. The FST maps normalized token
+ *   sequences to PlaceEntry arrays, pre-computing the valid interpretations for every prefix of
+ *   every place name in the gazetteer.
+ *
+ *   Build pipeline: open WOF DB → query spr + names → normalize names → insert into trie → attach
+ *   PlaceEntry at terminals → return FSTMatcher.
+ */
+
+import { DatabaseSync } from "node:sqlite"
+
+import type { FSTNode } from "./fst-matcher.ts"
+import { FSTMatcher, normalizeTokens } from "./fst-matcher.ts"
+import type { BuildFSTOpts, BuildFSTResult, FSTProvenance, PlaceEntry, PlacetypeID } from "./fst-types.ts"
+
+const DEFAULT_PLACETYPES: PlacetypeID[] = [
+	"country",
+	"region",
+	"county",
+	"locality",
+	"localadmin",
+	"borough",
+	"neighbourhood",
+]
+const DEFAULT_COUNTRIES = ["US"]
+const DEFAULT_LANGUAGES = ["eng", ""]
+
+interface SprRow {
+	id: number
+	name: string
+	placetype: string
+	parent_id: number
+	latitude: number
+	longitude: number
+}
+
+interface NameRow {
+	id: number
+	name: string
+	language: string
+	privateuse: string
+}
+
+interface PopulationRow {
+	id: number
+	population: number
+}
+
+export function buildFSTFromWOF(opts: BuildFSTOpts): {
+	matcher: FSTMatcher
+	provenance: FSTProvenance
+	result: BuildFSTResult
+} {
+	const countries = opts.countries ?? DEFAULT_COUNTRIES
+	const placetypes = opts.placetypes ?? DEFAULT_PLACETYPES
+	const languages = opts.languages ?? DEFAULT_LANGUAGES
+	const progress = opts.onProgress ?? (() => {})
+
+	progress("open", opts.dbPath)
+	const db = new DatabaseSync(opts.dbPath, { open: true })
+
+	// Phase 1: Load all matching SPR rows.
+	progress("spr", `Loading places for countries=[${countries}], placetypes=[${placetypes}]`)
+	const placeholders = (arr: string[]) => arr.map(() => "?").join(",")
+	const sprStmt = db.prepare(
+		`SELECT id, name, placetype, parent_id, latitude, longitude
+		 FROM spr
+		 WHERE is_current = 1
+		   AND country IN (${placeholders(countries)})
+		   AND placetype IN (${placeholders(placetypes)})`
+	)
+	const sprRows = sprStmt.all(...countries, ...placetypes) as unknown as SprRow[]
+	progress("spr", `Loaded ${sprRows.length} places`)
+
+	// Phase 2: Build a lookup for parent chain resolution.
+	const sprByID = new Map<number, SprRow>()
+
+	for (const row of sprRows) {
+		sprByID.set(row.id, row)
+	}
+
+	// Also load parent rows that might be outside our placetype filter (e.g., country for region).
+	const parentStmt = db.prepare("SELECT id, name, placetype, parent_id, latitude, longitude FROM spr WHERE id = ?")
+
+	// Fallback: use ancestors table when parent_id is a sentinel (-1, -4, etc.).
+	let ancestorStmt: ReturnType<typeof db.prepare> | null = null
+
+	try {
+		ancestorStmt = db.prepare(
+			`SELECT DISTINCT ancestor_id FROM ancestors
+			 WHERE id = ? AND ancestor_placetype IN ('country', 'region', 'county')
+			 ORDER BY CASE ancestor_placetype
+			   WHEN 'county' THEN 1
+			   WHEN 'region' THEN 2
+			   WHEN 'country' THEN 3
+			 END`
+		)
+	} catch {
+		progress("ancestors", "No ancestors table — sentinel parent_ids will produce empty chains")
+	}
+
+	function resolveParentChain(id: number): number[] {
+		const row = sprByID.get(id)
+
+		if (!row) return []
+
+		// If parent_id is a sentinel (≤ 0), use ancestors table.
+		if (row.parent_id <= 0 && ancestorStmt) {
+			const ancestors = ancestorStmt.all(id) as unknown as Array<{ ancestor_id: number }>
+
+			return ancestors.map((a) => a.ancestor_id).filter((aid) => aid !== id)
+		}
+
+		// Normal case: walk parent_id chain.
+		const chain: number[] = []
+		let current = row.parent_id
+		const seen = new Set<number>([id])
+
+		while (current > 0 && !seen.has(current)) {
+			seen.add(current)
+			chain.push(current)
+			let parentRow = sprByID.get(current)
+
+			if (!parentRow) {
+				const fetched = parentStmt.get(current) as unknown as SprRow | undefined
+
+				if (!fetched) break
+				parentRow = fetched
+				sprByID.set(current, parentRow)
+			}
+
+			if (parentRow.parent_id > 0 && parentRow.parent_id !== current) {
+				current = parentRow.parent_id
+			} else {
+				break
+			}
+		}
+
+		return chain
+	}
+
+	// Phase 3: Load importance data (Wikipedia-based, falls back to population-scaled).
+	// See docs/articles/concepts/importance-vs-population.md for the two-signal contract.
+	progress("importance", "Loading importance data")
+	const importanceMap = new Map<number, number>()
+
+	try {
+		const impStmt = db.prepare("SELECT id, importance FROM place_importance")
+		const impRows = impStmt.all() as unknown as Array<{ id: number; importance: number }>
+
+		for (const row of impRows) {
+			importanceMap.set(row.id, row.importance)
+		}
+		progress("importance", `Loaded ${importanceMap.size} importance scores`)
+	} catch {
+		progress("importance", "No place_importance table — falling back to population")
+
+		try {
+			const popStmt = db.prepare("SELECT id, population FROM place_population")
+			const popRows = popStmt.all() as unknown as PopulationRow[]
+
+			for (const row of popRows) {
+				const normalized = row.population > 0 ? Math.min(1.0, Math.log2(1 + row.population / 1000) / 14) : 0
+				importanceMap.set(row.id, normalized)
+			}
+		} catch {
+			progress("importance", "No place_population either — using 0 for all")
+		}
+	}
+
+	// Phase 4: Load names for matching places.
+	progress("names", "Loading name variants")
+	const placeIds = sprRows.map((r) => r.id)
+	const namesByPlace = new Map<number, string[]>()
+
+	const allLanguages = languages.includes("*")
+
+	for (let i = 0; i < placeIds.length; i += 500) {
+		const chunk = placeIds.slice(i, i + 500)
+		const idPlaceholders = chunk.map(() => "?").join(",")
+		const nameStmt = allLanguages
+			? db.prepare(`SELECT id, name, language, privateuse FROM names WHERE id IN (${idPlaceholders})`)
+			: db.prepare(
+					`SELECT id, name, language, privateuse FROM names WHERE id IN (${idPlaceholders}) AND language IN (${languages.map(() => "?").join(",")})`
+				)
+		const nameRows = (allLanguages
+			? nameStmt.all(...chunk)
+			: nameStmt.all(...chunk, ...languages)) as unknown as NameRow[]
+
+		for (const row of nameRows) {
+			const existing = namesByPlace.get(row.id) ?? []
+
+			if (!existing.includes(row.name)) {
+				existing.push(row.name)
+			}
+			namesByPlace.set(row.id, existing)
+		}
+	}
+	progress("names", `Loaded names for ${namesByPlace.size} places`)
+
+	// Phase 5: Build the trie.
+	progress("trie", "Building trie")
+	const nodes: FSTNode[] = [{ edges: new Map(), places: [] }]
+
+	function insertName(tokens: string[], entry: PlaceEntry): void {
+		if (tokens.length === 0) return
+		let stateID = 0
+
+		for (const t of tokens) {
+			const node = nodes[stateID]!
+			let next = node.edges.get(t)
+
+			if (next === undefined) {
+				next = nodes.length
+				nodes.push({ edges: new Map(), places: [] })
+				node.edges.set(t, next)
+			}
+			stateID = next
+		}
+		// Deduplicate: don't add the same wofID twice at the same state.
+		const existing = nodes[stateID]!.places
+
+		if (!existing.some((p) => p.wofID === entry.wofID && p.placetype === entry.placetype)) {
+			existing.push(entry)
+		}
+	}
+
+	let insertCount = 0
+
+	for (const row of sprRows) {
+		const parentChain = resolveParentChain(row.id)
+		const entry: PlaceEntry = {
+			wofID: row.id,
+			placetype: row.placetype as PlacetypeID,
+			name: row.name,
+			parentChain,
+			importance: importanceMap.get(row.id) ?? 0,
+			lat: row.latitude,
+			lon: row.longitude,
+		}
+
+		// Insert the primary name from spr.
+		const primaryTokens = normalizeTokens(row.name)
+		insertName(primaryTokens, entry)
+		insertCount++
+
+		// Insert alt names from the names table.
+		const altNames = namesByPlace.get(row.id) ?? []
+
+		for (const altName of altNames) {
+			if (altName === row.name) continue
+			const altTokens = normalizeTokens(altName)
+
+			if (altTokens.length > 0 && altTokens.join(" ") !== primaryTokens.join(" ")) {
+				insertName(altTokens, entry)
+				insertCount++
+			}
+		}
+	}
+
+	db.close()
+	progress("done", `Built trie: ${nodes.length} states, ${insertCount} name insertions`)
+
+	const edgeCount = nodes.reduce((sum, n) => sum + n.edges.size, 0)
+	const matcher = FSTMatcher.fromNodes(nodes)
+	const provenance: FSTProvenance = {
+		builtAt: new Date().toISOString(),
+		countries,
+		stateCount: nodes.length,
+		placeCount: sprRows.length,
+		edgeCount,
+		nameInsertions: insertCount,
+		importanceMatches: importanceMap.size,
+		sourceDB: opts.dbPath,
+	}
+
+	return {
+		matcher,
+		provenance,
+		result: {
+			stateCount: nodes.length,
+			placeCount: sprRows.length,
+			edgeCount,
+			tokenCount: insertCount,
+		},
+	}
+}

package/fst-deserialize-web.ts

@@ -0,0 +1,164 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Browser-compatible FST deserializer. Uses DataView + TextDecoder instead of Node's Buffer so the
+ *   same binary format can be loaded in the browser via fetch(url).then(r => r.arrayBuffer()).
+ *
+ *   This is a read-only counterpart to fst-serialize.ts — serialization stays Node-only (it's a
+ *   build-time operation).
+ */
+
+import type { FSTNode } from "./fst-matcher.ts"
+import { FSTMatcher } from "./fst-matcher.ts"
+import type { FSTProvenance, PlaceEntry, PlacetypeID } from "./fst-types.ts"
+
+const HEADER_SIZE = 32
+const EDGE_ENTRY_SIZE = 8
+const PLACE_ENTRY_SIZE = 56
+const MAGIC_BYTES = [0x46, 0x53, 0x54, 0x00] // "FST\0"
+// Must track the serializer's VERSION (fst-serialize.ts, currently 4). The v3 provenance + v4
+// 16-byte-state/u32-count layout logic below already matches the Node deserializer; only this gate
+// was left stale at 2, so the browser FST loader rejected every real (v4) artifact.
+const MAX_VERSION = 4
+
+const PLACETYPE_ORDER: readonly PlacetypeID[] = [
+	"country",
+	"region",
+	"county",
+	"locality",
+	"localadmin",
+	"borough",
+	"neighbourhood",
+	"postalcode",
+	"campus",
+	"dependency",
+	"street_affix",
+]
+
+export function deserializeFSTWeb(input: ArrayBuffer | Uint8Array): FSTMatcher {
+	const bytes = input instanceof ArrayBuffer ? new Uint8Array(input) : input
+	const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
+	const decoder = new TextDecoder("utf-8")
+
+	if (bytes.byteLength < HEADER_SIZE) throw new Error("FST buffer too small for header")
+
+	if (
+		bytes[0] !== MAGIC_BYTES[0] ||
+		bytes[1] !== MAGIC_BYTES[1] ||
+		bytes[2] !== MAGIC_BYTES[2] ||
+		bytes[3] !== MAGIC_BYTES[3]
+	) {
+		throw new Error("FST magic mismatch")
+	}
+
+	const version = view.getUint16(4, true)
+
+	if (version < 1 || version > MAX_VERSION) {
+		throw new Error(`FST version ${version} unsupported (expected 1..${MAX_VERSION})`)
+	}
+	const isV2 = version >= 2
+
+	const stateCount = view.getUint32(8, true)
+	const edgeCount = view.getUint32(12, true)
+	const _placeCount = view.getUint32(16, true)
+	const stringCount = view.getUint32(20, true)
+	const stringBytes = view.getUint32(24, true)
+
+	let pos = HEADER_SIZE
+
+	// --- String table ---
+	const strOffsets = new Uint32Array(stringCount + 1)
+
+	for (let i = 0; i <= stringCount; i++) {
+		strOffsets[i] = view.getUint32(pos, true)
+		pos += 4
+	}
+	const strDataStart = pos
+	const strings: string[] = new Array(stringCount)
+
+	for (let i = 0; i < stringCount; i++) {
+		const start = strDataStart + strOffsets[i]!
+		const end = strDataStart + strOffsets[i + 1]!
+		strings[i] = decoder.decode(bytes.subarray(start, end))
+	}
+	pos += stringBytes
+
+	// --- State table ---
+	const stateEntrySize = version >= 4 ? 16 : 12
+	const stateTableStart = pos
+	const edgeTableStart = stateTableStart + stateCount * stateEntrySize
+	const placeTableStart = edgeTableStart + edgeCount * EDGE_ENTRY_SIZE
+
+	const nodes: FSTNode[] = new Array(stateCount)
+
+	for (let si = 0; si < stateCount; si++) {
+		const sp = stateTableStart + si * stateEntrySize
+		const edgeStart = view.getUint32(sp, true)
+		const placeStart = view.getUint32(sp + 4, true)
+		const edgeCountForState = version >= 4 ? view.getUint32(sp + 8, true) : view.getUint16(sp + 8, true)
+		const placeCountForState = version >= 4 ? view.getUint32(sp + 12, true) : view.getUint16(sp + 10, true)
+
+		const edges = new Map<string, number>()
+
+		for (let ei = 0; ei < edgeCountForState; ei++) {
+			const ep = edgeTableStart + (edgeStart + ei) * EDGE_ENTRY_SIZE
+			const stringIdx = view.getUint32(ep, true)
+			const target = view.getUint32(ep + 4, true)
+			edges.set(strings[stringIdx]!, target)
+		}
+
+		const places: PlaceEntry[] = new Array(placeCountForState)
+
+		for (let pi = 0; pi < placeCountForState; pi++) {
+			const pp = placeTableStart + (placeStart + pi) * PLACE_ENTRY_SIZE
+			const chainLen = view.getUint8(pp + 5)
+			const parentChain: number[] = []
+
+			for (let ci = 0; ci < chainLen; ci++) {
+				parentChain.push(view.getUint32(pp + 24 + ci * 4, true))
+			}
+			const rawImportance = isV2
+				? view.getFloat32(pp + 12, true)
+				: Math.min(1.0, Math.log2(1 + view.getUint32(pp + 12, true) / 1000) / 14)
+
+			places[pi] = {
+				wofID: view.getUint32(pp, true),
+				placetype: PLACETYPE_ORDER[view.getUint8(pp + 4)] ?? "locality",
+				name: strings[view.getUint32(pp + 8, true)]!,
+				importance: rawImportance,
+				lat: view.getFloat32(pp + 16, true),
+				lon: view.getFloat32(pp + 20, true),
+				parentChain,
+			}
+		}
+
+		nodes[si] = { edges, places }
+	}
+
+	return FSTMatcher.fromNodes(nodes)
+}
+
+export function readFSTProvenanceWeb(input: ArrayBuffer | Uint8Array): FSTProvenance | undefined {
+	const bytes = input instanceof ArrayBuffer ? new Uint8Array(input) : input
+	const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
+	const decoder = new TextDecoder("utf-8")
+
+	if (bytes.byteLength < HEADER_SIZE) return undefined
+	const version = view.getUint16(4, true)
+
+	if (version < 3) return undefined
+	const provenanceOffset = view.getUint32(28, true)
+
+	if (provenanceOffset === 0 || provenanceOffset >= bytes.byteLength) return undefined
+
+	try {
+		const jsonLen = view.getUint32(provenanceOffset, true)
+		const jsonStr = decoder.decode(bytes.subarray(provenanceOffset + 4, provenanceOffset + 4 + jsonLen))
+
+		return JSON.parse(jsonStr) as FSTProvenance
+	} catch {
+		return undefined
+	}
+}

package/fst-matcher.ts

@@ -0,0 +1,150 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   In-memory FST matcher. Built by `fst-builder.ts`, queried at runtime for emission priors and CLI
+ *   introspection. The structure is a deterministic trie over normalized tokens with PlaceEntry
+ *   arrays at accepting states.
+ */
+
+import type { FSTContinuation, FSTMatchResult, FSTQueryResult, PlaceEntry } from "./fst-types.ts"
+
+interface FSTNode {
+	edges: Map<string, number>
+	places: PlaceEntry[]
+}
+
+export class FSTMatcher {
+	private nodes: FSTNode[]
+
+	constructor(nodes: FSTNode[]) {
+		this.nodes = nodes
+	}
+
+	get stateCount(): number {
+		return this.nodes.length
+	}
+
+	get placeCount(): number {
+		let count = 0
+
+		for (const n of this.nodes) {
+			count += n.places.length
+		}
+
+		return count
+	}
+
+	walk(tokens: string[]): FSTMatchResult | null {
+		let stateID = 0
+
+		for (let i = 0; i < tokens.length; i++) {
+			const node = this.nodes[stateID]
+
+			if (!node) return null
+			const next = node.edges.get(tokens[i]!)
+
+			if (next === undefined) return null
+			stateID = next
+		}
+		const node = this.nodes[stateID]!
+
+		return { stateID, accepted: node.places.length > 0, depth: tokens.length }
+	}
+
+	walkFrom(prev: FSTMatchResult, token: string): FSTMatchResult | null {
+		const node = this.nodes[prev.stateID]
+
+		if (!node) return null
+		const next = node.edges.get(token)
+
+		if (next === undefined) return null
+		const target = this.nodes[next]!
+
+		return { stateID: next, accepted: target.places.length > 0, depth: prev.depth + 1 }
+	}
+
+	accepting(stateID: number): PlaceEntry[] {
+		return this.nodes[stateID]?.places ?? []
+	}
+
+	continuations(stateID: number): FSTContinuation[] {
+		const node = this.nodes[stateID]
+
+		if (!node) return []
+		const result: FSTContinuation[] = []
+
+		for (const [token, targetID] of node.edges) {
+			const target = this.nodes[targetID]!
+			result.push({
+				token,
+				targetState: targetID,
+				acceptingCount: target.places.length,
+			})
+		}
+
+		return result
+	}
+
+	query(text: string): FSTQueryResult {
+		const tokens = normalizeTokens(text)
+		const match = this.walk(tokens)
+
+		if (!match) {
+			// Walk as far as possible to find where we fall off
+			let stateID = 0
+			let depth = 0
+
+			for (const t of tokens) {
+				const node = this.nodes[stateID]
+
+				if (!node) break
+				const next = node.edges.get(t)
+
+				if (next === undefined) break
+				stateID = next
+				depth++
+			}
+
+			return {
+				path: tokens.slice(0, depth),
+				stateID,
+				accepting: this.accepting(stateID),
+				continuations: this.continuations(stateID),
+			}
+		}
+
+		return {
+			path: tokens,
+			stateID: match.stateID,
+			accepting: this.accepting(match.stateID),
+			continuations: this.continuations(match.stateID),
+		}
+	}
+
+	get nodeCount(): number {
+		return this.nodes.length
+	}
+
+	/** Expose the internal node array for serialization. */
+	toNodes(): readonly FSTNode[] {
+		return this.nodes
+	}
+
+	static fromNodes(nodes: FSTNode[]): FSTMatcher {
+		return new FSTMatcher(nodes)
+	}
+}
+
+/** Normalize text into FST tokens: lowercase, NFKC, strip punctuation, split on whitespace. */
+export function normalizeTokens(text: string): string[] {
+	return text
+		.normalize("NFKC")
+		.toLowerCase()
+		.replace(/[\p{P}\p{S}]/gu, "")
+		.split(/\s+/)
+		.filter((t) => t.length > 0)
+}
+
+export type { FSTNode }