@mailwoman/resolver-wof-sqlite 7.2.0 → 7.3.0

package/street-centroid.ts

@@ -0,0 +1,124 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   SQLite implementation of core's `StreetCentroidLookup` (#1042): the street-level tier BELOW the
+ *   exact address-point tier and ABOVE admin-centroid resolution. Given a street name (no house
+ *   number) plus a postcode/commune scope, it returns the street's CENTROID + an honest extent-derived
+ *   uncertainty from the derived `street-centroids-<cc>.db` roll-up.
+ *
+ *   Query-side normalization is THE shared normalizer (`street-normalize.ts`), selected per the shard's
+ *   `streetLocale`, so build-side and probe-side keys agree by construction. The commune scope folds
+ *   through `normalizeLocalityForKey` + `stripArrondissement` (BAN names Paris/Lyon/Marseille per
+ *   arrondissement; a query names the base commune).
+ *
+ *   Scope order is most-selective first: `postcode`, then the base commune. Each scope WEIGHTED-
+ *   aggregates (by `point_count`) across the matched rows in SQL, so a commune-scope probe returns the
+ *   street's grand centroid over every postcode/arrondissement it spans — one row, one hit. Matching is
+ *   exact-after-normalization only (no fuzzy street matching in this tier).
+ */
+
+import { DatabaseSync } from "node:sqlite"
+
+import type { StreetCentroidHit, StreetCentroidLookup } from "@mailwoman/resolver"
+
+import { hasTable } from "./sqlite-utils.ts"
+import {
+	normalizeLocalityForKey,
+	normalizeStreetForKeyLocale,
+	type StreetLocale,
+	stripArrondissement,
+} from "./street-normalize.ts"
+
+/** The weighted-centroid + extent + provenance an aggregate probe projects. `lat` is null when nothing matched. */
+interface AggRow {
+	lat: number | null
+	lon: number | null
+	min_lat: number | null
+	max_lat: number | null
+	min_lon: number | null
+	max_lon: number | null
+	source: string | null
+	release: string | null
+}
+
+/** Weighted-centroid aggregate over a WHERE-filtered set. `SUM(coord*n)/SUM(n)` reconstructs the grand centroid. */
+const AGG_SELECT =
+	"SUM(lat * point_count) / SUM(point_count) AS lat, " +
+	"SUM(lon * point_count) / SUM(point_count) AS lon, " +
+	"MIN(min_lat) AS min_lat, MAX(max_lat) AS max_lat, MIN(min_lon) AS min_lon, MAX(max_lon) AS max_lon, " +
+	"MAX(source) AS source, MAX(release) AS release"
+
+/** Half the bbox diagonal, in METERS — an honest coarse radius for a street centroid. */
+function extentRadiusM(minLat: number, maxLat: number, minLon: number, maxLon: number): number {
+	const R = 6_371_000
+	const toRad = (d: number): number => (d * Math.PI) / 180
+	const dLat = toRad(maxLat - minLat)
+	const dLon = toRad(maxLon - minLon)
+	const midLat = toRad((minLat + maxLat) / 2)
+	const a = Math.sin(dLat / 2) ** 2 + Math.cos(midLat) ** 2 * Math.sin(dLon / 2) ** 2
+	const diag = 2 * R * Math.asin(Math.min(1, Math.sqrt(a)))
+
+	return Math.round(diag / 2)
+}
+
+export class StreetCentroidSqliteLookup implements StreetCentroidLookup {
+	readonly #db: DatabaseSync
+	readonly #locale: StreetLocale
+	readonly #byPostcode: ReturnType<DatabaseSync["prepare"]> | undefined
+	readonly #byLocality: ReturnType<DatabaseSync["prepare"]> | undefined
+
+	/**
+	 * @param dbPath Shard path.
+	 * @param opts.streetLocale The street-normalization locale this shard was BUILT with — must match, or every key
+	 *   misses. Defaults to `"fr"` (BAN is the French national register; the tier is FR-only today).
+	 */
+	constructor(dbPath: string, opts: { streetLocale?: StreetLocale } = {}) {
+		this.#db = new DatabaseSync(dbPath, { readOnly: true })
+		this.#locale = opts.streetLocale ?? "fr"
+
+		// Degrade gracefully on an empty/tableless shard (interrupted build, stray 0-byte file): with no
+		// `street_centroid` table this lookup is a no-op miss, not a crash (mirrors the address-point reader).
+		if (hasTable(this.#db, "street_centroid")) {
+			this.#byPostcode = this.#db.prepare(
+				`SELECT ${AGG_SELECT} FROM street_centroid WHERE postcode = ? AND street_norm = ?`
+			)
+			this.#byLocality = this.#db.prepare(
+				`SELECT ${AGG_SELECT} FROM street_centroid WHERE locality_base = ? AND street_norm = ?`
+			)
+		}
+	}
+
+	find(query: { street: string; postcode?: string; locality?: string }): StreetCentroidHit | null {
+		if (!this.#byPostcode || !this.#byLocality) return null
+		const streetNorm = normalizeStreetForKeyLocale(query.street, this.#locale)
+
+		if (!streetNorm) return null
+
+		let row: AggRow | undefined
+
+		if (query.postcode?.trim()) {
+			row = this.#byPostcode.get(query.postcode.trim(), streetNorm) as AggRow | undefined
+		}
+
+		if ((!row || row.lat == null) && query.locality?.trim()) {
+			const base = stripArrondissement(normalizeLocalityForKey(query.locality))
+			row = this.#byLocality.get(base, streetNorm) as AggRow | undefined
+		}
+
+		if (!row || row.lat == null || row.lon == null) return null
+
+		return {
+			lat: row.lat,
+			lon: row.lon,
+			uncertaintyM: extentRadiusM(row.min_lat!, row.max_lat!, row.min_lon!, row.max_lon!),
+			source: row.source ?? "",
+			release: row.release ?? "",
+		}
+	}
+
+	close(): void {
+		this.#db.close()
+	}
+}

package/street-morphology-fst-builder.ts

@@ -0,0 +1,230 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build a street-morphology FST from libpostal's street_types dictionaries. The morphology FST maps
+ *   street-typing affixes (Street/Avenue/rue/Calle/Straße/...) to a single synthetic placetype
+ *   `"street_affix"` — distinct from the admin FST in source data, intent, and binary artifact.
+ *
+ *   The morphology FST closes the inference-time vacuum identified by the v0.6.1 postmortem: street
+ *   tokens have no admin-FST anchor, so synth-street training pushed the model toward over-emitting
+ *   `dependent_locality` on subcomponents. With the morphology FST, the neural decoder gets
+ *   positive evidence for street-typing affixes and the adjacent name tokens, plus negative
+ *   evidence away from `dependent_locality` on the same neighbours.
+ *
+ *   Design rationale + the four-layer street-supplement architecture lives in
+ *   `docs/articles/concepts/street-supplement-architecture.md`.
+ *
+ *   Source: `core/data/libpostal/dictionaries/{locale}/street_types.txt`. Each line is pipe-delimited
+ *   surface forms with the canonical form first: avenue|av|ave|aven|avenu|avn|avnu|avnue
+ *
+ *   Output: an `FSTMatcher` ready to serialize via `serializeFST` to e.g.
+ *   `fst-street-morphology.bin`.
+ */
+
+import { readdirSync, readFileSync, statSync } from "node:fs"
+import { join } from "node:path"
+
+import type { FSTNode } from "./fst-matcher.ts"
+import { FSTMatcher, normalizeTokens } from "./fst-matcher.ts"
+import type { FSTProvenance, PlaceEntry } from "./fst-types.ts"
+
+/**
+ * Reserved synthetic wofID base for street-morphology entries. 32-bit unsigned, well above any realistic WOF
+ * allocation. Reusing the same base across rebuilds keeps IDs stable for any consumer that caches them. See
+ * [[project-schema-storage-decision]] for the reserved range policy.
+ */
+const STREET_AFFIX_WOFID_BASE = 1_900_000_000
+
+const STREET_TYPES_FILENAME = "street_types.txt"
+
+export interface BuildStreetMorphologyFSTOpts {
+	/** Path to the `core/data/libpostal/dictionaries` directory containing per-locale subfolders. */
+	dictionariesDir: string
+	/**
+	 * Optional locale filter — only ingest these locale subfolders. Defaults to all that have a `street_types.txt`.
+	 */
+	locales?: string[]
+	/**
+	 * Minimum length (in characters, post-normalization) of variant surface forms to insert into the trie. Defaults to 3.
+	 *
+	 * Rationale: libpostal's street_types dictionaries contain 1-2 character abbreviations (`a`, `b`, `av`, `bd`, `br`,
+	 * ...) that collide with non-affix tokens at parse time — notably US state abbreviations (`OR`, `CA`, `ND`, `NY`),
+	 * single-letter unit designators, and arbitrary short tokens. Empirically these collisions push the morphology prior
+	 * to mis-tag state abbreviations as `street_suffix`. A minimum length of 3 retains useful forms (`ave`, `blvd`,
+	 * `rue`, `str`) while filtering out the noise.
+	 */
+	minVariantLength?: number
+	/** Optional progress callback. */
+	onProgress?: (phase: string, detail?: string) => void
+}
+
+export interface BuildStreetMorphologyFSTResult {
+	matcher: FSTMatcher
+	provenance: FSTProvenance
+	canonicalCount: number
+	variantCount: number
+	insertCount: number
+	locales: string[]
+}
+
+/**
+ * Parse one `street_types.txt` line into `{ canonical, variants }`. Canonical is the first token (pre-`|`); variants
+ * are all whitespace-stripped non-empty tokens including the canonical.
+ *
+ * Lines with no `|` are treated as a single-form entry where canonical == variant.
+ */
+function parseLine(line: string): { canonical: string; variants: string[] } | null {
+	const trimmed = line.trim()
+
+	if (trimmed.length === 0 || trimmed.startsWith("#")) return null
+	const parts = trimmed
+		.split("|")
+		.map((s) => s.trim())
+		.filter((s) => s.length > 0)
+
+	if (parts.length === 0) return null
+
+	return { canonical: parts[0]!, variants: parts }
+}
+
+export function buildStreetMorphologyFST(opts: BuildStreetMorphologyFSTOpts): BuildStreetMorphologyFSTResult {
+	const progress = opts.onProgress ?? (() => {})
+	const minVariantLength = opts.minVariantLength ?? 3
+
+	// Discover locales — either provided explicitly, or all directories containing street_types.txt.
+	let locales: string[]
+
+	if (opts.locales && opts.locales.length > 0) {
+		locales = opts.locales
+	} else {
+		locales = readdirSync(opts.dictionariesDir).filter((entry) => {
+			const localePath = join(opts.dictionariesDir, entry)
+
+			if (!statSync(localePath).isDirectory()) return false
+
+			try {
+				statSync(join(localePath, STREET_TYPES_FILENAME))
+
+				return true
+			} catch {
+				return false
+			}
+		})
+	}
+	progress("discover", `Found ${locales.length} locales with ${STREET_TYPES_FILENAME}`)
+
+	// Collect canonical → set-of-variants across all locales. Same canonical form may appear in
+	// multiple locales (e.g. "avenue" in en/fr); we union the variant sets.
+	const canonicalToVariants = new Map<string, Set<string>>()
+
+	for (const locale of locales) {
+		const filePath = join(opts.dictionariesDir, locale, STREET_TYPES_FILENAME)
+		const content = readFileSync(filePath, "utf8")
+
+		for (const line of content.split("\n")) {
+			const parsed = parseLine(line)
+
+			if (!parsed) continue
+			const existing = canonicalToVariants.get(parsed.canonical) ?? new Set<string>()
+
+			for (const variant of parsed.variants) {
+				existing.add(variant)
+			}
+			canonicalToVariants.set(parsed.canonical, existing)
+		}
+	}
+	progress("collect", `Collected ${canonicalToVariants.size} canonical affixes`)
+
+	// Assign stable synthetic wofIDs. Sort canonicals for determinism.
+	const sortedCanonicals = [...canonicalToVariants.keys()].sort()
+	const canonicalToWOFID = new Map<string, number>()
+
+	for (let i = 0; i < sortedCanonicals.length; i++) {
+		canonicalToWOFID.set(sortedCanonicals[i]!, STREET_AFFIX_WOFID_BASE + i)
+	}
+
+	// Build the trie. Each variant is inserted as a token sequence pointing to its canonical's
+	// PlaceEntry — so all variants of "avenue" (av/ave/aven/...) lead to the same terminal entry.
+	const nodes: FSTNode[] = [{ edges: new Map(), places: [] }]
+
+	function insertName(tokens: string[], entry: PlaceEntry): void {
+		if (tokens.length === 0) return
+		let stateID = 0
+
+		for (const t of tokens) {
+			const node = nodes[stateID]!
+			let next = node.edges.get(t)
+
+			if (next === undefined) {
+				next = nodes.length
+				nodes.push({ edges: new Map(), places: [] })
+				node.edges.set(t, next)
+			}
+			stateID = next
+		}
+		const existing = nodes[stateID]!.places
+
+		if (!existing.some((p) => p.wofID === entry.wofID && p.placetype === entry.placetype)) {
+			existing.push(entry)
+		}
+	}
+
+	let insertCount = 0
+	let variantCount = 0
+
+	for (const canonical of sortedCanonicals) {
+		const variants = canonicalToVariants.get(canonical)!
+		const wofID = canonicalToWOFID.get(canonical)!
+		const entry: PlaceEntry = {
+			wofID,
+			placetype: "street_affix",
+			name: canonical,
+			parentChain: [],
+			// Fixed importance: street affixes are structurally unambiguous (Avenue is almost never
+			// anything but street-typing). The morphology prior caps bias separately; this value
+			// just feeds the cap formula `importance * cap`.
+			importance: 1.0,
+			lat: 0,
+			lon: 0,
+		}
+
+		for (const variant of variants) {
+			const tokens = normalizeTokens(variant)
+
+			if (tokens.length === 0) continue
+			// Filter out collision-prone short surface forms — see `minVariantLength` docstring.
+			// We measure against the joined token form (no spaces) since FST keys are token sequences.
+			const joined = tokens.join("")
+
+			if (joined.length < minVariantLength) continue
+			insertName(tokens, entry)
+			insertCount++
+			variantCount++
+		}
+	}
+	progress("trie", `Built trie: ${nodes.length} states, ${insertCount} variant insertions`)
+
+	const edgeCount = nodes.reduce((sum, n) => sum + n.edges.size, 0)
+	const matcher = FSTMatcher.fromNodes(nodes)
+	const provenance: FSTProvenance = {
+		builtAt: new Date().toISOString(),
+		countries: locales, // Reuse `countries` slot for locale provenance — semantics differ from admin FST.
+		stateCount: nodes.length,
+		placeCount: sortedCanonicals.length,
+		edgeCount,
+		nameInsertions: insertCount,
+		importanceMatches: 0, // No importance scoring for morphology — fixed at 1.0.
+		sourceDB: opts.dictionariesDir,
+	}
+
+	return {
+		matcher,
+		provenance,
+		canonicalCount: sortedCanonicals.length,
+		variantCount,
+		insertCount,
+		locales,
+	}
+}

package/street-name-lookup.ts

@@ -0,0 +1,101 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   #727 stage-2 phase 4c — the SQLite backend for {@link StreetLocalityEvidence}.
+ *
+ *   Reads a street-name index (the FR instance = BAN `street-centroids-fr.db`, a `street_centroid`
+ *   table of `street_norm × locality_base × postcode` rows) and answers "does this street surface
+ *   exist as a name" for the k-best rerank. Sync-by-interface, `readOnly`, prepared statements,
+ *   graceful-degrade on a tableless shard — the same reader discipline as `AddressPointSqliteLookup`.
+ *
+ *   THE FOLD CONTRACT: the surface is folded with {@link foldStreetSurface} (the shared function),
+ *   and the DB's `street_norm` column MUST have been built with that SAME fold or every hyphenated /
+ *   apostrophe'd street silently misses. The current `street-centroids-fr.db` predates the contract
+ *   fold (it folded without hyphen/apostrophe normalization); it must be REBUILT with
+ *   `foldStreetSurface` + a `street_norm` index before this backend is wired in production. Until
+ *   then this class is correct-by-construction against a fixture built with the contract fold, and
+ *   the production rebuild is a tracked BAN-sdk follow-up.
+ */
+
+import { DatabaseSync } from "node:sqlite"
+
+import { foldStreetSurface, type StreetEvidenceScope, type StreetLocalityEvidence } from "@mailwoman/resolver"
+
+function hasTable(db: DatabaseSync, table: string): boolean {
+	const row = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ? LIMIT 1").get(table)
+
+	return row !== undefined
+}
+
+function hasColumn(db: DatabaseSync, table: string, column: string): boolean {
+	// `table` is a caller-controlled identifier (default `street_centroid`), not user input — safe to interpolate.
+	for (const row of db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>) {
+		if (row.name === column) return true
+	}
+
+	return false
+}
+
+export interface SQLiteStreetNameLookupOpts {
+	/** ISO-2 (upper-case) countries this index answers for. Default `["FR"]` (the BAN street-centroids instance). */
+	countries?: Iterable<string>
+	/** Table name. Default `street_centroid`. */
+	table?: string
+}
+
+/**
+ * A {@link StreetLocalityEvidence} backed by a street-name SQLite index. Positive evidence only: any doubt (missing
+ * table, read miss) returns `false`, so the rerank fails open to the model's ranking.
+ */
+export class SQLiteStreetNameLookup implements StreetLocalityEvidence {
+	readonly countries: ReadonlySet<string>
+	readonly #db: DatabaseSync
+	readonly #byName: ReturnType<DatabaseSync["prepare"]> | undefined
+	readonly #byNameLocality: ReturnType<DatabaseSync["prepare"]> | undefined
+	readonly #byNamePostcode: ReturnType<DatabaseSync["prepare"]> | undefined
+
+	constructor(dbPath: string, opts: SQLiteStreetNameLookupOpts = {}) {
+		this.countries = new Set([...(opts.countries ?? ["FR"])].map((c) => c.toUpperCase()))
+		this.#db = new DatabaseSync(dbPath, { readOnly: true })
+		const table = opts.table ?? "street_centroid"
+
+		// Degrade gracefully on an empty/tableless shard — a no-op miss, never a crash (#568 discipline).
+		if (hasTable(this.#db, table)) {
+			// Prefer the #727 phase-4c `name_key` column (foldStreetSurface, indexed by `idx_sc_name` for a direct seek);
+			// fall back to `street_norm` on a pre-rebuild shard (a skip-scan, but correct). The fold used to build
+			// `name_key` MUST match `foldStreetSurface` here (the fold-parity contract).
+			const keyCol = hasColumn(this.#db, table, "name_key") ? "name_key" : "street_norm"
+			this.#byName = this.#db.prepare(`SELECT 1 FROM ${table} WHERE ${keyCol} = ? LIMIT 1`)
+			this.#byNameLocality = this.#db.prepare(
+				`SELECT 1 FROM ${table} WHERE ${keyCol} = ? AND locality_base = ? LIMIT 1`
+			)
+			this.#byNamePostcode = this.#db.prepare(`SELECT 1 FROM ${table} WHERE ${keyCol} = ? AND postcode = ? LIMIT 1`)
+		}
+	}
+
+	hasStreetName(streetSurface: string, scope?: StreetEvidenceScope): boolean {
+		if (!this.#byName) return false
+		const norm = foldStreetSurface(streetSurface)
+
+		if (!norm) return false
+
+		// Scoped lookups tighten precision when the hypothesis carries a locality/postcode; a scoped MISS falls back to the
+		// unscoped probe (index incompleteness in the scope column is not evidence of absence — positive-evidence rule).
+		if (scope?.locality && this.#byNameLocality) {
+			if (this.#byNameLocality.get(norm, foldStreetSurface(scope.locality)) !== undefined) return true
+		}
+
+		if (scope?.postcode && this.#byNamePostcode) {
+			if (this.#byNamePostcode.get(norm, scope.postcode) !== undefined) return true
+		}
+
+		return this.#byName.get(norm) !== undefined
+	}
+
+	/** Close the underlying handle. */
+	close(): void {
+		this.#db.close()
+	}
+}