@mailwoman/resolver-wof-sqlite 7.2.0 → 7.2.1

package/street-morphology-fst-builder.ts

@@ -0,0 +1,230 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build a street-morphology FST from libpostal's street_types dictionaries. The morphology FST maps
+ *   street-typing affixes (Street/Avenue/rue/Calle/Straße/...) to a single synthetic placetype
+ *   `"street_affix"` — distinct from the admin FST in source data, intent, and binary artifact.
+ *
+ *   The morphology FST closes the inference-time vacuum identified by the v0.6.1 postmortem: street
+ *   tokens have no admin-FST anchor, so synth-street training pushed the model toward over-emitting
+ *   `dependent_locality` on subcomponents. With the morphology FST, the neural decoder gets
+ *   positive evidence for street-typing affixes and the adjacent name tokens, plus negative
+ *   evidence away from `dependent_locality` on the same neighbours.
+ *
+ *   Design rationale + the four-layer street-supplement architecture lives in
+ *   `docs/articles/concepts/street-supplement-architecture.md`.
+ *
+ *   Source: `core/data/libpostal/dictionaries/{locale}/street_types.txt`. Each line is pipe-delimited
+ *   surface forms with the canonical form first: avenue|av|ave|aven|avenu|avn|avnu|avnue
+ *
+ *   Output: an `FSTMatcher` ready to serialize via `serializeFST` to e.g.
+ *   `fst-street-morphology.bin`.
+ */
+
+import { readdirSync, readFileSync, statSync } from "node:fs"
+import { join } from "node:path"
+
+import type { FSTNode } from "./fst-matcher.ts"
+import { FSTMatcher, normalizeTokens } from "./fst-matcher.ts"
+import type { FSTProvenance, PlaceEntry } from "./fst-types.ts"
+
+/**
+ * Reserved synthetic wofID base for street-morphology entries. 32-bit unsigned, well above any realistic WOF
+ * allocation. Reusing the same base across rebuilds keeps IDs stable for any consumer that caches them. See
+ * [[project-schema-storage-decision]] for the reserved range policy.
+ */
+const STREET_AFFIX_WOFID_BASE = 1_900_000_000
+
+const STREET_TYPES_FILENAME = "street_types.txt"
+
+export interface BuildStreetMorphologyFSTOpts {
+	/** Path to the `core/data/libpostal/dictionaries` directory containing per-locale subfolders. */
+	dictionariesDir: string
+	/**
+	 * Optional locale filter — only ingest these locale subfolders. Defaults to all that have a `street_types.txt`.
+	 */
+	locales?: string[]
+	/**
+	 * Minimum length (in characters, post-normalization) of variant surface forms to insert into the trie. Defaults to 3.
+	 *
+	 * Rationale: libpostal's street_types dictionaries contain 1-2 character abbreviations (`a`, `b`, `av`, `bd`, `br`,
+	 * ...) that collide with non-affix tokens at parse time — notably US state abbreviations (`OR`, `CA`, `ND`, `NY`),
+	 * single-letter unit designators, and arbitrary short tokens. Empirically these collisions push the morphology prior
+	 * to mis-tag state abbreviations as `street_suffix`. A minimum length of 3 retains useful forms (`ave`, `blvd`,
+	 * `rue`, `str`) while filtering out the noise.
+	 */
+	minVariantLength?: number
+	/** Optional progress callback. */
+	onProgress?: (phase: string, detail?: string) => void
+}
+
+export interface BuildStreetMorphologyFSTResult {
+	matcher: FSTMatcher
+	provenance: FSTProvenance
+	canonicalCount: number
+	variantCount: number
+	insertCount: number
+	locales: string[]
+}
+
+/**
+ * Parse one `street_types.txt` line into `{ canonical, variants }`. Canonical is the first token (pre-`|`); variants
+ * are all whitespace-stripped non-empty tokens including the canonical.
+ *
+ * Lines with no `|` are treated as a single-form entry where canonical == variant.
+ */
+function parseLine(line: string): { canonical: string; variants: string[] } | null {
+	const trimmed = line.trim()
+
+	if (trimmed.length === 0 || trimmed.startsWith("#")) return null
+	const parts = trimmed
+		.split("|")
+		.map((s) => s.trim())
+		.filter((s) => s.length > 0)
+
+	if (parts.length === 0) return null
+
+	return { canonical: parts[0]!, variants: parts }
+}
+
+export function buildStreetMorphologyFST(opts: BuildStreetMorphologyFSTOpts): BuildStreetMorphologyFSTResult {
+	const progress = opts.onProgress ?? (() => {})
+	const minVariantLength = opts.minVariantLength ?? 3
+
+	// Discover locales — either provided explicitly, or all directories containing street_types.txt.
+	let locales: string[]
+
+	if (opts.locales && opts.locales.length > 0) {
+		locales = opts.locales
+	} else {
+		locales = readdirSync(opts.dictionariesDir).filter((entry) => {
+			const localePath = join(opts.dictionariesDir, entry)
+
+			if (!statSync(localePath).isDirectory()) return false
+
+			try {
+				statSync(join(localePath, STREET_TYPES_FILENAME))
+
+				return true
+			} catch {
+				return false
+			}
+		})
+	}
+	progress("discover", `Found ${locales.length} locales with ${STREET_TYPES_FILENAME}`)
+
+	// Collect canonical → set-of-variants across all locales. Same canonical form may appear in
+	// multiple locales (e.g. "avenue" in en/fr); we union the variant sets.
+	const canonicalToVariants = new Map<string, Set<string>>()
+
+	for (const locale of locales) {
+		const filePath = join(opts.dictionariesDir, locale, STREET_TYPES_FILENAME)
+		const content = readFileSync(filePath, "utf8")
+
+		for (const line of content.split("\n")) {
+			const parsed = parseLine(line)
+
+			if (!parsed) continue
+			const existing = canonicalToVariants.get(parsed.canonical) ?? new Set<string>()
+
+			for (const variant of parsed.variants) {
+				existing.add(variant)
+			}
+			canonicalToVariants.set(parsed.canonical, existing)
+		}
+	}
+	progress("collect", `Collected ${canonicalToVariants.size} canonical affixes`)
+
+	// Assign stable synthetic wofIDs. Sort canonicals for determinism.
+	const sortedCanonicals = [...canonicalToVariants.keys()].sort()
+	const canonicalToWOFID = new Map<string, number>()
+
+	for (let i = 0; i < sortedCanonicals.length; i++) {
+		canonicalToWOFID.set(sortedCanonicals[i]!, STREET_AFFIX_WOFID_BASE + i)
+	}
+
+	// Build the trie. Each variant is inserted as a token sequence pointing to its canonical's
+	// PlaceEntry — so all variants of "avenue" (av/ave/aven/...) lead to the same terminal entry.
+	const nodes: FSTNode[] = [{ edges: new Map(), places: [] }]
+
+	function insertName(tokens: string[], entry: PlaceEntry): void {
+		if (tokens.length === 0) return
+		let stateID = 0
+
+		for (const t of tokens) {
+			const node = nodes[stateID]!
+			let next = node.edges.get(t)
+
+			if (next === undefined) {
+				next = nodes.length
+				nodes.push({ edges: new Map(), places: [] })
+				node.edges.set(t, next)
+			}
+			stateID = next
+		}
+		const existing = nodes[stateID]!.places
+
+		if (!existing.some((p) => p.wofID === entry.wofID && p.placetype === entry.placetype)) {
+			existing.push(entry)
+		}
+	}
+
+	let insertCount = 0
+	let variantCount = 0
+
+	for (const canonical of sortedCanonicals) {
+		const variants = canonicalToVariants.get(canonical)!
+		const wofID = canonicalToWOFID.get(canonical)!
+		const entry: PlaceEntry = {
+			wofID,
+			placetype: "street_affix",
+			name: canonical,
+			parentChain: [],
+			// Fixed importance: street affixes are structurally unambiguous (Avenue is almost never
+			// anything but street-typing). The morphology prior caps bias separately; this value
+			// just feeds the cap formula `importance * cap`.
+			importance: 1.0,
+			lat: 0,
+			lon: 0,
+		}
+
+		for (const variant of variants) {
+			const tokens = normalizeTokens(variant)
+
+			if (tokens.length === 0) continue
+			// Filter out collision-prone short surface forms — see `minVariantLength` docstring.
+			// We measure against the joined token form (no spaces) since FST keys are token sequences.
+			const joined = tokens.join("")
+
+			if (joined.length < minVariantLength) continue
+			insertName(tokens, entry)
+			insertCount++
+			variantCount++
+		}
+	}
+	progress("trie", `Built trie: ${nodes.length} states, ${insertCount} variant insertions`)
+
+	const edgeCount = nodes.reduce((sum, n) => sum + n.edges.size, 0)
+	const matcher = FSTMatcher.fromNodes(nodes)
+	const provenance: FSTProvenance = {
+		builtAt: new Date().toISOString(),
+		countries: locales, // Reuse `countries` slot for locale provenance — semantics differ from admin FST.
+		stateCount: nodes.length,
+		placeCount: sortedCanonicals.length,
+		edgeCount,
+		nameInsertions: insertCount,
+		importanceMatches: 0, // No importance scoring for morphology — fixed at 1.0.
+		sourceDB: opts.dictionariesDir,
+	}
+
+	return {
+		matcher,
+		provenance,
+		canonicalCount: sortedCanonicals.length,
+		variantCount,
+		insertCount,
+		locales,
+	}
+}

package/street-name-lookup.ts

@@ -0,0 +1,101 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   #727 stage-2 phase 4c — the SQLite backend for {@link StreetLocalityEvidence}.
+ *
+ *   Reads a street-name index (the FR instance = BAN `street-centroids-fr.db`, a `street_centroid`
+ *   table of `street_norm × locality_base × postcode` rows) and answers "does this street surface
+ *   exist as a name" for the k-best rerank. Sync-by-interface, `readOnly`, prepared statements,
+ *   graceful-degrade on a tableless shard — the same reader discipline as `AddressPointSqliteLookup`.
+ *
+ *   THE FOLD CONTRACT: the surface is folded with {@link foldStreetSurface} (the shared function),
+ *   and the DB's `street_norm` column MUST have been built with that SAME fold or every hyphenated /
+ *   apostrophe'd street silently misses. The current `street-centroids-fr.db` predates the contract
+ *   fold (it folded without hyphen/apostrophe normalization); it must be REBUILT with
+ *   `foldStreetSurface` + a `street_norm` index before this backend is wired in production. Until
+ *   then this class is correct-by-construction against a fixture built with the contract fold, and
+ *   the production rebuild is a tracked BAN-sdk follow-up.
+ */
+
+import { DatabaseSync } from "node:sqlite"
+
+import { foldStreetSurface, type StreetEvidenceScope, type StreetLocalityEvidence } from "@mailwoman/resolver"
+
+function hasTable(db: DatabaseSync, table: string): boolean {
+	const row = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ? LIMIT 1").get(table)
+
+	return row !== undefined
+}
+
+function hasColumn(db: DatabaseSync, table: string, column: string): boolean {
+	// `table` is a caller-controlled identifier (default `street_centroid`), not user input — safe to interpolate.
+	for (const row of db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>) {
+		if (row.name === column) return true
+	}
+
+	return false
+}
+
+export interface SQLiteStreetNameLookupOpts {
+	/** ISO-2 (upper-case) countries this index answers for. Default `["FR"]` (the BAN street-centroids instance). */
+	countries?: Iterable<string>
+	/** Table name. Default `street_centroid`. */
+	table?: string
+}
+
+/**
+ * A {@link StreetLocalityEvidence} backed by a street-name SQLite index. Positive evidence only: any doubt (missing
+ * table, read miss) returns `false`, so the rerank fails open to the model's ranking.
+ */
+export class SQLiteStreetNameLookup implements StreetLocalityEvidence {
+	readonly countries: ReadonlySet<string>
+	readonly #db: DatabaseSync
+	readonly #byName: ReturnType<DatabaseSync["prepare"]> | undefined
+	readonly #byNameLocality: ReturnType<DatabaseSync["prepare"]> | undefined
+	readonly #byNamePostcode: ReturnType<DatabaseSync["prepare"]> | undefined
+
+	constructor(dbPath: string, opts: SQLiteStreetNameLookupOpts = {}) {
+		this.countries = new Set([...(opts.countries ?? ["FR"])].map((c) => c.toUpperCase()))
+		this.#db = new DatabaseSync(dbPath, { readOnly: true })
+		const table = opts.table ?? "street_centroid"
+
+		// Degrade gracefully on an empty/tableless shard — a no-op miss, never a crash (#568 discipline).
+		if (hasTable(this.#db, table)) {
+			// Prefer the #727 phase-4c `name_key` column (foldStreetSurface, indexed by `idx_sc_name` for a direct seek);
+			// fall back to `street_norm` on a pre-rebuild shard (a skip-scan, but correct). The fold used to build
+			// `name_key` MUST match `foldStreetSurface` here (the fold-parity contract).
+			const keyCol = hasColumn(this.#db, table, "name_key") ? "name_key" : "street_norm"
+			this.#byName = this.#db.prepare(`SELECT 1 FROM ${table} WHERE ${keyCol} = ? LIMIT 1`)
+			this.#byNameLocality = this.#db.prepare(
+				`SELECT 1 FROM ${table} WHERE ${keyCol} = ? AND locality_base = ? LIMIT 1`
+			)
+			this.#byNamePostcode = this.#db.prepare(`SELECT 1 FROM ${table} WHERE ${keyCol} = ? AND postcode = ? LIMIT 1`)
+		}
+	}
+
+	hasStreetName(streetSurface: string, scope?: StreetEvidenceScope): boolean {
+		if (!this.#byName) return false
+		const norm = foldStreetSurface(streetSurface)
+
+		if (!norm) return false
+
+		// Scoped lookups tighten precision when the hypothesis carries a locality/postcode; a scoped MISS falls back to the
+		// unscoped probe (index incompleteness in the scope column is not evidence of absence — positive-evidence rule).
+		if (scope?.locality && this.#byNameLocality) {
+			if (this.#byNameLocality.get(norm, foldStreetSurface(scope.locality)) !== undefined) return true
+		}
+
+		if (scope?.postcode && this.#byNamePostcode) {
+			if (this.#byNamePostcode.get(norm, scope.postcode) !== undefined) return true
+		}
+
+		return this.#byName.get(norm) !== undefined
+	}
+
+	/** Close the underlying handle. */
+	close(): void {
+		this.#db.close()
+	}
+}

package/street-normalize.ts

@@ -0,0 +1,302 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   THE street normalizer for the address-point tier (#476). One function, used by BOTH the shard
+ *   builder (`scripts/build-address-point-shard.ts`) and the lookup tier (`address-point.ts`) —
+ *   never two implementations (the PLACETYPE_ORDER lesson: parallel copies silently corrupt).
+ *
+ *   Normalization contract (deliberately aggressive — both sides apply the same function, so
+ *   collisions only need to be _consistent_, not linguistically perfect):
+ *
+ *   1. Lowercase, NFKD-fold diacritics, collapse whitespace, strip punctuation (periods, commas,
+ *        apostrophes).
+ *   2. Expand USPS directional abbreviations at the FIRST and LAST token position (`n` → `north`, `se` →
+ *        `southeast`) — Overture sources abbreviate inconsistently.
+ *   3. Canonicalize a trailing USPS street-type token via the codex suffix table to its canonical full
+ *        form (`st`/`str`/`street` → `street`).
+ *
+ *   Numbered streets are left as digits (`5th` stays `5th`); a SPELLED ordinal before a street suffix
+ *   folds to its digit form (`tenth street` → `10th street`, #723) so the grid-city ordinal
+ *   cross-streets the source data spells with digits become reachable.
+ */
+
+import { AbbreviationToDirectional, US_STREET_SUFFIX_LOOKUP } from "@mailwoman/codex/us"
+
+/**
+ * Spelled ordinal street names → their digit-ordinal form ("tenth" → "10th"), applied ONLY when a street-type suffix
+ * follows (#723 admin-tail) — so the ordinal cross-streets common in grid cities ("Tenth Street", "Fifth Avenue") match
+ * the shards' digit keys, WITHOUT rewriting ordinal-WORD names where the next token is not a suffix ("First National
+ * Bank Rd" stays "first national …"). Digit-source shards are unaffected (a digit token isn't in this map), so the
+ * existing keys need no rebuild; a future rebuild folds any spelled-source key the same way (the one-function
+ * discipline).
+ */
+const SPELLED_ORDINAL_TO_DIGIT = new Map<string, string>([
+	["first", "1st"],
+	["second", "2nd"],
+	["third", "3rd"],
+	["fourth", "4th"],
+	["fifth", "5th"],
+	["sixth", "6th"],
+	["seventh", "7th"],
+	["eighth", "8th"],
+	["ninth", "9th"],
+	["tenth", "10th"],
+	["eleventh", "11th"],
+	["twelfth", "12th"],
+	["thirteenth", "13th"],
+	["fourteenth", "14th"],
+	["fifteenth", "15th"],
+	["sixteenth", "16th"],
+	["seventeenth", "17th"],
+	["eighteenth", "18th"],
+	["nineteenth", "19th"],
+	["twentieth", "20th"],
+	["thirtieth", "30th"],
+	["fortieth", "40th"],
+	["fiftieth", "50th"],
+	["sixtieth", "60th"],
+	["seventieth", "70th"],
+	["eightieth", "80th"],
+	["ninetieth", "90th"],
+	["hundredth", "100th"],
+])
+
+/** Lowercase + diacritic-fold + punctuation strip + whitespace collapse. */
+function fold(input: string): string {
+	return input
+		.normalize("NFKD")
+		.replace(/[̀-ͯ]/g, "")
+		.toLowerCase()
+		.replace(/[.,'’]/g, "")
+		.replace(/\s+/g, " ")
+		.trim()
+}
+
+/**
+ * Normalize a street name for address-point keying. Same function at build time and lookup time — see module docstring
+ * for the contract.
+ */
+export function normalizeStreetForKey(street: string): string {
+	const tokens = fold(street).split(" ")
+
+	if (tokens.length === 0) return ""
+
+	// Spelled-ordinal street names → digit form when a street suffix follows ("Tenth Street" →
+	// "10th street", #723). Gated on the next token being a suffix so ordinal-WORD names are untouched.
+	for (let i = 0; i < tokens.length - 1; i++) {
+		const digit = SPELLED_ORDINAL_TO_DIGIT.get(tokens[i]!)
+
+		if (digit && US_STREET_SUFFIX_LOOKUP.has(tokens[i + 1]!)) {
+			tokens[i] = digit
+		}
+	}
+
+	// Directional expansion at the edges only ("N Main St" / "Main St N" — never interior
+	// tokens, where "W" may be an initial in a person-named street). The codex expands
+	// compounds to two words ("SE" → "SOUTH EAST"); we key on the spaceless form
+	// ("southeast"), and also merge an already-written two-token pair ("South East …").
+	const edgeDirectional = (raw: string) =>
+		AbbreviationToDirectional.get(raw.toUpperCase())?.toLowerCase().replace(" ", "")
+	const mergePair = (a?: string, b?: string) =>
+		a && b && /^(north|south)$/.test(a) && /^(east|west)$/.test(b) ? a + b : undefined
+
+	const leadPair = mergePair(tokens[0], tokens[1])
+
+	if (leadPair && tokens.length > 2) {
+		tokens.splice(0, 2, leadPair)
+	}
+	const first = edgeDirectional(tokens[0]!)
+
+	if (first && tokens.length > 1) {
+		tokens[0] = first
+	}
+
+	const tailPair = mergePair(tokens[tokens.length - 2], tokens[tokens.length - 1])
+
+	if (tailPair && tokens.length > 3) {
+		tokens.splice(tokens.length - 2, 2, tailPair)
+	}
+
+	if (tokens.length > 2) {
+		const last = edgeDirectional(tokens[tokens.length - 1]!)
+
+		if (last) {
+			tokens[tokens.length - 1] = last
+		}
+	}
+
+	// Street-type canonicalization via the codex table (lowercase keys, UPPER canonical
+	// values). The suffix is usually the last token, but sits second-to-last when a trailing
+	// directional follows ("Main St N") — check both positions, canonicalize the first hit.
+	for (const at of [tokens.length - 1, tokens.length - 2]) {
+		if (at < 1) continue // never canonicalize the only/first token ("Street Road" exists)
+		const canonical = US_STREET_SUFFIX_LOOKUP.get(tokens[at]!)
+
+		if (canonical) {
+			tokens[at] = canonical.toLowerCase()
+			break
+		}
+	}
+
+	return tokens.join(" ")
+}
+
+/**
+ * Street-name locale for the address-point key. The US path is the full USPS pipeline ({@link normalizeStreetForKey});
+ * the international paths fold + apply a SMALL, consistent per-locale type-token canonicalization. Same discipline as
+ * the US normalizer: build side and probe side call the identical function, so the key only needs to be CONSISTENT, not
+ * linguistically perfect — a folded "rue du chevaleret" keys the same on both sides whether or not we reorder the
+ * article, so no salient-token / multi-key index is built yet (deferred until probing shows the normalizer can't absorb
+ * the false-negatives).
+ */
+export type StreetLocale = "us" | "fr" | "de" | "nl"
+
+/**
+ * French street-type abbreviations → canonical full form, applied per token after {@link fold}. French address types
+ * LEAD the name ("Av. de…", "Bd …", "Pl. …") and "St"/"Ste" abbreviate Saint/Sainte inside names ("Rue St-Honoré" →
+ * "rue saint honore"). fold() has already stripped the trailing period, so the keys are point-free ("av", "bd").
+ */
+const FR_STREET_ABBREV = new Map<string, string>([
+	["av", "avenue"],
+	["ave", "avenue"],
+	["bd", "boulevard"],
+	["bld", "boulevard"],
+	["bvd", "boulevard"],
+	["boul", "boulevard"],
+	["pl", "place"],
+	["imp", "impasse"],
+	["all", "allee"],
+	["ch", "chemin"],
+	["che", "chemin"],
+	["sq", "square"],
+	["pas", "passage"],
+	["fg", "faubourg"],
+	["fbg", "faubourg"],
+	["rte", "route"],
+	["st", "saint"],
+	["ste", "sainte"],
+	["sts", "saints"],
+])
+
+/**
+ * Normalize a street name for the address-point key in a non-US locale. Same function build-side and probe-side (the
+ * one-function discipline). US delegates to {@link normalizeStreetForKey}.
+ *
+ * - **fr** — fold + expand leading type abbreviations and Saint/Sainte (token map).
+ * - **de** — fold + ß→ss + canonicalize the GLUED `-str(.)` suffix to `-strasse` ("Lindenstr." → "lindenstrasse",
+ *   "Lindenstraße" → "lindenstrasse"); an already-full "-strasse" is left intact.
+ * - **nl** — fold + canonicalize the glued `-str` suffix to `-straat` ("Kerkstr." → "kerkstraat").
+ */
+export function normalizeStreetForKeyLocale(street: string, locale: StreetLocale): string {
+	if (locale === "us") return normalizeStreetForKey(street)
+
+	// Hyphen → space so a compound name keys the same whether the source or the query writes the
+	// hyphen ("Champs-Élysées", "St-Honoré") or a space — both sides fold identically, so this is pure
+	// robustness. It also splits a hyphenated abbreviation ("St-Honoré" → "st honore") into tokens the
+	// per-locale type/Saint map can see.
+	const tokens = fold(street).replace(/ß/g, "ss").replace(/-/g, " ").split(/\s+/).filter(Boolean)
+
+	if (tokens.length === 0) return ""
+
+	switch (locale) {
+		case "fr":
+			for (let i = 0; i < tokens.length; i++) {
+				tokens[i] = FR_STREET_ABBREV.get(tokens[i]!) ?? tokens[i]!
+			}
+			break
+		case "de":
+			for (let i = 0; i < tokens.length; i++) {
+				const t = tokens[i]!
+
+				if (t.endsWith("str") && !t.endsWith("strasse")) {
+					tokens[i] = t.replace(/str$/, "strasse")
+				}
+			}
+			break
+		case "nl":
+			for (let i = 0; i < tokens.length; i++) {
+				const t = tokens[i]!
+
+				if (t.endsWith("str") && !t.endsWith("straat")) {
+					tokens[i] = t.replace(/str$/, "straat")
+				}
+			}
+			break
+	}
+
+	return tokens.join(" ")
+}
+
+/** Normalize a locality name for address-point keying (fold only — no street semantics). */
+export function normalizeLocalityForKey(locality: string): string {
+	return fold(locality)
+}
+
+/**
+ * Strip a trailing French arrondissement designator from a FOLDED commune key ("paris 8e arrondissement" → "paris",
+ * "lyon 1er arrondissement" → "lyon", "marseille 10e arrondissement" → "marseille"). Paris, Lyon and Marseille are the
+ * only French communes subdivided into _arrondissements municipaux_; a national register (BAN) names each row per
+ * arrondissement, but a query names the base commune ("Place Bellecour, Lyon", never "…, Lyon 2e"). Applied on BOTH
+ * sides of the #1042 street-centroid key — build-side (deriving the `locality_base` column) and query-side (folding the
+ * probe commune) — so the two agree by construction (the one-function discipline). Input must already be folded
+ * (lower-case, diacritic-stripped); a no-op for every other commune. Returns the input unchanged if the strip would
+ * empty it.
+ */
+export function stripArrondissement(localityNorm: string): string {
+	const stripped = localityNorm.replace(/\s+\d+(?:er|e)\s+arrondissement$/, "").trim()
+
+	return stripped || localityNorm
+}
+
+/**
+ * Strip a locality QUALIFIER for a query-side fallback — when an OA locality's exact normalized name misses the
+ * gazetteer's canonical name, retry with the qualifier removed. OA address data carries disambiguating qualifiers the
+ * gazetteer's canonical name omits: Austrian `Kraubath/Mur` and `Hart b.Graz` → `Hart`; Swiss `Lenk im Simmental` →
+ * `Lenk`, `Roche VD` → `Roche`; Danish `Odense S`, `Hurup Thy`. A FALLBACK ONLY — the exact name is tried first, and
+ * the region-bbox disambiguation resolves any base-name ambiguity downstream. The candidate table is unchanged (this is
+ * purely query-side); feed the result back through {@link normalizeLocalityForKey}. Returns "" when nothing was stripped
+ * (no point re-probing the identical key).
+ *
+ * Measured (`scripts/eval/candidate-recall.ts --strip-fallback`, EU OA holdouts): recovers AT 74.1→88.2% (+14.1pp), DK
+ * 91.5→96.2%, CH 90.4→92.6%; +1.3pp overall (diluted by the already-100% locales). Conservative by design — only the
+ * qualifier forms above; FI/PT/SI misses are untouched.
+ */
+export function stripLocalityQualifier(locality: string): string {
+	let s = locality.trim()
+
+	if (s.includes("/")) {
+		s = s.split("/")[0]!.trim()
+	} // "Kraubath/Mur", "St.Kanzian/Klopeiner See"
+	s = s.replace(/\s+[a-zà-ÿ]\.\s*\S.*$/iu, "") // abbreviated " b.Graz" / " o.Bleiburg" / " a.d. …"
+	s = s.replace(/\s+(im|an der|ob|bei|in der|unter|vor)\s+\S.*$/iu, "") // " im Simmental", " bei Graz"
+	s = s.replace(/\s+(S|N|E|W|V|Ø|Sø|Fyn|Thy|Sjælland|Jylland|[A-ZÅÄÖ]{2})$/u, "") // " S", " VD", " Thy"
+	s = s.trim()
+
+	return s === locality.trim() ? "" : s
+}
+
+/**
+ * Fold numbered-route designators to a canonical key, applied AFTER {@link normalizeStreetForKey}. Sources disagree
+ * systematically on how they spell a route: TIGER says `State Rte 100` / `US Hwy 5` where E911/Overture say `VT ROUTE
+ * 100` / `US ROUTE 5` — the dominant street-name miss class in the #483 interpolation eval (rural addresses live on
+ * routes). `us <designator> N…` folds to `us route N…`; `state <designator> N…` and `<2-letter-prefix> <designator> N…`
+ * (the state abbreviation form) fold to `state route N…`. Only digit-leading route numbers fold — `State Street` and
+ * friends never match.
+ *
+ * Used by BOTH the segment-shard builder (`scripts/build-interpolation-shard.ts`) and the interpolation lookup — same
+ * one-function discipline as {@link normalizeStreetForKey}. The address-point tier (#476) does NOT apply it yet:
+ * adopting it there requires a shard rebuild (noted on #483).
+ *
+ * A same-numbered US and state route stay DISTINCT keys (`us route 5` vs `state route 5`); only the BARE `route N` form
+ * is ambiguous (designator unknown) and it stays unfolded — a bare-route query therefore misses rather than guessing a
+ * designator.
+ */
+export function canonicalizeRouteKey(streetNorm: string): string {
+	const match = /^(us|state|[a-z]{2}) (?:route|rte|rt|highway|hwy) (\d.*)$/.exec(streetNorm)
+
+	if (!match) return streetNorm
+
+	return `${match[1] === "us" ? "us" : "state"} route ${match[2]}`
+}