@mailwoman/resolver-wof-sqlite 9.1.0 → 9.2.0

package/coincident-roles.ts

@@ -40,8 +40,11 @@
 
 import type { DatabaseSync } from "node:sqlite"
 
+import type { CoincidentLocality } from "@mailwoman/resolver"
 import { haversineKm } from "@mailwoman/spatial"
 
+import { allRows } from "./sqlite-utils.ts"
+
 /**
  * Table of places that hold more than one admin role — a locality that is also its county seat. Written by the
  * gazetteer build, read by the resolver when a coincident locality has to be chosen.
@@ -149,8 +152,8 @@ export function buildCoincidentRoles(
 	// Admin (region/county tier) ⋈ same-name DESCENDANT locality. `place_population` is optional (LEFT
 	// JOIN → 0 when absent). The relative-tolerance filter + relationship classification happen in JS so
 	// the SQL stays a plain join. `spr` exposes the bbox columns we need for the diagonal.
-	const candidates = db
-		.prepare(
+	const candidates = allRows<CandidateRow>(
+		db.prepare(
 			`SELECT r.id AS admin_id, r.placetype AS admin_placetype, r.country AS country, l.id AS locality_id,
 				r.latitude AS rlat, r.longitude AS rlon, l.latitude AS llat, l.longitude AS llon,
 				r.min_latitude, r.min_longitude, r.max_latitude, r.max_longitude,
@@ -163,7 +166,7 @@ export function buildCoincidentRoles(
 			WHERE r.placetype = 'region'
 				AND r.is_current != 0 AND r.is_deprecated = 0`
 		)
-		.all() as unknown as CandidateRow[]
+	)
 
 	onProgress("filtering", `${candidates.length} candidates`)
 
@@ -226,19 +229,19 @@ export function loadCoincidentRoles(db: DatabaseSync): Map<number, CoincidentRol
 
 	if (!coincidentRolesExists(db)) return map
 
-	const rows = db
-		.prepare(
-			`SELECT admin_id, locality_id, relationship_type, admin_placetype, distance_km, locality_population
-			FROM ${COINCIDENT_ROLES_TABLE}`
-		)
-		.all() as unknown as Array<{
+	const rows = allRows<{
 		admin_id: number
 		locality_id: number
 		relationship_type: CoincidentRole["relationshipType"]
 		admin_placetype: string
 		distance_km: number
 		locality_population: number
-	}>
+	}>(
+		db.prepare(
+			`SELECT admin_id, locality_id, relationship_type, admin_placetype, distance_km, locality_population
+			FROM ${COINCIDENT_ROLES_TABLE}`
+		)
+	)
 
 	for (const r of rows) {
 		const entry: CoincidentRole = {
@@ -260,3 +263,59 @@ export function loadCoincidentRoles(db: DatabaseSync): Map<number, CoincidentRol
 
 	return map
 }
+
+/**
+ * The relation joined with `spr`, as an in-memory map keyed by `admin_id` — the resolver-facing view of
+ * {@link loadCoincidentRoles}, carrying the canonical name and coordinates each coincident locality resolves to.
+ * Returns an empty map when the relation table is absent.
+ */
+export function loadCoincidentLocalities(db: DatabaseSync): Map<number, CoincidentLocality[]> {
+	const map = new Map<number, CoincidentLocality[]>()
+
+	if (coincidentRolesExists(db)) {
+		const rows = allRows<{
+			adminID: number
+			id: number
+			name: string
+			country: string
+			lat: number
+			lon: number
+			relationshipType: string
+			population: number
+			distanceKm: number
+		}>(
+			db.prepare(
+				`SELECT cr.admin_id AS adminID, s.id AS id, s.name AS name, s.country AS country,
+					s.latitude AS lat, s.longitude AS lon,
+					cr.relationship_type AS relationshipType, cr.locality_population AS population,
+					cr.distance_km AS distanceKm
+				FROM ${COINCIDENT_ROLES_TABLE} cr JOIN spr s ON s.id = cr.locality_id`
+			)
+		)
+
+		for (const r of rows) {
+			const candidate: CoincidentLocality = {
+				id: r.id,
+				name: r.name,
+				placetype: "locality",
+				country: r.country,
+				lat: r.lat,
+				lon: r.lon,
+				score: 0,
+				relationshipType: r.relationshipType,
+				population: r.population,
+				distanceKm: r.distanceKm,
+			}
+
+			const list = map.get(r.adminID)
+
+			if (list) {
+				list.push(candidate)
+			} else {
+				map.set(r.adminID, [candidate])
+			}
+		}
+	}
+
+	return map
+}

package/convention-schema.ts

@@ -0,0 +1,72 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Typed schema for the convention asset — the `address_convention` table `SqliteConventionSource` probes, plus the
+ *   `meta` provenance row every sealed artifact here carries. The interface is the read/write contract and
+ *   {@link createAddressConventionTable} creates the table, so a column added to one is a compile error against the
+ *   other.
+ */
+
+import type { Kysely } from "kysely"
+
+import { ADDRESS_CONVENTION_TABLE } from "./convention.ts"
+
+/**
+ * One convention profile, keyed by the WOF admin polygon it attaches to.
+ */
+export interface AddressConventionTable {
+	wof_id: number
+	/**
+	 * The `Convention` record, JSON-encoded. Parsed at read time by `SqliteConventionSource`.
+	 */
+	convention: string
+	/**
+	 * Provenance: why this row exists and where it came from.
+	 */
+	source: string
+}
+
+/**
+ * Key/value provenance for the sealed artifact.
+ */
+export interface ConventionMetaTable {
+	key: string
+	value: string | null
+}
+
+export interface ConventionDatabase {
+	address_convention: AddressConventionTable
+	meta: ConventionMetaTable
+}
+
+/**
+ * The slice of a Kysely handle the convention DDL touches. Kysely is invariant in its schema parameter, so naming only
+ * `schema` lets a builder holding a wider handle pass it without a cast.
+ */
+export type ConventionSchemaHandle = Pick<Kysely<ConventionDatabase>, "schema">
+
+/**
+ * Create `address_convention`. The table name comes from {@link ADDRESS_CONVENTION_TABLE} so the build script, the
+ * runtime source, and the shard auto-detect cannot drift apart.
+ */
+export async function createAddressConventionTable(db: ConventionSchemaHandle): Promise<void> {
+	await db.schema
+		.createTable(ADDRESS_CONVENTION_TABLE)
+		.addColumn("wof_id", "integer", (column) => column.primaryKey())
+		.addColumn("convention", "text", (column) => column.notNull())
+		.addColumn("source", "text", (column) => column.notNull())
+		.execute()
+}
+
+/**
+ * Create the artifact's `meta` table.
+ */
+export async function createConventionMetaTable(db: ConventionSchemaHandle): Promise<void> {
+	await db.schema
+		.createTable("meta")
+		.addColumn("key", "text", (column) => column.primaryKey())
+		.addColumn("value", "text")
+		.execute()
+}

package/convention.ts

@@ -142,8 +142,8 @@ export function mergeConventions(base: Convention, ...overrides: Array<Conventio
  * region, …, locality). Starts from `WORLD_DEFAULT` so every field is defined regardless of which (if any) ancestors
  * carry an override.
  */
-export function resolveConvention(source: ConventionSource, ancestorIds: readonly number[]): ResolvedConvention {
-	const layers = ancestorIds.map((id) => source.get(id))
+export function resolveConvention(source: ConventionSource, ancestorIDs: readonly number[]): ResolvedConvention {
+	const layers = ancestorIDs.map((id) => source.get(id))
 	const merged = mergeConventions(WORLD_DEFAULT, ...layers)
 
 	return {

package/coverage-manifest-schema.ts

@@ -33,7 +33,7 @@ import {
 } from "@mailwoman/core/resolver"
 import { sql, type Kysely } from "kysely"
 
-import { hasTable } from "./sqlite-utils.ts"
+import { allRows, hasTable } from "./sqlite-utils.ts"
 
 /**
  * One country's hard-filter coverage measurement — the storage form of {@link CountryCoverageFact}.
@@ -194,11 +194,11 @@ export function readGazetteerCoverageManifest(db: DatabaseSync): GazetteerArtifa
 	const countryCoverage = new Map<string, CountryCoverageFact>()
 
 	if (hasCoverage) {
-		const rows = db
-			.prepare(
+		const rows = allRows<CountryCoverageTable>(
+			db.prepare(
 				`SELECT country, hard_filter_safe, hard_resolve_rate, sample_size, measured_at, source FROM ${COUNTRY_COVERAGE_TABLE}`
 			)
-			.all() as unknown as CountryCoverageTable[]
+		)
 
 		for (const row of rows) {
 			const country = String(row.country).toUpperCase()
@@ -217,9 +217,9 @@ export function readGazetteerCoverageManifest(db: DatabaseSync): GazetteerArtifa
 	const countryBBoxes = new Map<string, CountryBBoxFact>()
 
 	if (hasBBox) {
-		const rows = db
-			.prepare(`SELECT country, lat_min, lat_max, lon_min, lon_max, source FROM ${COUNTRY_BBOX_TABLE}`)
-			.all() as unknown as CountryBBoxTable[]
+		const rows = allRows<CountryBBoxTable>(
+			db.prepare(`SELECT country, lat_min, lat_max, lon_min, lon_max, source FROM ${COUNTRY_BBOX_TABLE}`)
+		)
 
 		for (const row of rows) {
 			const country = String(row.country).toUpperCase()

package/currency-backfill.ts

@@ -0,0 +1,249 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Pass 1c of the candidate build (#1737): resurrect deprecated-with-no-successor WOF localities that a second
+ *   source independently attests. Extracted from `build-candidate.ts` as a cohesive unit — the gates, their measured
+ *   constants, and the GeoNames dump reader live together here; the build calls {@link resurrectCurrencyHoles} once,
+ *   between the primaries pass and the alias pass.
+ */
+
+import { existsSync } from "node:fs"
+import { resolve } from "node:path"
+import type { DatabaseSync } from "node:sqlite"
+
+import { isStrictlyFiner } from "@mailwoman/core/resources/whosonfirst"
+import { haversineKm } from "@mailwoman/spatial"
+import { TSVSpliterator } from "spliterator"
+
+import type { loadImportanceIndex } from "./candidate-importance.ts"
+import type { PlaceAttrs, StageRow } from "./candidate/place-attrs.ts"
+import { normalizeLocalityForKey } from "./street-normalize.ts"
+
+/**
+ * Corroboration radius for the currency backfill (#1737), km — both for the live-near blocker and the GeoNames
+ * attestation. Measured basis (2026-08-19 prototype, GB locality slice): at 10 km, 20 of 108 dead names resurrect —
+ * Rochester (Kent), Aldershot, Staines, Telford, Ebbw Vale among them — while the `Birmingham/Wolverhampton/…`
+ * conurbation blobs stay dead (no attestation) and Swansea/Wrexham stay out because a live same-name row already serves
+ * them within the radius.
+ */
+const CURRENCY_BACKFILL_RADIUS_KM = 10
+
+/**
+ * Minimum attestor population for a resurrection. The same prototype measured 44 of 108 dead GB names attested but
+ * under this floor — hamlet-scale ghosts whose absence from the index nobody has reported. A floor keeps the pass
+ * answering the measured defect (real settlements) rather than re-importing the tail WOF chose to prune.
+ */
+const CURRENCY_BACKFILL_POP_FLOOR = 1000
+
+/**
+ * Pass 1c (#1737): resurrect deprecated-with-no-successor localities that a second source independently attests.
+ *
+ * WOF deprecations WITH a successor need nothing — the successor indexes. A deprecation with `is_superseded = 0` on a
+ * populated place is the shape of an upstream mistake (Rochester Kent, Aldershot, Telford; 120 GB localities alone),
+ * and it is indistinguishable from a correct pruning at this layer without outside evidence. So every resurrection
+ * requires all three gates, positive evidence only:
+ *
+ * 1. NO live same-name spr row of any placetype within {@link CURRENCY_BACKFILL_RADIUS_KM} of the dead record — a live row
+ *    means the place is alive (possibly under another placetype) and there is no hole. A DISTANT same-name row is a
+ *    namesake and does not block.
+ * 2. A GeoNames feature-class-P attestation of the same folded name within the radius.
+ * 3. The attestor at or above {@link CURRENCY_BACKFILL_POP_FLOOR}.
+ *
+ * The staged row keeps the WOF identity — id, name, centroid, bbox, region ancestry — because the dead record's own
+ * data is not what is wrong with it. GeoNames contributes exactly two things: the attestation, and the population that
+ * lets the row stand in prominence races (the dead record's own population is absent). Each name is judged once per
+ * country; the resurrected place joins `attrs`, so the alias pass explodes its alt names like any primary's.
+ */
+export async function resurrectCurrencyHoles(ctx: {
+	src: DatabaseSync
+	tx: DatabaseSync
+	geonamesDir: string
+	countries: readonly string[]
+	attrs: Map<number, PlaceAttrs>
+	ccID: (code: string | null) => number
+	ptID: (pt: string | null) => number
+	regionOf: Map<number, number>
+	importance: ReturnType<typeof loadImportanceIndex> | undefined
+	stageRow: StageRow
+	progress: (phase: string, message: string) => void
+}): Promise<number> {
+	const deadStmt = ctx.src.prepare(
+		`SELECT id, name, placetype, latitude, longitude, min_latitude, min_longitude, max_latitude, max_longitude
+		 FROM spr
+		 WHERE country = ? AND placetype = 'locality'
+		   AND is_current = 0 AND is_deprecated = 1 AND is_superseded = 0`
+	)
+
+	const liveStmt = ctx.src.prepare(
+		`SELECT latitude, longitude, placetype FROM spr WHERE country = ? AND name = ? AND is_current != 0`
+	)
+
+	let total = 0
+
+	for (const country of ctx.countries) {
+		const cc = country.toUpperCase()
+		const dumpPath = resolve(ctx.geonamesDir, `${cc}.txt`)
+
+		if (!existsSync(dumpPath)) {
+			ctx.progress("currency-backfill", `${cc}: no GeoNames dump at ${dumpPath} — holes stay dead`)
+
+			continue
+		}
+
+		// Dead rows FIRST: a country with no deprecated-no-successor localities needs no attestors at all, and
+		// loading a national dump to judge zero rows is pure heap pressure on a build already near its ceiling
+		// (the first live run OOM'd in a later pass with JP/KR dumps loaded for 0 dead names each).
+		const dead = deadStmt.all(cc)
+
+		if (!dead.length) {
+			ctx.progress("currency-backfill", `${cc}: 0 dead names — dump not loaded`)
+
+			continue
+		}
+
+		// Only the dead names' own folded keys can ever be probed, so only those keys are worth holding —
+		// the rest of the national dump streams through without residency.
+		const deadKeys = new Set<string>()
+
+		for (const d of dead) {
+			const k = normalizeLocalityForKey(String(d.name ?? ""))
+
+			if (k) {
+				deadKeys.add(k)
+			}
+		}
+
+		// Folded name → P-class attestors. GeoNames columns by index: 1 name, 2 ascii, 4 lat, 5 lon,
+		// 6 feature_class, 14 population.
+		const attestors = new Map<string, { lat: number; lon: number; pop: number }[]>()
+
+		for await (const f of TSVSpliterator.fromAsync(dumpPath, { header: false })) {
+			if (f[6] !== "P") continue
+
+			const keys = [normalizeLocalityForKey(String(f[1] ?? "")), normalizeLocalityForKey(String(f[2] ?? ""))].filter(
+				(key) => key && deadKeys.has(key)
+			)
+
+			if (!keys.length) continue
+
+			const row = { lat: Number(f[4]), lon: Number(f[5]), pop: Number(f[14]) || 0 }
+
+			for (const key of new Set(keys)) {
+				const bag = attestors.get(key)
+
+				if (bag) {
+					bag.push(row)
+				} else {
+					attestors.set(key, [row])
+				}
+			}
+		}
+
+		let judged = 0
+		let blocked = 0
+		let unattested = 0
+		let floored = 0
+		let resurrected = 0
+		const seen = new Set<string>()
+
+		ctx.tx.exec("BEGIN")
+
+		for (const d of dead) {
+			const name = String(d.name ?? "")
+			const pkey = normalizeLocalityForKey(name)
+
+			if (!pkey || seen.has(pkey)) continue
+			seen.add(pkey)
+
+			judged++
+
+			const dLat = Number(d.latitude)
+			const dLon = Number(d.longitude)
+			// The dead-row query is scoped to `locality` today; read it from the row anyway so widening that query
+			// cannot silently start comparing every candidate against a hardcoded rung.
+			const deadPlacetype = String(d.placetype ?? "locality")
+
+			// A live row blocks only when it is AT LEAST AS COARSE as the dead one. The original gate compared name and
+			// distance alone, on the premise that a nearby same-name row means "the place is alive under another
+			// placetype" — true for a place recorded twice, false for a placetype DEMOTION, which is the shape that
+			// actually occurs: WOF retired `Gillingham` the locality (pop 101,187) and kept `Gillingham` the
+			// neighbourhood 3.2 km away, and the gate read the surviving CHILD as covering its own dead parent.
+			// Sixteen of seventeen GB refusals had exactly that shape (#1746).
+			//
+			// An UNRANKED placetype blocks, which is the conservative direction: this gate's failure mode is inventing
+			// a place, so a row we cannot rank is treated as covering rather than waved through.
+			const liveNear = liveStmt.all(cc, name).some((row) => {
+				if (haversineKm(dLat, dLon, Number(row.latitude), Number(row.longitude)) > CURRENCY_BACKFILL_RADIUS_KM) {
+					return false
+				}
+
+				// Blocks UNLESS the live row is strictly finer. The equal rung must still block — a live `locality`
+				// covers a dead `locality` — and an unranked placetype blocks too, since this gate's failure mode
+				// is inventing a place.
+				return isStrictlyFiner(String(row.placetype ?? ""), deadPlacetype) !== true
+			})
+
+			if (liveNear) {
+				blocked++
+
+				continue
+			}
+
+			const near = (attestors.get(pkey) ?? []).filter(
+				(g) => haversineKm(dLat, dLon, g.lat, g.lon) <= CURRENCY_BACKFILL_RADIUS_KM
+			)
+
+			if (!near.length) {
+				unattested++
+
+				continue
+			}
+
+			const pop = Math.max(...near.map((g) => g.pop))
+
+			if (pop < CURRENCY_BACKFILL_POP_FLOOR) {
+				floored++
+
+				continue
+			}
+
+			const sid = Number(d.id)
+
+			const a: PlaceAttrs = {
+				cid: ctx.ccID(cc),
+				rid: ctx.regionOf.get(sid) ?? 0,
+				ptid: ctx.ptID("locality"),
+				name,
+				lat: dLat,
+				lon: dLon,
+				mnLat: Number(d.min_latitude),
+				mnLon: Number(d.min_longitude),
+				mxLat: Number(d.max_latitude),
+				mxLon: Number(d.max_longitude),
+				pop,
+				neg: -Math.log10(pop + 1),
+				pkey,
+				imp: ctx.importance?.find(name, cc, "locality", dLat, dLon) ?? null,
+			}
+
+			ctx.attrs.set(sid, a)
+			ctx.stageRow(pkey, a, sid, 1)
+
+			resurrected++
+		}
+
+		ctx.tx.exec("COMMIT")
+
+		ctx.progress(
+			"currency-backfill",
+			`${cc}: ${resurrected} resurrected of ${judged} dead names ` +
+				`(${blocked} blocked by a live near row, ${unattested} unattested, ${floored} under the population floor)`
+		)
+
+		total += resurrected
+	}
+
+	return total
+}

package/exact-match.ts

@@ -0,0 +1,104 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The exact-match tier's name probes — the two case-folded equality lookups that decide whether a
+ *   candidate holds the query text as its own name, an alias, or an official name.
+ */
+
+import type { DatabaseSync } from "node:sqlite"
+
+import { aliasBagExactMatch } from "./fts.ts"
+
+/**
+ * Among `ids`, return the subset whose name OR any alias equals `text` case-insensitively — the exact-match tier for
+ * ranking. One indexed query over `<schema>.names`. When the shard has no `names` table (a slim DB built with
+ * `dropNames`, or a postcode-only shard), fall back to the self-contained `place_search` FTS content: its `alt_names`
+ * column is the same alias set joined on the boundary-preserving `ALIAS_SEPARATOR` (#523), so `aliasBagExactMatch`
+ * recovers the exact alias tier ("New York City" → New York) that the dropped `names` table used to provide.
+ */
+export function exactMatchIDs(db: DatabaseSync, schemaName: string, ids: number[], text: string): Set<number> {
+	const out = new Set<number>()
+	const trimmed = text.trim()
+
+	if (!ids.length || !trimmed) return out
+	const placeholders = ids.map(() => "?").join(", ")
+
+	try {
+		const rows = db
+			.prepare(`SELECT DISTINCT id FROM ${schemaName}.names WHERE id IN (${placeholders}) AND name = ? COLLATE NOCASE`)
+			.all(...ids, trimmed) as Array<{ id: number }>
+
+		for (const r of rows) {
+			out.add(r.id)
+		}
+
+		return out
+	} catch {
+		// No `names` table on this shard — fall through to the place_search alias bag.
+	}
+
+	try {
+		const rows = db
+			.prepare(`SELECT wof_id AS id, name, alt_names FROM ${schemaName}.place_search WHERE wof_id IN (${placeholders})`)
+			.all(...ids) as Array<{ id: number; name: string | null; alt_names: string | null }>
+
+		const norm = (s: string): string => s.toLowerCase().trim().replaceAll(/\s+/g, " ")
+		const needle = norm(trimmed)
+
+		for (const r of rows) {
+			if (r.name !== null && norm(r.name) === needle) {
+				out.add(r.id)
+			}
+		}
+
+		// Alias pass via the shared bag parser (#523). Separated bags (built since #523) get a true
+		// per-alias equality check, ungated — matching the `names`-table branch above, where an
+		// alias match counts as exact regardless of other candidates. Legacy bags (no separator)
+		// fall back to padded containment, gated on "no canonical exact in the pool" because their
+		// lost boundaries would otherwise false-promote interior fragments ("York" inside the alias
+		// "New York City") or cross-alias fragments ("York New" across "…York" + "New City…").
+		const anyCanonicalExact = out.size > 0
+
+		for (const r of rows) {
+			if (aliasBagExactMatch(r.alt_names, needle, anyCanonicalExact)) {
+				out.add(r.id)
+			}
+		}
+	} catch {
+		// Shard without place_search either → no exact-match tier. Falls back to weighted-sum order.
+	}
+
+	return out
+}
+
+/**
+ * Among `ids` (already known exact matches), the subset holding `text` as an OFFICIAL name (`names.official = 1`, the
+ * #940 ingest bit). Same COLLATE NOCASE semantics as {@link WOFSQLitePlaceLookup.#exactMatchIDs} so the two probes
+ * agree on what "equals the query" means. Fails soft on gazetteers built before #940 (no `official` column) — the
+ * sub-tier then behaves exactly as if `officialNameExact` were off.
+ */
+export function officialNameIDs(db: DatabaseSync, schemaName: string, ids: number[], text: string): Set<number> {
+	const out = new Set<number>()
+	const trimmed = text.trim()
+
+	if (!ids.length || !trimmed) return out
+	const placeholders = ids.map(() => "?").join(", ")
+
+	try {
+		const rows = db
+			.prepare(
+				`SELECT DISTINCT id FROM ${schemaName}.names WHERE id IN (${placeholders}) AND official = 1 AND name = ? COLLATE NOCASE`
+			)
+			.all(...ids, trimmed) as Array<{ id: number }>
+
+		for (const r of rows) {
+			out.add(r.id)
+		}
+	} catch {
+		// Pre-#940 gazetteer (no `official` column) or a names-less slim shard — feature inert.
+	}
+
+	return out
+}