@mailwoman/resolver-wof-sqlite 7.2.0 → 7.2.1

package/street-segment-schema.ts

@@ -0,0 +1,104 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Typed schema for the TIGER STREET-SEGMENT interpolation shards (`street-segments-<cc>-<st>.db`,
+ *   built by `scripts/build-interpolation-shard.ts` from TIGER EDGES) — the #483 Method-3 fallback
+ *   the resolver drops to when the address-point tier (Method 2) can't bracket. Single source of
+ *   truth for the columns the BUILDER writes and the READER ({@link StreetInterpolator}) probes, so
+ *   a column rename in one is a compile error in the other.
+ *
+ *   The builder reads geometry from shapefiles via DuckDB's spatial extension (raw `ST_Read` — see
+ *   AGENTS.md "Database / inline SQL") and writes here through `node:sqlite`. The hot positional
+ *   INSERT (a county's worth of edges) stays raw; its column list is derived from
+ *   {@link STREET_SEGMENT_COLUMNS} so it can't drift from the DDL.
+ */
+
+import type { Kysely } from "kysely"
+
+/**
+ * One TIGER street-segment edge: a `(from_hn, to_hn)` house-number range on one `side` of a named street, with the
+ * geometry the interpolator walks. `min_hn`/`max_hn` are the sorted bounds (the probe filters on them); `parity` is
+ * `odd`/`even`/`mixed`.
+ */
+export interface StreetSegmentTable {
+	/** Shared {@link normalizeStreetForKey} of the street — the build/query-consistent probe key. */
+	street_norm: string
+	/** `L` or `R` — the TIGER side the address range sits on. */
+	side: string
+	from_hn: number
+	to_hn: number
+	/** Sorted lower bound of `(from_hn, to_hn)` — the probe filters `min_hn <= n <= max_hn`. */
+	min_hn: number
+	/** Sorted upper bound of `(from_hn, to_hn)`. */
+	max_hn: number
+	/** `odd` | `even` | `mixed` — the house-number parity along the range. */
+	parity: string
+	postcode: string | null
+	/** 5-digit state+county FIPS the edge came from. */
+	county_fips: string
+	/** The street as it appeared in TIGER (kept for display / debugging). */
+	street_raw: string
+	/** GeoJSON LineString text (no SpatiaLite — read back with `JSON.parse`). */
+	geometry: string
+	/** Provenance: the dataset this edge came from (e.g. `tiger:edges`). */
+	source: string
+	/** The pinned TIGER release the edge was ingested from. */
+	release: string
+}
+
+/** The street-segment database schema for `new DatabaseClient<StreetSegmentDatabase>(...)`. */
+export interface StreetSegmentDatabase {
+	street_segment: StreetSegmentTable
+}
+
+/**
+ * The `street_segment` columns in INSERT order. The builder's positional prepared statement derives its placeholder
+ * list from this, so the positional order can't drift from the DDL / the reader.
+ */
+export const STREET_SEGMENT_COLUMNS = [
+	"street_norm",
+	"side",
+	"from_hn",
+	"to_hn",
+	"min_hn",
+	"max_hn",
+	"parity",
+	"postcode",
+	"county_fips",
+	"street_raw",
+	"geometry",
+	"source",
+	"release",
+] as const
+
+/** Create the `street_segment` table — called before the streaming bulk load. */
+export async function createStreetSegmentTable(db: Kysely<StreetSegmentDatabase>): Promise<void> {
+	await db.schema
+		.createTable("street_segment")
+		.addColumn("street_norm", "text", (c) => c.notNull())
+		.addColumn("side", "text", (c) => c.notNull())
+		.addColumn("from_hn", "integer", (c) => c.notNull())
+		.addColumn("to_hn", "integer", (c) => c.notNull())
+		.addColumn("min_hn", "integer", (c) => c.notNull())
+		.addColumn("max_hn", "integer", (c) => c.notNull())
+		.addColumn("parity", "text", (c) => c.notNull())
+		.addColumn("postcode", "text")
+		.addColumn("county_fips", "text", (c) => c.notNull())
+		.addColumn("street_raw", "text", (c) => c.notNull())
+		.addColumn("geometry", "text", (c) => c.notNull())
+		.addColumn("source", "text", (c) => c.notNull())
+		.addColumn("release", "text", (c) => c.notNull())
+		.execute()
+}
+
+/** Create the two probe indexes the reader relies on (postcode-scope, street-scope). */
+export async function createStreetSegmentIndexes(db: Kysely<StreetSegmentDatabase>): Promise<void> {
+	await db.schema
+		.createIndex("idx_seg_postcode")
+		.on("street_segment")
+		.columns(["postcode", "street_norm", "min_hn"])
+		.execute()
+	await db.schema.createIndex("idx_seg_street").on("street_segment").columns(["street_norm", "min_hn"]).execute()
+}

package/types.ts

@@ -0,0 +1,164 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Public surface for the WOF SQLite resolver — types only, no runtime.
+ *
+ *   These mirror the conceptual model described in `docs/plan/phases/PHASE_4_2_wof_sqlite.md`. Phase
+ *   4.3 will extend `PlaceCandidate` with the resolver-decorated fields that flow into
+ *   `AddressNode.source` / `sourceID` (e.g. an explicit `wofURI: "wof-admin:101751113"` form).
+ */
+
+/**
+ * The placetype taxonomy used by Who's On First. Ordered roughly from coarsest (country) to finest (address). See
+ * https://github.com/whosonfirst/whosonfirst-placetypes for the authoritative definitions of each.
+ *
+ * Phase 4.2 only emits the ones we actually look up; the union is open enough to extend later.
+ */
+export type WOFPlacetype =
+	| "country"
+	| "macroregion"
+	| "region"
+	| "macrocounty"
+	| "county"
+	| "localadmin"
+	| "locality"
+	| "borough"
+	| "neighbourhood"
+	| "microhood"
+	| "postalcode"
+	| "venue"
+	| "campus"
+	| "address"
+
+/**
+ * One candidate match for a place lookup.
+ *
+ * `score` is the post-boost ranking number — higher is better, but the scale is implementation- defined. Callers should
+ * treat it as ordinal, not absolute.
+ *
+ * `id` is the WOF place id. It's named generically (not `wof_id`) so the shape stays structurally compatible with
+ * `@mailwoman/resolver`'s `ResolvedPlace` — `WOFSqlitePlaceLookup` satisfies the generic `ResolverBackend` contract
+ * without an adapter shim.
+ *
+ * `distanceKm` is populated only when the query carried `near` (and the place has a centroid). Useful for downstream
+ * UIs that want to show "X km from you" alongside the result.
+ */
+export interface PlaceCandidate {
+	id: number
+	name: string
+	placetype: WOFPlacetype
+	/** ISO 3166-1 alpha-2 country code. */
+	country: string
+	lat: number
+	lon: number
+	parent_id?: number
+	score: number
+	distanceKm?: number
+	/**
+	 * True when this candidate's name OR an alias EXACTLY equals the query (the exact-match tier from
+	 * {@link RankingWeights.exactMatchTiering}). Surfaced so a downstream country re-rank (#369's postcode anchor in
+	 * `resolveTree`) can pin the country without crossing the tier — see the `exactMatch` field on `@mailwoman/core`'s
+	 * `ResolvedPlace`.
+	 */
+	exactMatch?: boolean
+	/**
+	 * Combined prominence (population term + best proximity-bias term, same additive units) — populated by the FTS
+	 * lookup; the exact-tier sort orders by THIS instead of raw population when the query carried proximity hints
+	 * (`near`/`bias`).
+	 */
+	prominence?: number
+	/**
+	 * Population from WOF's `wof:population` property. Only present when the candidate has it on record — WOF carries
+	 * population for ~15% of localities (mostly larger ones). Absent does NOT mean zero, just unknown.
+	 */
+	population?: number
+	/**
+	 * Bounding box from WOF's `spr.{min,max}_{latitude,longitude}` columns. Coarse outline for the place — a city's bbox
+	 * is the city's full extent, a postcode's is roughly the postcode polygon's envelope. Optional because not all
+	 * callers ask for it; implementations are free to omit when the underlying schema lacks the columns.
+	 */
+	bbox?: GeoBbox
+	/**
+	 * Set by the coordinate-first path when the chosen locality and the sibling postcode's containing locality are
+	 * geographically far apart — the postcode and the parsed city name disagree (a transposed / wrong-for-the-city
+	 * postcode). The candidate is still returned (the name wins for the locality), but the flag lets callers lower
+	 * confidence / surface the conflict rather than silently mislocate. A retrieval/BM25 geocoder can't raise this — it's
+	 * the falsehood-detection differentiator.
+	 */
+	mismatch?: boolean
+}
+
+/**
+ * A WGS-84 lat/lon point. Used as a proximity hint for `FindPlaceQuery.near`.
+ */
+export interface GeoPoint {
+	lat: number
+	lon: number
+}
+
+/**
+ * A WGS-84 bounding box. Used as a hard filter via `FindPlaceQuery.bbox`.
+ */
+export interface GeoBbox {
+	minLat: number
+	maxLat: number
+	minLon: number
+	maxLon: number
+}
+
+/**
+ * Query against the resolver.
+ *
+ * `text` is the only required field; everything else narrows the search. When `country` and `parentID` are both set,
+ * `parentID` wins (it's more specific).
+ *
+ * `near` and `bbox` are independent. `near` is a soft signal — candidates close to the point get a ranking boost but
+ * distant candidates aren't dropped. `bbox` is a hard filter — only candidates whose bbox intersects the query bbox are
+ * returned (uses the package-built R*Tree index when present; if the index is missing the option is silently ignored to
+ * preserve backwards compatibility).
+ *
+ * `near` may carry `maxDistanceKm` to escalate from a boost to a hard filter — candidates further than that distance
+ * from the point are dropped at the SQL level via an R*Tree pre-filter.
+ */
+export interface FindPlaceQuery {
+	text: string
+	placetype?: WOFPlacetype | WOFPlacetype[]
+	/** ISO 3166-1 alpha-2 — narrows to one country. */
+	country?: string
+	/** WOF place id — narrows to descendants of this place. */
+	parentID?: number
+	/**
+	 * Sibling postcode. When set on a `locality` query AND a `postcode_locality` table is present, triggers the
+	 * coordinate-first soft-score path: postcode→candidate localities are injected and scored `0.6·S_pc + 0.3·S_name +
+	 * 0.1·S_pop` against the FTS name-match set, recovering small localities the name-match alone misses. Ignored when no
+	 * postcode_locality shard is present.
+	 */
+	postcode?: string
+	/** Proximity hint — candidates close to this point get a ranking boost. */
+	near?: GeoPoint & { maxDistanceKm?: number }
+	/**
+	 * Ordered proximity-bias points (viewport center, user location, …), each optionally weighted (default 1.0, first
+	 * entry strongest by convention). SOFT — a re-rank signal, never a filter: with bias present, exact-tier candidates
+	 * order by combined prominence (population + the best decayed-distance term over these points) instead of population
+	 * alone, which is how an ambiguous bare postcode ("48026": Fraser MI vs Russi IT) follows the map view / the user.
+	 * Absent (and no `near`) → ranking is byte-identical to today. `near` is treated as a weight-1.0 bias point for
+	 * back-compat.
+	 */
+	bias?: Array<GeoPoint & { weight?: number }>
+	/** Bounding-box filter — only candidates whose bbox intersects this box are returned. */
+	bbox?: GeoBbox
+	/** Default 10. */
+	limit?: number
+}
+
+/**
+ * The pull-based lookup surface. Implementations resolve a `FindPlaceQuery` to a ranked list of `PlaceCandidate`s. The
+ * interface is async even though `node:sqlite` is sync — leaves room for `Worker`-backed implementations later without
+ * a public API break.
+ */
+export interface PlaceLookup {
+	findPlace(query: FindPlaceQuery): Promise<PlaceCandidate[]>
+	close(): void
+}

package/unified-schema.ts

@@ -0,0 +1,171 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Schema for the unified WOF SQLite database we build from cloned WOF GeoJSON repos
+ *   (`scripts/build-unified-wof.ts`). This is the CANONICAL gazetteer — we never use the
+ *   off-the-shelf geocode.earth prebuilt dumps (they assign different WOF ids to the same place;
+ *   see the `feedback-custom-wof-db-only` memory). The table/column names match the resolver's
+ *   expectations (`lookup.ts`) so `WOFSqlitePlaceLookup` works unchanged, INCLUDING the `ancestors`
+ *   table (which lookup.ts's parent-constraint subquery needs) — see `populateAncestors`. The
+ *   `place_search` FTS5 + `place_bbox` R*Tree are built separately by `build-fts` (fts.ts).
+ */
+
+import { DatabaseSync } from "node:sqlite"
+
+import { DatabaseClient } from "@mailwoman/core/kysley/client"
+
+import type { WOFDatabase } from "./schema.ts"
+
+export async function createUnifiedSchema(db: DatabaseSync): Promise<void> {
+	// PRAGMAs stay raw — not Kysely-modelled, and these tune the bulk build.
+	db.exec("PRAGMA journal_mode = WAL")
+	db.exec("PRAGMA busy_timeout = 10000")
+	db.exec("PRAGMA synchronous = OFF")
+
+	// `kdb` wraps `db` for the DDL (the house idiom); the caller owns `db`'s lifecycle, so we don't
+	// destroy it here. The bulk INSERTs (populateAncestors + build-unified-wof) stay on the raw handle.
+	const kdb = new DatabaseClient<WOFDatabase>({ database: db })
+
+	await kdb.schema
+		.createTable("spr")
+		.ifNotExists()
+		.addColumn("id", "integer", (c) => c.primaryKey())
+		.addColumn("parent_id", "integer", (c) => c.notNull().defaultTo(-1))
+		.addColumn("name", "text", (c) => c.notNull().defaultTo(""))
+		.addColumn("placetype", "text", (c) => c.notNull().defaultTo(""))
+		.addColumn("country", "text", (c) => c.notNull().defaultTo(""))
+		.addColumn("latitude", "real", (c) => c.notNull().defaultTo(0))
+		.addColumn("longitude", "real", (c) => c.notNull().defaultTo(0))
+		.addColumn("min_latitude", "real", (c) => c.notNull().defaultTo(0))
+		.addColumn("min_longitude", "real", (c) => c.notNull().defaultTo(0))
+		.addColumn("max_latitude", "real", (c) => c.notNull().defaultTo(0))
+		.addColumn("max_longitude", "real", (c) => c.notNull().defaultTo(0))
+		.addColumn("is_current", "integer", (c) => c.notNull().defaultTo(1))
+		.addColumn("is_deprecated", "integer", (c) => c.notNull().defaultTo(0))
+		.addColumn("is_ceased", "integer", (c) => c.notNull().defaultTo(0))
+		.addColumn("is_superseded", "integer", (c) => c.notNull().defaultTo(0))
+		.addColumn("is_superseding", "integer", (c) => c.notNull().defaultTo(0))
+		.addColumn("lastmodified", "integer", (c) => c.notNull().defaultTo(0))
+		.execute()
+
+	// `privateuse` carries WOF's x_<variant> kind (preferred | variant) / GeoNames' isPreferredName
+	// ("preferred" | ""). `official` is the #936 ingest bit: 1 when the row's language is an official
+	// language of the place's country (codex OFFICIAL_LANGUAGES) AND the row is a preferred form —
+	// x_variant rows tagged with an official language ("MSP", "Frisco") stay 0. Primary-name mirror
+	// rows stay 0 too: the name-exact tier already consults spr.name; `official` only marks the
+	// ALIASES eligible to join it. Both are ingest-time facts, never computed at query time.
+	await kdb.schema
+		.createTable("names")
+		.ifNotExists()
+		.addColumn("id", "integer", (c) => c.notNull())
+		.addColumn("name", "text", (c) => c.notNull())
+		.addColumn("placetype", "text", (c) => c.notNull().defaultTo(""))
+		.addColumn("country", "text", (c) => c.notNull().defaultTo(""))
+		.addColumn("language", "text", (c) => c.notNull().defaultTo(""))
+		.addColumn("privateuse", "text", (c) => c.notNull().defaultTo(""))
+		.addColumn("official", "integer", (c) => c.notNull().defaultTo(0))
+		.addColumn("lastmodified", "integer", (c) => c.notNull().defaultTo(0))
+		.execute()
+
+	await kdb.schema
+		.createTable("concordances")
+		.ifNotExists()
+		.addColumn("id", "integer", (c) => c.notNull())
+		.addColumn("other_id", "text", (c) => c.notNull())
+		.addColumn("other_source", "text", (c) => c.notNull())
+		.addColumn("lastmodified", "integer", (c) => c.notNull().defaultTo(0))
+		.execute()
+
+	await kdb.schema
+		.createTable("place_population")
+		.ifNotExists()
+		.addColumn("id", "integer", (c) => c.primaryKey())
+		.addColumn("population", "integer", (c) => c.notNull().defaultTo(0))
+		.execute()
+
+	// `ancestors` maps each place to every place above it in the hierarchy (and itself). The
+	// resolver's parent-constraint scopes a child lookup to a parent's descendants via
+	// `spr.id IN (SELECT id FROM ancestors WHERE ancestor_id = ?)`. The off-the-shelf WOF dumps
+	// ship this table; our build derives it from the parent_id chain (see populateAncestors) since
+	// we don't capture `wof:hierarchy`.
+	await kdb.schema
+		.createTable("ancestors")
+		.ifNotExists()
+		.addColumn("id", "integer", (c) => c.notNull())
+		.addColumn("ancestor_id", "integer", (c) => c.notNull())
+		.addColumn("ancestor_placetype", "text", (c) => c.notNull().defaultTo(""))
+		.addColumn("lastmodified", "integer", (c) => c.notNull().defaultTo(0))
+		.execute()
+}
+
+/**
+ * Populate the `ancestors` table by walking each place's `parent_id` chain in `spr` (transitive closure, including the
+ * place itself). Idempotent: drops + rebuilds the table contents. Returns the row count. Run after `spr` is fully
+ * ingested (build-unified-wof freeze phase) or standalone on an existing unified DB (`scripts/add-ancestors.ts`).
+ * Sentinel/negative parent_ids and cycles terminate the walk. ~4 rows/place average; a transaction keeps the ~5M
+ * inserts fast.
+ */
+export function populateAncestors(db: DatabaseSync): number {
+	db.exec("DELETE FROM ancestors")
+	const rows = db.prepare("SELECT id, parent_id, placetype FROM spr").all() as Array<{
+		id: number
+		parent_id: number
+		placetype: string
+	}>
+	const byID = new Map<number, { parent: number; placetype: string }>()
+
+	for (const r of rows) {
+		byID.set(r.id, { parent: r.parent_id, placetype: r.placetype })
+	}
+
+	const insert = db.prepare("INSERT INTO ancestors (id, ancestor_id, ancestor_placetype) VALUES (?, ?, ?)")
+	db.exec("BEGIN")
+	let count = 0
+
+	for (const r of rows) {
+		insert.run(r.id, r.id, r.placetype) // self
+		count++
+		const seen = new Set<number>([r.id])
+		let cur = r.parent_id
+
+		while (cur > 0 && !seen.has(cur)) {
+			const node = byID.get(cur)
+
+			if (!node) break
+			insert.run(r.id, cur, node.placetype)
+			count++
+			seen.add(cur)
+			cur = node.parent
+		}
+	}
+	db.exec("COMMIT")
+
+	return count
+}
+
+export async function createUnifiedIndexes(db: DatabaseSync): Promise<void> {
+	const kdb = new DatabaseClient<WOFDatabase>({ database: db })
+	await kdb.schema.createIndex("spr_by_placetype").ifNotExists().on("spr").column("placetype").execute()
+	await kdb.schema.createIndex("spr_by_country").ifNotExists().on("spr").column("country").execute()
+	await kdb.schema.createIndex("spr_by_parent").ifNotExists().on("spr").column("parent_id").execute()
+	await kdb.schema.createIndex("names_by_id").ifNotExists().on("names").column("id").execute()
+	await kdb.schema.createIndex("names_by_name").ifNotExists().on("names").column("name").execute()
+	await kdb.schema
+		.createIndex("concordances_by_id")
+		.ifNotExists()
+		.on("concordances")
+		.columns(["id", "lastmodified"])
+		.execute()
+	await kdb.schema
+		.createIndex("concordances_by_other_id")
+		.ifNotExists()
+		.on("concordances")
+		.columns(["other_source", "other_id"])
+		.execute()
+	// ancestor_id is the hot column (parent-constraint queries `WHERE ancestor_id = ?`); id supports
+	// the reverse lookup.
+	await kdb.schema.createIndex("ancestors_by_ancestor").ifNotExists().on("ancestors").column("ancestor_id").execute()
+	await kdb.schema.createIndex("ancestors_by_id").ifNotExists().on("ancestors").column("id").execute()
+}