@mailwoman/resolver-wof-sqlite 7.2.0 → 7.2.1

package/poi-schema.ts

@@ -0,0 +1,147 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Typed schema for poi.db — spatial layer #1 (spec §3.4). One clustered `WITHOUT ROWID` B-tree
+ *   keyed `(h3_cell, category_id, neg_rank, rowid_key)` so "everything near this res-9 cell" is a
+ *   contiguous key range (the byte-range/httpvfs access pattern, same discipline as the candidate
+ *   gazetteer). Rows carry denormalized name/brand/coords; category ids are small ints via the
+ *   `poi_category_codes` dictionary (poi-taxonomy category ids are the string side). The DB also
+ *   embeds the layer-contract tables from `@mailwoman/core/layers` — the builder writes the
+ *   manifest (tier `shipped`, spine `h3` res 9) and per-res-6-cell coverage.
+ */
+
+import type { DatabaseSync } from "node:sqlite"
+
+import type { LayerContractDatabase } from "@mailwoman/core/layers"
+import { sql, type Kysely } from "kysely"
+
+/** One POI row. Clustered PK: h3_cell → category_id → neg_rank → rowid_key. */
+export interface POITable {
+	/** 48-bit short H3 cell at res 9 (`latLngToCell` → `shortenH3Cell`). */
+	h3_cell: number
+	/** Small int from {@link POICategoryCodeTable}; 0 = uncategorized. */
+	category_id: number
+	/** `-log10(confidence + epsilon)` so ASC = most-confident-first within a cell+category. */
+	neg_rank: number
+	/** Uniquifier within the clustered key (builder-assigned monotonic int). */
+	rowid_key: number
+	name: string | null
+	/** Lowercased, diacritic-flattened probe key for exact name lookups. */
+	name_key: string | null
+	brand_wikidata: string | null
+	latitude: number
+	longitude: number
+	/** ISO 3166-1 alpha-2 (from the Overture partition). */
+	country: string
+	/** Overture existence confidence (already filtered ≥ 0.85 at build). */
+	confidence: number
+	/** GERS id — nullable METADATA ONLY, never a key (the #470 rule). */
+	gers_id: string | null
+}
+
+/**
+ * Staging mirror — every column nullable except the coords (the loader fills positionally; the materialize SELECT
+ * enforces completeness).
+ */
+export interface POIStageTable {
+	h3_cell: number | null
+	category_id: number | null
+	neg_rank: number | null
+	rowid_key: number | null
+	name: string | null
+	name_key: string | null
+	brand_wikidata: string | null
+	latitude: number
+	longitude: number
+	country: string | null
+	confidence: number | null
+	gers_id: string | null
+}
+
+/** `(id → poi-taxonomy category id)` dictionary, e.g. `3 → "cafe"`. */
+export interface POICategoryCodeTable {
+	id: number
+	category: string
+}
+
+export interface POIDatabase extends LayerContractDatabase {
+	poi: POITable
+	poi_stage: POIStageTable
+	poi_category_codes: POICategoryCodeTable
+}
+
+/** Clustered-key-order column list shared by builder + `INSERT INTO poi SELECT … FROM poi_stage`. */
+export const POI_COLUMNS = [
+	"h3_cell",
+	"category_id",
+	"neg_rank",
+	"rowid_key",
+	"name",
+	"name_key",
+	"brand_wikidata",
+	"latitude",
+	"longitude",
+	"country",
+	"confidence",
+	"gers_id",
+] as const
+
+export async function createPOIStagingTables(db: Kysely<POIDatabase>): Promise<void> {
+	await db.schema
+		.createTable("poi_category_codes")
+		.addColumn("id", "integer", (c) => c.primaryKey())
+		.addColumn("category", "text", (c) => c.unique())
+		.execute()
+	await db.schema
+		.createTable("poi_stage")
+		.addColumn("h3_cell", "integer")
+		.addColumn("category_id", "integer")
+		.addColumn("neg_rank", "real")
+		.addColumn("rowid_key", "integer")
+		.addColumn("name", "text")
+		.addColumn("name_key", "text")
+		.addColumn("brand_wikidata", "text")
+		.addColumn("latitude", "real")
+		.addColumn("longitude", "real")
+		.addColumn("country", "text")
+		.addColumn("confidence", "real")
+		.addColumn("gers_id", "text")
+		.execute()
+}
+
+export async function createPOITable(db: Kysely<POIDatabase>): Promise<void> {
+	await db.schema
+		.createTable("poi")
+		.addColumn("h3_cell", "integer", (c) => c.notNull())
+		.addColumn("category_id", "integer", (c) => c.notNull())
+		.addColumn("neg_rank", "real", (c) => c.notNull())
+		.addColumn("rowid_key", "integer", (c) => c.notNull())
+		.addColumn("name", "text")
+		.addColumn("name_key", "text")
+		.addColumn("brand_wikidata", "text")
+		.addColumn("latitude", "real", (c) => c.notNull())
+		.addColumn("longitude", "real", (c) => c.notNull())
+		.addColumn("country", "text", (c) => c.notNull())
+		.addColumn("confidence", "real", (c) => c.notNull())
+		.addColumn("gers_id", "text")
+		.addPrimaryKeyConstraint("poi_pk", ["h3_cell", "category_id", "neg_rank", "rowid_key"])
+		// `WITHOUT ROWID` has no first-class builder; the raw modifier is the idiomatic fallback.
+		.modifyEnd(sql`without rowid`)
+		.execute()
+}
+
+/** Secondary index for the FTS-hydration path. Builders call this AFTER the bulk materialize (index-after-load). */
+export async function createPOINameKeyIndex(db: Kysely<POIDatabase>): Promise<void> {
+	await db.schema.createIndex("poi_name_key").on("poi").column("name_key").execute()
+}
+
+export const POI_FTS_TABLE = "poi_search"
+
+/** FTS5 stays raw SQL by project rule (Kysely can't express virtual tables). Content-keyed by name_key. */
+export function createPOISearchFTS(db: DatabaseSync): void {
+	db.exec(
+		`CREATE VIRTUAL TABLE ${POI_FTS_TABLE} USING fts5(name, name_key UNINDEXED, h3_cell UNINDEXED, tokenize = 'unicode61')`
+	)
+}

package/postal-city-alias-lookup.ts

@@ -0,0 +1,89 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Node reader over the POSTAL-CITY ALIAS table (`postal-city-alias-<cc>.db`) — the observed
+ *   `postal_city → geo_locality` aliases per postcode (`build-postal-city-alias.ts`). Consumed by
+ *   {@link WOFSqlitePlaceLookup}'s coordinate-first locality scorer: a user-typed postal city
+ *   ("Antioch", postcode 37013) becomes a name-match alias for the geographic locality the postcode
+ *   actually sits in ("Nashville"), so the right place tiers to the top instead of a same-named
+ *   town in another state. Opt-in — the lookup is only constructed when a path is supplied, and
+ *   absent it the resolver is byte-identical.
+ *
+ *   The reader returns RAW divergent rows for a postcode; normalization + name-matching against the
+ *   candidate localities is the scorer's job (it owns the case/diacritic fold the soft name score
+ *   uses), keeping one normalizer in one place.
+ */
+
+import { DatabaseSync } from "node:sqlite"
+
+import { DatabaseClient } from "@mailwoman/core/kysley/client"
+
+import type { PostalCityAliasDatabase } from "./postal-city-alias-schema.ts"
+
+export interface WOFPostalCityAliasLookupOpts {
+	/** Path to a `postal-city-alias-<cc>.db` built by `build-postal-city-alias.ts`. Opened read-only. */
+	databasePath?: string
+	/** Pre-opened handle (tests / shared connections). Mutually exclusive with `databasePath`. */
+	database?: DatabaseSync
+}
+
+/** One divergent alias edge: the postal-system name and the geographic locality it maps to. */
+export interface PostalCityAlias {
+	/** The postal-system surface (what a user types). */
+	postalCity: string
+	/** The geographic locality name the postcode sits in (≈ the gazetteer's canonical name). */
+	geoLocality: string
+	/** Observed usage count — the evidence weight. */
+	n: number
+}
+
+/**
+ * Reader over `postal_city_alias`. The only query is a postcode-scoped probe for DIVERGENT rows (where the postal name
+ * differs from the geographic name — the rows that carry alias signal), issued via the typed Kysely query builder
+ * against {@link PostalCityAliasDatabase}.
+ */
+export class WOFPostalCityAliasLookup {
+	#db: DatabaseSync
+	#kdb: DatabaseClient<PostalCityAliasDatabase>
+	#ownsDB: boolean
+
+	constructor(opts: WOFPostalCityAliasLookupOpts) {
+		if (opts.database) {
+			this.#db = opts.database
+			this.#ownsDB = false
+		} else if (opts.databasePath) {
+			this.#db = new DatabaseSync(opts.databasePath, { readOnly: true })
+			this.#ownsDB = true
+		} else {
+			throw new Error("WOFPostalCityAliasLookup needs `databasePath` or `database`")
+		}
+		// `#kdb` wraps `#db` for the typed query; close() owns the raw handle directly (sync).
+		this.#kdb = new DatabaseClient<PostalCityAliasDatabase>({ database: this.#db })
+	}
+
+	/**
+	 * Divergent postal-city aliases for a postcode (empty when the postcode isn't in the table). The scorer groups these
+	 * by normalized `geoLocality` and appends the `postalCity` surfaces to the matching candidate locality's alias set.
+	 */
+	async getDivergentAliases(postcode: string): Promise<PostalCityAlias[]> {
+		const pc = postcode.trim()
+
+		if (!pc) return []
+		const rows = await this.#kdb
+			.selectFrom("postal_city_alias")
+			.select(["postal_city", "geo_locality", "n"])
+			.where("postcode", "=", pc)
+			.where("divergent", "=", 1)
+			.execute()
+
+		return rows.map((r) => ({ postalCity: String(r.postal_city), geoLocality: String(r.geo_locality), n: Number(r.n) }))
+	}
+
+	close(): void {
+		if (this.#ownsDB) {
+			this.#db.close()
+		}
+	}
+}

package/postal-city-alias-schema.ts

@@ -0,0 +1,75 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Typed schema for the POSTAL-CITY ALIAS table (`postal-city-alias-<cc>.db`, built by
+ *   `scripts/build-postal-city-alias.ts`) — the single source of truth for the columns shared by
+ *   the BUILDER and the READER ({@link WOFPostalCityAliasLookup}). Like {@link CandidateTable}, the
+ *   contract is a Kysely `Database` interface plus the table DDL as a string, so a column rename in
+ *   the builder is a compile error in the reader.
+ *
+ *   Provenance discipline (provenance-first): this is a SIBLING table to the PIP-derived
+ *   `postcode_locality` data, never mixed into it — one table, one provenance class. Each row is an
+ *   OBSERVED `(postcode, postal_city, geo_locality)` aggregate from Overture's `postal_city` field
+ *   with a usage count `n`; `divergent = 1` exactly when `postal_city != geo_locality` (the alias
+ *   signal — the only rows the resolver consumes).
+ */
+
+import type { Kysely } from "kysely"
+
+/**
+ * One observed postal-city aggregate. The natural key is `(postcode, postal_city, geo_locality)`; the builder enforces
+ * a `MIN_COUNT` floor on `n`, so every row is a non-trivial usage.
+ */
+export interface PostalCityAliasTable {
+	/** The postcode the aggregate is scoped to (the resolver probes by this). */
+	postcode: string
+	/** What the postal system calls the place (the surface a user is likely to type). */
+	postal_city: string
+	/** The geographic locality name the postcode actually sits in (≈ the gazetteer's canonical name). */
+	geo_locality: string
+	/** Observed row count — the evidence weight behind this alias. */
+	n: number
+	/** 1 when `postal_city != geo_locality` (the alias signal); 0 when they agree. */
+	divergent: number
+	/** Provenance: the dataset this aggregate came from (e.g. `overture:US`). */
+	source: string
+	/** The pinned data release the aggregate was computed from. */
+	release: string
+}
+
+/** The postal-city-alias database schema for `new DatabaseClient<PostalCityAliasDatabase>(...)`. */
+export interface PostalCityAliasDatabase {
+	postal_city_alias: PostalCityAliasTable
+}
+
+/** The `postal_city_alias` column order — the builder's INSERT derives its column list from this. */
+export const POSTAL_CITY_ALIAS_COLUMNS = [
+	"postcode",
+	"postal_city",
+	"geo_locality",
+	"n",
+	"divergent",
+	"source",
+	"release",
+] as const
+
+/**
+ * Create the `postal_city_alias` table + its two probe indexes. Kept here (not only in the builder) so tests can stand
+ * up a fixture DB with the exact production shape. Pass a {@link DatabaseClient} (or any `Kysely`) over the alias DB.
+ */
+export async function createPostalCityAliasTable(db: Kysely<PostalCityAliasDatabase>): Promise<void> {
+	await db.schema
+		.createTable("postal_city_alias")
+		.addColumn("postcode", "text", (c) => c.notNull())
+		.addColumn("postal_city", "text", (c) => c.notNull())
+		.addColumn("geo_locality", "text", (c) => c.notNull())
+		.addColumn("n", "integer", (c) => c.notNull())
+		.addColumn("divergent", "integer", (c) => c.notNull())
+		.addColumn("source", "text", (c) => c.notNull())
+		.addColumn("release", "text", (c) => c.notNull())
+		.execute()
+	await db.schema.createIndex("idx_pca_postcode").on("postal_city_alias").column("postcode").execute()
+	await db.schema.createIndex("idx_pca_pair").on("postal_city_alias").columns(["postal_city", "geo_locality"]).execute()
+}

package/postal-city-candidate-schema.ts

@@ -0,0 +1,81 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Typed schema for the POSTAL-CITY CANDIDATE side-index (#741 / #475) — a small `(name_key,
+ *   postcode) → geo-locality` table that lives alongside the byte-range `candidate` table so the
+ *   candidate-backend resolver (the demo/CLI default) can do what the FTS coordinate-first scorer
+ *   does: resolve a user-typed POSTAL city ("Antioch", 37013) to the geographic locality the
+ *   postcode sits in ("Nashville").
+ *
+ *   Why a SIDE-INDEX, not cloned `candidate` rows: the `candidate` B-tree is keyed `(name_key,
+ *   country_id, region_id, placetype_id, …)` and ranked population-first — it has no postcode
+ *   dimension. A cloned alias row was tested (#741) and falsified: a sentinel rank is
+ *   bare-name-safe but then loses to any in-region homonym, and there is no single rank that is
+ *   both. The fix is an EXACT `(name_key, postcode)` probe that bypasses population/region ranking
+ *   entirely — consulted only when the query carries a postcode, so the common no-postcode path is
+ *   untouched.
+ */
+
+import { sql, type Kysely } from "kysely"
+
+/**
+ * One postal-city → geo-locality edge, keyed exactly by `(name_key, postcode)`. The probe returns the geographic
+ * locality directly; the denormalized name/coord avoid a join back to `candidate`.
+ */
+export interface PostalCityCandidateTable {
+	/** {@link normalizeLocalityForKey} of the postal-city name — the build/query-consistent probe key. */
+	name_key: string
+	/** The postcode the alias is scoped to (the second half of the exact key). */
+	postcode: string
+	/** WOF id of the geographic locality the postcode sits in (the resolve target). */
+	spr_id: number
+	/** The geographic locality's display name. */
+	name: string
+	latitude: number
+	longitude: number
+}
+
+/**
+ * The postal-city-candidate database schema for `new DatabaseClient<PostalCityCandidateDatabase>(...)`.
+ */
+export interface PostalCityCandidateDatabase {
+	postal_city_candidate: PostalCityCandidateTable
+}
+
+/**
+ * The table name the lookup probes (existence-gated, so an old candidate.db without it is byte-stable).
+ */
+export const POSTAL_CITY_CANDIDATE_TABLE = "postal_city_candidate"
+
+/** Column order for the builder's positional INSERT. */
+export const POSTAL_CITY_CANDIDATE_COLUMNS = [
+	"name_key",
+	"postcode",
+	"spr_id",
+	"name",
+	"latitude",
+	"longitude",
+] as const
+
+/**
+ * Create the side-index — a clustered `WITHOUT ROWID` B-tree on `(name_key, postcode)` so the resolve is a single exact
+ * probe. Idempotent (`IF NOT EXISTS`); pass a {@link DatabaseClient} (or any `Kysely`) over the candidate DB. The
+ * Kysely schema-builder is the house idiom for table creation — see `AGENTS.md` (inline-SQL → Kysely).
+ */
+export async function createPostalCityCandidateTable(db: Kysely<PostalCityCandidateDatabase>): Promise<void> {
+	await db.schema
+		.createTable(POSTAL_CITY_CANDIDATE_TABLE)
+		.ifNotExists()
+		.addColumn("name_key", "text", (c) => c.notNull())
+		.addColumn("postcode", "text", (c) => c.notNull())
+		.addColumn("spr_id", "integer", (c) => c.notNull())
+		.addColumn("name", "text", (c) => c.notNull())
+		.addColumn("latitude", "real", (c) => c.notNull())
+		.addColumn("longitude", "real", (c) => c.notNull())
+		.addPrimaryKeyConstraint("postal_city_candidate_pk", ["name_key", "postcode"])
+		// `WITHOUT ROWID` has no first-class builder; the raw modifier is the idiomatic fallback.
+		.modifyEnd(sql`without rowid`)
+		.execute()
+}

package/postcode-point-lookup.ts

@@ -0,0 +1,64 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   SQLite-backed postcode lookup for the postcode anchor (#240). A thin exact-match resolver over
+ *   one or more `postalcode-*.db` shards (the `spr` schema built by `build-unified-wof --placetypes
+ *   postalcode`, then centroid-backfilled by `scripts/backfill-postcode-centroids.ts`).
+ *
+ *   This is the production implementation of the `PostcodeResolver` interface consumed by
+ *   `@mailwoman/neural`'s `extractPostcodeAnchors`. It is deliberately dumb: an indexed exact-match
+ *   on the postcode string across every shard, unioned. No FTS, no ranking, no proximity — the
+ *   anchor only needs "does this string exist as a postcode, in which countries, near where". A
+ *   future WASM build swaps this for an FST-backed resolver behind the same `lookup()` seam.
+ *
+ *   Why multiple shards instead of the multi-shard `WOFSqlitePlaceLookup`: that resolver routes a
+ *   query to ONE shard by placetype, but every postcode shard shares `placetype='postalcode'`, so a
+ *   single query could only ever hit one country's shard. The anchor needs the union across
+ *   countries to build its country posterior, so it queries each shard directly.
+ */
+
+import { DatabaseSync } from "node:sqlite"
+
+/**
+ * A gazetteer hit. `lat`/`lon` of 0 means the postcode is known but has no centroid (no admin parent).
+ */
+export interface PostcodePlace {
+	country: string
+	lat: number
+	lon: number
+}
+
+const LOOKUP_SQL =
+	"SELECT country, latitude AS lat, longitude AS lon FROM spr WHERE name = ? AND placetype = 'postalcode' AND is_current != 0"
+
+export class WOFPostcodeLookup {
+	readonly #dbs: DatabaseSync[]
+	readonly #stmts: ReturnType<DatabaseSync["prepare"]>[]
+
+	/** Open each shard read-only and prepare its exact-match statement. */
+	constructor(dbPaths: readonly string[]) {
+		this.#dbs = dbPaths.map((p) => new DatabaseSync(p, { readOnly: true }))
+		this.#stmts = this.#dbs.map((db) => db.prepare(LOOKUP_SQL))
+	}
+
+	/** Exact-match the postcode across every shard and union the rows. */
+	lookup(postcode: string): PostcodePlace[] {
+		const out: PostcodePlace[] = []
+
+		for (const stmt of this.#stmts) {
+			for (const row of stmt.all(postcode)) {
+				out.push({ country: String(row.country), lat: Number(row.lat), lon: Number(row.lon) })
+			}
+		}
+
+		return out
+	}
+
+	close(): void {
+		for (const db of this.#dbs) {
+			db.close()
+		}
+	}
+}