@mailwoman/resolver-wof-sqlite 9.1.0 → 9.2.0

package/candidate/place-attrs.ts

@@ -0,0 +1,44 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The per-place record every candidate-staging pass writes its rows from.
+ *
+ *   Pass 1 reduces each current `spr` row to one {@link PlaceAttrs}; every later pass (alias bags,
+ *   region abbreviations, country display names, the currency backfill, the shard folds) discovers
+ *   ADDITIONAL name keys for a place already in that map and stages a row against the same record.
+ *   That is what keeps each candidate row denormalized without re-reading the source, and it is why
+ *   a pass needs exactly four things to stage: the key it found, the place, the id the row hangs on,
+ *   and whether the key is the place's canonical name.
+ */
+
+export interface PlaceAttrs {
+	cid: number
+	rid: number
+	ptid: number
+	name: string
+	lat: number
+	lon: number
+	mnLat: number
+	mnLon: number
+	mxLat: number
+	mxLon: number
+	pop: number
+	neg: number
+	pkey: string
+	/**
+	 * The place's toponym-fame score, or null when the score source has no measurement for it (#28). A property of the
+	 * PLACE, so it rides {@link StageRow} onto the alias and abbrev rows too — that is how a bare `Moscow` reaches
+	 * Москва's score through the alias row that carries the key.
+	 */
+	imp: number | null
+}
+
+/**
+ * Stage one candidate row: a normalized name key, the place it belongs to, the source id the row hangs on, and whether
+ * the key is that place's canonical name (`is_primary`).
+ *
+ * `sid` is passed separately rather than read off the place because a shard fold and the alias pass stage rows for ids
+ * the admin `attrs` map never held.
+ */
+export type StageRow = (k: string, a: PlaceAttrs, sid: number, isPrimary: number) => void

package/candidate/shard-fold.ts

@@ -0,0 +1,137 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Pass 4 of the candidate build — fold a postcode or locality shard into the staging table.
+ */
+
+import { DatabaseSync } from "node:sqlite"
+
+import { normalizeLocalityForKey } from "../street-normalize.ts"
+import type { PlaceAttrs, StageRow } from "./place-attrs.ts"
+
+/**
+ * Fold ONE shard (`spr` rows at `shardPlacetype` carrying real coordinates) in, then pass 4b: the alias names hanging
+ * off that same shard's `names` table.
+ *
+ * Self-contained by construction — it shares only the staging statement and the code dictionaries with the admin passes
+ * above it, and nothing downstream reads anything it produces except the two counters it returns.
+ */
+export function foldShard(ctx: {
+	/**
+	 * The staging connection. The shard itself is opened read-only here and closed before returning.
+	 */
+	out: DatabaseSync
+	shardPath: string
+	shardPlacetype: "postalcode" | "locality"
+	ccID: (code: string | null) => number
+	ptID: (pt: string | null) => number
+	stageRow: StageRow
+	progress: (phase: string, message: string) => void
+}): { primaries: number; aliases: number } {
+	const { out, shardPath, shardPlacetype, ccID, ptID, stageRow, progress } = ctx
+
+	progress(shardPlacetype === "postalcode" ? "postcodes" : "localities", `reading ${shardPath}`)
+
+	const pc = new DatabaseSync(shardPath, { readOnly: true })
+	const pcPtid = ptID(shardPlacetype)
+	// Per-shard, not the admin `attrs` map: pass 1 only ever sees the admin DB, so the alias pass
+	// below has nothing to join against unless this primary loop records what it staged.
+	const pcAttrs = new Map<number, PlaceAttrs>()
+	let primaries = 0
+	let aliases = 0
+
+	out.exec("BEGIN")
+
+	for (const r of pc
+		.prepare(
+			`SELECT id, name, country, latitude, longitude,
+				min_latitude AS mnlat, min_longitude AS mnlon, max_latitude AS mxlat, max_longitude AS mxlon
+			 FROM spr WHERE placetype = ? AND latitude != 0 AND longitude != 0`
+		)
+		.iterate(shardPlacetype)) {
+		const name = String(r.name ?? "")
+		const key = normalizeLocalityForKey(name)
+
+		if (!key) continue
+
+		const lat = r.latitude as number
+		const lon = r.longitude as number
+
+		// region_id 0 (a postcode is unique by name+country — no same-name disambiguation); neg_rank 0
+		// (no population). bbox = the postcode's own min/max (falls back to the centroid point).
+		const a: PlaceAttrs = {
+			cid: ccID(r.country as string | null),
+			rid: 0,
+			ptid: pcPtid,
+			name,
+			lat,
+			lon,
+			mnLat: (r.mnlat as number) || lat,
+			mnLon: (r.mnlon as number) || lon,
+			mxLat: (r.mxlat as number) || lat,
+			mxLon: (r.mxlon as number) || lon,
+			pop: 0,
+			neg: 0,
+			pkey: key,
+			// A postcode has no toponym fame — nobody writes an encyclopedia article about SW1A 2AA — and
+			// the score source carries no `postalcode` rows to join against anyway. NULL is the truthful
+			// value: unmeasured, so the ranking key leaves postcode rows exactly where they were.
+			imp: null,
+		}
+
+		pcAttrs.set(Number(r.id), a)
+		stageRow(key, a, Number(r.id), 1)
+
+		primaries++
+	}
+
+	out.exec("COMMIT")
+
+	// --- pass 4b: postcode ALIAS names (#1495) ---
+	//
+	// The delivery-city names GeoNames supplies for a ZIP ("Brooklyn" for 11201) are written into
+	// the shard's `names` table by `postcode/centroid-fills.ts`'s `geonamesNameFill`. Everything
+	// downstream of `names` picked them up EXCEPT this build: `fts.ts` unions `spr.name` with every
+	// `names` row into `place_search.alt_names`, so the FTS backend resolved "Brooklyn" → 11201
+	// while the candidate backend — whose every row IS an exact-tier row — had no key for it at
+	// all. Pass 2 does the equivalent fold for admin places, but reads the ADMIN `place_search`,
+	// and `attrs` holds admin ids only, so a postcode shard could never reach it.
+	//
+	// Same discipline as pass 2: `is_primary = 0` (so `rankByPrimaryPreference` treats it as an
+	// alias, not a canonical postcode name), the row stays denormalized onto the POSTCODE's own
+	// spr_id/coords/bbox, and the display `name` stays the postcode — resolving "brooklyn" answers
+	// with place 11201, it does not rename the place to its delivery city.
+	const hasNames = pc.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='names'").get() !== undefined
+
+	if (hasNames) {
+		out.exec("BEGIN")
+
+		for (const r of pc.prepare("SELECT id, name FROM names").iterate()) {
+			const a = pcAttrs.get(Number(r.id))
+
+			if (!a) continue
+
+			const k = normalizeLocalityForKey(String(r.name ?? ""))
+
+			// The postcode's own key is already staged as the primary; `INSERT OR IGNORE` at
+			// materialization dedupes repeats, so this only skips the obvious self-alias.
+			if (!k || k === a.pkey) continue
+
+			stageRow(k, a, Number(r.id), 0)
+
+			aliases++
+		}
+
+		out.exec("COMMIT")
+	} else {
+		// Never a silent zero: real shards come from `createUnifiedSchema`, which always creates
+		// `names`. A shard without it has no alias surface to lose, but say so rather than reporting
+		// "0 aliases" from a table that was never read.
+		progress("postcode-aliases", `${shardPath} has no \`names\` table — no delivery-city aliases to fold`)
+	}
+
+	pc.close()
+
+	return { primaries, aliases }
+}

package/candidate-ancestors-schema.ts

@@ -0,0 +1,195 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Typed schema for the candidate gazetteer's ANCESTORS sidecar (`candidate_ancestor` +
+ *   `candidate_interval`) — the containment lineage the candidate table itself cannot answer, built
+ *   into the same `candidate.db` by `build-candidate.ts` and read by
+ *   {@link WOFCandidateTableLookup.ancestors}.
+ *
+ *   ENCODING: closure-lite rows, denormalized — one row per (place, ancestor) edge carrying the
+ *   parent's placetype, display name and folded key — rather than a fixed-slot id chain on the
+ *   candidate row. Decided by the two consumers:
+ *
+ *   1. The admin-coherence check needs the WINNER's chain as (placetype, name) pairs in one probe.
+ *      A fixed-slot `[id;8]` chain answers with ids, and every id then needs a name lookup the
+ *      artifact has no per-id table for — up to 8 indirections where the closure row has zero.
+ *   2. The account layer needs EVERY candidate under a `name_key` enumerable WITH its chain from
+ *      one artifact probe ("present-but-outranked, discriminated by containment"). That is the
+ *      candidate probe (contiguous) followed by one `spr_id`-clustered closure probe per candidate
+ *      — each a handful of adjacent pages.
+ *
+ *   Denormalizing the parent onto the edge is the same discipline as the candidate table itself:
+ *   this artifact is read over HTTP byte ranges, where a join to a dimension table scatters page
+ *   fetches, and a `WITHOUT ROWID` B-tree clustered on `(spr_id, depth)` keeps a whole chain in
+ *   1-2 pages. The repeated parent strings are the price of the zero-join read, paid at build time.
+ *
+ *   `depth` is 1 for the NEAREST ancestor (deepest containment tier), increasing outward to the
+ *   country — the same nearest-first order `ancestry.ts` serves for the FTS backend, so the two
+ *   backends' `ancestors()` agree by construction. The order within a place is deterministic:
+ *   containment depth descending (`placetypeDepth`), then ancestor id ascending, capped at
+ *   {@link MAX_ANCESTOR_DEPTH}. `parent_name_key` is the SHARED {@link normalizeLocalityForKey}
+ *   fold — the same fold `candidate.name_key` is built with, so a chain entry and a candidate key
+ *   compare under one normalizer.
+ *
+ *   `candidate_interval` carries pre/post-order labels over the CANONICAL-PARENT FOREST, assigned
+ *   at build time: `a` contains `d` ⟺ `a.pre <= d.pre AND d.post <= a.post` — O(1) in either
+ *   direction with no chain scan and no knowledge of either side's tier — and the descendants of
+ *   `a` are the contiguous range `pre BETWEEN a.pre AND a.post`. Interval labels are classically
+ *   avoided for their relabel-on-update cost; this database is a sealed read-only artifact rebuilt
+ *   whole, which is exactly the regime where that cost is void.
+ *
+ *   THE DAG CAVEAT, and the recorded choice: WOF places can carry more than one parent (multiple
+ *   hierarchies, ambiguous boundaries). `candidate_ancestor` keeps EVERY parent — the closure rows
+ *   are the complete containment record. A single interval pair can only encode a tree, so the
+ *   interval forest links each place to ONE canonical parent: its depth-1 edge — the finest
+ *   containment tier, lowest ancestor id — the same MIN-stability convention the candidate table's
+ *   `region_id` stamp uses. A containment question about a NON-canonical hierarchy must consult the
+ *   closure rows; the interval answer for it is `false`, which is why interval verdicts are
+ *   "contained along the canonical hierarchy", never "not contained at all".
+ *
+ *   ABSENCE SEMANTICS (meaning-of-zero): a place with no `candidate_interval` row has no recorded
+ *   ancestry in the source (shard-fed postcodes and localities, isolated places, cycle-skipped
+ *   rows). Absence is UNVERIFIABLE, never a containment verdict.
+ */
+
+import { sql, type Kysely } from "kysely"
+
+// Type-only and circular on purpose (candidate-schema extends CandidateAncestorsDatabase): Kysely's
+// DB parameter is invariant, so the DDL functions must take the FULL database type their caller
+// holds. Erased at runtime.
+import type { CandidateDatabase } from "./candidate-schema.ts"
+import type { NameKey } from "./street-normalize.ts"
+
+/**
+ * The deepest chain the sidecar stores per place. WOF containment within the resolvable placetypes (country …
+ * microhood) never legitimately exceeds this; anything past it is source noise the build drops (and counts) rather than
+ * stores.
+ */
+export const MAX_ANCESTOR_DEPTH = 8
+
+/**
+ * The closure-row table's name — probed with `hasTable` by the reader, so an artifact predating the sidecar degrades to
+ * "no ancestors capability" rather than `no such table`.
+ */
+export const CANDIDATE_ANCESTOR_TABLE = "candidate_ancestor"
+
+/**
+ * The interval-label table's name — the closure table's seal-time sibling; existence-gated the same way.
+ */
+export const CANDIDATE_INTERVAL_TABLE = "candidate_interval"
+
+/**
+ * One (place, ancestor) edge, denormalized so a chain read is a single clustered probe (no join).
+ */
+export interface CandidateAncestorTable {
+	/**
+	 * WOF id of the place whose chain this row belongs to — the same `spr_id` the candidate table resolves to.
+	 */
+	spr_id: number
+	/**
+	 * 1 = nearest ancestor, increasing outward to the country. Deterministic within a place (containment depth
+	 * descending, then ancestor id ascending), so `(spr_id, depth)` is a stable primary key across rebuilds.
+	 */
+	depth: number
+	/**
+	 * WOF id of the ancestor.
+	 */
+	parent_spr_id: number
+	/**
+	 * Small int from the shared `placetype_codes` dictionary (the same one the candidate table uses).
+	 */
+	parent_placetype_id: number
+	/**
+	 * The ancestor's canonical display name — what {@link Ancestor.name} serves, matching the FTS backend's register.
+	 */
+	parent_name: string
+	/**
+	 * The SHARED {@link normalizeLocalityForKey} fold of `parent_name` — comparable against `candidate.name_key` and
+	 * against a query-side fold under one normalizer, by construction.
+	 */
+	parent_name_key: NameKey
+}
+
+/**
+ * Pre/post-order labels over the canonical-parent forest — one row per place WITH recorded ancestry (see the module
+ * docstring for absence semantics). `pre < post` always; labels are unique across the artifact.
+ */
+export interface CandidateIntervalTable {
+	spr_id: number
+	pre: number
+	post: number
+}
+
+/**
+ * The sidecar tables, for a `Kysely` view over the candidate DB. `CandidateDatabase` (candidate-schema.ts) extends
+ * this, so the builder's one typed client sees both families.
+ */
+export interface CandidateAncestorsDatabase {
+	candidate_ancestor: CandidateAncestorTable
+	candidate_interval: CandidateIntervalTable
+}
+
+/**
+ * The `candidate_ancestor` columns in clustered-key order — the first two ARE the primary key, and the builder's
+ * positional `INSERT` binds by this order. Keep in sync with {@link CandidateAncestorTable}.
+ */
+export const CANDIDATE_ANCESTOR_COLUMNS = [
+	"spr_id",
+	"depth",
+	"parent_spr_id",
+	"parent_placetype_id",
+	"parent_name",
+	"parent_name_key",
+] as const
+
+/**
+ * Create the clustered closure table. The builder inserts in `(spr_id, depth)` order so the B-tree leaves are
+ * contiguous per place — the byte-range read discipline.
+ */
+export async function createCandidateAncestorTable(db: Kysely<CandidateDatabase>): Promise<void> {
+	await db.schema
+		.createTable(CANDIDATE_ANCESTOR_TABLE)
+		.addColumn("spr_id", "integer", (c) => c.notNull())
+		.addColumn("depth", "integer", (c) => c.notNull())
+		.addColumn("parent_spr_id", "integer", (c) => c.notNull())
+		.addColumn("parent_placetype_id", "integer", (c) => c.notNull())
+		.addColumn("parent_name", "text", (c) => c.notNull())
+		.addColumn("parent_name_key", "text", (c) => c.notNull())
+		.addPrimaryKeyConstraint("candidate_ancestor_pk", ["spr_id", "depth"])
+		// `WITHOUT ROWID` has no first-class builder; the raw modifier is the idiomatic fallback.
+		.modifyEnd(sql`without rowid`)
+		.execute()
+}
+
+/**
+ * Create the interval-label table. Small rows probed by primary key — the `WITHOUT ROWID` win case.
+ */
+export async function createCandidateIntervalTable(db: Kysely<CandidateDatabase>): Promise<void> {
+	await db.schema
+		.createTable(CANDIDATE_INTERVAL_TABLE)
+		.addColumn("spr_id", "integer", (c) => c.primaryKey())
+		.addColumn("pre", "integer", (c) => c.notNull())
+		.addColumn("post", "integer", (c) => c.notNull())
+		.modifyEnd(sql`without rowid`)
+		.execute()
+}
+
+/**
+ * One place's interval label — the shape both sides of a containment comparison read.
+ */
+export interface IntervalLabel {
+	pre: number
+	post: number
+}
+
+/**
+ * Does `outer` contain `inner` along the canonical hierarchy? Reflexive: a place contains itself (containment
+ * degenerates to identity, the same reading the admin-coherence check gives a self-match). The shared FUNCTION for
+ * every consumer of the labels — a re-derived comparison risks disagreeing at exactly the strict/inclusive boundary
+ * this line settles.
+ */
+export function intervalContains(outer: IntervalLabel, inner: IntervalLabel): boolean {
+	return outer.pre <= inner.pre && inner.post <= outer.post
+}

package/candidate-fts.ts

@@ -9,8 +9,10 @@
  *   raw `name`), so a diacritic-stripped query (`munchen`) trigram-matches the stored `munchen`
  *   rather than missing a raw `München`. The trigram tokenizer makes MATCH a substring/fuzzy
  *   operation; the reader ({@link WOFCandidateTableLookup}) OR's the query's trigrams to fetch a
- *   loose set, then re-ranks by the SAME `trigramJaccard` the admin/FTS backend uses, so a typo
- *   resolves identically on either.
+ *   loose set, then re-ranks it with a WORD-level similarity. The trigram index is the candidate
+ *   GENERATOR, not the scorer — trigram Jaccard scores a true transposition correction BELOW a wrong
+ *   answer, because shared generic suffixes count as evidence and transpositions count against it.
+ *   The receipts are on `candidate-lookup.ts`'s `FUZZY_FETCH` and `WORD_FUZZY_MIN`.
  *
  *   This is what unifies the two gazetteers: the candidate B-tree stays the common,
  *   byte-range-optimal fast path (the browser's contiguous probe), and FTS5 is consulted ONLY on an

package/candidate-importance.ts

@@ -43,7 +43,8 @@
 
 import { DatabaseSync } from "node:sqlite"
 
-import { haversineKm } from "./geo.ts"
+import { haversineKm } from "@mailwoman/spatial"
+
 import { normalizeLocalityForKey } from "./street-normalize.ts"
 
 /**