@mailwoman/resolver-wof-sqlite 9.2.0 → 9.3.0

package/{coverage-manifest-schema.ts → lib/coverage-manifest-schema.ts}

@@ -8,9 +8,9 @@
  *   updated at gazetteer REBUILD, not by a hand-edited code PR after someone remembers:
  *
  *   - `country_coverage`: the hard-country-filter coverage record (#743/#194) — per-country
- *     promote-gate verdicts + the measured hard-resolve rates that used to live in a code comment on
+ *     promotion-eval verdicts + the measured hard-resolve rates that used to live in a code comment on
  *     `HARD_PLACE_COUNTRY_SAFELIST`. Presence = measured; `hard_filter_safe = 0` = measured and
- *     FAILED the gate (FI 69.5%, PL 77.8%) — distinguishable from a country never measured at all
+ *     FAILED the check (FI 69.5%, PL 77.8%) — distinguishable from a country never measured at all
  *     (the meaning-of-zero rule, `docs/engineering/reference/layer-contract.mdx`).
  *   - `country_bbox`: the coarse guard-B plausibility boxes that used to live in
  *     `resolver/plausibility.ts`'s `COUNTRY_BBOX`. An absent row fails open (never trips the guard),
@@ -23,17 +23,16 @@
  *   manifest returns `undefined` and every consumer falls back to the code constants byte-identically.
  */
 
-import type { DatabaseSync } from "node:sqlite"
-
 import {
 	hardCountrySafelistFromCoverage,
 	type CountryBBoxFact,
 	type CountryCoverageFact,
 	type GazetteerArtifactCoverage,
 } from "@mailwoman/core/resolver"
+import type { DatabaseClient } from "@mailwoman/sqlite/client"
 import { sql, type Kysely } from "kysely"
 
-import { allRows, hasTable } from "./sqlite-utils.ts"
+import { allRows, hasTable } from "#sqlite-utils"
 
 /**
  * One country's hard-filter coverage measurement — the storage form of {@link CountryCoverageFact}.
@@ -44,7 +43,7 @@ export interface CountryCoverageTable {
 	 */
 	country: string
 	/**
-	 * 0/1 — the promote-gate verdict (a verdict column, NOT re-derived from the rate; see the fact type's docstring).
+	 * 0/1 — the promotion-eval verdict (a verdict column, NOT re-derived from the rate; see the fact type's docstring).
 	 */
 	hard_filter_safe: number
 	/**
@@ -56,11 +55,11 @@ export interface CountryCoverageTable {
 	 */
 	sample_size: number | null
 	/**
-	 * ISO-8601 date of the measurement / promote gate.
+	 * ISO-8601 date of the measurement / promote eval.
 	 */
 	measured_at: string
 	/**
-	 * The receipt: which panel/gate produced this row.
+	 * The receipt: which panel/check produced this row.
 	 */
 	source: string
 }
@@ -92,7 +91,7 @@ export interface GazetteerCoverageDatabase {
 }
 
 /**
- * Table names the lookup probes (existence-gated, so a candidate.db built before the manifest is byte-stable).
+ * Table names the lookup probes (existence-restricted, so a candidate.db built before the manifest is byte-stable).
  */
 export const COUNTRY_COVERAGE_TABLE = "country_coverage"
 /**
@@ -185,7 +184,7 @@ export async function writeGazetteerCoverageManifest(
  * purpose: this runs inside {@link WOFCandidateTableLookup}'s synchronous constructor (the sync-reader carve-out in
  * `AGENTS.md`), and the tables are a few dozen rows read once per open.
  */
-export function readGazetteerCoverageManifest(db: DatabaseSync): GazetteerArtifactCoverage | undefined {
+export function readGazetteerCoverageManifest<DB>(db: DatabaseClient<DB>): GazetteerArtifactCoverage | undefined {
 	const hasCoverage = hasTable(db, COUNTRY_COVERAGE_TABLE)
 	const hasBBox = hasTable(db, COUNTRY_BBOX_TABLE)
 

package/{currency-backfill.ts → lib/currency-backfill.ts}

@@ -4,22 +4,23 @@
  * @author Teffen Ellis, et al.
  *
  *   Pass 1c of the candidate build (#1737): resurrect deprecated-with-no-successor WOF localities that a second
- *   source independently attests. Extracted from `build-candidate.ts` as a cohesive unit — the gates, their measured
+ *   source independently attests. Extracted from `build-candidate.ts` as a cohesive unit — the checks, their measured
  *   constants, and the GeoNames dump reader live together here; the build calls {@link resurrectCurrencyHoles} once,
  *   between the primaries pass and the alias pass.
  */
 
-import { existsSync } from "node:fs"
-import { resolve } from "node:path"
-import type { DatabaseSync } from "node:sqlite"
-
+import { pathExists } from "@mailwoman/core/fs/readers"
 import { isStrictlyFiner } from "@mailwoman/core/resources/whosonfirst"
 import { haversineKm } from "@mailwoman/spatial"
+import type { DatabaseClient } from "@mailwoman/sqlite/client"
+import { resolvePath } from "path-ts"
 import { TSVSpliterator } from "spliterator"
 
-import type { loadImportanceIndex } from "./candidate-importance.ts"
-import type { PlaceAttrs, StageRow } from "./candidate/place-attrs.ts"
-import { normalizeLocalityForKey } from "./street-normalize.ts"
+import type { loadImportanceIndex } from "#candidate-importance"
+import type { CandidateDatabase } from "#candidate-schema"
+import type { PlaceAttrs, StageRow } from "#candidate/place-attrs"
+import type { WOFDatabase } from "#schema"
+import { normalizeLocalityForKey } from "#street/normalize"
 
 /**
  * Corroboration radius for the currency backfill (#1737), km — both for the live-near blocker and the GeoNames
@@ -43,7 +44,7 @@ const CURRENCY_BACKFILL_POP_FLOOR = 1000
  * WOF deprecations WITH a successor need nothing — the successor indexes. A deprecation with `is_superseded = 0` on a
  * populated place is the shape of an upstream mistake (Rochester Kent, Aldershot, Telford; 120 GB localities alone),
  * and it is indistinguishable from a correct pruning at this layer without outside evidence. So every resurrection
- * requires all three gates, positive evidence only:
+ * requires all three conditions, positive evidence only:
  *
  * 1. NO live same-name spr row of any placetype within {@link CURRENCY_BACKFILL_RADIUS_KM} of the dead record — a live row
  *    means the place is alive (possibly under another placetype) and there is no hole. A DISTANT same-name row is a
@@ -56,9 +57,48 @@ const CURRENCY_BACKFILL_POP_FLOOR = 1000
  * lets the row stand in prominence races (the dead record's own population is absent). Each name is judged once per
  * country; the resurrected place joins `attrs`, so the alias pass explodes its alt names like any primary's.
  */
+export interface CurrencyBackfillOutcomes {
+	judged: number
+	blocked: number
+	unattested: number
+	floored: number
+	resurrected: number
+}
+
+/**
+ * One country's read of the check — every dead name judged, by outcome and by the dead row's placetype — so a census
+ * can say what a wider dead-row query admits before a build carries it.
+ */
+export interface CurrencyBackfillCountryReport extends CurrencyBackfillOutcomes {
+	country: string
+	deadPlacetypes: readonly string[]
+	dumpPresent: boolean
+	byDeadPlacetype: Record<string, CurrencyBackfillOutcomes>
+	/**
+	 * The first resurrected names, `placetype:name`, so a census reads what a wider query admits — capped, because the
+	 * report is a receipt and not the table.
+	 */
+	sample: string[]
+}
+
+const REPORT_SAMPLE_SIZE = 25
+
+/**
+ * The placetypes a dead row may carry to be judged at all. `locality` is the shipped default; `localadmin` is the
+ * second cause #1746 named (one GB row of seventeen) and is admitted through this option once its census is read.
+ */
+export const DEFAULT_DEAD_PLACETYPES: readonly string[] = ["locality"]
+
+function emptyOutcomes(): CurrencyBackfillOutcomes {
+	return { judged: 0, blocked: 0, unattested: 0, floored: 0, resurrected: 0 }
+}
+
 export async function resurrectCurrencyHoles(ctx: {
-	src: DatabaseSync
-	tx: DatabaseSync
+	src: DatabaseClient<WOFDatabase>
+	/**
+	 * The candidate build's transaction. Required unless `dryRun` — a dry run judges every row and stages nothing.
+	 */
+	tx?: DatabaseClient<CandidateDatabase>
 	geonamesDir: string
 	countries: readonly string[]
 	attrs: Map<number, PlaceAttrs>
@@ -68,11 +108,32 @@ export async function resurrectCurrencyHoles(ctx: {
 	importance: ReturnType<typeof loadImportanceIndex> | undefined
 	stageRow: StageRow
 	progress: (phase: string, message: string) => void
+	/**
+	 * Which dead placetypes are judged. Default {@link DEFAULT_DEAD_PLACETYPES}.
+	 */
+	deadPlacetypes?: readonly string[]
+	/**
+	 * Judge and count, stage nothing — the census mode. No transaction is opened.
+	 */
+	dryRun?: boolean
+	/**
+	 * Receives one report per country judged (a country with no dump or no dead rows reports zero outcomes).
+	 */
+	onCountry?: (report: CurrencyBackfillCountryReport) => void
 }): Promise<number> {
+	const deadPlacetypes = ctx.deadPlacetypes ?? DEFAULT_DEAD_PLACETYPES
+	const dryRun = ctx.dryRun === true
+
+	if (!dryRun && !ctx.tx) {
+		throw new Error(
+			"resurrectCurrencyHoles: a build run needs the candidate transaction (tx); pass dryRun to judge only"
+		)
+	}
+
 	const deadStmt = ctx.src.prepare(
 		`SELECT id, name, placetype, latitude, longitude, min_latitude, min_longitude, max_latitude, max_longitude
 		 FROM spr
-		 WHERE country = ? AND placetype = 'locality'
+		 WHERE country = ? AND placetype IN (${deadPlacetypes.map(() => "?").join(", ")})
 		   AND is_current = 0 AND is_deprecated = 1 AND is_superseded = 0`
 	)
 
@@ -84,10 +145,20 @@ export async function resurrectCurrencyHoles(ctx: {
 
 	for (const country of ctx.countries) {
 		const cc = country.toUpperCase()
-		const dumpPath = resolve(ctx.geonamesDir, `${cc}.txt`)
+		const dumpPath = resolvePath(ctx.geonamesDir, `${cc}.txt`)
+
+		const report: CurrencyBackfillCountryReport = {
+			country: cc,
+			deadPlacetypes,
+			dumpPresent: await pathExists(dumpPath),
+			...emptyOutcomes(),
+			byDeadPlacetype: {},
+			sample: [],
+		}
 
-		if (!existsSync(dumpPath)) {
+		if (!report.dumpPresent) {
 			ctx.progress("currency-backfill", `${cc}: no GeoNames dump at ${dumpPath} — holes stay dead`)
+			ctx.onCountry?.(report)
 
 			continue
 		}
@@ -95,10 +166,11 @@ export async function resurrectCurrencyHoles(ctx: {
 		// Dead rows FIRST: a country with no deprecated-no-successor localities needs no attestors at all, and
 		// loading a national dump to judge zero rows is pure heap pressure on a build already near its ceiling
 		// (the first live run OOM'd in a later pass with JP/KR dumps loaded for 0 dead names each).
-		const dead = deadStmt.all(cc)
+		const dead = deadStmt.all(cc, ...deadPlacetypes)
 
 		if (!dead.length) {
 			ctx.progress("currency-backfill", `${cc}: 0 dead names — dump not loaded`)
+			ctx.onCountry?.(report)
 
 			continue
 		}
@@ -141,14 +213,18 @@ export async function resurrectCurrencyHoles(ctx: {
 			}
 		}
 
-		let judged = 0
-		let blocked = 0
-		let unattested = 0
-		let floored = 0
-		let resurrected = 0
 		const seen = new Set<string>()
 
-		ctx.tx.exec("BEGIN")
+		const count = (deadPlacetype: string, outcome: keyof CurrencyBackfillOutcomes): void => {
+			report[outcome] += 1
+			const bucket = (report.byDeadPlacetype[deadPlacetype] ??= emptyOutcomes())
+
+			bucket[outcome] += 1
+		}
+
+		if (!dryRun) {
+			ctx.tx!.exec("BEGIN")
+		}
 
 		for (const d of dead) {
 			const name = String(d.name ?? "")
@@ -156,23 +232,22 @@ export async function resurrectCurrencyHoles(ctx: {
 
 			if (!pkey || seen.has(pkey)) continue
 			seen.add(pkey)
+			// Read from the row: the query admits whatever `deadPlacetypes` names, and the rank comparison below must
+			// judge each candidate against ITS dead rung, never a hardcoded one.
+			const deadPlacetype = String(d.placetype ?? "locality")
 
-			judged++
-
+			count(deadPlacetype, "judged")
 			const dLat = Number(d.latitude)
 			const dLon = Number(d.longitude)
-			// The dead-row query is scoped to `locality` today; read it from the row anyway so widening that query
-			// cannot silently start comparing every candidate against a hardcoded rung.
-			const deadPlacetype = String(d.placetype ?? "locality")
 
-			// A live row blocks only when it is AT LEAST AS COARSE as the dead one. The original gate compared name and
+			// A live row blocks only when it is AT LEAST AS COARSE as the dead one. The original check compared name and
 			// distance alone, on the premise that a nearby same-name row means "the place is alive under another
 			// placetype" — true for a place recorded twice, false for a placetype DEMOTION, which is the shape that
 			// actually occurs: WOF retired `Gillingham` the locality (pop 101,187) and kept `Gillingham` the
-			// neighbourhood 3.2 km away, and the gate read the surviving CHILD as covering its own dead parent.
+			// neighbourhood 3.2 km away, and the check read the surviving CHILD as covering its own dead parent.
 			// Sixteen of seventeen GB refusals had exactly that shape (#1746).
 			//
-			// An UNRANKED placetype blocks, which is the conservative direction: this gate's failure mode is inventing
+			// An UNRANKED placetype blocks, which is the conservative direction: this check's failure mode is inventing
 			// a place, so a row we cannot rank is treated as covering rather than waved through.
 			const liveNear = liveStmt.all(cc, name).some((row) => {
 				if (haversineKm(dLat, dLon, Number(row.latitude), Number(row.longitude)) > CURRENCY_BACKFILL_RADIUS_KM) {
@@ -180,13 +255,13 @@ export async function resurrectCurrencyHoles(ctx: {
 				}
 
 				// Blocks UNLESS the live row is strictly finer. The equal rung must still block — a live `locality`
-				// covers a dead `locality` — and an unranked placetype blocks too, since this gate's failure mode
+				// covers a dead `locality` — and an unranked placetype blocks too, since this check's failure mode
 				// is inventing a place.
 				return isStrictlyFiner(String(row.placetype ?? ""), deadPlacetype) !== true
 			})
 
 			if (liveNear) {
-				blocked++
+				count(deadPlacetype, "blocked")
 
 				continue
 			}
@@ -196,7 +271,7 @@ export async function resurrectCurrencyHoles(ctx: {
 			)
 
 			if (!near.length) {
-				unattested++
+				count(deadPlacetype, "unattested")
 
 				continue
 			}
@@ -204,17 +279,25 @@ export async function resurrectCurrencyHoles(ctx: {
 			const pop = Math.max(...near.map((g) => g.pop))
 
 			if (pop < CURRENCY_BACKFILL_POP_FLOOR) {
-				floored++
+				count(deadPlacetype, "floored")
 
 				continue
 			}
 
+			count(deadPlacetype, "resurrected")
+
+			if (report.sample.length < REPORT_SAMPLE_SIZE) {
+				report.sample.push(`${deadPlacetype}:${name}`)
+			}
+
+			if (dryRun) continue
+
 			const sid = Number(d.id)
 
 			const a: PlaceAttrs = {
 				cid: ctx.ccID(cc),
 				rid: ctx.regionOf.get(sid) ?? 0,
-				ptid: ctx.ptID("locality"),
+				ptid: ctx.ptID(deadPlacetype),
 				name,
 				lat: dLat,
 				lon: dLon,
@@ -225,24 +308,26 @@ export async function resurrectCurrencyHoles(ctx: {
 				pop,
 				neg: -Math.log10(pop + 1),
 				pkey,
-				imp: ctx.importance?.find(name, cc, "locality", dLat, dLon) ?? null,
+				imp: ctx.importance?.find(name, cc, deadPlacetype, dLat, dLon) ?? null,
 			}
 
 			ctx.attrs.set(sid, a)
 			ctx.stageRow(pkey, a, sid, 1)
-
-			resurrected++
 		}
 
-		ctx.tx.exec("COMMIT")
+		if (!dryRun) {
+			ctx.tx!.exec("COMMIT")
+		}
 
 		ctx.progress(
 			"currency-backfill",
-			`${cc}: ${resurrected} resurrected of ${judged} dead names ` +
-				`(${blocked} blocked by a live near row, ${unattested} unattested, ${floored} under the population floor)`
+			`${cc}: ${report.resurrected} resurrected of ${report.judged} dead names ` +
+				`(${report.blocked} blocked by a live near row, ${report.unattested} unattested, ` +
+				`${report.floored} under the population floor)${dryRun ? " — dry run, nothing staged" : ""}`
 		)
 
-		total += resurrected
+		ctx.onCountry?.(report)
+		total += report.resurrected
 	}
 
 	return total

package/lib/env.ts

@@ -0,0 +1,50 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Environment variables naming the WOF SQLite artifacts this resolver opens, and the development weights the overlay
+ *   linker materializes. CLI flags (`--resolve-db`, …) override these at the call site.
+ */
+
+import { $public as corePublic, liveEnv } from "@mailwoman/core/env"
+import { z } from "zod"
+
+/**
+ * Gazetteer and resolver database paths, plus the development weights overlay.
+ */
+export const PublicWOFSQLiteEnvSchema = z.object({
+	MAILWOMAN_WOF_DB: z.string().optional().meta({
+		title: "Who's On First database",
+		description: "Path to a full Who's On First SQLite distribution used by the search resolver backend.",
+	}),
+	MAILWOMAN_WOF_ADMIN_DB: z.string().optional().meta({
+		title: "Who's On First admin database",
+		description: "Path to the full Who's On First distribution used for reverse geocoding.",
+	}),
+	MAILWOMAN_WOF_POLYGONS_DB: z.string().optional().meta({
+		title: "Who's On First polygon database",
+		description: "Path to the polygon sidecar used for exact reverse-geocode containment.",
+	}),
+	MAILWOMAN_CANDIDATE_DB: z.string().optional().meta({
+		title: "Candidate database",
+		description: "Path to candidate.db; set to `none` to pin the full-distribution search backend.",
+	}),
+	MAILWOMAN_POSTAL_CITY_ALIAS_DB: z.string().optional().meta({
+		title: "Postal city alias database",
+		description: "Path to postal-city alias data used for alias-aware resolution of U.S. mailing addresses.",
+	}),
+	MAILWOMAN_DEV_MODEL: z.string().optional().meta({
+		title: "Development model",
+		description: "Path to the model artifact linked for local neural-weights development.",
+	}),
+	MAILWOMAN_DEV_TOKENIZER: z.string().optional().meta({
+		title: "Development tokenizer",
+		description: "Path to the tokenizer artifact linked for local neural-weights development.",
+	}),
+})
+
+/**
+ * Live resolver settings over core's, sharing core's getters and cached values.
+ */
+export const $public = liveEnv(PublicWOFSQLiteEnvSchema, corePublic)

package/{exact-match.ts → lib/exact-match.ts}

@@ -7,18 +7,23 @@
  *   candidate holds the query text as its own name, an alias, or an official name.
  */
 
-import type { DatabaseSync } from "node:sqlite"
+import type { DatabaseClient } from "@mailwoman/sqlite/client"
 
-import { aliasBagExactMatch } from "./fts.ts"
+import { aliasBagExactMatch, foldQueryText } from "#fts/index"
 
 /**
  * Among `ids`, return the subset whose name OR any alias equals `text` case-insensitively — the exact-match tier for
- * ranking. One indexed query over `<schema>.names`. When the shard has no `names` table (a slim DB built with
- * `dropNames`, or a postcode-only shard), fall back to the self-contained `place_search` FTS content: its `alt_names`
+ * ranking. One indexed query over `<schema>.names`. When the extract has no `names` table (a slim DB built with
+ * `dropNames`, or a postcode-only extract), fall back to the self-contained `place_search` FTS content: its `alt_names`
  * column is the same alias set joined on the boundary-preserving `ALIAS_SEPARATOR` (#523), so `aliasBagExactMatch`
  * recovers the exact alias tier ("New York City" → New York) that the dropped `names` table used to provide.
  */
-export function exactMatchIDs(db: DatabaseSync, schemaName: string, ids: number[], text: string): Set<number> {
+export function exactMatchIDs<DB>(
+	db: DatabaseClient<DB>,
+	schemaName: string,
+	ids: number[],
+	text: string
+): Set<number> {
 	const out = new Set<number>()
 	const trimmed = text.trim()
 
@@ -36,7 +41,7 @@ export function exactMatchIDs(db: DatabaseSync, schemaName: string, ids: number[
 
 		return out
 	} catch {
-		// No `names` table on this shard — fall through to the place_search alias bag.
+		// No `names` table on this extract — fall through to the place_search alias bag.
 	}
 
 	try {
@@ -44,19 +49,18 @@ export function exactMatchIDs(db: DatabaseSync, schemaName: string, ids: number[
 			.prepare(`SELECT wof_id AS id, name, alt_names FROM ${schemaName}.place_search WHERE wof_id IN (${placeholders})`)
 			.all(...ids) as Array<{ id: number; name: string | null; alt_names: string | null }>
 
-		const norm = (s: string): string => s.toLowerCase().trim().replaceAll(/\s+/g, " ")
-		const needle = norm(trimmed)
+		const needle = foldQueryText(trimmed)
 
 		for (const r of rows) {
-			if (r.name !== null && norm(r.name) === needle) {
+			if (r.name !== null && foldQueryText(r.name) === needle) {
 				out.add(r.id)
 			}
 		}
 
 		// Alias pass via the shared bag parser (#523). Separated bags (built since #523) get a true
-		// per-alias equality check, ungated — matching the `names`-table branch above, where an
+		// per-alias equality check, unrestricted — matching the `names`-table branch above, where an
 		// alias match counts as exact regardless of other candidates. Legacy bags (no separator)
-		// fall back to padded containment, gated on "no canonical exact in the pool" because their
+		// fall back to padded containment, conditioned on "no canonical exact in the pool" because their
 		// lost boundaries would otherwise false-promote interior fragments ("York" inside the alias
 		// "New York City") or cross-alias fragments ("York New" across "…York" + "New City…").
 		const anyCanonicalExact = out.size > 0
@@ -67,7 +71,7 @@ export function exactMatchIDs(db: DatabaseSync, schemaName: string, ids: number[
 			}
 		}
 	} catch {
-		// Shard without place_search either → no exact-match tier. Falls back to weighted-sum order.
+		// Extract without place_search either → no exact-match tier. Falls back to weighted-sum order.
 	}
 
 	return out
@@ -79,7 +83,12 @@ export function exactMatchIDs(db: DatabaseSync, schemaName: string, ids: number[
  * agree on what "equals the query" means. Fails soft on gazetteers built before #940 (no `official` column) — the
  * sub-tier then behaves exactly as if `officialNameExact` were off.
  */
-export function officialNameIDs(db: DatabaseSync, schemaName: string, ids: number[], text: string): Set<number> {
+export function officialNameIDs<DB>(
+	db: DatabaseClient<DB>,
+	schemaName: string,
+	ids: number[],
+	text: string
+): Set<number> {
 	const out = new Set<number>()
 	const trimmed = text.trim()
 
@@ -97,7 +106,7 @@ export function officialNameIDs(db: DatabaseSync, schemaName: string, ids: numbe
 			out.add(r.id)
 		}
 	} catch {
-		// Pre-#940 gazetteer (no `official` column) or a names-less slim shard — feature inert.
+		// Pre-#940 gazetteer (no `official` column) or a names-less slim extract — feature inert.
 	}
 
 	return out