@mailwoman/resolver-wof-sqlite 9.2.0 → 9.3.0

package/{build-candidate.ts → lib/build-candidate.ts}

@@ -22,7 +22,7 @@
  *
  *   The name_key normalizer is the SHARED {@link normalizeLocalityForKey} — the query side (the demo
  *   resolver {@link WOFCandidateTableLookup}) MUST use the same function, the one-normalizer
- *   discipline the address-point shard uses, so build/query stay consistent by construction.
+ *   discipline the address-point extract uses, so build/query stay consistent by construction.
  *
  *   Measured (2026-06-20, vs the 2.6 GB full-DB FTS): ~5 M rows; ~12 range fetches per 8-query
  *   session (the full DB needs 243); US locality 96.8% (region bbox), EU coord parity 88.6%.
@@ -39,36 +39,36 @@
  *   `candidate-ancestors-schema.ts` owns the encoding decision and the DAG/absence semantics.
  */
 
-import { existsSync, rmSync } from "node:fs"
-import { DatabaseSync } from "node:sqlite"
-
 import { COUNTRY_POPULATION } from "@mailwoman/codex/country"
-import { DatabaseClient } from "@mailwoman/core/kysley/client"
+import { pathExists } from "@mailwoman/core/fs/readers"
+import { removePath } from "@mailwoman/core/fs/writers"
+import { DatabaseClient } from "@mailwoman/sqlite/client"
 
-import { createCandidateFTS } from "./candidate-fts.ts"
-import { IMPORTANCE_JOIN_GATE_KM, loadImportanceIndex } from "./candidate-importance.ts"
+import { createCandidateFTS } from "#candidate-fts"
+import { IMPORTANCE_JOIN_RADIUS_KM, loadImportanceIndex } from "#candidate-importance"
 import {
 	CANDIDATE_COLUMNS,
 	createCandidateStagingTables,
 	createCandidateTable,
 	type CandidateDatabase,
-} from "./candidate-schema.ts"
-import { explodeAliasBags } from "./candidate/alias-bags.ts"
-import { buildAncestorsSidecar } from "./candidate/ancestors-sidecar.ts"
-import { stageCountryDisplayNames } from "./candidate/country-display-names.ts"
-import { GLOSS_KEY_THRESHOLD, stampNameRoles } from "./candidate/name-roles.ts"
-import type { PlaceAttrs } from "./candidate/place-attrs.ts"
-import { foldShard } from "./candidate/shard-fold.ts"
-import { createCapitalTable } from "./capital-schema.ts"
-import type { CapitalPoint } from "./capitals.ts"
-import { resurrectCurrencyHoles } from "./currency-backfill.ts"
-import { normalizeLocalityForKey } from "./street-normalize.ts"
+} from "#candidate-schema"
+import { explodeAliasBags } from "#candidate/alias-bags"
+import { buildAncestorsSidecar } from "#candidate/ancestors-sidecar"
+import { stageCountryDisplayNames } from "#candidate/country-display-names"
+import { foldExtract } from "#candidate/extract-fold"
+import { GLOSS_KEY_THRESHOLD, stampNameRoles } from "#candidate/name-roles"
+import type { PlaceAttrs } from "#candidate/place-attrs"
+import { createCapitalTable } from "#capital-schema"
+import type { CapitalPoint } from "#capitals"
+import { resurrectCurrencyHoles } from "#currency-backfill"
+import type { WOFDatabase } from "#schema"
+import { normalizeLocalityForKey } from "#street/normalize"
 
 // The build's contract is this module path; the passes behind it live in `./candidate/`. Re-exported
 // here so a consumer never has to know which pass owns which name.
-export { stageCountryDisplayNames } from "./candidate/country-display-names.ts"
-export { GLOSS_EXCLUDED_PLACETYPES, GLOSS_KEY_THRESHOLD } from "./candidate/name-roles.ts"
-export type { PlaceAttrs } from "./candidate/place-attrs.ts"
+export { stageCountryDisplayNames } from "#candidate/country-display-names"
+export { GLOSS_EXCLUDED_PLACETYPES, GLOSS_KEY_THRESHOLD } from "#candidate/name-roles"
+export type { PlaceAttrs } from "#candidate/place-attrs"
 
 export interface BuildCandidateOptions {
 	/**
@@ -86,20 +86,20 @@ export interface BuildCandidateOptions {
 	 */
 	capitals?: readonly CapitalPoint[]
 	/**
-	 * Optional postcode shards (`spr` rows with `placetype='postalcode'` + real coords, e.g. postalcode-us.db) — folded
+	 * Optional postcode extracts (`spr` rows with `placetype='postalcode'` + real coords, e.g. postalcode-us.db) — folded
 	 * in as `postalcode` candidate rows so `findPlace(postalcode)` resolves a ZIP directly (the demo's primary postcode
 	 * path; the postcode-*.bin anchor stays the fallback). Matches the slim wof-hot.db, which took one such postcode DB.
 	 *
-	 * Each shard's `names` table is folded in too (#1495) — that's where the GeoNames delivery-city names live
+	 * Each extract's `names` table is folded in too (#1495) — that's where the GeoNames delivery-city names live
 	 * ("Brooklyn" for 11201), and they were previously reachable only through FTS.
 	 */
 	postcodes?: string[]
 	/**
-	 * Optional LOCALITY shards (`spr` rows with `placetype='locality'` + real coords, e.g. localities-nz-linz.db — the
-	 * #1564 NZ suburb tier) — folded through the same shard loop as the postcode shards, staged as `locality` candidate
-	 * rows with no region scope and UNMEASURED population (`neg_rank 0`: a shard row ranks behind any populated namesake
-	 * and wins only where its key is the answer). Each shard's `names` table folds as aliases, `is_primary = 0`, same as
-	 * the delivery-city pass.
+	 * Optional LOCALITY extracts (`spr` rows with `placetype='locality'` + real coords, e.g. localities-nz-linz.db — the
+	 * #1564 NZ suburb tier) — folded through the same extract loop as the postcode extracts, staged as `locality`
+	 * candidate rows with no region scope and UNMEASURED population (`neg_rank 0`: a extract row ranks behind any
+	 * populated namesake and wins only where its key is the answer). Each extract's `names` table folds as aliases,
+	 * `is_primary = 0`, same as the delivery-city pass.
 	 */
 	localities?: string[]
 	/**
@@ -118,21 +118,30 @@ export interface BuildCandidateOptions {
 	 * Cross-source currency backfill (#1737). WOF carries deprecated-with-no-successor records for real, populated
 	 * settlements (Rochester Kent, Aldershot, Telford — 120 GB localities alone), and the currency filter correctly drops
 	 * them, leaving holes no ranking can fill. When this option is set, pass 1c resurrects a dead locality ONLY under
-	 * three gates, positive evidence throughout: no live same-name row of any placetype near the dead record (a distant
-	 * same-name row is a NAMESAKE and does not block — Rochester, Northumberland pop 318 must not veto Rochester, Kent);
-	 * an independent GeoNames P-class attestation of the same folded name within {@link CURRENCY_BACKFILL_RADIUS_KM}; and
-	 * the attestor at or above {@link CURRENCY_BACKFILL_POP_FLOOR}. The staged row keeps the real WOF id, name, centroid
-	 * and bbox — GeoNames only ATTESTS the place and supplies the population that lets it stand in prominence races.
-	 * `countries` are judged only where `<cc>.txt` exists under `geonamesDir`; absent dumps are skipped loudly.
+	 * three conditions, positive evidence throughout: no live same-name row of any placetype near the dead record (a
+	 * distant same-name row is a NAMESAKE and does not block — Rochester, Northumberland pop 318 must not veto Rochester,
+	 * Kent); an independent GeoNames P-class attestation of the same folded name within
+	 * {@link CURRENCY_BACKFILL_RADIUS_KM}; and the attestor at or above {@link CURRENCY_BACKFILL_POP_FLOOR}. The staged
+	 * row keeps the real WOF id, name, centroid and bbox — GeoNames only ATTESTS the place and supplies the population
+	 * that lets it stand in prominence races. `countries` are judged only where `<cc>.txt` exists under `geonamesDir`;
+	 * absent dumps are skipped loudly.
 	 */
-	currencyBackfill?: { geonamesDir: string; countries: readonly string[] }
+	currencyBackfill?: {
+		geonamesDir: string
+		countries: readonly string[]
+		/**
+		 * The dead placetypes the resurrection judges; `resurrectCurrencyHoles`'s default (`locality`) when absent. The
+		 * `localadmin` widening #1746 named is admitted here once its census is read.
+		 */
+		deadPlacetypes?: readonly string[]
+	}
 	/**
 	 * Optional progress callback for CLI / test introspection.
 	 */
 	onProgress?: (phase: string, message: string) => void
 	/**
 	 * Key-count threshold for the gloss anomaly detector — {@link GLOSS_KEY_THRESHOLD} unless a test passes a
-	 * fixture-scale value. The production number is the #1730 sweep's own cut (4,000 places at >= 50 keys).
+	 * fixture-scale value. The production number is the #1730 sweep's own threshold (4,000 places at >= 50 keys).
 	 */
 	glossKeyThreshold?: number
 }
@@ -145,8 +154,8 @@ export interface BuildCandidateResult {
 	abbrevs: number
 	postcodes: number
 	/**
-	 * Delivery-city (and other `names`-table) aliases folded onto postcode rows — #1495. Zero here means the shards
-	 * carried no alias names, NOT that the pass was skipped: a shard with no `names` table reports that separately
+	 * Delivery-city (and other `names`-table) aliases folded onto postcode rows — #1495. Zero here means the extracts
+	 * carried no alias names, NOT that the pass was skipped: a extract with no `names` table reports that separately
 	 * through `onProgress`.
 	 */
 	postcodeAliases: number
@@ -161,7 +170,7 @@ export interface BuildCandidateResult {
 	ancestorPlaces: number
 	/**
 	 * Places that received a pre/post interval label — the canonical-parent forest's node count. Places outside it (no
-	 * recorded ancestry, shard rows, cycle-skipped) have NO label: containment against them is unverifiable, never
+	 * recorded ancestry, extract rows, cycle-skipped) have NO label: containment against them is unverifiable, never
 	 * false.
 	 */
 	intervalPlaces: number
@@ -174,12 +183,12 @@ export interface BuildCandidateResult {
 	importanceScored?: number
 	/**
 	 * Places whose `(name_key, country, placetype)` matched a scored group but whose nearest scored centroid was outside
-	 * {@link IMPORTANCE_JOIN_GATE_KM} — a different town wearing the same name, refused rather than scored.
+	 * {@link IMPORTANCE_JOIN_RADIUS_KM} — a different town wearing the same name, refused rather than scored.
 	 *
 	 * Worth watching across rebuilds: a jump here means the score source and the admin source have drifted apart, and the
 	 * join is being asked to guess.
 	 */
-	importanceGated?: number
+	importanceFiltered?: number
 	/**
 	 * Alias rows stamped `name_role = 'gloss'` — the anomaly detector's certain core (#1730).
 	 */
@@ -202,16 +211,16 @@ export interface BuildCandidateResult {
 export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<BuildCandidateResult> {
 	const progress = opts.onProgress ?? (() => {})
 
-	if (existsSync(opts.output)) {
-		rmSync(opts.output)
+	if (await pathExists(opts.output)) {
+		await removePath(opts.output)
 	}
 
-	const src = new DatabaseSync(opts.input, { readOnly: true })
-	const out = new DatabaseSync(opts.output)
+	using src = new DatabaseClient<WOFDatabase>(opts.input, { readOnly: true })
+	await using kdb = new DatabaseClient<CandidateDatabase>(opts.output)
 	// Build-tuning pragmas (raw — Kysely doesn't model PRAGMA). The code dictionaries + the transient
 	// staging table come from the SHARED schema DDL, so they can't drift from {@link CandidateDatabase}.
-	out.exec("PRAGMA page_size=8192; PRAGMA journal_mode=OFF; PRAGMA synchronous=OFF; PRAGMA cache_size=-2000000;")
-	const kdb = new DatabaseClient<CandidateDatabase>({ database: out })
+	kdb.exec("PRAGMA page_size=8192; PRAGMA journal_mode=OFF; PRAGMA synchronous=OFF; PRAGMA cache_size=-2000000;")
+
 	await createCandidateStagingTables(kdb)
 
 	// --- compact code maps (country/placetype → small int, shrinks the clustered key). The ids are
@@ -296,13 +305,13 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
 	// The hot path — millions of clustered rows. Kept a single positional prepared statement (the fastest
 	// node:sqlite insert) rather than a per-row query builder. Placeholders come from CANDIDATE_COLUMNS so
 	// the column COUNT can't drift; the positional run() args below MUST stay in CANDIDATE_COLUMNS order.
-	const insStage = out.prepare(`INSERT INTO cand_stage VALUES (${CANDIDATE_COLUMNS.map(() => "?").join(", ")})`)
+	const insStage = kdb.prepare(`INSERT INTO cand_stage VALUES (${CANDIDATE_COLUMNS.map(() => "?").join(", ")})`)
 
 	// --- pass 1: primaries (and the per-place attrs the alias/abbrev passes reuse) ---
 	progress("primaries", "indexing place names")
 	const attrs = new Map<number, PlaceAttrs>()
 	let nPrim = 0
-	out.exec("BEGIN")
+	kdb.exec("BEGIN")
 
 	for (const r of src
 		.prepare(
@@ -374,14 +383,14 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
 		}
 	}
 
-	out.exec("COMMIT")
+	kdb.exec("COMMIT")
 	progress("primaries", `${nPrim.toLocaleString()} primaries; ${attrs.size.toLocaleString()} places`)
 
 	if (importance) {
 		progress(
 			"importance",
 			`${importance.matched.toLocaleString()} places scored; ` +
-				`${importance.gated.toLocaleString()} refused (nearest same-name place > ${IMPORTANCE_JOIN_GATE_KM} km away)`
+				`${importance.refused.toLocaleString()} refused (nearest same-name place > ${IMPORTANCE_JOIN_RADIUS_KM} km away)`
 		)
 	}
 
@@ -417,18 +426,19 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
 			iso2ByID: new Map([...ccodes].map(([code, id]) => [id, code])),
 			countryPtID: ptID("country"),
 			stageRow,
-			tx: out,
+			tx: kdb,
 		}).toLocaleString()} country surfaces`
 	)
 
-	// --- pass 1c: cross-source currency backfill (#1737 — resurrectCurrencyHoles owns the gates). Runs BEFORE
+	// --- pass 1c: cross-source currency backfill (#1737 — resurrectCurrencyHoles owns the checks). Runs BEFORE
 	// the alias pass so a resurrected place's alt names explode like any primary's. ---
 	if (opts.currencyBackfill) {
 		const nBackfill = await resurrectCurrencyHoles({
 			src,
-			tx: out,
+			tx: kdb,
 			geonamesDir: opts.currencyBackfill.geonamesDir,
 			countries: opts.currencyBackfill.countries,
+			...(opts.currencyBackfill.deadPlacetypes ? { deadPlacetypes: opts.currencyBackfill.deadPlacetypes } : {}),
 			attrs,
 			ccID,
 			ptID,
@@ -447,12 +457,12 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
 
 	// --- pass 2: distinct normalized aliases from place_search.alt_names (explodeAliasBags owns the loop) ---
 	progress("aliases", "exploding alias bags")
-	const { nAlias, keyCounts } = explodeAliasBags(src, out, attrs, stageRow)
+	const { nAlias, keyCounts } = explodeAliasBags(src, kdb, attrs, stageRow)
 	progress("aliases", `${nAlias.toLocaleString()} aliases`)
 
 	// --- pass 3: region abbreviations (place_abbr) ---
 	let nAbbr = 0
-	out.exec("BEGIN")
+	kdb.exec("BEGIN")
 
 	for (const r of src.prepare("SELECT id, abbr FROM place_abbr").iterate()) {
 		const a = attrs.get(Number(r.id))
@@ -466,7 +476,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
 		nAbbr++
 	}
 
-	out.exec("COMMIT")
+	kdb.exec("COMMIT")
 	progress("abbrevs", `${nAbbr.toLocaleString()} abbrevs`)
 
 	// --- pass 3b: name roles (#1730 prototype — stampNameRoles owns the detectors) ---
@@ -474,7 +484,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
 	// interval tables, and neither reads the other's output. Ordered by label only.
 	const roles = stampNameRoles({
 		src,
-		out,
+		out: kdb,
 		attrs,
 		keyCounts,
 		glossThreshold: opts.glossKeyThreshold ?? GLOSS_KEY_THRESHOLD,
@@ -484,17 +494,17 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
 	})
 
 	// --- pass 3c: the ancestors sidecar (candidate-ancestors-schema.ts owns the encoding decision) ---
-	const sidecar = await buildAncestorsSidecar({ src, out, kdb, attrs, ptID, progress })
+	const sidecar = await buildAncestorsSidecar({ src, out: kdb, attrs, ptID, progress })
 
-	// --- pass 4 + 4b: postcode and locality shards (foldShard owns the per-shard loop) ---
+	// --- pass 4 + 4b: postcode and locality extracts (foldExtract owns the per-extract loop) ---
 	let nPostcode = 0
 	let nPostcodeAlias = 0
 
 	for (const pcDB of opts.postcodes ?? []) {
-		const folded = foldShard({
-			out,
-			shardPath: pcDB,
-			shardPlacetype: "postalcode",
+		const folded = foldExtract({
+			out: kdb,
+			extractPath: pcDB,
+			extractPlacetype: "postalcode",
 			ccID,
 			ptID,
 			stageRow,
@@ -512,10 +522,10 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
 	let nLocality = 0
 
 	for (const locDB of opts.localities ?? []) {
-		const folded = foldShard({
-			out,
-			shardPath: locDB,
-			shardPlacetype: "locality",
+		const folded = foldExtract({
+			out: kdb,
+			extractPath: locDB,
+			extractPlacetype: "locality",
 			ccID,
 			ptID,
 			stageRow,
@@ -526,7 +536,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
 	}
 
 	if (nLocality > 0) {
-		progress("localities", `${nLocality.toLocaleString()} shard localities folded`)
+		progress("localities", `${nLocality.toLocaleString()} extract localities folded`)
 	}
 
 	// --- code dictionaries: typed batch inserts via kdb (a few hundred rows — Kysely is clean here) ---
@@ -574,26 +584,23 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
 	await createCandidateTable(kdb)
 	// OR IGNORE: an abbrev/alias can normalize to a place's primary key (same place, same rank) → any one
 	// row. The bulk sorted INSERT…SELECT (clustered materialization) stays raw — a single hot bulk statement.
-	out.exec(`INSERT OR IGNORE INTO candidate (${cols}) SELECT ${cols} FROM cand_stage ORDER BY ${keyOrder};`)
+	kdb.exec(`INSERT OR IGNORE INTO candidate (${cols}) SELECT ${cols} FROM cand_stage ORDER BY ${keyOrder};`)
 	await kdb.schema.dropTable("cand_stage").execute()
 	// Typo-tolerant fallback index (the unified gazetteer's second mode): the exact name_key probe can't
 	// recover misspellings, so FTS5-trigram over `name` lets the reader fuzzy-match on an exact+strip miss.
 	progress("fts", "building FTS5-trigram fuzzy index")
-	createCandidateFTS(out)
+	createCandidateFTS(kdb)
 	// page_size MUST be set right before VACUUM: node:sqlite initializes the file at the 4096 default on
 	// `new DatabaseSync`, so the creation-time pragma is a no-op — only a VACUUM rebuilds at the new size.
 	// 8192 matches the sql.js-httpvfs 64 KiB request chunk cleanly (8 pages) and shallows the B-tree.
-	out.exec("PRAGMA page_size=8192")
-	out.exec("VACUUM")
+	kdb.exec("PRAGMA page_size=8192")
+	kdb.exec("VACUUM")
 
 	const { n: rows } = await kdb
 		.selectFrom("candidate")
 		.select((eb) => eb.fn.countAll<number>().as("n"))
 		.executeTakeFirstOrThrow()
 
-	src.close()
-	await kdb.destroy()
-
 	// closes the underlying `out` connection
 	return {
 		rows,
@@ -607,6 +614,6 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
 		ancestorPlaces: sidecar.ancestorPlaces,
 		intervalPlaces: sidecar.intervalPlaces,
 		...roles,
-		...(importance ? { importanceScored: importance.matched, importanceGated: importance.gated } : {}),
+		...(importance ? { importanceScored: importance.matched, importanceFiltered: importance.refused } : {}),
 	}
 }