@mailwoman/resolver-wof-sqlite 9.4.0 → 10.1.0

package/lib/ancestry/backfill.ts

@@ -3,98 +3,50 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Repair the truncated ancestry that {@link populateAncestors} (the parent_id closure in
- *   unified-schema.ts) leaves wherever the closure dead-ends before reaching the top.
- *
- *   Root cause (#440 / #832): a place that straddles multiple parents — New York City spans five
- *   counties (its boroughs), London 30+ — carries `wof:parent_id = -4`, so the parent_id closure
- *   dead-ends and the place gets NO region/county/country ancestry. The resolver's region-descendant
- *   filter then can't reach it: given "New York, NY", NYC (with no NY-state ancestor) is excluded and
- *   a correctly-parented namesake ("New York Mills", pop 3,190) wins over NYC's 8.8M. The same defect
- *   orphans London, Singapore, and ~2,850 other localities — the most demo-visible queries.
- *
- *   **The dead end is inherited by children (#1445).** Repairing the `-4` place itself does not repair
- *   anything BELOW it: the closure walks Brooklyn → New York and stops, because New York's own
- *   `parent_id` is `-4`. Brooklyn-the-borough (pop 2.5M) therefore carried exactly two ancestor rows —
- *   itself and New York — with no county, region or country, and the only US locality-tier place named
- *   "Brooklyn" that survived a New-York-State descendant filter was Pillar Point, a Jefferson County
- *   hamlet 411 km away carrying "Brooklyn" as an alternate name. All five NYC boroughs, every London
- *   borough and Hyderabad's zones were in the same state.
- *
- *   So the candidate test is NOT "has only a self ancestor" — that misses every child of a repaired
- *   place, which by construction has two. It is **"has no `country`-tier ancestor"**. Country is the
- *   universal terminal for every non-{@link TOP_PLACETYPES} placetype, so its absence is exactly the
- *   signal that the chain dead-ended somewhere, at whatever depth. On a wide-coverage build this
- *   selects ~24k places against 2.55M rows.
- *
- *   A place whose `wof:hierarchy` genuinely stops short is NOT a candidate and needs no repair: the
- *   source is the authority on what a place should have. American Samoa's localities, for instance,
- *   have `{country_id, locality_id}` and no region in WOF itself — the artifact matching that is
- *   correct, not truncated.
- *
- *   The authoritative hierarchy IS in the source geojson: `wof:hierarchy` is an array of branches,
- *   each a `<placetype>_id` → id map (region_id, county_id, country_id, …), fully populated even when
- *   parent_id is -4. This reads it for every candidate and inserts the missing ancestor rows (one per
- *   distinct ancestor across branches).
- *
- *   MUST run AFTER populateAncestors and BEFORE the build freezes (VACUUM INTO), so the rows land in
- *   the shipped artifact — `scripts/build-unified-wof.ts` Phase 3 calls it inline. The standalone
- *   `scripts/backfill-ancestors-from-hierarchy.ts` is a thin CLI over the same function for ad-hoc
- *   repair of an already-built DB. Idempotent by the per-pair existence check, not by the candidate
- *   test: each (id, ancestor_id) is inserted at most once, so a second run over the same DB adds
- *   nothing.
+ *   Repair the truncated ancestry left by the `parent_id` closure. Must run after `populateAncestors` and before the build freezes.
  */
 
-import { readDirectoryEntries } from "@mailwoman/core/fs/readers"
 import { readWOFFeature } from "@mailwoman/core/resources/whosonfirst"
 import type { DatabaseClient } from "@mailwoman/sqlite/client"
-import { join, resolvePath, type PathBuilderLike } from "path-ts"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
+import { Globerator } from "spliterator/node/fs"
 
 import type { WOFDatabase } from "#schema"
 
-/**
- * Genuinely top-level placetypes — they never have (or need) an ancestor, so skip them.
- */
 const TOP_PLACETYPES = new Set(["country", "continent", "empire", "ocean", "marinearea", "planet"])
 
 export interface AncestryBackfillResult {
-	/**
-	 * Places that gained at least one ancestor row.
-	 */
 	placesFixed: number
-	/**
-	 * Total ancestor rows inserted.
-	 */
 	rowsAdded: number
 	/**
-	 * Candidates whose source geojson could not be found (non-WOF backfilled places, or repos not present locally) —
-	 * skipped, not an error.
+	 * Candidates whose source geojson was not found (non-WOF backfilled places or absent repos),
+	 * skipped rather than treated as an error.
 	 */
 	noGeojson: number
 }
 
 /**
- * Discover the `data` directories under a WOF repos root that hold attached geojson, e.g.
- * `<root>/whosonfirst-data/whosonfirst-data-admin-us/data`. Resolves an id to its geojson via these roots. Accepts both
- * the nested lab layout (a `whosonfirst-data` group dir holding the admin repos) and a flat layout (admin repos
- * directly under the root); searches at most two directory levels deep.
+ * Discover the `data` directories under a WOF repos root that hold attached geojson,
+ * accepting both the nested lab layout and a flat layout at most two directory levels deep.
  */
-export async function discoverAdminDataRoots(reposRoot: PathBuilderLike): Promise<string[]> {
-	const roots: string[] = []
+export async function discoverAdminDataRoots(reposRoot: PathBuilderLike): Promise<PathBuilder[]> {
+	const roots: PathBuilder[] = []
 
-	const visit = async (dir: string, depth: number): Promise<void> => {
+	const visit = async (dir: PathBuilder, depth: number): Promise<void> => {
 		if (depth > 2) return
 
 		let names: string[]
 
 		try {
-			names = (await readDirectoryEntries(dir)).filter((e) => e.isDirectory()).map((e) => e.name)
+			names = (await Globerator.from("*", { cwd: dir, withFileTypes: true, onlyFiles: false }).toArray())
+				.filter((entry) => entry.isDirectory())
+				.map((entry) => entry.name)
 		} catch {
 			return
 		}
 
 		for (const name of names) {
-			const child = join(dir, name)
+			const child = dir(name)
 
 			if (name === "data") {
 				roots.push(child)
@@ -104,15 +56,13 @@ export async function discoverAdminDataRoots(reposRoot: PathBuilderLike): Promis
 		}
 	}
 
-	await visit(resolvePath(reposRoot), 0)
+	await visit(PathBuilder.from(reposRoot), 0)
 
 	return roots
 }
 
-// `<placetype>_id` key → ancestor placetype. WOF hierarchy keys are e.g. region_id, county_id. Self
-// is filtered downstream by the `aid === id` check, so we do NOT special-case locality here: for a
-// locality candidate `locality_id` IS self (dropped by aid===id), but for a neighbourhood candidate
-// `locality_id` is its PARENT locality — a real ancestor we must keep.
+// The `aid === id` check filters out a place's own `locality_id`.
+// For a neighborhood, that id identifies a real ancestor.
 function placetypeFromKey(key: string): string | null {
 	if (!key.endsWith("_id")) return null
 
@@ -120,28 +70,19 @@ function placetypeFromKey(key: string): string | null {
 }
 
 /**
- * Insert missing ancestor rows for every place whose ancestry chain dead-ended before reaching a country, by reading
- * `wof:hierarchy` from its source geojson under `geojsonRoots` (see {@link discoverAdminDataRoots}). Runs inside a
- * single transaction; caller owns connection lifecycle (open, WAL checkpoint, close).
+ * Insert missing ancestor rows for places whose ancestry chain ended before reaching a country.
  *
- * `opts.maxID` bounds the candidate scan to ids BELOW it — pass the synthetic-id base (`OVERTURE_ID_BASE`, 8e12) so the
- * backfill considers only real WOF places. Overture/GeoNames rows carry synthetic ids and have NO `wof:hierarchy`
- * geojson, so probing them is pure waste: on a wide-coverage DB the country-less set is millions of Overture/GeoNames
- * leaf localities, and the per-candidate geojson probe across every repo root turns a seconds-long WOF-only pass into a
- * ~40-minute one (their ancestry comes from the parent_id closure, not this backfill). Correctness-preserving — the
- * skipped rows would have `noGeojson`-skipped anyway. Omit `maxID` (default) for the legacy WOF-only DBs.
+ * The function reads `wof:hierarchy` and runs in one transaction under the caller's connection.
+ * `opts.maxID` bounds the scan to avoid probing synthetic-id Overture and GeoNames rows.
  */
 export async function backfillAncestorsFromHierarchy(
 	db: DatabaseClient<WOFDatabase>,
-	geojsonRoots: readonly string[],
+	geojsonRoots: readonly PathBuilderLike[],
 	opts: { maxID?: number } = {}
 ): Promise<AncestryBackfillResult> {
 	const maxID = opts.maxID ?? Number.MAX_SAFE_INTEGER
 
-	// "No country-tier ancestor" is the dead-end signal at any depth — see the module docstring. The
-	// earlier "<= 1 ancestor row" test only caught the dead end's origin, never the children that
-	// inherit it (a child of a repaired -4 place has two rows: itself and that parent) (#1445).
-	// The id bound is stated first so SQLite prunes by the PK index before the NOT EXISTS runs at all.
+	// The id bound is stated first so SQLite prunes by the PK index before the not-exists runs.
 	const candidateBase = db
 		.selectFrom("spr")
 		.where("id", "<", maxID)
@@ -159,12 +100,8 @@ export async function backfillAncestorsFromHierarchy(
 
 	const candidates = await candidateBase.select(["id", "placetype"]).execute()
 
-	// Every candidate's existing ancestors in ONE query rather than an indexed read each. The widened
-	// candidate test made that per-candidate read the dominant cost of the pass, and the set is bounded:
-	// a candidate reaching this point has a handful of rows at most. The candidate set rides in as the
-	// SAME predicate re-issued as a subquery, never a materialized `IN (?, ?, …)` list — node:sqlite
-	// caps a statement at 32,766 bound variables and the wide-coverage build has 67,521 candidates
-	// (measured 2026-08-04).
+	// Keep the candidate set as a subquery instead of materializing an `IN` list.
+	// node:sqlite limits bound variables to 32,766.
 	const alreadyPresent = new Map<number, Set<number>>()
 
 	for (const row of await db
@@ -204,7 +141,6 @@ export async function backfillAncestorsFromHierarchy(
 			continue
 		}
 
-		// Collect distinct (ancestor_id, placetype) across all hierarchy branches, excluding self.
 		const seen = new Map<number, string>()
 
 		for (const branch of hierarchy) {

package/lib/{ancestry/index.ts → ancestry.ts}

@@ -3,25 +3,26 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   The shared ancestor-lineage walk over the WOF `ancestors` table — one place's containment chain
- *   joined with `spr` for canonical names + centroids, ordered NEAREST-FIRST (deepest placetype
- *   first, country last).
+ *   The shared ancestor-lineage walk over the WOF `ancestors` table: one place's containment chain
+ *   joined with `spr` for canonical names and centroids, ordered nearest-first with the deepest
+ *   placetype first and the country last.
  *
- *   Factored out of `WOFSQLitePlaceLookup.ancestors()` (#404) so the reverse geocoder (`reverse.ts`,
- *   #484) reuses the SAME walk instead of growing a second one. The placetype-specificity ordering
- *   lives here as `PLACETYPE_DEPTH` — a single TS map instead of the previous SQL CASE, and
- *   extended below `localadmin` (locality/borough/neighbourhood/microhood now rank correctly
- *   instead of sorting last; forward resolution rarely saw those as ANCESTOR placetypes, reverse
- *   geocoding always does).
+ *   The reverse geocoder and `WOFSQLitePlaceLookup.ancestors()` share this walk. Placetype
+ *   specificity lives here in `PLACETYPE_DEPTH`, a single TypeScript map that extends below
+ *   `localadmin` so locality, borough, neighborhood and microhood rank correctly. Forward
+ *   resolution rarely sees those as ancestor placetypes, while reverse geocoding always does.
  */
 
 import { allRows } from "@mailwoman/core/utils"
 import type { DatabaseClient } from "@mailwoman/sqlite/client"
 
 /**
- * WOF placetype → containment depth, coarsest = 1. Higher = finer. Placetypes we never resolve (continent, empire, …)
- * map to 0 and sort last. NOT the same table as the FST's `PLACETYPE_ORDER` (fst-serialize.ts) — that one is a
- * serialization order, this one is containment depth.
+ * WOF placetype → containment depth, coarsest = 1.
+ *
+ * Higher = finer. Placetypes we never resolve (continent, empire, …) map to 0 and sort last.
+ *
+ * This differs from the FST's `PLACETYPE_ORDER` (fst-serialize.ts), which is a serialization order.
+ * This one is containment depth.
  */
 export const PLACETYPE_DEPTH: Readonly<Record<string, number>> = {
 	country: 1,
@@ -38,7 +39,7 @@ export const PLACETYPE_DEPTH: Readonly<Record<string, number>> = {
 }
 
 /**
- * Containment depth for a placetype — 0 (sorts coarsest) when unknown.
+ * Containment depth for a placetype, 0 when unknown (sorts coarsest).
  */
 export function placetypeDepth(placetype: string): number {
 	return PLACETYPE_DEPTH[placetype] ?? 0
@@ -57,8 +58,11 @@ export interface AncestorPlaceRow {
 }
 
 /**
- * The ancestor lineage of `id` — self excluded, nearest-first. Returns `[]` when the place has no recorded ancestry.
- * NOT memoized here; `WOFSQLitePlaceLookup` keeps its own per-id cache.
+ * The ancestor lineage of `id`, self excluded, nearest-first.
+ *
+ * @returns `[]` when the place has no recorded ancestry.
+ * Not memoized here.
+ * `WOFSQLitePlaceLookup` keeps its own per-id cache.
  */
 export function ancestorLineage<DB>(db: DatabaseClient<DB>, id: number, schemaName = "main"): AncestorPlaceRow[] {
 	const rows = allRows<AncestorPlaceRow>(