@mailwoman/resolver-wof-sqlite 10.7.0 → 11.0.0

package/lib/address/point/interpolation.ts

@@ -30,7 +30,7 @@
  */
 
 import type { InterpolationLookup } from "@mailwoman/core/resolver"
-import { haversineKm } from "@mailwoman/spatial"
+import { haversine } from "@mailwoman/spatial/geometries/point/distance"
 import { DatabaseClient } from "@mailwoman/sqlite/client"
 
 import type { AddressPointDatabase } from "#address/point/schema"
@@ -185,7 +185,7 @@ function interpolateFromNeighbors(rows: readonly PointRow[], n: number): Interpo
 
 	if (below && above) {
 		const t = (n - below.n) / (above.n - below.n)
-		const spanM = haversineKm(below.lat, below.lon, above.lat, above.lon) * 1000
+		const spanM = haversine(below.lat, below.lon, above.lat, above.lon) * 1000
 
 		return {
 			lat: below.lat + (above.lat - below.lat) * t,
@@ -212,8 +212,8 @@ function interpolateFromNeighbors(rows: readonly PointRow[], n: number): Interpo
 
 	const lat = far.lat + (near.lat - far.lat) * t
 	const lon = far.lon + (near.lon - far.lon) * t
-	const pairM = haversineKm(near.lat, near.lon, far.lat, far.lon) * 1000
-	const overshootM = haversineKm(lat, lon, near.lat, near.lon) * 1000
+	const pairM = haversine(near.lat, near.lon, far.lat, far.lon) * 1000
+	const overshootM = haversine(lat, lon, near.lat, near.lon) * 1000
 
 	return {
 		lat,

package/lib/address/point/spine-schema.ts

@@ -0,0 +1,59 @@
+/**
+ * @copyright Sister Software.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The shared rooftop schema with an H3 spine column and the layer-interface tables.
+ *
+ *   The OpenStreetMap and G-NAF rooftop builders write this shape. Readers of existing address points
+ *   project only the columns of `address_point` in `#address/point/schema`, so the H3 spine and the
+ *   layer-interface tables leave their query path unchanged.
+ */
+
+import { createLayerCoverageTable, createLayerManifestTable, type layerschemadatabase } from "@mailwoman/core/layers"
+import type { Kysely } from "kysely"
+
+import { ADDRESS_POINT_COLUMNS, type AddressPointTable, createAddressPointTable } from "#address/point/schema"
+
+/**
+ * Jitter-stable address/POI join cell.
+ * Matches @mailwoman/address-id.
+ */
+export const ADDRESS_POINT_H3_RESOLUTION = 9
+
+export interface SpineAddressPointTable extends AddressPointTable {
+	/**
+	 * 48-bit shortened H3 cell stored as a safe SQLite integer.
+	 */
+	h3_cell: number
+}
+
+export interface SpineAddressPointDatabase extends layerschemadatabase {
+	address_point: SpineAddressPointTable
+}
+
+/**
+ * Positional insert order: the shared address-point fields followed by the H3 spine.
+ */
+export const SPINE_ADDRESS_POINT_COLUMNS = [...ADDRESS_POINT_COLUMNS, "h3_cell"] as const
+
+/**
+ * Create the legacy-compatible domain table plus the H3 spine and the layer-interface tables.
+ */
+export async function createSpineAddressPointTables(db: Kysely<SpineAddressPointDatabase>): Promise<void> {
+	await createAddressPointTable(db)
+
+	await db.schema
+		.alterTable("address_point")
+		.addColumn("h3_cell", "integer", (column) => column.notNull())
+		.execute()
+
+	await createLayerManifestTable(db)
+	await createLayerCoverageTable(db)
+}
+
+/**
+ * Build after the bulk load so the spine index is not updated on every streaming insert.
+ */
+export async function createSpineAddressPointIndexes(db: Kysely<SpineAddressPointDatabase>): Promise<void> {
+	await db.schema.createIndex("idx_ap_h3").on("address_point").column("h3_cell").execute()
+}

package/lib/address/point.ts

@@ -7,6 +7,7 @@
 
 import type { AddressPointHit, AddressPointLookup } from "@mailwoman/core/resolver"
 import { DatabaseClient } from "@mailwoman/sqlite/client"
+import { indexExists } from "@mailwoman/sqlite/introspection"
 import type { PathBuilderLike } from "path-ts"
 
 import type { AddressPointDatabase, AddressPointTable } from "#address/point/schema"
@@ -35,6 +36,27 @@ type AddressPointRow = Pick<AddressPointTable, "lat" | "lon" | "source" | "relea
  */
 const SELECT_COLS = "lat, lon, source, release, locality_norm, postcode"
 
+/**
+ * The widest span of numbers a written range covers on one building or block.
+ */
+const WIDEST_HOUSE_NUMBER_RANGE = 200
+
+/**
+ * Returns whether two hyphenated numbers read as one number rather than a range of houses.
+ *
+ * `8-9 Upper St` is a range, and joining it would answer the different house at number 89.
+ * `19-4074` and Queens's `123-45` cannot be ranges, because a range climbs by a short step.
+ */
+function isJoinedHyphenNumber(number: string): boolean {
+	const parts = /^(\d+)-(\d+)$/.exec(number)
+
+	if (!parts) return false
+
+	const span = Number(parts[2]) - Number(parts[1])
+
+	return span < 0 || span > WIDEST_HOUSE_NUMBER_RANGE
+}
+
 /**
  * Finds an address point by exact normalized `(street, number)` within a postcode,
  * a locality, or a bounding box.
@@ -73,6 +95,7 @@ export class AddressPointSqliteLookup<DB extends AddressPointDatabase = AddressP
 	readonly #byLocalityTail:
 		| PreparedGet<[street: StreetKey, number: string, tailPattern: string], AddressPointRow>
 		| undefined
+	readonly #hasStreetIndex: boolean = false
 	readonly #byBbox:
 		| PreparedGet<
 				[street: StreetKey, number: string, minLat: number, maxLat: number, minLon: number, maxLon: number],
@@ -140,6 +163,8 @@ export class AddressPointSqliteLookup<DB extends AddressPointDatabase = AddressP
 				 WHERE locality_norm = ? AND street_norm = ? AND number = ? LIMIT 1`
 			)
 
+			this.#hasStreetIndex = indexExists(this.#db, "idx_ap_street")
+
 			this.#byBbox = prepareGet(
 				this.#db,
 				`SELECT ${SELECT_COLS} FROM address_point
@@ -220,6 +245,23 @@ export class AddressPointSqliteLookup<DB extends AddressPointDatabase = AddressP
 	) {
 		let row = this.#probe(streetNorm, number, query)
 
+		// A hyphenated number can be one number written two ways: Hawaii's `19-4074`
+		// is stored as `194074` by some sources.
+		// This runs ahead of the range fallback, because that fallback reads it as house number 19.
+		if (!row && isJoinedHyphenNumber(number)) {
+			row = this.#probe(streetNorm, number.replace("-", ""), query)
+		}
+
+		// A grid number leads with a compass letter.
+		// One writer's `N3971` is another source's `n 3971`.
+		if (!row) {
+			const grid = /^([nesw])(?:\s+)?(\d+)$/.exec(number)
+
+			if (grid) {
+				row = this.#probe(streetNorm, number.includes(" ") ? `${grid[1]}${grid[2]}` : `${grid[1]} ${grid[2]}`, query)
+			}
+		}
+
 		// The registers store one number per point, so a range such as "385-387" falls back to its low end.
 		if (!row) {
 			const low = /^(\d+[a-z]?)-\d+[a-z]?$/.exec(number)?.[1]
@@ -307,7 +349,8 @@ export class AddressPointSqliteLookup<DB extends AddressPointDatabase = AddressP
 		}
 
 		// The bounding box serves extracts whose points have no postcode or locality, such as OSM.
-		if (!row && query.bbox) {
+		// A database without the street index skips this search, because each miss would scan the whole table.
+		if (!row && query.bbox && this.#hasStreetIndex) {
 			const b = query.bbox
 			const candidate = this.#byBbox!(streetNorm, number, b.minLat, b.maxLat, b.minLon, b.maxLon)
 

package/lib/ancestry/backfill.ts

@@ -6,7 +6,8 @@
  *   Repair the truncated ancestry left by the `parent_id` closure. Must run after `populateAncestors` and before the build freezes.
  */
 
-import { readWOFFeature } from "@mailwoman/core/resources/whosonfirst"
+import { readWOFFeature } from "@mailwoman/gazetteer/whosonfirst"
+import { hierarchyAncestorsInAnyBranch } from "@mailwoman/gazetteer/whosonfirst/hierarchy"
 import type { DatabaseClient } from "@mailwoman/sqlite/client"
 import { PathBuilder, type PathBuilderLike } from "path-ts"
 import { Globerator } from "spliterator/node/fs"
@@ -61,14 +62,6 @@ export async function discoverAdminDataRoots(reposRoot: PathBuilderLike): Promis
 	return roots
 }
 
-// The `aid === id` check filters out a place's own `locality_id`.
-// For a neighborhood, that id identifies a real ancestor.
-function placetypeFromKey(key: string): string | null {
-	if (!key.endsWith("_id")) return null
-
-	return key.slice(0, -3)
-}
-
 /**
  * Insert missing ancestor rows for places whose ancestry chain ended before reaching a country.
  *
@@ -141,22 +134,7 @@ export async function backfillAncestorsFromHierarchy(
 			continue
 		}
 
-		const seen = new Map<number, string>()
-
-		for (const branch of hierarchy) {
-			for (const [key, val] of Object.entries(branch)) {
-				const pt = placetypeFromKey(key)
-
-				if (!pt) continue
-				const aid = Number(val)
-
-				if (!Number.isFinite(aid) || aid <= 0 || aid === id) continue
-
-				if (!seen.has(aid)) {
-					seen.set(aid, pt)
-				}
-			}
-		}
+		const seen = hierarchyAncestorsInAnyBranch(id, hierarchy)
 
 		let present = alreadyPresent.get(id)
 

package/lib/build/candidate.ts

@@ -5,6 +5,7 @@
  */
 
 import { COUNTRY_POPULATION } from "@mailwoman/codex/country"
+import { bareAdminName } from "@mailwoman/codex/country/bare-admin-names"
 import { pathExists } from "@mailwoman/core/fs/readers/stat"
 import { removePath } from "@mailwoman/core/fs/writers"
 import { stringifyJSON } from "@mailwoman/core/json"
@@ -14,6 +15,7 @@ import type { PathBuilderLike } from "path-ts"
 import { POPULATION_CORRECTIONS } from "#build/population-corrections"
 import { explodeAliasBags } from "#candidate/alias-bags"
 import { buildAncestorsSidecar } from "#candidate/ancestors/sidecar"
+import { stageBareAdminNames } from "#candidate/bare-admin-names"
 import { stageCountryDisplayNames } from "#candidate/country-display-names"
 import { foldExtract } from "#candidate/extract-fold"
 import { createCandidateFTS } from "#candidate/fts"
@@ -205,6 +207,8 @@ export interface BuildCandidateResult {
  * Builds the candidate lookup database from a WOF admin database and replaces any existing output.
  * The builder adds an FTS5 trigram index for fuzzy lookup.
  */
+// repo-health-ignore export-name-affix -- writes candidate.db, and the gazetteer
+// pipeline's buildCandidate runs a whole build around it
 export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<BuildCandidateResult> {
 	const progress = opts.onProgress ?? (() => {})
 
@@ -226,7 +230,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
 		const c = (code || "??").toUpperCase()
 		let id = ccodes.get(c)
 
-		if (id === undefined) {
+		if (typeof id !== "number") {
 			id = ccodes.size
 			ccodes.set(c, id)
 		}
@@ -238,7 +242,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
 		const p = pt || ""
 		let id = ptcodes.get(p)
 
-		if (id === undefined) {
+		if (typeof id !== "number") {
 			id = ptcodes.size
 			ptcodes.set(p, id)
 		}
@@ -306,7 +310,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
 		const ptid = ptID(r.placetype as string | null)
 		const rid = regionOf.get(sid) ?? 0
 
-		const wofPop = r.pop === null || r.pop === undefined ? null : Number(r.pop)
+		const wofPop = r.pop === null || !r.pop ? null : Number(r.pop)
 
 		const pop =
 			POPULATION_CORRECTIONS[sid]?.population ??
@@ -406,6 +410,16 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
 		}).toLocaleString()} country surfaces`
 	)
 
+	progress(
+		"bare-admin-names",
+		`${stageBareAdminNames({
+			attrs,
+			iso2ByID: new Map([...ccodes].map(([code, id]) => [id, code])),
+			stageRow,
+			tx: kdb,
+		}).toLocaleString()} bare admin names`
+	)
+
 	if (opts.currencyBackfill) {
 		const nBackfill = await resurrectCurrencyHoles({
 			src,
@@ -429,14 +443,17 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
 
 	progress("aliases", "exploding alias bags")
 
-	const { nAlias, keyCounts, regionOfficialRefused } = explodeAliasBags(src, kdb, attrs, stageRow, {
+	const iso2ByCountryID = new Map([...ccodes].map(([code, id]) => [id, code]))
+
+	const { nAlias, nBare, keyCounts, regionOfficialRefused } = explodeAliasBags(src, kdb, attrs, stageRow, {
 		regionPlacetypeID: ptID("region"),
 		ccID,
+		bareName: (countryID, alias) => bareAdminName(iso2ByCountryID.get(countryID) ?? "", alias),
 	})
 
 	progress(
 		"aliases",
-		`${nAlias.toLocaleString()} aliases` +
+		`${nAlias.toLocaleString()} aliases; ${nBare.toLocaleString()} bare admin aliases` +
 			(regionOfficialRefused
 				? `; ${regionOfficialRefused.toLocaleString()} region aliases refused as another region's official name`
 				: "")

package/lib/build/slim.ts

@@ -54,7 +54,6 @@ export type SlimBuildPhase =
 	| "postcode"
 	| "names"
 	| "place_population"
-	| "coincident_roles"
 	| "place_abbr"
 	| "fts"
 	| "vacuum"
@@ -314,24 +313,6 @@ async function copyFromSource(
 				.onConflict((oc) => oc.doNothing())
 				.execute()
 		}
-
-		// `ancestors` is intentionally not copied, so the derived `coincident_roles`
-		// table is copied and filtered to surviving spr ids.
-		const relationSchema = out
-			.prepare(`SELECT sql FROM src.sqlite_master WHERE type = 'table' AND name = 'coincident_roles'`)
-			.get() as { sql?: string } | undefined
-
-		if (relationSchema?.sql) {
-			progress("coincident_roles", `${inputPath}: copying dual-role relation`)
-			out.exec(relationSchema.sql.replace(/CREATE TABLE/i, "CREATE TABLE IF NOT EXISTS"))
-
-			out.exec(
-				`INSERT OR IGNORE INTO coincident_roles SELECT * FROM src.coincident_roles
-					WHERE admin_id IN (SELECT id FROM spr) AND locality_id IN (SELECT id FROM spr)`
-			)
-
-			out.exec(`CREATE INDEX IF NOT EXISTS coincident_roles_by_admin ON coincident_roles (admin_id)`)
-		}
 	} finally {
 		out.exec(`DETACH DATABASE src;`)
 	}

package/lib/candidate/alias-bags.ts

@@ -61,9 +61,20 @@ export function explodeAliasBags(
 	out: DatabaseClient<CandidateDatabase>,
 	attrs: Map<number, PlaceAttrs>,
 	stageRow: StageRow,
-	opts: { regionPlacetypeID?: number; ccID?: (code: string | null) => number } = {}
-): { nAlias: number; keyCounts: Map<number, number>; regionOfficialRefused: number } {
+	opts: {
+		regionPlacetypeID?: number
+		ccID?: (code: string | null) => number
+		/**
+		 * The bare form of an alias in its place's country, staged as one more alias: `渋谷` for the alias `渋谷区`.
+		 *
+		 * A Japanese place's primary name is romanized, so its suffixed name reaches
+		 * the candidate only as an alias.
+		 */
+		bareName?: (countryID: number, alias: string) => string | null
+	} = {}
+): { nAlias: number; nBare: number; keyCounts: Map<number, number>; regionOfficialRefused: number } {
 	let nAlias = 0
+	let nBare = 0
 	let regionOfficialRefused = 0
 	const keyCounts = new Map<number, number>()
 
@@ -87,24 +98,31 @@ export function explodeAliasBags(
 		// includes surrounding whitespace and the last one is empty.
 		// `normalizeLocalityForKey` folds both away.
 		for (const piece of alt.split(ALIAS_SEPARATOR)) {
-			const k = normalizeLocalityForKey(piece)
+			const bare = opts.bareName?.(a.cid, piece.trim())
+			const keys = [normalizeLocalityForKey(piece), bare ? normalizeLocalityForKey(bare) : ""]
 
-			if (!k || seen.has(k)) continue
-			seen.add(k)
+			for (const [position, k] of keys.entries()) {
+				if (!k || seen.has(k)) continue
+				seen.add(k)
 
-			if (isRegion) {
-				const holder = official.get(`${a.cid}\0${k}`)
+				if (isRegion) {
+					const holder = official.get(`${a.cid}\0${k}`)
 
-				if (typeof holder === "number" && holder !== id) {
-					regionOfficialRefused++
+					if (typeof holder === "number" && holder !== id) {
+						regionOfficialRefused++
 
-					continue
+						continue
+					}
 				}
-			}
 
-			stageRow(k, a, id, 0)
+				stageRow(k, a, id, 0)
 
-			nAlias++
+				if (position) {
+					nBare++
+				} else {
+					nAlias++
+				}
+			}
 		}
 
 		keyCounts.set(id, seen.size)
@@ -112,5 +130,5 @@ export function explodeAliasBags(
 
 	out.exec("COMMIT")
 
-	return { nAlias, keyCounts, regionOfficialRefused }
+	return { nAlias, nBare, keyCounts, regionOfficialRefused }
 }

package/lib/candidate/bare-admin-names.ts

@@ -0,0 +1,54 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file A candidate build pass that keys each admin place under its bare name as well as its full one.
+ */
+
+import { bareAdminName } from "@mailwoman/codex/country/bare-admin-names"
+
+import type { PlaceAttrs, StageRow } from "#candidate/place-attrs"
+import { normalizeLocalityForKey } from "#street/normalize"
+
+/**
+ * Stages an alias row under the bare form of every place name a country writes with an admin-unit suffix.
+ *
+ * A bare `渋谷` otherwise reaches only the places whose full name is `渋谷`, a Hyogo
+ * and a Chiba locality with no recorded population, and never Tokyo's `渋谷区`.
+ * The alias row includes the ward's own population, so the candidate ordering
+ * ranks it above the unmeasured exact matches.
+ *
+ * @returns The number of alias rows staged.
+ */
+export function stageBareAdminNames(ctx: {
+	attrs: Map<number, PlaceAttrs>
+	iso2ByID: Map<number, string>
+	stageRow: StageRow
+	tx: { exec(sql: string): void }
+}): number {
+	let staged = 0
+
+	ctx.tx.exec("BEGIN")
+
+	for (const [sid, a] of ctx.attrs) {
+		const iso2 = ctx.iso2ByID.get(a.cid)
+
+		if (!iso2) continue
+
+		const bare = bareAdminName(iso2, a.name)
+
+		if (!bare) continue
+
+		const key = normalizeLocalityForKey(bare)
+
+		if (!key || key === a.pkey) continue
+
+		ctx.stageRow(key, a, sid, 0)
+
+		staged++
+	}
+
+	ctx.tx.exec("COMMIT")
+
+	return staged
+}

package/lib/candidate/importance.ts

@@ -29,7 +29,7 @@
  *   population assigned it.
  */
 
-import { haversineKm } from "@mailwoman/spatial"
+import { haversine } from "@mailwoman/spatial/geometries/point/distance"
 import { DatabaseClient } from "@mailwoman/sqlite/client"
 import type { PathBuilderLike } from "path-ts"
 
@@ -144,7 +144,7 @@ export class ImportanceIndex {
 		let bestKm = Infinity
 
 		for (const place of group) {
-			const km = haversineKm(lat, lon, place.lat, place.lon)
+			const km = haversine(lat, lon, place.lat, place.lon)
 
 			if (km < bestKm) {
 				bestKm = km