@mailwoman/resolver-wof-sqlite 10.0.0 → 10.2.0

package/lib/name-score.ts

@@ -3,13 +3,13 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Soft name matching for locality ranking: case/diacritic folding, padded character trigrams, and the
+ *   Soft name matching for locality ranking: case/diacritic folding, padded character trigrams and the
  *   trigram-Jaccard score built on them. Shared by the FTS lookup and the candidate-table backend so
  *   both rank the same typo identically — the whole reason this is one module rather than two copies.
  */
 
 /**
- * Case-fold + strip diacritics + collapse punctuation — for the coord-first soft name match.
+ * Case-fold + strip diacritics + collapse punctuation, for the coord-first soft name match.
  */
 export function cfNormalize(s: string): string {
 	return s
@@ -35,11 +35,13 @@ function trigrams(s: string): Set<string> {
 }
 
 /**
- * Character-trigram Jaccard ∈ [0,1] — tolerant of the swallowed-leading-char fragments ("auen" vs "plauen") and minor
- * misspellings without a heavyweight edit-distance pass. The fuzzy component of {@link softNameScore}, and private to
- * this module: the candidate backend scores typos with a WORD-level measure instead, on receipts that falsify trigram
- * Jaccard for that job (see `candidate-lookup.ts`'s `FUZZY_FETCH`). A shared name here would assert a parity the two
- * backends do not have.
+ * Character-trigram Jaccard ∈ [0,1] — tolerant of the swallowed-leading-char fragments
+ * ("auen" vs "plauen") and minor misspellings without a heavyweight edit-distance pass.
+ *
+ * The fuzzy component of {@link softNameScore}, and private to this module:
+ * the candidate backend scores typos with a word-level measure instead, on receipts that
+ * falsify trigram Jaccard for that job (see `candidate-lookup.ts`'s `FUZZY_FETCH`).
+ * A shared name here would assert a parity the two backends do not have.
  */
 function trigramJaccard(a: string, b: string): number {
 	const A = trigrams(a)

package/lib/nsul/lookup.ts

@@ -3,38 +3,38 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Node reader for `nsul.db` — the GB UPRN → unit-postcode register (`nsul/schema.ts`). Two probes,
+ *   Node reader for `nsul.db` — the GB uprn → unit-postcode register (`nsul/schema.ts`). Two probes,
  *   both synchronous `.prepare()` hits in the `uprn/lookup.ts` style:
  *
- *   - **`postcodeForUPRN(uprn)`**: primary-key hit on the `WITHOUT ROWID` table.
- *   - **`uprnsForPostcode(postcode)`**: the `pcds_compact` index, answering every UPRN the register
+ *   - **`postcodeForUPRN(uprn)`**: primary-key hit on the `without rowid` table.
+ *   - **`uprnsForPostcode(postcode)`**: the `pcds_compact` index, answering every uprn the register
  *     assigns to one unit postcode with the point OS publishes for each — the "assigned points and
  *     their bound" the physical-constraint design record names as the prior's soft structure.
  *
  *   ## `null` and `[]` are claims, scoped by coverage
  *
- *   The register designates GB complete (every UPRN in AddressBase with a Code-Point Open postcode),
+ *   The register designates GB complete (every uprn in AddressBase with a Code-Point Open postcode),
  *   and the builder writes `layer_coverage` with basis `designated` for every cell the register
- *   touches. Inside a covered cell an empty answer is evidence of absence: no GB UPRN by that number
- *   carries a Code-Point postcode, or no UPRN carries that postcode. Two absences the reader cannot
- *   tell from those are recorded in `nsul_meta` as counts rather than as rows — a UPRN whose `PCDS` is
- *   null (its postcode is not in Code-Point Open) and one Open UPRN publishes no coordinate for — so a
- *   caller building negative evidence reads the coverage table and those counts, not this reader
- *   alone. Outside coverage (Northern Ireland, the Isle of Man, the Channel Islands) the answer is
- *   UNKNOWN, per the meaning-of-zero rule.
+ *   touches. Inside a covered cell an empty answer is evidence of absence: no GB uprn by that number
+ *   has a Code-Point postcode, or no UPRN has that postcode. Two absences the reader cannot
+ *   tell from those are recorded in `nsul_meta` as counts rather than as rows. A uprn whose `pcds` is
+ *   null (its postcode is not in Code-Point Open) and one Open uprn publishes no coordinate for — so a
+ *   caller building negative evidence reads the coverage table and those counts rather than this reader
+ *   by itself. Outside coverage (Northern Ireland, the Isle of Man, the Channel Islands) the answer is
+ *   unknown, per the meaning-of-zero rule.
  */
 
-import { DatabaseClient } from "@mailwoman/sqlite/client"
+import { SQLiteLookup, type SQLiteLookupOptions } from "@mailwoman/sqlite/lookup"
 
 import { compactPostcode, type NSULDatabase } from "#nsul/schema"
 import { prepareAll, prepareGet } from "#sqlite-utils"
 
 /**
- * The unit postcode the register assigns to a UPRN, in both stored forms.
+ * The unit postcode the register assigns to a uprn, in both stored forms.
  */
 export interface NSULPostcode {
 	/**
-	 * As NSUL writes it — `RG40 4HR`.
+	 * As nsul writes it — `RG40 4HR`.
 	 */
 	pcds: string
 	/**
@@ -44,7 +44,7 @@ export interface NSULPostcode {
 }
 
 /**
- * One UPRN assigned to a unit postcode, with the point `uprn.db` holds for it.
+ * One uprn assigned to a unit postcode, with the point `uprn.db` holds for it.
  */
 export interface NSULAssignedPoint {
 	uprn: number
@@ -56,16 +56,13 @@ export interface NSULAssignedPoint {
 	h3Cell: number
 }
 
-export interface NSULLookupOpts {
-	/**
-	 * Path to a `nsul.db` built by `mailwoman`'s gazetteer pipeline. Opened read-only.
-	 */
-	databasePath?: string
-	/**
-	 * Pre-opened handle (tests / shared connections). Mutually exclusive with `databasePath`.
-	 */
-	database?: DatabaseClient<NSULDatabase>
-}
+/**
+ * Where an {@link NSULLookup} reads from.
+ *
+ * The source is a `nsul.db` built by `mailwoman`'s gazetteer pipeline, opened read-only,
+ * or a connection the caller already holds.
+ */
+export type NSULLookupOpts = SQLiteLookupOptions<NSULDatabase>
 
 interface PostcodeRow {
 	pcds: string
@@ -80,43 +77,31 @@ interface PointRow {
 }
 
 /**
- * Node reader over `nsul.db`. `implements Disposable` so callers can `using lookup = new NSULLookup(...)` — the same
- * precedent as `UPRNLookup`.
+ * Node reader over `nsul.db`.
+ *
+ * Disposable, so callers can `using lookup = new NSULLookup(...)`.
  */
-export class NSULLookup implements Disposable {
-	#db: DatabaseClient<NSULDatabase>
-	/**
-	 * Resources this instance opened. A connection handed in by a caller is NOT in here, so disposal cannot reach it —
-	 * ownership is membership rather than a flag a later branch has to check.
-	 */
-	readonly #resources = new DisposableStack()
-
-	readonly #postcodeProbe: (uprn: number) => PostcodeRow | undefined
+export class NSULLookup extends SQLiteLookup<NSULDatabase> {
+	readonly #postcodeProbe: (uprn: number) => PostcodeRow | null
 	readonly #pointsProbe: (pcdsCompact: string) => PointRow[]
 
 	constructor(opts: NSULLookupOpts) {
-		if (opts.database) {
-			this.#db = opts.database
-		} else if (opts.databasePath) {
-			this.#db = this.#resources.use(new DatabaseClient<NSULDatabase>(opts.databasePath, { readOnly: true }))
-		} else {
-			throw new Error("NSULLookup needs `databasePath` or `database`")
-		}
+		super(opts)
 
 		this.#postcodeProbe = prepareGet<[number], PostcodeRow, NSULDatabase>(
-			this.#db,
+			this.database,
 			"SELECT pcds, pcds_compact FROM uprn_postcode WHERE uprn = ?"
 		)
 
 		this.#pointsProbe = prepareAll<[string], PointRow, NSULDatabase>(
-			this.#db,
+			this.database,
 			"SELECT uprn, lat, lon, h3_cell FROM uprn_postcode WHERE pcds_compact = ? ORDER BY uprn"
 		)
 	}
 
 	/**
-	 * The unit postcode the register assigns to `uprn`, or `null` when the register holds no row for it (see the module
-	 * docstring for what that `null` claims).
+	 * The unit postcode the register assigns to `uprn`, or `null` when the register holds
+	 * no row for it (see the module docstring for what that `null` claims).
 	 */
 	postcodeForUPRN(uprn: number): NSULPostcode | null {
 		const row = this.#postcodeProbe(uprn)
@@ -125,9 +110,11 @@ export class NSULLookup implements Disposable {
 	}
 
 	/**
-	 * Every UPRN the register assigns to one unit postcode, with its published point, in ascending UPRN order. The key is
-	 * compacted through {@link compactPostcode} first, so `PO21 1HR` and `PO211HR` answer identically. An empty array is
-	 * the register's answer, scoped as the module docstring says.
+	 * Every uprn the register assigns to one unit postcode, with its published point, in ascending uprn order.
+	 *
+	 * The key is compacted through {@link compactPostcode} first, so `PO21 1HR`
+	 * and `PO211HR` answer identically.
+	 * An empty array is the register's answer, scoped as the module docstring describes.
 	 */
 	uprnsForPostcode(postcode: string): NSULAssignedPoint[] {
 		return this.#pointsProbe(compactPostcode(postcode)).map((row) => ({
@@ -137,8 +124,4 @@ export class NSULLookup implements Disposable {
 			h3Cell: row.h3_cell,
 		}))
 	}
-
-	[Symbol.dispose](): void {
-		this.#resources[Symbol.dispose]()
-	}
 }

package/lib/nsul/schema.ts

@@ -3,44 +3,21 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Typed schema for `nsul.db` — the GB UPRN → unit-postcode register: the ONS **National Statistics
- *   UPRN Lookup** (NSUL) joined to the WGS84 point OS Open UPRN publishes for the same UPRN. One row per
- *   GB UPRN that carries a unit postcode AND has a published coordinate; the postcode is Code-Point
- *   Open's, so this table and `postcode-gb.bin` agree by construction on the universe of unit postcodes.
- *
- *   ## Why this table exists
- *
- *   Code-Point Open gives one coordinate per unit postcode; Open UPRN gives one coordinate per
- *   addressable object; neither says which object carries which postcode. NSUL does, as field `PCDS`,
- *   under OGL-UK-3.0. The `PO`-area measurement that produced this design (recorded on #1975) showed a
- *   nearest-centroid reconstruction of the register is exact on 69.6% of 531,266 UPRNs and a building
- *   footprint adds nothing, so the register is stored rather than inferred.
- *
- *   ## Shape
- *
- *   `uprn_postcode` is `WITHOUT ROWID` keyed on the UPRN: small fixed rows, probed by primary key, with
- *   one secondary index on `pcds_compact`. The postcode is stored twice on purpose — `pcds` exactly as
- *   NSUL writes it (`RG40 4HR`, one space) and `pcds_compact` with the space removed (`RG404HR`), which
- *   is the form Code-Point Open's `spr.name` uses, so the two artifacts join on a column rather than an
- *   expression. `lat`/`lon`/`h3_cell` are COPIED from `uprn.db` at build time by UPRN join — OS's own
- *   WGS84 columns, never a reprojection of NSUL's `GRIDGB1E`/`GRIDGB1N` — so a coordinate here is
- *   byte-identical to the one `uprn.db` holds for the same UPRN.
- *
- *   The DB also embeds the layer-contract tables from `@mailwoman/core/layers`; the builder
- *   (`packages/mailwoman/lib/gazetteer-pipeline/nsul-layer.ts`) writes the manifest and coverage.
+ * Typed schema for `nsul.db`, the GB UPRN-to-unit-postcode register.
  */
 
-import type { LayerContractDatabase } from "@mailwoman/core/layers"
+import type { layerschemadatabase } from "@mailwoman/core/layers"
 import { sql, type Kysely } from "kysely"
 
 /**
- * Resolution of the `h3_cell` column — the same res-9 spine `uprn.db` keys on, because the value is copied from it.
+ * Resolution of the `h3_cell` column, matching the res-9 spine `uprn.db` keys on
+ * because the value is copied from it.
  */
 export const NSUL_H3_RESOLUTION = 9
 
 /**
- * Resolution of the layer's `layer_coverage` cells — coarse, per the contract, and the same as `uprn.db`'s so the two
- * layers' coverage tables describe the same cells.
+ * Resolution of the layer's `layer_coverage` cells, matching `uprn.db`'s
+ * so both coverage tables describe the same cells.
  */
 export const NSUL_COVERAGE_H3_RESOLUTION = 6
 
@@ -49,16 +26,16 @@ export const NSUL_COVERAGE_H3_RESOLUTION = 6
  */
 export interface UPRNPostcodeTable {
 	/**
-	 * The Unique Property Reference Number — up to 12 digits, so always within `Number.MAX_SAFE_INTEGER`.
+	 * The Unique Property Reference Number, up to 12 digits, so always within `Number.MAX_SAFE_INTEGER`.
 	 */
 	uprn: number
 	/**
-	 * The unit postcode as NSUL writes it — outward code, one space, inward code (`RG40 4HR`).
+	 * The unit postcode as nsul writes it: outward code, one space, inward code (`RG40 4HR`).
 	 */
 	pcds: string
 	/**
-	 * {@link pcds} with the space removed (`RG404HR`) — the form Code-Point Open's `spr.name` carries, and the column
-	 * `uprnsForPostcode` probes.
+	 * {@link pcds} with the space removed (`RG404HR`), the form Code-Point Open's
+	 * `spr.name` contains and `uprnsForPostcode` probes.
 	 */
 	pcds_compact: string
 	/**
@@ -76,23 +53,20 @@ export interface UPRNPostcodeTable {
 }
 
 /**
- * Build-provenance key/value pairs the fixed `layer_manifest` columns have no room for: the accounting counts, the
- * header as found, the per-region row counts, the four attribution lines.
+ * Build-provenance key/value pairs the fixed `layer_manifest` columns have no room for.
  */
 export interface NSULMetaTable {
 	key: string
 	value: string
 }
 
-export interface NSULDatabase extends LayerContractDatabase {
+export interface NSULDatabase extends layerschemadatabase {
 	uprn_postcode: UPRNPostcodeTable
 	nsul_meta: NSULMetaTable
 }
 
 /**
- * The compact form of a unit postcode: every space removed, upper-cased. `RG40 4HR` → `RG404HR`. The ONE derivation
- * both the builder and every consumer share, so a caller holding NSUL's spaced form and one holding Code-Point's
- * compact form reach the same key.
+ * The compact form of a unit postcode, every space removed and upper-cased.
  */
 export function compactPostcode(pcds: string): string {
 	return pcds.replaceAll(/\s+/g, "").toUpperCase()
@@ -120,10 +94,9 @@ export async function createNSULMetaTable(db: Kysely<NSULDatabase>): Promise<voi
 }
 
 /**
- * The `pcds_compact` index the `uprnsForPostcode` probe reads. Builders call this AFTER the bulk load
- * (index-after-load). There is no index on the spaced `pcds`: it is derivable from `pcds_compact` through
- * {@link compactPostcode}, and a second index over 40 million rows would buy nothing a caller cannot get by compacting
- * its key first.
+ * The `pcds_compact` index the `uprnsForPostcode` probe reads.
+ *
+ * There is no index on the spaced `pcds` because it is derivable through {@link compactPostcode}.
  */
 export async function createNSULIndexes(db: Kysely<NSULDatabase>): Promise<void> {
 	await db.schema.createIndex("uprn_postcode_pcds_compact").on("uprn_postcode").column("pcds_compact").execute()

package/lib/paths.ts

@@ -0,0 +1,159 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Where the databases this resolver reads live under the data root's `db/` group.
+ *
+ *   Each constant is a lazy path builder: it reads `$MAILWOMAN_DATA_ROOT` when a path is requested,
+ *   so a later environment change or a test stub is honored.
+ *   Call it with a filename, as in `wofDatabasePath("admin-global-priority.db")`.
+ *   A caller holding another data root uses the matching `…DatabaseRoot(dataRoot)` function.
+ */
+
+import { databaseRootPath, dataRootPath } from "@mailwoman/core/data-root"
+import type { PathBuilder, PathBuilderLike } from "path-ts"
+
+/**
+ * `<dataRoot>/db/wof`, which holds the admin, candidate and postcode databases.
+ */
+export function wofDatabaseRoot(dataRoot: PathBuilderLike): PathBuilder {
+	return databaseRootPath(dataRoot)("wof")
+}
+
+/**
+ * `$MAILWOMAN_DATA_ROOT/db/wof`.
+ *
+ * @see {@link wofDatabaseRoot} for another data root.
+ */
+export const wofDatabasePath: PathBuilder = wofDatabaseRoot(dataRootPath())
+
+/**
+ * `<dataRoot>/db/address-points`, which holds the national rooftop address-point databases.
+ */
+export function addressPointDatabaseRoot(dataRoot: PathBuilderLike): PathBuilder {
+	return databaseRootPath(dataRoot)("address-points")
+}
+
+/**
+ * `$MAILWOMAN_DATA_ROOT/db/address-points`.
+ */
+export const addressPointDatabasePath: PathBuilder = addressPointDatabaseRoot(dataRootPath())
+
+/**
+ * `<dataRoot>/db/interpolation`, which holds the street-segment interpolation databases.
+ */
+export function interpolationDatabaseRoot(dataRoot: PathBuilderLike): PathBuilder {
+	return databaseRootPath(dataRoot)("interpolation")
+}
+
+/**
+ * `$MAILWOMAN_DATA_ROOT/db/interpolation`.
+ */
+export const interpolationDatabasePath: PathBuilder = interpolationDatabaseRoot(dataRootPath())
+
+/**
+ * `<dataRoot>/db/poi`, which holds `poi.db` and its per-source builds.
+ */
+export function poiDatabaseRoot(dataRoot: PathBuilderLike): PathBuilder {
+	return databaseRootPath(dataRoot)("poi")
+}
+
+/**
+ * `$MAILWOMAN_DATA_ROOT/db/poi`.
+ */
+export const poiDatabasePath: PathBuilder = poiDatabaseRoot(dataRootPath())
+
+/**
+ * `$MAILWOMAN_DATA_ROOT/db/uprn`, which holds the UPRN point database.
+ */
+export const uprnDatabasePath: PathBuilder = databaseRootPath(dataRootPath())("uprn")
+
+/**
+ * `$MAILWOMAN_DATA_ROOT/db/nsul`, which holds the NSUL postcode layer and its source vintages.
+ */
+export const nsulDatabasePath: PathBuilder = databaseRootPath(dataRootPath())("nsul")
+
+/**
+ * Default WOF extracts for FTS when `--wof-db` is not provided.
+ *
+ * Includes the global admin-priority extract plus postcode extracts.
+ * The routing in `pickExtractForPlacetype` sends each postcode query to the
+ * extract that claims that country.
+ *
+ * All paths are under `dataRoot`.
+ * It defaults to `$MAILWOMAN_DATA_ROOT`, and callers may pass `--data-root`.
+ * Returns a fresh array each call.
+ *
+ * Callers usually filter with `existsSync`, so missing files are skipped.
+ *
+ * This runtime list is intentionally smaller than `DEFAULT_POSTCODE_DATABASES`
+ * (`mailwoman/gazetteer-pipeline/index.ts`), because these databases are attached live at boot.
+ *
+ * Notes:
+ * - `postalcode-geonames-tail.db` now contains FI/CZ/SK/SI/DK/no/HR/PL/SE.
+ * - `postalcode-ni-osm.db` is build-local (ODbL, OSM `addr:postcode`) and may be absent.
+ *   A missing file is filtered out.
+ *   This is the only GB-claiming extract in the list.
+ *   Code-Point Open is absent.
+ */
+export function wofExtractPaths(dataRoot: PathBuilderLike = dataRootPath()): string[] {
+	return Object.values(wofExtractPathsByName(dataRoot))
+}
+
+/**
+ * The extract set {@link wofExtractPaths} lists, keyed by role so a caller can
+ * name one without indexing a tuple.
+ */
+export interface WOFExtractPaths {
+	/**
+	 * The global admin-priority extract, where every admin lookup starts.
+	 */
+	adminGlobalPriority: string
+	/**
+	 * US ZIP codes.
+	 */
+	postalcodeUS: string
+	/**
+	 * The nine-country namesake set FI/CZ/SK/SI/DK/no/HR/PL/SE (see {@link wofExtractPaths}).
+	 */
+	postalcodeGeonamesTail: string
+	/**
+	 * The international postcode extract (FR/DE/ES/IT/NL and the other countries
+	 * routed here by `pickExtractForPlacetype`).
+	 */
+	postalcodeIntl: string
+	/**
+	 * The NL PC6 full-postcode extract from CBS via pdok (`scripts/build-postalcode-nl-pc6.ts`), the
+	 * data the lookup's NL PC6 ladder (`"1012 LG"` → joined `"1012LG"` → 4-digit stem) resolves against.
+	 */
+	postalcodeNLPC6: string
+	/**
+	 * Northern Ireland (BT) from OpenStreetMap, 4,757 of 50,032 live NI postcodes (9.5 %).
+	 *
+	 * It is the only coverage for the hole Code-Point Open leaves.
+	 *
+	 * ODbL, build-local, 2.5 MB.
+	 * A miss on a BT code means not attested in OSM.
+	 *
+	 * An unknown postcode abstains, so the extract is strictly additive.
+	 * Rebuild: `mailwoman gazetteer build postcode-ni-osm`.
+	 */
+	postalcodeNIOSM: string
+}
+
+/**
+ * {@link wofExtractPaths} as a record keyed by source, in the same order the runtime attaches them.
+ */
+export function wofExtractPathsByName(dataRoot: PathBuilderLike = dataRootPath()): WOFExtractPaths {
+	const wof = wofDatabaseRoot(dataRoot)
+
+	return {
+		adminGlobalPriority: wof("admin-global-priority.db").toString(),
+		postalcodeUS: wof("postalcode-us.db").toString(),
+		postalcodeGeonamesTail: wof("postalcode-geonames-tail.db").toString(),
+		postalcodeIntl: wof("postalcode-intl.db").toString(),
+		postalcodeNLPC6: wof("postalcode-nl-pc6.db").toString(),
+		postalcodeNIOSM: wof("postalcode-ni-osm.db").toString(),
+	}
+}