@mailwoman/resolver-wof-sqlite 10.3.0 → 10.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/lib/address/point/schema.ts +1 -1
- package/lib/address/point.ts +81 -3
- package/lib/candidate/ancestors/sidecar.ts +3 -3
- package/lib/candidate/importance.ts +2 -2
- package/lib/candidate/own-name.ts +1 -1
- package/lib/coincident-roles.ts +1 -1
- package/lib/currency-backfill.ts +2 -2
- package/lib/extracts.ts +1 -1
- package/lib/fst/autocomplete.ts +3 -3
- package/lib/fst/deserialize-web.ts +1 -1
- package/lib/fst/format.ts +1 -1
- package/lib/geonames/postal.ts +3 -3
- package/lib/interpolation.ts +2 -2
- package/lib/lookup.ts +1 -1
- package/lib/nsul/lookup.ts +33 -0
- package/lib/nsul/schema.ts +6 -0
- package/lib/paths.ts +1 -1
- package/lib/place-importance-schema.ts +1 -1
- package/lib/poi/lookup.ts +2 -2
- package/lib/polygon-schema.ts +1 -1
- package/lib/postal/city/alias/lookup.ts +1 -1
- package/lib/postal/city/alias/schema.ts +2 -2
- package/lib/proximity-rerank.ts +1 -1
- package/lib/search-fetch.ts +5 -5
- package/lib/street/centroid/schema.ts +1 -1
- package/lib/street/morphology/fst/builder.ts +34 -14
- package/lib/street/normalize.ts +1 -1
- package/lib/street/segment-schema.ts +1 -1
- package/lib/types.ts +2 -1
- package/lib/uprn/lookup.ts +1 -1
- package/lib/weights-overlay-linker.ts +41 -1
- package/out/address/point/schema.js +1 -1
- package/out/address/point/schema.js.map +1 -1
- package/out/address/point.d.ts +9 -0
- package/out/address/point.d.ts.map +1 -1
- package/out/address/point.js +60 -2
- package/out/address/point.js.map +1 -1
- package/out/candidate/ancestors/sidecar.js +3 -3
- package/out/candidate/ancestors/sidecar.js.map +1 -1
- package/out/candidate/importance.d.ts +2 -2
- package/out/candidate/importance.js +1 -1
- package/out/candidate/own-name.js +1 -1
- package/out/coincident-roles.d.ts +1 -1
- package/out/coincident-roles.js +1 -1
- package/out/currency-backfill.d.ts +2 -2
- package/out/extracts.d.ts +1 -1
- package/out/fst/autocomplete.js +3 -3
- package/out/fst/deserialize-web.js +1 -1
- package/out/fst/format.d.ts +1 -1
- package/out/fst/format.js +1 -1
- package/out/geonames/postal.d.ts +3 -3
- package/out/geonames/postal.js +2 -2
- package/out/interpolation.js +2 -2
- package/out/lookup.js +1 -1
- package/out/nsul/lookup.d.ts +18 -0
- package/out/nsul/lookup.d.ts.map +1 -1
- package/out/nsul/lookup.js +11 -0
- package/out/nsul/lookup.js.map +1 -1
- package/out/nsul/schema.d.ts +5 -0
- package/out/nsul/schema.d.ts.map +1 -1
- package/out/nsul/schema.js +1 -0
- package/out/nsul/schema.js.map +1 -1
- package/out/paths.d.ts +1 -1
- package/out/paths.js +1 -1
- package/out/place-importance-schema.d.ts +1 -1
- package/out/poi/lookup.d.ts +1 -1
- package/out/poi/lookup.js +1 -1
- package/out/polygon-schema.d.ts +1 -1
- package/out/postal/city/alias/lookup.d.ts +1 -1
- package/out/postal/city/alias/lookup.js +1 -1
- package/out/postal/city/alias/schema.d.ts +2 -2
- package/out/postal/city/alias/schema.js +1 -1
- package/out/proximity-rerank.js +1 -1
- package/out/proximity-rerank.js.map +1 -1
- package/out/search-fetch.js +5 -5
- package/out/search-fetch.js.map +1 -1
- package/out/street/centroid/schema.d.ts +1 -1
- package/out/street/morphology/fst/builder.d.ts +21 -9
- package/out/street/morphology/fst/builder.d.ts.map +1 -1
- package/out/street/morphology/fst/builder.js +14 -5
- package/out/street/morphology/fst/builder.js.map +1 -1
- package/out/street/normalize.js +1 -1
- package/out/street/normalize.js.map +1 -1
- package/out/street/segment-schema.d.ts +1 -1
- package/out/street/segment-schema.js +1 -1
- package/out/types.d.ts +2 -1
- package/out/types.d.ts.map +1 -1
- package/out/uprn/lookup.js +1 -1
- package/out/weights-overlay-linker.d.ts +11 -0
- package/out/weights-overlay-linker.d.ts.map +1 -1
- package/out/weights-overlay-linker.js +22 -1
- package/out/weights-overlay-linker.js.map +1 -1
- package/package.json +10 -10
package/README.md
CHANGED
|
@@ -33,14 +33,14 @@ for (const c of candidates) {
|
|
|
33
33
|
}
|
|
34
34
|
```
|
|
35
35
|
|
|
36
|
-
## A database that cannot answer
|
|
36
|
+
## A database that cannot answer throws on construction
|
|
37
37
|
|
|
38
38
|
Databases are `ATTACH`ed by a schema name **derived from the filename**, and queries route to them by matching that name against the requested placetype — `postalcode_us` serves `postalcode`. Two ways that used to fail without logging, and both now throw when you build the lookup:
|
|
39
39
|
|
|
40
|
-
- The name does not route. `postcode-ca-overture.db` derives `postcode_ca_overture`, and the router tests `startsWith("postalcode_")` — **"postcode" is not "postalcode"**. It held 843,739 Canadian codes and answered every query with zero hits
|
|
41
|
-
- The database
|
|
40
|
+
- The name does not route. `postcode-ca-overture.db` derives `postcode_ca_overture`, and the router tests `startsWith("postalcode_")` — **"postcode" is not "postalcode"**. It held 843,739 Canadian codes and answered every query with zero hits. A caller could not tell that result from "this country has no places".
|
|
41
|
+
- The database has `spr` but no `place_search`. It routes, then dies mid-`SELECT`.
|
|
42
42
|
|
|
43
|
-
The predicate is the **table rather than the filename**: a database
|
|
43
|
+
The predicate is the **table rather than the filename**: a database with an `spr` table is claiming to be a place database, and every lookup path here reaches the FTS index. A relation-table database like `postcode-locality-<cc>.db` has no `spr`, never makes that claim, and is exempt. That exemption keeps the documented default database list working.
|
|
44
44
|
|
|
45
45
|
```
|
|
46
46
|
WOFSQLitePlaceLookup: …/postcode-ca-overture.db carries "spr" but no "place_search" table, so it
|
|
@@ -118,8 +118,8 @@ The CLI:
|
|
|
118
118
|
- Populates it from `spr` + `names` (alternate-name concatenation included).
|
|
119
119
|
- Builds the `place_bbox` R*Tree virtual table from `spr.min\_*`/`spr.max\_\*` columns for the
|
|
120
120
|
proximity + bbox query support.
|
|
121
|
-
- Reports progress to stderr per phase
|
|
122
|
-
|
|
121
|
+
- Reports progress to stderr per phase, in the order `checking`, `creating`, `populating`, `creating-bbox`,
|
|
122
|
+
`populating-bbox` and `done`.
|
|
123
123
|
- Exits 0 with a no-op message if both indexes already exist.
|
|
124
124
|
|
|
125
125
|
```bash
|
|
@@ -131,7 +131,7 @@ mailwoman gazetteer build fts /path/to/wof.db --drop
|
|
|
131
131
|
|
|
132
132
|
### Slim builds (retired CLI — module only)
|
|
133
133
|
|
|
134
|
-
Builds a trimmed WOF SQLite distribution sized for browser-side deployments (Path B of the demo plan). The full admin-US distribution is ~4 GB
|
|
134
|
+
Builds a trimmed WOF SQLite distribution sized for browser-side deployments (Path B of the demo plan). The full admin-US distribution is ~4 GB. A slim US bundle with the top-1k localities by population plus all postcodes lands at **~35 MB**, small enough to ship as a static asset.
|
|
135
135
|
|
|
136
136
|
```bash
|
|
137
137
|
# Defaults: --top 1000 localities, --countries US, drops geojson after building aux tables
|
|
@@ -158,7 +158,7 @@ What survives in the slim DB:
|
|
|
158
158
|
- All `names` rows for the selected place IDs
|
|
159
159
|
- Fresh `place_search` (FTS5), `place_bbox` (R\*Tree), `place_population` aux tables
|
|
160
160
|
|
|
161
|
-
What gets dropped: the `geojson` table
|
|
161
|
+
What gets dropped: the `geojson` table. That table is build-time only. `lookup.ts` never reads it at query time, and it accounts for ~95% of the on-disk size. The `place_population` aux table consumes `wof:population` from geojson before we drop it.
|
|
162
162
|
|
|
163
163
|
`WOFSQLitePlaceLookup` opens the slim DB without any code change. Out-of-set queries (a locality not in the top-K) correctly return zero hits.
|
|
164
164
|
|
|
@@ -186,7 +186,7 @@ The resolver scores candidates by:
|
|
|
186
186
|
4. - `countryMatchBoost` when the country filter matches.
|
|
187
187
|
5. - `directChildBoost` / `descendantBoost` when `parentId` is set.
|
|
188
188
|
6. - `proximityBoost / (1 + distanceKm / proximityScaleKm)` when `near: {lat, lon}` is set — decays
|
|
189
|
-
smoothly with distance from the user's position. At distance 0 the boost is full magnitude
|
|
189
|
+
smoothly with distance from the user's position. At distance 0 the boost is full magnitude, and at
|
|
190
190
|
`proximityScaleKm` (default 100 km) it's half.
|
|
191
191
|
7. − `lengthPenaltyWeight` × excess-length penalty (favors short matches over long matches on short
|
|
192
192
|
queries).
|
|
@@ -237,7 +237,7 @@ Defaults are in `lookup.ts::DEFAULT_WEIGHTS`.
|
|
|
237
237
|
are all stripped safely before going to FTS5. Per-token rules:
|
|
238
238
|
|
|
239
239
|
- **Bare tokens** (`"Paris"`, `"62701"`) become FTS5 **phrase matches**: `"Paris"` matches places
|
|
240
|
-
|
|
240
|
+
whose name is exactly "Paris", `"62701"` matches the postcode 62701 exactly.
|
|
241
241
|
- **A trailing `*`** (`"627*"`, `"Pari*"`) becomes FTS5 **prefix syntax**: `627*` matches every
|
|
242
242
|
postcode starting with 627, `Pari*` matches Paris / Parishville / etc. The caller explicitly
|
|
243
243
|
signals "prefix"; bare tokens stay phrase-matched for safety.
|
|
@@ -249,21 +249,21 @@ matching the Los Angeles ZIP corridor.
|
|
|
249
249
|
|
|
250
250
|
## Attribution (CC-BY 4.0)
|
|
251
251
|
|
|
252
|
-
Who's On First data is licensed [CC-BY 4.0](https://creativecommons.org/licenses/by/4.0/). Downstream applications shipping resolved results from this package **must**
|
|
252
|
+
Who's On First data is licensed [CC-BY 4.0](https://creativecommons.org/licenses/by/4.0/). Downstream applications shipping resolved results from this package **must** display an attribution notice — for example:
|
|
253
253
|
|
|
254
254
|
> Place data via [Who's On First](https://whosonfirst.org/) © Mapzen + contributors, CC-BY 4.0.
|
|
255
255
|
|
|
256
|
-
This package itself is AGPL-3.0
|
|
256
|
+
This package itself is AGPL-3.0, and the WOF data it indexes is CC-BY 4.0. The two licenses are separate — your application must comply with both.
|
|
257
257
|
|
|
258
258
|
## Integration tests
|
|
259
259
|
|
|
260
|
-
`resolver-wof-sqlite/integration.test.ts` exercises the resolver against a real WOF SQLite distribution. The suite is **skipped** when no DB is present — set `MAILWOMAN_WOF_DB` to override the lookup path, otherwise it defaults to `$MAILWOMAN_DATA_ROOT/db/wof/whosonfirst-data-admin-us-latest.db` (the canonical lab location). CI runs against the fixture-only suites
|
|
260
|
+
`resolver-wof-sqlite/integration.test.ts` exercises the resolver against a real WOF SQLite distribution. The suite is **skipped** when no DB is present — set `MAILWOMAN_WOF_DB` to override the lookup path, otherwise it defaults to `$MAILWOMAN_DATA_ROOT/db/wof/whosonfirst-data-admin-us-latest.db` (the canonical lab location). CI runs against the fixture-only suites. Operators with real WOF data locally get an extra layer of validation.
|
|
261
261
|
|
|
262
262
|
Coverage includes: placetype filtering, country filtering, the empty-result case, FTS5 special-character sanitization, Japanese alt-name resolution, parent-constrained lookup, and a performance budget (`findPlace` < 250 ms against the 142 k-row US admin database).
|
|
263
263
|
|
|
264
264
|
## Concurrency model
|
|
265
265
|
|
|
266
|
-
This package opens a single `node:sqlite` connection per `WOFSQLitePlaceLookup` instance. SQLite is single-writer / multiple-reader
|
|
266
|
+
This package opens a single `node:sqlite` connection per `WOFSQLitePlaceLookup` instance. SQLite is single-writer / multiple-reader, and the Kysely wrapper around the connection serializes all queries through a mutex. For high-concurrency HTTP servers, instantiate one resolver per request handler or per pool slot — sharing a single instance across concurrent requests is safe (queries queue) but won't parallelize across cores.
|
|
267
267
|
|
|
268
268
|
## License
|
|
269
269
|
|
|
@@ -126,7 +126,7 @@ export async function createAddressPointIndexes(db: AddressPointSchemaHandle): P
|
|
|
126
126
|
.execute()
|
|
127
127
|
|
|
128
128
|
await db.schema.createIndex("idx_ap_streetkey").on("address_point").columns(["postcode", "street_key"]).execute()
|
|
129
|
-
// Street-first index for the bbox scope: OSM points
|
|
129
|
+
// Street-first index for the bbox scope: some OSM points have no postcode or locality,
|
|
130
130
|
// so the reader scopes a `(street_norm, number)` probe by the resolved locality's bbox.
|
|
131
131
|
await db.schema.createIndex("idx_ap_street").on("address_point").columns(["street_norm", "number"]).execute()
|
|
132
132
|
}
|
package/lib/address/point.ts
CHANGED
|
@@ -52,10 +52,18 @@ export class AddressPointSqliteLookup<DB extends AddressPointDatabase = AddressP
|
|
|
52
52
|
readonly #db: DatabaseClient<DB>
|
|
53
53
|
readonly #locale: StreetLocale
|
|
54
54
|
readonly #localityKeys: "full" | "abbreviated"
|
|
55
|
+
readonly #localityLevel: "place" | "finest"
|
|
55
56
|
readonly #byPostcode: PreparedGet<[postcode: string, street: StreetKey, number: string], AddressPointRow> | undefined
|
|
56
57
|
readonly #byPostcodeLocality:
|
|
57
58
|
| PreparedGet<[postcode: string, locality: NameKey, street: StreetKey, number: string], AddressPointRow>
|
|
58
59
|
| undefined
|
|
60
|
+
/**
|
|
61
|
+
* This query counts the distinct stored localities that hold one (postcode, street, number) key.
|
|
62
|
+
* Unit siblings share a locality, so they count once.
|
|
63
|
+
*/
|
|
64
|
+
readonly #postcodeKeyPlaces:
|
|
65
|
+
| PreparedGet<[postcode: string, street: StreetKey, number: string], { places: number }>
|
|
66
|
+
| undefined
|
|
59
67
|
readonly #byLocality: PreparedGet<[locality: NameKey, street: StreetKey, number: string], AddressPointRow> | undefined
|
|
60
68
|
/**
|
|
61
69
|
* This query matches the locality key by its suffix, for a `zh` query that gives the 鄉鎮市區 without its 縣市.
|
|
@@ -82,16 +90,29 @@ export class AddressPointSqliteLookup<DB extends AddressPointDatabase = AddressP
|
|
|
82
90
|
* Some counties abbreviate them (`addi` for Addison).
|
|
83
91
|
* Other counties replace them with the parent town.
|
|
84
92
|
*
|
|
93
|
+
* @param opts.localityLevel The kind of place the extract stores in `locality_norm`.
|
|
94
|
+
* With `"place"`, `locality_norm` holds the place an address line gives, such as a BAN commune.
|
|
95
|
+
* With `"finest"`, `locality_norm` holds the finest place the source recorded.
|
|
96
|
+
* That place can be a part of the place the line gives.
|
|
97
|
+
* The OSM builder stores `addr:suburb` before `addr:city`, so
|
|
98
|
+
* `Teschener Weg 5, 14163 Berlin` is stored under `zehlendorf`.
|
|
99
|
+
*
|
|
85
100
|
* @defaultValue `"us"`
|
|
86
101
|
* @defaultValue `"abbreviated"` for `"us"` and `"full"` for every other locale.
|
|
102
|
+
* @defaultValue `"place"`
|
|
87
103
|
*/
|
|
88
104
|
constructor(
|
|
89
105
|
dbPath: PathBuilderLike,
|
|
90
|
-
opts: {
|
|
106
|
+
opts: {
|
|
107
|
+
streetLocale?: StreetLocale
|
|
108
|
+
localityKeys?: "full" | "abbreviated"
|
|
109
|
+
localityLevel?: "place" | "finest"
|
|
110
|
+
} = {}
|
|
91
111
|
) {
|
|
92
112
|
this.#db = new DatabaseClient<DB>(dbPath, { readOnly: true })
|
|
93
113
|
this.#locale = opts.streetLocale ?? "us"
|
|
94
114
|
this.#localityKeys = opts.localityKeys ?? (this.#locale === "us" ? "abbreviated" : "full")
|
|
115
|
+
this.#localityLevel = opts.localityLevel ?? "place"
|
|
95
116
|
|
|
96
117
|
// An extract without an `address_point` table, such as an interrupted build, makes every lookup miss.
|
|
97
118
|
if (hasTable(this.#db, "address_point")) {
|
|
@@ -107,6 +128,12 @@ export class AddressPointSqliteLookup<DB extends AddressPointDatabase = AddressP
|
|
|
107
128
|
WHERE postcode = ? AND locality_norm = ? AND street_norm = ? AND number = ? LIMIT 1`
|
|
108
129
|
)
|
|
109
130
|
|
|
131
|
+
this.#postcodeKeyPlaces = prepareGet(
|
|
132
|
+
this.#db,
|
|
133
|
+
`SELECT COUNT(DISTINCT locality_norm) AS places FROM address_point
|
|
134
|
+
WHERE postcode = ? AND street_norm = ? AND number = ?`
|
|
135
|
+
)
|
|
136
|
+
|
|
110
137
|
this.#byLocality = prepareGet(
|
|
111
138
|
this.#db,
|
|
112
139
|
`SELECT ${SELECT_COLS} FROM address_point
|
|
@@ -246,6 +273,8 @@ export class AddressPointSqliteLookup<DB extends AddressPointDatabase = AddressP
|
|
|
246
273
|
}
|
|
247
274
|
): AddressPointRow | null {
|
|
248
275
|
let row: AddressPointRow | null = null
|
|
276
|
+
// A postcode row stored under a different locality waits until the probe for the query's own locality misses.
|
|
277
|
+
let heldBack: AddressPointRow | null = null
|
|
249
278
|
|
|
250
279
|
if (query.postcode) {
|
|
251
280
|
const postcode = query.postcode.trim()
|
|
@@ -257,7 +286,11 @@ export class AddressPointSqliteLookup<DB extends AddressPointDatabase = AddressP
|
|
|
257
286
|
const agreeing = localityKey ? this.#byPostcodeLocality!(postcode, localityKey, streetNorm, number) : null
|
|
258
287
|
const candidate = agreeing ?? this.#byPostcode!(postcode, streetNorm, number)
|
|
259
288
|
|
|
260
|
-
|
|
289
|
+
if (candidate && !this.#scopeContradicts(candidate, query)) {
|
|
290
|
+
row = candidate
|
|
291
|
+
} else if (candidate && this.#isSublocalityRow(candidate, postcode, streetNorm, number, query.bbox)) {
|
|
292
|
+
heldBack = candidate
|
|
293
|
+
}
|
|
261
294
|
}
|
|
262
295
|
|
|
263
296
|
if (!row && query.locality) {
|
|
@@ -269,6 +302,10 @@ export class AddressPointSqliteLookup<DB extends AddressPointDatabase = AddressP
|
|
|
269
302
|
row = this.#byLocalityTail(streetNorm, number, `%${query.localityTail}`)
|
|
270
303
|
}
|
|
271
304
|
|
|
305
|
+
if (!row && heldBack) {
|
|
306
|
+
row = heldBack
|
|
307
|
+
}
|
|
308
|
+
|
|
272
309
|
// The bounding box serves extracts whose points have no postcode or locality, such as OSM.
|
|
273
310
|
if (!row && query.bbox) {
|
|
274
311
|
const b = query.bbox
|
|
@@ -299,17 +336,58 @@ export class AddressPointSqliteLookup<DB extends AddressPointDatabase = AddressP
|
|
|
299
336
|
* A row field that is empty never contradicts.
|
|
300
337
|
* The locality is compared only when the extract's `localityKeys` is `"full"`,
|
|
301
338
|
* because abbreviated keys would reject correct rows.
|
|
339
|
+
*
|
|
340
|
+
* Two locality keys agree when they are equal or when one is the leading words of the other.
|
|
341
|
+
* A German disambiguated name such as `Lauta b Hoyerswerda` or `Freiberg Sachs` begins with the place
|
|
342
|
+
* name that OSM's `addr:city` records, and a word boundary keeps `lauta` apart from `lautabach`.
|
|
302
343
|
*/
|
|
303
344
|
#scopeContradicts(row: AddressPointRow, query: { postcode?: string; locality?: string }): boolean {
|
|
304
345
|
if (query.postcode && row.postcode && row.postcode.trim() !== query.postcode.trim()) return true
|
|
305
346
|
|
|
306
347
|
if (this.#localityKeys === "full" && query.locality && row.locality_norm) {
|
|
307
|
-
|
|
348
|
+
const queryKey = this.#localityKey(query.locality)
|
|
349
|
+
|
|
350
|
+
const [shorter, longer] =
|
|
351
|
+
row.locality_norm.length <= queryKey.length ? [row.locality_norm, queryKey] : [queryKey, row.locality_norm]
|
|
352
|
+
|
|
353
|
+
return shorter !== longer && !longer.startsWith(`${shorter} `)
|
|
308
354
|
}
|
|
309
355
|
|
|
310
356
|
return false
|
|
311
357
|
}
|
|
312
358
|
|
|
359
|
+
/**
|
|
360
|
+
* Returns whether a postcode row stored under a different locality lies in a part of the queried place.
|
|
361
|
+
*
|
|
362
|
+
* Only an extract whose `localityLevel` is `"finest"` can store a part of a place.
|
|
363
|
+
* In a `"place"` extract, a different stored locality is a different place.
|
|
364
|
+
*
|
|
365
|
+
* In a `"finest"` extract, the row is accepted as part of the queried place when two conditions hold.
|
|
366
|
+
*
|
|
367
|
+
* No other stored locality holds the same (postcode, street, number) key.
|
|
368
|
+
* When the caller supplies the box around the query's resolved locality, the row lies inside it.
|
|
369
|
+
*
|
|
370
|
+
* The caller probes the query's own locality before it uses this row.
|
|
371
|
+
* A postcode key that two places share therefore still answers the place the query gives.
|
|
372
|
+
*/
|
|
373
|
+
#isSublocalityRow(
|
|
374
|
+
row: AddressPointRow,
|
|
375
|
+
postcode: string,
|
|
376
|
+
streetNorm: StreetKey,
|
|
377
|
+
number: string,
|
|
378
|
+
bbox: { minLat: number; maxLat: number; minLon: number; maxLon: number } | null
|
|
379
|
+
): boolean {
|
|
380
|
+
if (this.#localityLevel !== "finest") return false
|
|
381
|
+
|
|
382
|
+
if (row.postcode && row.postcode.trim() !== postcode) return false
|
|
383
|
+
|
|
384
|
+
if ((this.#postcodeKeyPlaces?.(postcode, streetNorm, number)?.places ?? 0) !== 1) return false
|
|
385
|
+
|
|
386
|
+
if (!bbox) return true
|
|
387
|
+
|
|
388
|
+
return row.lat >= bbox.minLat && row.lat <= bbox.maxLat && row.lon >= bbox.minLon && row.lon <= bbox.maxLon
|
|
389
|
+
}
|
|
390
|
+
|
|
313
391
|
[Symbol.dispose](): void {
|
|
314
392
|
this.#db[Symbol.dispose]()
|
|
315
393
|
}
|
|
@@ -132,13 +132,13 @@ export async function buildAncestorsSidecar(ctx: {
|
|
|
132
132
|
const postOf = new Map<number, number>()
|
|
133
133
|
|
|
134
134
|
for (const kids of childrenOf.values()) {
|
|
135
|
-
// oxlint-disable-next-line unicorn/no-array-sort -- sorts an array this pass
|
|
135
|
+
// oxlint-disable-next-line unicorn/no-array-sort -- sorts an array this pass built
|
|
136
136
|
kids.sort((a, b) => a - b)
|
|
137
137
|
}
|
|
138
138
|
|
|
139
139
|
const roots = [...forest].filter((id) => !canonicalParentOf.has(id))
|
|
140
140
|
|
|
141
|
-
// oxlint-disable-next-line unicorn/no-array-sort -- sorts an array this pass
|
|
141
|
+
// oxlint-disable-next-line unicorn/no-array-sort -- sorts an array this pass built
|
|
142
142
|
roots.sort((a, b) => a - b)
|
|
143
143
|
|
|
144
144
|
let counter = 0
|
|
@@ -174,7 +174,7 @@ export async function buildAncestorsSidecar(ctx: {
|
|
|
174
174
|
const insInterval = out.prepare(`INSERT INTO ${CANDIDATE_INTERVAL_TABLE} VALUES (?, ?, ?)`)
|
|
175
175
|
const labeled = [...preOf.keys()]
|
|
176
176
|
|
|
177
|
-
// oxlint-disable-next-line unicorn/no-array-sort -- sorts an array this pass
|
|
177
|
+
// oxlint-disable-next-line unicorn/no-array-sort -- sorts an array this pass built
|
|
178
178
|
labeled.sort((a, b) => a - b)
|
|
179
179
|
|
|
180
180
|
out.exec("BEGIN")
|
|
@@ -63,7 +63,7 @@ interface ScoredPlace {
|
|
|
63
63
|
/**
|
|
64
64
|
* What {@link loadImportanceIndex} measured while reading the source.
|
|
65
65
|
*
|
|
66
|
-
* Reported by the builder so a run reports how much of the gazetteer it
|
|
66
|
+
* Reported by the builder so a run reports how much of the gazetteer it scored,
|
|
67
67
|
* rather than leaving the caller to infer it from a column full of nulls.
|
|
68
68
|
*/
|
|
69
69
|
export interface ImportanceIndexStats {
|
|
@@ -108,7 +108,7 @@ export class ImportanceIndex {
|
|
|
108
108
|
* The key matched a scored group, but the nearest member was outside the radius,
|
|
109
109
|
* so it is a different place wearing the same name.
|
|
110
110
|
*
|
|
111
|
-
*
|
|
111
|
+
* Compare this number across rebuilds.
|
|
112
112
|
* A jump means the score source and the admin source have drifted apart
|
|
113
113
|
* and the join is being asked to guess.
|
|
114
114
|
* It does not mean the radius is too tight.
|
|
@@ -31,7 +31,7 @@ export const VARIANT_SIMILARITY_MIN = 0.85
|
|
|
31
31
|
* BGN/pcgn-flavored Cyrillic romanization, folded to the name-key alphabet.
|
|
32
32
|
*
|
|
33
33
|
* Digraph outputs (zh, kh, ts, ch, sh, shch, yu, ya) match the dominant transliteration
|
|
34
|
-
* conventions the gazetteer's Latin aliases
|
|
34
|
+
* conventions the gazetteer's Latin aliases use.
|
|
35
35
|
* The `w`/`v` and `kh`/`h` style variance between systems is what the edit-similarity threshold absorbs.
|
|
36
36
|
*/
|
|
37
37
|
const CYRILLIC_TO_LATIN: Record<string, string> = {
|
package/lib/coincident-roles.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
*
|
|
6
6
|
* `buildCoincidentRoles` derives the coincident-roles relation into the unified gazetteer.
|
|
7
7
|
*
|
|
8
|
-
*
|
|
8
|
+
* Some places occupy multiple admin tiers under one name: German city-states (Berlin, Hamburg,
|
|
9
9
|
* Bremen), Italian provinces whose names match their capital (Milano, Varese), Spanish
|
|
10
10
|
* provinces-after-capitals, UK unitary authorities, JP prefectures, NL province-capitals (Utrecht,
|
|
11
11
|
* Groningen), and Shanghai. When an address surfaces only the admin role because the parser
|
package/lib/currency-backfill.ts
CHANGED
|
@@ -24,8 +24,8 @@ const CURRENCY_BACKFILL_RADIUS_KM = 10
|
|
|
24
24
|
const CURRENCY_BACKFILL_POP_FLOOR = 1000
|
|
25
25
|
|
|
26
26
|
/**
|
|
27
|
-
* Restore a deprecated locality only when no
|
|
28
|
-
* attests the same folded name above {@link CURRENCY_BACKFILL_POP_FLOOR}.
|
|
27
|
+
* Restore a deprecated locality only when no live namesake exists within CURRENCY_BACKFILL_RADIUS_KM
|
|
28
|
+
* and GeoNames attests the same folded name above {@link CURRENCY_BACKFILL_POP_FLOOR}.
|
|
29
29
|
*/
|
|
30
30
|
export interface CurrencyBackfillResults {
|
|
31
31
|
judged: number
|
package/lib/extracts.ts
CHANGED
|
@@ -55,7 +55,7 @@ export function deriveSchemaName(path: PathBuilderLike): string {
|
|
|
55
55
|
/**
|
|
56
56
|
* Per-extract configuration.
|
|
57
57
|
*
|
|
58
|
-
* The simple form is
|
|
58
|
+
* The simple form is a path string.
|
|
59
59
|
* The schema name is derived from it.
|
|
60
60
|
*
|
|
61
61
|
* The object form lets callers override the derived schema name (useful when a filename
|
package/lib/fst/autocomplete.ts
CHANGED
|
@@ -109,9 +109,9 @@ class FSTReader implements AncestrieReaderLike<PlaceEntry> {
|
|
|
109
109
|
* Parent chains of the entries this reader has served, id-keyed.
|
|
110
110
|
*
|
|
111
111
|
* A place's chain is identical across its surfaces (it is place-row data), so last-write-wins is safe.
|
|
112
|
-
* The algorithm asks {@link FSTReader.ancestorsOf} only for ids it
|
|
113
|
-
* from {@link FSTReader.entriesAt}, so serving from this memo answers
|
|
114
|
-
* real call without an artifact-wide id index.
|
|
112
|
+
* The algorithm asks {@link FSTReader.ancestorsOf} only for ids it received in the
|
|
113
|
+
* same call from {@link FSTReader.entriesAt}, so serving from this memo answers
|
|
114
|
+
* every real call without an artifact-wide id index.
|
|
115
115
|
*/
|
|
116
116
|
readonly #chains = new Map<number, number[]>()
|
|
117
117
|
|
|
@@ -127,7 +127,7 @@ export function deserializeFSTWeb(input: ArrayBuffer | Uint8Array): FSTMatcher {
|
|
|
127
127
|
|
|
128
128
|
// v1 stored a raw population u32 here. v2-v4 stored the conflated `importance`
|
|
129
129
|
// float. v5 stored the referential score.
|
|
130
|
-
// The Node deserializer explains why a v1 value is
|
|
130
|
+
// The Node deserializer explains why a v1 value is the referential score.
|
|
131
131
|
const referential = isV2
|
|
132
132
|
? view.getFloat32(pp + 12, true)
|
|
133
133
|
: Math.min(1, Math.log2(1 + view.getUint32(pp + 12, true) / 1000) / 14)
|
package/lib/fst/format.ts
CHANGED
package/lib/geonames/postal.ts
CHANGED
|
@@ -122,8 +122,8 @@ function collapseDuplicatePoints(points: readonly PostcodePoint[]): PostcodePoin
|
|
|
122
122
|
* and a multi-member group is one of its members.
|
|
123
123
|
* The bound applies only while the members are distinct, so duplicate points are collapsed first.
|
|
124
124
|
*
|
|
125
|
-
* Separate counts for N rows at one coordinate would weight that value by
|
|
126
|
-
* inherited it. {@link MedoidSupport.distinctPoints} reports
|
|
125
|
+
* Separate counts for N rows at one coordinate would weight that value by the count of settlements that
|
|
126
|
+
* inherited it. {@link MedoidSupport.distinctPoints} reports the count of points the answer rested on.
|
|
127
127
|
*
|
|
128
128
|
* Distance is squared-Euclidean in degrees rather than haversine.
|
|
129
129
|
* At the scale a postcode spans, both methods produce the same ranking.
|
|
@@ -194,7 +194,7 @@ export interface GeonamesPostalIngestResult {
|
|
|
194
194
|
/**
|
|
195
195
|
* Per-country count of inserted codes the dump listed on several rows that all refer to one point.
|
|
196
196
|
*
|
|
197
|
-
* The coordinate rests on a single value
|
|
197
|
+
* The coordinate rests on a single value regardless of the count of settlements under the code.
|
|
198
198
|
* Reported because the point is still the best the source offers.
|
|
199
199
|
*
|
|
200
200
|
* A consumer weighing postal coverage needs to know how much of it is this.
|
package/lib/interpolation.ts
CHANGED
|
@@ -68,7 +68,7 @@ export interface InterpolationQuery {
|
|
|
68
68
|
|
|
69
69
|
/**
|
|
70
70
|
* Acceptance geometry for the `near` tie-break: the winning group must be within this
|
|
71
|
-
*
|
|
71
|
+
* number of kilometers and the runner-up at least {@link NEAR_DOMINANCE} times farther.
|
|
72
72
|
*/
|
|
73
73
|
const NEAR_MAX_KM = 25
|
|
74
74
|
|
|
@@ -223,7 +223,7 @@ export class StreetInterpolator<
|
|
|
223
223
|
}
|
|
224
224
|
|
|
225
225
|
/**
|
|
226
|
-
* Resolve one key variant's covering rows to an answer, or null when
|
|
226
|
+
* Resolve one key variant's covering rows to an answer, or null when the rows do not support one.
|
|
227
227
|
*/
|
|
228
228
|
#answerFromRows(rows: SegmentRow[], n: number, query: InterpolationQuery): InterpolatedHit | null {
|
|
229
229
|
if (!rows.length) return null
|
package/lib/lookup.ts
CHANGED
|
@@ -569,7 +569,7 @@ export class WOFSQLitePlaceLookup extends SQLiteLookup<WOFDatabase> implements P
|
|
|
569
569
|
|
|
570
570
|
/**
|
|
571
571
|
* Coordinate-first locality resolution: union the postcode's containing
|
|
572
|
-
* and
|
|
572
|
+
* and closest neighboring localities with the FTS name candidates and soft-score them,
|
|
573
573
|
* returning null when the postcode is not in the table.
|
|
574
574
|
*/
|
|
575
575
|
async #findLocalityCoordFirst(
|
package/lib/nsul/lookup.ts
CHANGED
|
@@ -24,6 +24,7 @@
|
|
|
24
24
|
* unknown, per the meaning-of-zero rule.
|
|
25
25
|
*/
|
|
26
26
|
|
|
27
|
+
import type { UkCountryCode } from "@mailwoman/codex/gb/country"
|
|
27
28
|
import { SQLiteLookup, type SQLiteLookupOptions } from "@mailwoman/sqlite/lookup"
|
|
28
29
|
|
|
29
30
|
import { compactPostcode, type NSULDatabase } from "#nsul/schema"
|
|
@@ -64,6 +65,22 @@ export interface NSULAssignedPoint {
|
|
|
64
65
|
*/
|
|
65
66
|
export type NSULLookupOpts = SQLiteLookupOptions<NSULDatabase>
|
|
66
67
|
|
|
68
|
+
/**
|
|
69
|
+
* The UPRNs of one unit postcode that lie in one constituent country.
|
|
70
|
+
*/
|
|
71
|
+
export interface NSULPostcodeCountry {
|
|
72
|
+
/**
|
|
73
|
+
* The ISO 3166-2:GB constituent country (`ENG`, `SCT`, `WLS`).
|
|
74
|
+
*/
|
|
75
|
+
country: UkCountryCode
|
|
76
|
+
uprns: number
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
interface CountryRow {
|
|
80
|
+
ctry: UkCountryCode
|
|
81
|
+
uprns: number
|
|
82
|
+
}
|
|
83
|
+
|
|
67
84
|
interface PostcodeRow {
|
|
68
85
|
pcds: string
|
|
69
86
|
pcds_compact: string
|
|
@@ -84,6 +101,7 @@ interface PointRow {
|
|
|
84
101
|
export class NSULLookup extends SQLiteLookup<NSULDatabase> {
|
|
85
102
|
readonly #postcodeProbe: (uprn: number) => PostcodeRow | null
|
|
86
103
|
readonly #pointsProbe: (pcdsCompact: string) => PointRow[]
|
|
104
|
+
readonly #countriesProbe: (pcdsCompact: string) => CountryRow[]
|
|
87
105
|
|
|
88
106
|
constructor(opts: NSULLookupOpts) {
|
|
89
107
|
super(opts)
|
|
@@ -97,6 +115,21 @@ export class NSULLookup extends SQLiteLookup<NSULDatabase> {
|
|
|
97
115
|
this.database,
|
|
98
116
|
"SELECT uprn, lat, lon, h3_cell FROM uprn_postcode WHERE pcds_compact = ? ORDER BY uprn"
|
|
99
117
|
)
|
|
118
|
+
|
|
119
|
+
this.#countriesProbe = prepareAll<[string], CountryRow, NSULDatabase>(
|
|
120
|
+
this.database,
|
|
121
|
+
"SELECT ctry, count(*) AS uprns FROM uprn_postcode WHERE pcds_compact = ? GROUP BY ctry ORDER BY uprns DESC, ctry"
|
|
122
|
+
)
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* The constituent countries one unit postcode's UPRNs lie in, with the UPRN count in each, most first.
|
|
127
|
+
*
|
|
128
|
+
* A postcode on the England–Wales or England–Scotland border can return two entries.
|
|
129
|
+
* An empty array means the register assigns no UPRN to the postcode.
|
|
130
|
+
*/
|
|
131
|
+
countriesForPostcode(postcode: string): NSULPostcodeCountry[] {
|
|
132
|
+
return this.#countriesProbe(compactPostcode(postcode)).map((row) => ({ country: row.ctry, uprns: row.uprns }))
|
|
100
133
|
}
|
|
101
134
|
|
|
102
135
|
/**
|
package/lib/nsul/schema.ts
CHANGED
|
@@ -38,6 +38,11 @@ export interface UPRNPostcodeTable {
|
|
|
38
38
|
* `spr.name` contains and `uprnsForPostcode` probes.
|
|
39
39
|
*/
|
|
40
40
|
pcds_compact: string
|
|
41
|
+
/**
|
|
42
|
+
* The ISO 3166-2:GB constituent country the UPRN lies in (`ENG`, `SCT`, `WLS`),
|
|
43
|
+
* read from NSUL's `CTRY25CD` column.
|
|
44
|
+
*/
|
|
45
|
+
ctry: string
|
|
41
46
|
/**
|
|
42
47
|
* WGS84 latitude, copied from `uprn.db` for the same UPRN.
|
|
43
48
|
*/
|
|
@@ -78,6 +83,7 @@ export async function createUPRNPostcodeTable(db: Kysely<NSULDatabase>): Promise
|
|
|
78
83
|
.addColumn("uprn", "integer", (c) => c.primaryKey())
|
|
79
84
|
.addColumn("pcds", "text", (c) => c.notNull())
|
|
80
85
|
.addColumn("pcds_compact", "text", (c) => c.notNull())
|
|
86
|
+
.addColumn("ctry", "text", (c) => c.notNull())
|
|
81
87
|
.addColumn("lat", "real", (c) => c.notNull())
|
|
82
88
|
.addColumn("lon", "real", (c) => c.notNull())
|
|
83
89
|
.addColumn("h3_cell", "integer", (c) => c.notNull())
|
package/lib/paths.ts
CHANGED
|
@@ -85,7 +85,7 @@ export const nsulDatabasePath: PathBuilder = databaseRootPath(dataRootPath())("n
|
|
|
85
85
|
* It defaults to `$MAILWOMAN_DATA_ROOT`, and callers may pass `--data-root`.
|
|
86
86
|
* Returns a fresh array each call.
|
|
87
87
|
*
|
|
88
|
-
*
|
|
88
|
+
* A caller can filter with `existsSync`, so missing files are skipped.
|
|
89
89
|
*
|
|
90
90
|
* This runtime list is intentionally smaller than `DEFAULT_POSTCODE_DATABASES`
|
|
91
91
|
* (`mailwoman/gazetteer-pipeline/index.ts`), because these databases are attached live at boot.
|
|
@@ -21,7 +21,7 @@ export interface PlaceImportanceTable {
|
|
|
21
21
|
* It is never NULL.
|
|
22
22
|
* A place without a population row scores `0`.
|
|
23
23
|
*
|
|
24
|
-
* That score represents no population evidence, so it
|
|
24
|
+
* That score represents no population evidence, so it receives no boost or penalty.
|
|
25
25
|
*/
|
|
26
26
|
referential: number
|
|
27
27
|
|
package/lib/poi/lookup.ts
CHANGED
|
@@ -81,7 +81,7 @@ export interface POISearchQuery {
|
|
|
81
81
|
*/
|
|
82
82
|
center?: { latitude: number; longitude: number }
|
|
83
83
|
/**
|
|
84
|
-
* Ring budget:
|
|
84
|
+
* Ring budget: the count of res-9 k-rings to expand before giving up.
|
|
85
85
|
*
|
|
86
86
|
* Counts ring 0, so k reaches `maxRings - 1`.
|
|
87
87
|
*
|
|
@@ -251,7 +251,7 @@ export class POILookup<DB extends POIDatabase = POIDatabase> extends SQLiteLooku
|
|
|
251
251
|
// `ring` starts at 0 (the origin cell itself), so this loop's k reaches `maxRings - 1`.
|
|
252
252
|
for (let ring = 0; ring < maxRings; ring++) {
|
|
253
253
|
// gridDisk(origin, ring) returns the whole disk out to `ring`, so diffing against
|
|
254
|
-
// the already-probed set derives
|
|
254
|
+
// the already-probed set derives only this ring's new cells.
|
|
255
255
|
const diskCells = gridDisk(origin, ring) as string[]
|
|
256
256
|
const newCells = diskCells.filter((cell) => !seenCells.has(cell))
|
|
257
257
|
|
package/lib/polygon-schema.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* `postal_city → geo_locality` aliases per postcode (`build-postal-city-alias.ts`). Consumed by
|
|
8
8
|
* {@link WOFSQLitePlaceLookup}'s coordinate-first locality scorer: a user-typed postal city
|
|
9
9
|
* ("Antioch", postcode 37013) becomes a name-match alias for the geographic locality the postcode
|
|
10
|
-
*
|
|
10
|
+
* sits in ("Nashville"), so the right place tiers to the top instead of a place with the same name
|
|
11
11
|
* town in another state. Opt-in — the lookup is only constructed when a path is supplied.
|
|
12
12
|
* absent it the resolver is byte-identical.
|
|
13
13
|
*
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
6
|
* Typed schema for the postal-city alias table (`postal-city-alias-<cc>.db`, built by
|
|
7
|
-
* `scripts/build-postal-city-alias.ts`) — the
|
|
7
|
+
* `scripts/build-postal-city-alias.ts`) — the one declaration of the columns shared by
|
|
8
8
|
* the builder and the reader ({@link WOFPostalCityAliasLookup}). Like {@link CandidateTable}, the
|
|
9
9
|
* interface is a Kysely `Database` interface plus the table DDL as a string, so a column rename in
|
|
10
10
|
* the builder is a compile error in the reader.
|
|
@@ -34,7 +34,7 @@ export interface PostalCityAliasTable {
|
|
|
34
34
|
*/
|
|
35
35
|
postal_city: string
|
|
36
36
|
/**
|
|
37
|
-
* The geographic locality name the postcode
|
|
37
|
+
* The geographic locality name the postcode sits in (≈ the gazetteer's canonical name).
|
|
38
38
|
*/
|
|
39
39
|
geo_locality: string
|
|
40
40
|
/**
|
package/lib/proximity-rerank.ts
CHANGED
|
@@ -88,7 +88,7 @@ export function combinedProminence(candidate: ProximityRerankable, bias: readonl
|
|
|
88
88
|
const popTerm = POP_BOOST * Math.min(1, Math.max(0, popBase) / POP_SCALE_LOG10)
|
|
89
89
|
let proxTerm = 0
|
|
90
90
|
|
|
91
|
-
// A candidate at
|
|
91
|
+
// A candidate at 0,0 (Null Island) has no coordinate, so it receives no nearness term.
|
|
92
92
|
// A 0,0 pair interpreted as a real coordinate would receive an enormous score.
|
|
93
93
|
if (!(candidate.lat === 0 && candidate.lon === 0)) {
|
|
94
94
|
for (const b of bias) {
|
package/lib/search-fetch.ts
CHANGED
|
@@ -30,7 +30,7 @@ const SHORT_QUERY_MAX_LENGTH = 3
|
|
|
30
30
|
/**
|
|
31
31
|
* Over-fetch floor for short (≤3-character) queries.
|
|
32
32
|
*
|
|
33
|
-
*
|
|
33
|
+
* The target case is a region abbreviation such as "NY" or "VT".
|
|
34
34
|
*
|
|
35
35
|
* An exact-abbrev holder's BM25 is poor (long multilingual alt-name document),
|
|
36
36
|
* so the normal `limit * 4` window can drop it before `exactMatchTiering` promotes
|
|
@@ -41,7 +41,7 @@ const SHORT_QUERY_MAX_LENGTH = 3
|
|
|
41
41
|
const SHORT_QUERY_OVERFETCH = 200
|
|
42
42
|
|
|
43
43
|
/**
|
|
44
|
-
*
|
|
44
|
+
* The count of rows the population-ordered companion fetch adds to the candidate pool.
|
|
45
45
|
*
|
|
46
46
|
* Small on purpose.
|
|
47
47
|
* Its only job is to guarantee the famous holders of a name enter the pool at all.
|
|
@@ -184,10 +184,10 @@ export function fetchSearchRows<DB>(options: {
|
|
|
184
184
|
// and builds no join at all.
|
|
185
185
|
const { select: encyclopedicSelect, join: encyclopedicJoin } = encyclopedicClauses.get(sch)!
|
|
186
186
|
|
|
187
|
-
// Push the population boost into the order BY when the index is available, so famous
|
|
188
|
-
// (whose long alt-name lists hurt BM25)
|
|
187
|
+
// Push the population boost into the order BY when the index is available, so famous
|
|
188
|
+
// places (whose long alt-name lists hurt BM25) make it into the over-fetch window.
|
|
189
189
|
// The TS post-scoring will still compute the same boost for the final score.
|
|
190
|
-
// This
|
|
190
|
+
// This ordering only decides which rows enter the candidate set.
|
|
191
191
|
//
|
|
192
192
|
// Formula: rank_adjusted = bm25 - populationBoost * min(1.0, log10(1 + pop) / scaleLog10)
|
|
193
193
|
// Lower rank_adjusted = better (matches SQLite's bm25 convention of "more negative = better").
|