@mailwoman/resolver-wof-sqlite 9.0.0 → 9.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build-candidate.ts +97 -4
- package/candidate-importance.ts +227 -0
- package/candidate-lookup.ts +144 -7
- package/candidate-schema.ts +29 -0
- package/fst-autocomplete.ts +22 -11
- package/fst-builder.ts +25 -35
- package/fst-deserialize-web.ts +43 -7
- package/fst-serialize.ts +68 -12
- package/fst-types.ts +35 -1
- package/fts.ts +12 -0
- package/index.ts +8 -0
- package/lookup.ts +44 -5
- package/out/build-candidate.d.ts +33 -0
- package/out/build-candidate.d.ts.map +1 -1
- package/out/build-candidate.js +37 -4
- package/out/build-candidate.js.map +1 -1
- package/out/candidate-importance.d.ts +132 -0
- package/out/candidate-importance.d.ts.map +1 -0
- package/out/candidate-importance.js +174 -0
- package/out/candidate-importance.js.map +1 -0
- package/out/candidate-lookup.d.ts +9 -1
- package/out/candidate-lookup.d.ts.map +1 -1
- package/out/candidate-lookup.js +115 -6
- package/out/candidate-lookup.js.map +1 -1
- package/out/candidate-schema.d.ts +25 -1
- package/out/candidate-schema.d.ts.map +1 -1
- package/out/candidate-schema.js +5 -0
- package/out/candidate-schema.js.map +1 -1
- package/out/fst-autocomplete.d.ts +13 -3
- package/out/fst-autocomplete.d.ts.map +1 -1
- package/out/fst-autocomplete.js +10 -9
- package/out/fst-autocomplete.js.map +1 -1
- package/out/fst-builder.d.ts.map +1 -1
- package/out/fst-builder.js +21 -28
- package/out/fst-builder.js.map +1 -1
- package/out/fst-deserialize-web.d.ts.map +1 -1
- package/out/fst-deserialize-web.js +36 -7
- package/out/fst-deserialize-web.js.map +1 -1
- package/out/fst-serialize.d.ts +14 -4
- package/out/fst-serialize.d.ts.map +1 -1
- package/out/fst-serialize.js +60 -12
- package/out/fst-serialize.js.map +1 -1
- package/out/fst-types.d.ts +35 -1
- package/out/fst-types.d.ts.map +1 -1
- package/out/fts.d.ts +11 -0
- package/out/fts.d.ts.map +1 -1
- package/out/fts.js +11 -0
- package/out/fts.js.map +1 -1
- package/out/index.d.ts +1 -0
- package/out/index.d.ts.map +1 -1
- package/out/index.js +1 -0
- package/out/index.js.map +1 -1
- package/out/lookup.d.ts.map +1 -1
- package/out/lookup.js +37 -5
- package/out/lookup.js.map +1 -1
- package/out/place-importance-schema.d.ts +189 -0
- package/out/place-importance-schema.d.ts.map +1 -0
- package/out/place-importance-schema.js +242 -0
- package/out/place-importance-schema.js.map +1 -0
- package/out/sqlite-utils.d.ts +12 -0
- package/out/sqlite-utils.d.ts.map +1 -1
- package/out/sqlite-utils.js +20 -0
- package/out/sqlite-utils.js.map +1 -1
- package/out/street-morphology-fst-builder.d.ts.map +1 -1
- package/out/street-morphology-fst-builder.js +5 -4
- package/out/street-morphology-fst-builder.js.map +1 -1
- package/out/street-normalize.d.ts +2 -1
- package/out/street-normalize.d.ts.map +1 -1
- package/out/street-normalize.js +2 -1
- package/out/street-normalize.js.map +1 -1
- package/out/types.d.ts +39 -0
- package/out/types.d.ts.map +1 -1
- package/package.json +14 -6
- package/place-importance-schema.ts +353 -0
- package/sqlite-utils.ts +21 -0
- package/street-morphology-fst-builder.ts +5 -4
- package/street-normalize.ts +3 -2
- package/types.ts +39 -0
package/build-candidate.ts
CHANGED
|
@@ -26,6 +26,12 @@
|
|
|
26
26
|
*
|
|
27
27
|
* Measured (2026-06-20, vs the 2.6 GB full-DB FTS): ~5 M rows; ~12 range fetches per 8-query
|
|
28
28
|
* session (the full DB needs 243); US locality 96.8% (region bbox), EU coord parity 88.6%.
|
|
29
|
+
*
|
|
30
|
+
* #28 adds one more denormalized field, `importance` — the toponym-fame prior that decides a BARE
|
|
31
|
+
* city name, joined in from a separate score source by name rather than by id (see
|
|
32
|
+
* `candidate-importance.ts`, which owns that join and explains why the id would be wrong). It is
|
|
33
|
+
* optional: without {@link BuildCandidateOptions.importance} the column is NULL on every row, which
|
|
34
|
+
* the consumer reads as unmeasured and ignores.
|
|
29
35
|
*/
|
|
30
36
|
|
|
31
37
|
import { existsSync, rmSync } from "node:fs"
|
|
@@ -34,6 +40,7 @@ import { DatabaseSync } from "node:sqlite"
|
|
|
34
40
|
import { DatabaseClient } from "@mailwoman/core/kysley/client"
|
|
35
41
|
|
|
36
42
|
import { createCandidateFTS } from "./candidate-fts.ts"
|
|
43
|
+
import { IMPORTANCE_JOIN_GATE_KM, loadImportanceIndex } from "./candidate-importance.ts"
|
|
37
44
|
import {
|
|
38
45
|
CANDIDATE_COLUMNS,
|
|
39
46
|
createCandidateStagingTables,
|
|
@@ -65,6 +72,18 @@ export interface BuildCandidateOptions {
|
|
|
65
72
|
* ("Brooklyn" for 11201), and they were previously reachable only through FTS.
|
|
66
73
|
*/
|
|
67
74
|
postcodes?: string[]
|
|
75
|
+
/**
|
|
76
|
+
* Optional WOF admin database carrying a `place_importance` table — the source of the `importance` column (#28), the
|
|
77
|
+
* toponym-fame prior that decides the bare-city-name class. Joined by `(name_key, country, placetype)` + nearest
|
|
78
|
+
* centroid, NOT by id; see `candidate-importance.ts` for why the id join silently drops the foreign homonyms the
|
|
79
|
+
* prior exists to demote.
|
|
80
|
+
*
|
|
81
|
+
* Omit it and every row's `importance` is NULL — unmeasured, which is what the consumer's positive-evidence-only rule
|
|
82
|
+
* already treats as "do not participate", so the artifact is byte-identical to a pre-#28 build except for the empty
|
|
83
|
+
* column. That is the honest degradation and it is the DEFAULT: a caller with no score source must not get a
|
|
84
|
+
* population-derived stand-in written into a column that means fame.
|
|
85
|
+
*/
|
|
86
|
+
importance?: string
|
|
68
87
|
/**
|
|
69
88
|
* Optional progress callback for CLI / test introspection.
|
|
70
89
|
*/
|
|
@@ -84,6 +103,21 @@ export interface BuildCandidateResult {
|
|
|
84
103
|
* through `onProgress`.
|
|
85
104
|
*/
|
|
86
105
|
postcodeAliases: number
|
|
106
|
+
/**
|
|
107
|
+
* Places that took an `importance` score from the join (#28).
|
|
108
|
+
*
|
|
109
|
+
* `undefined` and `0` mean different things. `undefined` is "the pass did not run" — no score source was given. A `0`
|
|
110
|
+
* would be the source matching NOTHING, which is a finding. Never collapse the two.
|
|
111
|
+
*/
|
|
112
|
+
importanceScored?: number
|
|
113
|
+
/**
|
|
114
|
+
* Places whose `(name_key, country, placetype)` matched a scored group but whose nearest scored centroid was outside
|
|
115
|
+
* {@link IMPORTANCE_JOIN_GATE_KM} — a different town wearing the same name, refused rather than scored.
|
|
116
|
+
*
|
|
117
|
+
* Worth watching across rebuilds: a jump here means the score source and the admin source have drifted apart, and the
|
|
118
|
+
* join is being asked to guess.
|
|
119
|
+
*/
|
|
120
|
+
importanceGated?: number
|
|
87
121
|
}
|
|
88
122
|
|
|
89
123
|
interface PlaceAttrs {
|
|
@@ -100,6 +134,12 @@ interface PlaceAttrs {
|
|
|
100
134
|
pop: number
|
|
101
135
|
neg: number
|
|
102
136
|
pkey: string
|
|
137
|
+
/**
|
|
138
|
+
* The place's toponym-fame score, or null when the score source has no measurement for it (#28). A property of the
|
|
139
|
+
* PLACE, so it rides {@link stageRow} onto the alias and abbrev rows too — that is how a bare `Moscow` reaches
|
|
140
|
+
* Москва's score through the alias row that carries the key.
|
|
141
|
+
*/
|
|
142
|
+
imp: number | null
|
|
103
143
|
}
|
|
104
144
|
|
|
105
145
|
export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<BuildCandidateResult> {
|
|
@@ -146,6 +186,25 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
146
186
|
return id
|
|
147
187
|
}
|
|
148
188
|
|
|
189
|
+
// --- importance source (#28): loaded BEFORE pass 1, which is the only pass that sees a place's
|
|
190
|
+
// name/country/placetype/centroid together. Absent → every row's `importance` stays NULL. ---
|
|
191
|
+
let importance: ReturnType<typeof loadImportanceIndex> | undefined
|
|
192
|
+
|
|
193
|
+
if (opts.importance) {
|
|
194
|
+
progress("importance", `loading place_importance from ${opts.importance}`)
|
|
195
|
+
importance = loadImportanceIndex(opts.importance)
|
|
196
|
+
|
|
197
|
+
progress(
|
|
198
|
+
"importance",
|
|
199
|
+
`${importance.stats.places.toLocaleString()} scored places in ${importance.stats.keys.toLocaleString()} (name, country, placetype) groups` +
|
|
200
|
+
(importance.stats.unkeyable ? `; ${importance.stats.unkeyable.toLocaleString()} unkeyable names skipped` : "")
|
|
201
|
+
)
|
|
202
|
+
} else {
|
|
203
|
+
// Never a silent column of nulls: a build without a score source produces one, and the reason has
|
|
204
|
+
// to be visible in the log rather than inferred from the artifact.
|
|
205
|
+
progress("importance", "no score source given — `importance` will be NULL on every row")
|
|
206
|
+
}
|
|
207
|
+
|
|
149
208
|
// --- region_id per place (its region-tier ancestor) for same-name disambiguation ---
|
|
150
209
|
progress("region", "loading region ancestry")
|
|
151
210
|
const regionOf = new Map<number, number>()
|
|
@@ -206,14 +265,16 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
206
265
|
const neg = -Math.log10(pop + 1)
|
|
207
266
|
const name = String(r.name ?? "")
|
|
208
267
|
const pkey = normalizeLocalityForKey(name)
|
|
268
|
+
const lat = r.lat as number
|
|
269
|
+
const lon = r.lon as number
|
|
209
270
|
|
|
210
271
|
const a: PlaceAttrs = {
|
|
211
272
|
cid,
|
|
212
273
|
rid,
|
|
213
274
|
ptid,
|
|
214
275
|
name,
|
|
215
|
-
lat
|
|
216
|
-
lon
|
|
276
|
+
lat,
|
|
277
|
+
lon,
|
|
217
278
|
mnLat: r.mnlat as number,
|
|
218
279
|
mnLon: r.mnlon as number,
|
|
219
280
|
mxLat: r.mxlat as number,
|
|
@@ -221,12 +282,30 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
221
282
|
pop,
|
|
222
283
|
neg,
|
|
223
284
|
pkey,
|
|
285
|
+
imp: importance?.find(name, r.country as string | null, r.placetype as string | null, lat, lon) ?? null,
|
|
224
286
|
}
|
|
225
287
|
|
|
226
288
|
attrs.set(sid, a)
|
|
227
289
|
|
|
228
290
|
if (pkey) {
|
|
229
|
-
insStage.run(
|
|
291
|
+
insStage.run(
|
|
292
|
+
pkey,
|
|
293
|
+
cid,
|
|
294
|
+
rid,
|
|
295
|
+
ptid,
|
|
296
|
+
neg,
|
|
297
|
+
sid,
|
|
298
|
+
name,
|
|
299
|
+
a.lat,
|
|
300
|
+
a.lon,
|
|
301
|
+
a.mnLat,
|
|
302
|
+
a.mnLon,
|
|
303
|
+
a.mxLat,
|
|
304
|
+
a.mxLon,
|
|
305
|
+
pop,
|
|
306
|
+
1,
|
|
307
|
+
a.imp
|
|
308
|
+
)
|
|
230
309
|
|
|
231
310
|
nPrim++
|
|
232
311
|
}
|
|
@@ -235,6 +314,14 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
235
314
|
out.exec("COMMIT")
|
|
236
315
|
progress("primaries", `${nPrim.toLocaleString()} primaries; ${attrs.size.toLocaleString()} places`)
|
|
237
316
|
|
|
317
|
+
if (importance) {
|
|
318
|
+
progress(
|
|
319
|
+
"importance",
|
|
320
|
+
`${importance.matched.toLocaleString()} places scored; ` +
|
|
321
|
+
`${importance.gated.toLocaleString()} refused (nearest same-name place > ${IMPORTANCE_JOIN_GATE_KM} km away)`
|
|
322
|
+
)
|
|
323
|
+
}
|
|
324
|
+
|
|
238
325
|
const stageRow = (k: string, a: PlaceAttrs, sid: number, isPrimary: number): void => {
|
|
239
326
|
insStage.run(
|
|
240
327
|
k,
|
|
@@ -251,7 +338,8 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
251
338
|
a.mxLat,
|
|
252
339
|
a.mxLon,
|
|
253
340
|
a.pop,
|
|
254
|
-
isPrimary
|
|
341
|
+
isPrimary,
|
|
342
|
+
a.imp
|
|
255
343
|
)
|
|
256
344
|
}
|
|
257
345
|
|
|
@@ -352,6 +440,10 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
352
440
|
pop: 0,
|
|
353
441
|
neg: 0,
|
|
354
442
|
pkey: key,
|
|
443
|
+
// A postcode has no toponym fame — nobody writes an encyclopedia article about SW1A 2AA — and
|
|
444
|
+
// the score source carries no `postalcode` rows to join against anyway. NULL is the truthful
|
|
445
|
+
// value: unmeasured, so the ranking key leaves postcode rows exactly where they were.
|
|
446
|
+
imp: null,
|
|
355
447
|
}
|
|
356
448
|
|
|
357
449
|
pcAttrs.set(Number(r.id), a)
|
|
@@ -478,5 +570,6 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
478
570
|
abbrevs: nAbbr,
|
|
479
571
|
postcodes: nPostcode,
|
|
480
572
|
postcodeAliases: nPostcodeAlias,
|
|
573
|
+
...(importance ? { importanceScored: importance.matched, importanceGated: importance.gated } : {}),
|
|
481
574
|
}
|
|
482
575
|
}
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* The score side of the candidate build (#28) — load a WOF admin database's `place_importance` into
|
|
7
|
+
* a lookup the candidate builder can probe per place, so every candidate row can carry the
|
|
8
|
+
* toponym-fame prior a bare city name is decided on.
|
|
9
|
+
*
|
|
10
|
+
* ## Why this joins on the NAME and not on the id
|
|
11
|
+
*
|
|
12
|
+
* The obvious join is `candidate.spr_id = place_importance.id`, and it is wrong here. The score
|
|
13
|
+
* source (`admin-global-priority-importance.db`) and the candidate build's admin source are
|
|
14
|
+
* DIFFERENT SNAPSHOTS, and they disagree about ids for exactly the rows that matter: Whitby,
|
|
15
|
+
* Ontario is `8143502164401` in the shipped `candidate.db` and `8000001156384` in the score source
|
|
16
|
+
* — the Overture-sourced rows were re-keyed between the two. An id join silently drops every one of
|
|
17
|
+
* them, which means it drops precisely the FOREIGN HOMONYMS the fame prior exists to demote, and it
|
|
18
|
+
* drops them invisibly: the build succeeds, the column is populated, and the ranking is inert on
|
|
19
|
+
* the queries it was built for.
|
|
20
|
+
*
|
|
21
|
+
* So the join key is `(name_key, country, placetype)` — the same {@link normalizeLocalityForKey} the
|
|
22
|
+
* candidate build uses for its probe key, so the two sides fold identically by construction.
|
|
23
|
+
*
|
|
24
|
+
* ## Why the key alone is not enough
|
|
25
|
+
*
|
|
26
|
+
* `(warwick, US, locality)` names ELEVEN different places. Taking the group's max would give every
|
|
27
|
+
* Warwick in America the fame of Warwick, Rhode Island, which is the fan-out defect
|
|
28
|
+
* `importance-fanout.ts` documents one layer up, re-introduced at the join. So the group is
|
|
29
|
+
* disambiguated GEOGRAPHICALLY: the nearest centroid wins, and only within
|
|
30
|
+
* {@link IMPORTANCE_JOIN_GATE_KM}. Two artifacts describing the same settlement put its centroid in
|
|
31
|
+
* almost the same place; two same-named towns in one country do not.
|
|
32
|
+
*
|
|
33
|
+
* ## What lands in the column
|
|
34
|
+
*
|
|
35
|
+
* `place_importance.importance` VERBATIM — the pre-split conflation (encyclopedia-derived where
|
|
36
|
+
* the concordance matched, a population-derived proxy everywhere else). See
|
|
37
|
+
* {@link CandidateTable.importance} for why the split `encyclopedic` channel is deliberately NOT
|
|
38
|
+
* what is written here, with the measurement that settled it.
|
|
39
|
+
*
|
|
40
|
+
* A place with no match gets NULL. NULL is UNMEASURED, never zero — the consumer
|
|
41
|
+
* (`resolver/toponym-prior.ts`) leaves an unmeasured candidate exactly where population put it.
|
|
42
|
+
*/
|
|
43
|
+
|
|
44
|
+
import { DatabaseSync } from "node:sqlite"
|
|
45
|
+
|
|
46
|
+
import { haversineKm } from "./geo.ts"
|
|
47
|
+
import { normalizeLocalityForKey } from "./street-normalize.ts"
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* How far apart two artifacts may put the same place's centroid and still be read as the same place.
|
|
51
|
+
*
|
|
52
|
+
* MEASURED, not guessed (2026-08-10, `admin-global-priority.db` × `admin-global-priority-importance.db`; 4,476,245
|
|
53
|
+
* current locality-tier places against 676,790 scored ones, 1,020,099 of which matched a scored group by `(name_key,
|
|
54
|
+
* country, placetype)`). The nearest-centroid distance is **exactly 0.00 km for 577,080 of them (56.6%)** — the two
|
|
55
|
+
* snapshots agree to the bit — and 656,755 (64.4%) are inside 500 m.
|
|
56
|
+
*
|
|
57
|
+
* What sets the gate is where that mode ENDS, and the per-kilometre density says so plainly. It falls from 1,076
|
|
58
|
+
* places/km over 3–5 km to a trough of **441 places/km over 7–10 km**, then climbs back and flattens onto a plateau of
|
|
59
|
+
* 760–780 places/km from 30 km out to 100 km and beyond. That plateau is the background rate of two DIFFERENT towns
|
|
60
|
+
* wearing one name in one country, and it does not decay with distance because there is no reason it should. 10 km is
|
|
61
|
+
* the floor between the two populations. Admitting it scores 679,163 places (66.6% of the matched set); pushing the
|
|
62
|
+
* gate to 25 km buys 8,722 more, and by then better than half of each additional kilometre is the wrong town.
|
|
63
|
+
*
|
|
64
|
+
* The four-row bare-GB board is insensitive across this whole range — 5 km and 25 km were both measured and select
|
|
65
|
+
* identical rows — so the value is chosen by what the join MEANS, not by what it scores. The gate is the definition of
|
|
66
|
+
* "this is the same place"; widening it past the floor starts handing one town's fame to another.
|
|
67
|
+
*/
|
|
68
|
+
export const IMPORTANCE_JOIN_GATE_KM = 10
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* One scored place from the source: where it is, and what it scored.
|
|
72
|
+
*/
|
|
73
|
+
interface ScoredPlace {
|
|
74
|
+
lat: number
|
|
75
|
+
lon: number
|
|
76
|
+
importance: number
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* What {@link loadImportanceIndex} measured while reading the source. Reported by the builder so a run says how much of
|
|
81
|
+
* the gazetteer it actually scored, rather than leaving the caller to infer it from a column full of nulls.
|
|
82
|
+
*/
|
|
83
|
+
export interface ImportanceIndexStats {
|
|
84
|
+
/**
|
|
85
|
+
* Scored places read out of the source.
|
|
86
|
+
*/
|
|
87
|
+
places: number
|
|
88
|
+
/**
|
|
89
|
+
* Distinct `(name_key, country, placetype)` groups they fall into.
|
|
90
|
+
*/
|
|
91
|
+
keys: number
|
|
92
|
+
/**
|
|
93
|
+
* Places whose name folded to the empty key and could never be joined (non-Latin punctuation-only names, blanks).
|
|
94
|
+
*/
|
|
95
|
+
unkeyable: number
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* `(name_key, country, placetype)` → the scored places under it. The separator is U+0000, which no WOF name carries and
|
|
100
|
+
* no fold can produce, so the three fields can't smear into one another.
|
|
101
|
+
*/
|
|
102
|
+
function groupKey(nameKey: string, country: string | null, placetype: string | null): string {
|
|
103
|
+
return `${nameKey}\u0000${(country ?? "").toUpperCase()}\u0000${placetype ?? ""}`
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* A loaded `place_importance`, probed by name + country + placetype + position.
|
|
108
|
+
*/
|
|
109
|
+
export class ImportanceIndex {
|
|
110
|
+
readonly #groups: Map<string, ScoredPlace[]>
|
|
111
|
+
readonly stats: ImportanceIndexStats
|
|
112
|
+
/**
|
|
113
|
+
* Places {@link find} matched inside the gate.
|
|
114
|
+
*/
|
|
115
|
+
matched = 0
|
|
116
|
+
/**
|
|
117
|
+
* Places {@link find} REFUSED — the key matched a scored group, but the nearest member of it was outside the gate, so
|
|
118
|
+
* it is a different place wearing the same name.
|
|
119
|
+
*
|
|
120
|
+
* This is the number worth watching across rebuilds. A jump means the score source and the admin source have drifted
|
|
121
|
+
* apart and the join is being asked to guess; it does not mean the gate is too tight.
|
|
122
|
+
*/
|
|
123
|
+
gated = 0
|
|
124
|
+
|
|
125
|
+
constructor(groups: Map<string, ScoredPlace[]>, stats: ImportanceIndexStats) {
|
|
126
|
+
this.#groups = groups
|
|
127
|
+
this.stats = stats
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* The importance of the scored place nearest `(lat, lon)` sharing `name`'s folded key, `country` and `placetype`, or
|
|
132
|
+
* null when there is no such place within {@link IMPORTANCE_JOIN_GATE_KM}.
|
|
133
|
+
*
|
|
134
|
+
* Null is UNMEASURED. Never substitute a zero, and never fall back to a population-derived value here — the source
|
|
135
|
+
* column already carries that fallback where it has one, and inventing a second one would make an absence
|
|
136
|
+
* indistinguishable from a measurement.
|
|
137
|
+
*/
|
|
138
|
+
find(name: string, country: string | null, placetype: string | null, lat: number, lon: number): number | null {
|
|
139
|
+
const nameKey = normalizeLocalityForKey(name)
|
|
140
|
+
|
|
141
|
+
if (!nameKey) return null
|
|
142
|
+
|
|
143
|
+
const group = this.#groups.get(groupKey(nameKey, country, placetype))
|
|
144
|
+
|
|
145
|
+
if (!group) return null
|
|
146
|
+
|
|
147
|
+
let best: ScoredPlace | undefined
|
|
148
|
+
let bestKm = Infinity
|
|
149
|
+
|
|
150
|
+
for (const place of group) {
|
|
151
|
+
const km = haversineKm(lat, lon, place.lat, place.lon)
|
|
152
|
+
|
|
153
|
+
if (km < bestKm) {
|
|
154
|
+
bestKm = km
|
|
155
|
+
best = place
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
if (!best || bestKm > IMPORTANCE_JOIN_GATE_KM) {
|
|
160
|
+
this.gated++
|
|
161
|
+
|
|
162
|
+
return null
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
this.matched++
|
|
166
|
+
|
|
167
|
+
return best.importance
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Read `place_importance` (joined to `spr` for the name/country/placetype/centroid) out of a WOF admin database into an
|
|
173
|
+
* {@link ImportanceIndex}.
|
|
174
|
+
*
|
|
175
|
+
* Only CURRENT, non-deprecated places are indexed — a superseded row's score belongs to a place the gazetteer no longer
|
|
176
|
+
* carries, and letting it win the nearest-centroid contest would hand a live place a dead one's fame.
|
|
177
|
+
*
|
|
178
|
+
* The whole table is held in memory on purpose. The 2026-08-10 source holds 676,790 scored places in 544,823 groups,
|
|
179
|
+
* and the build probes it once for every one of its ~4.8 M places; the alternative is a prepared statement per place
|
|
180
|
+
* against a 3.7 GB database. Measured end to end, loading the index plus probing all 4.48 M locality-tier places takes
|
|
181
|
+
* 25 s.
|
|
182
|
+
*/
|
|
183
|
+
export function loadImportanceIndex(databasePath: string): ImportanceIndex {
|
|
184
|
+
const db = new DatabaseSync(databasePath, { readOnly: true })
|
|
185
|
+
|
|
186
|
+
try {
|
|
187
|
+
const groups = new Map<string, ScoredPlace[]>()
|
|
188
|
+
let places = 0
|
|
189
|
+
let unkeyable = 0
|
|
190
|
+
|
|
191
|
+
for (const row of db
|
|
192
|
+
.prepare(
|
|
193
|
+
`SELECT s.name AS name, s.country AS country, s.placetype AS placetype,
|
|
194
|
+
s.latitude AS latitude, s.longitude AS longitude, i.importance AS importance
|
|
195
|
+
FROM place_importance i JOIN spr s ON s.id = i.id
|
|
196
|
+
WHERE s.is_current != 0 AND s.is_deprecated = 0`
|
|
197
|
+
)
|
|
198
|
+
.iterate()) {
|
|
199
|
+
const importance = Number(row.importance)
|
|
200
|
+
|
|
201
|
+
if (!Number.isFinite(importance)) continue
|
|
202
|
+
const nameKey = normalizeLocalityForKey(String(row.name ?? ""))
|
|
203
|
+
|
|
204
|
+
if (!nameKey) {
|
|
205
|
+
unkeyable++
|
|
206
|
+
|
|
207
|
+
continue
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
const key = groupKey(nameKey, row.country as string | null, row.placetype as string | null)
|
|
211
|
+
let group = groups.get(key)
|
|
212
|
+
|
|
213
|
+
if (!group) {
|
|
214
|
+
group = []
|
|
215
|
+
groups.set(key, group)
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
group.push({ lat: Number(row.latitude), lon: Number(row.longitude), importance })
|
|
219
|
+
|
|
220
|
+
places++
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
return new ImportanceIndex(groups, { places, keys: groups.size, unkeyable })
|
|
224
|
+
} finally {
|
|
225
|
+
db.close()
|
|
226
|
+
}
|
|
227
|
+
}
|
package/candidate-lookup.ts
CHANGED
|
@@ -32,8 +32,9 @@ import type { CandidateTable, CountryCodeTable, PlacetypeCodeTable } from "./can
|
|
|
32
32
|
import { readGazetteerCoverageManifest } from "./coverage-manifest-schema.ts"
|
|
33
33
|
import { haversineKm } from "./geo.ts"
|
|
34
34
|
import { trigramJaccard } from "./lookup.ts"
|
|
35
|
+
import { referentialFromPopulation } from "./place-importance-schema.ts"
|
|
35
36
|
import { POSTAL_CITY_CANDIDATE_TABLE, type PostalCityCandidateTable } from "./postal-city-candidate-schema.ts"
|
|
36
|
-
import { hasTable } from "./sqlite-utils.ts"
|
|
37
|
+
import { hasColumn, hasTable } from "./sqlite-utils.ts"
|
|
37
38
|
import { normalizeLocalityForKey, stripLocalityQualifier } from "./street-normalize.ts"
|
|
38
39
|
import type { FindPlaceQuery, PlaceCandidate, PlaceLookup, WOFPlacetype } from "./types.ts"
|
|
39
40
|
|
|
@@ -66,7 +67,14 @@ type CandidateRow = Pick<
|
|
|
66
67
|
| "max_lon"
|
|
67
68
|
| "neg_rank"
|
|
68
69
|
| "is_primary"
|
|
69
|
-
|
|
70
|
+
| "population"
|
|
71
|
+
> &
|
|
72
|
+
// `importance` is OPTIONAL on the row rather than `number | null`, because whether the SELECT names it
|
|
73
|
+
// depends on the artifact: a candidate.db built before #28 has no such column and the probe leaves it
|
|
74
|
+
// out (see `#importanceSelect`). `undefined` therefore means "this build cannot tell you", which is the
|
|
75
|
+
// same answer as `null`'s "the score source had no measurement" — both are UNMEASURED, and the emit
|
|
76
|
+
// below collapses them into the one thing the consumer understands: no `importance` field at all.
|
|
77
|
+
Partial<Pick<CandidateTable, "importance">>
|
|
70
78
|
|
|
71
79
|
/**
|
|
72
80
|
* FTS5-trigram over-fetch before the trigram-Jaccard re-rank, and the minimum similarity to count as a fuzzy hit (below
|
|
@@ -75,6 +83,14 @@ type CandidateRow = Pick<
|
|
|
75
83
|
const FUZZY_FETCH = 40
|
|
76
84
|
const FUZZY_MIN = 0.34
|
|
77
85
|
|
|
86
|
+
/**
|
|
87
|
+
* Postcode-containment re-rank gate radius (km) — the SAME value the resolver's country pass measures at
|
|
88
|
+
* (`POSTCODE_COUNTRY_COHERENCE_GATE_KM`, resolver/postcode-country-coherence.ts): a locality within this distance of
|
|
89
|
+
* the postcode's own centroid counts as "containing" it. One number, two passes — a divergence here would make the two
|
|
90
|
+
* mechanisms disagree about what is proximal.
|
|
91
|
+
*/
|
|
92
|
+
const POSTCODE_CONTAINMENT_GATE_KM = 25
|
|
93
|
+
|
|
78
94
|
/**
|
|
79
95
|
* Bounded PRIMARY-NAME preference across a CROSS-COUNTRY name collision (the `is_primary` ranking signal).
|
|
80
96
|
*
|
|
@@ -125,6 +141,14 @@ export type RankedRow<R> = R & {
|
|
|
125
141
|
* tier, and a same-country nickname (San Francisco's "Frisco") is never touched.
|
|
126
142
|
*/
|
|
127
143
|
demoted: boolean
|
|
144
|
+
/**
|
|
145
|
+
* True when this row came from the TYPO-CORRECTOR tier — the FTS5-trigram fallback that fires only after the exact
|
|
146
|
+
* and qualifier-strip probes both missed. Such a row answers a query the gazetteer does not contain, so it is a fuzzy
|
|
147
|
+
* match by construction and must not claim `exactMatch` (#17). Recall is unaffected: the row is still returned, still
|
|
148
|
+
* ranked, still resolvable — it just stops asserting a match quality it does not have, which is what the FTS backend
|
|
149
|
+
* has always done and what every `exactMatch`-filtering consumer assumed.
|
|
150
|
+
*/
|
|
151
|
+
fuzzy?: boolean
|
|
128
152
|
}
|
|
129
153
|
|
|
130
154
|
/**
|
|
@@ -228,6 +252,13 @@ export class WOFCandidateTableLookup implements PlaceLookup {
|
|
|
228
252
|
* (the hard-country coverage gate, guard-B plausibility) falls back to its code constants byte-identically.
|
|
229
253
|
*/
|
|
230
254
|
readonly artifactCoverage: GazetteerArtifactCoverage | undefined
|
|
255
|
+
/**
|
|
256
|
+
* `", importance"` when this artifact carries the #28 fame column, `""` when it does not — spliced into the probe's
|
|
257
|
+
* SELECT list. Existence-gated exactly like `#ftsProbe` and `#postalCityProbe` above, and for the same reason: a
|
|
258
|
+
* candidate.db built before the column is a valid artifact, and naming a column it lacks would turn a stale gazetteer
|
|
259
|
+
* into `no such column` on the first keystroke rather than into "no fame signal", which is what it is.
|
|
260
|
+
*/
|
|
261
|
+
readonly #importanceSelect: string
|
|
231
262
|
|
|
232
263
|
constructor(opts: WOFCandidateTableLookupOpts) {
|
|
233
264
|
if (opts.database) {
|
|
@@ -273,6 +304,9 @@ export class WOFCandidateTableLookup implements PlaceLookup {
|
|
|
273
304
|
this.#nameKeyExistsProbe = this.#db.prepare("SELECT 1 FROM candidate WHERE name_key = ? LIMIT 1")
|
|
274
305
|
}
|
|
275
306
|
|
|
307
|
+
// #28 fame column: probed ONCE here (it runs a PRAGMA, and `findPlace` is per-keystroke hot).
|
|
308
|
+
this.#importanceSelect = hasColumn(this.#db, "candidate", "importance") ? ", importance" : ""
|
|
309
|
+
|
|
276
310
|
// Coverage manifest (survey candidate #2): the artifact's own coverage facts, existence-gated like
|
|
277
311
|
// the probes above — a candidate.db built before the manifest reads `undefined` and consumers keep
|
|
278
312
|
// their code-constant fallbacks byte-identically.
|
|
@@ -289,6 +323,38 @@ export class WOFCandidateTableLookup implements PlaceLookup {
|
|
|
289
323
|
return expandPlacetypeFilter(want as readonly string[]).includes("locality")
|
|
290
324
|
}
|
|
291
325
|
|
|
326
|
+
/**
|
|
327
|
+
* The postcode-containment anchor: the postcode's own centroid row in the candidate table, keyed whitespace-stripped
|
|
328
|
+
* (#920 — the same fold the build applies to postcode rows), country-scoped when the query is, first
|
|
329
|
+
* coordinate-bearing row wins. null when the candidate table carries no such postcode — the re-rank then abstains,
|
|
330
|
+
* because a recall gap is not evidence for the name match. Meaning-of-zero: a 0,0 row is the build's unlocated
|
|
331
|
+
* sentinel, never a real centroid.
|
|
332
|
+
*/
|
|
333
|
+
#postcodeAnchor(postcode: string, country?: string): { lat: number; lon: number } | null {
|
|
334
|
+
const placetypeID = this.#placetypeToID.get("postalcode")
|
|
335
|
+
|
|
336
|
+
if (placetypeID === undefined) return null
|
|
337
|
+
|
|
338
|
+
const conds = ["name_key = ?", "placetype_id = ?"]
|
|
339
|
+
const params: Array<string | number> = [postcode.replaceAll(/\s+/g, ""), placetypeID]
|
|
340
|
+
|
|
341
|
+
if (country) {
|
|
342
|
+
const countryID = this.#countryToID.get(country.toUpperCase())
|
|
343
|
+
|
|
344
|
+
if (countryID === undefined) return null // a country the candidate table doesn't carry
|
|
345
|
+
conds.push("country_id = ?")
|
|
346
|
+
params.push(countryID)
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
const row = this.#db
|
|
350
|
+
.prepare(`SELECT latitude, longitude FROM candidate WHERE ${conds.join(" AND ")} ORDER BY neg_rank ASC LIMIT 1`)
|
|
351
|
+
.get(...params) as { latitude: number; longitude: number } | undefined
|
|
352
|
+
|
|
353
|
+
if (!row || (Number(row.latitude) === 0 && Number(row.longitude) === 0)) return null
|
|
354
|
+
|
|
355
|
+
return { lat: Number(row.latitude), lon: Number(row.longitude) }
|
|
356
|
+
}
|
|
357
|
+
|
|
292
358
|
async findPlace(query: FindPlaceQuery): Promise<PlaceCandidate[]> {
|
|
293
359
|
let text = (query.text ?? "").trim()
|
|
294
360
|
|
|
@@ -392,9 +458,16 @@ export class WOFCandidateTableLookup implements PlaceLookup {
|
|
|
392
458
|
// intended primary even when a cluster of more-populous foreign aliases sits ahead of it. `is_primary`
|
|
393
459
|
// + `country_id` feed that re-rank. A single-country probe (a country filter, or all rows same
|
|
394
460
|
// country) re-ranks to the identical population order, so the common path is untouched.
|
|
461
|
+
// `population` rides along for the REFERENTIAL score on the result (ROAD_TO_V9 §2) — one more
|
|
462
|
+
// column off a clustered row the probe already reads, and it is NOT what the probe orders by:
|
|
463
|
+
// `neg_rank` remains the sort key, so this changes no ordering, only what the result reports.
|
|
464
|
+
// `importance` (#28) rides along on the same terms, and there is deliberately NO `ORDER BY` on it:
|
|
465
|
+
// the fame prior is applied by the RESOLVER (`resolver/toponym-prior.ts`), which alone knows
|
|
466
|
+
// whether the query was bare enough to deserve it. A backend that pre-sorted by fame would apply
|
|
467
|
+
// it to every lookup, including the qualified addresses the D-rule guard exists to protect.
|
|
395
468
|
const sql =
|
|
396
|
-
"SELECT spr_id, name, country_id, placetype_id, latitude, longitude, min_lat, min_lon, max_lat, max_lon, neg_rank, is_primary " +
|
|
397
|
-
|
|
469
|
+
"SELECT spr_id, name, country_id, placetype_id, latitude, longitude, min_lat, min_lon, max_lat, max_lon, neg_rank, is_primary, population" +
|
|
470
|
+
`${this.#importanceSelect} FROM candidate WHERE ${conds.join(" AND ")} ORDER BY neg_rank ASC LIMIT ?`
|
|
398
471
|
|
|
399
472
|
const fetched = this.#db.prepare(sql).all(...params, Math.max(limit, RERANK_FETCH)) as unknown as CandidateRow[]
|
|
400
473
|
|
|
@@ -457,7 +530,9 @@ export class WOFCandidateTableLookup implements PlaceLookup {
|
|
|
457
530
|
for (const h of ranked) {
|
|
458
531
|
if (seen.has(h.nk)) continue
|
|
459
532
|
seen.add(h.nk)
|
|
460
|
-
rows
|
|
533
|
+
// #17: stamp the tier. These rows answer a name the gazetteer does not carry, so they are
|
|
534
|
+
// fuzzy matches and `exactMatch` below must say so — see `RankedRow.fuzzy`.
|
|
535
|
+
rows.push(...probe(h.nk, regionID).map((r) => ({ ...r, fuzzy: true })))
|
|
461
536
|
|
|
462
537
|
if (rows.length >= limit) break
|
|
463
538
|
}
|
|
@@ -478,6 +553,44 @@ export class WOFCandidateTableLookup implements PlaceLookup {
|
|
|
478
553
|
rows = cascade(undefined)
|
|
479
554
|
}
|
|
480
555
|
|
|
556
|
+
// Postcode-containment coherence (#31, Mechanism 2): re-rank the rows by proximity to the postcode's
|
|
557
|
+
// own centroid, so the locality that CONTAINS the postcode wins the name-match tie (the "Paris" that
|
|
558
|
+
// holds 75001, not the one that holds a 75001-free namesake). The resolver sends this flag on locality
|
|
559
|
+
// lookups when `ResolveOpts.postcodeContainmentCoherence` is on. Strictly beneath the #741 postal-city
|
|
560
|
+
// short-circuit above — an exact (name, postcode) hit IS the answer and outranks any re-rank — and after
|
|
561
|
+
// the region-scope fallback, so it sees the final row set. Rows within the gate sort by distance first;
|
|
562
|
+
// the out-gate tail keeps its original population-first order. No in-gate row, or no postcode row in
|
|
563
|
+
// the candidate table → unchanged (byte-identical to the flag-off path).
|
|
564
|
+
if (
|
|
565
|
+
query.postcode &&
|
|
566
|
+
query.postcodeContainmentCoherence === true &&
|
|
567
|
+
this.#wantsLocality(query.placetype) &&
|
|
568
|
+
rows.length > 1
|
|
569
|
+
) {
|
|
570
|
+
const anchor = this.#postcodeAnchor(query.postcode, query.country)
|
|
571
|
+
|
|
572
|
+
if (anchor) {
|
|
573
|
+
const inGate: Array<{ row: RankedRow<CandidateRow>; distanceKm: number }> = []
|
|
574
|
+
const outGate: RankedRow<CandidateRow>[] = []
|
|
575
|
+
|
|
576
|
+
for (const row of rows) {
|
|
577
|
+
const distanceKm = haversineKm(anchor.lat, anchor.lon, Number(row.latitude), Number(row.longitude))
|
|
578
|
+
|
|
579
|
+
if (distanceKm <= POSTCODE_CONTAINMENT_GATE_KM) {
|
|
580
|
+
inGate.push({ row, distanceKm })
|
|
581
|
+
} else {
|
|
582
|
+
outGate.push(row)
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
if (inGate.length) {
|
|
587
|
+
// oxlint-disable-next-line unicorn/no-array-sort -- sorts a freshly-built array; toSorted would double-allocate on a hot path
|
|
588
|
+
inGate.sort((a, b) => a.distanceKm - b.distanceKm)
|
|
589
|
+
rows = [...inGate.map(({ row }) => row), ...outGate]
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
|
|
481
594
|
const candidates = rows.map((row): PlaceCandidate => {
|
|
482
595
|
const hasBbox = row.min_lat != null && row.max_lat != null && row.min_lon != null && row.max_lon != null
|
|
483
596
|
|
|
@@ -502,8 +615,32 @@ export class WOFCandidateTableLookup implements PlaceLookup {
|
|
|
502
615
|
// Every candidate row IS an exact normalized-name (or alias/abbrev) match — the cascade's exact tier
|
|
503
616
|
// accepts alias-exact hits ("New York City" → New York) the same as canonical — EXCEPT a cross-country
|
|
504
617
|
// alias that lost the bounded contest to a same-key primary (`demoted`): it drops to the partial tier so
|
|
505
|
-
// the walk's country posterior can't cross back over the primary (see `RankedRow.demoted`)
|
|
506
|
-
|
|
618
|
+
// the walk's country posterior can't cross back over the primary (see `RankedRow.demoted`) — and
|
|
619
|
+
// EXCEPT a row the typo-corrector produced (`fuzzy`), which by definition answers a name the
|
|
620
|
+
// gazetteer does not carry (see `RankedRow.fuzzy`).
|
|
621
|
+
exactMatch: !row.demoted && !row.fuzzy,
|
|
622
|
+
// The two-score split's carry (ROAD_TO_V9 §2). `referential` names the prominence this
|
|
623
|
+
// backend has always ordered by — `neg_rank` IS `-log10(population + 1)`, so the score and
|
|
624
|
+
// the sort key are two readings of the same number.
|
|
625
|
+
...(row.population === null || row.population <= 0
|
|
626
|
+
? {}
|
|
627
|
+
: { population: row.population, referential: referentialFromPopulation(row.population) }),
|
|
628
|
+
// #28: the fame prior, from the `importance` column the candidate build joins in. Emitted ONLY
|
|
629
|
+
// when the artifact measured this place — an absent field is what `rankByImportance` reads as
|
|
630
|
+
// "does not participate", and a 0 would be a claim nobody made (meaning-of-zero).
|
|
631
|
+
//
|
|
632
|
+
// The field name matches the column because they hold the same thing: the score source's
|
|
633
|
+
// BLENDED prior — the concordance's encyclopedia-derived channel where a concordance matched, a
|
|
634
|
+
// population-derived proxy everywhere else. It is NOT the strict `encyclopedic` channel
|
|
635
|
+
// `place-importance-schema.ts` defines, and it deliberately does not land in that field: the
|
|
636
|
+
// strict channel was measured on 2026-08-10 and covers eleven countries, none of them CA/AU/RU,
|
|
637
|
+
// which makes it inert on three of the four homonym contests the prior exists to settle.
|
|
638
|
+
// `PlaceCandidate.encyclopedic` stays reserved for a strict-channel source (the FTS backend's
|
|
639
|
+
// clauses are strict and today emit NULL for everything — no shipped admin DB has the split
|
|
640
|
+
// table at all). See `candidate-schema.ts` → {@link CandidateTable.importance}.
|
|
641
|
+
...(typeof row.importance === "number" && Number.isFinite(row.importance)
|
|
642
|
+
? { importance: row.importance }
|
|
643
|
+
: {}),
|
|
507
644
|
...(hasBbox
|
|
508
645
|
? {
|
|
509
646
|
bbox: {
|