@mailwoman/resolver-wof-sqlite 8.6.0 → 9.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ancestry-backfill.ts +103 -64
- package/build-candidate.ts +230 -29
- package/candidate-importance.ts +227 -0
- package/candidate-lookup.ts +159 -9
- package/candidate-schema.ts +29 -0
- package/coverage-manifest-schema.ts +1 -1
- package/fst-autocomplete.ts +22 -11
- package/fst-builder.ts +76 -50
- package/fst-deserialize-web.ts +46 -8
- package/fst-freshness.ts +333 -0
- package/fst-serialize.ts +78 -13
- package/fst-types.ts +58 -1
- package/fts.ts +12 -0
- package/geo.ts +16 -41
- package/geonames-aliases.ts +105 -37
- package/geonames-postal.ts +65 -21
- package/index.ts +16 -1
- package/interpolation.ts +2 -1
- package/lookup.ts +44 -5
- package/out/ancestry-backfill.d.ts +37 -18
- package/out/ancestry-backfill.d.ts.map +1 -1
- package/out/ancestry-backfill.js +82 -52
- package/out/ancestry-backfill.js.map +1 -1
- package/out/build-candidate.d.ts +42 -0
- package/out/build-candidate.d.ts.map +1 -1
- package/out/build-candidate.js +141 -13
- package/out/build-candidate.js.map +1 -1
- package/out/candidate-importance.d.ts +132 -0
- package/out/candidate-importance.d.ts.map +1 -0
- package/out/candidate-importance.js +174 -0
- package/out/candidate-importance.js.map +1 -0
- package/out/candidate-lookup.d.ts +9 -1
- package/out/candidate-lookup.d.ts.map +1 -1
- package/out/candidate-lookup.js +127 -8
- package/out/candidate-lookup.js.map +1 -1
- package/out/candidate-schema.d.ts +25 -1
- package/out/candidate-schema.d.ts.map +1 -1
- package/out/candidate-schema.js +5 -0
- package/out/candidate-schema.js.map +1 -1
- package/out/coverage-manifest-schema.d.ts +1 -1
- package/out/coverage-manifest-schema.js +1 -1
- package/out/fst-autocomplete.d.ts +13 -3
- package/out/fst-autocomplete.d.ts.map +1 -1
- package/out/fst-autocomplete.js +10 -9
- package/out/fst-autocomplete.js.map +1 -1
- package/out/fst-builder.d.ts.map +1 -1
- package/out/fst-builder.js +64 -40
- package/out/fst-builder.js.map +1 -1
- package/out/fst-deserialize-web.d.ts.map +1 -1
- package/out/fst-deserialize-web.js +38 -8
- package/out/fst-deserialize-web.js.map +1 -1
- package/out/fst-freshness.d.ts +138 -0
- package/out/fst-freshness.d.ts.map +1 -0
- package/out/fst-freshness.js +238 -0
- package/out/fst-freshness.js.map +1 -0
- package/out/fst-serialize.d.ts +19 -3
- package/out/fst-serialize.d.ts.map +1 -1
- package/out/fst-serialize.js +68 -13
- package/out/fst-serialize.js.map +1 -1
- package/out/fst-types.d.ts +61 -1
- package/out/fst-types.d.ts.map +1 -1
- package/out/fts.d.ts +11 -0
- package/out/fts.d.ts.map +1 -1
- package/out/fts.js +11 -0
- package/out/fts.js.map +1 -1
- package/out/geo.d.ts +10 -13
- package/out/geo.d.ts.map +1 -1
- package/out/geo.js +15 -35
- package/out/geo.js.map +1 -1
- package/out/geonames-aliases.d.ts +23 -1
- package/out/geonames-aliases.d.ts.map +1 -1
- package/out/geonames-aliases.js +88 -33
- package/out/geonames-aliases.js.map +1 -1
- package/out/geonames-postal.d.ts +22 -1
- package/out/geonames-postal.d.ts.map +1 -1
- package/out/geonames-postal.js +50 -16
- package/out/geonames-postal.js.map +1 -1
- package/out/index.d.ts +3 -1
- package/out/index.d.ts.map +1 -1
- package/out/index.js +3 -1
- package/out/index.js.map +1 -1
- package/out/interpolation.d.ts.map +1 -1
- package/out/interpolation.js +2 -1
- package/out/interpolation.js.map +1 -1
- package/out/lookup.d.ts.map +1 -1
- package/out/lookup.js +37 -5
- package/out/lookup.js.map +1 -1
- package/out/place-importance-schema.d.ts +189 -0
- package/out/place-importance-schema.d.ts.map +1 -0
- package/out/place-importance-schema.js +242 -0
- package/out/place-importance-schema.js.map +1 -0
- package/out/poi-lookup.d.ts +1 -1
- package/out/poi-lookup.js +3 -3
- package/out/reverse.d.ts.map +1 -1
- package/out/reverse.js +3 -9
- package/out/reverse.js.map +1 -1
- package/out/sqlite-convention-source.d.ts.map +1 -1
- package/out/sqlite-convention-source.js +4 -3
- package/out/sqlite-convention-source.js.map +1 -1
- package/out/sqlite-utils.d.ts +12 -0
- package/out/sqlite-utils.d.ts.map +1 -1
- package/out/sqlite-utils.js +20 -0
- package/out/sqlite-utils.js.map +1 -1
- package/out/street-morphology-fst-builder.d.ts.map +1 -1
- package/out/street-morphology-fst-builder.js +7 -5
- package/out/street-morphology-fst-builder.js.map +1 -1
- package/out/street-normalize.d.ts +2 -1
- package/out/street-normalize.d.ts.map +1 -1
- package/out/street-normalize.js +2 -1
- package/out/street-normalize.js.map +1 -1
- package/out/types.d.ts +39 -0
- package/out/types.d.ts.map +1 -1
- package/package.json +24 -6
- package/place-importance-schema.ts +353 -0
- package/poi-lookup.ts +3 -3
- package/reverse.ts +4 -9
- package/sqlite-convention-source.ts +5 -3
- package/sqlite-utils.ts +21 -0
- package/street-morphology-fst-builder.ts +8 -5
- package/street-normalize.ts +3 -2
- package/types.ts +39 -0
package/ancestry-backfill.ts
CHANGED
|
@@ -3,9 +3,8 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
* Repair the
|
|
7
|
-
* unified-schema.ts) leaves
|
|
8
|
-
* multi-parent" sentinel.
|
|
6
|
+
* Repair the truncated ancestry that {@link populateAncestors} (the parent_id closure in
|
|
7
|
+
* unified-schema.ts) leaves wherever the closure dead-ends before reaching the top.
|
|
9
8
|
*
|
|
10
9
|
* Root cause (#440 / #832): a place that straddles multiple parents — New York City spans five
|
|
11
10
|
* counties (its boroughs), London 30+ — carries `wof:parent_id = -4`, so the parent_id closure
|
|
@@ -14,22 +13,47 @@
|
|
|
14
13
|
* a correctly-parented namesake ("New York Mills", pop 3,190) wins over NYC's 8.8M. The same defect
|
|
15
14
|
* orphans London, Singapore, and ~2,850 other localities — the most demo-visible queries.
|
|
16
15
|
*
|
|
16
|
+
* **The dead end is inherited by children (#1445).** Repairing the `-4` place itself does not repair
|
|
17
|
+
* anything BELOW it: the closure walks Brooklyn → New York and stops, because New York's own
|
|
18
|
+
* `parent_id` is `-4`. Brooklyn-the-borough (pop 2.5M) therefore carried exactly two ancestor rows —
|
|
19
|
+
* itself and New York — with no county, region or country, and the only US locality-tier place named
|
|
20
|
+
* "Brooklyn" that survived a New-York-State descendant filter was Pillar Point, a Jefferson County
|
|
21
|
+
* hamlet 411 km away carrying "Brooklyn" as an alternate name. All five NYC boroughs, every London
|
|
22
|
+
* borough and Hyderabad's zones were in the same state.
|
|
23
|
+
*
|
|
24
|
+
* So the candidate test is NOT "has only a self ancestor" — that misses every child of a repaired
|
|
25
|
+
* place, which by construction has two. It is **"has no `country`-tier ancestor"**. Country is the
|
|
26
|
+
* universal terminal for every non-{@link TOP_PLACETYPES} placetype, so its absence is exactly the
|
|
27
|
+
* signal that the chain dead-ended somewhere, at whatever depth. On a wide-coverage build this
|
|
28
|
+
* selects ~24k places against 2.55M rows.
|
|
29
|
+
*
|
|
30
|
+
* A place whose `wof:hierarchy` genuinely stops short is NOT a candidate and needs no repair: the
|
|
31
|
+
* source is the authority on what a place should have. American Samoa's localities, for instance,
|
|
32
|
+
* have `{country_id, locality_id}` and no region in WOF itself — the artifact matching that is
|
|
33
|
+
* correct, not truncated.
|
|
34
|
+
*
|
|
17
35
|
* The authoritative hierarchy IS in the source geojson: `wof:hierarchy` is an array of branches,
|
|
18
36
|
* each a `<placetype>_id` → id map (region_id, county_id, country_id, …), fully populated even when
|
|
19
|
-
* parent_id is -4. This reads it for every
|
|
20
|
-
*
|
|
37
|
+
* parent_id is -4. This reads it for every candidate and inserts the missing ancestor rows (one per
|
|
38
|
+
* distinct ancestor across branches).
|
|
21
39
|
*
|
|
22
40
|
* MUST run AFTER populateAncestors and BEFORE the build freezes (VACUUM INTO), so the rows land in
|
|
23
41
|
* the shipped artifact — `scripts/build-unified-wof.ts` Phase 3 calls it inline. The standalone
|
|
24
42
|
* `scripts/backfill-ancestors-from-hierarchy.ts` is a thin CLI over the same function for ad-hoc
|
|
25
|
-
* repair of an already-built DB. Idempotent
|
|
26
|
-
*
|
|
43
|
+
* repair of an already-built DB. Idempotent by the per-pair existence check, not by the candidate
|
|
44
|
+
* test: each (id, ancestor_id) is inserted at most once, so a second run over the same DB adds
|
|
45
|
+
* nothing.
|
|
27
46
|
*/
|
|
28
47
|
|
|
29
|
-
import {
|
|
30
|
-
import { join } from "node:path"
|
|
48
|
+
import { readdirSync } from "node:fs"
|
|
31
49
|
import type { DatabaseSync } from "node:sqlite"
|
|
32
50
|
|
|
51
|
+
import { DatabaseClient } from "@mailwoman/core/kysley/client"
|
|
52
|
+
import { readWOFFeature } from "@mailwoman/core/resources/whosonfirst"
|
|
53
|
+
import { join } from "path-ts"
|
|
54
|
+
|
|
55
|
+
import type { WOFDatabase } from "./schema.ts"
|
|
56
|
+
|
|
33
57
|
/**
|
|
34
58
|
* Genuinely top-level placetypes — they never have (or need) an ancestor, so skip them.
|
|
35
59
|
*/
|
|
@@ -45,8 +69,8 @@ export interface AncestryBackfillResult {
|
|
|
45
69
|
*/
|
|
46
70
|
rowsAdded: number
|
|
47
71
|
/**
|
|
48
|
-
*
|
|
49
|
-
*
|
|
72
|
+
* Candidates whose source geojson could not be found (non-WOF backfilled places, or repos not present locally) —
|
|
73
|
+
* skipped, not an error.
|
|
50
74
|
*/
|
|
51
75
|
noGeojson: number
|
|
52
76
|
}
|
|
@@ -89,34 +113,6 @@ export function discoverAdminDataRoots(reposRoot: string): string[] {
|
|
|
89
113
|
return roots
|
|
90
114
|
}
|
|
91
115
|
|
|
92
|
-
/**
|
|
93
|
-
* WOF geojson lives sharded: an id resolves to `<3-char chunks>/<id>.geojson` under each data root.
|
|
94
|
-
*/
|
|
95
|
-
function geojsonForID(id: number, roots: readonly string[]): Record<string, unknown> | null {
|
|
96
|
-
const s = String(id)
|
|
97
|
-
const chunks: string[] = []
|
|
98
|
-
|
|
99
|
-
for (let i = 0; i < s.length; i += 3) {
|
|
100
|
-
chunks.push(s.slice(i, i + 3))
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
const rel = join(chunks.join("/"), `${s}.geojson`)
|
|
104
|
-
|
|
105
|
-
for (const root of roots) {
|
|
106
|
-
const fp = join(root, rel)
|
|
107
|
-
|
|
108
|
-
if (existsSync(fp)) {
|
|
109
|
-
try {
|
|
110
|
-
return JSON.parse(readFileSync(fp, "utf8")) as Record<string, unknown>
|
|
111
|
-
} catch {
|
|
112
|
-
return null
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
return null
|
|
118
|
-
}
|
|
119
|
-
|
|
120
116
|
// `<placetype>_id` key → ancestor placetype. WOF hierarchy keys are e.g. region_id, county_id. Self
|
|
121
117
|
// is filtered downstream by the `aid === id` check, so we do NOT special-case locality here: for a
|
|
122
118
|
// locality candidate `locality_id` IS self (dropped by aid===id), but for a neighbourhood candidate
|
|
@@ -128,48 +124,82 @@ function placetypeFromKey(key: string): string | null {
|
|
|
128
124
|
}
|
|
129
125
|
|
|
130
126
|
/**
|
|
131
|
-
* Insert missing ancestor rows for
|
|
132
|
-
* `geojsonRoots` (see {@link discoverAdminDataRoots}). Runs inside a
|
|
133
|
-
* lifecycle (open, WAL checkpoint, close).
|
|
127
|
+
* Insert missing ancestor rows for every place whose ancestry chain dead-ended before reaching a country, by reading
|
|
128
|
+
* `wof:hierarchy` from its source geojson under `geojsonRoots` (see {@link discoverAdminDataRoots}). Runs inside a
|
|
129
|
+
* single transaction; caller owns connection lifecycle (open, WAL checkpoint, close).
|
|
134
130
|
*
|
|
135
|
-
* `opts.
|
|
131
|
+
* `opts.maxID` bounds the candidate scan to ids BELOW it — pass the synthetic-id base (`OVERTURE_ID_BASE`, 8e12) so the
|
|
136
132
|
* backfill considers only real WOF places. Overture/GeoNames rows carry synthetic ids and have NO `wof:hierarchy`
|
|
137
|
-
* geojson, so probing them is pure waste: on a wide-coverage DB the
|
|
138
|
-
* localities, and the per-candidate geojson probe across every repo root turns a seconds-long WOF-only pass into a
|
|
133
|
+
* geojson, so probing them is pure waste: on a wide-coverage DB the country-less set is millions of Overture/GeoNames
|
|
134
|
+
* leaf localities, and the per-candidate geojson probe across every repo root turns a seconds-long WOF-only pass into a
|
|
139
135
|
* ~40-minute one (their ancestry comes from the parent_id closure, not this backfill). Correctness-preserving — the
|
|
140
|
-
* skipped rows would have `noGeojson`-skipped anyway. Omit `
|
|
136
|
+
* skipped rows would have `noGeojson`-skipped anyway. Omit `maxID` (default) for the legacy WOF-only DBs.
|
|
141
137
|
*/
|
|
142
|
-
export function backfillAncestorsFromHierarchy(
|
|
138
|
+
export async function backfillAncestorsFromHierarchy(
|
|
143
139
|
db: DatabaseSync,
|
|
144
140
|
geojsonRoots: readonly string[],
|
|
145
|
-
opts: {
|
|
146
|
-
): AncestryBackfillResult {
|
|
147
|
-
const
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
141
|
+
opts: { maxID?: number } = {}
|
|
142
|
+
): Promise<AncestryBackfillResult> {
|
|
143
|
+
const maxID = opts.maxID ?? Number.MAX_SAFE_INTEGER
|
|
144
|
+
const kdb = new DatabaseClient<WOFDatabase>({ database: db })
|
|
145
|
+
|
|
146
|
+
// "No country-tier ancestor" is the dead-end signal at any depth — see the module docstring. The
|
|
147
|
+
// earlier "<= 1 ancestor row" test only caught the dead end's origin, never the children that
|
|
148
|
+
// inherit it (a child of a repaired -4 place has two rows: itself and that parent) (#1445).
|
|
149
|
+
// The id bound is stated first so SQLite prunes by the PK index before the NOT EXISTS runs at all.
|
|
150
|
+
const candidateBase = kdb
|
|
151
|
+
.selectFrom("spr")
|
|
152
|
+
.where("id", "<", maxID)
|
|
153
|
+
.where((eb) =>
|
|
154
|
+
eb.not(
|
|
155
|
+
eb.exists(
|
|
156
|
+
eb
|
|
157
|
+
.selectFrom("ancestors as a")
|
|
158
|
+
.select("a.id")
|
|
159
|
+
.whereRef("a.id", "=", "spr.id")
|
|
160
|
+
.where("a.ancestor_placetype", "=", "country")
|
|
161
|
+
)
|
|
162
|
+
)
|
|
154
163
|
)
|
|
155
|
-
|
|
164
|
+
|
|
165
|
+
const candidates = await candidateBase.select(["id", "placetype"]).execute()
|
|
166
|
+
|
|
167
|
+
// Every candidate's existing ancestors in ONE query rather than an indexed read each. The widened
|
|
168
|
+
// candidate test made that per-candidate read the dominant cost of the pass, and the set is bounded:
|
|
169
|
+
// a candidate reaching this point has a handful of rows at most. The candidate set rides in as the
|
|
170
|
+
// SAME predicate re-issued as a subquery, never a materialized `IN (?, ?, …)` list — node:sqlite
|
|
171
|
+
// caps a statement at 32,766 bound variables and the wide-coverage build has 67,521 candidates
|
|
172
|
+
// (measured 2026-08-04).
|
|
173
|
+
const alreadyPresent = new Map<number, Set<number>>()
|
|
174
|
+
|
|
175
|
+
for (const row of await kdb
|
|
176
|
+
.selectFrom("ancestors")
|
|
177
|
+
.select(["id", "ancestor_id"])
|
|
178
|
+
.where("id", "in", candidateBase.select("spr.id"))
|
|
179
|
+
.execute()) {
|
|
180
|
+
let set = alreadyPresent.get(row.id)
|
|
181
|
+
|
|
182
|
+
if (!set) {
|
|
183
|
+
set = new Set()
|
|
184
|
+
alreadyPresent.set(row.id, set)
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
set.add(Number(row.ancestor_id))
|
|
188
|
+
}
|
|
156
189
|
|
|
157
190
|
const insert = db.prepare(
|
|
158
191
|
"INSERT INTO ancestors (id, ancestor_id, ancestor_placetype, lastmodified) VALUES (?, ?, ?, 0)"
|
|
159
192
|
)
|
|
160
193
|
|
|
161
|
-
const hasRow = db.prepare("SELECT 1 FROM ancestors WHERE id = ? AND ancestor_id = ? LIMIT 1")
|
|
162
|
-
|
|
163
194
|
let placesFixed = 0
|
|
164
195
|
let rowsAdded = 0
|
|
165
196
|
let noGeojson = 0
|
|
166
197
|
db.exec("BEGIN")
|
|
167
198
|
|
|
168
199
|
for (const { id, placetype } of candidates) {
|
|
169
|
-
if (TOP_PLACETYPES.has(placetype)) continue
|
|
170
|
-
const gj =
|
|
171
|
-
const
|
|
172
|
-
const hierarchy = (props?.["wof:hierarchy"] ?? null) as Array<Record<string, number>> | null
|
|
200
|
+
if (placetype && TOP_PLACETYPES.has(placetype)) continue
|
|
201
|
+
const gj = readWOFFeature(id, geojsonRoots)
|
|
202
|
+
const hierarchy = gj?.properties?.["wof:hierarchy"]
|
|
173
203
|
|
|
174
204
|
if (!hierarchy || !hierarchy.length) {
|
|
175
205
|
if (!gj) {
|
|
@@ -197,11 +227,20 @@ export function backfillAncestorsFromHierarchy(
|
|
|
197
227
|
}
|
|
198
228
|
}
|
|
199
229
|
|
|
230
|
+
let present = alreadyPresent.get(id)
|
|
231
|
+
|
|
232
|
+
if (!present) {
|
|
233
|
+
present = new Set()
|
|
234
|
+
alreadyPresent.set(id, present)
|
|
235
|
+
}
|
|
236
|
+
|
|
200
237
|
let added = 0
|
|
201
238
|
|
|
202
239
|
for (const [aid, pt] of seen) {
|
|
203
|
-
if (
|
|
240
|
+
if (present.has(aid)) continue
|
|
241
|
+
|
|
204
242
|
insert.run(id, aid, pt)
|
|
243
|
+
present.add(aid)
|
|
205
244
|
|
|
206
245
|
added++
|
|
207
246
|
}
|
package/build-candidate.ts
CHANGED
|
@@ -26,6 +26,12 @@
|
|
|
26
26
|
*
|
|
27
27
|
* Measured (2026-06-20, vs the 2.6 GB full-DB FTS): ~5 M rows; ~12 range fetches per 8-query
|
|
28
28
|
* session (the full DB needs 243); US locality 96.8% (region bbox), EU coord parity 88.6%.
|
|
29
|
+
*
|
|
30
|
+
* #28 adds one more denormalized field, `importance` — the toponym-fame prior that decides a BARE
|
|
31
|
+
* city name, joined in from a separate score source by name rather than by id (see
|
|
32
|
+
* `candidate-importance.ts`, which owns that join and explains why the id would be wrong). It is
|
|
33
|
+
* optional: without {@link BuildCandidateOptions.importance} the column is NULL on every row, which
|
|
34
|
+
* the consumer reads as unmeasured and ignores.
|
|
29
35
|
*/
|
|
30
36
|
|
|
31
37
|
import { existsSync, rmSync } from "node:fs"
|
|
@@ -34,6 +40,7 @@ import { DatabaseSync } from "node:sqlite"
|
|
|
34
40
|
import { DatabaseClient } from "@mailwoman/core/kysley/client"
|
|
35
41
|
|
|
36
42
|
import { createCandidateFTS } from "./candidate-fts.ts"
|
|
43
|
+
import { IMPORTANCE_JOIN_GATE_KM, loadImportanceIndex } from "./candidate-importance.ts"
|
|
37
44
|
import {
|
|
38
45
|
CANDIDATE_COLUMNS,
|
|
39
46
|
createCandidateStagingTables,
|
|
@@ -60,8 +67,23 @@ export interface BuildCandidateOptions {
|
|
|
60
67
|
* Optional postcode shards (`spr` rows with `placetype='postalcode'` + real coords, e.g. postalcode-us.db) — folded
|
|
61
68
|
* in as `postalcode` candidate rows so `findPlace(postalcode)` resolves a ZIP directly (the demo's primary postcode
|
|
62
69
|
* path; the postcode-*.bin anchor stays the fallback). Matches the slim wof-hot.db, which took one such postcode DB.
|
|
70
|
+
*
|
|
71
|
+
* Each shard's `names` table is folded in too (#1495) — that's where the GeoNames delivery-city names live
|
|
72
|
+
* ("Brooklyn" for 11201), and they were previously reachable only through FTS.
|
|
63
73
|
*/
|
|
64
74
|
postcodes?: string[]
|
|
75
|
+
/**
|
|
76
|
+
* Optional WOF admin database carrying a `place_importance` table — the source of the `importance` column (#28), the
|
|
77
|
+
* toponym-fame prior that decides the bare-city-name class. Joined by `(name_key, country, placetype)` + nearest
|
|
78
|
+
* centroid, NOT by id; see `candidate-importance.ts` for why the id join silently drops the foreign homonyms the
|
|
79
|
+
* prior exists to demote.
|
|
80
|
+
*
|
|
81
|
+
* Omit it and every row's `importance` is NULL — unmeasured, which is what the consumer's positive-evidence-only rule
|
|
82
|
+
* already treats as "do not participate", so the artifact is byte-identical to a pre-#28 build except for the empty
|
|
83
|
+
* column. That is the honest degradation and it is the DEFAULT: a caller with no score source must not get a
|
|
84
|
+
* population-derived stand-in written into a column that means fame.
|
|
85
|
+
*/
|
|
86
|
+
importance?: string
|
|
65
87
|
/**
|
|
66
88
|
* Optional progress callback for CLI / test introspection.
|
|
67
89
|
*/
|
|
@@ -75,6 +97,27 @@ export interface BuildCandidateResult {
|
|
|
75
97
|
aliases: number
|
|
76
98
|
abbrevs: number
|
|
77
99
|
postcodes: number
|
|
100
|
+
/**
|
|
101
|
+
* Delivery-city (and other `names`-table) aliases folded onto postcode rows — #1495. Zero here means the shards
|
|
102
|
+
* carried no alias names, NOT that the pass was skipped: a shard with no `names` table reports that separately
|
|
103
|
+
* through `onProgress`.
|
|
104
|
+
*/
|
|
105
|
+
postcodeAliases: number
|
|
106
|
+
/**
|
|
107
|
+
* Places that took an `importance` score from the join (#28).
|
|
108
|
+
*
|
|
109
|
+
* `undefined` and `0` mean different things. `undefined` is "the pass did not run" — no score source was given. A `0`
|
|
110
|
+
* would be the source matching NOTHING, which is a finding. Never collapse the two.
|
|
111
|
+
*/
|
|
112
|
+
importanceScored?: number
|
|
113
|
+
/**
|
|
114
|
+
* Places whose `(name_key, country, placetype)` matched a scored group but whose nearest scored centroid was outside
|
|
115
|
+
* {@link IMPORTANCE_JOIN_GATE_KM} — a different town wearing the same name, refused rather than scored.
|
|
116
|
+
*
|
|
117
|
+
* Worth watching across rebuilds: a jump here means the score source and the admin source have drifted apart, and the
|
|
118
|
+
* join is being asked to guess.
|
|
119
|
+
*/
|
|
120
|
+
importanceGated?: number
|
|
78
121
|
}
|
|
79
122
|
|
|
80
123
|
interface PlaceAttrs {
|
|
@@ -91,6 +134,12 @@ interface PlaceAttrs {
|
|
|
91
134
|
pop: number
|
|
92
135
|
neg: number
|
|
93
136
|
pkey: string
|
|
137
|
+
/**
|
|
138
|
+
* The place's toponym-fame score, or null when the score source has no measurement for it (#28). A property of the
|
|
139
|
+
* PLACE, so it rides {@link stageRow} onto the alias and abbrev rows too — that is how a bare `Moscow` reaches
|
|
140
|
+
* Москва's score through the alias row that carries the key.
|
|
141
|
+
*/
|
|
142
|
+
imp: number | null
|
|
94
143
|
}
|
|
95
144
|
|
|
96
145
|
export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<BuildCandidateResult> {
|
|
@@ -137,15 +186,55 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
137
186
|
return id
|
|
138
187
|
}
|
|
139
188
|
|
|
189
|
+
// --- importance source (#28): loaded BEFORE pass 1, which is the only pass that sees a place's
|
|
190
|
+
// name/country/placetype/centroid together. Absent → every row's `importance` stays NULL. ---
|
|
191
|
+
let importance: ReturnType<typeof loadImportanceIndex> | undefined
|
|
192
|
+
|
|
193
|
+
if (opts.importance) {
|
|
194
|
+
progress("importance", `loading place_importance from ${opts.importance}`)
|
|
195
|
+
importance = loadImportanceIndex(opts.importance)
|
|
196
|
+
|
|
197
|
+
progress(
|
|
198
|
+
"importance",
|
|
199
|
+
`${importance.stats.places.toLocaleString()} scored places in ${importance.stats.keys.toLocaleString()} (name, country, placetype) groups` +
|
|
200
|
+
(importance.stats.unkeyable ? `; ${importance.stats.unkeyable.toLocaleString()} unkeyable names skipped` : "")
|
|
201
|
+
)
|
|
202
|
+
} else {
|
|
203
|
+
// Never a silent column of nulls: a build without a score source produces one, and the reason has
|
|
204
|
+
// to be visible in the log rather than inferred from the artifact.
|
|
205
|
+
progress("importance", "no score source given — `importance` will be NULL on every row")
|
|
206
|
+
}
|
|
207
|
+
|
|
140
208
|
// --- region_id per place (its region-tier ancestor) for same-name disambiguation ---
|
|
141
209
|
progress("region", "loading region ancestry")
|
|
142
210
|
const regionOf = new Map<number, number>()
|
|
143
|
-
|
|
144
|
-
|
|
211
|
+
let multiRegion = 0
|
|
212
|
+
|
|
213
|
+
// A place can carry more than one region-tier ancestor — 59 do on the 2026-08-03 artifact (42
|
|
214
|
+
// localities, mostly Chinese places on an ambiguous boundary). One `region_id` column holds one of
|
|
215
|
+
// them, and which one it ought to be is a real question this is not the place to answer.
|
|
216
|
+
//
|
|
217
|
+
// MIN is arbitrary but STABLE: an unordered pick lets the stamp for those places differ between two
|
|
218
|
+
// builds of the same source. The count is logged because the number is expected to grow, and should
|
|
219
|
+
// be visible rather than inferred.
|
|
220
|
+
for (const r of src
|
|
221
|
+
.prepare(
|
|
222
|
+
"SELECT id, MIN(ancestor_id) AS ancestor_id, COUNT(DISTINCT ancestor_id) AS n" +
|
|
223
|
+
" FROM ancestors WHERE ancestor_placetype='region' GROUP BY id"
|
|
224
|
+
)
|
|
225
|
+
.iterate()) {
|
|
145
226
|
regionOf.set(Number(r.id), Number(r.ancestor_id))
|
|
227
|
+
|
|
228
|
+
if (Number(r.n) > 1) {
|
|
229
|
+
multiRegion++
|
|
230
|
+
}
|
|
146
231
|
}
|
|
147
232
|
|
|
148
|
-
progress(
|
|
233
|
+
progress(
|
|
234
|
+
"region",
|
|
235
|
+
`${regionOf.size.toLocaleString()} places carry a region` +
|
|
236
|
+
(multiRegion ? ` (${multiRegion.toLocaleString()} carry more than one; stamped with the lowest id)` : "")
|
|
237
|
+
)
|
|
149
238
|
|
|
150
239
|
// The hot path — millions of clustered rows. Kept a single positional prepared statement (the fastest
|
|
151
240
|
// node:sqlite insert) rather than a per-row query builder. Placeholders come from CANDIDATE_COLUMNS so
|
|
@@ -176,14 +265,16 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
176
265
|
const neg = -Math.log10(pop + 1)
|
|
177
266
|
const name = String(r.name ?? "")
|
|
178
267
|
const pkey = normalizeLocalityForKey(name)
|
|
268
|
+
const lat = r.lat as number
|
|
269
|
+
const lon = r.lon as number
|
|
179
270
|
|
|
180
271
|
const a: PlaceAttrs = {
|
|
181
272
|
cid,
|
|
182
273
|
rid,
|
|
183
274
|
ptid,
|
|
184
275
|
name,
|
|
185
|
-
lat
|
|
186
|
-
lon
|
|
276
|
+
lat,
|
|
277
|
+
lon,
|
|
187
278
|
mnLat: r.mnlat as number,
|
|
188
279
|
mnLon: r.mnlon as number,
|
|
189
280
|
mxLat: r.mxlat as number,
|
|
@@ -191,12 +282,30 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
191
282
|
pop,
|
|
192
283
|
neg,
|
|
193
284
|
pkey,
|
|
285
|
+
imp: importance?.find(name, r.country as string | null, r.placetype as string | null, lat, lon) ?? null,
|
|
194
286
|
}
|
|
195
287
|
|
|
196
288
|
attrs.set(sid, a)
|
|
197
289
|
|
|
198
290
|
if (pkey) {
|
|
199
|
-
insStage.run(
|
|
291
|
+
insStage.run(
|
|
292
|
+
pkey,
|
|
293
|
+
cid,
|
|
294
|
+
rid,
|
|
295
|
+
ptid,
|
|
296
|
+
neg,
|
|
297
|
+
sid,
|
|
298
|
+
name,
|
|
299
|
+
a.lat,
|
|
300
|
+
a.lon,
|
|
301
|
+
a.mnLat,
|
|
302
|
+
a.mnLon,
|
|
303
|
+
a.mxLat,
|
|
304
|
+
a.mxLon,
|
|
305
|
+
pop,
|
|
306
|
+
1,
|
|
307
|
+
a.imp
|
|
308
|
+
)
|
|
200
309
|
|
|
201
310
|
nPrim++
|
|
202
311
|
}
|
|
@@ -205,6 +314,14 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
205
314
|
out.exec("COMMIT")
|
|
206
315
|
progress("primaries", `${nPrim.toLocaleString()} primaries; ${attrs.size.toLocaleString()} places`)
|
|
207
316
|
|
|
317
|
+
if (importance) {
|
|
318
|
+
progress(
|
|
319
|
+
"importance",
|
|
320
|
+
`${importance.matched.toLocaleString()} places scored; ` +
|
|
321
|
+
`${importance.gated.toLocaleString()} refused (nearest same-name place > ${IMPORTANCE_JOIN_GATE_KM} km away)`
|
|
322
|
+
)
|
|
323
|
+
}
|
|
324
|
+
|
|
208
325
|
const stageRow = (k: string, a: PlaceAttrs, sid: number, isPrimary: number): void => {
|
|
209
326
|
insStage.run(
|
|
210
327
|
k,
|
|
@@ -221,7 +338,8 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
221
338
|
a.mxLat,
|
|
222
339
|
a.mxLon,
|
|
223
340
|
a.pop,
|
|
224
|
-
isPrimary
|
|
341
|
+
isPrimary,
|
|
342
|
+
a.imp
|
|
225
343
|
)
|
|
226
344
|
}
|
|
227
345
|
|
|
@@ -270,13 +388,25 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
270
388
|
out.exec("COMMIT")
|
|
271
389
|
progress("abbrevs", `${nAbbr.toLocaleString()} abbrevs`)
|
|
272
390
|
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
391
|
+
/**
|
|
392
|
+
* Pass 4 — fold ONE postcode shard (`spr` rows with `placetype='postalcode'`) in, then pass 4b: the delivery-city
|
|
393
|
+
* aliases hanging off the same shard's `names` table.
|
|
394
|
+
*
|
|
395
|
+
* Extracted rather than inlined because the shard loop is self-contained — it shares only the staging statement and
|
|
396
|
+
* the code dictionaries with the passes above, and nothing after it reads anything it produces except the two
|
|
397
|
+
* counters it returns.
|
|
398
|
+
*/
|
|
399
|
+
const foldPostcodeShard = (pcDB: string): { primaries: number; aliases: number } => {
|
|
277
400
|
progress("postcodes", `reading ${pcDB}`)
|
|
401
|
+
|
|
278
402
|
const pc = new DatabaseSync(pcDB, { readOnly: true })
|
|
279
403
|
const pcPtid = ptID("postalcode")
|
|
404
|
+
// Per-shard, not the admin `attrs` map: pass 1 only ever sees the admin DB, so the alias pass
|
|
405
|
+
// below has nothing to join against unless this primary loop records what it staged.
|
|
406
|
+
const pcAttrs = new Map<number, PlaceAttrs>()
|
|
407
|
+
let primaries = 0
|
|
408
|
+
let aliases = 0
|
|
409
|
+
|
|
280
410
|
out.exec("BEGIN")
|
|
281
411
|
|
|
282
412
|
for (const r of pc
|
|
@@ -290,38 +420,100 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
290
420
|
const key = normalizeLocalityForKey(name)
|
|
291
421
|
|
|
292
422
|
if (!key) continue
|
|
423
|
+
|
|
293
424
|
const lat = r.latitude as number
|
|
294
425
|
const lon = r.longitude as number
|
|
295
426
|
|
|
296
427
|
// region_id 0 (a postcode is unique by name+country — no same-name disambiguation); neg_rank 0
|
|
297
428
|
// (no population). bbox = the postcode's own min/max (falls back to the centroid point).
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
pcPtid,
|
|
303
|
-
0,
|
|
304
|
-
Number(r.id),
|
|
429
|
+
const a: PlaceAttrs = {
|
|
430
|
+
cid: ccID(r.country as string | null),
|
|
431
|
+
rid: 0,
|
|
432
|
+
ptid: pcPtid,
|
|
305
433
|
name,
|
|
306
434
|
lat,
|
|
307
435
|
lon,
|
|
308
|
-
(r.mnlat as number) || lat,
|
|
309
|
-
(r.mnlon as number) || lon,
|
|
310
|
-
(r.mxlat as number) || lat,
|
|
311
|
-
(r.mxlon as number) || lon,
|
|
312
|
-
0,
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
436
|
+
mnLat: (r.mnlat as number) || lat,
|
|
437
|
+
mnLon: (r.mnlon as number) || lon,
|
|
438
|
+
mxLat: (r.mxlat as number) || lat,
|
|
439
|
+
mxLon: (r.mxlon as number) || lon,
|
|
440
|
+
pop: 0,
|
|
441
|
+
neg: 0,
|
|
442
|
+
pkey: key,
|
|
443
|
+
// A postcode has no toponym fame — nobody writes an encyclopedia article about SW1A 2AA — and
|
|
444
|
+
// the score source carries no `postalcode` rows to join against anyway. NULL is the truthful
|
|
445
|
+
// value: unmeasured, so the ranking key leaves postcode rows exactly where they were.
|
|
446
|
+
imp: null,
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
pcAttrs.set(Number(r.id), a)
|
|
450
|
+
stageRow(key, a, Number(r.id), 1)
|
|
451
|
+
|
|
452
|
+
primaries++
|
|
317
453
|
}
|
|
318
454
|
|
|
319
455
|
out.exec("COMMIT")
|
|
456
|
+
|
|
457
|
+
// --- pass 4b: postcode ALIAS names (#1495) ---
|
|
458
|
+
//
|
|
459
|
+
// The delivery-city names GeoNames supplies for a ZIP ("Brooklyn" for 11201) are written into
|
|
460
|
+
// the shard's `names` table by `postcode/centroid-fills.ts`'s `geonamesNameFill`. Everything
|
|
461
|
+
// downstream of `names` picked them up EXCEPT this build: `fts.ts` unions `spr.name` with every
|
|
462
|
+
// `names` row into `place_search.alt_names`, so the FTS backend resolved "Brooklyn" → 11201
|
|
463
|
+
// while the candidate backend — whose every row IS an exact-tier row — had no key for it at
|
|
464
|
+
// all. Pass 2 does the equivalent fold for admin places, but reads the ADMIN `place_search`,
|
|
465
|
+
// and `attrs` holds admin ids only, so a postcode shard could never reach it.
|
|
466
|
+
//
|
|
467
|
+
// Same discipline as pass 2: `is_primary = 0` (so `rankByPrimaryPreference` treats it as an
|
|
468
|
+
// alias, not a canonical postcode name), the row stays denormalized onto the POSTCODE's own
|
|
469
|
+
// spr_id/coords/bbox, and the display `name` stays the postcode — resolving "brooklyn" answers
|
|
470
|
+
// with place 11201, it does not rename the place to its delivery city.
|
|
471
|
+
const hasNames = pc.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='names'").get() !== undefined
|
|
472
|
+
|
|
473
|
+
if (hasNames) {
|
|
474
|
+
out.exec("BEGIN")
|
|
475
|
+
|
|
476
|
+
for (const r of pc.prepare("SELECT id, name FROM names").iterate()) {
|
|
477
|
+
const a = pcAttrs.get(Number(r.id))
|
|
478
|
+
|
|
479
|
+
if (!a) continue
|
|
480
|
+
|
|
481
|
+
const k = normalizeLocalityForKey(String(r.name ?? ""))
|
|
482
|
+
|
|
483
|
+
// The postcode's own key is already staged as the primary; `INSERT OR IGNORE` at
|
|
484
|
+
// materialization dedupes repeats, so this only skips the obvious self-alias.
|
|
485
|
+
if (!k || k === a.pkey) continue
|
|
486
|
+
|
|
487
|
+
stageRow(k, a, Number(r.id), 0)
|
|
488
|
+
|
|
489
|
+
aliases++
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
out.exec("COMMIT")
|
|
493
|
+
} else {
|
|
494
|
+
// Never a silent zero: real shards come from `createUnifiedSchema`, which always creates
|
|
495
|
+
// `names`. A shard without it has no alias surface to lose, but say so rather than reporting
|
|
496
|
+
// "0 aliases" from a table that was never read.
|
|
497
|
+
progress("postcode-aliases", `${pcDB} has no \`names\` table — no delivery-city aliases to fold`)
|
|
498
|
+
}
|
|
499
|
+
|
|
320
500
|
pc.close()
|
|
501
|
+
|
|
502
|
+
return { primaries, aliases }
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
let nPostcode = 0
|
|
506
|
+
let nPostcodeAlias = 0
|
|
507
|
+
|
|
508
|
+
for (const pcDB of opts.postcodes ?? []) {
|
|
509
|
+
const folded = foldPostcodeShard(pcDB)
|
|
510
|
+
|
|
511
|
+
nPostcode += folded.primaries
|
|
512
|
+
nPostcodeAlias += folded.aliases
|
|
321
513
|
}
|
|
322
514
|
|
|
323
515
|
if (nPostcode > 0) {
|
|
324
|
-
progress("postcodes", `${nPostcode.toLocaleString()} postcodes`)
|
|
516
|
+
progress("postcodes", `${nPostcode.toLocaleString()} postcodes; ${nPostcodeAlias.toLocaleString()} aliases`)
|
|
325
517
|
}
|
|
326
518
|
|
|
327
519
|
// --- code dictionaries: typed batch inserts via kdb (a few hundred rows — Kysely is clean here) ---
|
|
@@ -370,5 +562,14 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
370
562
|
await kdb.destroy()
|
|
371
563
|
|
|
372
564
|
// closes the underlying `out` connection
|
|
373
|
-
return {
|
|
565
|
+
return {
|
|
566
|
+
rows,
|
|
567
|
+
places: attrs.size,
|
|
568
|
+
primaries: nPrim,
|
|
569
|
+
aliases: nAlias,
|
|
570
|
+
abbrevs: nAbbr,
|
|
571
|
+
postcodes: nPostcode,
|
|
572
|
+
postcodeAliases: nPostcodeAlias,
|
|
573
|
+
...(importance ? { importanceScored: importance.matched, importanceGated: importance.gated } : {}),
|
|
574
|
+
}
|
|
374
575
|
}
|