@mailwoman/resolver-wof-sqlite 8.6.0 → 9.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/ancestry-backfill.ts +103 -64
  2. package/build-candidate.ts +230 -29
  3. package/candidate-importance.ts +227 -0
  4. package/candidate-lookup.ts +159 -9
  5. package/candidate-schema.ts +29 -0
  6. package/coverage-manifest-schema.ts +1 -1
  7. package/fst-autocomplete.ts +22 -11
  8. package/fst-builder.ts +76 -50
  9. package/fst-deserialize-web.ts +46 -8
  10. package/fst-freshness.ts +333 -0
  11. package/fst-serialize.ts +78 -13
  12. package/fst-types.ts +58 -1
  13. package/fts.ts +12 -0
  14. package/geo.ts +16 -41
  15. package/geonames-aliases.ts +105 -37
  16. package/geonames-postal.ts +65 -21
  17. package/index.ts +16 -1
  18. package/interpolation.ts +2 -1
  19. package/lookup.ts +44 -5
  20. package/out/ancestry-backfill.d.ts +37 -18
  21. package/out/ancestry-backfill.d.ts.map +1 -1
  22. package/out/ancestry-backfill.js +82 -52
  23. package/out/ancestry-backfill.js.map +1 -1
  24. package/out/build-candidate.d.ts +42 -0
  25. package/out/build-candidate.d.ts.map +1 -1
  26. package/out/build-candidate.js +141 -13
  27. package/out/build-candidate.js.map +1 -1
  28. package/out/candidate-importance.d.ts +132 -0
  29. package/out/candidate-importance.d.ts.map +1 -0
  30. package/out/candidate-importance.js +174 -0
  31. package/out/candidate-importance.js.map +1 -0
  32. package/out/candidate-lookup.d.ts +9 -1
  33. package/out/candidate-lookup.d.ts.map +1 -1
  34. package/out/candidate-lookup.js +127 -8
  35. package/out/candidate-lookup.js.map +1 -1
  36. package/out/candidate-schema.d.ts +25 -1
  37. package/out/candidate-schema.d.ts.map +1 -1
  38. package/out/candidate-schema.js +5 -0
  39. package/out/candidate-schema.js.map +1 -1
  40. package/out/coverage-manifest-schema.d.ts +1 -1
  41. package/out/coverage-manifest-schema.js +1 -1
  42. package/out/fst-autocomplete.d.ts +13 -3
  43. package/out/fst-autocomplete.d.ts.map +1 -1
  44. package/out/fst-autocomplete.js +10 -9
  45. package/out/fst-autocomplete.js.map +1 -1
  46. package/out/fst-builder.d.ts.map +1 -1
  47. package/out/fst-builder.js +64 -40
  48. package/out/fst-builder.js.map +1 -1
  49. package/out/fst-deserialize-web.d.ts.map +1 -1
  50. package/out/fst-deserialize-web.js +38 -8
  51. package/out/fst-deserialize-web.js.map +1 -1
  52. package/out/fst-freshness.d.ts +138 -0
  53. package/out/fst-freshness.d.ts.map +1 -0
  54. package/out/fst-freshness.js +238 -0
  55. package/out/fst-freshness.js.map +1 -0
  56. package/out/fst-serialize.d.ts +19 -3
  57. package/out/fst-serialize.d.ts.map +1 -1
  58. package/out/fst-serialize.js +68 -13
  59. package/out/fst-serialize.js.map +1 -1
  60. package/out/fst-types.d.ts +61 -1
  61. package/out/fst-types.d.ts.map +1 -1
  62. package/out/fts.d.ts +11 -0
  63. package/out/fts.d.ts.map +1 -1
  64. package/out/fts.js +11 -0
  65. package/out/fts.js.map +1 -1
  66. package/out/geo.d.ts +10 -13
  67. package/out/geo.d.ts.map +1 -1
  68. package/out/geo.js +15 -35
  69. package/out/geo.js.map +1 -1
  70. package/out/geonames-aliases.d.ts +23 -1
  71. package/out/geonames-aliases.d.ts.map +1 -1
  72. package/out/geonames-aliases.js +88 -33
  73. package/out/geonames-aliases.js.map +1 -1
  74. package/out/geonames-postal.d.ts +22 -1
  75. package/out/geonames-postal.d.ts.map +1 -1
  76. package/out/geonames-postal.js +50 -16
  77. package/out/geonames-postal.js.map +1 -1
  78. package/out/index.d.ts +3 -1
  79. package/out/index.d.ts.map +1 -1
  80. package/out/index.js +3 -1
  81. package/out/index.js.map +1 -1
  82. package/out/interpolation.d.ts.map +1 -1
  83. package/out/interpolation.js +2 -1
  84. package/out/interpolation.js.map +1 -1
  85. package/out/lookup.d.ts.map +1 -1
  86. package/out/lookup.js +37 -5
  87. package/out/lookup.js.map +1 -1
  88. package/out/place-importance-schema.d.ts +189 -0
  89. package/out/place-importance-schema.d.ts.map +1 -0
  90. package/out/place-importance-schema.js +242 -0
  91. package/out/place-importance-schema.js.map +1 -0
  92. package/out/poi-lookup.d.ts +1 -1
  93. package/out/poi-lookup.js +3 -3
  94. package/out/reverse.d.ts.map +1 -1
  95. package/out/reverse.js +3 -9
  96. package/out/reverse.js.map +1 -1
  97. package/out/sqlite-convention-source.d.ts.map +1 -1
  98. package/out/sqlite-convention-source.js +4 -3
  99. package/out/sqlite-convention-source.js.map +1 -1
  100. package/out/sqlite-utils.d.ts +12 -0
  101. package/out/sqlite-utils.d.ts.map +1 -1
  102. package/out/sqlite-utils.js +20 -0
  103. package/out/sqlite-utils.js.map +1 -1
  104. package/out/street-morphology-fst-builder.d.ts.map +1 -1
  105. package/out/street-morphology-fst-builder.js +7 -5
  106. package/out/street-morphology-fst-builder.js.map +1 -1
  107. package/out/street-normalize.d.ts +2 -1
  108. package/out/street-normalize.d.ts.map +1 -1
  109. package/out/street-normalize.js +2 -1
  110. package/out/street-normalize.js.map +1 -1
  111. package/out/types.d.ts +39 -0
  112. package/out/types.d.ts.map +1 -1
  113. package/package.json +24 -6
  114. package/place-importance-schema.ts +353 -0
  115. package/poi-lookup.ts +3 -3
  116. package/reverse.ts +4 -9
  117. package/sqlite-convention-source.ts +5 -3
  118. package/sqlite-utils.ts +21 -0
  119. package/street-morphology-fst-builder.ts +8 -5
  120. package/street-normalize.ts +3 -2
  121. package/types.ts +39 -0
@@ -3,9 +3,8 @@
3
3
  * @license AGPL-3.0
4
4
  * @author Teffen Ellis, et al.
5
5
  *
6
- * Repair the only-self ancestry that {@link populateAncestors} (the parent_id closure in
7
- * unified-schema.ts) leaves for places whose `wof:parent_id` is the WOF `-4` "ambiguous /
8
- * multi-parent" sentinel.
6
+ * Repair the truncated ancestry that {@link populateAncestors} (the parent_id closure in
7
+ * unified-schema.ts) leaves wherever the closure dead-ends before reaching the top.
9
8
  *
10
9
  * Root cause (#440 / #832): a place that straddles multiple parents — New York City spans five
11
10
  * counties (its boroughs), London 30+ — carries `wof:parent_id = -4`, so the parent_id closure
@@ -14,22 +13,47 @@
14
13
  * a correctly-parented namesake ("New York Mills", pop 3,190) wins over NYC's 8.8M. The same defect
15
14
  * orphans London, Singapore, and ~2,850 other localities — the most demo-visible queries.
16
15
  *
16
+ * **The dead end is inherited by children (#1445).** Repairing the `-4` place itself does not repair
17
+ * anything BELOW it: the closure walks Brooklyn → New York and stops, because New York's own
18
+ * `parent_id` is `-4`. Brooklyn-the-borough (pop 2.5M) therefore carried exactly two ancestor rows —
19
+ * itself and New York — with no county, region or country, and the only US locality-tier place named
20
+ * "Brooklyn" that survived a New-York-State descendant filter was Pillar Point, a Jefferson County
21
+ * hamlet 411 km away carrying "Brooklyn" as an alternate name. All five NYC boroughs, every London
22
+ * borough and Hyderabad's zones were in the same state.
23
+ *
24
+ * So the candidate test is NOT "has only a self ancestor" — that misses every child of a repaired
25
+ * place, which by construction has two. It is **"has no `country`-tier ancestor"**. Country is the
26
+ * universal terminal for every non-{@link TOP_PLACETYPES} placetype, so its absence is exactly the
27
+ * signal that the chain dead-ended somewhere, at whatever depth. On a wide-coverage build this
28
+ * selects ~24k places against 2.55M rows.
29
+ *
30
+ * A place whose `wof:hierarchy` genuinely stops short is NOT a candidate and needs no repair: the
31
+ * source is the authority on what a place should have. American Samoa's localities, for instance,
32
+ * have `{country_id, locality_id}` and no region in WOF itself — the artifact matching that is
33
+ * correct, not truncated.
34
+ *
17
35
  * The authoritative hierarchy IS in the source geojson: `wof:hierarchy` is an array of branches,
18
36
  * each a `<placetype>_id` → id map (region_id, county_id, country_id, …), fully populated even when
19
- * parent_id is -4. This reads it for every only-self place and inserts the missing ancestor rows
20
- * (one per distinct ancestor across branches).
37
+ * parent_id is -4. This reads it for every candidate and inserts the missing ancestor rows (one per
38
+ * distinct ancestor across branches).
21
39
  *
22
40
  * MUST run AFTER populateAncestors and BEFORE the build freezes (VACUUM INTO), so the rows land in
23
41
  * the shipped artifact — `scripts/build-unified-wof.ts` Phase 3 calls it inline. The standalone
24
42
  * `scripts/backfill-ancestors-from-hierarchy.ts` is a thin CLI over the same function for ad-hoc
25
- * repair of an already-built DB. Idempotent: only touches places with <= 1 ancestor row (self), and
26
- * inserts each (id, ancestor_id) at most once.
43
+ * repair of an already-built DB. Idempotent by the per-pair existence check, not by the candidate
44
+ * test: each (id, ancestor_id) is inserted at most once, so a second run over the same DB adds
45
+ * nothing.
27
46
  */
28
47
 
29
- import { existsSync, readdirSync, readFileSync } from "node:fs"
30
- import { join } from "node:path"
48
+ import { readdirSync } from "node:fs"
31
49
  import type { DatabaseSync } from "node:sqlite"
32
50
 
51
+ import { DatabaseClient } from "@mailwoman/core/kysley/client"
52
+ import { readWOFFeature } from "@mailwoman/core/resources/whosonfirst"
53
+ import { join } from "path-ts"
54
+
55
+ import type { WOFDatabase } from "./schema.ts"
56
+
33
57
  /**
34
58
  * Genuinely top-level placetypes — they never have (or need) an ancestor, so skip them.
35
59
  */
@@ -45,8 +69,8 @@ export interface AncestryBackfillResult {
45
69
  */
46
70
  rowsAdded: number
47
71
  /**
48
- * Only-self candidates whose source geojson could not be found (non-WOF backfilled places, or repos not present
49
- * locally) — skipped, not an error.
72
+ * Candidates whose source geojson could not be found (non-WOF backfilled places, or repos not present locally) —
73
+ * skipped, not an error.
50
74
  */
51
75
  noGeojson: number
52
76
  }
@@ -89,34 +113,6 @@ export function discoverAdminDataRoots(reposRoot: string): string[] {
89
113
  return roots
90
114
  }
91
115
 
92
- /**
93
- * WOF geojson lives sharded: an id resolves to `<3-char chunks>/<id>.geojson` under each data root.
94
- */
95
- function geojsonForID(id: number, roots: readonly string[]): Record<string, unknown> | null {
96
- const s = String(id)
97
- const chunks: string[] = []
98
-
99
- for (let i = 0; i < s.length; i += 3) {
100
- chunks.push(s.slice(i, i + 3))
101
- }
102
-
103
- const rel = join(chunks.join("/"), `${s}.geojson`)
104
-
105
- for (const root of roots) {
106
- const fp = join(root, rel)
107
-
108
- if (existsSync(fp)) {
109
- try {
110
- return JSON.parse(readFileSync(fp, "utf8")) as Record<string, unknown>
111
- } catch {
112
- return null
113
- }
114
- }
115
- }
116
-
117
- return null
118
- }
119
-
120
116
  // `<placetype>_id` key → ancestor placetype. WOF hierarchy keys are e.g. region_id, county_id. Self
121
117
  // is filtered downstream by the `aid === id` check, so we do NOT special-case locality here: for a
122
118
  // locality candidate `locality_id` IS self (dropped by aid===id), but for a neighbourhood candidate
@@ -128,48 +124,82 @@ function placetypeFromKey(key: string): string | null {
128
124
  }
129
125
 
130
126
  /**
131
- * Insert missing ancestor rows for only-self places by reading `wof:hierarchy` from their source geojson under
132
- * `geojsonRoots` (see {@link discoverAdminDataRoots}). Runs inside a single transaction; caller owns connection
133
- * lifecycle (open, WAL checkpoint, close).
127
+ * Insert missing ancestor rows for every place whose ancestry chain dead-ended before reaching a country, by reading
128
+ * `wof:hierarchy` from its source geojson under `geojsonRoots` (see {@link discoverAdminDataRoots}). Runs inside a
129
+ * single transaction; caller owns connection lifecycle (open, WAL checkpoint, close).
134
130
  *
135
- * `opts.maxId` bounds the candidate scan to ids BELOW it — pass the synthetic-id base (`OVERTURE_ID_BASE`, 8e12) so the
131
+ * `opts.maxID` bounds the candidate scan to ids BELOW it — pass the synthetic-id base (`OVERTURE_ID_BASE`, 8e12) so the
136
132
  * backfill considers only real WOF places. Overture/GeoNames rows carry synthetic ids and have NO `wof:hierarchy`
137
- * geojson, so probing them is pure waste: on a wide-coverage DB the only-self set is millions of Overture/GeoNames leaf
138
- * localities, and the per-candidate geojson probe across every repo root turns a seconds-long WOF-only pass into a
133
+ * geojson, so probing them is pure waste: on a wide-coverage DB the country-less set is millions of Overture/GeoNames
134
+ * leaf localities, and the per-candidate geojson probe across every repo root turns a seconds-long WOF-only pass into a
139
135
  * ~40-minute one (their ancestry comes from the parent_id closure, not this backfill). Correctness-preserving — the
140
- * skipped rows would have `noGeojson`-skipped anyway. Omit `maxId` (default) for the legacy WOF-only DBs.
136
+ * skipped rows would have `noGeojson`-skipped anyway. Omit `maxID` (default) for the legacy WOF-only DBs.
141
137
  */
142
- export function backfillAncestorsFromHierarchy(
138
+ export async function backfillAncestorsFromHierarchy(
143
139
  db: DatabaseSync,
144
140
  geojsonRoots: readonly string[],
145
- opts: { maxId?: number } = {}
146
- ): AncestryBackfillResult {
147
- const maxId = opts.maxId ?? Number.MAX_SAFE_INTEGER
148
-
149
- // `s.id < ?` first lets SQLite prune by the PK index before the correlated only-self subquery runs at all.
150
- const candidates = db
151
- .prepare(
152
- `SELECT s.id AS id, s.placetype AS placetype FROM spr s
153
- WHERE s.id < ? AND (SELECT count(*) FROM ancestors a WHERE a.id = s.id) <= 1`
141
+ opts: { maxID?: number } = {}
142
+ ): Promise<AncestryBackfillResult> {
143
+ const maxID = opts.maxID ?? Number.MAX_SAFE_INTEGER
144
+ const kdb = new DatabaseClient<WOFDatabase>({ database: db })
145
+
146
+ // "No country-tier ancestor" is the dead-end signal at any depth — see the module docstring. The
147
+ // earlier "<= 1 ancestor row" test only caught the dead end's origin, never the children that
148
+ // inherit it (a child of a repaired -4 place has two rows: itself and that parent) (#1445).
149
+ // The id bound is stated first so SQLite prunes by the PK index before the NOT EXISTS runs at all.
150
+ const candidateBase = kdb
151
+ .selectFrom("spr")
152
+ .where("id", "<", maxID)
153
+ .where((eb) =>
154
+ eb.not(
155
+ eb.exists(
156
+ eb
157
+ .selectFrom("ancestors as a")
158
+ .select("a.id")
159
+ .whereRef("a.id", "=", "spr.id")
160
+ .where("a.ancestor_placetype", "=", "country")
161
+ )
162
+ )
154
163
  )
155
- .all(maxId) as Array<{ id: number; placetype: string }>
164
+
165
+ const candidates = await candidateBase.select(["id", "placetype"]).execute()
166
+
167
+ // Every candidate's existing ancestors in ONE query rather than an indexed read each. The widened
168
+ // candidate test made that per-candidate read the dominant cost of the pass, and the set is bounded:
169
+ // a candidate reaching this point has a handful of rows at most. The candidate set rides in as the
170
+ // SAME predicate re-issued as a subquery, never a materialized `IN (?, ?, …)` list — node:sqlite
171
+ // caps a statement at 32,766 bound variables and the wide-coverage build has 67,521 candidates
172
+ // (measured 2026-08-04).
173
+ const alreadyPresent = new Map<number, Set<number>>()
174
+
175
+ for (const row of await kdb
176
+ .selectFrom("ancestors")
177
+ .select(["id", "ancestor_id"])
178
+ .where("id", "in", candidateBase.select("spr.id"))
179
+ .execute()) {
180
+ let set = alreadyPresent.get(row.id)
181
+
182
+ if (!set) {
183
+ set = new Set()
184
+ alreadyPresent.set(row.id, set)
185
+ }
186
+
187
+ set.add(Number(row.ancestor_id))
188
+ }
156
189
 
157
190
  const insert = db.prepare(
158
191
  "INSERT INTO ancestors (id, ancestor_id, ancestor_placetype, lastmodified) VALUES (?, ?, ?, 0)"
159
192
  )
160
193
 
161
- const hasRow = db.prepare("SELECT 1 FROM ancestors WHERE id = ? AND ancestor_id = ? LIMIT 1")
162
-
163
194
  let placesFixed = 0
164
195
  let rowsAdded = 0
165
196
  let noGeojson = 0
166
197
  db.exec("BEGIN")
167
198
 
168
199
  for (const { id, placetype } of candidates) {
169
- if (TOP_PLACETYPES.has(placetype)) continue
170
- const gj = geojsonForID(id, geojsonRoots)
171
- const props = (gj?.["properties"] ?? null) as Record<string, unknown> | null
172
- const hierarchy = (props?.["wof:hierarchy"] ?? null) as Array<Record<string, number>> | null
200
+ if (placetype && TOP_PLACETYPES.has(placetype)) continue
201
+ const gj = readWOFFeature(id, geojsonRoots)
202
+ const hierarchy = gj?.properties?.["wof:hierarchy"]
173
203
 
174
204
  if (!hierarchy || !hierarchy.length) {
175
205
  if (!gj) {
@@ -197,11 +227,20 @@ export function backfillAncestorsFromHierarchy(
197
227
  }
198
228
  }
199
229
 
230
+ let present = alreadyPresent.get(id)
231
+
232
+ if (!present) {
233
+ present = new Set()
234
+ alreadyPresent.set(id, present)
235
+ }
236
+
200
237
  let added = 0
201
238
 
202
239
  for (const [aid, pt] of seen) {
203
- if (hasRow.get(id, aid)) continue
240
+ if (present.has(aid)) continue
241
+
204
242
  insert.run(id, aid, pt)
243
+ present.add(aid)
205
244
 
206
245
  added++
207
246
  }
@@ -26,6 +26,12 @@
26
26
  *
27
27
  * Measured (2026-06-20, vs the 2.6 GB full-DB FTS): ~5 M rows; ~12 range fetches per 8-query
28
28
  * session (the full DB needs 243); US locality 96.8% (region bbox), EU coord parity 88.6%.
29
+ *
30
+ * #28 adds one more denormalized field, `importance` — the toponym-fame prior that decides a BARE
31
+ * city name, joined in from a separate score source by name rather than by id (see
32
+ * `candidate-importance.ts`, which owns that join and explains why the id would be wrong). It is
33
+ * optional: without {@link BuildCandidateOptions.importance} the column is NULL on every row, which
34
+ * the consumer reads as unmeasured and ignores.
29
35
  */
30
36
 
31
37
  import { existsSync, rmSync } from "node:fs"
@@ -34,6 +40,7 @@ import { DatabaseSync } from "node:sqlite"
34
40
  import { DatabaseClient } from "@mailwoman/core/kysley/client"
35
41
 
36
42
  import { createCandidateFTS } from "./candidate-fts.ts"
43
+ import { IMPORTANCE_JOIN_GATE_KM, loadImportanceIndex } from "./candidate-importance.ts"
37
44
  import {
38
45
  CANDIDATE_COLUMNS,
39
46
  createCandidateStagingTables,
@@ -60,8 +67,23 @@ export interface BuildCandidateOptions {
60
67
  * Optional postcode shards (`spr` rows with `placetype='postalcode'` + real coords, e.g. postalcode-us.db) — folded
61
68
  * in as `postalcode` candidate rows so `findPlace(postalcode)` resolves a ZIP directly (the demo's primary postcode
62
69
  * path; the postcode-*.bin anchor stays the fallback). Matches the slim wof-hot.db, which took one such postcode DB.
70
+ *
71
+ * Each shard's `names` table is folded in too (#1495) — that's where the GeoNames delivery-city names live
72
+ * ("Brooklyn" for 11201), and they were previously reachable only through FTS.
63
73
  */
64
74
  postcodes?: string[]
75
+ /**
76
+ * Optional WOF admin database carrying a `place_importance` table — the source of the `importance` column (#28), the
77
+ * toponym-fame prior that decides the bare-city-name class. Joined by `(name_key, country, placetype)` + nearest
78
+ * centroid, NOT by id; see `candidate-importance.ts` for why the id join silently drops the foreign homonyms the
79
+ * prior exists to demote.
80
+ *
81
+ * Omit it and every row's `importance` is NULL — unmeasured, which is what the consumer's positive-evidence-only rule
82
+ * already treats as "do not participate", so the artifact is byte-identical to a pre-#28 build except for the empty
83
+ * column. That is the honest degradation and it is the DEFAULT: a caller with no score source must not get a
84
+ * population-derived stand-in written into a column that means fame.
85
+ */
86
+ importance?: string
65
87
  /**
66
88
  * Optional progress callback for CLI / test introspection.
67
89
  */
@@ -75,6 +97,27 @@ export interface BuildCandidateResult {
75
97
  aliases: number
76
98
  abbrevs: number
77
99
  postcodes: number
100
+ /**
101
+ * Delivery-city (and other `names`-table) aliases folded onto postcode rows — #1495. Zero here means the shards
102
+ * carried no alias names, NOT that the pass was skipped: a shard with no `names` table reports that separately
103
+ * through `onProgress`.
104
+ */
105
+ postcodeAliases: number
106
+ /**
107
+ * Places that took an `importance` score from the join (#28).
108
+ *
109
+ * `undefined` and `0` mean different things. `undefined` is "the pass did not run" — no score source was given. A `0`
110
+ * would be the source matching NOTHING, which is a finding. Never collapse the two.
111
+ */
112
+ importanceScored?: number
113
+ /**
114
+ * Places whose `(name_key, country, placetype)` matched a scored group but whose nearest scored centroid was outside
115
+ * {@link IMPORTANCE_JOIN_GATE_KM} — a different town wearing the same name, refused rather than scored.
116
+ *
117
+ * Worth watching across rebuilds: a jump here means the score source and the admin source have drifted apart, and the
118
+ * join is being asked to guess.
119
+ */
120
+ importanceGated?: number
78
121
  }
79
122
 
80
123
  interface PlaceAttrs {
@@ -91,6 +134,12 @@ interface PlaceAttrs {
91
134
  pop: number
92
135
  neg: number
93
136
  pkey: string
137
+ /**
138
+ * The place's toponym-fame score, or null when the score source has no measurement for it (#28). A property of the
139
+ * PLACE, so it rides {@link stageRow} onto the alias and abbrev rows too — that is how a bare `Moscow` reaches
140
+ * Москва's score through the alias row that carries the key.
141
+ */
142
+ imp: number | null
94
143
  }
95
144
 
96
145
  export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<BuildCandidateResult> {
@@ -137,15 +186,55 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
137
186
  return id
138
187
  }
139
188
 
189
+ // --- importance source (#28): loaded BEFORE pass 1, which is the only pass that sees a place's
190
+ // name/country/placetype/centroid together. Absent → every row's `importance` stays NULL. ---
191
+ let importance: ReturnType<typeof loadImportanceIndex> | undefined
192
+
193
+ if (opts.importance) {
194
+ progress("importance", `loading place_importance from ${opts.importance}`)
195
+ importance = loadImportanceIndex(opts.importance)
196
+
197
+ progress(
198
+ "importance",
199
+ `${importance.stats.places.toLocaleString()} scored places in ${importance.stats.keys.toLocaleString()} (name, country, placetype) groups` +
200
+ (importance.stats.unkeyable ? `; ${importance.stats.unkeyable.toLocaleString()} unkeyable names skipped` : "")
201
+ )
202
+ } else {
203
+ // Never a silent column of nulls: a build without a score source produces one, and the reason has
204
+ // to be visible in the log rather than inferred from the artifact.
205
+ progress("importance", "no score source given — `importance` will be NULL on every row")
206
+ }
207
+
140
208
  // --- region_id per place (its region-tier ancestor) for same-name disambiguation ---
141
209
  progress("region", "loading region ancestry")
142
210
  const regionOf = new Map<number, number>()
143
-
144
- for (const r of src.prepare("SELECT id, ancestor_id FROM ancestors WHERE ancestor_placetype='region'").iterate()) {
211
+ let multiRegion = 0
212
+
213
+ // A place can carry more than one region-tier ancestor — 59 do on the 2026-08-03 artifact (42
214
+ // localities, mostly Chinese places on an ambiguous boundary). One `region_id` column holds one of
215
+ // them, and which one it ought to be is a real question this is not the place to answer.
216
+ //
217
+ // MIN is arbitrary but STABLE: an unordered pick lets the stamp for those places differ between two
218
+ // builds of the same source. The count is logged because the number is expected to grow, and should
219
+ // be visible rather than inferred.
220
+ for (const r of src
221
+ .prepare(
222
+ "SELECT id, MIN(ancestor_id) AS ancestor_id, COUNT(DISTINCT ancestor_id) AS n" +
223
+ " FROM ancestors WHERE ancestor_placetype='region' GROUP BY id"
224
+ )
225
+ .iterate()) {
145
226
  regionOf.set(Number(r.id), Number(r.ancestor_id))
227
+
228
+ if (Number(r.n) > 1) {
229
+ multiRegion++
230
+ }
146
231
  }
147
232
 
148
- progress("region", `${regionOf.size.toLocaleString()} places carry a region`)
233
+ progress(
234
+ "region",
235
+ `${regionOf.size.toLocaleString()} places carry a region` +
236
+ (multiRegion ? ` (${multiRegion.toLocaleString()} carry more than one; stamped with the lowest id)` : "")
237
+ )
149
238
 
150
239
  // The hot path — millions of clustered rows. Kept a single positional prepared statement (the fastest
151
240
  // node:sqlite insert) rather than a per-row query builder. Placeholders come from CANDIDATE_COLUMNS so
@@ -176,14 +265,16 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
176
265
  const neg = -Math.log10(pop + 1)
177
266
  const name = String(r.name ?? "")
178
267
  const pkey = normalizeLocalityForKey(name)
268
+ const lat = r.lat as number
269
+ const lon = r.lon as number
179
270
 
180
271
  const a: PlaceAttrs = {
181
272
  cid,
182
273
  rid,
183
274
  ptid,
184
275
  name,
185
- lat: r.lat as number,
186
- lon: r.lon as number,
276
+ lat,
277
+ lon,
187
278
  mnLat: r.mnlat as number,
188
279
  mnLon: r.mnlon as number,
189
280
  mxLat: r.mxlat as number,
@@ -191,12 +282,30 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
191
282
  pop,
192
283
  neg,
193
284
  pkey,
285
+ imp: importance?.find(name, r.country as string | null, r.placetype as string | null, lat, lon) ?? null,
194
286
  }
195
287
 
196
288
  attrs.set(sid, a)
197
289
 
198
290
  if (pkey) {
199
- insStage.run(pkey, cid, rid, ptid, neg, sid, name, a.lat, a.lon, a.mnLat, a.mnLon, a.mxLat, a.mxLon, pop, 1)
291
+ insStage.run(
292
+ pkey,
293
+ cid,
294
+ rid,
295
+ ptid,
296
+ neg,
297
+ sid,
298
+ name,
299
+ a.lat,
300
+ a.lon,
301
+ a.mnLat,
302
+ a.mnLon,
303
+ a.mxLat,
304
+ a.mxLon,
305
+ pop,
306
+ 1,
307
+ a.imp
308
+ )
200
309
 
201
310
  nPrim++
202
311
  }
@@ -205,6 +314,14 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
205
314
  out.exec("COMMIT")
206
315
  progress("primaries", `${nPrim.toLocaleString()} primaries; ${attrs.size.toLocaleString()} places`)
207
316
 
317
+ if (importance) {
318
+ progress(
319
+ "importance",
320
+ `${importance.matched.toLocaleString()} places scored; ` +
321
+ `${importance.gated.toLocaleString()} refused (nearest same-name place > ${IMPORTANCE_JOIN_GATE_KM} km away)`
322
+ )
323
+ }
324
+
208
325
  const stageRow = (k: string, a: PlaceAttrs, sid: number, isPrimary: number): void => {
209
326
  insStage.run(
210
327
  k,
@@ -221,7 +338,8 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
221
338
  a.mxLat,
222
339
  a.mxLon,
223
340
  a.pop,
224
- isPrimary
341
+ isPrimary,
342
+ a.imp
225
343
  )
226
344
  }
227
345
 
@@ -270,13 +388,25 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
270
388
  out.exec("COMMIT")
271
389
  progress("abbrevs", `${nAbbr.toLocaleString()} abbrevs`)
272
390
 
273
- // --- pass 4: postcodes (separate shards: spr placetype='postalcode' with real coords) ---
274
- let nPostcode = 0
275
-
276
- for (const pcDB of opts.postcodes ?? []) {
391
+ /**
392
+ * Pass 4 — fold ONE postcode shard (`spr` rows with `placetype='postalcode'`) in, then pass 4b: the delivery-city
393
+ * aliases hanging off the same shard's `names` table.
394
+ *
395
+ * Extracted rather than inlined because the shard loop is self-contained — it shares only the staging statement and
396
+ * the code dictionaries with the passes above, and nothing after it reads anything it produces except the two
397
+ * counters it returns.
398
+ */
399
+ const foldPostcodeShard = (pcDB: string): { primaries: number; aliases: number } => {
277
400
  progress("postcodes", `reading ${pcDB}`)
401
+
278
402
  const pc = new DatabaseSync(pcDB, { readOnly: true })
279
403
  const pcPtid = ptID("postalcode")
404
+ // Per-shard, not the admin `attrs` map: pass 1 only ever sees the admin DB, so the alias pass
405
+ // below has nothing to join against unless this primary loop records what it staged.
406
+ const pcAttrs = new Map<number, PlaceAttrs>()
407
+ let primaries = 0
408
+ let aliases = 0
409
+
280
410
  out.exec("BEGIN")
281
411
 
282
412
  for (const r of pc
@@ -290,38 +420,100 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
290
420
  const key = normalizeLocalityForKey(name)
291
421
 
292
422
  if (!key) continue
423
+
293
424
  const lat = r.latitude as number
294
425
  const lon = r.longitude as number
295
426
 
296
427
  // region_id 0 (a postcode is unique by name+country — no same-name disambiguation); neg_rank 0
297
428
  // (no population). bbox = the postcode's own min/max (falls back to the centroid point).
298
- insStage.run(
299
- key,
300
- ccID(r.country as string | null),
301
- 0,
302
- pcPtid,
303
- 0,
304
- Number(r.id),
429
+ const a: PlaceAttrs = {
430
+ cid: ccID(r.country as string | null),
431
+ rid: 0,
432
+ ptid: pcPtid,
305
433
  name,
306
434
  lat,
307
435
  lon,
308
- (r.mnlat as number) || lat,
309
- (r.mnlon as number) || lon,
310
- (r.mxlat as number) || lat,
311
- (r.mxlon as number) || lon,
312
- 0,
313
- 1
314
- )
315
-
316
- nPostcode++
436
+ mnLat: (r.mnlat as number) || lat,
437
+ mnLon: (r.mnlon as number) || lon,
438
+ mxLat: (r.mxlat as number) || lat,
439
+ mxLon: (r.mxlon as number) || lon,
440
+ pop: 0,
441
+ neg: 0,
442
+ pkey: key,
443
+ // A postcode has no toponym fame — nobody writes an encyclopedia article about SW1A 2AA — and
444
+ // the score source carries no `postalcode` rows to join against anyway. NULL is the truthful
445
+ // value: unmeasured, so the ranking key leaves postcode rows exactly where they were.
446
+ imp: null,
447
+ }
448
+
449
+ pcAttrs.set(Number(r.id), a)
450
+ stageRow(key, a, Number(r.id), 1)
451
+
452
+ primaries++
317
453
  }
318
454
 
319
455
  out.exec("COMMIT")
456
+
457
+ // --- pass 4b: postcode ALIAS names (#1495) ---
458
+ //
459
+ // The delivery-city names GeoNames supplies for a ZIP ("Brooklyn" for 11201) are written into
460
+ // the shard's `names` table by `postcode/centroid-fills.ts`'s `geonamesNameFill`. Everything
461
+ // downstream of `names` picked them up EXCEPT this build: `fts.ts` unions `spr.name` with every
462
+ // `names` row into `place_search.alt_names`, so the FTS backend resolved "Brooklyn" → 11201
463
+ // while the candidate backend — whose every row IS an exact-tier row — had no key for it at
464
+ // all. Pass 2 does the equivalent fold for admin places, but reads the ADMIN `place_search`,
465
+ // and `attrs` holds admin ids only, so a postcode shard could never reach it.
466
+ //
467
+ // Same discipline as pass 2: `is_primary = 0` (so `rankByPrimaryPreference` treats it as an
468
+ // alias, not a canonical postcode name), the row stays denormalized onto the POSTCODE's own
469
+ // spr_id/coords/bbox, and the display `name` stays the postcode — resolving "brooklyn" answers
470
+ // with place 11201, it does not rename the place to its delivery city.
471
+ const hasNames = pc.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='names'").get() !== undefined
472
+
473
+ if (hasNames) {
474
+ out.exec("BEGIN")
475
+
476
+ for (const r of pc.prepare("SELECT id, name FROM names").iterate()) {
477
+ const a = pcAttrs.get(Number(r.id))
478
+
479
+ if (!a) continue
480
+
481
+ const k = normalizeLocalityForKey(String(r.name ?? ""))
482
+
483
+ // The postcode's own key is already staged as the primary; `INSERT OR IGNORE` at
484
+ // materialization dedupes repeats, so this only skips the obvious self-alias.
485
+ if (!k || k === a.pkey) continue
486
+
487
+ stageRow(k, a, Number(r.id), 0)
488
+
489
+ aliases++
490
+ }
491
+
492
+ out.exec("COMMIT")
493
+ } else {
494
+ // Never a silent zero: real shards come from `createUnifiedSchema`, which always creates
495
+ // `names`. A shard without it has no alias surface to lose, but say so rather than reporting
496
+ // "0 aliases" from a table that was never read.
497
+ progress("postcode-aliases", `${pcDB} has no \`names\` table — no delivery-city aliases to fold`)
498
+ }
499
+
320
500
  pc.close()
501
+
502
+ return { primaries, aliases }
503
+ }
504
+
505
+ let nPostcode = 0
506
+ let nPostcodeAlias = 0
507
+
508
+ for (const pcDB of opts.postcodes ?? []) {
509
+ const folded = foldPostcodeShard(pcDB)
510
+
511
+ nPostcode += folded.primaries
512
+ nPostcodeAlias += folded.aliases
321
513
  }
322
514
 
323
515
  if (nPostcode > 0) {
324
- progress("postcodes", `${nPostcode.toLocaleString()} postcodes`)
516
+ progress("postcodes", `${nPostcode.toLocaleString()} postcodes; ${nPostcodeAlias.toLocaleString()} aliases`)
325
517
  }
326
518
 
327
519
  // --- code dictionaries: typed batch inserts via kdb (a few hundred rows — Kysely is clean here) ---
@@ -370,5 +562,14 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
370
562
  await kdb.destroy()
371
563
 
372
564
  // closes the underlying `out` connection
373
- return { rows, places: attrs.size, primaries: nPrim, aliases: nAlias, abbrevs: nAbbr, postcodes: nPostcode }
565
+ return {
566
+ rows,
567
+ places: attrs.size,
568
+ primaries: nPrim,
569
+ aliases: nAlias,
570
+ abbrevs: nAbbr,
571
+ postcodes: nPostcode,
572
+ postcodeAliases: nPostcodeAlias,
573
+ ...(importance ? { importanceScored: importance.matched, importanceGated: importance.gated } : {}),
574
+ }
374
575
  }