@mailwoman/resolver-wof-sqlite 9.0.0 → 9.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (297) hide show
  1. package/README.md +28 -9
  2. package/address-point-interpolation.ts +18 -8
  3. package/address-point-schema.ts +18 -6
  4. package/address-point.ts +111 -18
  5. package/ancestry.ts +9 -6
  6. package/build-candidate.ts +287 -157
  7. package/build-slim.ts +3 -3
  8. package/candidate/alias-bags.ts +54 -0
  9. package/candidate/ancestors-sidecar.ts +206 -0
  10. package/candidate/country-display-names.ts +79 -0
  11. package/candidate/name-roles.ts +237 -0
  12. package/candidate/own-name.ts +146 -0
  13. package/candidate/place-attrs.ts +44 -0
  14. package/candidate/shard-fold.ts +137 -0
  15. package/candidate-ancestors-schema.ts +195 -0
  16. package/candidate-fts.ts +4 -2
  17. package/candidate-importance.ts +228 -0
  18. package/candidate-lookup.ts +564 -174
  19. package/candidate-schema.ts +60 -3
  20. package/candidate-scoring.ts +268 -0
  21. package/capital-schema.ts +90 -0
  22. package/capitals.ts +148 -0
  23. package/coincident-roles.ts +69 -10
  24. package/convention-schema.ts +72 -0
  25. package/convention.ts +2 -2
  26. package/coverage-manifest-schema.ts +7 -7
  27. package/currency-backfill.ts +249 -0
  28. package/exact-match.ts +104 -0
  29. package/fst-autocomplete.ts +105 -122
  30. package/fst-builder.ts +39 -47
  31. package/fst-deserialize-web.ts +43 -7
  32. package/fst-freshness.ts +2 -2
  33. package/fst-serialize.ts +68 -12
  34. package/fst-types.ts +35 -1
  35. package/fts-query.ts +1 -1
  36. package/fts.ts +16 -4
  37. package/geonames-postal.ts +2 -2
  38. package/index.ts +26 -14
  39. package/interpolation.ts +113 -19
  40. package/lookup.ts +118 -560
  41. package/name-score.ts +6 -4
  42. package/out/address-point-interpolation.d.ts.map +1 -1
  43. package/out/address-point-interpolation.js +13 -7
  44. package/out/address-point-interpolation.js.map +1 -1
  45. package/out/address-point-schema.d.ts +16 -6
  46. package/out/address-point-schema.d.ts.map +1 -1
  47. package/out/address-point-schema.js.map +1 -1
  48. package/out/address-point.d.ts.map +1 -1
  49. package/out/address-point.js +70 -14
  50. package/out/address-point.js.map +1 -1
  51. package/out/ancestry.d.ts +2 -2
  52. package/out/ancestry.d.ts.map +1 -1
  53. package/out/ancestry.js +5 -6
  54. package/out/ancestry.js.map +1 -1
  55. package/out/build-candidate.d.ts +108 -0
  56. package/out/build-candidate.d.ts.map +1 -1
  57. package/out/build-candidate.js +151 -120
  58. package/out/build-candidate.js.map +1 -1
  59. package/out/build-slim.d.ts +1 -1
  60. package/out/build-slim.js +3 -3
  61. package/out/build-slim.js.map +1 -1
  62. package/out/candidate/alias-bags.d.ts +17 -0
  63. package/out/candidate/alias-bags.d.ts.map +1 -0
  64. package/out/candidate/alias-bags.js +39 -0
  65. package/out/candidate/alias-bags.js.map +1 -0
  66. package/out/candidate/ancestors-sidecar.d.ts +33 -0
  67. package/out/candidate/ancestors-sidecar.d.ts.map +1 -0
  68. package/out/candidate/ancestors-sidecar.js +140 -0
  69. package/out/candidate/ancestors-sidecar.js.map +1 -0
  70. package/out/candidate/country-display-names.d.ts +35 -0
  71. package/out/candidate/country-display-names.d.ts.map +1 -0
  72. package/out/candidate/country-display-names.js +59 -0
  73. package/out/candidate/country-display-names.js.map +1 -0
  74. package/out/candidate/name-roles.d.ts +55 -0
  75. package/out/candidate/name-roles.d.ts.map +1 -0
  76. package/out/candidate/name-roles.js +165 -0
  77. package/out/candidate/name-roles.js.map +1 -0
  78. package/out/candidate/own-name.d.ts +50 -0
  79. package/out/candidate/own-name.d.ts.map +1 -0
  80. package/out/candidate/own-name.js +132 -0
  81. package/out/candidate/own-name.js.map +1 -0
  82. package/out/candidate/place-attrs.d.ts +43 -0
  83. package/out/candidate/place-attrs.d.ts.map +1 -0
  84. package/out/candidate/place-attrs.js +15 -0
  85. package/out/candidate/place-attrs.js.map +1 -0
  86. package/out/candidate/shard-fold.d.ts +31 -0
  87. package/out/candidate/shard-fold.d.ts.map +1 -0
  88. package/out/candidate/shard-fold.js +104 -0
  89. package/out/candidate/shard-fold.js.map +1 -0
  90. package/out/candidate-ancestors-schema.d.ts +150 -0
  91. package/out/candidate-ancestors-schema.d.ts.map +1 -0
  92. package/out/candidate-ancestors-schema.js +123 -0
  93. package/out/candidate-ancestors-schema.js.map +1 -0
  94. package/out/candidate-fts.d.ts +4 -2
  95. package/out/candidate-fts.d.ts.map +1 -1
  96. package/out/candidate-fts.js +4 -2
  97. package/out/candidate-fts.js.map +1 -1
  98. package/out/candidate-importance.d.ts +132 -0
  99. package/out/candidate-importance.d.ts.map +1 -0
  100. package/out/candidate-importance.js +174 -0
  101. package/out/candidate-importance.js.map +1 -0
  102. package/out/candidate-lookup.d.ts +22 -37
  103. package/out/candidate-lookup.d.ts.map +1 -1
  104. package/out/candidate-lookup.js +446 -132
  105. package/out/candidate-lookup.js.map +1 -1
  106. package/out/candidate-schema.d.ts +52 -4
  107. package/out/candidate-schema.d.ts.map +1 -1
  108. package/out/candidate-schema.js +8 -0
  109. package/out/candidate-schema.js.map +1 -1
  110. package/out/candidate-scoring.d.ts +34 -0
  111. package/out/candidate-scoring.d.ts.map +1 -0
  112. package/out/candidate-scoring.js +200 -0
  113. package/out/candidate-scoring.js.map +1 -0
  114. package/out/capital-schema.d.ts +51 -0
  115. package/out/capital-schema.d.ts.map +1 -0
  116. package/out/capital-schema.js +63 -0
  117. package/out/capital-schema.js.map +1 -0
  118. package/out/capitals.d.ts +69 -0
  119. package/out/capitals.d.ts.map +1 -0
  120. package/out/capitals.js +98 -0
  121. package/out/capitals.js.map +1 -0
  122. package/out/coincident-roles.d.ts +7 -0
  123. package/out/coincident-roles.d.ts.map +1 -1
  124. package/out/coincident-roles.js +42 -8
  125. package/out/coincident-roles.js.map +1 -1
  126. package/out/convention-schema.d.ts +51 -0
  127. package/out/convention-schema.d.ts.map +1 -0
  128. package/out/convention-schema.js +34 -0
  129. package/out/convention-schema.js.map +1 -0
  130. package/out/convention.d.ts +1 -1
  131. package/out/convention.js +2 -2
  132. package/out/coverage-manifest-schema.js +3 -7
  133. package/out/coverage-manifest-schema.js.map +1 -1
  134. package/out/currency-backfill.d.ts +46 -0
  135. package/out/currency-backfill.d.ts.map +1 -0
  136. package/out/currency-backfill.js +180 -0
  137. package/out/currency-backfill.js.map +1 -0
  138. package/out/exact-match.d.ts +25 -0
  139. package/out/exact-match.d.ts.map +1 -0
  140. package/out/exact-match.js +89 -0
  141. package/out/exact-match.js.map +1 -0
  142. package/out/fst-autocomplete.d.ts +24 -14
  143. package/out/fst-autocomplete.d.ts.map +1 -1
  144. package/out/fst-autocomplete.js +84 -100
  145. package/out/fst-autocomplete.js.map +1 -1
  146. package/out/fst-builder.d.ts.map +1 -1
  147. package/out/fst-builder.js +32 -40
  148. package/out/fst-builder.js.map +1 -1
  149. package/out/fst-deserialize-web.d.ts.map +1 -1
  150. package/out/fst-deserialize-web.js +36 -7
  151. package/out/fst-deserialize-web.js.map +1 -1
  152. package/out/fst-freshness.d.ts +2 -2
  153. package/out/fst-freshness.js +2 -2
  154. package/out/fst-serialize.d.ts +14 -4
  155. package/out/fst-serialize.d.ts.map +1 -1
  156. package/out/fst-serialize.js +60 -12
  157. package/out/fst-serialize.js.map +1 -1
  158. package/out/fst-types.d.ts +35 -1
  159. package/out/fst-types.d.ts.map +1 -1
  160. package/out/fts-query.js +1 -1
  161. package/out/fts-query.js.map +1 -1
  162. package/out/fts.d.ts +15 -4
  163. package/out/fts.d.ts.map +1 -1
  164. package/out/fts.js +15 -4
  165. package/out/fts.js.map +1 -1
  166. package/out/geonames-postal.d.ts +2 -2
  167. package/out/geonames-postal.js +2 -2
  168. package/out/index.d.ts +4 -2
  169. package/out/index.d.ts.map +1 -1
  170. package/out/index.js +3 -2
  171. package/out/index.js.map +1 -1
  172. package/out/interpolation.d.ts +8 -0
  173. package/out/interpolation.d.ts.map +1 -1
  174. package/out/interpolation.js +91 -19
  175. package/out/interpolation.js.map +1 -1
  176. package/out/lookup.d.ts +4 -5
  177. package/out/lookup.d.ts.map +1 -1
  178. package/out/lookup.js +102 -444
  179. package/out/lookup.js.map +1 -1
  180. package/out/name-score.d.ts +0 -10
  181. package/out/name-score.d.ts.map +1 -1
  182. package/out/name-score.js +6 -4
  183. package/out/name-score.js.map +1 -1
  184. package/out/place-importance-schema.d.ts +226 -0
  185. package/out/place-importance-schema.d.ts.map +1 -0
  186. package/out/place-importance-schema.js +288 -0
  187. package/out/place-importance-schema.js.map +1 -0
  188. package/out/poi-lookup.d.ts +1 -1
  189. package/out/poi-lookup.d.ts.map +1 -1
  190. package/out/poi-lookup.js +12 -13
  191. package/out/poi-lookup.js.map +1 -1
  192. package/out/poi-schema.d.ts +7 -3
  193. package/out/poi-schema.d.ts.map +1 -1
  194. package/out/poi-schema.js.map +1 -1
  195. package/out/polygon-schema.d.ts +37 -0
  196. package/out/polygon-schema.d.ts.map +1 -0
  197. package/out/polygon-schema.js +23 -0
  198. package/out/polygon-schema.js.map +1 -0
  199. package/out/postal-city-alias-lookup.d.ts +1 -1
  200. package/out/postal-city-alias-lookup.js +1 -1
  201. package/out/postal-city-candidate-schema.d.ts +2 -1
  202. package/out/postal-city-candidate-schema.d.ts.map +1 -1
  203. package/out/postal-city-candidate-schema.js.map +1 -1
  204. package/out/postcode-point-lookup.d.ts +1 -1
  205. package/out/postcode-point-lookup.js +1 -1
  206. package/out/primary-preference.d.ts +125 -0
  207. package/out/primary-preference.d.ts.map +1 -0
  208. package/out/primary-preference.js +138 -0
  209. package/out/primary-preference.js.map +1 -0
  210. package/out/proximity-rerank.d.ts +77 -0
  211. package/out/proximity-rerank.d.ts.map +1 -0
  212. package/out/proximity-rerank.js +86 -0
  213. package/out/proximity-rerank.js.map +1 -0
  214. package/out/region-keys.d.ts +47 -0
  215. package/out/region-keys.d.ts.map +1 -0
  216. package/out/region-keys.js +121 -0
  217. package/out/region-keys.js.map +1 -0
  218. package/out/reverse.d.ts.map +1 -1
  219. package/out/reverse.js +6 -9
  220. package/out/reverse.js.map +1 -1
  221. package/out/schema.d.ts +1 -1
  222. package/out/search-fetch.d.ts +57 -0
  223. package/out/search-fetch.d.ts.map +1 -0
  224. package/out/search-fetch.js +183 -0
  225. package/out/search-fetch.js.map +1 -0
  226. package/out/sharding.d.ts +3 -3
  227. package/out/sharding.js +1 -1
  228. package/out/sqlite-convention-source.d.ts +1 -1
  229. package/out/sqlite-convention-source.js +1 -1
  230. package/out/sqlite-utils.d.ts +31 -1
  231. package/out/sqlite-utils.d.ts.map +1 -1
  232. package/out/sqlite-utils.js +38 -0
  233. package/out/sqlite-utils.js.map +1 -1
  234. package/out/street-centroid-schema.d.ts +7 -2
  235. package/out/street-centroid-schema.d.ts.map +1 -1
  236. package/out/street-centroid-schema.js.map +1 -1
  237. package/out/street-centroid.d.ts.map +1 -1
  238. package/out/street-centroid.js +7 -7
  239. package/out/street-centroid.js.map +1 -1
  240. package/out/street-morphology-fst-builder.d.ts.map +1 -1
  241. package/out/street-morphology-fst-builder.js +5 -4
  242. package/out/street-morphology-fst-builder.js.map +1 -1
  243. package/out/street-normalize.d.ts +83 -9
  244. package/out/street-normalize.d.ts.map +1 -1
  245. package/out/street-normalize.js +177 -10
  246. package/out/street-normalize.js.map +1 -1
  247. package/out/street-segment-schema.d.ts +6 -2
  248. package/out/street-segment-schema.d.ts.map +1 -1
  249. package/out/street-segment-schema.js.map +1 -1
  250. package/out/types.d.ts +74 -1
  251. package/out/types.d.ts.map +1 -1
  252. package/out/unified-schema.d.ts +1 -1
  253. package/out/unified-schema.js +1 -1
  254. package/out/uprn-lookup.d.ts +85 -0
  255. package/out/uprn-lookup.d.ts.map +1 -0
  256. package/out/uprn-lookup.js +152 -0
  257. package/out/uprn-lookup.js.map +1 -0
  258. package/out/uprn-schema.d.ts +93 -0
  259. package/out/uprn-schema.d.ts.map +1 -0
  260. package/out/uprn-schema.js +78 -0
  261. package/out/uprn-schema.js.map +1 -0
  262. package/out/weights-overlay-linker.d.ts +141 -0
  263. package/out/weights-overlay-linker.d.ts.map +1 -0
  264. package/out/weights-overlay-linker.js +259 -0
  265. package/out/weights-overlay-linker.js.map +1 -0
  266. package/package.json +296 -16
  267. package/place-importance-schema.ts +402 -0
  268. package/poi-lookup.ts +12 -13
  269. package/poi-schema.ts +8 -3
  270. package/polygon-schema.ts +47 -0
  271. package/postal-city-alias-lookup.ts +1 -1
  272. package/postal-city-candidate-schema.ts +3 -1
  273. package/postcode-point-lookup.ts +1 -1
  274. package/primary-preference.ts +207 -0
  275. package/proximity-rerank.ts +120 -0
  276. package/region-keys.ts +144 -0
  277. package/reverse.ts +17 -16
  278. package/schema.ts +1 -1
  279. package/search-fetch.ts +256 -0
  280. package/sharding.ts +3 -3
  281. package/sqlite-convention-source.ts +1 -1
  282. package/sqlite-utils.ts +63 -1
  283. package/street-centroid-schema.ts +8 -2
  284. package/street-centroid.ts +13 -8
  285. package/street-morphology-fst-builder.ts +5 -4
  286. package/street-normalize.ts +254 -24
  287. package/street-segment-schema.ts +7 -2
  288. package/types.ts +74 -1
  289. package/unified-schema.ts +1 -1
  290. package/uprn-lookup.ts +210 -0
  291. package/uprn-schema.ts +124 -0
  292. package/weights-overlay-linker.ts +377 -0
  293. package/geo.ts +0 -121
  294. package/out/geo.d.ts +0 -74
  295. package/out/geo.d.ts.map +0 -1
  296. package/out/geo.js +0 -71
  297. package/out/geo.js.map +0 -1
@@ -0,0 +1,402 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * The TWO-SCORE SPLIT (ROAD_TO_V9 §2 R1) — typed schema, table builder, and the two derivations, in
7
+ * one module so the read contract and the DDL cannot drift.
8
+ *
9
+ * THE POLICY THIS ENCODES, ratified 2026-08-06: **the importance of a knowledge-base article is not
10
+ * the probability that this is the place the user means.** The geocoder ranks by REFERENTIAL
11
+ * likelihood; encyclopedic importance is carried as data and is never the ranking key. So the one
12
+ * `place_importance.importance` column — which was a Wikipedia score where the concordance join
13
+ * landed and a population-derived pseudo-score everywhere else — becomes two named columns that can
14
+ * never be confused for one another:
15
+ *
16
+ * - {@link PlaceImportanceTable.referential} — population-anchored, ALWAYS derivable, the ranking
17
+ * backbone. {@link referentialFromPopulation} is the normalization the FST builder's population
18
+ * fallback has always used; naming it here is the whole change.
19
+ * - {@link PlaceImportanceTable.encyclopedic} — the fan-out-guarded Wikipedia join
20
+ * (`importance-fanout.ts`, #1497). NULLABLE, and null means ABSENT, never "an importance of
21
+ * zero": ~1.5 M of the 1.54 M rows in the 2026-08-05 build have no Wikipedia article at all, and
22
+ * a consumer that reads a 0 there would be reading a fact nobody recorded.
23
+ *
24
+ * WHY SAINT-DENIS IS THE TEST. The Seine-Saint-Denis suburb (pop 96,128) carries encyclopedic
25
+ * 0.1173; the Aude hamlet (pop 418) carries 0.5683 — the encyclopedic signal ranks the hamlet 4.8x
26
+ * ABOVE the place every user means. Referentially the suburb wins by 230x on population. One score
27
+ * cannot serve both readers, which is why there are two.
28
+ *
29
+ * THE LEGACY COLUMN STAYS, AND IS DERIVED. `importance` is written by {@link blendImportance} — the
30
+ * bounded blend the bare-toponym fame consumer (#28) ranks on. It is the CONFLATION, and nothing new
31
+ * should read it; new code reads the split columns.
32
+ */
33
+
34
+ import type { DatabaseSync } from "node:sqlite"
35
+
36
+ import { referentialFromPopulation } from "@mailwoman/core/resolver"
37
+ import type { Kysely } from "kysely"
38
+
39
+ import { allRows } from "./sqlite-utils.ts"
40
+
41
+ //#region Schema
42
+
43
+ /**
44
+ * One row of `place_importance`. Keyed by WOF place id.
45
+ */
46
+ export interface PlaceImportanceTable {
47
+ id: number
48
+ /**
49
+ * Population-anchored referential likelihood in [0, 1] — see {@link referentialFromPopulation}. NOT NULL because it is
50
+ * always derivable: a place with no population row scores 0, which here genuinely means "no population evidence", the
51
+ * same state the ranking has always treated as "no boost, never a penalty".
52
+ */
53
+ referential: number
54
+ /**
55
+ * Fan-out-guarded Wikipedia importance in [0, 1], or NULL when this place has no surviving concordance. NULL is
56
+ * ABSENCE — never coalesce it to 0 in a consumer, and never rank on it at all.
57
+ */
58
+ encyclopedic: number | null
59
+ /**
60
+ * DEPRECATED — the pre-split conflation, written by {@link blendImportance} so the bare-toponym fame consumer (#28)
61
+ * keeps one cross-bearer scale. New code reads {@link PlaceImportanceTable.referential} (to rank) or
62
+ * {@link PlaceImportanceTable.encyclopedic} (to display).
63
+ */
64
+ importance: number
65
+ }
66
+
67
+ /**
68
+ * The `place_importance` slice of a WOF admin database, for `new DatabaseClient<PlaceImportanceDatabase>(...)`.
69
+ */
70
+ export interface PlaceImportanceDatabase {
71
+ place_importance: PlaceImportanceTable
72
+ }
73
+
74
+ /**
75
+ * The columns in declaration order. The builder's INSERT derives its column list from this, so a field added to
76
+ * {@link PlaceImportanceTable} without a matching DDL column is a compile error at the insert site.
77
+ */
78
+ export const PLACE_IMPORTANCE_COLUMNS = [
79
+ "id",
80
+ "referential",
81
+ "encyclopedic",
82
+ "importance",
83
+ ] as const satisfies readonly (keyof PlaceImportanceTable)[]
84
+
85
+ /**
86
+ * Create `place_importance` at the split schema. Drops any existing table first — this is a rebuild-in-place step of
87
+ * `mailwoman gazetteer importance`, never a migration.
88
+ */
89
+ export async function createPlaceImportanceTable(db: Kysely<PlaceImportanceDatabase>): Promise<void> {
90
+ await db.schema.dropTable("place_importance").ifExists().execute()
91
+
92
+ await db.schema
93
+ .createTable("place_importance")
94
+ .addColumn("id", "integer", (c) => c.primaryKey())
95
+ .addColumn("referential", "real", (c) => c.notNull())
96
+ .addColumn("encyclopedic", "real")
97
+ .addColumn("importance", "real", (c) => c.notNull())
98
+ .execute()
99
+ }
100
+
101
+ //#endregion
102
+
103
+ //#region The referential derivation
104
+
105
+ /**
106
+ * Re-exported from `@mailwoman/core/resolver`, which is where the derivation lives so that `@mailwoman/resolver`
107
+ * (backend-agnostic — it cannot import this package) reads the same number. Re-exported HERE so the schema module stays
108
+ * the one-stop read for the table: the column and the function that fills it are one hop apart.
109
+ */
110
+ export {
111
+ compareReferential,
112
+ REFERENTIAL_LOG2_SCALE,
113
+ REFERENTIAL_POPULATION_DIVISOR,
114
+ REFERENTIAL_SATURATION_POPULATION,
115
+ referentialFromPopulation,
116
+ } from "@mailwoman/core/resolver"
117
+
118
+ //#endregion
119
+
120
+ //#region The legacy blend
121
+
122
+ /**
123
+ * The most the encyclopedic channel may raise a place's blended importance above its population-anchored referential
124
+ * score.
125
+ *
126
+ * In referential units 0.25 is 3.5 population doublings (the referential curve divides log2 by 14), so an article can
127
+ * promote a place as if it were up to ~11x its recorded population — never more. The bound exists because the two
128
+ * channels' scales CROSS at the article floor: merely having a Wikipedia article scores ~0.25–0.35, which exceeds the
129
+ * referential score of a mid-size town, so an unbounded blend ranks a 136-person village with an article above a
130
+ * 16,026-person town without one.
131
+ *
132
+ * The value is bracketed by two decided contests, measured on the 2026-08-24 staging build:
133
+ *
134
+ * - `> 0.2282`, or bare `Whitby` stops answering Whitby GB (pop 13,130, referential 0.2729, encyclopedic 0.5496) over
135
+ * Whitby CA (pop 128,377, referential 0.5011) — the #28 design case the fame prior exists to serve.
136
+ * - `< 0.2790`, or bare `Tó`/`To` answers Tó PT (pop 136, referential 0.0131, encyclopedic 0.3375) over Tô BF (pop
137
+ * 16,026, no article) — the `bf-gloss-to-*` board pair.
138
+ *
139
+ * 0.25 sits mid-interval with ~0.02 margin to each bound.
140
+ */
141
+ export const ENCYCLOPEDIC_BOOST_CAP = 0.25
142
+
143
+ /**
144
+ * The legacy `importance` blend — one cross-bearer fame scale for the #28 consumer, derived from the two split
145
+ * channels:
146
+ *
147
+ * - No article → the referential score.
148
+ * - No population evidence (`referential` 0) → the encyclopedic score stands alone: there is nothing to bound the
149
+ * article's claim against, and a constant cap would demote every famous place WOF records no population for
150
+ * (meaning-of-zero: referential 0 is "unmeasured", not "tiny").
151
+ * - Both present → the encyclopedic value clamped to at most {@link ENCYCLOPEDIC_BOOST_CAP} above the referential score,
152
+ * and never below it. The floor half repairs the downward inversion (the Seine-Saint-Denis suburb's weak article
153
+ * scored 0.1173 and REPLACED its referential 0.4716 under the old `COALESCE`, so a 418-person Aude hamlet outranked
154
+ * it 4.8x); the cap half repairs the upward one (`Tó`, above).
155
+ *
156
+ * Scale of the clamp on the 2026-08-24 staging build: of 628,202 article-bearing rows, 209,738 sit above the cap and
157
+ * 13,888 sit below their referential floor; the 305,168 article-without-population rows pass through unchanged.
158
+ */
159
+ export function blendImportance(referential: number, encyclopedic: number | null | undefined): number {
160
+ if (encyclopedic === null || encyclopedic === undefined) return referential
161
+
162
+ if (referential <= 0) return encyclopedic
163
+
164
+ return Math.max(referential, Math.min(encyclopedic, referential + ENCYCLOPEDIC_BOOST_CAP))
165
+ }
166
+
167
+ //#endregion
168
+
169
+ //#region Reading a database that may or may not carry the split
170
+
171
+ /**
172
+ * Where a reader's two scores came from. Recorded into the FST stamp so an artifact says, in its own provenance,
173
+ * whether its encyclopedic channel is real, reconstructed, or absent.
174
+ */
175
+ export const IMPORTANCE_SPLIT_SOURCES = {
176
+ /**
177
+ * `place_importance` carries `referential` + `encyclopedic` — the post-split build.
178
+ */
179
+ splitColumns: "split-columns",
180
+ /**
181
+ * `place_importance` carries only the conflated `importance`; the split was RECONSTRUCTED against `place_population`
182
+ * (see {@link splitLegacyImportance}).
183
+ */
184
+ legacyReconstructed: "legacy-reconstructed",
185
+ /**
186
+ * No `place_importance` at all — referential from `place_population`, encyclopedic absent for every place.
187
+ */
188
+ populationOnly: "population-only",
189
+ /**
190
+ * Neither table. Every score is 0, and 0 here means the database carries no salience evidence whatsoever.
191
+ */
192
+ none: "none",
193
+ } as const
194
+
195
+ export type ImportanceSplitSource = (typeof IMPORTANCE_SPLIT_SOURCES)[keyof typeof IMPORTANCE_SPLIT_SOURCES]
196
+
197
+ /**
198
+ * The `SELECT` term and `LEFT JOIN` a name lookup needs in order to CARRY `encyclopedic` onto its results, probed
199
+ * against `schemaName`'s `place_importance`.
200
+ *
201
+ * THE PROBE IS A COLUMN, NOT A TABLE, and that is the whole reason this lives here rather than beside a caller's other
202
+ * table probes. The pre-split table exists and holds a single conflated `importance` column whose value is a Wikipedia
203
+ * score on some rows and a population proxy on others, with nothing in the row to say which — reading that as
204
+ * encyclopedic would surface the exact confusion ROAD_TO_V9 §2 exists to end.
205
+ *
206
+ * There is deliberately no ORDER BY counterpart, and there should never be one: §2's policy is that this score is
207
+ * carried and never ranked on. Without the column the select degrades to a literal `NULL` and the join is the empty
208
+ * string, so a pre-split shard's query plan is byte-identical to what it was before the split. No shipped gazetteer
209
+ * carries the column yet, so today that degraded form is the only one anything builds.
210
+ *
211
+ * Call it ONCE per shard and cache the result — it runs a `PRAGMA`, and the callers are per-keystroke hot.
212
+ */
213
+ export function encyclopedicClauses(db: DatabaseSync, schemaName: string): { select: string; join: string } {
214
+ let present: boolean
215
+
216
+ try {
217
+ const rows = allRows<{ name: string }>(db.prepare(`PRAGMA ${schemaName}.table_info(place_importance)`))
218
+
219
+ present = rows.some((r) => r.name === "encyclopedic")
220
+ } catch {
221
+ present = false
222
+ }
223
+
224
+ if (!present) return { select: "NULL AS encyclopedic", join: "" }
225
+
226
+ return {
227
+ select: "place_importance.encyclopedic AS encyclopedic",
228
+ join: `LEFT JOIN ${schemaName}.place_importance ON place_importance.id = spr.id`,
229
+ }
230
+ }
231
+
232
+ /**
233
+ * The tolerance {@link splitLegacyImportance} treats as "this value reproduces the population curve". Eight ULP —
234
+ * comfortably wider than the one-ULP `log2` spread measured between CPython and V8 (see that function's docstring), and
235
+ * ~1e-15 absolute against scores Nominatim publishes to four decimals.
236
+ */
237
+ export const LEGACY_FALLBACK_EPSILON = 8 * Number.EPSILON
238
+
239
+ /**
240
+ * Split one row of a LEGACY (pre-split) `place_importance` table back into its two components.
241
+ *
242
+ * WHY THIS IS RECOVERABLE AT ALL. The legacy builder ran in two passes: Wikipedia scores first, then `INSERT OR IGNORE`
243
+ * of `min(1, log2(1+pop/1000)/14)` for every place with a population that Wikipedia had missed. So a legacy value is a
244
+ * fallback row IFF it reproduces {@link referentialFromPopulation} of that place's population — the two passes wrote
245
+ * different arithmetic, and a score from Nominatim's four-decimal TSV landing on the log2 curve is a measure-zero
246
+ * event.
247
+ *
248
+ * THE COMPARISON IS ULP-TOLERANT, AND THAT IS NOT DEFENSIVE PADDING. The first version compared for exact bit equality,
249
+ * which is correct — in the runtime that wrote the values. Cross-checking the same rule in CPython returned 166,638
250
+ * encyclopedic rows against Node's 133,096: `math.log2` and V8's `Math.log2` disagree by one ULP on 33,542 of the 1.5 M
251
+ * inputs (worked example: wof 85803233, population 21,299 — stored 0.31992193633838988953, CPython
252
+ * 0.31992193633838994504, delta 5.55e-17). Bit equality would therefore INVENT 33,542 encyclopedic scores for anyone
253
+ * who ported this rule to another runtime, and invented data is the failure mode this whole module exists to end. The
254
+ * tolerance is a few ULP of the score's own magnitude; a genuine Wikipedia value that close to the population curve is
255
+ * not distinguishable from it by any consequence.
256
+ *
257
+ * MEASURED, not reasoned (2026-08-06, `wof/fst-staging-2026-08-05/admin-global-priority-importance.db`): 1,543,753 rows
258
+ * split **1,410,657 fallback / 133,096 encyclopedic**, and the arithmetic closes on itself — 1,410,657 + 108,861
259
+ * (encyclopedic rows that ALSO have a population) = 1,519,518, which is exactly the count of `place_population` rows
260
+ * with `population > 0`, i.e. every row the fallback pass could have written. The remaining 24,235 encyclopedic rows
261
+ * have no population row at all. Under the exact-equality rule, Node found ZERO mismatches within one ULP, so the
262
+ * tolerance changes no classification on this database — it only makes the answer runtime-independent.
263
+ *
264
+ * Referential is NOT read out of the legacy column under any branch — it is always re-derived from population, because
265
+ * a legacy Wikipedia row overwrote whatever population would have said.
266
+ */
267
+ export function splitLegacyImportance(
268
+ legacy: number | undefined,
269
+ population: number | null | undefined
270
+ ): { referential: number; encyclopedic?: number } {
271
+ const referential = referentialFromPopulation(population)
272
+
273
+ if (legacy === undefined) return { referential }
274
+
275
+ if (Math.abs(legacy - referential) <= LEGACY_FALLBACK_EPSILON * Math.max(referential, 1)) return { referential }
276
+
277
+ return { referential, encyclopedic: legacy }
278
+ }
279
+
280
+ /**
281
+ * The two score maps a builder needs, plus the provenance of how they were obtained.
282
+ */
283
+ export interface ImportanceSplit {
284
+ /**
285
+ * WOF id → referential likelihood. Sparse: absent means 0 (no population evidence).
286
+ */
287
+ referential: Map<number, number>
288
+ /**
289
+ * WOF id → encyclopedic importance. Sparse, and ABSENCE IS ABSENCE — never fill a 0 in.
290
+ */
291
+ encyclopedic: Map<number, number>
292
+ source: ImportanceSplitSource
293
+ /**
294
+ * Rows the legacy reconstruction attributed to the population fallback (only meaningful under
295
+ * {@link IMPORTANCE_SPLIT_SOURCES.legacyReconstructed}).
296
+ */
297
+ legacyFallbackRows: number
298
+ }
299
+
300
+ /**
301
+ * Does `table` exist in `db`, and if so which of `columns` does it have?
302
+ */
303
+ function tableColumns(db: DatabaseSync, table: string): Set<string> {
304
+ try {
305
+ const rows = allRows<{ name: string }>(db.prepare(`PRAGMA table_info(${table})`))
306
+
307
+ return new Set(rows.map((r) => r.name))
308
+ } catch {
309
+ return new Set()
310
+ }
311
+ }
312
+
313
+ /**
314
+ * Load both scores from a WOF admin database, whatever schema generation it is at.
315
+ *
316
+ * Handles all four states in {@link IMPORTANCE_SPLIT_SOURCES} so callers never branch on schema themselves — the FST
317
+ * builder in particular must read the shipped population-only databases, the read-only 2026-08-05 staging database
318
+ * (legacy conflated column), and post-split builds with one code path.
319
+ */
320
+ export function loadImportanceSplit(db: DatabaseSync): ImportanceSplit {
321
+ const referential = new Map<number, number>()
322
+ const encyclopedic = new Map<number, number>()
323
+ const population = new Map<number, number>()
324
+
325
+ const populationColumns = tableColumns(db, "place_population")
326
+
327
+ if (populationColumns.has("population")) {
328
+ const rows = allRows<{
329
+ id: number
330
+ population: number
331
+ }>(db.prepare("SELECT id, population FROM place_population"))
332
+
333
+ for (const row of rows) {
334
+ population.set(row.id, row.population)
335
+ const score = referentialFromPopulation(row.population)
336
+
337
+ if (score > 0) {
338
+ referential.set(row.id, score)
339
+ }
340
+ }
341
+ }
342
+
343
+ const importanceColumns = tableColumns(db, "place_importance")
344
+
345
+ if (importanceColumns.has("referential")) {
346
+ // Post-split build: the columns ARE the contract. Referential is read verbatim rather than
347
+ // re-derived, so a build that scored referential differently stays visible instead of being
348
+ // silently overwritten by this reader's own formula.
349
+ const rows = allRows<{
350
+ id: number
351
+ referential: number
352
+ encyclopedic: number | null
353
+ }>(db.prepare("SELECT id, referential, encyclopedic FROM place_importance"))
354
+
355
+ for (const row of rows) {
356
+ if (row.referential > 0) {
357
+ referential.set(row.id, row.referential)
358
+ }
359
+
360
+ if (row.encyclopedic !== null) {
361
+ encyclopedic.set(row.id, row.encyclopedic)
362
+ }
363
+ }
364
+
365
+ return { referential, encyclopedic, source: IMPORTANCE_SPLIT_SOURCES.splitColumns, legacyFallbackRows: 0 }
366
+ }
367
+
368
+ if (importanceColumns.has("importance")) {
369
+ const rows = allRows<{
370
+ id: number
371
+ importance: number
372
+ }>(db.prepare("SELECT id, importance FROM place_importance"))
373
+
374
+ let legacyFallbackRows = 0
375
+
376
+ for (const row of rows) {
377
+ const split = splitLegacyImportance(row.importance, population.get(row.id))
378
+
379
+ if (split.encyclopedic === undefined) {
380
+ legacyFallbackRows++
381
+ } else {
382
+ encyclopedic.set(row.id, split.encyclopedic)
383
+ }
384
+ }
385
+
386
+ return {
387
+ referential,
388
+ encyclopedic,
389
+ source: IMPORTANCE_SPLIT_SOURCES.legacyReconstructed,
390
+ legacyFallbackRows,
391
+ }
392
+ }
393
+
394
+ return {
395
+ referential,
396
+ encyclopedic,
397
+ source: referential.size ? IMPORTANCE_SPLIT_SOURCES.populationOnly : IMPORTANCE_SPLIT_SOURCES.none,
398
+ legacyFallbackRows: 0,
399
+ }
400
+ }
401
+
402
+ //#endregion
package/poi-lookup.ts CHANGED
@@ -31,6 +31,7 @@ import { haversineKm, shortCellToInt, type H3Cell } from "@mailwoman/spatial"
31
31
  import { gridDisk, latLngToCell } from "h3-js"
32
32
 
33
33
  import type { POICategoryCodeTable, POITable } from "./poi-schema.ts"
34
+ import { allRows } from "./sqlite-utils.ts"
34
35
 
35
36
  /**
36
37
  * Resolution the `poi` table's `h3_cell` column is keyed at — matches the builder (spec §3.4).
@@ -141,7 +142,7 @@ type POIRow = Pick<
141
142
  /**
142
143
  * Node reader over `poi.db`. `implements Disposable` so callers can `using lookup = new POILookup(...)` (or call
143
144
  * `[Symbol.dispose]()` explicitly), the same precedent as {@link WOFCandidateTableLookup} /
144
- * {@link WOFSqlitePlaceLookup}.
145
+ * {@link WOFSQLitePlaceLookup}.
145
146
  */
146
147
  export class POILookup implements Disposable {
147
148
  #db: DatabaseSync
@@ -175,9 +176,7 @@ export class POILookup implements Disposable {
175
176
 
176
177
  // The category dictionary is tiny (poi-taxonomy's category count) — load it once at
177
178
  // construction so `search` never round-trips to it.
178
- for (const r of this.#db
179
- .prepare("SELECT id, category FROM poi_category_codes")
180
- .all() as unknown as POICategoryCodeTable[]) {
179
+ for (const r of allRows<POICategoryCodeTable>(this.#db.prepare("SELECT id, category FROM poi_category_codes"))) {
181
180
  this.#categoryToID.set(String(r.category), Number(r.id))
182
181
  this.#idToCategory.set(Number(r.id), String(r.category))
183
182
  }
@@ -226,7 +225,7 @@ export class POILookup implements Disposable {
226
225
  * nearest at any distance — the reach ceiling k-ring hits on sparse brand rows is gone.
227
226
  */
228
227
  #searchBrand(brandWikidata: string, center: { latitude: number; longitude: number }, limit: number): POISearchHit[] {
229
- const rows = this.#brandProbe.all(brandWikidata) as unknown as POIRow[]
228
+ const rows = allRows<POIRow>(this.#brandProbe, brandWikidata)
230
229
 
231
230
  return sortByDistance(rows, center)
232
231
  .filter(
@@ -242,7 +241,7 @@ export class POILookup implements Disposable {
242
241
  #searchKRing(query: POISearchQuery, limit: number): POISearchHit[] {
243
242
  const center = query.center!
244
243
  const maxRings = query.maxRings ?? DEFAULT_MAX_RINGS
245
- const categoryIds: number[] = []
244
+ const categoryIDs: number[] = []
246
245
 
247
246
  // `categoryIDs` (the fan-out list) supersedes the single `categoryID`; either way, resolve each id through the
248
247
  // dictionary and drop the ones the db doesn't carry (Overture-taxonomy drift, or an identity id with no rows).
@@ -252,12 +251,12 @@ export class POILookup implements Disposable {
252
251
  const resolved = this.#categoryToID.get(id)
253
252
 
254
253
  if (resolved !== undefined) {
255
- categoryIds.push(resolved)
254
+ categoryIDs.push(resolved)
256
255
  }
257
256
  }
258
257
 
259
258
  // No resolvable leaf (every id unknown to the dictionary) can't have rows — a clean miss, not a throw.
260
- if (!categoryIds.length) return []
259
+ if (!categoryIDs.length) return []
261
260
 
262
261
  const origin = latLngToCell(center.latitude, center.longitude, POI_H3_RESOLUTION) as H3Cell
263
262
  const seenCells = new Set<string>()
@@ -276,8 +275,8 @@ export class POILookup implements Disposable {
276
275
 
277
276
  // Fan-out: probe every resolved Overture leaf for this canonical category, unioning the rows. The
278
277
  // post-ring distance sort + `slice(0, limit)` below dedupes the pool down to the nearest `limit`.
279
- for (const categoryID of categoryIds) {
280
- rows.push(...(this.#categoryCellProbe.all(shortCell, categoryID, limit) as unknown as POIRow[]))
278
+ for (const categoryID of categoryIDs) {
279
+ rows.push(...allRows<POIRow>(this.#categoryCellProbe, shortCell, categoryID, limit))
281
280
  }
282
281
  }
283
282
 
@@ -301,7 +300,7 @@ export class POILookup implements Disposable {
301
300
 
302
301
  if (!matchQuery) return []
303
302
 
304
- const ftsHits = this.#nameFTSProbe.all(matchQuery, limit) as unknown as Array<{ name_key: string | null }>
303
+ const ftsHits = allRows<{ name_key: string | null }>(this.#nameFTSProbe, matchQuery, limit)
305
304
  const uniqueKeys: string[] = []
306
305
  const seenKeys = new Set<string>()
307
306
 
@@ -350,7 +349,7 @@ export class POILookup implements Disposable {
350
349
  const placeholders = nameKeys.map(() => "?").join(", ")
351
350
  const stmt = this.#db.prepare(`SELECT ${columns} FROM poi WHERE name_key IN (${placeholders})`)
352
351
 
353
- return stmt.all(...nameKeys) as unknown as POIRow[]
352
+ return allRows<POIRow>(stmt, ...nameKeys)
354
353
  }
355
354
 
356
355
  close(): void {
@@ -406,7 +405,7 @@ function sanitizePOINameQuery(text: string): string {
406
405
  .replaceAll(/["*:]/g, "")
407
406
  .trim()
408
407
  .split(/\s+/u)
409
- .filter(Boolean)
408
+ .filter((token) => token.length > 0)
410
409
  .map((token) => `"${token.replaceAll('"', '""')}"`)
411
410
  .join(" ")
412
411
  }
package/poi-schema.ts CHANGED
@@ -17,6 +17,8 @@ import type { DatabaseSync } from "node:sqlite"
17
17
  import type { LayerContractDatabase } from "@mailwoman/core/layers"
18
18
  import { sql, type Kysely } from "kysely"
19
19
 
20
+ import type { NameKey } from "./street-normalize.ts"
21
+
20
22
  /**
21
23
  * One POI row. Clustered PK: h3_cell → category_id → neg_rank → rowid_key.
22
24
  */
@@ -39,9 +41,12 @@ export interface POITable {
39
41
  rowid_key: number
40
42
  name: string | null
41
43
  /**
42
- * Lowercased, diacritic-flattened probe key for exact name lookups.
44
+ * Probe key for exact name lookups, minted by {@link normalizeLocalityForKey} at build AND at query.
45
+ *
46
+ * Branded because a `toLowerCase()` approximation of the fold is still a `string`: it binds to the parameter, returns
47
+ * fewer rows, and the shortfall reads as a coverage gap in the data rather than a defect in the probe.
43
48
  */
44
- name_key: string | null
49
+ name_key: NameKey | null
45
50
  brand_wikidata: string | null
46
51
  latitude: number
47
52
  longitude: number
@@ -69,7 +74,7 @@ export interface POIStageTable {
69
74
  neg_rank: number | null
70
75
  rowid_key: number | null
71
76
  name: string | null
72
- name_key: string | null
77
+ name_key: NameKey | null
73
78
  brand_wikidata: string | null
74
79
  latitude: number
75
80
  longitude: number
@@ -0,0 +1,47 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * Typed schema for the polygon sidecar (`wof-polygons.db`) — the one table `WOFReverseGeocoder` probes for a
7
+ * place's geometry. The interface is the read/write contract and {@link createPolygonsTable} creates the table, so a
8
+ * column added to one is a compile error against the other.
9
+ *
10
+ * The sidecar is OPTIONAL to the reverse geocoder: without it every result falls back to a centroid, so the reader
11
+ * checks for the table's presence rather than assuming it.
12
+ */
13
+
14
+ import type { Kysely } from "kysely"
15
+
16
+ /**
17
+ * One admin polygon, keyed by WOF id.
18
+ */
19
+ export interface PolygonsTable {
20
+ id: number
21
+ /**
22
+ * The GeoJSON geometry, JSON-encoded. A row that fails to parse reads as no-polygon, never as an error — a malformed
23
+ * geometry must not fail the whole reverse query.
24
+ */
25
+ geom: string
26
+ }
27
+
28
+ export interface PolygonDatabase {
29
+ polygons: PolygonsTable
30
+ }
31
+
32
+ /**
33
+ * The slice of a Kysely handle the polygon DDL touches. Kysely is invariant in its schema parameter, so naming only
34
+ * `schema` lets a builder holding a wider handle pass it without a cast.
35
+ */
36
+ export type PolygonSchemaHandle = Pick<Kysely<PolygonDatabase>, "schema">
37
+
38
+ /**
39
+ * Create the `polygons` table — called before the streaming bulk load.
40
+ */
41
+ export async function createPolygonsTable(db: PolygonSchemaHandle): Promise<void> {
42
+ await db.schema
43
+ .createTable("polygons")
44
+ .addColumn("id", "integer", (column) => column.primaryKey())
45
+ .addColumn("geom", "text", (column) => column.notNull())
46
+ .execute()
47
+ }
@@ -5,7 +5,7 @@
5
5
  *
6
6
  * Node reader over the POSTAL-CITY ALIAS table (`postal-city-alias-<cc>.db`) — the observed
7
7
  * `postal_city → geo_locality` aliases per postcode (`build-postal-city-alias.ts`). Consumed by
8
- * {@link WOFSqlitePlaceLookup}'s coordinate-first locality scorer: a user-typed postal city
8
+ * {@link WOFSQLitePlaceLookup}'s coordinate-first locality scorer: a user-typed postal city
9
9
  * ("Antioch", postcode 37013) becomes a name-match alias for the geographic locality the postcode
10
10
  * actually sits in ("Nashville"), so the right place tiers to the top instead of a same-named
11
11
  * town in another state. Opt-in — the lookup is only constructed when a path is supplied, and
@@ -20,6 +20,8 @@
20
20
 
21
21
  import { sql, type Kysely } from "kysely"
22
22
 
23
+ import type { NameKey } from "./street-normalize.ts"
24
+
23
25
  /**
24
26
  * One postal-city → geo-locality edge, keyed exactly by `(name_key, postcode)`. The probe returns the geographic
25
27
  * locality directly; the denormalized name/coord avoid a join back to `candidate`.
@@ -28,7 +30,7 @@ export interface PostalCityCandidateTable {
28
30
  /**
29
31
  * {@link normalizeLocalityForKey} of the postal-city name — the build/query-consistent probe key.
30
32
  */
31
- name_key: string
33
+ name_key: NameKey
32
34
  /**
33
35
  * The postcode the alias is scoped to (the second half of the exact key).
34
36
  */
@@ -13,7 +13,7 @@
13
13
  * anchor only needs "does this string exist as a postcode, in which countries, near where". A
14
14
  * future WASM build swaps this for an FST-backed resolver behind the same `lookup()` seam.
15
15
  *
16
- * Why multiple shards instead of the multi-shard `WOFSqlitePlaceLookup`: that resolver routes a
16
+ * Why multiple shards instead of the multi-shard `WOFSQLitePlaceLookup`: that resolver routes a
17
17
  * query to ONE shard by placetype, but every postcode shard shares `placetype='postalcode'`, so a
18
18
  * single query could only ever hit one country's shard. The anchor needs the union across
19
19
  * countries to build its country posterior, so it queries each shard directly.