@mailwoman/resolver-wof-sqlite 9.1.0 → 9.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (278) hide show
  1. package/README.md +28 -9
  2. package/address-point-interpolation.ts +18 -8
  3. package/address-point-schema.ts +18 -6
  4. package/address-point.ts +111 -18
  5. package/ancestry.ts +9 -6
  6. package/build-candidate.ts +200 -163
  7. package/build-slim.ts +3 -3
  8. package/candidate/alias-bags.ts +54 -0
  9. package/candidate/ancestors-sidecar.ts +206 -0
  10. package/candidate/country-display-names.ts +79 -0
  11. package/candidate/name-roles.ts +237 -0
  12. package/candidate/own-name.ts +146 -0
  13. package/candidate/place-attrs.ts +44 -0
  14. package/candidate/shard-fold.ts +137 -0
  15. package/candidate-ancestors-schema.ts +195 -0
  16. package/candidate-fts.ts +4 -2
  17. package/candidate-importance.ts +2 -1
  18. package/candidate-lookup.ts +439 -186
  19. package/candidate-schema.ts +33 -5
  20. package/candidate-scoring.ts +268 -0
  21. package/capital-schema.ts +90 -0
  22. package/capitals.ts +148 -0
  23. package/coincident-roles.ts +69 -10
  24. package/convention-schema.ts +72 -0
  25. package/convention.ts +2 -2
  26. package/coverage-manifest-schema.ts +7 -7
  27. package/currency-backfill.ts +249 -0
  28. package/exact-match.ts +104 -0
  29. package/fst-autocomplete.ts +91 -119
  30. package/fst-builder.ts +14 -12
  31. package/fst-freshness.ts +2 -2
  32. package/fts-query.ts +1 -1
  33. package/fts.ts +4 -4
  34. package/geonames-postal.ts +2 -2
  35. package/index.ts +18 -14
  36. package/interpolation.ts +113 -19
  37. package/lookup.ts +110 -591
  38. package/name-score.ts +6 -4
  39. package/out/address-point-interpolation.d.ts.map +1 -1
  40. package/out/address-point-interpolation.js +13 -7
  41. package/out/address-point-interpolation.js.map +1 -1
  42. package/out/address-point-schema.d.ts +16 -6
  43. package/out/address-point-schema.d.ts.map +1 -1
  44. package/out/address-point-schema.js.map +1 -1
  45. package/out/address-point.d.ts.map +1 -1
  46. package/out/address-point.js +70 -14
  47. package/out/address-point.js.map +1 -1
  48. package/out/ancestry.d.ts +2 -2
  49. package/out/ancestry.d.ts.map +1 -1
  50. package/out/ancestry.js +5 -6
  51. package/out/ancestry.js.map +1 -1
  52. package/out/build-candidate.d.ts +75 -0
  53. package/out/build-candidate.d.ts.map +1 -1
  54. package/out/build-candidate.js +120 -122
  55. package/out/build-candidate.js.map +1 -1
  56. package/out/build-slim.d.ts +1 -1
  57. package/out/build-slim.js +3 -3
  58. package/out/build-slim.js.map +1 -1
  59. package/out/candidate/alias-bags.d.ts +17 -0
  60. package/out/candidate/alias-bags.d.ts.map +1 -0
  61. package/out/candidate/alias-bags.js +39 -0
  62. package/out/candidate/alias-bags.js.map +1 -0
  63. package/out/candidate/ancestors-sidecar.d.ts +33 -0
  64. package/out/candidate/ancestors-sidecar.d.ts.map +1 -0
  65. package/out/candidate/ancestors-sidecar.js +140 -0
  66. package/out/candidate/ancestors-sidecar.js.map +1 -0
  67. package/out/candidate/country-display-names.d.ts +35 -0
  68. package/out/candidate/country-display-names.d.ts.map +1 -0
  69. package/out/candidate/country-display-names.js +59 -0
  70. package/out/candidate/country-display-names.js.map +1 -0
  71. package/out/candidate/name-roles.d.ts +55 -0
  72. package/out/candidate/name-roles.d.ts.map +1 -0
  73. package/out/candidate/name-roles.js +165 -0
  74. package/out/candidate/name-roles.js.map +1 -0
  75. package/out/candidate/own-name.d.ts +50 -0
  76. package/out/candidate/own-name.d.ts.map +1 -0
  77. package/out/candidate/own-name.js +132 -0
  78. package/out/candidate/own-name.js.map +1 -0
  79. package/out/candidate/place-attrs.d.ts +43 -0
  80. package/out/candidate/place-attrs.d.ts.map +1 -0
  81. package/out/candidate/place-attrs.js +15 -0
  82. package/out/candidate/place-attrs.js.map +1 -0
  83. package/out/candidate/shard-fold.d.ts +31 -0
  84. package/out/candidate/shard-fold.d.ts.map +1 -0
  85. package/out/candidate/shard-fold.js +104 -0
  86. package/out/candidate/shard-fold.js.map +1 -0
  87. package/out/candidate-ancestors-schema.d.ts +150 -0
  88. package/out/candidate-ancestors-schema.d.ts.map +1 -0
  89. package/out/candidate-ancestors-schema.js +123 -0
  90. package/out/candidate-ancestors-schema.js.map +1 -0
  91. package/out/candidate-fts.d.ts +4 -2
  92. package/out/candidate-fts.d.ts.map +1 -1
  93. package/out/candidate-fts.js +4 -2
  94. package/out/candidate-fts.js.map +1 -1
  95. package/out/candidate-importance.d.ts.map +1 -1
  96. package/out/candidate-importance.js +1 -1
  97. package/out/candidate-importance.js.map +1 -1
  98. package/out/candidate-lookup.d.ts +22 -45
  99. package/out/candidate-lookup.d.ts.map +1 -1
  100. package/out/candidate-lookup.js +340 -135
  101. package/out/candidate-lookup.js.map +1 -1
  102. package/out/candidate-schema.d.ts +30 -6
  103. package/out/candidate-schema.d.ts.map +1 -1
  104. package/out/candidate-schema.js +3 -0
  105. package/out/candidate-schema.js.map +1 -1
  106. package/out/candidate-scoring.d.ts +34 -0
  107. package/out/candidate-scoring.d.ts.map +1 -0
  108. package/out/candidate-scoring.js +200 -0
  109. package/out/candidate-scoring.js.map +1 -0
  110. package/out/capital-schema.d.ts +51 -0
  111. package/out/capital-schema.d.ts.map +1 -0
  112. package/out/capital-schema.js +63 -0
  113. package/out/capital-schema.js.map +1 -0
  114. package/out/capitals.d.ts +69 -0
  115. package/out/capitals.d.ts.map +1 -0
  116. package/out/capitals.js +98 -0
  117. package/out/capitals.js.map +1 -0
  118. package/out/coincident-roles.d.ts +7 -0
  119. package/out/coincident-roles.d.ts.map +1 -1
  120. package/out/coincident-roles.js +42 -8
  121. package/out/coincident-roles.js.map +1 -1
  122. package/out/convention-schema.d.ts +51 -0
  123. package/out/convention-schema.d.ts.map +1 -0
  124. package/out/convention-schema.js +34 -0
  125. package/out/convention-schema.js.map +1 -0
  126. package/out/convention.d.ts +1 -1
  127. package/out/convention.js +2 -2
  128. package/out/coverage-manifest-schema.js +3 -7
  129. package/out/coverage-manifest-schema.js.map +1 -1
  130. package/out/currency-backfill.d.ts +46 -0
  131. package/out/currency-backfill.d.ts.map +1 -0
  132. package/out/currency-backfill.js +180 -0
  133. package/out/currency-backfill.js.map +1 -0
  134. package/out/exact-match.d.ts +25 -0
  135. package/out/exact-match.d.ts.map +1 -0
  136. package/out/exact-match.js +89 -0
  137. package/out/exact-match.js.map +1 -0
  138. package/out/fst-autocomplete.d.ts +11 -11
  139. package/out/fst-autocomplete.d.ts.map +1 -1
  140. package/out/fst-autocomplete.js +82 -99
  141. package/out/fst-autocomplete.js.map +1 -1
  142. package/out/fst-builder.d.ts.map +1 -1
  143. package/out/fst-builder.js +11 -12
  144. package/out/fst-builder.js.map +1 -1
  145. package/out/fst-freshness.d.ts +2 -2
  146. package/out/fst-freshness.js +2 -2
  147. package/out/fts-query.js +1 -1
  148. package/out/fts-query.js.map +1 -1
  149. package/out/fts.d.ts +4 -4
  150. package/out/fts.js +4 -4
  151. package/out/geonames-postal.d.ts +2 -2
  152. package/out/geonames-postal.js +2 -2
  153. package/out/index.d.ts +3 -2
  154. package/out/index.d.ts.map +1 -1
  155. package/out/index.js +2 -2
  156. package/out/index.js.map +1 -1
  157. package/out/interpolation.d.ts +8 -0
  158. package/out/interpolation.d.ts.map +1 -1
  159. package/out/interpolation.js +91 -19
  160. package/out/interpolation.js.map +1 -1
  161. package/out/lookup.d.ts +4 -5
  162. package/out/lookup.d.ts.map +1 -1
  163. package/out/lookup.js +94 -468
  164. package/out/lookup.js.map +1 -1
  165. package/out/name-score.d.ts +0 -10
  166. package/out/name-score.d.ts.map +1 -1
  167. package/out/name-score.js +6 -4
  168. package/out/name-score.js.map +1 -1
  169. package/out/place-importance-schema.d.ts +42 -5
  170. package/out/place-importance-schema.d.ts.map +1 -1
  171. package/out/place-importance-schema.js +54 -8
  172. package/out/place-importance-schema.js.map +1 -1
  173. package/out/poi-lookup.d.ts +1 -1
  174. package/out/poi-lookup.d.ts.map +1 -1
  175. package/out/poi-lookup.js +12 -13
  176. package/out/poi-lookup.js.map +1 -1
  177. package/out/poi-schema.d.ts +7 -3
  178. package/out/poi-schema.d.ts.map +1 -1
  179. package/out/poi-schema.js.map +1 -1
  180. package/out/polygon-schema.d.ts +37 -0
  181. package/out/polygon-schema.d.ts.map +1 -0
  182. package/out/polygon-schema.js +23 -0
  183. package/out/polygon-schema.js.map +1 -0
  184. package/out/postal-city-alias-lookup.d.ts +1 -1
  185. package/out/postal-city-alias-lookup.js +1 -1
  186. package/out/postal-city-candidate-schema.d.ts +2 -1
  187. package/out/postal-city-candidate-schema.d.ts.map +1 -1
  188. package/out/postal-city-candidate-schema.js.map +1 -1
  189. package/out/postcode-point-lookup.d.ts +1 -1
  190. package/out/postcode-point-lookup.js +1 -1
  191. package/out/primary-preference.d.ts +125 -0
  192. package/out/primary-preference.d.ts.map +1 -0
  193. package/out/primary-preference.js +138 -0
  194. package/out/primary-preference.js.map +1 -0
  195. package/out/proximity-rerank.d.ts +77 -0
  196. package/out/proximity-rerank.d.ts.map +1 -0
  197. package/out/proximity-rerank.js +86 -0
  198. package/out/proximity-rerank.js.map +1 -0
  199. package/out/region-keys.d.ts +47 -0
  200. package/out/region-keys.d.ts.map +1 -0
  201. package/out/region-keys.js +121 -0
  202. package/out/region-keys.js.map +1 -0
  203. package/out/reverse.d.ts.map +1 -1
  204. package/out/reverse.js +6 -9
  205. package/out/reverse.js.map +1 -1
  206. package/out/schema.d.ts +1 -1
  207. package/out/search-fetch.d.ts +57 -0
  208. package/out/search-fetch.d.ts.map +1 -0
  209. package/out/search-fetch.js +183 -0
  210. package/out/search-fetch.js.map +1 -0
  211. package/out/sharding.d.ts +3 -3
  212. package/out/sharding.js +1 -1
  213. package/out/sqlite-convention-source.d.ts +1 -1
  214. package/out/sqlite-convention-source.js +1 -1
  215. package/out/sqlite-utils.d.ts +19 -1
  216. package/out/sqlite-utils.d.ts.map +1 -1
  217. package/out/sqlite-utils.js +19 -1
  218. package/out/sqlite-utils.js.map +1 -1
  219. package/out/street-centroid-schema.d.ts +7 -2
  220. package/out/street-centroid-schema.d.ts.map +1 -1
  221. package/out/street-centroid-schema.js.map +1 -1
  222. package/out/street-centroid.d.ts.map +1 -1
  223. package/out/street-centroid.js +7 -7
  224. package/out/street-centroid.js.map +1 -1
  225. package/out/street-normalize.d.ts +82 -9
  226. package/out/street-normalize.d.ts.map +1 -1
  227. package/out/street-normalize.js +175 -9
  228. package/out/street-normalize.js.map +1 -1
  229. package/out/street-segment-schema.d.ts +6 -2
  230. package/out/street-segment-schema.d.ts.map +1 -1
  231. package/out/street-segment-schema.js.map +1 -1
  232. package/out/types.d.ts +35 -1
  233. package/out/types.d.ts.map +1 -1
  234. package/out/unified-schema.d.ts +1 -1
  235. package/out/unified-schema.js +1 -1
  236. package/out/uprn-lookup.d.ts +85 -0
  237. package/out/uprn-lookup.d.ts.map +1 -0
  238. package/out/uprn-lookup.js +152 -0
  239. package/out/uprn-lookup.js.map +1 -0
  240. package/out/uprn-schema.d.ts +93 -0
  241. package/out/uprn-schema.d.ts.map +1 -0
  242. package/out/uprn-schema.js +78 -0
  243. package/out/uprn-schema.js.map +1 -0
  244. package/out/weights-overlay-linker.d.ts +141 -0
  245. package/out/weights-overlay-linker.d.ts.map +1 -0
  246. package/out/weights-overlay-linker.js +259 -0
  247. package/out/weights-overlay-linker.js.map +1 -0
  248. package/package.json +288 -16
  249. package/place-importance-schema.ts +64 -15
  250. package/poi-lookup.ts +12 -13
  251. package/poi-schema.ts +8 -3
  252. package/polygon-schema.ts +47 -0
  253. package/postal-city-alias-lookup.ts +1 -1
  254. package/postal-city-candidate-schema.ts +3 -1
  255. package/postcode-point-lookup.ts +1 -1
  256. package/primary-preference.ts +207 -0
  257. package/proximity-rerank.ts +120 -0
  258. package/region-keys.ts +144 -0
  259. package/reverse.ts +17 -16
  260. package/schema.ts +1 -1
  261. package/search-fetch.ts +256 -0
  262. package/sharding.ts +3 -3
  263. package/sqlite-convention-source.ts +1 -1
  264. package/sqlite-utils.ts +43 -2
  265. package/street-centroid-schema.ts +8 -2
  266. package/street-centroid.ts +13 -8
  267. package/street-normalize.ts +252 -23
  268. package/street-segment-schema.ts +7 -2
  269. package/types.ts +35 -1
  270. package/unified-schema.ts +1 -1
  271. package/uprn-lookup.ts +210 -0
  272. package/uprn-schema.ts +124 -0
  273. package/weights-overlay-linker.ts +377 -0
  274. package/geo.ts +0 -121
  275. package/out/geo.d.ts +0 -74
  276. package/out/geo.d.ts.map +0 -1
  277. package/out/geo.js +0 -71
  278. package/out/geo.js.map +0 -1
@@ -0,0 +1,206 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ * @file Pass 3b of the candidate build — the containment sidecar (closure rows + interval labels).
6
+ */
7
+
8
+ import type { DatabaseSync } from "node:sqlite"
9
+
10
+ import type { DatabaseClient } from "@mailwoman/core/kysley/client"
11
+
12
+ import { placetypeDepth } from "../ancestry.ts"
13
+ import {
14
+ CANDIDATE_ANCESTOR_COLUMNS,
15
+ CANDIDATE_ANCESTOR_TABLE,
16
+ CANDIDATE_INTERVAL_TABLE,
17
+ createCandidateAncestorTable,
18
+ createCandidateIntervalTable,
19
+ MAX_ANCESTOR_DEPTH,
20
+ } from "../candidate-ancestors-schema.ts"
21
+ import type { CandidateDatabase } from "../candidate-schema.ts"
22
+ import type { PlaceAttrs } from "./place-attrs.ts"
23
+
24
+ /**
25
+ * Pass 3b — the ancestors sidecar: closure rows + interval labels (candidate-ancestors-schema.ts owns the encoding
26
+ * decision and the DAG/absence semantics). Reads the same source `ancestors` table the region stamp reads,
27
+ * denormalizing each edge with the parent's name/key from `attrs`, streamed `ORDER BY id` so the clustered `(spr_id,
28
+ * depth)` insert is sorted — the contiguous-leaves discipline of the candidate table itself.
29
+ *
30
+ * Excluded by policy: self rows, and placetypes outside the containment ladder (continent, empire, …: `placetypeDepth`
31
+ * 0) — they discriminate nothing a consumer of this sidecar checks. An edge to a parent with no current `spr` row has
32
+ * no name to denormalize; it is dropped and counted rather than stored blind.
33
+ */
34
+ export async function buildAncestorsSidecar(ctx: {
35
+ src: DatabaseSync
36
+ out: DatabaseSync
37
+ kdb: DatabaseClient<CandidateDatabase>
38
+ attrs: Map<number, PlaceAttrs>
39
+ ptID: (pt: string | null) => number
40
+ progress: (phase: string, message: string) => void
41
+ }): Promise<{ ancestorRows: number; ancestorPlaces: number; intervalPlaces: number }> {
42
+ const { src, out, attrs, ptID, progress } = ctx
43
+
44
+ progress("ancestors", "building containment sidecar (closure rows + interval labels)")
45
+ await createCandidateAncestorTable(ctx.kdb)
46
+ await createCandidateIntervalTable(ctx.kdb)
47
+
48
+ const insAncestor = out.prepare(
49
+ `INSERT INTO ${CANDIDATE_ANCESTOR_TABLE} VALUES (${CANDIDATE_ANCESTOR_COLUMNS.map(() => "?").join(", ")})`
50
+ )
51
+
52
+ // The canonical-parent forest the interval labels are computed over. One parent per place — the
53
+ // depth-1 edge (finest containment tier, lowest ancestor id; the `regionOf` MIN-stability
54
+ // convention). ALL parents stay in the closure rows; only the interval tree canonicalizes.
55
+ const canonicalParentOf = new Map<number, number>()
56
+ const childrenOf = new Map<number, number[]>()
57
+ const forest = new Set<number>()
58
+
59
+ let ancestorRows = 0
60
+ let ancestorPlaces = 0
61
+ let droppedParents = 0
62
+
63
+ // Per-child edge buffer; the stream below is grouped by child id, so each flush owns one place.
64
+ let childID = -1
65
+ let edges: Array<{ aid: number; apt: string }> = []
66
+
67
+ const flush = (): void => {
68
+ if (childID < 0 || !edges.length) return
69
+
70
+ // Deterministic nearest-first: containment depth descending, then ancestor id ascending —
71
+ // the FTS backend's `ancestorLineage` ordering, made stable across rebuilds.
72
+ edges.sort((a, b) => placetypeDepth(b.apt) - placetypeDepth(a.apt) || a.aid - b.aid)
73
+
74
+ if (edges.length > MAX_ANCESTOR_DEPTH) {
75
+ edges = edges.slice(0, MAX_ANCESTOR_DEPTH)
76
+ }
77
+
78
+ ancestorPlaces++
79
+
80
+ for (const [i, edge] of edges.entries()) {
81
+ const parent = attrs.get(edge.aid)!
82
+
83
+ insAncestor.run(childID, i + 1, edge.aid, ptID(edge.apt), parent.name, parent.pkey)
84
+
85
+ ancestorRows++
86
+ }
87
+
88
+ const canonical = edges[0]!.aid
89
+
90
+ canonicalParentOf.set(childID, canonical)
91
+
92
+ const siblings = childrenOf.get(canonical)
93
+
94
+ if (siblings) {
95
+ siblings.push(childID)
96
+ } else {
97
+ childrenOf.set(canonical, [childID])
98
+ }
99
+
100
+ forest.add(childID)
101
+ forest.add(canonical)
102
+ }
103
+
104
+ out.exec("BEGIN")
105
+
106
+ for (const r of src
107
+ .prepare("SELECT id, ancestor_id, ancestor_placetype FROM ancestors WHERE ancestor_id != id ORDER BY id")
108
+ .iterate()) {
109
+ const id = Number(r.id)
110
+
111
+ if (id !== childID) {
112
+ flush()
113
+ childID = id
114
+ edges = []
115
+ }
116
+
117
+ if (!attrs.has(id)) continue
118
+
119
+ const apt = String(r.ancestor_placetype ?? "")
120
+
121
+ if (placetypeDepth(apt) === 0) continue
122
+
123
+ const aid = Number(r.ancestor_id)
124
+
125
+ if (!attrs.has(aid)) {
126
+ droppedParents++
127
+
128
+ continue
129
+ }
130
+
131
+ edges.push({ aid, apt })
132
+ }
133
+
134
+ flush()
135
+ out.exec("COMMIT")
136
+
137
+ // Interval labels: pre/post-order DFS over the canonical-parent forest. Root order and child
138
+ // order are id-ascending so the labels are stable across rebuilds of the same source.
139
+ const preOf = new Map<number, number>()
140
+ const postOf = new Map<number, number>()
141
+
142
+ for (const kids of childrenOf.values()) {
143
+ // oxlint-disable-next-line unicorn/no-array-sort -- sorts an array this pass just built
144
+ kids.sort((a, b) => a - b)
145
+ }
146
+
147
+ const roots = [...forest].filter((id) => !canonicalParentOf.has(id))
148
+
149
+ // oxlint-disable-next-line unicorn/no-array-sort -- sorts an array this pass just built
150
+ roots.sort((a, b) => a - b)
151
+
152
+ let counter = 0
153
+
154
+ for (const root of roots) {
155
+ preOf.set(root, counter++)
156
+ const stack: Array<{ id: number; next: number }> = [{ id: root, next: 0 }]
157
+
158
+ while (stack.length) {
159
+ const top = stack.at(-1)!
160
+ const kids = childrenOf.get(top.id)
161
+
162
+ if (kids && top.next < kids.length) {
163
+ const kid = kids[top.next++]!
164
+
165
+ // Each child holds exactly one canonical parent, so a labeled node here means the
166
+ // grouping upstream broke — skip rather than corrupt the numbering.
167
+ if (preOf.has(kid)) continue
168
+
169
+ preOf.set(kid, counter++)
170
+ stack.push({ id: kid, next: 0 })
171
+ } else {
172
+ postOf.set(top.id, counter++)
173
+ stack.pop()
174
+ }
175
+ }
176
+ }
177
+
178
+ // A canonical-parent CYCLE (corrupt source ancestry) leaves its members unreachable from any
179
+ // root: they simply receive no label, and containment against them reads unverifiable — the
180
+ // absence semantics the schema module states. Counted so a jump is visible across rebuilds.
181
+ const cycleSkipped = forest.size - preOf.size
182
+
183
+ const insInterval = out.prepare(`INSERT INTO ${CANDIDATE_INTERVAL_TABLE} VALUES (?, ?, ?)`)
184
+ const labeled = [...preOf.keys()]
185
+
186
+ // oxlint-disable-next-line unicorn/no-array-sort -- sorts an array this pass just built
187
+ labeled.sort((a, b) => a - b)
188
+
189
+ out.exec("BEGIN")
190
+
191
+ for (const id of labeled) {
192
+ insInterval.run(id, preOf.get(id)!, postOf.get(id)!)
193
+ }
194
+
195
+ out.exec("COMMIT")
196
+
197
+ progress(
198
+ "ancestors",
199
+ `${ancestorRows.toLocaleString()} closure rows across ${ancestorPlaces.toLocaleString()} places; ` +
200
+ `${preOf.size.toLocaleString()} interval labels` +
201
+ (droppedParents ? `; ${droppedParents.toLocaleString()} edges dropped (parent has no current spr row)` : "") +
202
+ (cycleSkipped ? `; ${cycleSkipped.toLocaleString()} places skipped (canonical-parent cycle)` : "")
203
+ )
204
+
205
+ return { ancestorRows, ancestorPlaces, intervalPlaces: preOf.size }
206
+ }
@@ -0,0 +1,79 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ * @file Pass 1b of the candidate build — fold ICU's country display names onto the country rows.
6
+ */
7
+
8
+ import { enumerateCountryDisplayNames } from "@mailwoman/codex/country"
9
+
10
+ import { normalizeLocalityForKey } from "../street-normalize.ts"
11
+ import type { PlaceAttrs, StageRow } from "./place-attrs.ts"
12
+
13
+ /**
14
+ * Fold every country surface ICU knows onto that country's candidate row (#1678 thread 1).
15
+ *
16
+ * A bare `格鲁吉亚` (Georgia the country) resolved to NOTHING while `佐治亚州` (Georgia the US state) resolved correctly, and
17
+ * the model gave both the same wrong `locality` tag — so the tag was never the variable. Measured 2026-08-15: 140 of
18
+ * 237 country rows are synthetic and carry a canonical English name and nothing else; WOF holds no Chinese country
19
+ * names at all; and the GeoNames alias fold filters through a Latin-script regex, so neither existing source could ever
20
+ * supply them.
21
+ *
22
+ * `Intl.DisplayNames` already knows every one — ~280 regions, ~5,244 surfaces, from the same ICU the runtime uses for
23
+ * every other locale-sensitive operation. No download, no vendored corpus, no snapshot to drift.
24
+ *
25
+ * `is_primary = 0`: these are NAMES THE WORLD USES, not the country's canonical name. The display `name` stays whatever
26
+ * the gazetteer already had, so resolving `格鲁吉亚` answers with the Georgia country row rather than renaming it.
27
+ *
28
+ * Returns the row count so the caller can report it — a zero means ICU supplied nothing, which is a different fact from
29
+ * the pass not having run.
30
+ */
31
+ export function stageCountryDisplayNames(ctx: {
32
+ attrs: Map<number, PlaceAttrs>
33
+ iso2ByID: Map<number, string>
34
+ countryPtID: number
35
+ stageRow: StageRow
36
+ tx: { exec(sql: string): void }
37
+ }): number {
38
+ // One country row per ISO2. Where a code has several (historic rows surviving the is_current filter), the most
39
+ // populous wins — the same tiebreak the ranking uses everywhere else.
40
+ const countryByISO2 = new Map<string, { sid: number; a: PlaceAttrs }>()
41
+
42
+ for (const [sid, a] of ctx.attrs) {
43
+ if (a.ptid !== ctx.countryPtID) continue
44
+
45
+ const iso2 = ctx.iso2ByID.get(a.cid)
46
+
47
+ if (!iso2 || iso2 === "??") continue
48
+
49
+ const held = countryByISO2.get(iso2)
50
+
51
+ if (!held || a.pop > held.a.pop) {
52
+ countryByISO2.set(iso2, { sid, a })
53
+ }
54
+ }
55
+
56
+ let staged = 0
57
+
58
+ ctx.tx.exec("BEGIN")
59
+
60
+ for (const { iso2, name } of enumerateCountryDisplayNames()) {
61
+ const target = countryByISO2.get(iso2)
62
+
63
+ if (!target) continue
64
+
65
+ const k = normalizeLocalityForKey(name)
66
+
67
+ // The country's own key is already staged as its primary; INSERT OR IGNORE at materialization dedupes the
68
+ // rest, so this only skips the obvious self-alias.
69
+ if (!k || k === target.a.pkey) continue
70
+
71
+ ctx.stageRow(k, target.a, target.sid, 0)
72
+
73
+ staged++
74
+ }
75
+
76
+ ctx.tx.exec("COMMIT")
77
+
78
+ return staged
79
+ }
@@ -0,0 +1,237 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ * @file Pass 3c of the candidate build — the `name_role` detectors and the cuts they are judged by.
6
+ */
7
+
8
+ import type { DatabaseSync } from "node:sqlite"
9
+
10
+ import { isOfficialLanguage } from "@mailwoman/codex/country"
11
+
12
+ import { normalizeLocalityForKey } from "../street-normalize.ts"
13
+ import { isOwnNameVariant } from "./own-name.ts"
14
+ import type { PlaceAttrs } from "./place-attrs.ts"
15
+
16
+ /**
17
+ * Key-count cut for the gloss anomaly detector (#1730) — the sweep's own boundary: 4,000 places carried >= 50 keys, and
18
+ * a legitimate famous place at that count (New York, 176 keys) is separated by the PROMINENCE gate, never by this
19
+ * number alone.
20
+ */
21
+ export const GLOSS_KEY_THRESHOLD = 50
22
+
23
+ /**
24
+ * Placetypes the gloss detector never flags. A country or region legitimately carries a name in every language — that
25
+ * is what an exonym set IS — so key volume discriminates nothing there. The detector's population is the non-admin
26
+ * tail, where a place named by a common noun ("Poisson", "Sunday") accumulating 200+ translations is a
27
+ * machine-translated gloss set, not fame.
28
+ */
29
+ export const GLOSS_EXCLUDED_PLACETYPES: ReadonlySet<string> = new Set([
30
+ "country",
31
+ "dependency",
32
+ "disputed",
33
+ "empire",
34
+ "macroregion",
35
+ "region",
36
+ "macrocounty",
37
+ "county",
38
+ ])
39
+
40
+ /**
41
+ * Pass 3c — the #1730 name-role prototype: two independent detectors over the staged rows, WRITE-ONLY in this
42
+ * generation (no ranking consumer; the rank penalty is its own D-rule-gated step with the `gloss_key` board as
43
+ * tripwire). Both stamp `is_primary = 0` rows only — a place's canonical name and the `place_abbr` region abbreviations
44
+ * are never a gloss or a variant.
45
+ *
46
+ * - `gloss` is ANOMALY-based, and stamps only the certain core: key volume at/over the threshold + a non-admin placetype
47
+ *
48
+ * - NO measured prominence (population absent AND importance unmeasured). Provenance cannot separate a gloss from an
49
+ * exonym — WOF imported both as `x_preferred` — and prominence is what rescues New York/Paris.
50
+ * - `abbr` is PROVENANCE-based — the #936 signal: a WOF `variant` name in one of the country's official languages (or
51
+ * English), measured there at a 13× key-collision rate. A source without a `names` table (fixture-scale admin DBs)
52
+ * skips this detector loudly.
53
+ *
54
+ * Returns the stamp counts plus the census the prototype exists to report: how much of the ≥-threshold key tail carries
55
+ * any role.
56
+ */
57
+ export function stampNameRoles(ctx: {
58
+ src: DatabaseSync
59
+ out: DatabaseSync
60
+ attrs: Map<number, PlaceAttrs>
61
+ keyCounts: Map<number, number>
62
+ glossThreshold: number
63
+ ptcodes: Map<string, number>
64
+ ccodes: Map<string, number>
65
+ progress: (phase: string, message: string) => void
66
+ }): { roleGloss: number; roleAbbr: number; roleVariant: number; keyTailPlaces: number; keyTailWithRole: number } {
67
+ const { src, out, attrs, keyCounts, glossThreshold, progress } = ctx
68
+ progress("roles", "stamping name roles (gloss anomaly + abbr provenance)")
69
+
70
+ const ptNameByID = new Map([...ctx.ptcodes].map(([placetype, id]) => [id, placetype]))
71
+ const iso2ByCID = new Map([...ctx.ccodes].map(([code, id]) => [id, code]))
72
+
73
+ let keyTailPlaces = 0
74
+ const glossSids: number[] = []
75
+
76
+ for (const [sid, count] of keyCounts) {
77
+ if (count < glossThreshold) continue
78
+
79
+ keyTailPlaces++
80
+ const a = attrs.get(sid)
81
+
82
+ if (!a) continue
83
+
84
+ if (GLOSS_EXCLUDED_PLACETYPES.has(ptNameByID.get(a.ptid) ?? "")) continue
85
+
86
+ if (a.pop > 0 || a.imp != null) continue
87
+ glossSids.push(sid)
88
+ }
89
+
90
+ out.exec(
91
+ "CREATE TEMP TABLE role_key (spr_id INTEGER NOT NULL, name_key TEXT NOT NULL, PRIMARY KEY (spr_id, name_key)) WITHOUT ROWID"
92
+ )
93
+
94
+ const insRoleKey = out.prepare("INSERT OR IGNORE INTO role_key VALUES (?, ?)")
95
+
96
+ // Zero abbr stamps from a skipped detector is a different fact from zero variants found.
97
+ const hasSourceNames =
98
+ src.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='names'").get() !== undefined
99
+
100
+ if (hasSourceNames) {
101
+ out.exec("BEGIN")
102
+
103
+ // Two provenance routes into the same stamp: WOF's abbreviation/short name KINDS arrive in the
104
+ // LANGUAGE column ('abbr'/'short' — 280 rows, measured 2026-08-18, Toledo's 'TO' among them) and
105
+ // qualify by kind alone; everything else qualifies as a variant in an official language.
106
+ for (const r of src
107
+ .prepare("SELECT id, name, language FROM names WHERE privateuse = 'variant' OR language IN ('abbr', 'short')")
108
+ .iterate()) {
109
+ const a = attrs.get(Number(r.id))
110
+
111
+ if (!a) continue
112
+ const language = String(r.language ?? "")
113
+
114
+ if (language !== "abbr" && language !== "short") {
115
+ const iso2 = iso2ByCID.get(a.cid) ?? "??"
116
+
117
+ if (language !== "eng" && !isOfficialLanguage(iso2, language)) continue
118
+ }
119
+
120
+ const k = normalizeLocalityForKey(String(r.name ?? ""))
121
+
122
+ if (!k || k === a.pkey) continue
123
+ insRoleKey.run(Number(r.id), k)
124
+ }
125
+
126
+ out.exec("COMMIT")
127
+ } else {
128
+ progress("roles", "source carries no `names` table — abbr detector skipped (gloss still runs)")
129
+ }
130
+
131
+ // Stamp order is precedence: the provenance-based abbr first, then the own-name variant verdict,
132
+ // then gloss fills what neither claimed.
133
+ const roleAbbr = Number(
134
+ out
135
+ .prepare(
136
+ `UPDATE cand_stage SET name_role = 'abbr'
137
+ WHERE is_primary = 0 AND EXISTS (
138
+ SELECT 1 FROM role_key rk WHERE rk.spr_id = cand_stage.spr_id AND rk.name_key = cand_stage.name_key)`
139
+ )
140
+ .run().changes
141
+ )
142
+
143
+ // --- variant detector (#1882): the alias surface is the holder's OWN primary name in another
144
+ // orthography — romanization, spacing/diacritic variant, or abbreviation expansion. The verdict
145
+ // is per (alias key, primary key) pair, so it runs in JS over the still-unstamped alias rows;
146
+ // an uncovered script answers no-verdict and stamps nothing (own-name.ts owns the predicate and
147
+ // its measured threshold). Runs BEFORE gloss on purpose: a surface that IS the place's own name
148
+ // is not a translation, whatever the key volume says.
149
+ out.exec(
150
+ "CREATE TEMP TABLE variant_key (spr_id INTEGER NOT NULL, name_key TEXT NOT NULL, PRIMARY KEY (spr_id, name_key)) WITHOUT ROWID"
151
+ )
152
+
153
+ const insVariantKey = out.prepare("INSERT OR IGNORE INTO variant_key VALUES (?, ?)")
154
+ let variantScanned = 0
155
+
156
+ out.exec("BEGIN")
157
+
158
+ for (const r of out
159
+ .prepare("SELECT DISTINCT spr_id, name_key FROM cand_stage WHERE is_primary = 0 AND name_role IS NULL")
160
+ .iterate()) {
161
+ variantScanned++
162
+ const a = attrs.get(Number(r.spr_id))
163
+
164
+ if (!a?.pkey) continue
165
+
166
+ if (isOwnNameVariant(String(a.pkey), String(r.name_key))) {
167
+ insVariantKey.run(Number(r.spr_id), String(r.name_key))
168
+ }
169
+ }
170
+
171
+ out.exec("COMMIT")
172
+
173
+ const roleVariant = Number(
174
+ out
175
+ .prepare(
176
+ `UPDATE cand_stage SET name_role = 'variant'
177
+ WHERE is_primary = 0 AND name_role IS NULL AND EXISTS (
178
+ SELECT 1 FROM variant_key vk WHERE vk.spr_id = cand_stage.spr_id AND vk.name_key = cand_stage.name_key)`
179
+ )
180
+ .run().changes
181
+ )
182
+
183
+ out.exec("DROP TABLE variant_key")
184
+
185
+ progress(
186
+ "roles",
187
+ `variant: ${roleVariant.toLocaleString()} of ${variantScanned.toLocaleString()} unstamped alias keys are own-name variants`
188
+ )
189
+
190
+ out.exec("CREATE TEMP TABLE role_place (spr_id INTEGER PRIMARY KEY) WITHOUT ROWID")
191
+ out.exec("CREATE TEMP TABLE key_tail (spr_id INTEGER PRIMARY KEY) WITHOUT ROWID")
192
+ const insRolePlace = out.prepare("INSERT OR IGNORE INTO role_place VALUES (?)")
193
+ const insKeyTail = out.prepare("INSERT OR IGNORE INTO key_tail VALUES (?)")
194
+ out.exec("BEGIN")
195
+
196
+ for (const sid of glossSids) {
197
+ insRolePlace.run(sid)
198
+ }
199
+
200
+ for (const [sid, count] of keyCounts) {
201
+ if (count >= glossThreshold) {
202
+ insKeyTail.run(sid)
203
+ }
204
+ }
205
+
206
+ out.exec("COMMIT")
207
+
208
+ const roleGloss = Number(
209
+ out
210
+ .prepare(
211
+ `UPDATE cand_stage SET name_role = 'gloss'
212
+ WHERE is_primary = 0 AND name_role IS NULL AND spr_id IN (SELECT spr_id FROM role_place)`
213
+ )
214
+ .run().changes
215
+ )
216
+
217
+ const keyTailWithRole = Number(
218
+ out
219
+ .prepare(
220
+ `SELECT count(DISTINCT spr_id) AS n FROM cand_stage
221
+ WHERE name_role IS NOT NULL AND spr_id IN (SELECT spr_id FROM key_tail)`
222
+ )
223
+ .get()!["n"]
224
+ )
225
+
226
+ out.exec("DROP TABLE role_key")
227
+ out.exec("DROP TABLE role_place")
228
+ out.exec("DROP TABLE key_tail")
229
+
230
+ progress(
231
+ "roles",
232
+ `${roleAbbr.toLocaleString()} abbr + ${roleGloss.toLocaleString()} gloss rows stamped; ` +
233
+ `key tail (>= ${glossThreshold} keys): ${keyTailWithRole.toLocaleString()} of ${keyTailPlaces.toLocaleString()} places carry a role`
234
+ )
235
+
236
+ return { roleGloss, roleAbbr, roleVariant, keyTailPlaces, keyTailWithRole }
237
+ }
@@ -0,0 +1,146 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * The OWN-NAME VARIANT predicate (#1882): is an alias surface the holder's own primary name in
7
+ * another orthography — a romanization (`Брэст` → `brest`), a spacing/diacritic variant
8
+ * (`George Town` → `georgetown`), an abbreviation expansion (`St. George's` → `Saint George's`)
9
+ * — rather than a different name that merely shares the folded key?
10
+ *
11
+ * The comparator is `levenshteinSimilarity`, NOT Jaro-Winkler: JW's common-prefix bonus scores
12
+ * the load-bearing NEGATIVE case (`chanchun` vs `cancun`, 0.925) above real positives
13
+ * (`saint george s` vs `st georges` expanded, 0.914), so no JW threshold separates them.
14
+ * Measured on the #1882 census contests, edit similarity separates every case with margin:
15
+ *
16
+ * - IN: `brest`/`brest` 1.0 · `george town`/`georgetown` 0.909 · `saint george s`/`saint georges`
17
+ * 0.929 · `adamovka`/`adamowka` 0.875
18
+ * - OUT: `lievin`/`levin` 0.833 (Liévin FR vs Levin NZ — different places with near-identical
19
+ * names; the panel's `41 Weraroa Road, Levin` row needs this side) · `chanchun`/`cancun` 0.75 ·
20
+ * `augsburg`/`augusta` 0.375 · `west bay`/`west end` 0.625 · `derry`/`londonderry` 0.455
21
+ * (Derry/Londonderry is a DUAL NAME, not a variant — its own follow-up on #1882)
22
+ *
23
+ * An unhandled script (Arabic, Hebrew, CJK — the romanizer covers Cyrillic only) answers NULL,
24
+ * never "different name": absence of a verdict must not stamp anything (the meaning-of-zero rule).
25
+ * The #1882 census counted 396 demoted Arabic/Hebrew-primary aliases left unclassified by this.
26
+ */
27
+
28
+ import { levenshteinSimilarity } from "@mailwoman/match/comparators"
29
+
30
+ /**
31
+ * Edit-similarity floor for the variant verdict. The measured band: nearest admitted pair 0.875
32
+ * (`adamovka`/`adamowka`), nearest refused pair 0.833 (`lievin`/`levin`).
33
+ */
34
+ export const VARIANT_SIMILARITY_MIN = 0.85
35
+
36
+ /**
37
+ * BGN/PCGN-flavored Cyrillic romanization, folded to the name-key alphabet. Digraph outputs (zh, kh, ts, ch, sh, shch,
38
+ * yu, ya) match the dominant transliteration conventions the gazetteer's Latin aliases actually use; the `w`/`v` and
39
+ * `kh`/`h` style variance between systems is what the edit-similarity threshold absorbs.
40
+ */
41
+ const CYRILLIC_TO_LATIN: Record<string, string> = {
42
+ а: "a",
43
+ б: "b",
44
+ в: "v",
45
+ г: "g",
46
+ д: "d",
47
+ е: "e",
48
+ ё: "e",
49
+ ж: "zh",
50
+ з: "z",
51
+ и: "i",
52
+ і: "i",
53
+ й: "i",
54
+ к: "k",
55
+ л: "l",
56
+ м: "m",
57
+ н: "n",
58
+ о: "o",
59
+ п: "p",
60
+ р: "r",
61
+ с: "s",
62
+ т: "t",
63
+ у: "u",
64
+ ў: "u",
65
+ ф: "f",
66
+ х: "kh",
67
+ ц: "ts",
68
+ ч: "ch",
69
+ ш: "sh",
70
+ щ: "shch",
71
+ ъ: "",
72
+ ы: "y",
73
+ ь: "",
74
+ э: "e",
75
+ ю: "yu",
76
+ я: "ya",
77
+ ґ: "g",
78
+ є: "e",
79
+ ї: "i",
80
+ }
81
+
82
+ /**
83
+ * Leading-word abbreviations expanded BEFORE comparison, so `st georges` meets `saint george s` inside the edit
84
+ * threshold. Whole-word only — `st` inside `stanley` never expands.
85
+ */
86
+ const NAME_ABBREVIATIONS: ReadonlyArray<[RegExp, string]> = [
87
+ [/\bst\b/g, "saint"],
88
+ [/\bste\b/g, "sainte"],
89
+ [/\bmt\b/g, "mount"],
90
+ [/\bft\b/g, "fort"],
91
+ ]
92
+
93
+ /**
94
+ * Expand the whole-word abbreviations above.
95
+ */
96
+ export function expandNameAbbreviations(key: string): string {
97
+ let out = key
98
+
99
+ for (const [pattern, replacement] of NAME_ABBREVIATIONS) {
100
+ out = out.replaceAll(pattern, replacement)
101
+ }
102
+
103
+ return out
104
+ }
105
+
106
+ /**
107
+ * Romanize a folded name key to the a–z0–9/space alphabet. `null` when characters outside the covered scripts remain —
108
+ * an unhandled script is NO VERDICT, not a mismatch.
109
+ */
110
+ export function romanizeNameKey(key: string): string | null {
111
+ let out = ""
112
+
113
+ for (const ch of key.toLowerCase()) {
114
+ out += CYRILLIC_TO_LATIN[ch] ?? ch
115
+ }
116
+
117
+ out = out
118
+ .normalize("NFD")
119
+ .replaceAll(/[̀-ͯ]/g, "")
120
+ .replaceAll(/\s+/g, " ")
121
+ .trim()
122
+
123
+ return /^[a-z0-9 ]*$/.test(out) ? out : null
124
+ }
125
+
126
+ /**
127
+ * Edit similarity between a holder's primary name key and one of its alias keys, both romanized and
128
+ * abbreviation-expanded. `null` when either side's script is uncovered.
129
+ */
130
+ export function ownNameSimilarity(primaryKey: string, aliasKey: string): number | null {
131
+ const primary = romanizeNameKey(primaryKey)
132
+ const alias = romanizeNameKey(aliasKey)
133
+
134
+ if (primary == null || alias == null || !primary || !alias) return null
135
+
136
+ return levenshteinSimilarity(expandNameAbbreviations(primary), expandNameAbbreviations(alias))
137
+ }
138
+
139
+ /**
140
+ * The stamp predicate: the alias surface is the holder's own name in another orthography.
141
+ */
142
+ export function isOwnNameVariant(primaryKey: string, aliasKey: string): boolean {
143
+ const similarity = ownNameSimilarity(primaryKey, aliasKey)
144
+
145
+ return similarity != null && similarity >= VARIANT_SIMILARITY_MIN
146
+ }