@mailwoman/resolver-wof-sqlite 10.7.0 → 10.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/lib/address/point/interpolation.ts +4 -4
  2. package/lib/address/point/spine-schema.ts +59 -0
  3. package/lib/ancestry/backfill.ts +3 -25
  4. package/lib/build/candidate.ts +22 -5
  5. package/lib/candidate/alias-bags.ts +32 -14
  6. package/lib/candidate/bare-admin-names.ts +54 -0
  7. package/lib/candidate/importance.ts +2 -2
  8. package/lib/candidate/lookup.ts +54 -26
  9. package/lib/candidate/own-name.ts +1 -5
  10. package/lib/candidate/scoring.ts +4 -4
  11. package/lib/capitals.ts +2 -2
  12. package/lib/coincident-roles.ts +3 -3
  13. package/lib/currency-backfill.ts +4 -4
  14. package/lib/fst/autocomplete.ts +1 -1
  15. package/lib/fst/binary-matcher.ts +379 -0
  16. package/lib/fst/builder.ts +1 -1
  17. package/lib/fst/deserialize-web.ts +14 -143
  18. package/lib/fst/matcher.ts +3 -3
  19. package/lib/fst/serialize.ts +1 -1
  20. package/lib/fst.ts +1 -0
  21. package/lib/geonames/aliases.ts +2 -2
  22. package/lib/interpolation.ts +3 -2
  23. package/lib/lookup.ts +6 -6
  24. package/lib/nsul/lookup.ts +2 -1
  25. package/lib/nsul/schema.ts +2 -8
  26. package/lib/place-importance-schema.ts +1 -1
  27. package/lib/poi/lookup.ts +14 -14
  28. package/lib/proximity-rerank.ts +2 -2
  29. package/lib/reverse.ts +8 -5
  30. package/lib/search-fetch.ts +3 -3
  31. package/lib/sqlite/convention-source.ts +1 -1
  32. package/lib/street/centroid.ts +2 -2
  33. package/lib/street/morphology/fst/builder.ts +3 -3
  34. package/lib/street/morphology/fst/loader.ts +3 -3
  35. package/lib/street/normalize.ts +59 -1
  36. package/lib/types.ts +8 -0
  37. package/lib/uprn/existence.ts +0 -1
  38. package/lib/uprn/lookup.ts +5 -5
  39. package/lib/uprn/schema.ts +3 -4
  40. package/lib/weights-overlay-linker.ts +4 -4
  41. package/out/address/point/interpolation.js +4 -4
  42. package/out/address/point/interpolation.js.map +1 -1
  43. package/out/address/point/spine-schema.d.ts +40 -0
  44. package/out/address/point/spine-schema.d.ts.map +1 -0
  45. package/out/address/point/spine-schema.js +40 -0
  46. package/out/address/point/spine-schema.js.map +1 -0
  47. package/out/ancestry/backfill.d.ts.map +1 -1
  48. package/out/ancestry/backfill.js +3 -22
  49. package/out/ancestry/backfill.js.map +1 -1
  50. package/out/build/candidate.d.ts.map +1 -1
  51. package/out/build/candidate.js +17 -5
  52. package/out/build/candidate.js.map +1 -1
  53. package/out/candidate/alias-bags.d.ts +8 -0
  54. package/out/candidate/alias-bags.d.ts.map +1 -1
  55. package/out/candidate/alias-bags.js +20 -11
  56. package/out/candidate/alias-bags.js.map +1 -1
  57. package/out/candidate/bare-admin-names.d.ts +26 -0
  58. package/out/candidate/bare-admin-names.d.ts.map +1 -0
  59. package/out/candidate/bare-admin-names.js +38 -0
  60. package/out/candidate/bare-admin-names.js.map +1 -0
  61. package/out/candidate/importance.js +2 -2
  62. package/out/candidate/importance.js.map +1 -1
  63. package/out/candidate/lookup.d.ts.map +1 -1
  64. package/out/candidate/lookup.js +35 -19
  65. package/out/candidate/lookup.js.map +1 -1
  66. package/out/candidate/own-name.d.ts.map +1 -1
  67. package/out/candidate/own-name.js +1 -5
  68. package/out/candidate/own-name.js.map +1 -1
  69. package/out/candidate/scoring.js +4 -4
  70. package/out/candidate/scoring.js.map +1 -1
  71. package/out/capitals.js +2 -2
  72. package/out/capitals.js.map +1 -1
  73. package/out/coincident-roles.js +3 -3
  74. package/out/coincident-roles.js.map +1 -1
  75. package/out/currency-backfill.js +4 -4
  76. package/out/currency-backfill.js.map +1 -1
  77. package/out/env.d.ts +3 -0
  78. package/out/env.d.ts.map +1 -1
  79. package/out/fst/autocomplete.js +1 -1
  80. package/out/fst/autocomplete.js.map +1 -1
  81. package/out/fst/binary-matcher.d.ts +34 -0
  82. package/out/fst/binary-matcher.d.ts.map +1 -0
  83. package/out/fst/binary-matcher.js +294 -0
  84. package/out/fst/binary-matcher.js.map +1 -0
  85. package/out/fst/builder.js +1 -1
  86. package/out/fst/builder.js.map +1 -1
  87. package/out/fst/deserialize-web.d.ts +11 -6
  88. package/out/fst/deserialize-web.d.ts.map +1 -1
  89. package/out/fst/deserialize-web.js +12 -94
  90. package/out/fst/deserialize-web.js.map +1 -1
  91. package/out/fst/matcher.js +3 -3
  92. package/out/fst/matcher.js.map +1 -1
  93. package/out/fst/serialize.js +1 -1
  94. package/out/fst/serialize.js.map +1 -1
  95. package/out/fst.d.ts +1 -0
  96. package/out/fst.d.ts.map +1 -1
  97. package/out/fst.js +1 -0
  98. package/out/fst.js.map +1 -1
  99. package/out/geonames/aliases.js +2 -2
  100. package/out/geonames/aliases.js.map +1 -1
  101. package/out/interpolation.d.ts.map +1 -1
  102. package/out/interpolation.js +3 -2
  103. package/out/interpolation.js.map +1 -1
  104. package/out/lookup.d.ts +2 -2
  105. package/out/lookup.d.ts.map +1 -1
  106. package/out/lookup.js +4 -4
  107. package/out/lookup.js.map +1 -1
  108. package/out/nsul/lookup.d.ts +1 -1
  109. package/out/nsul/lookup.d.ts.map +1 -1
  110. package/out/nsul/lookup.js +1 -1
  111. package/out/nsul/lookup.js.map +1 -1
  112. package/out/nsul/schema.d.ts +2 -5
  113. package/out/nsul/schema.d.ts.map +1 -1
  114. package/out/nsul/schema.js +2 -7
  115. package/out/nsul/schema.js.map +1 -1
  116. package/out/place-importance-schema.js +1 -1
  117. package/out/place-importance-schema.js.map +1 -1
  118. package/out/poi/lookup.d.ts +3 -3
  119. package/out/poi/lookup.d.ts.map +1 -1
  120. package/out/poi/lookup.js +9 -9
  121. package/out/poi/lookup.js.map +1 -1
  122. package/out/proximity-rerank.js +2 -2
  123. package/out/proximity-rerank.js.map +1 -1
  124. package/out/reverse.d.ts.map +1 -1
  125. package/out/reverse.js +7 -5
  126. package/out/reverse.js.map +1 -1
  127. package/out/search-fetch.js +3 -3
  128. package/out/search-fetch.js.map +1 -1
  129. package/out/sqlite/convention-source.js +1 -1
  130. package/out/sqlite/convention-source.js.map +1 -1
  131. package/out/street/centroid.js +2 -2
  132. package/out/street/centroid.js.map +1 -1
  133. package/out/street/morphology/fst/builder.d.ts +2 -2
  134. package/out/street/morphology/fst/builder.js +2 -2
  135. package/out/street/morphology/fst/builder.js.map +1 -1
  136. package/out/street/morphology/fst/loader.d.ts +2 -2
  137. package/out/street/morphology/fst/loader.js +2 -2
  138. package/out/street/morphology/fst/loader.js.map +1 -1
  139. package/out/street/normalize.d.ts +16 -0
  140. package/out/street/normalize.d.ts.map +1 -1
  141. package/out/street/normalize.js +51 -1
  142. package/out/street/normalize.js.map +1 -1
  143. package/out/types.d.ts +7 -0
  144. package/out/types.d.ts.map +1 -1
  145. package/out/uprn/existence.d.ts.map +1 -1
  146. package/out/uprn/existence.js.map +1 -1
  147. package/out/uprn/lookup.d.ts.map +1 -1
  148. package/out/uprn/lookup.js +3 -3
  149. package/out/uprn/lookup.js.map +1 -1
  150. package/out/uprn/schema.d.ts +1 -1
  151. package/out/uprn/schema.d.ts.map +1 -1
  152. package/out/uprn/schema.js +1 -1
  153. package/out/uprn/schema.js.map +1 -1
  154. package/out/weights-overlay-linker.js +4 -4
  155. package/out/weights-overlay-linker.js.map +1 -1
  156. package/package.json +16 -13
@@ -0,0 +1,379 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * An FST matcher that answers from the serialized binary in place.
7
+ *
8
+ * {@link FSTMatcher} holds one `Map` per state and one object per place entry. For a national gazetteer those
9
+ * objects take several times the size of the file. This matcher keeps the file's bytes and two typed-array hash tables: one from
10
+ * an edge token to its string index, and one from a (state, string index) pair to the edge's target state. Place
11
+ * entries are decoded when a query reaches their state.
12
+ */
13
+
14
+ import type { AncestrieMatch } from "@mailwoman/ancestrie"
15
+ import { hashFNV1a } from "@mailwoman/core/coarse-placer/fnv-hash"
16
+
17
+ import {
18
+ EDGE_ENTRY_SIZE,
19
+ ENCYCLOPEDIC_OFFSET,
20
+ FST_FORMAT_VERSION,
21
+ FST_MAGIC_BYTES,
22
+ HEADER_SIZE,
23
+ LEGACY_PLACE_ENTRY_SIZE,
24
+ NARROW_STATE_ENTRY_SIZE,
25
+ PLACE_FLAG_HAS_ENCYCLOPEDIC,
26
+ PLACETYPE_ORDER,
27
+ SPLIT_PLACE_ENTRY_SIZE,
28
+ VERSION_TWO_SCORE_SPLIT,
29
+ VERSION_WIDE_STATE_COUNTERS,
30
+ WIDE_STATE_ENTRY_SIZE,
31
+ } from "#fst/format"
32
+ import { normalizeTokens } from "#fst/matcher"
33
+ import type { FSTContinuation, FSTQueryResult, PlaceEntry } from "#fst/types"
34
+
35
+ const EMPTY_SLOT = -1
36
+
37
+ /**
38
+ * Returns the smallest power of two at least twice `count`, so each open-addressing
39
+ * table stays at most half full.
40
+ */
41
+ function tableCapacity(count: number): number {
42
+ let capacity = 16
43
+
44
+ while (capacity < count * 2) {
45
+ capacity *= 2
46
+ }
47
+
48
+ return capacity
49
+ }
50
+
51
+ /**
52
+ * Mixes a state and a string index into one 32-bit hash.
53
+ */
54
+ function edgeHash(stateID: number, stringIndex: number): number {
55
+ let h = Math.imul(stateID ^ 0x9e_37_79_b9, 0x85_eb_ca_6b) ^ Math.imul(stringIndex ^ 0xc2_b2_ae_35, 0x27_d4_eb_2f)
56
+
57
+ h ^= h >>> 15
58
+
59
+ return Math.imul(h, 0x2c_1b_3c_6d) >>> 0
60
+ }
61
+
62
+ /**
63
+ * Reads an FST binary in the format `serializeFST` writes, versions 1 through
64
+ * {@link FST_FORMAT_VERSION}, without decoding it into objects.
65
+ */
66
+ export class BinaryFSTMatcher {
67
+ readonly stateCount: number
68
+ readonly #view: DataView
69
+ readonly #bytes: Uint8Array
70
+ readonly #decoder = new TextDecoder("utf-8")
71
+ readonly #version: number
72
+ readonly #isWide: boolean
73
+ readonly #isSplit: boolean
74
+ readonly #hasAmbiguity: boolean
75
+ readonly #stateEntrySize: number
76
+ readonly #placeEntrySize: number
77
+ readonly #stringDataStart: number
78
+ readonly #stateTableStart: number
79
+ readonly #edgeTableStart: number
80
+ readonly #placeTableStart: number
81
+ readonly #stringOffsets: Uint32Array
82
+ readonly #tokenSlots: Int32Array
83
+ readonly #edgeSlotState: Int32Array
84
+ readonly #edgeSlotString: Uint32Array
85
+ readonly #edgeSlotTarget: Uint32Array
86
+ #placeCount: number | null = null
87
+
88
+ constructor(input: ArrayBuffer | Uint8Array) {
89
+ const bytes = input instanceof ArrayBuffer ? new Uint8Array(input) : input
90
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
91
+
92
+ if (bytes.byteLength < HEADER_SIZE) throw new Error("FST buffer too small for header")
93
+
94
+ if (FST_MAGIC_BYTES.some((byte, index) => bytes[index] !== byte)) {
95
+ throw new Error("FST magic mismatch")
96
+ }
97
+
98
+ const version = view.getUint16(4, true)
99
+
100
+ if (version < 1 || version > FST_FORMAT_VERSION) {
101
+ throw new Error(`FST version ${version} unsupported (expected 1..${FST_FORMAT_VERSION})`)
102
+ }
103
+
104
+ this.#bytes = bytes
105
+ this.#view = view
106
+ this.#version = version
107
+ this.#isWide = version >= VERSION_WIDE_STATE_COUNTERS
108
+ this.#isSplit = version >= VERSION_TWO_SCORE_SPLIT
109
+ // Flags bit 0, as `serializeFST` writes it: place rows include surface-ambiguity data.
110
+ this.#hasAmbiguity = (view.getUint16(6, true) & 1) === 1
111
+ this.#stateEntrySize = this.#isWide ? WIDE_STATE_ENTRY_SIZE : NARROW_STATE_ENTRY_SIZE
112
+ // Version 5 grew the place entry by the encyclopedic float.
113
+ this.#placeEntrySize = this.#isSplit ? SPLIT_PLACE_ENTRY_SIZE : LEGACY_PLACE_ENTRY_SIZE
114
+
115
+ const stateCount = view.getUint32(8, true)
116
+ const edgeCount = view.getUint32(12, true)
117
+ const stringCount = view.getUint32(20, true)
118
+ const stringBytes = view.getUint32(24, true)
119
+
120
+ this.stateCount = stateCount
121
+ this.#stringOffsets = new Uint32Array(stringCount + 1)
122
+
123
+ for (let index = 0; index <= stringCount; index++) {
124
+ this.#stringOffsets[index] = view.getUint32(HEADER_SIZE + index * 4, true)
125
+ }
126
+
127
+ this.#stringDataStart = HEADER_SIZE + (stringCount + 1) * 4
128
+ this.#stateTableStart = this.#stringDataStart + stringBytes
129
+ this.#edgeTableStart = this.#stateTableStart + stateCount * this.#stateEntrySize
130
+ this.#placeTableStart = this.#edgeTableStart + edgeCount * EDGE_ENTRY_SIZE
131
+
132
+ this.#tokenSlots = new Int32Array(tableCapacity(stringCount)).fill(EMPTY_SLOT)
133
+
134
+ const edgeCapacity = tableCapacity(edgeCount)
135
+
136
+ this.#edgeSlotState = new Int32Array(edgeCapacity).fill(EMPTY_SLOT)
137
+ this.#edgeSlotString = new Uint32Array(edgeCapacity)
138
+ this.#edgeSlotTarget = new Uint32Array(edgeCapacity)
139
+
140
+ const indexedTokens = new Uint8Array(stringCount)
141
+
142
+ for (let stateID = 0; stateID < stateCount; stateID++) {
143
+ const { edgeStart, edgeCount: stateEdgeCount } = this.#stateEntry(stateID)
144
+
145
+ for (let edge = 0; edge < stateEdgeCount; edge++) {
146
+ const entry = this.#edgeTableStart + (edgeStart + edge) * EDGE_ENTRY_SIZE
147
+ const stringIndex = view.getUint32(entry, true)
148
+
149
+ if (!indexedTokens[stringIndex]) {
150
+ indexedTokens[stringIndex] = 1
151
+ this.#indexToken(stringIndex)
152
+ }
153
+
154
+ this.#indexEdge(stateID, stringIndex, view.getUint32(entry + 4, true))
155
+ }
156
+ }
157
+ }
158
+
159
+ /**
160
+ * The number of place entries over all states.
161
+ */
162
+ get placeCount(): number {
163
+ if (this.#placeCount === null) {
164
+ let count = 0
165
+
166
+ for (let stateID = 0; stateID < this.stateCount; stateID++) {
167
+ count += this.#stateEntry(stateID).placeCount
168
+ }
169
+
170
+ this.#placeCount = count
171
+ }
172
+
173
+ return this.#placeCount
174
+ }
175
+
176
+ get nodeCount(): number {
177
+ return this.stateCount
178
+ }
179
+
180
+ walk(tokens: string[]): AncestrieMatch | null {
181
+ let stateID = 0
182
+
183
+ for (const token of tokens) {
184
+ if (stateID >= this.stateCount) return null
185
+
186
+ const next = this.#target(stateID, token)
187
+
188
+ // No edge enters the root.
189
+ // A target of 0 therefore marks a missing edge.
190
+ if (!next) return null
191
+ stateID = next
192
+ }
193
+
194
+ return { stateID, accepted: this.#stateEntry(stateID).placeCount > 0, depth: tokens.length }
195
+ }
196
+
197
+ walkFrom(prev: Pick<AncestrieMatch, "stateID" | "depth">, token: string): AncestrieMatch | null {
198
+ if (prev.stateID >= this.stateCount) return null
199
+
200
+ const next = this.#target(prev.stateID, token)
201
+
202
+ if (!next) return null
203
+
204
+ return { stateID: next, accepted: this.#stateEntry(next).placeCount > 0, depth: prev.depth + 1 }
205
+ }
206
+
207
+ accepting(stateID: number): PlaceEntry[] {
208
+ if (stateID < 0 || stateID >= this.stateCount) return []
209
+
210
+ const { placeStart, placeCount } = this.#stateEntry(stateID)
211
+ const places: PlaceEntry[] = new Array(placeCount)
212
+
213
+ for (let index = 0; index < placeCount; index++) {
214
+ places[index] = this.#placeAt(placeStart + index)
215
+ }
216
+
217
+ return places
218
+ }
219
+
220
+ continuations(stateID: number): FSTContinuation[] {
221
+ if (stateID < 0 || stateID >= this.stateCount) return []
222
+
223
+ const { edgeStart, edgeCount } = this.#stateEntry(stateID)
224
+ const result: FSTContinuation[] = []
225
+
226
+ for (let edge = 0; edge < edgeCount; edge++) {
227
+ const entry = this.#edgeTableStart + (edgeStart + edge) * EDGE_ENTRY_SIZE
228
+ const targetState = this.#view.getUint32(entry + 4, true)
229
+
230
+ result.push({
231
+ token: this.#string(this.#view.getUint32(entry, true)),
232
+ targetState,
233
+ acceptingCount: this.#stateEntry(targetState).placeCount,
234
+ })
235
+ }
236
+
237
+ return result
238
+ }
239
+
240
+ query(text: string): FSTQueryResult {
241
+ const tokens = normalizeTokens(text)
242
+ let stateID = 0
243
+ let depth = 0
244
+
245
+ for (const token of tokens) {
246
+ const next = this.#target(stateID, token)
247
+
248
+ if (!next) break
249
+ stateID = next
250
+
251
+ depth++
252
+ }
253
+
254
+ return {
255
+ path: tokens.slice(0, depth),
256
+ stateID,
257
+ accepting: this.accepting(stateID),
258
+ continuations: this.continuations(stateID),
259
+ }
260
+ }
261
+
262
+ #stateEntry(stateID: number): { edgeStart: number; placeStart: number; edgeCount: number; placeCount: number } {
263
+ const entry = this.#stateTableStart + stateID * this.#stateEntrySize
264
+ const view = this.#view
265
+
266
+ return {
267
+ edgeStart: view.getUint32(entry, true),
268
+ placeStart: view.getUint32(entry + 4, true),
269
+ edgeCount: this.#isWide ? view.getUint32(entry + 8, true) : view.getUint16(entry + 8, true),
270
+ placeCount: this.#isWide ? view.getUint32(entry + 12, true) : view.getUint16(entry + 10, true),
271
+ }
272
+ }
273
+
274
+ #string(stringIndex: number): string {
275
+ const start = this.#stringDataStart + this.#stringOffsets[stringIndex]!
276
+ const end = this.#stringDataStart + this.#stringOffsets[stringIndex + 1]!
277
+
278
+ return this.#decoder.decode(this.#bytes.subarray(start, end))
279
+ }
280
+
281
+ #indexToken(stringIndex: number): void {
282
+ const mask = this.#tokenSlots.length - 1
283
+ let slot = hashFNV1a(this.#string(stringIndex)) & mask
284
+
285
+ while (this.#tokenSlots[slot] !== EMPTY_SLOT) {
286
+ slot = (slot + 1) & mask
287
+ }
288
+
289
+ this.#tokenSlots[slot] = stringIndex
290
+ }
291
+
292
+ /**
293
+ * Returns the string index of `token` among the edge tokens, or -1 when no edge has that token.
294
+ */
295
+ #tokenIndex(token: string): number {
296
+ const mask = this.#tokenSlots.length - 1
297
+ let slot = hashFNV1a(token) & mask
298
+
299
+ for (;;) {
300
+ const stringIndex = this.#tokenSlots[slot]!
301
+
302
+ if (stringIndex === EMPTY_SLOT) return -1
303
+
304
+ if (this.#string(stringIndex) === token) return stringIndex
305
+ slot = (slot + 1) & mask
306
+ }
307
+ }
308
+
309
+ #indexEdge(stateID: number, stringIndex: number, target: number): void {
310
+ const mask = this.#edgeSlotState.length - 1
311
+ let slot = edgeHash(stateID, stringIndex) & mask
312
+
313
+ for (;;) {
314
+ const occupant = this.#edgeSlotState[slot]!
315
+
316
+ if (occupant === EMPTY_SLOT || (occupant === stateID && this.#edgeSlotString[slot] === stringIndex)) break
317
+ slot = (slot + 1) & mask
318
+ }
319
+
320
+ // A repeated (state, token) pair keeps its last target, as `Map.set` does in the object deserializer.
321
+ this.#edgeSlotState[slot] = stateID
322
+ this.#edgeSlotString[slot] = stringIndex
323
+ this.#edgeSlotTarget[slot] = target
324
+ }
325
+
326
+ /**
327
+ * Returns the state the edge labeled `token` leads to from `stateID`, or 0 when there is no such edge.
328
+ */
329
+ #target(stateID: number, token: string): number {
330
+ const stringIndex = this.#tokenIndex(token)
331
+
332
+ if (stringIndex < 0) return 0
333
+
334
+ const mask = this.#edgeSlotState.length - 1
335
+ let slot = edgeHash(stateID, stringIndex) & mask
336
+
337
+ for (;;) {
338
+ const occupant = this.#edgeSlotState[slot]!
339
+
340
+ if (occupant === EMPTY_SLOT) return 0
341
+
342
+ if (occupant === stateID && this.#edgeSlotString[slot] === stringIndex) return this.#edgeSlotTarget[slot]!
343
+ slot = (slot + 1) & mask
344
+ }
345
+ }
346
+
347
+ #placeAt(placeIndex: number): PlaceEntry {
348
+ const view = this.#view
349
+ const entry = this.#placeTableStart + placeIndex * this.#placeEntrySize
350
+ const chainLength = view.getUint8(entry + 5)
351
+ const parentChain: number[] = new Array(chainLength)
352
+
353
+ for (let index = 0; index < chainLength; index++) {
354
+ parentChain[index] = view.getUint32(entry + 24 + index * 4, true)
355
+ }
356
+
357
+ // Version 1 stored a raw population u32 here, and later versions store the referential score.
358
+ // `deserializeFST` explains why a version 1 value converts to the referential score this way.
359
+ const referential =
360
+ this.#version >= 2
361
+ ? view.getFloat32(entry + 12, true)
362
+ : Math.min(1, Math.log2(1 + view.getUint32(entry + 12, true) / 1000) / 14)
363
+
364
+ const hasEncyclopedic =
365
+ this.#isSplit && (view.getUint8(entry + 7) & PLACE_FLAG_HAS_ENCYCLOPEDIC) === PLACE_FLAG_HAS_ENCYCLOPEDIC
366
+
367
+ return {
368
+ wofID: view.getUint32(entry, true),
369
+ placetype: PLACETYPE_ORDER[view.getUint8(entry + 4)] ?? "locality",
370
+ name: this.#string(view.getUint32(entry + 8, true)),
371
+ referential,
372
+ lat: view.getFloat32(entry + 16, true),
373
+ lon: view.getFloat32(entry + 20, true),
374
+ parentChain,
375
+ crossCountryBranches: this.#hasAmbiguity ? view.getUint8(entry + 6) : null,
376
+ encyclopedic: hasEncyclopedic ? view.getFloat32(entry + ENCYCLOPEDIC_OFFSET, true) : null,
377
+ }
378
+ }
379
+ }
@@ -260,7 +260,7 @@ export async function buildFSTFromWOF(opts: BuildFSTOpts): Promise<{
260
260
  const node = nodes[stateID]!
261
261
  let next = node.edges.get(t)
262
262
 
263
- if (next === undefined) {
263
+ if (!next) {
264
264
  next = nodes.length
265
265
  nodes.push({ edges: new Map(), places: [] })
266
266
  node.edges.set(t, next)
@@ -3,155 +3,26 @@
3
3
  * @license AGPL-3.0
4
4
  * @author Teffen Ellis, et al.
5
5
  *
6
- * Browser-compatible FST deserializer. Uses DataView + TextDecoder instead of Node's Buffer so the
7
- * same binary format loads in the browser through fetch(url).then(r => r.arrayBuffer()).
6
+ * Browser-compatible FST reader. Uses DataView + TextDecoder instead of Node's Buffer so the same binary format
7
+ * loads in the browser and in a Cloudflare Worker through fetch(url).then(r => r.arrayBuffer()).
8
8
  *
9
- * This is a read-only counterpart to fst-serialize.ts. Serialization stays Node-only because it is
10
- * a build-time operation.
9
+ * This is a read-only counterpart to serialize.ts. Serialization stays Node-only because it is a build-time
10
+ * operation.
11
11
  */
12
12
 
13
13
  import { tryParsingJSON } from "@mailwoman/core/json"
14
14
 
15
- import {
16
- EDGE_ENTRY_SIZE,
17
- ENCYCLOPEDIC_OFFSET,
18
- FST_FORMAT_VERSION,
19
- FST_MAGIC_BYTES,
20
- HEADER_SIZE,
21
- LEGACY_PLACE_ENTRY_SIZE,
22
- NARROW_STATE_ENTRY_SIZE,
23
- PLACE_FLAG_HAS_ENCYCLOPEDIC,
24
- PLACETYPE_ORDER,
25
- SPLIT_PLACE_ENTRY_SIZE,
26
- VERSION_TWO_SCORE_SPLIT,
27
- VERSION_WIDE_STATE_COUNTERS,
28
- VERSION_WITH_METADATA,
29
- WIDE_STATE_ENTRY_SIZE,
30
- } from "#fst/format"
31
- import type { FSTNode } from "#fst/matcher"
32
- import { FSTMatcher } from "#fst/matcher"
33
- import type { FSTProvenance, PlaceEntry } from "#fst/types"
15
+ import { BinaryFSTMatcher } from "#fst/binary-matcher"
16
+ import { HEADER_SIZE, VERSION_WITH_METADATA } from "#fst/format"
17
+ import type { FSTProvenance } from "#fst/types"
34
18
 
35
- export function deserializeFSTWeb(input: ArrayBuffer | Uint8Array): FSTMatcher {
36
- const bytes = input instanceof ArrayBuffer ? new Uint8Array(input) : input
37
- const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
38
- const decoder = new TextDecoder("utf-8")
39
-
40
- if (bytes.byteLength < HEADER_SIZE) throw new Error("FST buffer too small for header")
41
-
42
- if (
43
- bytes[0] !== FST_MAGIC_BYTES[0] ||
44
- bytes[1] !== FST_MAGIC_BYTES[1] ||
45
- bytes[2] !== FST_MAGIC_BYTES[2] ||
46
- bytes[3] !== FST_MAGIC_BYTES[3]
47
- ) {
48
- throw new Error("FST magic mismatch")
49
- }
50
-
51
- const version = view.getUint16(4, true)
52
-
53
- if (version < 1 || version > FST_FORMAT_VERSION) {
54
- throw new Error(`FST version ${version} unsupported (expected 1..${FST_FORMAT_VERSION})`)
55
- }
56
-
57
- const isV2 = version >= 2
58
- const isSplit = version >= VERSION_TWO_SCORE_SPLIT
59
- // flags bit0 (mirrors fst-serialize.ts): place rows include surface-ambiguity data.
60
- const hasAmbiguity = (view.getUint16(6, true) & 1) === 1
61
-
62
- const stateCount = view.getUint32(8, true)
63
- const edgeCount = view.getUint32(12, true)
64
- const _placeCount = view.getUint32(16, true)
65
- const stringCount = view.getUint32(20, true)
66
- const stringBytes = view.getUint32(24, true)
67
-
68
- let pos = HEADER_SIZE
69
-
70
- const strOffsets = new Uint32Array(stringCount + 1)
71
-
72
- for (let i = 0; i <= stringCount; i++) {
73
- strOffsets[i] = view.getUint32(pos, true)
74
- pos += 4
75
- }
76
-
77
- const strDataStart = pos
78
- const strings: string[] = new Array(stringCount)
79
-
80
- for (let i = 0; i < stringCount; i++) {
81
- const start = strDataStart + strOffsets[i]!
82
- const end = strDataStart + strOffsets[i + 1]!
83
- strings[i] = decoder.decode(bytes.subarray(start, end))
84
- }
85
-
86
- pos += stringBytes
87
-
88
- const stateEntrySize = version >= VERSION_WIDE_STATE_COUNTERS ? WIDE_STATE_ENTRY_SIZE : NARROW_STATE_ENTRY_SIZE
89
- // v5 grew the place entry by the encyclopedic float. v4-and-below files are read at the old stride.
90
- const placeEntrySize = isSplit ? SPLIT_PLACE_ENTRY_SIZE : LEGACY_PLACE_ENTRY_SIZE
91
- const stateTableStart = pos
92
- const edgeTableStart = stateTableStart + stateCount * stateEntrySize
93
- const placeTableStart = edgeTableStart + edgeCount * EDGE_ENTRY_SIZE
94
-
95
- const nodes: FSTNode[] = new Array(stateCount)
96
-
97
- for (let si = 0; si < stateCount; si++) {
98
- const sp = stateTableStart + si * stateEntrySize
99
- const edgeStart = view.getUint32(sp, true)
100
- const placeStart = view.getUint32(sp + 4, true)
101
-
102
- const edgeCountForState =
103
- version >= VERSION_WIDE_STATE_COUNTERS ? view.getUint32(sp + 8, true) : view.getUint16(sp + 8, true)
104
-
105
- const placeCountForState =
106
- version >= VERSION_WIDE_STATE_COUNTERS ? view.getUint32(sp + 12, true) : view.getUint16(sp + 10, true)
107
-
108
- const edges = new Map<string, number>()
109
-
110
- for (let ei = 0; ei < edgeCountForState; ei++) {
111
- const ep = edgeTableStart + (edgeStart + ei) * EDGE_ENTRY_SIZE
112
- const stringIdx = view.getUint32(ep, true)
113
- const target = view.getUint32(ep + 4, true)
114
- edges.set(strings[stringIdx]!, target)
115
- }
116
-
117
- const places: PlaceEntry[] = new Array(placeCountForState)
118
-
119
- for (let pi = 0; pi < placeCountForState; pi++) {
120
- const pp = placeTableStart + (placeStart + pi) * placeEntrySize
121
- const chainLen = view.getUint8(pp + 5)
122
- const parentChain: number[] = []
123
-
124
- for (let ci = 0; ci < chainLen; ci++) {
125
- parentChain.push(view.getUint32(pp + 24 + ci * 4, true))
126
- }
127
-
128
- // v1 stored a raw population u32 here. v2-v4 stored the conflated `importance`
129
- // float. v5 stored the referential score.
130
- // The Node deserializer explains why a v1 value is the referential score.
131
- const referential = isV2
132
- ? view.getFloat32(pp + 12, true)
133
- : Math.min(1, Math.log2(1 + view.getUint32(pp + 12, true) / 1000) / 14)
134
-
135
- const hasEncyclopedic =
136
- isSplit && (view.getUint8(pp + 7) & PLACE_FLAG_HAS_ENCYCLOPEDIC) === PLACE_FLAG_HAS_ENCYCLOPEDIC
137
-
138
- places[pi] = {
139
- wofID: view.getUint32(pp, true),
140
- placetype: PLACETYPE_ORDER[view.getUint8(pp + 4)] ?? "locality",
141
- name: strings[view.getUint32(pp + 8, true)]!,
142
- referential,
143
- lat: view.getFloat32(pp + 16, true),
144
- lon: view.getFloat32(pp + 20, true),
145
- parentChain,
146
- crossCountryBranches: hasAmbiguity ? view.getUint8(pp + 6) : null,
147
- encyclopedic: hasEncyclopedic ? view.getFloat32(pp + ENCYCLOPEDIC_OFFSET, true) : null,
148
- }
149
- }
150
-
151
- nodes[si] = { edges, places }
152
- }
153
-
154
- return FSTMatcher.fromNodes(nodes)
19
+ /**
20
+ * Opens an FST binary for matching, answering queries from the bytes in place.
21
+ *
22
+ * The matcher keeps a reference to `input`, so the caller must not modify it afterward.
23
+ */
24
+ export function deserializeFSTWeb(input: ArrayBuffer | Uint8Array): BinaryFSTMatcher {
25
+ return new BinaryFSTMatcher(input)
155
26
  }
156
27
 
157
28
  export function readFSTProvenanceWeb(input: ArrayBuffer | Uint8Array): FSTProvenance | null {
@@ -47,7 +47,7 @@ export class FSTMatcher {
47
47
  if (!node) return null
48
48
  const next = node.edges.get(token)
49
49
 
50
- if (next === undefined) return null
50
+ if (!next) return null
51
51
  stateID = next
52
52
  }
53
53
 
@@ -62,7 +62,7 @@ export class FSTMatcher {
62
62
  if (!node) return null
63
63
  const next = node.edges.get(token)
64
64
 
65
- if (next === undefined) return null
65
+ if (!next) return null
66
66
  const target = this.nodes[next]!
67
67
 
68
68
  return { stateID: next, accepted: target.places.length > 0, depth: prev.depth + 1 }
@@ -106,7 +106,7 @@ export class FSTMatcher {
106
106
  if (!node) break
107
107
  const next = node.edges.get(t)
108
108
 
109
- if (next === undefined) break
109
+ if (!next) break
110
110
  stateID = next
111
111
 
112
112
  depth++
@@ -71,7 +71,7 @@ export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): B
71
71
  function intern(s: string): number {
72
72
  let idx = stringMap.get(s)
73
73
 
74
- if (idx === undefined) {
74
+ if (typeof idx !== "number") {
75
75
  idx = strings.length
76
76
  strings.push(s)
77
77
  stringMap.set(s, idx)
package/lib/fst.ts CHANGED
@@ -5,6 +5,7 @@
5
5
  */
6
6
 
7
7
  export * from "#fst/autocomplete"
8
+ export * from "#fst/binary-matcher"
8
9
  export * from "#fst/builder"
9
10
  export * from "#fst/deserialize-web"
10
11
  export * from "#fst/format"
@@ -220,7 +220,7 @@ export async function ingestGeonamesAliases(
220
220
  let refused = 0
221
221
 
222
222
  const addAdmin = opts?.adminForCountries?.has(cc) ?? false
223
- const v2File = opts?.alternateDir ? resolvePathBuilder(opts.alternateDir, `${cc}.txt`) : undefined
223
+ const v2File = opts?.alternateDir ? resolvePathBuilder(opts.alternateDir, `${cc}.txt`) : null
224
224
  const readV2 = Boolean(v2File && (await pathExists(v2File)))
225
225
 
226
226
  const wanted = new Set<number>()
@@ -242,7 +242,7 @@ export async function ingestGeonamesAliases(
242
242
  }
243
243
  }
244
244
 
245
- const v2 = readV2 ? await parseAlternateNamesV2(v2File!, cc, wanted) : undefined
245
+ const v2 = readV2 ? await parseAlternateNamesV2(v2File!, cc, wanted) : null
246
246
 
247
247
  let countryID = -1
248
248
  const adminMap = new Map<string, number>()
@@ -8,7 +8,8 @@
8
8
 
9
9
  import { parseJSONStrict } from "@mailwoman/core/json"
10
10
  import type { InterpolationLookup } from "@mailwoman/core/resolver"
11
- import { clampFraction, haversineKm, pointAlong } from "@mailwoman/spatial"
11
+ import { haversine } from "@mailwoman/spatial/geometries/point/distance"
12
+ import { clampFraction, pointAlong } from "@mailwoman/spatial/polyline"
12
13
  import { DatabaseClient } from "@mailwoman/sqlite/client"
13
14
 
14
15
  import { hasTable, prepareAll, type PreparedAll } from "#sqlite-utils"
@@ -100,7 +101,7 @@ function nearestPostcodeGroup(pool: readonly SegmentRow[], near: { lat: number;
100
101
  const [firstVertex] = parseJSONStrict<[number, number][]>(row.geometry)
101
102
 
102
103
  if (firstVertex) {
103
- km = haversineKm(near.lat, near.lon, firstVertex[1], firstVertex[0])
104
+ km = haversine(near.lat, near.lon, firstVertex[1], firstVertex[0])
104
105
  }
105
106
  } catch {
106
107
  // Unparseable geometry: this row cannot be sited, so it cannot win the tie-break.
package/lib/lookup.ts CHANGED
@@ -9,7 +9,7 @@
9
9
  import { expandPlacetypeFilter } from "@mailwoman/codex/placetype-map"
10
10
  import type { Ancestor, CoincidentLocality } from "@mailwoman/core/resolver"
11
11
  import { allRows } from "@mailwoman/core/utils"
12
- import { haversineKm } from "@mailwoman/spatial"
12
+ import { haversine } from "@mailwoman/spatial/geometries/point/distance"
13
13
  import type { DatabaseClient, SQLInputValue } from "@mailwoman/sqlite/client"
14
14
  import { SQLiteLookup } from "@mailwoman/sqlite/lookup"
15
15
  import type { PathBuilderLike } from "path-ts"
@@ -81,7 +81,7 @@ export interface WOFSQLitePlaceLookupOpts {
81
81
  *
82
82
  * Absent, every alias code path is skipped and the resolver is byte-identical.
83
83
  */
84
- postalCityAliases?: WOFPostalCityAliasLookup
84
+ postalCityAliases?: WOFPostalCityAliasLookup | null
85
85
  }
86
86
 
87
87
  /**
@@ -168,7 +168,7 @@ export class WOFSQLitePlaceLookup extends SQLiteLookup<WOFDatabase> implements P
168
168
  */
169
169
  readonly #postalCityAliases: WOFPostalCityAliasLookup | null
170
170
 
171
- constructor(opts: WOFSQLitePlaceLookupOpts, weights?: Partial<RankingWeights>) {
171
+ constructor(opts: WOFSQLitePlaceLookupOpts, weights?: Partial<RankingWeights> | null) {
172
172
  if (opts.database && opts.databasePath) {
173
173
  throw new Error("WOFSQLitePlaceLookup: pass either `database` or `databasePath`, not both")
174
174
  }
@@ -549,7 +549,7 @@ export class WOFSQLitePlaceLookup extends SQLiteLookup<WOFDatabase> implements P
549
549
  #countryWOFId(code: string): number | null {
550
550
  const cached = this.#countryWOFIdCache.get(code)
551
551
 
552
- if (cached !== undefined) return cached
552
+ if (cached) return cached
553
553
  let id: number | null
554
554
 
555
555
  try {
@@ -666,12 +666,12 @@ export class WOFSQLitePlaceLookup extends SQLiteLookup<WOFDatabase> implements P
666
666
  // oxlint-disable-next-line unicorn/no-array-sort -- sorts a freshly-built array. toSorted would double-allocate on a hot path
667
667
  .sort((a, b) => b.containing - a.containing || a.dist - b.dist)[0]
668
668
 
669
- const anchor = anchorRow ? merged.get(anchorRow.id) : undefined
669
+ const anchor = anchorRow ? merged.get(anchorRow.id) : null
670
670
 
671
671
  if (
672
672
  anchor &&
673
673
  (top.id as number) !== anchorRow!.id &&
674
- haversineKm(top.lat, top.lon, anchor.lat, anchor.lon) > CF_MISMATCH_KM
674
+ haversine(top.lat, top.lon, anchor.lat, anchor.lon) > CF_MISMATCH_KM
675
675
  ) {
676
676
  top.mismatch = true
677
677
  }