@mailwoman/resolver-wof-sqlite 7.1.0 → 7.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/address-point-interpolation.ts +207 -0
- package/address-point-schema.ts +107 -0
- package/address-point.ts +122 -0
- package/ancestry-backfill.ts +205 -0
- package/ancestry.ts +70 -0
- package/build-candidate.ts +351 -0
- package/build-slim.ts +394 -0
- package/candidate-fts.ts +43 -0
- package/candidate-lookup.ts +382 -0
- package/candidate-schema.ts +166 -0
- package/coincident-roles.ts +240 -0
- package/convention.ts +152 -0
- package/fst-autocomplete.ts +187 -0
- package/fst-builder.ts +291 -0
- package/fst-deserialize-web.ts +164 -0
- package/fst-matcher.ts +150 -0
- package/fst-serialize.ts +311 -0
- package/fst-types.ts +78 -0
- package/fts.ts +318 -0
- package/geo.ts +140 -0
- package/geonames-aliases.ts +317 -0
- package/geonames-postal.ts +150 -0
- package/index.ts +117 -0
- package/interpolation.ts +232 -0
- package/lookup.ts +1498 -0
- package/out/poi-lookup.d.ts +74 -0
- package/out/poi-lookup.d.ts.map +1 -0
- package/out/poi-lookup.js +226 -0
- package/out/poi-lookup.js.map +1 -0
- package/out/poi-schema.d.ts +77 -0
- package/out/poi-schema.d.ts.map +1 -0
- package/out/poi-schema.js +81 -0
- package/out/poi-schema.js.map +1 -0
- package/package.json +33 -8
- package/poi-lookup.ts +319 -0
- package/poi-schema.ts +147 -0
- package/postal-city-alias-lookup.ts +89 -0
- package/postal-city-alias-schema.ts +75 -0
- package/postal-city-candidate-schema.ts +81 -0
- package/postcode-point-lookup.ts +64 -0
- package/reverse.ts +429 -0
- package/schema.ts +176 -0
- package/sharding.ts +235 -0
- package/sqlite-convention-source.ts +61 -0
- package/sqlite-utils.ts +25 -0
- package/street-centroid-schema.ts +124 -0
- package/street-centroid.ts +124 -0
- package/street-morphology-fst-builder.ts +230 -0
- package/street-name-lookup.ts +101 -0
- package/street-normalize.ts +302 -0
- package/street-segment-schema.ts +104 -0
- package/types.ts +164 -0
- package/unified-schema.ts +171 -0
package/poi-lookup.ts
ADDED
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Node reader for `poi.db` (spec §3.4) — the res-9 k-ring reader over the clustered `poi`
|
|
7
|
+
* `WITHOUT ROWID` B-tree Task 1's schema (`poi-schema.ts`) builds. Three search modes share one
|
|
8
|
+
* artifact:
|
|
9
|
+
*
|
|
10
|
+
* - **Category**: `latLngToCell(center, 9)` → `gridDisk` ring-by-ring expansion, probing each
|
|
11
|
+
* cell's clustered `(h3_cell, category_id, neg_rank, …)` range. Rings accumulate until `limit`
|
|
12
|
+
* rows are on hand after a completed ring, or `maxRings` is exhausted; the pool is sorted by
|
|
13
|
+
* haversine distance from `center` after every ring.
|
|
14
|
+
* - **Brand**: the same k-ring walk, but probing on `brand_wikidata` instead — category
|
|
15
|
+
* unconstrained, so a chain's rows surface regardless of how they were categorized.
|
|
16
|
+
* - **Name**: FTS5 `MATCH` against the `poi_search` virtual table, hydrated back to full rows by
|
|
17
|
+
* `name_key`. No center required; if one is given, hits are still distance-sorted.
|
|
18
|
+
*
|
|
19
|
+
* `latLngToCell`/`gridDisk` come from `h3-js`; the 48-bit short-cell packing that turns a raw H3
|
|
20
|
+
* cell into the integer `poi.h3_cell` stores is `@mailwoman/spatial`'s `shortenH3Cell` — that math
|
|
21
|
+
* is NEVER reimplemented here (see AGENTS.md on `@mailwoman/spatial` being the one true home for
|
|
22
|
+
* it).
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { DatabaseSync } from "node:sqlite"
|
|
26
|
+
|
|
27
|
+
import { haversineKm, shortenH3Cell, type H3Cell } from "@mailwoman/spatial"
|
|
28
|
+
import { gridDisk, latLngToCell } from "h3-js"
|
|
29
|
+
|
|
30
|
+
import type { POICategoryCodeTable, POITable } from "./poi-schema.ts"
|
|
31
|
+
|
|
32
|
+
/** Resolution the `poi` table's `h3_cell` column is keyed at — matches the builder (spec §3.4). */
|
|
33
|
+
export const POI_H3_RESOLUTION = 9
|
|
34
|
+
|
|
35
|
+
/** Ring budget default: 12 res-9 k-rings ≈ ~4 km. */
|
|
36
|
+
const DEFAULT_MAX_RINGS = 12
|
|
37
|
+
|
|
38
|
+
/** Row-count default when a query doesn't specify `limit`. */
|
|
39
|
+
const DEFAULT_LIMIT = 20
|
|
40
|
+
|
|
41
|
+
export interface POISearchQuery {
|
|
42
|
+
/** Poi-taxonomy category id (string side of the dictionary). Ignored when `brandWikidata` is also set — brand wins. */
|
|
43
|
+
categoryID?: string
|
|
44
|
+
/** Wikidata QID for brand-exact search. */
|
|
45
|
+
brandWikidata?: string
|
|
46
|
+
/** Free-text name (FTS5). */
|
|
47
|
+
name?: string
|
|
48
|
+
/** Search center. Required for category/brand queries (k-ring expansion). */
|
|
49
|
+
center?: { latitude: number; longitude: number }
|
|
50
|
+
/**
|
|
51
|
+
* Ring budget: how many res-9 k-rings to expand before giving up (default 12 ≈ ~4 km). Counts ring 0, so k reaches
|
|
52
|
+
* `maxRings - 1`.
|
|
53
|
+
*/
|
|
54
|
+
maxRings?: number
|
|
55
|
+
limit?: number
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export interface POISearchHit {
|
|
59
|
+
name: string | null
|
|
60
|
+
categoryID: string | null
|
|
61
|
+
brandWikidata: string | null
|
|
62
|
+
latitude: number
|
|
63
|
+
longitude: number
|
|
64
|
+
country: string
|
|
65
|
+
confidence: number
|
|
66
|
+
distanceM?: number
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export interface POILookupOpts {
|
|
70
|
+
/** Path to a `poi.db` built by the (future) POI builder. Opened read-only. */
|
|
71
|
+
databasePath?: string
|
|
72
|
+
/** Pre-opened handle (tests / shared connections). Mutually exclusive with `databasePath`. */
|
|
73
|
+
database?: DatabaseSync
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** The `poi` columns every search mode hydrates — a typed projection of the SHARED {@link POITable}. */
|
|
77
|
+
type POIRow = Pick<
|
|
78
|
+
POITable,
|
|
79
|
+
"name" | "category_id" | "brand_wikidata" | "latitude" | "longitude" | "country" | "confidence" | "name_key"
|
|
80
|
+
>
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Node reader over `poi.db`. `implements Disposable` so callers can `using lookup = new POILookup(...)` (or call
|
|
84
|
+
* `[Symbol.dispose]()` explicitly), the same precedent as {@link WOFCandidateTableLookup} /
|
|
85
|
+
* {@link WOFSqlitePlaceLookup}.
|
|
86
|
+
*/
|
|
87
|
+
export class POILookup implements Disposable {
|
|
88
|
+
#db: DatabaseSync
|
|
89
|
+
#ownsDB: boolean
|
|
90
|
+
readonly #categoryToID = new Map<string, number>()
|
|
91
|
+
readonly #idToCategory = new Map<number, string>()
|
|
92
|
+
|
|
93
|
+
/** `(h3_cell, category_id)` → the cell's category-clustered range, most-confident-first. */
|
|
94
|
+
readonly #categoryCellProbe: ReturnType<DatabaseSync["prepare"]>
|
|
95
|
+
/** `(h3_cell, brand_wikidata)` → the cell's brand-matched rows (category unconstrained). */
|
|
96
|
+
readonly #brandCellProbe: ReturnType<DatabaseSync["prepare"]>
|
|
97
|
+
/** FTS5 `MATCH` over `poi_search`, returning candidate `name_key`s to hydrate. */
|
|
98
|
+
readonly #nameFTSProbe: ReturnType<DatabaseSync["prepare"]>
|
|
99
|
+
|
|
100
|
+
constructor(opts: POILookupOpts) {
|
|
101
|
+
if (opts.database) {
|
|
102
|
+
this.#db = opts.database
|
|
103
|
+
this.#ownsDB = false
|
|
104
|
+
} else if (opts.databasePath) {
|
|
105
|
+
this.#db = new DatabaseSync(opts.databasePath, { readOnly: true })
|
|
106
|
+
this.#ownsDB = true
|
|
107
|
+
} else {
|
|
108
|
+
throw new Error("POILookup needs `databasePath` or `database`")
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// The category dictionary is tiny (poi-taxonomy's category count) — load it once at
|
|
112
|
+
// construction so `search` never round-trips to it.
|
|
113
|
+
for (const r of this.#db
|
|
114
|
+
.prepare("SELECT id, category FROM poi_category_codes")
|
|
115
|
+
.all() as unknown as POICategoryCodeTable[]) {
|
|
116
|
+
this.#categoryToID.set(String(r.category), Number(r.id))
|
|
117
|
+
this.#idToCategory.set(Number(r.id), String(r.category))
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
const columns = "name, category_id, brand_wikidata, latitude, longitude, country, confidence, name_key"
|
|
121
|
+
|
|
122
|
+
this.#categoryCellProbe = this.#db.prepare(
|
|
123
|
+
`SELECT ${columns} FROM poi WHERE h3_cell = ? AND category_id = ? ORDER BY neg_rank ASC LIMIT ?`
|
|
124
|
+
)
|
|
125
|
+
this.#brandCellProbe = this.#db.prepare(
|
|
126
|
+
`SELECT ${columns} FROM poi WHERE h3_cell = ? AND brand_wikidata = ? ORDER BY neg_rank ASC LIMIT ?`
|
|
127
|
+
)
|
|
128
|
+
this.#nameFTSProbe = this.#db.prepare(
|
|
129
|
+
"SELECT name_key FROM poi_search WHERE poi_search MATCH ? ORDER BY bm25(poi_search) LIMIT ?"
|
|
130
|
+
)
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
search(query: POISearchQuery): POISearchHit[] {
|
|
134
|
+
const limit = Math.max(1, query.limit ?? DEFAULT_LIMIT)
|
|
135
|
+
|
|
136
|
+
if (query.name) {
|
|
137
|
+
return this.#searchByName(query.name, limit, query.center)
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
if (query.categoryID || query.brandWikidata) {
|
|
141
|
+
if (!query.center) {
|
|
142
|
+
throw new Error("POILookup.search: category/brand search requires a `center`")
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
return this.#searchKRing(query, limit)
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
return []
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** Category or brand path: k-ring expansion from `query.center`'s res-9 cell, probing each new ring's cells. */
|
|
152
|
+
#searchKRing(query: POISearchQuery, limit: number): POISearchHit[] {
|
|
153
|
+
const center = query.center!
|
|
154
|
+
const maxRings = query.maxRings ?? DEFAULT_MAX_RINGS
|
|
155
|
+
let categoryId: number | undefined
|
|
156
|
+
|
|
157
|
+
// brandWikidata wins over categoryID when both are set — see POISearchQuery.categoryID.
|
|
158
|
+
if (!query.brandWikidata) {
|
|
159
|
+
categoryId = this.#categoryToID.get(query.categoryID!)
|
|
160
|
+
|
|
161
|
+
// An unknown category (not in the dictionary) can't have rows — a clean miss, not a throw.
|
|
162
|
+
if (categoryId === undefined) return []
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const origin = latLngToCell(center.latitude, center.longitude, POI_H3_RESOLUTION) as H3Cell
|
|
166
|
+
const seenCells = new Set<string>()
|
|
167
|
+
let rows: POIRow[] = []
|
|
168
|
+
|
|
169
|
+
// `ring` starts at 0 (the origin cell itself), so this loop's k reaches `maxRings - 1`.
|
|
170
|
+
for (let ring = 0; ring < maxRings; ring++) {
|
|
171
|
+
// gridDisk(origin, ring) returns the WHOLE disk out to `ring`; diffing against what's
|
|
172
|
+
// already been probed derives just this ring's new cells.
|
|
173
|
+
const diskCells = gridDisk(origin, ring) as string[]
|
|
174
|
+
const newCells = diskCells.filter((cell) => !seenCells.has(cell))
|
|
175
|
+
|
|
176
|
+
for (const cell of newCells) {
|
|
177
|
+
seenCells.add(cell)
|
|
178
|
+
const shortCell = h3CellToInt(cell as H3Cell)
|
|
179
|
+
|
|
180
|
+
const hits = query.brandWikidata
|
|
181
|
+
? (this.#brandCellProbe.all(shortCell, query.brandWikidata, limit) as unknown as POIRow[])
|
|
182
|
+
: (this.#categoryCellProbe.all(shortCell, categoryId!, limit) as unknown as POIRow[])
|
|
183
|
+
|
|
184
|
+
rows.push(...hits)
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
rows = sortByDistance(rows, center)
|
|
188
|
+
|
|
189
|
+
if (rows.length >= limit) break
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
return rows.slice(0, limit).map((row) => toHit(row, this.#idToCategory, center))
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Name path: FTS5 MATCH → hydrate by name_key. No center required; distance-sorts if one is given anyway.
|
|
197
|
+
*
|
|
198
|
+
* Hydration is ONE batched `WHERE name_key IN (...)` query over the FTS hits' unique `name_key`s, not a per-hit probe
|
|
199
|
+
* — with up to `limit` FTS hits, a per-hit probe was up to `limit` full table scans before `createPOINameKeyIndex`
|
|
200
|
+
* (poi-schema.ts) + this batching.
|
|
201
|
+
*/
|
|
202
|
+
#searchByName(name: string, limit: number, center?: { latitude: number; longitude: number }): POISearchHit[] {
|
|
203
|
+
const matchQuery = sanitizePOINameQuery(name)
|
|
204
|
+
|
|
205
|
+
if (!matchQuery) return []
|
|
206
|
+
|
|
207
|
+
const ftsHits = this.#nameFTSProbe.all(matchQuery, limit) as unknown as Array<{ name_key: string | null }>
|
|
208
|
+
const uniqueKeys: string[] = []
|
|
209
|
+
const seenKeys = new Set<string>()
|
|
210
|
+
|
|
211
|
+
for (const hit of ftsHits) {
|
|
212
|
+
if (!hit.name_key || seenKeys.has(hit.name_key)) continue
|
|
213
|
+
seenKeys.add(hit.name_key)
|
|
214
|
+
uniqueKeys.push(hit.name_key)
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
if (uniqueKeys.length === 0) return []
|
|
218
|
+
|
|
219
|
+
const hydrated = this.#hydrateByNameKeys(uniqueKeys)
|
|
220
|
+
const rowsByKey = new Map<string, POIRow[]>()
|
|
221
|
+
|
|
222
|
+
for (const row of hydrated) {
|
|
223
|
+
if (row.name_key === null) continue
|
|
224
|
+
const bucket = rowsByKey.get(row.name_key)
|
|
225
|
+
|
|
226
|
+
if (bucket) {
|
|
227
|
+
bucket.push(row)
|
|
228
|
+
} else {
|
|
229
|
+
rowsByKey.set(row.name_key, [row])
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
let rows: POIRow[] = []
|
|
234
|
+
|
|
235
|
+
for (const key of uniqueKeys) {
|
|
236
|
+
rows.push(...(rowsByKey.get(key) ?? []))
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
if (center) {
|
|
240
|
+
rows = sortByDistance(rows, center)
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
return rows.slice(0, limit).map((row) => toHit(row, this.#idToCategory, center))
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* Batched hydration for the FTS name path: `WHERE name_key IN (?, ?, …)`, one query for the whole batch instead of
|
|
248
|
+
* one probe per FTS hit. This is a cold path (name search only) with variable arity per call, so the statement is
|
|
249
|
+
* prepared fresh each time rather than cached.
|
|
250
|
+
*/
|
|
251
|
+
#hydrateByNameKeys(nameKeys: string[]): POIRow[] {
|
|
252
|
+
const columns = "name, category_id, brand_wikidata, latitude, longitude, country, confidence, name_key"
|
|
253
|
+
const placeholders = nameKeys.map(() => "?").join(", ")
|
|
254
|
+
const stmt = this.#db.prepare(`SELECT ${columns} FROM poi WHERE name_key IN (${placeholders})`)
|
|
255
|
+
|
|
256
|
+
return stmt.all(...nameKeys) as unknown as POIRow[]
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
close(): void {
|
|
260
|
+
if (this.#ownsDB) {
|
|
261
|
+
this.#db.close()
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
[Symbol.dispose](): void {
|
|
266
|
+
this.close()
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/** `poi.h3_cell` is the SHORTENED (48-bit) cell — never the full h3-js cell string. */
|
|
271
|
+
function h3CellToInt(cell: H3Cell): number {
|
|
272
|
+
return Number(BigInt(`0x${shortenH3Cell(cell)}`))
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
function sortByDistance(rows: POIRow[], center: { latitude: number; longitude: number }): POIRow[] {
|
|
276
|
+
return [...rows].sort(
|
|
277
|
+
(a, b) =>
|
|
278
|
+
haversineKm(center.latitude, center.longitude, a.latitude, a.longitude) -
|
|
279
|
+
haversineKm(center.latitude, center.longitude, b.latitude, b.longitude)
|
|
280
|
+
)
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
function toHit(
|
|
284
|
+
row: POIRow,
|
|
285
|
+
idToCategory: ReadonlyMap<number, string>,
|
|
286
|
+
center?: { latitude: number; longitude: number }
|
|
287
|
+
): POISearchHit {
|
|
288
|
+
return {
|
|
289
|
+
name: row.name,
|
|
290
|
+
categoryID: row.category_id !== 0 ? (idToCategory.get(row.category_id) ?? null) : null,
|
|
291
|
+
brandWikidata: row.brand_wikidata,
|
|
292
|
+
latitude: row.latitude,
|
|
293
|
+
longitude: row.longitude,
|
|
294
|
+
country: row.country,
|
|
295
|
+
confidence: row.confidence,
|
|
296
|
+
...(center
|
|
297
|
+
? { distanceM: haversineKm(center.latitude, center.longitude, row.latitude, row.longitude) * 1000 }
|
|
298
|
+
: {}),
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/**
|
|
303
|
+
* Sanitize free text into an FTS5-safe MATCH query: strip the characters FTS5 would otherwise read as syntax (`"`
|
|
304
|
+
* phrase delimiters, `*` prefix wildcards, `:` column-filter separators), then phrase-quote each whitespace-separated
|
|
305
|
+
* token (AND-joined).
|
|
306
|
+
*
|
|
307
|
+
* `resolver-wof-sqlite` already has this discipline — `lookup.ts`'s `sanitizeFTSQuery` — but that function is
|
|
308
|
+
* module-private there (not re-exported from `fts.ts` or the package's `index.ts`), so this replicates the same
|
|
309
|
+
* discipline locally rather than reaching across the module boundary for a private helper.
|
|
310
|
+
*/
|
|
311
|
+
function sanitizePOINameQuery(text: string): string {
|
|
312
|
+
return text
|
|
313
|
+
.replace(/["*:]/g, "")
|
|
314
|
+
.trim()
|
|
315
|
+
.split(/\s+/u)
|
|
316
|
+
.filter(Boolean)
|
|
317
|
+
.map((token) => `"${token.replace(/"/g, '""')}"`)
|
|
318
|
+
.join(" ")
|
|
319
|
+
}
|
package/poi-schema.ts
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Typed schema for poi.db — spatial layer #1 (spec §3.4). One clustered `WITHOUT ROWID` B-tree
|
|
7
|
+
* keyed `(h3_cell, category_id, neg_rank, rowid_key)` so "everything near this res-9 cell" is a
|
|
8
|
+
* contiguous key range (the byte-range/httpvfs access pattern, same discipline as the candidate
|
|
9
|
+
* gazetteer). Rows carry denormalized name/brand/coords; category ids are small ints via the
|
|
10
|
+
* `poi_category_codes` dictionary (poi-taxonomy category ids are the string side). The DB also
|
|
11
|
+
* embeds the layer-contract tables from `@mailwoman/core/layers` — the builder writes the
|
|
12
|
+
* manifest (tier `shipped`, spine `h3` res 9) and per-res-6-cell coverage.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import type { DatabaseSync } from "node:sqlite"
|
|
16
|
+
|
|
17
|
+
import type { LayerContractDatabase } from "@mailwoman/core/layers"
|
|
18
|
+
import { sql, type Kysely } from "kysely"
|
|
19
|
+
|
|
20
|
+
/** One POI row. Clustered PK: h3_cell → category_id → neg_rank → rowid_key. */
|
|
21
|
+
export interface POITable {
|
|
22
|
+
/** 48-bit short H3 cell at res 9 (`latLngToCell` → `shortenH3Cell`). */
|
|
23
|
+
h3_cell: number
|
|
24
|
+
/** Small int from {@link POICategoryCodeTable}; 0 = uncategorized. */
|
|
25
|
+
category_id: number
|
|
26
|
+
/** `-log10(confidence + epsilon)` so ASC = most-confident-first within a cell+category. */
|
|
27
|
+
neg_rank: number
|
|
28
|
+
/** Uniquifier within the clustered key (builder-assigned monotonic int). */
|
|
29
|
+
rowid_key: number
|
|
30
|
+
name: string | null
|
|
31
|
+
/** Lowercased, diacritic-flattened probe key for exact name lookups. */
|
|
32
|
+
name_key: string | null
|
|
33
|
+
brand_wikidata: string | null
|
|
34
|
+
latitude: number
|
|
35
|
+
longitude: number
|
|
36
|
+
/** ISO 3166-1 alpha-2 (from the Overture partition). */
|
|
37
|
+
country: string
|
|
38
|
+
/** Overture existence confidence (already filtered ≥ 0.85 at build). */
|
|
39
|
+
confidence: number
|
|
40
|
+
/** GERS id — nullable METADATA ONLY, never a key (the #470 rule). */
|
|
41
|
+
gers_id: string | null
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Staging mirror — every column nullable except the coords (the loader fills positionally; the materialize SELECT
|
|
46
|
+
* enforces completeness).
|
|
47
|
+
*/
|
|
48
|
+
export interface POIStageTable {
|
|
49
|
+
h3_cell: number | null
|
|
50
|
+
category_id: number | null
|
|
51
|
+
neg_rank: number | null
|
|
52
|
+
rowid_key: number | null
|
|
53
|
+
name: string | null
|
|
54
|
+
name_key: string | null
|
|
55
|
+
brand_wikidata: string | null
|
|
56
|
+
latitude: number
|
|
57
|
+
longitude: number
|
|
58
|
+
country: string | null
|
|
59
|
+
confidence: number | null
|
|
60
|
+
gers_id: string | null
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** `(id → poi-taxonomy category id)` dictionary, e.g. `3 → "cafe"`. */
|
|
64
|
+
export interface POICategoryCodeTable {
|
|
65
|
+
id: number
|
|
66
|
+
category: string
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export interface POIDatabase extends LayerContractDatabase {
|
|
70
|
+
poi: POITable
|
|
71
|
+
poi_stage: POIStageTable
|
|
72
|
+
poi_category_codes: POICategoryCodeTable
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Clustered-key-order column list shared by builder + `INSERT INTO poi SELECT … FROM poi_stage`. */
|
|
76
|
+
export const POI_COLUMNS = [
|
|
77
|
+
"h3_cell",
|
|
78
|
+
"category_id",
|
|
79
|
+
"neg_rank",
|
|
80
|
+
"rowid_key",
|
|
81
|
+
"name",
|
|
82
|
+
"name_key",
|
|
83
|
+
"brand_wikidata",
|
|
84
|
+
"latitude",
|
|
85
|
+
"longitude",
|
|
86
|
+
"country",
|
|
87
|
+
"confidence",
|
|
88
|
+
"gers_id",
|
|
89
|
+
] as const
|
|
90
|
+
|
|
91
|
+
export async function createPOIStagingTables(db: Kysely<POIDatabase>): Promise<void> {
|
|
92
|
+
await db.schema
|
|
93
|
+
.createTable("poi_category_codes")
|
|
94
|
+
.addColumn("id", "integer", (c) => c.primaryKey())
|
|
95
|
+
.addColumn("category", "text", (c) => c.unique())
|
|
96
|
+
.execute()
|
|
97
|
+
await db.schema
|
|
98
|
+
.createTable("poi_stage")
|
|
99
|
+
.addColumn("h3_cell", "integer")
|
|
100
|
+
.addColumn("category_id", "integer")
|
|
101
|
+
.addColumn("neg_rank", "real")
|
|
102
|
+
.addColumn("rowid_key", "integer")
|
|
103
|
+
.addColumn("name", "text")
|
|
104
|
+
.addColumn("name_key", "text")
|
|
105
|
+
.addColumn("brand_wikidata", "text")
|
|
106
|
+
.addColumn("latitude", "real")
|
|
107
|
+
.addColumn("longitude", "real")
|
|
108
|
+
.addColumn("country", "text")
|
|
109
|
+
.addColumn("confidence", "real")
|
|
110
|
+
.addColumn("gers_id", "text")
|
|
111
|
+
.execute()
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export async function createPOITable(db: Kysely<POIDatabase>): Promise<void> {
|
|
115
|
+
await db.schema
|
|
116
|
+
.createTable("poi")
|
|
117
|
+
.addColumn("h3_cell", "integer", (c) => c.notNull())
|
|
118
|
+
.addColumn("category_id", "integer", (c) => c.notNull())
|
|
119
|
+
.addColumn("neg_rank", "real", (c) => c.notNull())
|
|
120
|
+
.addColumn("rowid_key", "integer", (c) => c.notNull())
|
|
121
|
+
.addColumn("name", "text")
|
|
122
|
+
.addColumn("name_key", "text")
|
|
123
|
+
.addColumn("brand_wikidata", "text")
|
|
124
|
+
.addColumn("latitude", "real", (c) => c.notNull())
|
|
125
|
+
.addColumn("longitude", "real", (c) => c.notNull())
|
|
126
|
+
.addColumn("country", "text", (c) => c.notNull())
|
|
127
|
+
.addColumn("confidence", "real", (c) => c.notNull())
|
|
128
|
+
.addColumn("gers_id", "text")
|
|
129
|
+
.addPrimaryKeyConstraint("poi_pk", ["h3_cell", "category_id", "neg_rank", "rowid_key"])
|
|
130
|
+
// `WITHOUT ROWID` has no first-class builder; the raw modifier is the idiomatic fallback.
|
|
131
|
+
.modifyEnd(sql`without rowid`)
|
|
132
|
+
.execute()
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Secondary index for the FTS-hydration path. Builders call this AFTER the bulk materialize (index-after-load). */
|
|
136
|
+
export async function createPOINameKeyIndex(db: Kysely<POIDatabase>): Promise<void> {
|
|
137
|
+
await db.schema.createIndex("poi_name_key").on("poi").column("name_key").execute()
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export const POI_FTS_TABLE = "poi_search"
|
|
141
|
+
|
|
142
|
+
/** FTS5 stays raw SQL by project rule (Kysely can't express virtual tables). Content-keyed by name_key. */
|
|
143
|
+
export function createPOISearchFTS(db: DatabaseSync): void {
|
|
144
|
+
db.exec(
|
|
145
|
+
`CREATE VIRTUAL TABLE ${POI_FTS_TABLE} USING fts5(name, name_key UNINDEXED, h3_cell UNINDEXED, tokenize = 'unicode61')`
|
|
146
|
+
)
|
|
147
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Node reader over the POSTAL-CITY ALIAS table (`postal-city-alias-<cc>.db`) — the observed
|
|
7
|
+
* `postal_city → geo_locality` aliases per postcode (`build-postal-city-alias.ts`). Consumed by
|
|
8
|
+
* {@link WOFSqlitePlaceLookup}'s coordinate-first locality scorer: a user-typed postal city
|
|
9
|
+
* ("Antioch", postcode 37013) becomes a name-match alias for the geographic locality the postcode
|
|
10
|
+
* actually sits in ("Nashville"), so the right place tiers to the top instead of a same-named
|
|
11
|
+
* town in another state. Opt-in — the lookup is only constructed when a path is supplied, and
|
|
12
|
+
* absent it the resolver is byte-identical.
|
|
13
|
+
*
|
|
14
|
+
* The reader returns RAW divergent rows for a postcode; normalization + name-matching against the
|
|
15
|
+
* candidate localities is the scorer's job (it owns the case/diacritic fold the soft name score
|
|
16
|
+
* uses), keeping one normalizer in one place.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { DatabaseSync } from "node:sqlite"
|
|
20
|
+
|
|
21
|
+
import { DatabaseClient } from "@mailwoman/core/kysley/client"
|
|
22
|
+
|
|
23
|
+
import type { PostalCityAliasDatabase } from "./postal-city-alias-schema.ts"
|
|
24
|
+
|
|
25
|
+
export interface WOFPostalCityAliasLookupOpts {
|
|
26
|
+
/** Path to a `postal-city-alias-<cc>.db` built by `build-postal-city-alias.ts`. Opened read-only. */
|
|
27
|
+
databasePath?: string
|
|
28
|
+
/** Pre-opened handle (tests / shared connections). Mutually exclusive with `databasePath`. */
|
|
29
|
+
database?: DatabaseSync
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** One divergent alias edge: the postal-system name and the geographic locality it maps to. */
|
|
33
|
+
export interface PostalCityAlias {
|
|
34
|
+
/** The postal-system surface (what a user types). */
|
|
35
|
+
postalCity: string
|
|
36
|
+
/** The geographic locality name the postcode sits in (≈ the gazetteer's canonical name). */
|
|
37
|
+
geoLocality: string
|
|
38
|
+
/** Observed usage count — the evidence weight. */
|
|
39
|
+
n: number
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Reader over `postal_city_alias`. The only query is a postcode-scoped probe for DIVERGENT rows (where the postal name
|
|
44
|
+
* differs from the geographic name — the rows that carry alias signal), issued via the typed Kysely query builder
|
|
45
|
+
* against {@link PostalCityAliasDatabase}.
|
|
46
|
+
*/
|
|
47
|
+
export class WOFPostalCityAliasLookup {
|
|
48
|
+
#db: DatabaseSync
|
|
49
|
+
#kdb: DatabaseClient<PostalCityAliasDatabase>
|
|
50
|
+
#ownsDB: boolean
|
|
51
|
+
|
|
52
|
+
constructor(opts: WOFPostalCityAliasLookupOpts) {
|
|
53
|
+
if (opts.database) {
|
|
54
|
+
this.#db = opts.database
|
|
55
|
+
this.#ownsDB = false
|
|
56
|
+
} else if (opts.databasePath) {
|
|
57
|
+
this.#db = new DatabaseSync(opts.databasePath, { readOnly: true })
|
|
58
|
+
this.#ownsDB = true
|
|
59
|
+
} else {
|
|
60
|
+
throw new Error("WOFPostalCityAliasLookup needs `databasePath` or `database`")
|
|
61
|
+
}
|
|
62
|
+
// `#kdb` wraps `#db` for the typed query; close() owns the raw handle directly (sync).
|
|
63
|
+
this.#kdb = new DatabaseClient<PostalCityAliasDatabase>({ database: this.#db })
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Divergent postal-city aliases for a postcode (empty when the postcode isn't in the table). The scorer groups these
|
|
68
|
+
* by normalized `geoLocality` and appends the `postalCity` surfaces to the matching candidate locality's alias set.
|
|
69
|
+
*/
|
|
70
|
+
async getDivergentAliases(postcode: string): Promise<PostalCityAlias[]> {
|
|
71
|
+
const pc = postcode.trim()
|
|
72
|
+
|
|
73
|
+
if (!pc) return []
|
|
74
|
+
const rows = await this.#kdb
|
|
75
|
+
.selectFrom("postal_city_alias")
|
|
76
|
+
.select(["postal_city", "geo_locality", "n"])
|
|
77
|
+
.where("postcode", "=", pc)
|
|
78
|
+
.where("divergent", "=", 1)
|
|
79
|
+
.execute()
|
|
80
|
+
|
|
81
|
+
return rows.map((r) => ({ postalCity: String(r.postal_city), geoLocality: String(r.geo_locality), n: Number(r.n) }))
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
close(): void {
|
|
85
|
+
if (this.#ownsDB) {
|
|
86
|
+
this.#db.close()
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Typed schema for the POSTAL-CITY ALIAS table (`postal-city-alias-<cc>.db`, built by
|
|
7
|
+
* `scripts/build-postal-city-alias.ts`) — the single source of truth for the columns shared by
|
|
8
|
+
* the BUILDER and the READER ({@link WOFPostalCityAliasLookup}). Like {@link CandidateTable}, the
|
|
9
|
+
* contract is a Kysely `Database` interface plus the table DDL as a string, so a column rename in
|
|
10
|
+
* the builder is a compile error in the reader.
|
|
11
|
+
*
|
|
12
|
+
* Provenance discipline (provenance-first): this is a SIBLING table to the PIP-derived
|
|
13
|
+
* `postcode_locality` data, never mixed into it — one table, one provenance class. Each row is an
|
|
14
|
+
* OBSERVED `(postcode, postal_city, geo_locality)` aggregate from Overture's `postal_city` field
|
|
15
|
+
* with a usage count `n`; `divergent = 1` exactly when `postal_city != geo_locality` (the alias
|
|
16
|
+
* signal — the only rows the resolver consumes).
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import type { Kysely } from "kysely"
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* One observed postal-city aggregate. The natural key is `(postcode, postal_city, geo_locality)`; the builder enforces
|
|
23
|
+
* a `MIN_COUNT` floor on `n`, so every row is a non-trivial usage.
|
|
24
|
+
*/
|
|
25
|
+
export interface PostalCityAliasTable {
|
|
26
|
+
/** The postcode the aggregate is scoped to (the resolver probes by this). */
|
|
27
|
+
postcode: string
|
|
28
|
+
/** What the postal system calls the place (the surface a user is likely to type). */
|
|
29
|
+
postal_city: string
|
|
30
|
+
/** The geographic locality name the postcode actually sits in (≈ the gazetteer's canonical name). */
|
|
31
|
+
geo_locality: string
|
|
32
|
+
/** Observed row count — the evidence weight behind this alias. */
|
|
33
|
+
n: number
|
|
34
|
+
/** 1 when `postal_city != geo_locality` (the alias signal); 0 when they agree. */
|
|
35
|
+
divergent: number
|
|
36
|
+
/** Provenance: the dataset this aggregate came from (e.g. `overture:US`). */
|
|
37
|
+
source: string
|
|
38
|
+
/** The pinned data release the aggregate was computed from. */
|
|
39
|
+
release: string
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** The postal-city-alias database schema for `new DatabaseClient<PostalCityAliasDatabase>(...)`. */
|
|
43
|
+
export interface PostalCityAliasDatabase {
|
|
44
|
+
postal_city_alias: PostalCityAliasTable
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** The `postal_city_alias` column order — the builder's INSERT derives its column list from this. */
|
|
48
|
+
export const POSTAL_CITY_ALIAS_COLUMNS = [
|
|
49
|
+
"postcode",
|
|
50
|
+
"postal_city",
|
|
51
|
+
"geo_locality",
|
|
52
|
+
"n",
|
|
53
|
+
"divergent",
|
|
54
|
+
"source",
|
|
55
|
+
"release",
|
|
56
|
+
] as const
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Create the `postal_city_alias` table + its two probe indexes. Kept here (not only in the builder) so tests can stand
|
|
60
|
+
* up a fixture DB with the exact production shape. Pass a {@link DatabaseClient} (or any `Kysely`) over the alias DB.
|
|
61
|
+
*/
|
|
62
|
+
export async function createPostalCityAliasTable(db: Kysely<PostalCityAliasDatabase>): Promise<void> {
|
|
63
|
+
await db.schema
|
|
64
|
+
.createTable("postal_city_alias")
|
|
65
|
+
.addColumn("postcode", "text", (c) => c.notNull())
|
|
66
|
+
.addColumn("postal_city", "text", (c) => c.notNull())
|
|
67
|
+
.addColumn("geo_locality", "text", (c) => c.notNull())
|
|
68
|
+
.addColumn("n", "integer", (c) => c.notNull())
|
|
69
|
+
.addColumn("divergent", "integer", (c) => c.notNull())
|
|
70
|
+
.addColumn("source", "text", (c) => c.notNull())
|
|
71
|
+
.addColumn("release", "text", (c) => c.notNull())
|
|
72
|
+
.execute()
|
|
73
|
+
await db.schema.createIndex("idx_pca_postcode").on("postal_city_alias").column("postcode").execute()
|
|
74
|
+
await db.schema.createIndex("idx_pca_pair").on("postal_city_alias").columns(["postal_city", "geo_locality"]).execute()
|
|
75
|
+
}
|