@mailwoman/resolver-wof-sqlite 7.2.0 → 7.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/address-point-interpolation.ts +207 -0
- package/address-point-schema.ts +107 -0
- package/address-point.ts +122 -0
- package/ancestry-backfill.ts +205 -0
- package/ancestry.ts +70 -0
- package/build-candidate.ts +351 -0
- package/build-slim.ts +394 -0
- package/candidate-fts.ts +43 -0
- package/candidate-lookup.ts +382 -0
- package/candidate-schema.ts +166 -0
- package/coincident-roles.ts +240 -0
- package/convention.ts +152 -0
- package/fst-autocomplete.ts +187 -0
- package/fst-builder.ts +291 -0
- package/fst-deserialize-web.ts +164 -0
- package/fst-matcher.ts +150 -0
- package/fst-serialize.ts +311 -0
- package/fst-types.ts +78 -0
- package/fts.ts +318 -0
- package/geo.ts +140 -0
- package/geonames-aliases.ts +317 -0
- package/geonames-postal.ts +150 -0
- package/index.ts +117 -0
- package/interpolation.ts +232 -0
- package/lookup.ts +1498 -0
- package/out/poi-lookup.d.ts +14 -2
- package/out/poi-lookup.d.ts.map +1 -1
- package/out/poi-lookup.js +55 -21
- package/out/poi-lookup.js.map +1 -1
- package/out/poi-schema.d.ts +9 -0
- package/out/poi-schema.d.ts.map +1 -1
- package/out/poi-schema.js +16 -0
- package/out/poi-schema.js.map +1 -1
- package/out/reverse.d.ts +8 -1
- package/out/reverse.d.ts.map +1 -1
- package/out/reverse.js +10 -1
- package/out/reverse.js.map +1 -1
- package/package.json +168 -82
- package/poi-lookup.ts +375 -0
- package/poi-schema.ts +164 -0
- package/postal-city-alias-lookup.ts +89 -0
- package/postal-city-alias-schema.ts +75 -0
- package/postal-city-candidate-schema.ts +81 -0
- package/postcode-point-lookup.ts +64 -0
- package/reverse.ts +439 -0
- package/schema.ts +176 -0
- package/sharding.ts +235 -0
- package/sqlite-convention-source.ts +61 -0
- package/sqlite-utils.ts +25 -0
- package/street-centroid-schema.ts +124 -0
- package/street-centroid.ts +124 -0
- package/street-morphology-fst-builder.ts +230 -0
- package/street-name-lookup.ts +101 -0
- package/street-normalize.ts +302 -0
- package/street-segment-schema.ts +104 -0
- package/types.ts +164 -0
- package/unified-schema.ts +171 -0
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Node-side {@link PlaceLookup} over the byte-range CANDIDATE table (`build-candidate.ts`) — the
|
|
7
|
+
* SAME gazetteer the browser demo resolves against ({@link WOFCandidateTableLookup} in
|
|
8
|
+
* `docs/src/shared/httpvfs-resolver.ts`), but reading a LOCAL `candidate.db` via `node:sqlite`
|
|
9
|
+
* instead of sql.js-httpvfs. This is what makes the server/CLI resolver match the demo: one
|
|
10
|
+
* lookup surface, one artifact, one ranking.
|
|
11
|
+
*
|
|
12
|
+
* The query is a single contiguous probe on the `WITHOUT ROWID` B-tree keyed `(name_key,
|
|
13
|
+
* country_id, region_id, placetype_id, neg_rank, spr_id)`. `name_key` is the SHARED
|
|
14
|
+
* {@link normalizeLocalityForKey} (build- and query-consistent), each row is denormalized (display
|
|
15
|
+
* `name`, centroid, bbox), and population rank is precomputed into `neg_rank` — so the result is
|
|
16
|
+
* POPULATION-FIRST and COUNTRY-AGNOSTIC (when no `country` filter is given), exactly like the
|
|
17
|
+
* demo. That's the deliberate divergence from {@link WOFSqlitePlaceLookup}'s FTS/bm25 ranking: a
|
|
18
|
+
* bare "Moscow" resolves to the 10.4 M-pop Russian city, not whichever same-name US township bm25
|
|
19
|
+
* floats to the top.
|
|
20
|
+
*
|
|
21
|
+
* Disambiguation rides the same mechanism the cascade already uses: a parsed region resolves to its
|
|
22
|
+
* stored bbox and the locality query is point-in-bbox-filtered on the candidate centroid (the
|
|
23
|
+
* `bbox` field on {@link FindPlaceQuery}).
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { DatabaseSync } from "node:sqlite"
|
|
27
|
+
|
|
28
|
+
import { expandPlacetypeFilter } from "@mailwoman/resolver"
|
|
29
|
+
|
|
30
|
+
import { CANDIDATE_FTS_TABLE } from "./candidate-fts.ts"
|
|
31
|
+
import type { CandidateTable, CountryCodeTable, PlacetypeCodeTable } from "./candidate-schema.ts"
|
|
32
|
+
import { haversineKm } from "./geo.ts"
|
|
33
|
+
import { trigramJaccard } from "./lookup.ts"
|
|
34
|
+
import { POSTAL_CITY_CANDIDATE_TABLE, type PostalCityCandidateTable } from "./postal-city-candidate-schema.ts"
|
|
35
|
+
import { hasTable } from "./sqlite-utils.ts"
|
|
36
|
+
import { normalizeLocalityForKey, stripLocalityQualifier } from "./street-normalize.ts"
|
|
37
|
+
import type { FindPlaceQuery, PlaceCandidate, PlaceLookup, WOFPlacetype } from "./types.ts"
|
|
38
|
+
|
|
39
|
+
export interface WOFCandidateTableLookupOpts {
|
|
40
|
+
/** Path to a `candidate.db` built by `build-candidate.ts`. Opened read-only. */
|
|
41
|
+
databasePath?: string
|
|
42
|
+
/** Pre-opened handle (tests / shared connections). Mutually exclusive with `databasePath`. */
|
|
43
|
+
database?: DatabaseSync
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* The candidate columns this lookup probes — a typed projection of the SHARED {@link CandidateTable}, so a column rename
|
|
48
|
+
* in `build-candidate` (the writer) is a compile error here (the reader).
|
|
49
|
+
*/
|
|
50
|
+
type CandidateRow = Pick<
|
|
51
|
+
CandidateTable,
|
|
52
|
+
| "spr_id"
|
|
53
|
+
| "name"
|
|
54
|
+
| "country_id"
|
|
55
|
+
| "placetype_id"
|
|
56
|
+
| "latitude"
|
|
57
|
+
| "longitude"
|
|
58
|
+
| "min_lat"
|
|
59
|
+
| "min_lon"
|
|
60
|
+
| "max_lat"
|
|
61
|
+
| "max_lon"
|
|
62
|
+
| "neg_rank"
|
|
63
|
+
>
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* FTS5-trigram over-fetch before the trigram-Jaccard re-rank, and the minimum similarity to count as a fuzzy hit (below
|
|
67
|
+
* it the trigram overlap is noise, e.g. unrelated same-trigram names). Tunable.
|
|
68
|
+
*/
|
|
69
|
+
const FUZZY_FETCH = 40
|
|
70
|
+
const FUZZY_MIN = 0.34
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Unpadded character-trigrams of `s`, OR'd into an FTS5 trigram MATCH query (each quoted so FTS treats it as a literal
|
|
74
|
+
* term). Returns "" when `s` is shorter than a trigram or yields no clean grams — the caller then skips the fuzzy
|
|
75
|
+
* probe.
|
|
76
|
+
*/
|
|
77
|
+
function ftsTrigramQuery(s: string): string {
|
|
78
|
+
const grams = new Set<string>()
|
|
79
|
+
|
|
80
|
+
for (let i = 0; i + 3 <= s.length; i++) {
|
|
81
|
+
const g = s.slice(i, i + 3)
|
|
82
|
+
|
|
83
|
+
if (/^[\p{L}\p{N} ]{3}$/u.test(g)) {
|
|
84
|
+
grams.add(g)
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
return [...grams].map((g) => `"${g}"`).join(" OR ")
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Node {@link PlaceLookup} over `candidate.db`. Drop-in for {@link WOFSqlitePlaceLookup} in `createWOFResolver(backend)`
|
|
93
|
+
* — same `findPlace` contract, population-first ranking.
|
|
94
|
+
*/
|
|
95
|
+
export class WOFCandidateTableLookup implements PlaceLookup {
|
|
96
|
+
#db: DatabaseSync
|
|
97
|
+
#ownsDB: boolean
|
|
98
|
+
readonly #countryToID = new Map<string, number>()
|
|
99
|
+
readonly #idToCountry = new Map<number, string>()
|
|
100
|
+
readonly #placetypeToID = new Map<string, number>()
|
|
101
|
+
readonly #idToPlacetype = new Map<number, string>()
|
|
102
|
+
/**
|
|
103
|
+
* Prepared `(name_key, postcode)` probe for the #741 postal-city side-index — `undefined` when the
|
|
104
|
+
* `postal_city_candidate` table isn't present, so a candidate.db built without it is byte-stable.
|
|
105
|
+
*/
|
|
106
|
+
readonly #postalCityProbe: ReturnType<DatabaseSync["prepare"]> | undefined
|
|
107
|
+
/**
|
|
108
|
+
* Prepared FTS5-trigram MATCH probe for the typo-tolerant fallback — `undefined` when the `candidate_fts` index isn't
|
|
109
|
+
* present, so a candidate.db built without it is byte-stable (the fuzzy path is skipped, exactly like today).
|
|
110
|
+
*/
|
|
111
|
+
readonly #ftsProbe: ReturnType<DatabaseSync["prepare"]> | undefined
|
|
112
|
+
/**
|
|
113
|
+
* Prepared UNFILTERED existence probe (`name_key` present anywhere, ignoring country/placetype/bbox). Gates the fuzzy
|
|
114
|
+
* fallback: fuzzy is a TYPO corrector, so it engages only when the name doesn't exist in the gazetteer at all. A name
|
|
115
|
+
* that DOES exist but missed under the active filter is a filter miss (e.g. a placer misroute "Vienna, Austria"→IT),
|
|
116
|
+
* not a spelling miss — fuzzing it would scrape an unrelated same-country place and defeat the cascade's
|
|
117
|
+
* country-agnostic retry. Prepared only alongside `#ftsProbe`.
|
|
118
|
+
*/
|
|
119
|
+
readonly #nameKeyExistsProbe: ReturnType<DatabaseSync["prepare"]> | undefined
|
|
120
|
+
|
|
121
|
+
constructor(opts: WOFCandidateTableLookupOpts) {
|
|
122
|
+
if (opts.database) {
|
|
123
|
+
this.#db = opts.database
|
|
124
|
+
this.#ownsDB = false
|
|
125
|
+
} else if (opts.databasePath) {
|
|
126
|
+
this.#db = new DatabaseSync(opts.databasePath, { readOnly: true })
|
|
127
|
+
this.#ownsDB = true
|
|
128
|
+
} else {
|
|
129
|
+
throw new Error("WOFCandidateTableLookup needs `databasePath` or `database`")
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// The code tables are tiny (country/placetype dictionaries) — load them once at construction so
|
|
133
|
+
// `findPlace` is a single B-tree probe with no dictionary round-trip.
|
|
134
|
+
for (const r of this.#db.prepare("SELECT id, code FROM country_codes").all() as unknown as CountryCodeTable[]) {
|
|
135
|
+
const code = String(r.code).toUpperCase()
|
|
136
|
+
this.#countryToID.set(code, Number(r.id))
|
|
137
|
+
this.#idToCountry.set(Number(r.id), code)
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
for (const r of this.#db
|
|
141
|
+
.prepare("SELECT id, placetype FROM placetype_codes")
|
|
142
|
+
.all() as unknown as PlacetypeCodeTable[]) {
|
|
143
|
+
this.#placetypeToID.set(String(r.placetype), Number(r.id))
|
|
144
|
+
this.#idToPlacetype.set(Number(r.id), String(r.placetype))
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// #741 postal-city side-index: prepare the exact probe only if the table is present. Absent →
|
|
148
|
+
// `#postalCityProbe` stays undefined → findPlace skips the postal-city path → byte-stable.
|
|
149
|
+
if (hasTable(this.#db, POSTAL_CITY_CANDIDATE_TABLE)) {
|
|
150
|
+
this.#postalCityProbe = this.#db.prepare(
|
|
151
|
+
`SELECT spr_id, name, latitude, longitude FROM ${POSTAL_CITY_CANDIDATE_TABLE} WHERE name_key = ? AND postcode = ? LIMIT 1`
|
|
152
|
+
)
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// FTS5-trigram fuzzy fallback: prepare the MATCH probe only if the index is present (the unified
|
|
156
|
+
// gazetteer carries it; an older candidate.db doesn't → the fuzzy path is skipped, byte-stable).
|
|
157
|
+
if (hasTable(this.#db, CANDIDATE_FTS_TABLE)) {
|
|
158
|
+
this.#ftsProbe = this.#db.prepare(
|
|
159
|
+
`SELECT name_key FROM ${CANDIDATE_FTS_TABLE} WHERE ${CANDIDATE_FTS_TABLE} MATCH ? ORDER BY bm25(${CANDIDATE_FTS_TABLE}) LIMIT ?`
|
|
160
|
+
)
|
|
161
|
+
this.#nameKeyExistsProbe = this.#db.prepare("SELECT 1 FROM candidate WHERE name_key = ? LIMIT 1")
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** Does this query want a locality-tier place? Postal-city aliases (#741) are all localities. */
|
|
166
|
+
#wantsLocality(placetype: FindPlaceQuery["placetype"]): boolean {
|
|
167
|
+
if (!placetype) return true
|
|
168
|
+
const want = Array.isArray(placetype) ? placetype : [placetype]
|
|
169
|
+
|
|
170
|
+
return expandPlacetypeFilter(want as readonly string[]).includes("locality")
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
async findPlace(query: FindPlaceQuery): Promise<PlaceCandidate[]> {
|
|
174
|
+
let text = (query.text ?? "").trim()
|
|
175
|
+
|
|
176
|
+
if (!text) return []
|
|
177
|
+
|
|
178
|
+
// #920 name law, candidate-key edition: postcode rows are keyed by their whitespace-stripped
|
|
179
|
+
// form at build (the GeoNames fold normalizes '624 66' → '62466'), so a postcode-typed query
|
|
180
|
+
// strips internal whitespace before keying. Postcode-only — locality names keep their spaces.
|
|
181
|
+
if ([query.placetype].flat().includes("postalcode")) {
|
|
182
|
+
text = text.replace(/\s+/g, "")
|
|
183
|
+
}
|
|
184
|
+
const nameKey = normalizeLocalityForKey(text)
|
|
185
|
+
|
|
186
|
+
if (!nameKey) return []
|
|
187
|
+
|
|
188
|
+
// #741: postcode-keyed postal-city alias. An exact `(name_key, postcode)` hit resolves a
|
|
189
|
+
// user-typed POSTAL city ("Antioch", 37013) to the geographic locality the postcode sits in
|
|
190
|
+
// ("Nashville"), bypassing the population/region ranking that can't see the postcode. Gated on
|
|
191
|
+
// the side-index being present, a postcode in the query, and a locality-tier request — so the
|
|
192
|
+
// common (no-postcode / non-locality) path is untouched. A hit short-circuits: the postcode is
|
|
193
|
+
// an exact, high-confidence disambiguator, so we return the single geographic locality.
|
|
194
|
+
if (query.postcode && this.#postalCityProbe && this.#wantsLocality(query.placetype)) {
|
|
195
|
+
const hit = this.#postalCityProbe.get(nameKey, query.postcode.trim()) as
|
|
196
|
+
| Pick<PostalCityCandidateTable, "spr_id" | "name" | "latitude" | "longitude">
|
|
197
|
+
| undefined
|
|
198
|
+
|
|
199
|
+
if (hit) {
|
|
200
|
+
return [
|
|
201
|
+
{
|
|
202
|
+
id: Number(hit.spr_id),
|
|
203
|
+
name: String(hit.name ?? ""),
|
|
204
|
+
placetype: "locality" as WOFPlacetype,
|
|
205
|
+
country: query.country?.toUpperCase() ?? "",
|
|
206
|
+
lat: Number(hit.latitude),
|
|
207
|
+
lon: Number(hit.longitude),
|
|
208
|
+
score: 1,
|
|
209
|
+
exactMatch: true,
|
|
210
|
+
},
|
|
211
|
+
]
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
const limit = Math.max(1, query.limit ?? 10)
|
|
216
|
+
|
|
217
|
+
// Filter conds shared by the exact-key + strip-fallback probes (everything but name_key).
|
|
218
|
+
const filters: string[] = []
|
|
219
|
+
const filterParams: Array<string | number> = []
|
|
220
|
+
|
|
221
|
+
if (query.country) {
|
|
222
|
+
const cid = this.#countryToID.get(query.country.toUpperCase())
|
|
223
|
+
|
|
224
|
+
if (cid === undefined) return [] // a country the candidate table doesn't carry
|
|
225
|
+
filters.push("country_id = ?")
|
|
226
|
+
filterParams.push(cid)
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
if (query.placetype) {
|
|
230
|
+
// Shared placetype-equivalence expansion (a `locality` query must also reach borough /
|
|
231
|
+
// localadmin). `postalcode` maps to no admin placetype here → empty → no rows.
|
|
232
|
+
const want = Array.isArray(query.placetype) ? query.placetype : [query.placetype]
|
|
233
|
+
const ids = expandPlacetypeFilter(want as readonly string[])
|
|
234
|
+
.map((t) => this.#placetypeToID.get(t))
|
|
235
|
+
.filter((v): v is number => v !== undefined)
|
|
236
|
+
|
|
237
|
+
if (ids.length === 0) return []
|
|
238
|
+
filters.push(`placetype_id IN (${ids.map(() => "?").join(",")})`)
|
|
239
|
+
filterParams.push(...ids)
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
if (query.bbox) {
|
|
243
|
+
const b = query.bbox
|
|
244
|
+
filters.push("latitude BETWEEN ? AND ? AND longitude BETWEEN ? AND ?")
|
|
245
|
+
filterParams.push(b.minLat, b.maxLat, b.minLon, b.maxLon)
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
const probe = (nk: string): CandidateRow[] => {
|
|
249
|
+
const conds = ["name_key = ?", ...filters]
|
|
250
|
+
const sql =
|
|
251
|
+
"SELECT spr_id, name, country_id, placetype_id, latitude, longitude, min_lat, min_lon, max_lat, max_lon, neg_rank " +
|
|
252
|
+
`FROM candidate WHERE ${conds.join(" AND ")} ORDER BY neg_rank ASC LIMIT ?`
|
|
253
|
+
|
|
254
|
+
return this.#db.prepare(sql).all(nk, ...filterParams, limit) as unknown as CandidateRow[]
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
let rows = probe(nameKey)
|
|
258
|
+
|
|
259
|
+
if (rows.length === 0) {
|
|
260
|
+
// Query-side qualifier-strip fallback: an OA locality with a qualifier the gazetteer's
|
|
261
|
+
// canonical name omits ("Lenk im Simmental" → "Lenk", "Roche VD"). Tried ONLY on an exact
|
|
262
|
+
// miss; the cascade's region bbox disambiguates any base-name ambiguity.
|
|
263
|
+
const strippedKey = normalizeLocalityForKey(stripLocalityQualifier(text))
|
|
264
|
+
|
|
265
|
+
if (strippedKey && strippedKey !== nameKey) {
|
|
266
|
+
rows = probe(strippedKey)
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// Typo-tolerant fallback (the unified gazetteer's fuzzy mode): an exact + strip miss may be a
|
|
271
|
+
// misspelling the normalized key can't reach. FTS5-trigram fetches a loose set; we re-rank by
|
|
272
|
+
// trigram-Jaccard (the admin backend's measure) and probe the best name_keys, so a typo resolves
|
|
273
|
+
// the same on either backend. The country/placetype/bbox filters still apply via `probe`. Skipped
|
|
274
|
+
// when the index is absent (byte-stable for an older candidate.db).
|
|
275
|
+
//
|
|
276
|
+
// Gate: only when the name doesn't exist in the gazetteer AT ALL (unfiltered). A name that exists
|
|
277
|
+
// but missed under the active country/placetype/bbox filter is a FILTER miss, not a spelling miss
|
|
278
|
+
// — fuzzing it scrapes an unrelated same-filter place ("Vienna, Austria" misrouted to IT would
|
|
279
|
+
// pull a tiny Italian name_key near Siena) and masks the cascade's country-agnostic retry that
|
|
280
|
+
// correctly lands population-first Vienna AT. The exact/strip probes already covered the real name.
|
|
281
|
+
if (rows.length === 0 && this.#ftsProbe && this.#nameKeyExistsProbe && !this.#nameKeyExistsProbe.get(nameKey)) {
|
|
282
|
+
const match = ftsTrigramQuery(nameKey)
|
|
283
|
+
|
|
284
|
+
if (match) {
|
|
285
|
+
const hits = this.#ftsProbe.all(match, FUZZY_FETCH) as unknown as Array<{ name_key: string }>
|
|
286
|
+
const ranked = hits
|
|
287
|
+
.map((h) => ({ nk: String(h.name_key), s: trigramJaccard(nameKey, String(h.name_key)) }))
|
|
288
|
+
.filter((h) => h.s >= FUZZY_MIN)
|
|
289
|
+
.sort((a, b) => b.s - a.s)
|
|
290
|
+
const seen = new Set<string>()
|
|
291
|
+
|
|
292
|
+
for (const h of ranked) {
|
|
293
|
+
if (seen.has(h.nk)) continue
|
|
294
|
+
seen.add(h.nk)
|
|
295
|
+
rows.push(...probe(h.nk))
|
|
296
|
+
|
|
297
|
+
if (rows.length >= limit) break
|
|
298
|
+
}
|
|
299
|
+
rows = rows.slice(0, limit)
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
const candidates = rows.map((row): PlaceCandidate => {
|
|
304
|
+
const hasBbox = row.min_lat != null && row.max_lat != null && row.min_lon != null && row.max_lon != null
|
|
305
|
+
|
|
306
|
+
return {
|
|
307
|
+
id: Number(row.spr_id),
|
|
308
|
+
name: String(row.name ?? ""),
|
|
309
|
+
placetype: (this.#idToPlacetype.get(Number(row.placetype_id)) ?? "") as WOFPlacetype,
|
|
310
|
+
// Surfaced so the cascade can country-gate a postcode by the resolved locality (an ambiguous
|
|
311
|
+
// international postcode like 10115 = Berlin DE AND New York US must not out-resolve the city).
|
|
312
|
+
country: this.#idToCountry.get(Number(row.country_id)) ?? "",
|
|
313
|
+
lat: Number(row.latitude),
|
|
314
|
+
lon: Number(row.longitude),
|
|
315
|
+
score: -Number(row.neg_rank),
|
|
316
|
+
// Every candidate row IS an exact normalized-name (or alias/abbrev) match — the cascade's
|
|
317
|
+
// exact tier accepts alias-exact hits ("New York City" → New York) the same as canonical.
|
|
318
|
+
exactMatch: true,
|
|
319
|
+
...(hasBbox
|
|
320
|
+
? {
|
|
321
|
+
bbox: {
|
|
322
|
+
minLat: Number(row.min_lat),
|
|
323
|
+
maxLat: Number(row.max_lat),
|
|
324
|
+
minLon: Number(row.min_lon),
|
|
325
|
+
maxLon: Number(row.max_lon),
|
|
326
|
+
},
|
|
327
|
+
}
|
|
328
|
+
: {}),
|
|
329
|
+
}
|
|
330
|
+
})
|
|
331
|
+
|
|
332
|
+
// Proximity re-rank (#938): with bias hints (the demo's map viewport / user location), re-sort the
|
|
333
|
+
// exact-match candidates by the SAME prominence the FTS server uses (lookup.ts) — population and
|
|
334
|
+
// nearness in one additive scale — so an in-view namesake wins a tie without a hard filter. Byte-
|
|
335
|
+
// identical to the plain population order when no bias is passed. `score` here is -neg_rank =
|
|
336
|
+
// log10(population + 1), so popTerm is the server formula read straight off it. Constants MIRROR
|
|
337
|
+
// lookup.ts's DEFAULT_WEIGHTS (biasBoost 4, populationBoost 4, populationScaleLog10 6,
|
|
338
|
+
// proximityScaleKm 100) — the #861 server↔demo parity contract; keep them in lockstep.
|
|
339
|
+
if (query.bias && query.bias.length > 0) {
|
|
340
|
+
const BIAS_BOOST = 4.0
|
|
341
|
+
const POP_BOOST = 4.0
|
|
342
|
+
const POP_SCALE_LOG10 = 6
|
|
343
|
+
// SHARPER than lookup.ts's 100 km on purpose: this backend's `score` is log-population ALONE
|
|
344
|
+
// (no bm25 document term), so the population signal is weaker relative to the bias and the
|
|
345
|
+
// gentle 100 km decay let a 230 km-distant alias-exact township ("Paris Township", OH) edge
|
|
346
|
+
// out a global city ("Paris", FR) from a nearby view. A ~30 km scale keeps the boost to
|
|
347
|
+
// candidates the user is actually LOOKING at — an in-view namesake still wins (Dublin, OH from
|
|
348
|
+
// an Ohio view), a distant one no longer does (Paris stays FR from a Michigan view).
|
|
349
|
+
const PROX_SCALE_KM = 30
|
|
350
|
+
const prominence = (c: PlaceCandidate): number => {
|
|
351
|
+
const popTerm = POP_BOOST * Math.min(1, Math.max(0, c.score) / POP_SCALE_LOG10)
|
|
352
|
+
let proxTerm = 0
|
|
353
|
+
|
|
354
|
+
if (!(c.lat === 0 && c.lon === 0)) {
|
|
355
|
+
for (const b of query.bias!) {
|
|
356
|
+
const d = haversineKm(b.lat, b.lon, c.lat, c.lon)
|
|
357
|
+
const term = (BIAS_BOOST * (b.weight ?? 1)) / (1 + d / PROX_SCALE_KM)
|
|
358
|
+
|
|
359
|
+
if (term > proxTerm) {
|
|
360
|
+
proxTerm = term
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
return popTerm + proxTerm
|
|
366
|
+
}
|
|
367
|
+
// Stable within equal prominence (preserves the population order the B-tree already gave).
|
|
368
|
+
candidates
|
|
369
|
+
.map((c, i) => ({ c, i, p: prominence(c) }))
|
|
370
|
+
.sort((a, b) => b.p - a.p || a.i - b.i)
|
|
371
|
+
.forEach((x, j) => (candidates[j] = x.c))
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
return candidates
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
close(): void {
|
|
378
|
+
if (this.#ownsDB) {
|
|
379
|
+
this.#db.close()
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Typed schema for the byte-range CANDIDATE gazetteer (`candidate.db`) — the single source of truth
|
|
7
|
+
* for the columns shared by the BUILDER ({@link buildCandidateTable}) and the READERS (the Node
|
|
8
|
+
* {@link WOFCandidateTableLookup} + the browser `httpvfs-resolver.ts`). Before this module each
|
|
9
|
+
* side hand-wrote the column list; a rename in one place broke the other at runtime. Now the
|
|
10
|
+
* contract is a Kysely `Database` interface (`new DatabaseClient<CandidateDatabase>(...)` for
|
|
11
|
+
* typed inserts) plus the table DDL as strings — so a column change is a compile error on every
|
|
12
|
+
* consumer.
|
|
13
|
+
*
|
|
14
|
+
* `cand_stage` is the transient staging table the builder bulk-loads; `candidate` is the clustered
|
|
15
|
+
* `WITHOUT ROWID` B-tree it's materialized into (same columns). The reader queries `candidate`.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { sql, type Kysely } from "kysely"
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* One candidate row. `name_key` + the four small int keys + `neg_rank` + `spr_id` form the clustered primary key; the
|
|
22
|
+
* rest is denormalized so a resolve is one probe (no join to `spr`). Coordinates + bbox + name are nullable at the SQL
|
|
23
|
+
* level (a postcode shard row may lack a bbox).
|
|
24
|
+
*/
|
|
25
|
+
export interface CandidateTable {
|
|
26
|
+
/** The shared {@link normalizeLocalityForKey} of the name/alias — the probe key. */
|
|
27
|
+
name_key: string
|
|
28
|
+
/** Small int from {@link CountryCodeTable} (shrinks the clustered key). */
|
|
29
|
+
country_id: number
|
|
30
|
+
/** The place's region-tier ancestor id, or 0 (carried for the future region 2-step). */
|
|
31
|
+
region_id: number
|
|
32
|
+
/** Small int from {@link PlacetypeCodeTable}. */
|
|
33
|
+
placetype_id: number
|
|
34
|
+
/**
|
|
35
|
+
* `-log10(population + 1)` — ASC order = highest-population first. 0 for postcodes (no population).
|
|
36
|
+
*/
|
|
37
|
+
neg_rank: number
|
|
38
|
+
/** WOF id of the place this row resolves to. */
|
|
39
|
+
spr_id: number
|
|
40
|
+
name: string | null
|
|
41
|
+
latitude: number | null
|
|
42
|
+
longitude: number | null
|
|
43
|
+
min_lat: number | null
|
|
44
|
+
min_lon: number | null
|
|
45
|
+
max_lat: number | null
|
|
46
|
+
max_lon: number | null
|
|
47
|
+
population: number | null
|
|
48
|
+
/** 1 when the row is the place's canonical name (vs an alias/abbrev). */
|
|
49
|
+
is_primary: number | null
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** `(id → ISO country code)` dictionary. */
|
|
53
|
+
export interface CountryCodeTable {
|
|
54
|
+
id: number
|
|
55
|
+
code: string
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** `(id → placetype)` dictionary. */
|
|
59
|
+
export interface PlacetypeCodeTable {
|
|
60
|
+
id: number
|
|
61
|
+
placetype: string
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** The candidate database schema for `new DatabaseClient<CandidateDatabase>(...)`. */
|
|
65
|
+
export interface CandidateDatabase {
|
|
66
|
+
/** The clustered `WITHOUT ROWID` lookup table the reader probes. */
|
|
67
|
+
candidate: CandidateTable
|
|
68
|
+
/** Transient staging table (same columns); dropped once `candidate` is materialized. */
|
|
69
|
+
cand_stage: CandidateTable
|
|
70
|
+
country_codes: CountryCodeTable
|
|
71
|
+
placetype_codes: PlacetypeCodeTable
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* The `candidate`/`cand_stage` columns in clustered-key order. The materialization `INSERT INTO candidate SELECT … FROM
|
|
76
|
+
* cand_stage` derives its column list from this, so the two tables can't drift. Keep in sync with
|
|
77
|
+
* {@link CandidateTable}.
|
|
78
|
+
*/
|
|
79
|
+
export const CANDIDATE_COLUMNS = [
|
|
80
|
+
"name_key",
|
|
81
|
+
"country_id",
|
|
82
|
+
"region_id",
|
|
83
|
+
"placetype_id",
|
|
84
|
+
"neg_rank",
|
|
85
|
+
"spr_id",
|
|
86
|
+
"name",
|
|
87
|
+
"latitude",
|
|
88
|
+
"longitude",
|
|
89
|
+
"min_lat",
|
|
90
|
+
"min_lon",
|
|
91
|
+
"max_lat",
|
|
92
|
+
"max_lon",
|
|
93
|
+
"population",
|
|
94
|
+
"is_primary",
|
|
95
|
+
] as const
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Create the code dictionaries + the transient staging table — called before the build's load passes. `cand_stage`
|
|
99
|
+
* mirrors {@link CandidateTable} but every column is nullable (the loader fills them positionally). Pass a
|
|
100
|
+
* {@link DatabaseClient} (or any `Kysely`) over the candidate DB.
|
|
101
|
+
*/
|
|
102
|
+
export async function createCandidateStagingTables(db: Kysely<CandidateDatabase>): Promise<void> {
|
|
103
|
+
await db.schema
|
|
104
|
+
.createTable("country_codes")
|
|
105
|
+
.addColumn("id", "integer", (c) => c.primaryKey())
|
|
106
|
+
.addColumn("code", "text", (c) => c.unique())
|
|
107
|
+
.execute()
|
|
108
|
+
await db.schema
|
|
109
|
+
.createTable("placetype_codes")
|
|
110
|
+
.addColumn("id", "integer", (c) => c.primaryKey())
|
|
111
|
+
.addColumn("placetype", "text", (c) => c.unique())
|
|
112
|
+
.execute()
|
|
113
|
+
await db.schema
|
|
114
|
+
.createTable("cand_stage")
|
|
115
|
+
.addColumn("name_key", "text")
|
|
116
|
+
.addColumn("country_id", "integer")
|
|
117
|
+
.addColumn("region_id", "integer")
|
|
118
|
+
.addColumn("placetype_id", "integer")
|
|
119
|
+
.addColumn("neg_rank", "real")
|
|
120
|
+
.addColumn("spr_id", "integer")
|
|
121
|
+
.addColumn("name", "text")
|
|
122
|
+
.addColumn("latitude", "real")
|
|
123
|
+
.addColumn("longitude", "real")
|
|
124
|
+
.addColumn("min_lat", "real")
|
|
125
|
+
.addColumn("min_lon", "real")
|
|
126
|
+
.addColumn("max_lat", "real")
|
|
127
|
+
.addColumn("max_lon", "real")
|
|
128
|
+
.addColumn("population", "integer")
|
|
129
|
+
.addColumn("is_primary", "integer")
|
|
130
|
+
.execute()
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Create the clustered `WITHOUT ROWID` lookup table — called after staging, before the VACUUM. The first six columns
|
|
135
|
+
* form the clustered primary key (population-ranked via `neg_rank`).
|
|
136
|
+
*/
|
|
137
|
+
export async function createCandidateTable(db: Kysely<CandidateDatabase>): Promise<void> {
|
|
138
|
+
await db.schema
|
|
139
|
+
.createTable("candidate")
|
|
140
|
+
.addColumn("name_key", "text", (c) => c.notNull())
|
|
141
|
+
.addColumn("country_id", "integer", (c) => c.notNull())
|
|
142
|
+
.addColumn("region_id", "integer", (c) => c.notNull())
|
|
143
|
+
.addColumn("placetype_id", "integer", (c) => c.notNull())
|
|
144
|
+
.addColumn("neg_rank", "real", (c) => c.notNull())
|
|
145
|
+
.addColumn("spr_id", "integer", (c) => c.notNull())
|
|
146
|
+
.addColumn("name", "text")
|
|
147
|
+
.addColumn("latitude", "real")
|
|
148
|
+
.addColumn("longitude", "real")
|
|
149
|
+
.addColumn("min_lat", "real")
|
|
150
|
+
.addColumn("min_lon", "real")
|
|
151
|
+
.addColumn("max_lat", "real")
|
|
152
|
+
.addColumn("max_lon", "real")
|
|
153
|
+
.addColumn("population", "integer")
|
|
154
|
+
.addColumn("is_primary", "integer")
|
|
155
|
+
.addPrimaryKeyConstraint("candidate_pk", [
|
|
156
|
+
"name_key",
|
|
157
|
+
"country_id",
|
|
158
|
+
"region_id",
|
|
159
|
+
"placetype_id",
|
|
160
|
+
"neg_rank",
|
|
161
|
+
"spr_id",
|
|
162
|
+
])
|
|
163
|
+
// `WITHOUT ROWID` has no first-class builder; the raw modifier is the idiomatic fallback.
|
|
164
|
+
.modifyEnd(sql`without rowid`)
|
|
165
|
+
.execute()
|
|
166
|
+
}
|