@mailwoman/resolver-wof-sqlite 9.0.0 → 9.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -9
- package/address-point-interpolation.ts +18 -8
- package/address-point-schema.ts +18 -6
- package/address-point.ts +111 -18
- package/ancestry.ts +9 -6
- package/build-candidate.ts +287 -157
- package/build-slim.ts +3 -3
- package/candidate/alias-bags.ts +54 -0
- package/candidate/ancestors-sidecar.ts +206 -0
- package/candidate/country-display-names.ts +79 -0
- package/candidate/name-roles.ts +237 -0
- package/candidate/own-name.ts +146 -0
- package/candidate/place-attrs.ts +44 -0
- package/candidate/shard-fold.ts +137 -0
- package/candidate-ancestors-schema.ts +195 -0
- package/candidate-fts.ts +4 -2
- package/candidate-importance.ts +228 -0
- package/candidate-lookup.ts +564 -174
- package/candidate-schema.ts +60 -3
- package/candidate-scoring.ts +268 -0
- package/capital-schema.ts +90 -0
- package/capitals.ts +148 -0
- package/coincident-roles.ts +69 -10
- package/convention-schema.ts +72 -0
- package/convention.ts +2 -2
- package/coverage-manifest-schema.ts +7 -7
- package/currency-backfill.ts +249 -0
- package/exact-match.ts +104 -0
- package/fst-autocomplete.ts +105 -122
- package/fst-builder.ts +39 -47
- package/fst-deserialize-web.ts +43 -7
- package/fst-freshness.ts +2 -2
- package/fst-serialize.ts +68 -12
- package/fst-types.ts +35 -1
- package/fts-query.ts +1 -1
- package/fts.ts +16 -4
- package/geonames-postal.ts +2 -2
- package/index.ts +26 -14
- package/interpolation.ts +113 -19
- package/lookup.ts +118 -560
- package/name-score.ts +6 -4
- package/out/address-point-interpolation.d.ts.map +1 -1
- package/out/address-point-interpolation.js +13 -7
- package/out/address-point-interpolation.js.map +1 -1
- package/out/address-point-schema.d.ts +16 -6
- package/out/address-point-schema.d.ts.map +1 -1
- package/out/address-point-schema.js.map +1 -1
- package/out/address-point.d.ts.map +1 -1
- package/out/address-point.js +70 -14
- package/out/address-point.js.map +1 -1
- package/out/ancestry.d.ts +2 -2
- package/out/ancestry.d.ts.map +1 -1
- package/out/ancestry.js +5 -6
- package/out/ancestry.js.map +1 -1
- package/out/build-candidate.d.ts +108 -0
- package/out/build-candidate.d.ts.map +1 -1
- package/out/build-candidate.js +151 -120
- package/out/build-candidate.js.map +1 -1
- package/out/build-slim.d.ts +1 -1
- package/out/build-slim.js +3 -3
- package/out/build-slim.js.map +1 -1
- package/out/candidate/alias-bags.d.ts +17 -0
- package/out/candidate/alias-bags.d.ts.map +1 -0
- package/out/candidate/alias-bags.js +39 -0
- package/out/candidate/alias-bags.js.map +1 -0
- package/out/candidate/ancestors-sidecar.d.ts +33 -0
- package/out/candidate/ancestors-sidecar.d.ts.map +1 -0
- package/out/candidate/ancestors-sidecar.js +140 -0
- package/out/candidate/ancestors-sidecar.js.map +1 -0
- package/out/candidate/country-display-names.d.ts +35 -0
- package/out/candidate/country-display-names.d.ts.map +1 -0
- package/out/candidate/country-display-names.js +59 -0
- package/out/candidate/country-display-names.js.map +1 -0
- package/out/candidate/name-roles.d.ts +55 -0
- package/out/candidate/name-roles.d.ts.map +1 -0
- package/out/candidate/name-roles.js +165 -0
- package/out/candidate/name-roles.js.map +1 -0
- package/out/candidate/own-name.d.ts +50 -0
- package/out/candidate/own-name.d.ts.map +1 -0
- package/out/candidate/own-name.js +132 -0
- package/out/candidate/own-name.js.map +1 -0
- package/out/candidate/place-attrs.d.ts +43 -0
- package/out/candidate/place-attrs.d.ts.map +1 -0
- package/out/candidate/place-attrs.js +15 -0
- package/out/candidate/place-attrs.js.map +1 -0
- package/out/candidate/shard-fold.d.ts +31 -0
- package/out/candidate/shard-fold.d.ts.map +1 -0
- package/out/candidate/shard-fold.js +104 -0
- package/out/candidate/shard-fold.js.map +1 -0
- package/out/candidate-ancestors-schema.d.ts +150 -0
- package/out/candidate-ancestors-schema.d.ts.map +1 -0
- package/out/candidate-ancestors-schema.js +123 -0
- package/out/candidate-ancestors-schema.js.map +1 -0
- package/out/candidate-fts.d.ts +4 -2
- package/out/candidate-fts.d.ts.map +1 -1
- package/out/candidate-fts.js +4 -2
- package/out/candidate-fts.js.map +1 -1
- package/out/candidate-importance.d.ts +132 -0
- package/out/candidate-importance.d.ts.map +1 -0
- package/out/candidate-importance.js +174 -0
- package/out/candidate-importance.js.map +1 -0
- package/out/candidate-lookup.d.ts +22 -37
- package/out/candidate-lookup.d.ts.map +1 -1
- package/out/candidate-lookup.js +446 -132
- package/out/candidate-lookup.js.map +1 -1
- package/out/candidate-schema.d.ts +52 -4
- package/out/candidate-schema.d.ts.map +1 -1
- package/out/candidate-schema.js +8 -0
- package/out/candidate-schema.js.map +1 -1
- package/out/candidate-scoring.d.ts +34 -0
- package/out/candidate-scoring.d.ts.map +1 -0
- package/out/candidate-scoring.js +200 -0
- package/out/candidate-scoring.js.map +1 -0
- package/out/capital-schema.d.ts +51 -0
- package/out/capital-schema.d.ts.map +1 -0
- package/out/capital-schema.js +63 -0
- package/out/capital-schema.js.map +1 -0
- package/out/capitals.d.ts +69 -0
- package/out/capitals.d.ts.map +1 -0
- package/out/capitals.js +98 -0
- package/out/capitals.js.map +1 -0
- package/out/coincident-roles.d.ts +7 -0
- package/out/coincident-roles.d.ts.map +1 -1
- package/out/coincident-roles.js +42 -8
- package/out/coincident-roles.js.map +1 -1
- package/out/convention-schema.d.ts +51 -0
- package/out/convention-schema.d.ts.map +1 -0
- package/out/convention-schema.js +34 -0
- package/out/convention-schema.js.map +1 -0
- package/out/convention.d.ts +1 -1
- package/out/convention.js +2 -2
- package/out/coverage-manifest-schema.js +3 -7
- package/out/coverage-manifest-schema.js.map +1 -1
- package/out/currency-backfill.d.ts +46 -0
- package/out/currency-backfill.d.ts.map +1 -0
- package/out/currency-backfill.js +180 -0
- package/out/currency-backfill.js.map +1 -0
- package/out/exact-match.d.ts +25 -0
- package/out/exact-match.d.ts.map +1 -0
- package/out/exact-match.js +89 -0
- package/out/exact-match.js.map +1 -0
- package/out/fst-autocomplete.d.ts +24 -14
- package/out/fst-autocomplete.d.ts.map +1 -1
- package/out/fst-autocomplete.js +84 -100
- package/out/fst-autocomplete.js.map +1 -1
- package/out/fst-builder.d.ts.map +1 -1
- package/out/fst-builder.js +32 -40
- package/out/fst-builder.js.map +1 -1
- package/out/fst-deserialize-web.d.ts.map +1 -1
- package/out/fst-deserialize-web.js +36 -7
- package/out/fst-deserialize-web.js.map +1 -1
- package/out/fst-freshness.d.ts +2 -2
- package/out/fst-freshness.js +2 -2
- package/out/fst-serialize.d.ts +14 -4
- package/out/fst-serialize.d.ts.map +1 -1
- package/out/fst-serialize.js +60 -12
- package/out/fst-serialize.js.map +1 -1
- package/out/fst-types.d.ts +35 -1
- package/out/fst-types.d.ts.map +1 -1
- package/out/fts-query.js +1 -1
- package/out/fts-query.js.map +1 -1
- package/out/fts.d.ts +15 -4
- package/out/fts.d.ts.map +1 -1
- package/out/fts.js +15 -4
- package/out/fts.js.map +1 -1
- package/out/geonames-postal.d.ts +2 -2
- package/out/geonames-postal.js +2 -2
- package/out/index.d.ts +4 -2
- package/out/index.d.ts.map +1 -1
- package/out/index.js +3 -2
- package/out/index.js.map +1 -1
- package/out/interpolation.d.ts +8 -0
- package/out/interpolation.d.ts.map +1 -1
- package/out/interpolation.js +91 -19
- package/out/interpolation.js.map +1 -1
- package/out/lookup.d.ts +4 -5
- package/out/lookup.d.ts.map +1 -1
- package/out/lookup.js +102 -444
- package/out/lookup.js.map +1 -1
- package/out/name-score.d.ts +0 -10
- package/out/name-score.d.ts.map +1 -1
- package/out/name-score.js +6 -4
- package/out/name-score.js.map +1 -1
- package/out/place-importance-schema.d.ts +226 -0
- package/out/place-importance-schema.d.ts.map +1 -0
- package/out/place-importance-schema.js +288 -0
- package/out/place-importance-schema.js.map +1 -0
- package/out/poi-lookup.d.ts +1 -1
- package/out/poi-lookup.d.ts.map +1 -1
- package/out/poi-lookup.js +12 -13
- package/out/poi-lookup.js.map +1 -1
- package/out/poi-schema.d.ts +7 -3
- package/out/poi-schema.d.ts.map +1 -1
- package/out/poi-schema.js.map +1 -1
- package/out/polygon-schema.d.ts +37 -0
- package/out/polygon-schema.d.ts.map +1 -0
- package/out/polygon-schema.js +23 -0
- package/out/polygon-schema.js.map +1 -0
- package/out/postal-city-alias-lookup.d.ts +1 -1
- package/out/postal-city-alias-lookup.js +1 -1
- package/out/postal-city-candidate-schema.d.ts +2 -1
- package/out/postal-city-candidate-schema.d.ts.map +1 -1
- package/out/postal-city-candidate-schema.js.map +1 -1
- package/out/postcode-point-lookup.d.ts +1 -1
- package/out/postcode-point-lookup.js +1 -1
- package/out/primary-preference.d.ts +125 -0
- package/out/primary-preference.d.ts.map +1 -0
- package/out/primary-preference.js +138 -0
- package/out/primary-preference.js.map +1 -0
- package/out/proximity-rerank.d.ts +77 -0
- package/out/proximity-rerank.d.ts.map +1 -0
- package/out/proximity-rerank.js +86 -0
- package/out/proximity-rerank.js.map +1 -0
- package/out/region-keys.d.ts +47 -0
- package/out/region-keys.d.ts.map +1 -0
- package/out/region-keys.js +121 -0
- package/out/region-keys.js.map +1 -0
- package/out/reverse.d.ts.map +1 -1
- package/out/reverse.js +6 -9
- package/out/reverse.js.map +1 -1
- package/out/schema.d.ts +1 -1
- package/out/search-fetch.d.ts +57 -0
- package/out/search-fetch.d.ts.map +1 -0
- package/out/search-fetch.js +183 -0
- package/out/search-fetch.js.map +1 -0
- package/out/sharding.d.ts +3 -3
- package/out/sharding.js +1 -1
- package/out/sqlite-convention-source.d.ts +1 -1
- package/out/sqlite-convention-source.js +1 -1
- package/out/sqlite-utils.d.ts +31 -1
- package/out/sqlite-utils.d.ts.map +1 -1
- package/out/sqlite-utils.js +38 -0
- package/out/sqlite-utils.js.map +1 -1
- package/out/street-centroid-schema.d.ts +7 -2
- package/out/street-centroid-schema.d.ts.map +1 -1
- package/out/street-centroid-schema.js.map +1 -1
- package/out/street-centroid.d.ts.map +1 -1
- package/out/street-centroid.js +7 -7
- package/out/street-centroid.js.map +1 -1
- package/out/street-morphology-fst-builder.d.ts.map +1 -1
- package/out/street-morphology-fst-builder.js +5 -4
- package/out/street-morphology-fst-builder.js.map +1 -1
- package/out/street-normalize.d.ts +83 -9
- package/out/street-normalize.d.ts.map +1 -1
- package/out/street-normalize.js +177 -10
- package/out/street-normalize.js.map +1 -1
- package/out/street-segment-schema.d.ts +6 -2
- package/out/street-segment-schema.d.ts.map +1 -1
- package/out/street-segment-schema.js.map +1 -1
- package/out/types.d.ts +74 -1
- package/out/types.d.ts.map +1 -1
- package/out/unified-schema.d.ts +1 -1
- package/out/unified-schema.js +1 -1
- package/out/uprn-lookup.d.ts +85 -0
- package/out/uprn-lookup.d.ts.map +1 -0
- package/out/uprn-lookup.js +152 -0
- package/out/uprn-lookup.js.map +1 -0
- package/out/uprn-schema.d.ts +93 -0
- package/out/uprn-schema.d.ts.map +1 -0
- package/out/uprn-schema.js +78 -0
- package/out/uprn-schema.js.map +1 -0
- package/out/weights-overlay-linker.d.ts +141 -0
- package/out/weights-overlay-linker.d.ts.map +1 -0
- package/out/weights-overlay-linker.js +259 -0
- package/out/weights-overlay-linker.js.map +1 -0
- package/package.json +296 -16
- package/place-importance-schema.ts +402 -0
- package/poi-lookup.ts +12 -13
- package/poi-schema.ts +8 -3
- package/polygon-schema.ts +47 -0
- package/postal-city-alias-lookup.ts +1 -1
- package/postal-city-candidate-schema.ts +3 -1
- package/postcode-point-lookup.ts +1 -1
- package/primary-preference.ts +207 -0
- package/proximity-rerank.ts +120 -0
- package/region-keys.ts +144 -0
- package/reverse.ts +17 -16
- package/schema.ts +1 -1
- package/search-fetch.ts +256 -0
- package/sharding.ts +3 -3
- package/sqlite-convention-source.ts +1 -1
- package/sqlite-utils.ts +63 -1
- package/street-centroid-schema.ts +8 -2
- package/street-centroid.ts +13 -8
- package/street-morphology-fst-builder.ts +5 -4
- package/street-normalize.ts +254 -24
- package/street-segment-schema.ts +7 -2
- package/types.ts +74 -1
- package/unified-schema.ts +1 -1
- package/uprn-lookup.ts +210 -0
- package/uprn-schema.ts +124 -0
- package/weights-overlay-linker.ts +377 -0
- package/geo.ts +0 -121
- package/out/geo.d.ts +0 -74
- package/out/geo.d.ts.map +0 -1
- package/out/geo.js +0 -71
- package/out/geo.js.map +0 -1
package/lookup.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
* `
|
|
6
|
+
* `WOFSQLitePlaceLookup` — the resolver implementation backed by `node:sqlite` + a Kysely-typed
|
|
7
7
|
* query layer where the queries are non-trivial, and raw SQL where they aren't (FTS5 MATCH, the
|
|
8
8
|
* FTS index build).
|
|
9
9
|
*
|
|
@@ -14,10 +14,12 @@ import { DatabaseSync, type SQLInputValue } from "node:sqlite"
|
|
|
14
14
|
|
|
15
15
|
import { SqliteDialect } from "@mailwoman/core/kysley/dialect"
|
|
16
16
|
import { expandPlacetypeFilter, type Ancestor, type CoincidentLocality } from "@mailwoman/resolver"
|
|
17
|
+
import { haversineKm } from "@mailwoman/spatial"
|
|
17
18
|
import { Kysely } from "kysely"
|
|
18
19
|
|
|
19
20
|
import { ancestorLineage } from "./ancestry.ts"
|
|
20
|
-
import {
|
|
21
|
+
import { candidateFromSearchRow, rankCandidates } from "./candidate-scoring.ts"
|
|
22
|
+
import { loadCoincidentLocalities } from "./coincident-roles.ts"
|
|
21
23
|
import {
|
|
22
24
|
ADDRESS_CONVENTION_TABLE,
|
|
23
25
|
resolveConvention,
|
|
@@ -29,19 +31,20 @@ import {
|
|
|
29
31
|
} from "./convention.ts"
|
|
30
32
|
import { normalizePlacetypes, sanitizeFTSQuery } from "./fts-query.ts"
|
|
31
33
|
import {
|
|
32
|
-
aliasBagExactMatch,
|
|
33
34
|
buildPlaceSearchFTS,
|
|
34
35
|
PLACE_BBOX_TABLE,
|
|
35
36
|
PLACE_POPULATION_TABLE,
|
|
37
|
+
PLACE_SEARCH_TABLE,
|
|
36
38
|
placeBboxExists,
|
|
37
39
|
placePopulationExists,
|
|
38
40
|
placeSearchFTSExists,
|
|
39
41
|
} from "./fts.ts"
|
|
40
|
-
import {
|
|
41
|
-
import {
|
|
42
|
+
import { cfNormalize, softNameScore } from "./name-score.ts"
|
|
43
|
+
import { encyclopedicClauses } from "./place-importance-schema.ts"
|
|
42
44
|
import type { WOFPostalCityAliasLookup } from "./postal-city-alias-lookup.ts"
|
|
43
45
|
import { DEFAULT_WEIGHTS, type RankingWeights } from "./ranking-weights.ts"
|
|
44
46
|
import type { WOFDatabase } from "./schema.ts"
|
|
47
|
+
import { fetchSearchRows, type RawSearchRow } from "./search-fetch.ts"
|
|
45
48
|
import {
|
|
46
49
|
pickShardForPlacetype,
|
|
47
50
|
pickShardsForPlacetype,
|
|
@@ -50,16 +53,10 @@ import {
|
|
|
50
53
|
type ShardConfig,
|
|
51
54
|
} from "./sharding.ts"
|
|
52
55
|
import { SqliteConventionSource } from "./sqlite-convention-source.ts"
|
|
56
|
+
import { allRows } from "./sqlite-utils.ts"
|
|
53
57
|
import type { FindPlaceQuery, PlaceCandidate, PlaceLookup, WOFPlacetype } from "./types.ts"
|
|
54
58
|
|
|
55
|
-
|
|
56
|
-
* Query length at or below which the FTS window is widened. A two- or three-character query is almost always a region
|
|
57
|
-
* abbreviation, where the exact match can otherwise fall outside the window behind higher-bm25 partial hits — "NY"
|
|
58
|
-
* losing to "New York".
|
|
59
|
-
*/
|
|
60
|
-
const SHORT_QUERY_MAX_LENGTH = 3
|
|
61
|
-
|
|
62
|
-
export interface WOFSqlitePlaceLookupOpts {
|
|
59
|
+
export interface WOFSQLitePlaceLookupOpts {
|
|
63
60
|
/**
|
|
64
61
|
* Path to the WOF SQLite distribution on disk. Mutually exclusive with `database`.
|
|
65
62
|
*
|
|
@@ -106,44 +103,25 @@ export interface WOFSqlitePlaceLookupOpts {
|
|
|
106
103
|
postalCityAliases?: WOFPostalCityAliasLookup
|
|
107
104
|
}
|
|
108
105
|
|
|
109
|
-
/**
|
|
110
|
-
* Over-fetch floor for SHORT (≤3-char) queries — region abbreviations like "NY"/"VT". An exact-abbrev holder's BM25 is
|
|
111
|
-
* poor (long multilingual alt-name document), so the normal `limit * 4` window can drop it before `exactMatchTiering`
|
|
112
|
-
* promotes it. 200 comfortably covers every same-abbrev region across the 12-country gazetteer (a 2-letter token
|
|
113
|
-
* matches a few dozen regions at most) while staying a cheap region-placetype fetch. See the `#fuzzyNameMatch`
|
|
114
|
-
* over-fetch comment.
|
|
115
|
-
*/
|
|
116
|
-
const SHORT_QUERY_OVERFETCH = 200
|
|
117
|
-
|
|
118
|
-
/**
|
|
119
|
-
* How many rows the population-ordered companion fetch (#905) adds to the candidate pool. Small on purpose: its only
|
|
120
|
-
* job is to guarantee the FAMOUS holders of a name enter the pool at all — for "Paris"-class floods the bm25 window is
|
|
121
|
-
* saturated by thousands of tiny same-name rows and no boost inside the bm25-based ORDER BY can rescue a candidate
|
|
122
|
-
* whose bm25 is length-poisoned by ~15 points (see the fetch-site comment).
|
|
123
|
-
*/
|
|
124
|
-
const POPULATION_FETCH_LIMIT = 15
|
|
125
|
-
|
|
126
|
-
interface RawSearchRow {
|
|
127
|
-
id: number
|
|
128
|
-
name: string
|
|
129
|
-
placetype: string
|
|
130
|
-
country: string | null
|
|
131
|
-
parent_id: number | null
|
|
132
|
-
rank: number // BM25 (lower = better in SQLite); we negate to get higher-is-better
|
|
133
|
-
lat: number | null
|
|
134
|
-
lon: number | null
|
|
135
|
-
min_latitude: number | null
|
|
136
|
-
max_latitude: number | null
|
|
137
|
-
min_longitude: number | null
|
|
138
|
-
max_longitude: number | null
|
|
139
|
-
population: number | null // from the place_population aux table; null when missing
|
|
140
|
-
}
|
|
141
|
-
|
|
142
106
|
/**
|
|
143
107
|
* The coordinate-first candidate table (scripts/build-postcode-locality.ts): postcode → containing
|
|
144
108
|
*
|
|
145
109
|
* - Nearby localities with WOF alt-name aliases.
|
|
146
110
|
*/
|
|
111
|
+
/**
|
|
112
|
+
* The placetypes `pickShardsForPlacetype`'s substring rule can route by name. Not every WOF placetype — only the ones a
|
|
113
|
+
* purpose-built shard is ever named for — so the diagnostic below can say "this name routes nowhere" without claiming
|
|
114
|
+
* to enumerate the gazetteer.
|
|
115
|
+
*/
|
|
116
|
+
const KNOWN_ROUTED_PLACETYPES: ReadonlyArray<string> = [
|
|
117
|
+
"postalcode",
|
|
118
|
+
"locality",
|
|
119
|
+
"region",
|
|
120
|
+
"county",
|
|
121
|
+
"country",
|
|
122
|
+
"venue",
|
|
123
|
+
]
|
|
124
|
+
|
|
147
125
|
const POSTCODE_LOCALITY_TABLE = "postcode_locality"
|
|
148
126
|
|
|
149
127
|
/**
|
|
@@ -159,9 +137,8 @@ const CF_PC_DECAY_KM = 8
|
|
|
159
137
|
* flagged, tight enough to catch a wrong city (hundreds of km).
|
|
160
138
|
*/
|
|
161
139
|
const CF_MISMATCH_KM = 50
|
|
162
|
-
const CF_MISMATCH_DELTA = 0.5
|
|
163
140
|
|
|
164
|
-
export class
|
|
141
|
+
export class WOFSQLitePlaceLookup implements PlaceLookup, Disposable {
|
|
165
142
|
readonly #db: DatabaseSync
|
|
166
143
|
readonly #ownsDB: boolean
|
|
167
144
|
readonly #kysely: Kysely<WOFDatabase>
|
|
@@ -179,6 +156,12 @@ export class WOFSqlitePlaceLookup implements PlaceLookup, Disposable {
|
|
|
179
156
|
* population boost is 0 for every row — preserves compatibility with DBs built before this feature shipped.
|
|
180
157
|
*/
|
|
181
158
|
readonly #hasPopulationIndex: Map<string, boolean>
|
|
159
|
+
/**
|
|
160
|
+
* Per-shard SELECT term + LEFT JOIN for the two-score split's `encyclopedic` carry (ROAD_TO_V9 §2 R1), probed and
|
|
161
|
+
* built once at construction. Degrades to `NULL AS encyclopedic` with no join on a pre-split shard — every shipped
|
|
162
|
+
* shard today. See {@link encyclopedicClauses} for why the probe is a column and not a table.
|
|
163
|
+
*/
|
|
164
|
+
readonly #encyclopedicClauses: Map<string, { select: string; join: string }>
|
|
182
165
|
/**
|
|
183
166
|
* Per-shard probe for the `postcode_locality` table (the coordinate-first candidate table, built by
|
|
184
167
|
* scripts/build-postcode-locality.ts). Cached at construction; null'd out when absent so the coord-first path
|
|
@@ -222,13 +205,13 @@ export class WOFSqlitePlaceLookup implements PlaceLookup, Disposable {
|
|
|
222
205
|
*/
|
|
223
206
|
readonly #postalCityAliases: WOFPostalCityAliasLookup | null
|
|
224
207
|
|
|
225
|
-
constructor(opts:
|
|
208
|
+
constructor(opts: WOFSQLitePlaceLookupOpts, weights?: Partial<RankingWeights>) {
|
|
226
209
|
if (opts.database && opts.databasePath) {
|
|
227
|
-
throw new Error("
|
|
210
|
+
throw new Error("WOFSQLitePlaceLookup: pass either `database` or `databasePath`, not both")
|
|
228
211
|
}
|
|
229
212
|
|
|
230
213
|
if (!opts.database && !opts.databasePath) {
|
|
231
|
-
throw new Error("
|
|
214
|
+
throw new Error("WOFSQLitePlaceLookup: one of `database` or `databasePath` is required")
|
|
232
215
|
}
|
|
233
216
|
|
|
234
217
|
if (opts.database) {
|
|
@@ -273,10 +256,55 @@ export class WOFSqlitePlaceLookup implements PlaceLookup, Disposable {
|
|
|
273
256
|
// sqlite_master. Cached at construction so findPlace doesn't query sqlite_master per call.
|
|
274
257
|
this.#hasBboxIndex = new Map()
|
|
275
258
|
this.#hasPopulationIndex = new Map()
|
|
259
|
+
this.#encyclopedicClauses = new Map()
|
|
276
260
|
|
|
277
261
|
for (const s of this.#shards) {
|
|
278
262
|
this.#hasBboxIndex.set(s.schemaName, this.#shardHasTable(s.schemaName, PLACE_BBOX_TABLE))
|
|
279
263
|
this.#hasPopulationIndex.set(s.schemaName, this.#shardHasTable(s.schemaName, PLACE_POPULATION_TABLE))
|
|
264
|
+
this.#encyclopedicClauses.set(s.schemaName, encyclopedicClauses(this.#db, s.schemaName))
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// Every lookup path here reaches `place_search`, and a shard without it fails in one of two ways
|
|
268
|
+
// that are both hard to read: an unroutable name returns zero hits (indistinguishable from "this
|
|
269
|
+
// country has no places") and a routable one throws mid-query from deep inside a SELECT. The
|
|
270
|
+
// unroutable half is the worse of the two — a shard reaches routing only through the name
|
|
271
|
+
// `deriveSchemaName` derives from its FILENAME, so a file spelled one letter off the placetype it
|
|
272
|
+
// serves answers with nothing while holding every row that was asked for.
|
|
273
|
+
//
|
|
274
|
+
// Two independent things bring a shard under the guard, and it needs both. Carrying `spr` is a
|
|
275
|
+
// CLAIM to be a place shard. Carrying a name that routes is an INVITATION to be queried as one, and
|
|
276
|
+
// it is made by the filename alone — so a database with no tables at all still gets picked, still
|
|
277
|
+
// answers no query, and still dies inside a SELECT. Testing only the claim lets an empty or
|
|
278
|
+
// truncated file past construction; testing only the name would exempt a correctly-named build
|
|
279
|
+
// input. A shard needs to fail neither test to be exempt.
|
|
280
|
+
//
|
|
281
|
+
// Exempt by design: `postcode-locality-<cc>.db` carries a relation table and nothing else, matches
|
|
282
|
+
// no routed placetype, and is part of the documented default shard list.
|
|
283
|
+
for (const s of this.#shards) {
|
|
284
|
+
if (s.schemaName === "main") continue
|
|
285
|
+
|
|
286
|
+
const routes = KNOWN_ROUTED_PLACETYPES.some(
|
|
287
|
+
(pt) => s.schemaName === pt || s.schemaName.startsWith(`${pt}_`) || s.schemaName.endsWith(`_${pt}`)
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
const claimsPlaceShard = this.#shardHasTable(s.schemaName, "spr")
|
|
291
|
+
|
|
292
|
+
if (!routes && !claimsPlaceShard) continue
|
|
293
|
+
|
|
294
|
+
if (this.#shardHasTable(s.schemaName, PLACE_SEARCH_TABLE)) continue
|
|
295
|
+
|
|
296
|
+
throw new Error(
|
|
297
|
+
`WOFSQLitePlaceLookup: ${s.path} ` +
|
|
298
|
+
(claimsPlaceShard
|
|
299
|
+
? `carries "spr" but no "${PLACE_SEARCH_TABLE}" table, so it cannot serve a lookup.`
|
|
300
|
+
: `is named for a routed placetype but carries neither "spr" nor "${PLACE_SEARCH_TABLE}", so every ` +
|
|
301
|
+
`query routed to it would die mid-SELECT. An empty or truncated file reads exactly like this.`) +
|
|
302
|
+
` Build it with the FTS index, or leave it out — it is usable as a BUILD input either way.` +
|
|
303
|
+
(routes
|
|
304
|
+
? ""
|
|
305
|
+
: ` Its schema name "${s.schemaName}" also matches no routed placetype (${KNOWN_ROUTED_PLACETYPES.join(", ")}), ` +
|
|
306
|
+
`so it would never have been queried even with the table — check the filename's spelling.`)
|
|
307
|
+
)
|
|
280
308
|
}
|
|
281
309
|
|
|
282
310
|
// #920 country-aware shard routing: probe each NON-MAIN shard's country set once at
|
|
@@ -421,54 +449,7 @@ export class WOFSqlitePlaceLookup implements PlaceLookup, Disposable {
|
|
|
421
449
|
if (!Number.isFinite(id)) return []
|
|
422
450
|
|
|
423
451
|
if (!this.#coincidentRolesCache) {
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
if (coincidentRolesExists(this.#db)) {
|
|
427
|
-
const rows = this.#db
|
|
428
|
-
.prepare(
|
|
429
|
-
`SELECT cr.admin_id AS adminID, s.id AS id, s.name AS name, s.country AS country,
|
|
430
|
-
s.latitude AS lat, s.longitude AS lon,
|
|
431
|
-
cr.relationship_type AS relationshipType, cr.locality_population AS population,
|
|
432
|
-
cr.distance_km AS distanceKm
|
|
433
|
-
FROM ${COINCIDENT_ROLES_TABLE} cr JOIN spr s ON s.id = cr.locality_id`
|
|
434
|
-
)
|
|
435
|
-
.all() as unknown as Array<{
|
|
436
|
-
adminID: number
|
|
437
|
-
id: number
|
|
438
|
-
name: string
|
|
439
|
-
country: string
|
|
440
|
-
lat: number
|
|
441
|
-
lon: number
|
|
442
|
-
relationshipType: string
|
|
443
|
-
population: number
|
|
444
|
-
distanceKm: number
|
|
445
|
-
}>
|
|
446
|
-
|
|
447
|
-
for (const r of rows) {
|
|
448
|
-
const candidate: CoincidentLocality = {
|
|
449
|
-
id: r.id,
|
|
450
|
-
name: r.name,
|
|
451
|
-
placetype: "locality",
|
|
452
|
-
country: r.country,
|
|
453
|
-
lat: r.lat,
|
|
454
|
-
lon: r.lon,
|
|
455
|
-
score: 0,
|
|
456
|
-
relationshipType: r.relationshipType,
|
|
457
|
-
population: r.population,
|
|
458
|
-
distanceKm: r.distanceKm,
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
const list = map.get(r.adminID)
|
|
462
|
-
|
|
463
|
-
if (list) {
|
|
464
|
-
list.push(candidate)
|
|
465
|
-
} else {
|
|
466
|
-
map.set(r.adminID, [candidate])
|
|
467
|
-
}
|
|
468
|
-
}
|
|
469
|
-
}
|
|
470
|
-
|
|
471
|
-
this.#coincidentRolesCache = map
|
|
452
|
+
this.#coincidentRolesCache = loadCoincidentLocalities(this.#db)
|
|
472
453
|
}
|
|
473
454
|
|
|
474
455
|
return this.#coincidentRolesCache.get(id) ?? []
|
|
@@ -513,7 +494,7 @@ export class WOFSqlitePlaceLookup implements PlaceLookup, Disposable {
|
|
|
513
494
|
this.#warnedUnknownStrategies.add(name)
|
|
514
495
|
|
|
515
496
|
console.warn(
|
|
516
|
-
`
|
|
497
|
+
`WOFSQLitePlaceLookup: a convention names strategy "${name}", which this build does not register ` +
|
|
517
498
|
`(known: ${[...this.#strategies.keys()].join(", ")}). Skipping it. If the convention asset was built ` +
|
|
518
499
|
`against a newer code revision, rebuild the asset for this one.`
|
|
519
500
|
)
|
|
@@ -540,18 +521,6 @@ export class WOFSqlitePlaceLookup implements PlaceLookup, Disposable {
|
|
|
540
521
|
async #fuzzyNameMatch(query: FindPlaceQuery, forceShard?: ResolvedShard): Promise<PlaceCandidate[]> {
|
|
541
522
|
const limit = query.limit ?? 10
|
|
542
523
|
|
|
543
|
-
// Over-fetch so post-scoring + exact-match tiering have room to re-rank. SHORT queries (a 2–3-char
|
|
544
|
-
// region abbreviation like "NY"/"VT") are the danger case the `exactMatchTiering` docstring flags:
|
|
545
|
-
// the exact-abbrev holder's BM25 is poor (its long multilingual alt-name document tanks the score),
|
|
546
|
-
// so under the normal `limit * 4` window it drops OUT of the candidate pool BEFORE tiering can
|
|
547
|
-
// promote it — "NY" then resolves to a token-matching foreign region (Highland, GB) instead of New
|
|
548
|
-
// York. Widen the window for short queries so the exact match is always present to be tiered.
|
|
549
|
-
// (Cross-country abbrev collisions — "VT" is BOTH Vermont and Viterbo — still need a country/
|
|
550
|
-
// postcode signal to disambiguate; this only rescues the window-drop class, not genuine ambiguity.
|
|
551
|
-
// With a `country` hint every abbrev resolves; bare + no-context lifts 7→10/15 US states.)
|
|
552
|
-
const ftsLimit =
|
|
553
|
-
query.text.trim().length <= SHORT_QUERY_MAX_LENGTH ? Math.max(limit * 4, SHORT_QUERY_OVERFETCH) : limit * 4
|
|
554
|
-
|
|
555
524
|
// Expand the placetype filter through the shared equivalence table (core/resolver): a
|
|
556
525
|
// `locality` query must also reach `borough` / `localadmin` rows — Brooklyn-the-borough
|
|
557
526
|
// (pop 2.5M) is a borough, not a locality, and a strict filter made it unreachable so the
|
|
@@ -614,353 +583,38 @@ export class WOFSqlitePlaceLookup implements PlaceLookup, Disposable {
|
|
|
614
583
|
countriesBySchema: this.#shardCountries,
|
|
615
584
|
})
|
|
616
585
|
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
if (query.country) {
|
|
633
|
-
where.push("spr.country = ?")
|
|
634
|
-
params.push(query.country)
|
|
635
|
-
}
|
|
636
|
-
|
|
637
|
-
if (query.parentID !== undefined) {
|
|
638
|
-
where.push(`(spr.parent_id = ? OR spr.id IN (SELECT id FROM ${sch}.ancestors WHERE ancestor_id = ?))`)
|
|
639
|
-
params.push(query.parentID, query.parentID)
|
|
640
|
-
}
|
|
641
|
-
|
|
642
|
-
// Bbox + near-with-radius are SQL-level filters via the R*Tree. We only emit the JOIN when
|
|
643
|
-
// the active shard has the R*Tree; missing-but-requested is silently treated as no-bbox-
|
|
644
|
-
// filter so legacy DBs / shards-without-bbox don't crash.
|
|
645
|
-
const shardHasBbox = this.#hasBboxIndex.get(sch) === true
|
|
646
|
-
const useBboxJoin = (query.bbox || query.near?.maxDistanceKm !== undefined) && shardHasBbox
|
|
647
|
-
let joinClause = `JOIN ${sch}.spr ON spr.id = place_search.wof_id`
|
|
648
|
-
|
|
649
|
-
if (useBboxJoin) {
|
|
650
|
-
joinClause += ` JOIN ${sch}.${PLACE_BBOX_TABLE} bbox ON bbox.id = spr.id`
|
|
651
|
-
// AABB intersection — both bbox sides must overlap. R*Tree handles this in O(log n).
|
|
652
|
-
const filterBox = query.bbox || bboxAround(query.near!.lat, query.near!.lon, query.near!.maxDistanceKm!)
|
|
653
|
-
where.push("bbox.min_lat <= ? AND bbox.max_lat >= ?", "bbox.min_lon <= ? AND bbox.max_lon >= ?")
|
|
654
|
-
params.push(filterBox.maxLat, filterBox.minLat, filterBox.maxLon, filterBox.minLon)
|
|
655
|
-
}
|
|
656
|
-
|
|
657
|
-
// LEFT JOIN the population aux table when present. Missing-on-this-shard means the SELECT
|
|
658
|
-
// just doesn't include the population column; the post-scoring loop treats it as 0.
|
|
659
|
-
const shardHasPopulation = this.#hasPopulationIndex.get(sch) === true
|
|
660
|
-
|
|
661
|
-
const populationSelect = shardHasPopulation
|
|
662
|
-
? `${PLACE_POPULATION_TABLE}.population AS population`
|
|
663
|
-
: `NULL AS population`
|
|
664
|
-
|
|
665
|
-
const populationJoin = shardHasPopulation
|
|
666
|
-
? `LEFT JOIN ${sch}.${PLACE_POPULATION_TABLE} ON ${PLACE_POPULATION_TABLE}.id = spr.id`
|
|
667
|
-
: ""
|
|
668
|
-
|
|
669
|
-
// Push the population boost into the ORDER BY when the index is available, so famous places
|
|
670
|
-
// (whose long alt-name lists hurt BM25) actually make it into the over-fetch window. The TS
|
|
671
|
-
// post-scoring will still compute the same boost for the final score; this just ensures the
|
|
672
|
-
// candidate set is right.
|
|
673
|
-
//
|
|
674
|
-
// Formula: rank_adjusted = bm25 - populationBoost * min(1.0, log10(1 + pop) / scaleLog10)
|
|
675
|
-
// Lower rank_adjusted = better (matches SQLite's bm25 convention of "more negative = better").
|
|
676
|
-
//
|
|
677
|
-
// #905 — do NOT reach for bm25 column weights here. Measured falsification (2026-07-02): FTS5's
|
|
678
|
-
// bm25 length normalization is polluted by the row's TOTAL document size, so identical 1-token
|
|
679
|
-
// `name` docs read −16.0 (empty alt_names) vs −0.43 (2.7 KB alt_names) EVEN with the alt_names
|
|
680
|
-
// column weighted to zero — no weighting isolates name relevance in this schema. The famous-
|
|
681
|
-
// holder guarantee lives in the population-ordered companion fetch below instead, and the
|
|
682
|
-
// exact tier breaks ties by population in the post-scoring sort.
|
|
683
|
-
const orderByExpr = shardHasPopulation
|
|
684
|
-
? `(bm25(place_search) - ? * MIN(1.0, COALESCE(log10(1.0 + ${PLACE_POPULATION_TABLE}.population), 0) / ?))`
|
|
685
|
-
: "bm25(place_search)"
|
|
686
|
-
|
|
687
|
-
// Schema-qualified FROM with bare-name MATCH — required syntax for FTS5 on attached schemas.
|
|
688
|
-
// See sharding.ts header for the gotcha that drove this design.
|
|
689
|
-
const stmt = this.#db.prepare(`
|
|
690
|
-
SELECT
|
|
691
|
-
spr.id AS id,
|
|
692
|
-
spr.name,
|
|
693
|
-
spr.placetype,
|
|
694
|
-
spr.country,
|
|
695
|
-
spr.parent_id,
|
|
696
|
-
bm25(place_search) AS rank,
|
|
697
|
-
spr.latitude AS lat,
|
|
698
|
-
spr.longitude AS lon,
|
|
699
|
-
spr.min_latitude, spr.max_latitude, spr.min_longitude, spr.max_longitude,
|
|
700
|
-
${populationSelect}
|
|
701
|
-
FROM ${sch}.place_search
|
|
702
|
-
${joinClause}
|
|
703
|
-
${populationJoin}
|
|
704
|
-
WHERE ${where.join(" AND ")}
|
|
705
|
-
ORDER BY ${orderByExpr} ASC
|
|
706
|
-
LIMIT ?
|
|
707
|
-
`)
|
|
708
|
-
|
|
709
|
-
if (shardHasPopulation) {
|
|
710
|
-
params.push(this.#weights.populationBoost, this.#weights.populationScaleLog10)
|
|
711
|
-
}
|
|
586
|
+
// bare schema name; safe to interpolate (validated at construction)
|
|
587
|
+
const sch = shard.schemaName
|
|
588
|
+
|
|
589
|
+
const rawRows = fetchSearchRows({
|
|
590
|
+
db: this.#db,
|
|
591
|
+
schemaName: sch,
|
|
592
|
+
query,
|
|
593
|
+
placetypes,
|
|
594
|
+
ftsQuery,
|
|
595
|
+
limit,
|
|
596
|
+
hasBboxIndex: this.#hasBboxIndex,
|
|
597
|
+
hasPopulationIndex: this.#hasPopulationIndex,
|
|
598
|
+
encyclopedicClauses: this.#encyclopedicClauses,
|
|
599
|
+
weights: this.#weights,
|
|
600
|
+
})
|
|
712
601
|
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
// ("Paris" matches thousands of gap-fill villages) the bm25-based window above cannot admit
|
|
719
|
-
// the famous holder — its bm25 is length-poisoned by the row's alias bulk (measured ~15 pts,
|
|
720
|
-
// vs a +4.0 boost cap), so FR Paris never even reaches post-scoring. This fetch makes the
|
|
721
|
-
// prominent holders of a name pool-complete BY CONSTRUCTION; the exact-tier sort below
|
|
722
|
-
// decides whether they win. Skipped without a population index (nothing to order by).
|
|
723
|
-
if (shardHasPopulation) {
|
|
724
|
-
const popStmt = this.#db.prepare(`
|
|
725
|
-
SELECT
|
|
726
|
-
spr.id AS id,
|
|
727
|
-
spr.name,
|
|
728
|
-
spr.placetype,
|
|
729
|
-
spr.country,
|
|
730
|
-
spr.parent_id,
|
|
731
|
-
bm25(place_search) AS rank,
|
|
732
|
-
spr.latitude AS lat,
|
|
733
|
-
spr.longitude AS lon,
|
|
734
|
-
spr.min_latitude, spr.max_latitude, spr.min_longitude, spr.max_longitude,
|
|
735
|
-
${populationSelect}
|
|
736
|
-
FROM ${sch}.place_search
|
|
737
|
-
${joinClause}
|
|
738
|
-
${populationJoin}
|
|
739
|
-
WHERE ${where.join(" AND ")}
|
|
740
|
-
ORDER BY COALESCE(${PLACE_POPULATION_TABLE}.population, 0) DESC
|
|
741
|
-
LIMIT ?
|
|
742
|
-
`)
|
|
743
|
-
|
|
744
|
-
const popParams = params.slice(0, -3) // drop the two boost params + ftsLimit
|
|
745
|
-
const seen = new Set(rawRows.map((r) => r.id))
|
|
746
|
-
|
|
747
|
-
for (const row of popStmt.all(...popParams, POPULATION_FETCH_LIMIT) as unknown as RawSearchRow[]) {
|
|
748
|
-
if (!seen.has(row.id)) {
|
|
749
|
-
rawRows.push(row)
|
|
750
|
-
}
|
|
751
|
-
}
|
|
602
|
+
const scoring = {
|
|
603
|
+
query,
|
|
604
|
+
placetypes,
|
|
605
|
+
queryLen: query.text.length,
|
|
606
|
+
weights: this.#weights,
|
|
752
607
|
}
|
|
753
608
|
|
|
754
|
-
const
|
|
755
|
-
|
|
756
|
-
const candidates = rawRows.map((row): PlaceCandidate => {
|
|
757
|
-
// SQLite's bm25() returns a lower-is-better score (negative for matches). Negate so we
|
|
758
|
-
// start from a higher-is-better baseline.
|
|
759
|
-
let score = -row.rank
|
|
760
|
-
|
|
761
|
-
if (placetypes && placetypes.length && placetypes.includes(row.placetype as WOFPlacetype)) {
|
|
762
|
-
score += this.#weights.placetypeMatchBoost
|
|
763
|
-
}
|
|
764
|
-
|
|
765
|
-
if (!placetypes && row.placetype === "locality") {
|
|
766
|
-
score += this.#weights.localityImplicitBoost
|
|
767
|
-
}
|
|
609
|
+
const candidates = rawRows.map((row) => candidateFromSearchRow(row, scoring))
|
|
768
610
|
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
score += row.parent_id === query.parentID ? this.#weights.directChildBoost : this.#weights.descendantBoost
|
|
775
|
-
}
|
|
776
|
-
|
|
777
|
-
const extraLen = Math.max(0, row.name.length - queryLen - 3)
|
|
778
|
-
score -= (this.#weights.lengthPenaltyWeight * extraLen) / 10
|
|
779
|
-
|
|
780
|
-
// Proximity boost: only applied when the query carries `near` AND the candidate has real
|
|
781
|
-
// coordinates. The formula decays smoothly with distance so close-but-not-exact hits
|
|
782
|
-
// still benefit; tunable via proximityBoost + proximityScaleKm.
|
|
783
|
-
let distanceKm: number | undefined
|
|
784
|
-
// The best decayed-distance term over `near` + every `bias` point (each point's term is
|
|
785
|
-
// scaled by its weight; the MAX wins — a candidate near ANY hint is "nearby"). Carried
|
|
786
|
-
// into the exact-tier prominence sort below when hints are present.
|
|
787
|
-
let proximityTerm = 0
|
|
788
|
-
|
|
789
|
-
if (row.lat !== null && row.lon !== null && !(row.lat === 0 && row.lon === 0)) {
|
|
790
|
-
const hints: Array<{ lat: number; lon: number; weight: number }> = []
|
|
791
|
-
|
|
792
|
-
if (query.near) {
|
|
793
|
-
hints.push({ lat: query.near.lat, lon: query.near.lon, weight: 1 })
|
|
794
|
-
}
|
|
795
|
-
|
|
796
|
-
for (const b of query.bias ?? []) {
|
|
797
|
-
hints.push({ lat: b.lat, lon: b.lon, weight: b.weight ?? 1 })
|
|
798
|
-
}
|
|
799
|
-
|
|
800
|
-
let scoreTerm = 0
|
|
801
|
-
|
|
802
|
-
for (const h of hints) {
|
|
803
|
-
const d = haversineKm(h.lat, h.lon, row.lat, row.lon)
|
|
804
|
-
const decay = h.weight / (1 + d / this.#weights.proximityScaleKm)
|
|
805
|
-
const prom = decay * this.#weights.biasBoost
|
|
806
|
-
|
|
807
|
-
if (prom > proximityTerm) {
|
|
808
|
-
proximityTerm = prom
|
|
809
|
-
distanceKm = d
|
|
810
|
-
scoreTerm = decay * this.#weights.proximityBoost
|
|
811
|
-
}
|
|
812
|
-
}
|
|
813
|
-
|
|
814
|
-
score += scoreTerm
|
|
815
|
-
}
|
|
816
|
-
|
|
817
|
-
// Population boost: capped at `populationBoost` magnitude at `10^populationScaleLog10`
|
|
818
|
-
// people. Missing population → no contribution. Never penalizes.
|
|
819
|
-
let popTerm = 0
|
|
820
|
-
|
|
821
|
-
if (row.population !== null && row.population > 0 && this.#weights.populationScaleLog10 > 0) {
|
|
822
|
-
const popLog = Math.log10(1 + row.population)
|
|
823
|
-
const popFraction = Math.min(1, popLog / this.#weights.populationScaleLog10)
|
|
824
|
-
popTerm = this.#weights.populationBoost * popFraction
|
|
825
|
-
score += popTerm
|
|
826
|
-
}
|
|
827
|
-
|
|
828
|
-
// Combined prominence for the exact-tier sort when proximity hints are present: population
|
|
829
|
-
// and nearness in the SAME additive units, so the map view / the user's location can win a
|
|
830
|
-
// cross-country postcode tie without a hard filter.
|
|
831
|
-
const prominence = popTerm + proximityTerm
|
|
832
|
-
|
|
833
|
-
const candidate: PlaceCandidate = {
|
|
834
|
-
id: row.id,
|
|
835
|
-
prominence,
|
|
836
|
-
name: row.name,
|
|
837
|
-
placetype: row.placetype as WOFPlacetype,
|
|
838
|
-
country: row.country ?? "",
|
|
839
|
-
lat: row.lat ?? 0,
|
|
840
|
-
lon: row.lon ?? 0,
|
|
841
|
-
parent_id: row.parent_id ?? undefined,
|
|
842
|
-
score,
|
|
843
|
-
}
|
|
844
|
-
|
|
845
|
-
if (distanceKm !== undefined) {
|
|
846
|
-
candidate.distanceKm = distanceKm
|
|
847
|
-
}
|
|
848
|
-
|
|
849
|
-
if (row.population !== null && row.population > 0) {
|
|
850
|
-
candidate.population = row.population
|
|
851
|
-
}
|
|
852
|
-
|
|
853
|
-
// Candidate bbox — parity with the WASM lookup (resolver-wof-wasm/lookup.ts), whose
|
|
854
|
-
// consumers (the demo cascade's region constraint) read it. Without this the Node
|
|
855
|
-
// backend's region→bbox constraint is dead and disambiguation falls to population
|
|
856
|
-
// ranking (the Springfield-IL→MO failure the #524 smoke eval caught).
|
|
857
|
-
if (
|
|
858
|
-
row.min_latitude != null &&
|
|
859
|
-
row.max_latitude != null &&
|
|
860
|
-
row.min_longitude != null &&
|
|
861
|
-
row.max_longitude != null
|
|
862
|
-
) {
|
|
863
|
-
candidate.bbox = {
|
|
864
|
-
minLat: row.min_latitude,
|
|
865
|
-
maxLat: row.max_latitude,
|
|
866
|
-
minLon: row.min_longitude,
|
|
867
|
-
maxLon: row.max_longitude,
|
|
868
|
-
}
|
|
869
|
-
}
|
|
870
|
-
|
|
871
|
-
return candidate
|
|
611
|
+
rankCandidates(candidates, {
|
|
612
|
+
db: this.#db,
|
|
613
|
+
schemaName: sch,
|
|
614
|
+
query,
|
|
615
|
+
weights: this.#weights,
|
|
872
616
|
})
|
|
873
617
|
|
|
874
|
-
// Exact-match tiering: a candidate whose name OR any alias equals the query text (case-folded)
|
|
875
|
-
// ranks above any partial match, with the weighted-sum score (incl. population) breaking ties
|
|
876
|
-
// WITHIN a tier. See the RankingWeights.exactMatchTiering docstring for why this aligns the
|
|
877
|
-
// population prior rather than overriding it. One cheap indexed lookup over the candidate ids.
|
|
878
|
-
// Runs even for a SINGLE candidate so `exactMatch` is stamped consistently (parity with the
|
|
879
|
-
// WASM lookup) — a sole alias hit ("New York City" → New York) must still carry the flag the
|
|
880
|
-
// demo cascade / #369 re-rank read.
|
|
881
|
-
if (this.#weights.exactMatchTiering && candidates.length) {
|
|
882
|
-
const exactIds = this.#exactMatchIds(
|
|
883
|
-
sch,
|
|
884
|
-
candidates.map((c) => c.id as number),
|
|
885
|
-
query.text
|
|
886
|
-
)
|
|
887
|
-
|
|
888
|
-
// Stamp the tier onto every candidate (not just when the tiering sort fires) so a downstream
|
|
889
|
-
// re-rank — #369's postcode-anchor country pin in `resolveTree` — can keep the country pin from
|
|
890
|
-
// crossing the exact/partial boundary ("ME" → Maine, not the more-populous Missouri).
|
|
891
|
-
for (const c of candidates) {
|
|
892
|
-
c.exactMatch = exactIds.has(c.id as number)
|
|
893
|
-
}
|
|
894
|
-
|
|
895
|
-
if (exactIds.size) {
|
|
896
|
-
// #905: WITHIN the exact tier, population is the PRIMARY key and the weighted score
|
|
897
|
-
// only breaks population ties. Exactness saturates text relevance, and the bm25
|
|
898
|
-
// residue inside `score` is length-noise (see the fetch-site comment), so letting it
|
|
899
|
-
// order the tier is what sent unscoped "Paris" to an Ohio township. The partial tier
|
|
900
|
-
// keeps score order — text relevance still means something there. This makes the
|
|
901
|
-
// exactMatchTiering docstring literal: match quality primary, prominence within.
|
|
902
|
-
//
|
|
903
|
-
// #912 sub-tier: a NAME-exact candidate (spr.name equals the query) outranks an
|
|
904
|
-
// ALIAS-exact one ('Paris' the place beats 'Paris Township' held via alias 'Paris').
|
|
905
|
-
// The place's own name is a stronger identity claim than an alias — aliases exist to
|
|
906
|
-
// widen recall, not to tie primaries. ME→Maine is untouched: 'ME' name-exact-matches
|
|
907
|
-
// nothing, so the alias sub-tier still decides there. Population orders within each
|
|
908
|
-
// sub-tier as before.
|
|
909
|
-
const norm = (v: string): string => v.toLowerCase().trim().replaceAll(/\s+/g, " ")
|
|
910
|
-
const needle = norm(query.text)
|
|
911
|
-
|
|
912
|
-
// #936 option 3: an OFFICIAL name (preferred form in an official language of the place's
|
|
913
|
-
// country, `names.official = 1`) counts as the place's own name for the sub-tier — "Åbo" is
|
|
914
|
-
// Turku's name, not merely its alias. Floor-gated on the holder's population (see the
|
|
915
|
-
// RankingWeights docstring for the measured 100k boundary). officialIds ⊆ exactIds by
|
|
916
|
-
// construction (official rows are names rows), so only the sub-tier KIND changes.
|
|
917
|
-
const officialIds = this.#weights.officialNameExact
|
|
918
|
-
? this.#officialNameIds(
|
|
919
|
-
sch,
|
|
920
|
-
candidates
|
|
921
|
-
.filter(
|
|
922
|
-
(c) => exactIds.has(c.id as number) && (c.population ?? 0) >= this.#weights.officialNameExactFloor
|
|
923
|
-
)
|
|
924
|
-
.map((c) => c.id as number),
|
|
925
|
-
query.text
|
|
926
|
-
)
|
|
927
|
-
: undefined
|
|
928
|
-
|
|
929
|
-
const kind = (c: PlaceCandidate): number => {
|
|
930
|
-
if (!exactIds.has(c.id as number)) return 0
|
|
931
|
-
|
|
932
|
-
if (norm(String(c.name ?? "")) === needle) return 2
|
|
933
|
-
|
|
934
|
-
return officialIds?.has(c.id as number) ? 2 : 1
|
|
935
|
-
}
|
|
936
|
-
|
|
937
|
-
// With proximity hints (near/bias), prominence (population + nearness, same units)
|
|
938
|
-
// replaces raw population as the within-tier key — the 48026 rule: the map view or
|
|
939
|
-
// the user's location breaks a cross-country postcode tie. Without hints, population
|
|
940
|
-
// ordering is byte-identical to before.
|
|
941
|
-
const hasHints = !!query.near || (query.bias?.length ?? 0) > 0
|
|
942
|
-
|
|
943
|
-
candidates.sort((a, b) => {
|
|
944
|
-
const ax = kind(a)
|
|
945
|
-
const bx = kind(b)
|
|
946
|
-
|
|
947
|
-
if (bx !== ax) return bx - ax
|
|
948
|
-
|
|
949
|
-
if (ax >= 1) {
|
|
950
|
-
if (hasHints) return (b.prominence ?? 0) - (a.prominence ?? 0) || b.score - a.score
|
|
951
|
-
|
|
952
|
-
return (b.population ?? 0) - (a.population ?? 0) || b.score - a.score
|
|
953
|
-
}
|
|
954
|
-
|
|
955
|
-
return b.score - a.score
|
|
956
|
-
})
|
|
957
|
-
|
|
958
|
-
return candidates.slice(0, limit)
|
|
959
|
-
}
|
|
960
|
-
}
|
|
961
|
-
|
|
962
|
-
candidates.sort((a, b) => b.score - a.score)
|
|
963
|
-
|
|
964
618
|
return candidates.slice(0, limit)
|
|
965
619
|
}
|
|
966
620
|
|
|
@@ -1034,12 +688,13 @@ export class WOFSqlitePlaceLookup implements PlaceLookup, Disposable {
|
|
|
1034
688
|
const pcWhere = query.country ? "postcode = ? AND country = ?" : "postcode = ?"
|
|
1035
689
|
const pcParams: SQLInputValue[] = query.country ? [pc, query.country] : [pc]
|
|
1036
690
|
|
|
1037
|
-
const pcRows =
|
|
1038
|
-
.prepare(
|
|
691
|
+
const pcRows = allRows<{ id: number; aliases: string | null; dist: number; containing: number }>(
|
|
692
|
+
this.#db.prepare(
|
|
1039
693
|
`SELECT locality_id AS id, aliases, distance_km AS dist, is_containing AS containing
|
|
1040
694
|
FROM ${sch}.${POSTCODE_LOCALITY_TABLE} WHERE ${pcWhere}`
|
|
1041
|
-
)
|
|
1042
|
-
|
|
695
|
+
),
|
|
696
|
+
...pcParams
|
|
697
|
+
)
|
|
1043
698
|
|
|
1044
699
|
if (!pcRows.length) return null
|
|
1045
700
|
|
|
@@ -1155,14 +810,15 @@ export class WOFSqlitePlaceLookup implements PlaceLookup, Disposable {
|
|
|
1155
810
|
const popJoin = hasPop ? `LEFT JOIN main.${PLACE_POPULATION_TABLE} pp ON pp.id = s.id` : ""
|
|
1156
811
|
const ph = ids.map(() => "?").join(", ")
|
|
1157
812
|
|
|
1158
|
-
const rows =
|
|
1159
|
-
.prepare(
|
|
813
|
+
const rows = allRows<RawSearchRow>(
|
|
814
|
+
this.#db.prepare(
|
|
1160
815
|
`SELECT s.id AS id, s.name AS name, s.country AS country, s.parent_id AS parent_id,
|
|
1161
816
|
s.latitude AS lat, s.longitude AS lon, s.placetype AS placetype, ${popSelect}
|
|
1162
817
|
FROM main.spr s ${popJoin}
|
|
1163
818
|
WHERE s.id IN (${ph}) AND s.is_current != 0`
|
|
1164
|
-
)
|
|
1165
|
-
|
|
819
|
+
),
|
|
820
|
+
...ids
|
|
821
|
+
)
|
|
1166
822
|
|
|
1167
823
|
return rows.map((row) => {
|
|
1168
824
|
const c: PlaceCandidate = {
|
|
@@ -1184,102 +840,6 @@ export class WOFSqlitePlaceLookup implements PlaceLookup, Disposable {
|
|
|
1184
840
|
})
|
|
1185
841
|
}
|
|
1186
842
|
|
|
1187
|
-
/**
|
|
1188
|
-
* Among `ids`, return the subset whose name OR any alias equals `text` case-insensitively — the exact-match tier for
|
|
1189
|
-
* ranking. One indexed query over `<schema>.names`. When the shard has no `names` table (a slim DB built with
|
|
1190
|
-
* `dropNames`, or a postcode-only shard), fall back to the self-contained `place_search` FTS content: its `alt_names`
|
|
1191
|
-
* column is the same alias set joined on the boundary-preserving `ALIAS_SEPARATOR` (#523), so `aliasBagExactMatch`
|
|
1192
|
-
* recovers the exact alias tier ("New York City" → New York) that the dropped `names` table used to provide.
|
|
1193
|
-
*/
|
|
1194
|
-
#exactMatchIds(schemaName: string, ids: number[], text: string): Set<number> {
|
|
1195
|
-
const out = new Set<number>()
|
|
1196
|
-
const trimmed = text.trim()
|
|
1197
|
-
|
|
1198
|
-
if (!ids.length || !trimmed) return out
|
|
1199
|
-
const placeholders = ids.map(() => "?").join(", ")
|
|
1200
|
-
|
|
1201
|
-
try {
|
|
1202
|
-
const rows = this.#db
|
|
1203
|
-
.prepare(
|
|
1204
|
-
`SELECT DISTINCT id FROM ${schemaName}.names WHERE id IN (${placeholders}) AND name = ? COLLATE NOCASE`
|
|
1205
|
-
)
|
|
1206
|
-
.all(...ids, trimmed) as Array<{ id: number }>
|
|
1207
|
-
|
|
1208
|
-
for (const r of rows) {
|
|
1209
|
-
out.add(r.id)
|
|
1210
|
-
}
|
|
1211
|
-
|
|
1212
|
-
return out
|
|
1213
|
-
} catch {
|
|
1214
|
-
// No `names` table on this shard — fall through to the place_search alias bag.
|
|
1215
|
-
}
|
|
1216
|
-
|
|
1217
|
-
try {
|
|
1218
|
-
const rows = this.#db
|
|
1219
|
-
.prepare(
|
|
1220
|
-
`SELECT wof_id AS id, name, alt_names FROM ${schemaName}.place_search WHERE wof_id IN (${placeholders})`
|
|
1221
|
-
)
|
|
1222
|
-
.all(...ids) as Array<{ id: number; name: string | null; alt_names: string | null }>
|
|
1223
|
-
|
|
1224
|
-
const norm = (s: string): string => s.toLowerCase().trim().replaceAll(/\s+/g, " ")
|
|
1225
|
-
const needle = norm(trimmed)
|
|
1226
|
-
|
|
1227
|
-
for (const r of rows) {
|
|
1228
|
-
if (r.name !== null && norm(r.name) === needle) {
|
|
1229
|
-
out.add(r.id)
|
|
1230
|
-
}
|
|
1231
|
-
}
|
|
1232
|
-
|
|
1233
|
-
// Alias pass via the shared bag parser (#523). Separated bags (built since #523) get a true
|
|
1234
|
-
// per-alias equality check, ungated — matching the `names`-table branch above, where an
|
|
1235
|
-
// alias match counts as exact regardless of other candidates. Legacy bags (no separator)
|
|
1236
|
-
// fall back to padded containment, gated on "no canonical exact in the pool" because their
|
|
1237
|
-
// lost boundaries would otherwise false-promote interior fragments ("York" inside the alias
|
|
1238
|
-
// "New York City") or cross-alias fragments ("York New" across "…York" + "New City…").
|
|
1239
|
-
const anyCanonicalExact = out.size > 0
|
|
1240
|
-
|
|
1241
|
-
for (const r of rows) {
|
|
1242
|
-
if (aliasBagExactMatch(r.alt_names, needle, anyCanonicalExact)) {
|
|
1243
|
-
out.add(r.id)
|
|
1244
|
-
}
|
|
1245
|
-
}
|
|
1246
|
-
} catch {
|
|
1247
|
-
// Shard without place_search either → no exact-match tier. Falls back to weighted-sum order.
|
|
1248
|
-
}
|
|
1249
|
-
|
|
1250
|
-
return out
|
|
1251
|
-
}
|
|
1252
|
-
|
|
1253
|
-
/**
|
|
1254
|
-
* Among `ids` (already known exact matches), the subset holding `text` as an OFFICIAL name (`names.official = 1`, the
|
|
1255
|
-
* #940 ingest bit). Same COLLATE NOCASE semantics as {@link WOFSqlitePlaceLookup.#exactMatchIds} so the two probes
|
|
1256
|
-
* agree on what "equals the query" means. Fails soft on gazetteers built before #940 (no `official` column) — the
|
|
1257
|
-
* sub-tier then behaves exactly as if `officialNameExact` were off.
|
|
1258
|
-
*/
|
|
1259
|
-
#officialNameIds(schemaName: string, ids: number[], text: string): Set<number> {
|
|
1260
|
-
const out = new Set<number>()
|
|
1261
|
-
const trimmed = text.trim()
|
|
1262
|
-
|
|
1263
|
-
if (!ids.length || !trimmed) return out
|
|
1264
|
-
const placeholders = ids.map(() => "?").join(", ")
|
|
1265
|
-
|
|
1266
|
-
try {
|
|
1267
|
-
const rows = this.#db
|
|
1268
|
-
.prepare(
|
|
1269
|
-
`SELECT DISTINCT id FROM ${schemaName}.names WHERE id IN (${placeholders}) AND official = 1 AND name = ? COLLATE NOCASE`
|
|
1270
|
-
)
|
|
1271
|
-
.all(...ids, trimmed) as Array<{ id: number }>
|
|
1272
|
-
|
|
1273
|
-
for (const r of rows) {
|
|
1274
|
-
out.add(r.id)
|
|
1275
|
-
}
|
|
1276
|
-
} catch {
|
|
1277
|
-
// Pre-#940 gazetteer (no `official` column) or a names-less slim shard — feature inert.
|
|
1278
|
-
}
|
|
1279
|
-
|
|
1280
|
-
return out
|
|
1281
|
-
}
|
|
1282
|
-
|
|
1283
843
|
close(): void {
|
|
1284
844
|
// Destroying the Kysely instance closes the underlying connection IF we own it. If the caller
|
|
1285
845
|
// passed in a pre-opened DatabaseSync (test fixture), respect their ownership.
|
|
@@ -1304,12 +864,10 @@ export class WOFSqlitePlaceLookup implements PlaceLookup, Disposable {
|
|
|
1304
864
|
#assertFTSExists(): void {
|
|
1305
865
|
if (!placeSearchFTSExists(this.#db)) {
|
|
1306
866
|
throw new Error(
|
|
1307
|
-
"
|
|
867
|
+
"WOFSQLitePlaceLookup: `place_search` FTS5 table is missing. Pass `buildFTS: true` to build it on open, or run `mailwoman gazetteer build fts <path-to-wof.db>` ahead of time (see resolver-wof-sqlite/README.md)."
|
|
1308
868
|
)
|
|
1309
869
|
}
|
|
1310
870
|
}
|
|
1311
871
|
}
|
|
1312
872
|
|
|
1313
|
-
export { trigramJaccard, trigrams } from "./name-score.ts"
|
|
1314
|
-
|
|
1315
873
|
export type { RankingWeights } from "./ranking-weights.ts"
|