@mailwoman/resolver-wof-sqlite 9.1.0 → 9.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -9
- package/address-point-interpolation.ts +18 -8
- package/address-point-schema.ts +18 -6
- package/address-point.ts +111 -18
- package/ancestry.ts +9 -6
- package/build-candidate.ts +200 -163
- package/build-slim.ts +3 -3
- package/candidate/alias-bags.ts +54 -0
- package/candidate/ancestors-sidecar.ts +206 -0
- package/candidate/country-display-names.ts +79 -0
- package/candidate/name-roles.ts +237 -0
- package/candidate/own-name.ts +146 -0
- package/candidate/place-attrs.ts +44 -0
- package/candidate/shard-fold.ts +137 -0
- package/candidate-ancestors-schema.ts +195 -0
- package/candidate-fts.ts +4 -2
- package/candidate-importance.ts +2 -1
- package/candidate-lookup.ts +439 -186
- package/candidate-schema.ts +33 -5
- package/candidate-scoring.ts +268 -0
- package/capital-schema.ts +90 -0
- package/capitals.ts +148 -0
- package/coincident-roles.ts +69 -10
- package/convention-schema.ts +72 -0
- package/convention.ts +2 -2
- package/coverage-manifest-schema.ts +7 -7
- package/currency-backfill.ts +249 -0
- package/exact-match.ts +104 -0
- package/fst-autocomplete.ts +91 -119
- package/fst-builder.ts +14 -12
- package/fst-freshness.ts +2 -2
- package/fts-query.ts +1 -1
- package/fts.ts +4 -4
- package/geonames-postal.ts +2 -2
- package/index.ts +18 -14
- package/interpolation.ts +113 -19
- package/lookup.ts +110 -591
- package/name-score.ts +6 -4
- package/out/address-point-interpolation.d.ts.map +1 -1
- package/out/address-point-interpolation.js +13 -7
- package/out/address-point-interpolation.js.map +1 -1
- package/out/address-point-schema.d.ts +16 -6
- package/out/address-point-schema.d.ts.map +1 -1
- package/out/address-point-schema.js.map +1 -1
- package/out/address-point.d.ts.map +1 -1
- package/out/address-point.js +70 -14
- package/out/address-point.js.map +1 -1
- package/out/ancestry.d.ts +2 -2
- package/out/ancestry.d.ts.map +1 -1
- package/out/ancestry.js +5 -6
- package/out/ancestry.js.map +1 -1
- package/out/build-candidate.d.ts +75 -0
- package/out/build-candidate.d.ts.map +1 -1
- package/out/build-candidate.js +120 -122
- package/out/build-candidate.js.map +1 -1
- package/out/build-slim.d.ts +1 -1
- package/out/build-slim.js +3 -3
- package/out/build-slim.js.map +1 -1
- package/out/candidate/alias-bags.d.ts +17 -0
- package/out/candidate/alias-bags.d.ts.map +1 -0
- package/out/candidate/alias-bags.js +39 -0
- package/out/candidate/alias-bags.js.map +1 -0
- package/out/candidate/ancestors-sidecar.d.ts +33 -0
- package/out/candidate/ancestors-sidecar.d.ts.map +1 -0
- package/out/candidate/ancestors-sidecar.js +140 -0
- package/out/candidate/ancestors-sidecar.js.map +1 -0
- package/out/candidate/country-display-names.d.ts +35 -0
- package/out/candidate/country-display-names.d.ts.map +1 -0
- package/out/candidate/country-display-names.js +59 -0
- package/out/candidate/country-display-names.js.map +1 -0
- package/out/candidate/name-roles.d.ts +55 -0
- package/out/candidate/name-roles.d.ts.map +1 -0
- package/out/candidate/name-roles.js +165 -0
- package/out/candidate/name-roles.js.map +1 -0
- package/out/candidate/own-name.d.ts +50 -0
- package/out/candidate/own-name.d.ts.map +1 -0
- package/out/candidate/own-name.js +132 -0
- package/out/candidate/own-name.js.map +1 -0
- package/out/candidate/place-attrs.d.ts +43 -0
- package/out/candidate/place-attrs.d.ts.map +1 -0
- package/out/candidate/place-attrs.js +15 -0
- package/out/candidate/place-attrs.js.map +1 -0
- package/out/candidate/shard-fold.d.ts +31 -0
- package/out/candidate/shard-fold.d.ts.map +1 -0
- package/out/candidate/shard-fold.js +104 -0
- package/out/candidate/shard-fold.js.map +1 -0
- package/out/candidate-ancestors-schema.d.ts +150 -0
- package/out/candidate-ancestors-schema.d.ts.map +1 -0
- package/out/candidate-ancestors-schema.js +123 -0
- package/out/candidate-ancestors-schema.js.map +1 -0
- package/out/candidate-fts.d.ts +4 -2
- package/out/candidate-fts.d.ts.map +1 -1
- package/out/candidate-fts.js +4 -2
- package/out/candidate-fts.js.map +1 -1
- package/out/candidate-importance.d.ts.map +1 -1
- package/out/candidate-importance.js +1 -1
- package/out/candidate-importance.js.map +1 -1
- package/out/candidate-lookup.d.ts +22 -45
- package/out/candidate-lookup.d.ts.map +1 -1
- package/out/candidate-lookup.js +340 -135
- package/out/candidate-lookup.js.map +1 -1
- package/out/candidate-schema.d.ts +30 -6
- package/out/candidate-schema.d.ts.map +1 -1
- package/out/candidate-schema.js +3 -0
- package/out/candidate-schema.js.map +1 -1
- package/out/candidate-scoring.d.ts +34 -0
- package/out/candidate-scoring.d.ts.map +1 -0
- package/out/candidate-scoring.js +200 -0
- package/out/candidate-scoring.js.map +1 -0
- package/out/capital-schema.d.ts +51 -0
- package/out/capital-schema.d.ts.map +1 -0
- package/out/capital-schema.js +63 -0
- package/out/capital-schema.js.map +1 -0
- package/out/capitals.d.ts +69 -0
- package/out/capitals.d.ts.map +1 -0
- package/out/capitals.js +98 -0
- package/out/capitals.js.map +1 -0
- package/out/coincident-roles.d.ts +7 -0
- package/out/coincident-roles.d.ts.map +1 -1
- package/out/coincident-roles.js +42 -8
- package/out/coincident-roles.js.map +1 -1
- package/out/convention-schema.d.ts +51 -0
- package/out/convention-schema.d.ts.map +1 -0
- package/out/convention-schema.js +34 -0
- package/out/convention-schema.js.map +1 -0
- package/out/convention.d.ts +1 -1
- package/out/convention.js +2 -2
- package/out/coverage-manifest-schema.js +3 -7
- package/out/coverage-manifest-schema.js.map +1 -1
- package/out/currency-backfill.d.ts +46 -0
- package/out/currency-backfill.d.ts.map +1 -0
- package/out/currency-backfill.js +180 -0
- package/out/currency-backfill.js.map +1 -0
- package/out/exact-match.d.ts +25 -0
- package/out/exact-match.d.ts.map +1 -0
- package/out/exact-match.js +89 -0
- package/out/exact-match.js.map +1 -0
- package/out/fst-autocomplete.d.ts +11 -11
- package/out/fst-autocomplete.d.ts.map +1 -1
- package/out/fst-autocomplete.js +82 -99
- package/out/fst-autocomplete.js.map +1 -1
- package/out/fst-builder.d.ts.map +1 -1
- package/out/fst-builder.js +11 -12
- package/out/fst-builder.js.map +1 -1
- package/out/fst-freshness.d.ts +2 -2
- package/out/fst-freshness.js +2 -2
- package/out/fts-query.js +1 -1
- package/out/fts-query.js.map +1 -1
- package/out/fts.d.ts +4 -4
- package/out/fts.js +4 -4
- package/out/geonames-postal.d.ts +2 -2
- package/out/geonames-postal.js +2 -2
- package/out/index.d.ts +3 -2
- package/out/index.d.ts.map +1 -1
- package/out/index.js +2 -2
- package/out/index.js.map +1 -1
- package/out/interpolation.d.ts +8 -0
- package/out/interpolation.d.ts.map +1 -1
- package/out/interpolation.js +91 -19
- package/out/interpolation.js.map +1 -1
- package/out/lookup.d.ts +4 -5
- package/out/lookup.d.ts.map +1 -1
- package/out/lookup.js +94 -468
- package/out/lookup.js.map +1 -1
- package/out/name-score.d.ts +0 -10
- package/out/name-score.d.ts.map +1 -1
- package/out/name-score.js +6 -4
- package/out/name-score.js.map +1 -1
- package/out/place-importance-schema.d.ts +42 -5
- package/out/place-importance-schema.d.ts.map +1 -1
- package/out/place-importance-schema.js +54 -8
- package/out/place-importance-schema.js.map +1 -1
- package/out/poi-lookup.d.ts +1 -1
- package/out/poi-lookup.d.ts.map +1 -1
- package/out/poi-lookup.js +12 -13
- package/out/poi-lookup.js.map +1 -1
- package/out/poi-schema.d.ts +7 -3
- package/out/poi-schema.d.ts.map +1 -1
- package/out/poi-schema.js.map +1 -1
- package/out/polygon-schema.d.ts +37 -0
- package/out/polygon-schema.d.ts.map +1 -0
- package/out/polygon-schema.js +23 -0
- package/out/polygon-schema.js.map +1 -0
- package/out/postal-city-alias-lookup.d.ts +1 -1
- package/out/postal-city-alias-lookup.js +1 -1
- package/out/postal-city-candidate-schema.d.ts +2 -1
- package/out/postal-city-candidate-schema.d.ts.map +1 -1
- package/out/postal-city-candidate-schema.js.map +1 -1
- package/out/postcode-point-lookup.d.ts +1 -1
- package/out/postcode-point-lookup.js +1 -1
- package/out/primary-preference.d.ts +125 -0
- package/out/primary-preference.d.ts.map +1 -0
- package/out/primary-preference.js +138 -0
- package/out/primary-preference.js.map +1 -0
- package/out/proximity-rerank.d.ts +77 -0
- package/out/proximity-rerank.d.ts.map +1 -0
- package/out/proximity-rerank.js +86 -0
- package/out/proximity-rerank.js.map +1 -0
- package/out/region-keys.d.ts +47 -0
- package/out/region-keys.d.ts.map +1 -0
- package/out/region-keys.js +121 -0
- package/out/region-keys.js.map +1 -0
- package/out/reverse.d.ts.map +1 -1
- package/out/reverse.js +6 -9
- package/out/reverse.js.map +1 -1
- package/out/schema.d.ts +1 -1
- package/out/search-fetch.d.ts +57 -0
- package/out/search-fetch.d.ts.map +1 -0
- package/out/search-fetch.js +183 -0
- package/out/search-fetch.js.map +1 -0
- package/out/sharding.d.ts +3 -3
- package/out/sharding.js +1 -1
- package/out/sqlite-convention-source.d.ts +1 -1
- package/out/sqlite-convention-source.js +1 -1
- package/out/sqlite-utils.d.ts +19 -1
- package/out/sqlite-utils.d.ts.map +1 -1
- package/out/sqlite-utils.js +19 -1
- package/out/sqlite-utils.js.map +1 -1
- package/out/street-centroid-schema.d.ts +7 -2
- package/out/street-centroid-schema.d.ts.map +1 -1
- package/out/street-centroid-schema.js.map +1 -1
- package/out/street-centroid.d.ts.map +1 -1
- package/out/street-centroid.js +7 -7
- package/out/street-centroid.js.map +1 -1
- package/out/street-normalize.d.ts +82 -9
- package/out/street-normalize.d.ts.map +1 -1
- package/out/street-normalize.js +175 -9
- package/out/street-normalize.js.map +1 -1
- package/out/street-segment-schema.d.ts +6 -2
- package/out/street-segment-schema.d.ts.map +1 -1
- package/out/street-segment-schema.js.map +1 -1
- package/out/types.d.ts +35 -1
- package/out/types.d.ts.map +1 -1
- package/out/unified-schema.d.ts +1 -1
- package/out/unified-schema.js +1 -1
- package/out/uprn-lookup.d.ts +85 -0
- package/out/uprn-lookup.d.ts.map +1 -0
- package/out/uprn-lookup.js +152 -0
- package/out/uprn-lookup.js.map +1 -0
- package/out/uprn-schema.d.ts +93 -0
- package/out/uprn-schema.d.ts.map +1 -0
- package/out/uprn-schema.js +78 -0
- package/out/uprn-schema.js.map +1 -0
- package/out/weights-overlay-linker.d.ts +141 -0
- package/out/weights-overlay-linker.d.ts.map +1 -0
- package/out/weights-overlay-linker.js +259 -0
- package/out/weights-overlay-linker.js.map +1 -0
- package/package.json +288 -16
- package/place-importance-schema.ts +64 -15
- package/poi-lookup.ts +12 -13
- package/poi-schema.ts +8 -3
- package/polygon-schema.ts +47 -0
- package/postal-city-alias-lookup.ts +1 -1
- package/postal-city-candidate-schema.ts +3 -1
- package/postcode-point-lookup.ts +1 -1
- package/primary-preference.ts +207 -0
- package/proximity-rerank.ts +120 -0
- package/region-keys.ts +144 -0
- package/reverse.ts +17 -16
- package/schema.ts +1 -1
- package/search-fetch.ts +256 -0
- package/sharding.ts +3 -3
- package/sqlite-convention-source.ts +1 -1
- package/sqlite-utils.ts +43 -2
- package/street-centroid-schema.ts +8 -2
- package/street-centroid.ts +13 -8
- package/street-normalize.ts +252 -23
- package/street-segment-schema.ts +7 -2
- package/types.ts +35 -1
- package/unified-schema.ts +1 -1
- package/uprn-lookup.ts +210 -0
- package/uprn-schema.ts +124 -0
- package/weights-overlay-linker.ts +377 -0
- package/geo.ts +0 -121
- package/out/geo.d.ts +0 -74
- package/out/geo.d.ts.map +0 -1
- package/out/geo.js +0 -71
- package/out/geo.js.map +0 -1
package/build-candidate.ts
CHANGED
|
@@ -32,11 +32,17 @@
|
|
|
32
32
|
* `candidate-importance.ts`, which owns that join and explains why the id would be wrong). It is
|
|
33
33
|
* optional: without {@link BuildCandidateOptions.importance} the column is NULL on every row, which
|
|
34
34
|
* the consumer reads as unmeasured and ignores.
|
|
35
|
+
*
|
|
36
|
+
* The build also materializes the ANCESTORS SIDECAR (`candidate_ancestor` closure rows +
|
|
37
|
+
* `candidate_interval` pre/post labels) from the source `ancestors` table — the containment
|
|
38
|
+
* lineage behind {@link WOFCandidateTableLookup.ancestors} and the admin-coherence check.
|
|
39
|
+
* `candidate-ancestors-schema.ts` owns the encoding decision and the DAG/absence semantics.
|
|
35
40
|
*/
|
|
36
41
|
|
|
37
42
|
import { existsSync, rmSync } from "node:fs"
|
|
38
43
|
import { DatabaseSync } from "node:sqlite"
|
|
39
44
|
|
|
45
|
+
import { COUNTRY_POPULATION } from "@mailwoman/codex/country"
|
|
40
46
|
import { DatabaseClient } from "@mailwoman/core/kysley/client"
|
|
41
47
|
|
|
42
48
|
import { createCandidateFTS } from "./candidate-fts.ts"
|
|
@@ -47,12 +53,22 @@ import {
|
|
|
47
53
|
createCandidateTable,
|
|
48
54
|
type CandidateDatabase,
|
|
49
55
|
} from "./candidate-schema.ts"
|
|
56
|
+
import { explodeAliasBags } from "./candidate/alias-bags.ts"
|
|
57
|
+
import { buildAncestorsSidecar } from "./candidate/ancestors-sidecar.ts"
|
|
58
|
+
import { stageCountryDisplayNames } from "./candidate/country-display-names.ts"
|
|
59
|
+
import { GLOSS_KEY_THRESHOLD, stampNameRoles } from "./candidate/name-roles.ts"
|
|
60
|
+
import type { PlaceAttrs } from "./candidate/place-attrs.ts"
|
|
61
|
+
import { foldShard } from "./candidate/shard-fold.ts"
|
|
62
|
+
import { createCapitalTable } from "./capital-schema.ts"
|
|
63
|
+
import type { CapitalPoint } from "./capitals.ts"
|
|
64
|
+
import { resurrectCurrencyHoles } from "./currency-backfill.ts"
|
|
50
65
|
import { normalizeLocalityForKey } from "./street-normalize.ts"
|
|
51
66
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
67
|
+
// The build's contract is this module path; the passes behind it live in `./candidate/`. Re-exported
|
|
68
|
+
// here so a consumer never has to know which pass owns which name.
|
|
69
|
+
export { stageCountryDisplayNames } from "./candidate/country-display-names.ts"
|
|
70
|
+
export { GLOSS_EXCLUDED_PLACETYPES, GLOSS_KEY_THRESHOLD } from "./candidate/name-roles.ts"
|
|
71
|
+
export type { PlaceAttrs } from "./candidate/place-attrs.ts"
|
|
56
72
|
|
|
57
73
|
export interface BuildCandidateOptions {
|
|
58
74
|
/**
|
|
@@ -63,6 +79,12 @@ export interface BuildCandidateOptions {
|
|
|
63
79
|
* Output candidate DB path (overwritten if present).
|
|
64
80
|
*/
|
|
65
81
|
output: string
|
|
82
|
+
/**
|
|
83
|
+
* The capital-status reference entries (#1880) to carry in-artifact — the parsed `data/gazetteer/capitals-v1.json`
|
|
84
|
+
* entries, passed by the CALLER because this module publishes to npm and must not read repo-root paths. Absent → the
|
|
85
|
+
* `capital` table is not created, and the session loader falls back to the repo file where one exists.
|
|
86
|
+
*/
|
|
87
|
+
capitals?: readonly CapitalPoint[]
|
|
66
88
|
/**
|
|
67
89
|
* Optional postcode shards (`spr` rows with `placetype='postalcode'` + real coords, e.g. postalcode-us.db) — folded
|
|
68
90
|
* in as `postalcode` candidate rows so `findPlace(postalcode)` resolves a ZIP directly (the demo's primary postcode
|
|
@@ -72,6 +94,14 @@ export interface BuildCandidateOptions {
|
|
|
72
94
|
* ("Brooklyn" for 11201), and they were previously reachable only through FTS.
|
|
73
95
|
*/
|
|
74
96
|
postcodes?: string[]
|
|
97
|
+
/**
|
|
98
|
+
* Optional LOCALITY shards (`spr` rows with `placetype='locality'` + real coords, e.g. localities-nz-linz.db — the
|
|
99
|
+
* #1564 NZ suburb tier) — folded through the same shard loop as the postcode shards, staged as `locality` candidate
|
|
100
|
+
* rows with no region scope and UNMEASURED population (`neg_rank 0`: a shard row ranks behind any populated namesake
|
|
101
|
+
* and wins only where its key is the answer). Each shard's `names` table folds as aliases, `is_primary = 0`, same as
|
|
102
|
+
* the delivery-city pass.
|
|
103
|
+
*/
|
|
104
|
+
localities?: string[]
|
|
75
105
|
/**
|
|
76
106
|
* Optional WOF admin database carrying a `place_importance` table — the source of the `importance` column (#28), the
|
|
77
107
|
* toponym-fame prior that decides the bare-city-name class. Joined by `(name_key, country, placetype)` + nearest
|
|
@@ -84,10 +114,27 @@ export interface BuildCandidateOptions {
|
|
|
84
114
|
* population-derived stand-in written into a column that means fame.
|
|
85
115
|
*/
|
|
86
116
|
importance?: string
|
|
117
|
+
/**
|
|
118
|
+
* Cross-source currency backfill (#1737). WOF carries deprecated-with-no-successor records for real, populated
|
|
119
|
+
* settlements (Rochester Kent, Aldershot, Telford — 120 GB localities alone), and the currency filter correctly drops
|
|
120
|
+
* them, leaving holes no ranking can fill. When this option is set, pass 1c resurrects a dead locality ONLY under
|
|
121
|
+
* three gates, positive evidence throughout: no live same-name row of any placetype near the dead record (a distant
|
|
122
|
+
* same-name row is a NAMESAKE and does not block — Rochester, Northumberland pop 318 must not veto Rochester, Kent);
|
|
123
|
+
* an independent GeoNames P-class attestation of the same folded name within {@link CURRENCY_BACKFILL_RADIUS_KM}; and
|
|
124
|
+
* the attestor at or above {@link CURRENCY_BACKFILL_POP_FLOOR}. The staged row keeps the real WOF id, name, centroid
|
|
125
|
+
* and bbox — GeoNames only ATTESTS the place and supplies the population that lets it stand in prominence races.
|
|
126
|
+
* `countries` are judged only where `<cc>.txt` exists under `geonamesDir`; absent dumps are skipped loudly.
|
|
127
|
+
*/
|
|
128
|
+
currencyBackfill?: { geonamesDir: string; countries: readonly string[] }
|
|
87
129
|
/**
|
|
88
130
|
* Optional progress callback for CLI / test introspection.
|
|
89
131
|
*/
|
|
90
132
|
onProgress?: (phase: string, message: string) => void
|
|
133
|
+
/**
|
|
134
|
+
* Key-count threshold for the gloss anomaly detector — {@link GLOSS_KEY_THRESHOLD} unless a test passes a
|
|
135
|
+
* fixture-scale value. The production number is the #1730 sweep's own cut (4,000 places at >= 50 keys).
|
|
136
|
+
*/
|
|
137
|
+
glossKeyThreshold?: number
|
|
91
138
|
}
|
|
92
139
|
|
|
93
140
|
export interface BuildCandidateResult {
|
|
@@ -103,6 +150,21 @@ export interface BuildCandidateResult {
|
|
|
103
150
|
* through `onProgress`.
|
|
104
151
|
*/
|
|
105
152
|
postcodeAliases: number
|
|
153
|
+
/**
|
|
154
|
+
* Closure rows in the `candidate_ancestor` sidecar (see candidate-ancestors-schema.ts). Zero means the source
|
|
155
|
+
* `ancestors` table contributed nothing within the resolvable placetypes — a finding, reported rather than implied.
|
|
156
|
+
*/
|
|
157
|
+
ancestorRows: number
|
|
158
|
+
/**
|
|
159
|
+
* Places carrying at least one closure row.
|
|
160
|
+
*/
|
|
161
|
+
ancestorPlaces: number
|
|
162
|
+
/**
|
|
163
|
+
* Places that received a pre/post interval label — the canonical-parent forest's node count. Places outside it (no
|
|
164
|
+
* recorded ancestry, shard rows, cycle-skipped) have NO label: containment against them is unverifiable, never
|
|
165
|
+
* false.
|
|
166
|
+
*/
|
|
167
|
+
intervalPlaces: number
|
|
106
168
|
/**
|
|
107
169
|
* Places that took an `importance` score from the join (#28).
|
|
108
170
|
*
|
|
@@ -118,28 +180,23 @@ export interface BuildCandidateResult {
|
|
|
118
180
|
* join is being asked to guess.
|
|
119
181
|
*/
|
|
120
182
|
importanceGated?: number
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
interface PlaceAttrs {
|
|
124
|
-
cid: number
|
|
125
|
-
rid: number
|
|
126
|
-
ptid: number
|
|
127
|
-
name: string
|
|
128
|
-
lat: number
|
|
129
|
-
lon: number
|
|
130
|
-
mnLat: number
|
|
131
|
-
mnLon: number
|
|
132
|
-
mxLat: number
|
|
133
|
-
mxLon: number
|
|
134
|
-
pop: number
|
|
135
|
-
neg: number
|
|
136
|
-
pkey: string
|
|
137
183
|
/**
|
|
138
|
-
*
|
|
139
|
-
|
|
140
|
-
|
|
184
|
+
* Alias rows stamped `name_role = 'gloss'` — the anomaly detector's certain core (#1730).
|
|
185
|
+
*/
|
|
186
|
+
roleGloss: number
|
|
187
|
+
/**
|
|
188
|
+
* Alias rows stamped `name_role = 'abbr'` — the variant-in-official-language provenance signal (#1730/#936).
|
|
189
|
+
*/
|
|
190
|
+
roleAbbr: number
|
|
191
|
+
/**
|
|
192
|
+
* Admin places whose staged key count reached {@link GLOSS_KEY_THRESHOLD} — the sweep's tail, reported so the stamped
|
|
193
|
+
* fraction has its denominator.
|
|
194
|
+
*/
|
|
195
|
+
keyTailPlaces: number
|
|
196
|
+
/**
|
|
197
|
+
* Of {@link BuildCandidateResult.keyTailPlaces}, how many carry at least one stamped role row.
|
|
141
198
|
*/
|
|
142
|
-
|
|
199
|
+
keyTailWithRole: number
|
|
143
200
|
}
|
|
144
201
|
|
|
145
202
|
export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<BuildCandidateResult> {
|
|
@@ -261,7 +318,12 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
261
318
|
const cid = ccID(r.country as string | null)
|
|
262
319
|
const ptid = ptID(r.placetype as string | null)
|
|
263
320
|
const rid = regionOf.get(sid) ?? 0
|
|
264
|
-
|
|
321
|
+
// A zero population on a COUNTRY row is a WOF absence artifact, never a real zero — 147 of 237
|
|
322
|
+
// primary country records carried none (measured 2026-08-18, #1650), which ranked those nations
|
|
323
|
+
// below any namesake hamlet in every prominence race ("Georgia" → Georgia VT). The codex table is
|
|
324
|
+
// the secondary source; a country absent from it too stays at zero honestly.
|
|
325
|
+
const wofPop = Number(r.pop) || 0
|
|
326
|
+
const pop = wofPop === 0 && r.placetype === "country" ? (COUNTRY_POPULATION[String(r.country ?? "")] ?? 0) : wofPop
|
|
265
327
|
const neg = -Math.log10(pop + 1)
|
|
266
328
|
const name = String(r.name ?? "")
|
|
267
329
|
const pkey = normalizeLocalityForKey(name)
|
|
@@ -304,7 +366,8 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
304
366
|
a.mxLon,
|
|
305
367
|
pop,
|
|
306
368
|
1,
|
|
307
|
-
a.imp
|
|
369
|
+
a.imp,
|
|
370
|
+
null
|
|
308
371
|
)
|
|
309
372
|
|
|
310
373
|
nPrim++
|
|
@@ -339,34 +402,52 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
339
402
|
a.mxLon,
|
|
340
403
|
a.pop,
|
|
341
404
|
isPrimary,
|
|
342
|
-
a.imp
|
|
405
|
+
a.imp,
|
|
406
|
+
null
|
|
343
407
|
)
|
|
344
408
|
}
|
|
345
409
|
|
|
346
|
-
// --- pass
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
const k = normalizeLocalityForKey(piece)
|
|
360
|
-
|
|
361
|
-
if (!k || seen.has(k)) continue
|
|
362
|
-
seen.add(k)
|
|
363
|
-
stageRow(k, a, Number(r.wof_id), 0)
|
|
410
|
+
// --- pass 1b: country surfaces across scripts (#1678 thread 1) — see stageCountryDisplayNames ---
|
|
411
|
+
// Never a silent zero: a runtime whose ICU lacks these locales degrades to fewer surfaces, and the count is how a
|
|
412
|
+
// reader tells that apart from the pass not having run.
|
|
413
|
+
progress(
|
|
414
|
+
"country-display-names",
|
|
415
|
+
`${stageCountryDisplayNames({
|
|
416
|
+
attrs,
|
|
417
|
+
iso2ByID: new Map([...ccodes].map(([code, id]) => [id, code])),
|
|
418
|
+
countryPtID: ptID("country"),
|
|
419
|
+
stageRow,
|
|
420
|
+
tx: out,
|
|
421
|
+
}).toLocaleString()} country surfaces`
|
|
422
|
+
)
|
|
364
423
|
|
|
365
|
-
|
|
366
|
-
|
|
424
|
+
// --- pass 1c: cross-source currency backfill (#1737 — resurrectCurrencyHoles owns the gates). Runs BEFORE
|
|
425
|
+
// the alias pass so a resurrected place's alt names explode like any primary's. ---
|
|
426
|
+
if (opts.currencyBackfill) {
|
|
427
|
+
const nBackfill = await resurrectCurrencyHoles({
|
|
428
|
+
src,
|
|
429
|
+
tx: out,
|
|
430
|
+
geonamesDir: opts.currencyBackfill.geonamesDir,
|
|
431
|
+
countries: opts.currencyBackfill.countries,
|
|
432
|
+
attrs,
|
|
433
|
+
ccID,
|
|
434
|
+
ptID,
|
|
435
|
+
regionOf,
|
|
436
|
+
importance,
|
|
437
|
+
stageRow,
|
|
438
|
+
progress,
|
|
439
|
+
})
|
|
440
|
+
|
|
441
|
+
progress("currency-backfill", `${nBackfill.toLocaleString()} resurrections staged`)
|
|
442
|
+
} else {
|
|
443
|
+
// Never a silent absence: a build without the option leaves the deprecated-no-successor holes dead,
|
|
444
|
+
// and the log must say so rather than leave it inferable only from a missing row.
|
|
445
|
+
progress("currency-backfill", "not configured — deprecated-no-successor holes stay dead (#1737)")
|
|
367
446
|
}
|
|
368
447
|
|
|
369
|
-
|
|
448
|
+
// --- pass 2: distinct normalized aliases from place_search.alt_names (explodeAliasBags owns the loop) ---
|
|
449
|
+
progress("aliases", "exploding alias bags")
|
|
450
|
+
const { nAlias, keyCounts } = explodeAliasBags(src, out, attrs, stageRow)
|
|
370
451
|
progress("aliases", `${nAlias.toLocaleString()} aliases`)
|
|
371
452
|
|
|
372
453
|
// --- pass 3: region abbreviations (place_abbr) ---
|
|
@@ -388,125 +469,37 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
388
469
|
out.exec("COMMIT")
|
|
389
470
|
progress("abbrevs", `${nAbbr.toLocaleString()} abbrevs`)
|
|
390
471
|
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
out.exec("BEGIN")
|
|
411
|
-
|
|
412
|
-
for (const r of pc
|
|
413
|
-
.prepare(
|
|
414
|
-
`SELECT id, name, country, latitude, longitude,
|
|
415
|
-
min_latitude AS mnlat, min_longitude AS mnlon, max_latitude AS mxlat, max_longitude AS mxlon
|
|
416
|
-
FROM spr WHERE placetype='postalcode' AND latitude != 0 AND longitude != 0`
|
|
417
|
-
)
|
|
418
|
-
.iterate()) {
|
|
419
|
-
const name = String(r.name ?? "")
|
|
420
|
-
const key = normalizeLocalityForKey(name)
|
|
421
|
-
|
|
422
|
-
if (!key) continue
|
|
423
|
-
|
|
424
|
-
const lat = r.latitude as number
|
|
425
|
-
const lon = r.longitude as number
|
|
426
|
-
|
|
427
|
-
// region_id 0 (a postcode is unique by name+country — no same-name disambiguation); neg_rank 0
|
|
428
|
-
// (no population). bbox = the postcode's own min/max (falls back to the centroid point).
|
|
429
|
-
const a: PlaceAttrs = {
|
|
430
|
-
cid: ccID(r.country as string | null),
|
|
431
|
-
rid: 0,
|
|
432
|
-
ptid: pcPtid,
|
|
433
|
-
name,
|
|
434
|
-
lat,
|
|
435
|
-
lon,
|
|
436
|
-
mnLat: (r.mnlat as number) || lat,
|
|
437
|
-
mnLon: (r.mnlon as number) || lon,
|
|
438
|
-
mxLat: (r.mxlat as number) || lat,
|
|
439
|
-
mxLon: (r.mxlon as number) || lon,
|
|
440
|
-
pop: 0,
|
|
441
|
-
neg: 0,
|
|
442
|
-
pkey: key,
|
|
443
|
-
// A postcode has no toponym fame — nobody writes an encyclopedia article about SW1A 2AA — and
|
|
444
|
-
// the score source carries no `postalcode` rows to join against anyway. NULL is the truthful
|
|
445
|
-
// value: unmeasured, so the ranking key leaves postcode rows exactly where they were.
|
|
446
|
-
imp: null,
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
pcAttrs.set(Number(r.id), a)
|
|
450
|
-
stageRow(key, a, Number(r.id), 1)
|
|
451
|
-
|
|
452
|
-
primaries++
|
|
453
|
-
}
|
|
454
|
-
|
|
455
|
-
out.exec("COMMIT")
|
|
456
|
-
|
|
457
|
-
// --- pass 4b: postcode ALIAS names (#1495) ---
|
|
458
|
-
//
|
|
459
|
-
// The delivery-city names GeoNames supplies for a ZIP ("Brooklyn" for 11201) are written into
|
|
460
|
-
// the shard's `names` table by `postcode/centroid-fills.ts`'s `geonamesNameFill`. Everything
|
|
461
|
-
// downstream of `names` picked them up EXCEPT this build: `fts.ts` unions `spr.name` with every
|
|
462
|
-
// `names` row into `place_search.alt_names`, so the FTS backend resolved "Brooklyn" → 11201
|
|
463
|
-
// while the candidate backend — whose every row IS an exact-tier row — had no key for it at
|
|
464
|
-
// all. Pass 2 does the equivalent fold for admin places, but reads the ADMIN `place_search`,
|
|
465
|
-
// and `attrs` holds admin ids only, so a postcode shard could never reach it.
|
|
466
|
-
//
|
|
467
|
-
// Same discipline as pass 2: `is_primary = 0` (so `rankByPrimaryPreference` treats it as an
|
|
468
|
-
// alias, not a canonical postcode name), the row stays denormalized onto the POSTCODE's own
|
|
469
|
-
// spr_id/coords/bbox, and the display `name` stays the postcode — resolving "brooklyn" answers
|
|
470
|
-
// with place 11201, it does not rename the place to its delivery city.
|
|
471
|
-
const hasNames = pc.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='names'").get() !== undefined
|
|
472
|
-
|
|
473
|
-
if (hasNames) {
|
|
474
|
-
out.exec("BEGIN")
|
|
475
|
-
|
|
476
|
-
for (const r of pc.prepare("SELECT id, name FROM names").iterate()) {
|
|
477
|
-
const a = pcAttrs.get(Number(r.id))
|
|
478
|
-
|
|
479
|
-
if (!a) continue
|
|
480
|
-
|
|
481
|
-
const k = normalizeLocalityForKey(String(r.name ?? ""))
|
|
482
|
-
|
|
483
|
-
// The postcode's own key is already staged as the primary; `INSERT OR IGNORE` at
|
|
484
|
-
// materialization dedupes repeats, so this only skips the obvious self-alias.
|
|
485
|
-
if (!k || k === a.pkey) continue
|
|
486
|
-
|
|
487
|
-
stageRow(k, a, Number(r.id), 0)
|
|
488
|
-
|
|
489
|
-
aliases++
|
|
490
|
-
}
|
|
491
|
-
|
|
492
|
-
out.exec("COMMIT")
|
|
493
|
-
} else {
|
|
494
|
-
// Never a silent zero: real shards come from `createUnifiedSchema`, which always creates
|
|
495
|
-
// `names`. A shard without it has no alias surface to lose, but say so rather than reporting
|
|
496
|
-
// "0 aliases" from a table that was never read.
|
|
497
|
-
progress("postcode-aliases", `${pcDB} has no \`names\` table — no delivery-city aliases to fold`)
|
|
498
|
-
}
|
|
499
|
-
|
|
500
|
-
pc.close()
|
|
501
|
-
|
|
502
|
-
return { primaries, aliases }
|
|
503
|
-
}
|
|
504
|
-
|
|
472
|
+
// --- pass 3b: name roles (#1730 prototype — stampNameRoles owns the detectors) ---
|
|
473
|
+
// Independent of the sidecar below: this writes `cand_stage.name_role`, that writes the ancestor and
|
|
474
|
+
// interval tables, and neither reads the other's output. Ordered by label only.
|
|
475
|
+
const roles = stampNameRoles({
|
|
476
|
+
src,
|
|
477
|
+
out,
|
|
478
|
+
attrs,
|
|
479
|
+
keyCounts,
|
|
480
|
+
glossThreshold: opts.glossKeyThreshold ?? GLOSS_KEY_THRESHOLD,
|
|
481
|
+
ptcodes,
|
|
482
|
+
ccodes,
|
|
483
|
+
progress,
|
|
484
|
+
})
|
|
485
|
+
|
|
486
|
+
// --- pass 3c: the ancestors sidecar (candidate-ancestors-schema.ts owns the encoding decision) ---
|
|
487
|
+
const sidecar = await buildAncestorsSidecar({ src, out, kdb, attrs, ptID, progress })
|
|
488
|
+
|
|
489
|
+
// --- pass 4 + 4b: postcode and locality shards (foldShard owns the per-shard loop) ---
|
|
505
490
|
let nPostcode = 0
|
|
506
491
|
let nPostcodeAlias = 0
|
|
507
492
|
|
|
508
493
|
for (const pcDB of opts.postcodes ?? []) {
|
|
509
|
-
const folded =
|
|
494
|
+
const folded = foldShard({
|
|
495
|
+
out,
|
|
496
|
+
shardPath: pcDB,
|
|
497
|
+
shardPlacetype: "postalcode",
|
|
498
|
+
ccID,
|
|
499
|
+
ptID,
|
|
500
|
+
stageRow,
|
|
501
|
+
progress,
|
|
502
|
+
})
|
|
510
503
|
|
|
511
504
|
nPostcode += folded.primaries
|
|
512
505
|
nPostcodeAlias += folded.aliases
|
|
@@ -516,6 +509,26 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
516
509
|
progress("postcodes", `${nPostcode.toLocaleString()} postcodes; ${nPostcodeAlias.toLocaleString()} aliases`)
|
|
517
510
|
}
|
|
518
511
|
|
|
512
|
+
let nLocality = 0
|
|
513
|
+
|
|
514
|
+
for (const locDB of opts.localities ?? []) {
|
|
515
|
+
const folded = foldShard({
|
|
516
|
+
out,
|
|
517
|
+
shardPath: locDB,
|
|
518
|
+
shardPlacetype: "locality",
|
|
519
|
+
ccID,
|
|
520
|
+
ptID,
|
|
521
|
+
stageRow,
|
|
522
|
+
progress,
|
|
523
|
+
})
|
|
524
|
+
|
|
525
|
+
nLocality += folded.primaries
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
if (nLocality > 0) {
|
|
529
|
+
progress("localities", `${nLocality.toLocaleString()} shard localities folded`)
|
|
530
|
+
}
|
|
531
|
+
|
|
519
532
|
// --- code dictionaries: typed batch inserts via kdb (a few hundred rows — Kysely is clean here) ---
|
|
520
533
|
if (ccodes.size) {
|
|
521
534
|
await kdb
|
|
@@ -531,6 +544,26 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
531
544
|
.execute()
|
|
532
545
|
}
|
|
533
546
|
|
|
547
|
+
// --- capital-status reference (#1880's distribution home) — cold, small (~3.7k rows), typed ---
|
|
548
|
+
if (opts.capitals?.length) {
|
|
549
|
+
await createCapitalTable<CandidateDatabase>(kdb)
|
|
550
|
+
|
|
551
|
+
await kdb
|
|
552
|
+
.insertInto("capital")
|
|
553
|
+
.values(
|
|
554
|
+
opts.capitals.map((entry) => ({
|
|
555
|
+
country: entry.country,
|
|
556
|
+
latitude: entry.latitude,
|
|
557
|
+
longitude: entry.longitude,
|
|
558
|
+
level: entry.level,
|
|
559
|
+
keys: JSON.stringify(entry.k),
|
|
560
|
+
}))
|
|
561
|
+
)
|
|
562
|
+
.execute()
|
|
563
|
+
|
|
564
|
+
progress("capitals", `${opts.capitals.length.toLocaleString()} capital-reference rows carried in-artifact`)
|
|
565
|
+
}
|
|
566
|
+
|
|
534
567
|
// --- materialize the clustered WITHOUT ROWID table (sorted insert → contiguous leaves) ---
|
|
535
568
|
progress("cluster", "building clustered candidate table + VACUUM")
|
|
536
569
|
// Column list + clustered-key order are sourced from CANDIDATE_COLUMNS (the first 6 ARE the PRIMARY
|
|
@@ -570,6 +603,10 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
570
603
|
abbrevs: nAbbr,
|
|
571
604
|
postcodes: nPostcode,
|
|
572
605
|
postcodeAliases: nPostcodeAlias,
|
|
606
|
+
ancestorRows: sidecar.ancestorRows,
|
|
607
|
+
ancestorPlaces: sidecar.ancestorPlaces,
|
|
608
|
+
intervalPlaces: sidecar.intervalPlaces,
|
|
609
|
+
...roles,
|
|
573
610
|
...(importance ? { importanceScored: importance.matched, importanceGated: importance.gated } : {}),
|
|
574
611
|
}
|
|
575
612
|
}
|
package/build-slim.ts
CHANGED
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
*
|
|
27
27
|
* The output DB has the resolver-facing schema: `spr`, `names`, `place_population`, plus the
|
|
28
28
|
* `place_search` FTS5 / `place_bbox` R*Tree virtual tables rebuilt against the trimmed row set
|
|
29
|
-
* (both derive purely from `spr` + `names` — see `fts.ts`). That means `
|
|
29
|
+
* (both derive purely from `spr` + `names` — see `fts.ts`). That means `WOFSQLitePlaceLookup`
|
|
30
30
|
* opens the slim DB without any code change — it sees a smaller universe, nothing more.
|
|
31
31
|
*
|
|
32
32
|
* Multi-shard inputs (e.g. admin + postcode) are processed in sequence; selected rows accumulate
|
|
@@ -223,10 +223,10 @@ export async function buildSlimWOFDatabase(opts: BuildSlimOptions): Promise<Buil
|
|
|
223
223
|
|
|
224
224
|
// Materialize region/state ABBREVIATIONS into a standalone `place_abbr (id, abbr)` table BEFORE
|
|
225
225
|
// `names` is (optionally) dropped. The full DB lets the resolver tier an exact-abbrev match by
|
|
226
|
-
// querying `names` (`#
|
|
226
|
+
// querying `names` (`#exactMatchIDs`), but the slim DB drops `names` for size — so the
|
|
227
227
|
// browser resolver gets its own tiny lookup (~hundreds of rows) to do the same data-driven
|
|
228
228
|
// exact-abbrev tiering ("VT" → Vermont, not a token-matching foreign region) instead of the
|
|
229
|
-
// demo's hardcoded
|
|
229
|
+
// demo's hardcoded region-abbreviation map (since deleted). Sourced from the `language='abbr'` rows
|
|
230
230
|
// `add-region-abbrevs.ts` wrote, already filtered to surviving spr ids via the names copy. The
|
|
231
231
|
// table is always created (empty when the source predates the abbrev enrichment) so the
|
|
232
232
|
// resolver can query it unconditionally.
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
* @file Pass 2 of the candidate build — explode `place_search.alt_names` into distinct alias rows.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type { DatabaseSync } from "node:sqlite"
|
|
9
|
+
|
|
10
|
+
import { ALIAS_SEPARATOR } from "../fts.ts"
|
|
11
|
+
import { normalizeLocalityForKey } from "../street-normalize.ts"
|
|
12
|
+
import type { PlaceAttrs, StageRow } from "./place-attrs.ts"
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Pass 2 — explode each place's `place_search.alt_names` bag into distinct-key alias rows (`is_primary = 0`), and count
|
|
16
|
+
* each place's distinct staged keys (primary included) — the gloss detector's key-count signal (#1730).
|
|
17
|
+
*/
|
|
18
|
+
export function explodeAliasBags(
|
|
19
|
+
src: DatabaseSync,
|
|
20
|
+
out: DatabaseSync,
|
|
21
|
+
attrs: Map<number, PlaceAttrs>,
|
|
22
|
+
stageRow: StageRow
|
|
23
|
+
): { nAlias: number; keyCounts: Map<number, number> } {
|
|
24
|
+
let nAlias = 0
|
|
25
|
+
const keyCounts = new Map<number, number>()
|
|
26
|
+
out.exec("BEGIN")
|
|
27
|
+
|
|
28
|
+
for (const r of src.prepare("SELECT wof_id, alt_names FROM place_search").iterate()) {
|
|
29
|
+
const a = attrs.get(Number(r.wof_id))
|
|
30
|
+
const alt = r.alt_names as string | null
|
|
31
|
+
|
|
32
|
+
if (!a || !alt) continue
|
|
33
|
+
const seen = new Set<string>([a.pkey])
|
|
34
|
+
|
|
35
|
+
// The writer space-pads each separator and appends a trailing one, so every piece arrives with
|
|
36
|
+
// surrounding whitespace and the last one is empty. `normalizeLocalityForKey` folds both away,
|
|
37
|
+
// and the empty tail falls out at the `!k` guard below.
|
|
38
|
+
for (const piece of alt.split(ALIAS_SEPARATOR)) {
|
|
39
|
+
const k = normalizeLocalityForKey(piece)
|
|
40
|
+
|
|
41
|
+
if (!k || seen.has(k)) continue
|
|
42
|
+
seen.add(k)
|
|
43
|
+
stageRow(k, a, Number(r.wof_id), 0)
|
|
44
|
+
|
|
45
|
+
nAlias++
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
keyCounts.set(Number(r.wof_id), seen.size)
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
out.exec("COMMIT")
|
|
52
|
+
|
|
53
|
+
return { nAlias, keyCounts }
|
|
54
|
+
}
|