@mailwoman/resolver-wof-sqlite 9.0.0 → 9.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -9
- package/address-point-interpolation.ts +18 -8
- package/address-point-schema.ts +18 -6
- package/address-point.ts +111 -18
- package/ancestry.ts +9 -6
- package/build-candidate.ts +287 -157
- package/build-slim.ts +3 -3
- package/candidate/alias-bags.ts +54 -0
- package/candidate/ancestors-sidecar.ts +206 -0
- package/candidate/country-display-names.ts +79 -0
- package/candidate/name-roles.ts +237 -0
- package/candidate/own-name.ts +146 -0
- package/candidate/place-attrs.ts +44 -0
- package/candidate/shard-fold.ts +137 -0
- package/candidate-ancestors-schema.ts +195 -0
- package/candidate-fts.ts +4 -2
- package/candidate-importance.ts +228 -0
- package/candidate-lookup.ts +564 -174
- package/candidate-schema.ts +60 -3
- package/candidate-scoring.ts +268 -0
- package/capital-schema.ts +90 -0
- package/capitals.ts +148 -0
- package/coincident-roles.ts +69 -10
- package/convention-schema.ts +72 -0
- package/convention.ts +2 -2
- package/coverage-manifest-schema.ts +7 -7
- package/currency-backfill.ts +249 -0
- package/exact-match.ts +104 -0
- package/fst-autocomplete.ts +105 -122
- package/fst-builder.ts +39 -47
- package/fst-deserialize-web.ts +43 -7
- package/fst-freshness.ts +2 -2
- package/fst-serialize.ts +68 -12
- package/fst-types.ts +35 -1
- package/fts-query.ts +1 -1
- package/fts.ts +16 -4
- package/geonames-postal.ts +2 -2
- package/index.ts +26 -14
- package/interpolation.ts +113 -19
- package/lookup.ts +118 -560
- package/name-score.ts +6 -4
- package/out/address-point-interpolation.d.ts.map +1 -1
- package/out/address-point-interpolation.js +13 -7
- package/out/address-point-interpolation.js.map +1 -1
- package/out/address-point-schema.d.ts +16 -6
- package/out/address-point-schema.d.ts.map +1 -1
- package/out/address-point-schema.js.map +1 -1
- package/out/address-point.d.ts.map +1 -1
- package/out/address-point.js +70 -14
- package/out/address-point.js.map +1 -1
- package/out/ancestry.d.ts +2 -2
- package/out/ancestry.d.ts.map +1 -1
- package/out/ancestry.js +5 -6
- package/out/ancestry.js.map +1 -1
- package/out/build-candidate.d.ts +108 -0
- package/out/build-candidate.d.ts.map +1 -1
- package/out/build-candidate.js +151 -120
- package/out/build-candidate.js.map +1 -1
- package/out/build-slim.d.ts +1 -1
- package/out/build-slim.js +3 -3
- package/out/build-slim.js.map +1 -1
- package/out/candidate/alias-bags.d.ts +17 -0
- package/out/candidate/alias-bags.d.ts.map +1 -0
- package/out/candidate/alias-bags.js +39 -0
- package/out/candidate/alias-bags.js.map +1 -0
- package/out/candidate/ancestors-sidecar.d.ts +33 -0
- package/out/candidate/ancestors-sidecar.d.ts.map +1 -0
- package/out/candidate/ancestors-sidecar.js +140 -0
- package/out/candidate/ancestors-sidecar.js.map +1 -0
- package/out/candidate/country-display-names.d.ts +35 -0
- package/out/candidate/country-display-names.d.ts.map +1 -0
- package/out/candidate/country-display-names.js +59 -0
- package/out/candidate/country-display-names.js.map +1 -0
- package/out/candidate/name-roles.d.ts +55 -0
- package/out/candidate/name-roles.d.ts.map +1 -0
- package/out/candidate/name-roles.js +165 -0
- package/out/candidate/name-roles.js.map +1 -0
- package/out/candidate/own-name.d.ts +50 -0
- package/out/candidate/own-name.d.ts.map +1 -0
- package/out/candidate/own-name.js +132 -0
- package/out/candidate/own-name.js.map +1 -0
- package/out/candidate/place-attrs.d.ts +43 -0
- package/out/candidate/place-attrs.d.ts.map +1 -0
- package/out/candidate/place-attrs.js +15 -0
- package/out/candidate/place-attrs.js.map +1 -0
- package/out/candidate/shard-fold.d.ts +31 -0
- package/out/candidate/shard-fold.d.ts.map +1 -0
- package/out/candidate/shard-fold.js +104 -0
- package/out/candidate/shard-fold.js.map +1 -0
- package/out/candidate-ancestors-schema.d.ts +150 -0
- package/out/candidate-ancestors-schema.d.ts.map +1 -0
- package/out/candidate-ancestors-schema.js +123 -0
- package/out/candidate-ancestors-schema.js.map +1 -0
- package/out/candidate-fts.d.ts +4 -2
- package/out/candidate-fts.d.ts.map +1 -1
- package/out/candidate-fts.js +4 -2
- package/out/candidate-fts.js.map +1 -1
- package/out/candidate-importance.d.ts +132 -0
- package/out/candidate-importance.d.ts.map +1 -0
- package/out/candidate-importance.js +174 -0
- package/out/candidate-importance.js.map +1 -0
- package/out/candidate-lookup.d.ts +22 -37
- package/out/candidate-lookup.d.ts.map +1 -1
- package/out/candidate-lookup.js +446 -132
- package/out/candidate-lookup.js.map +1 -1
- package/out/candidate-schema.d.ts +52 -4
- package/out/candidate-schema.d.ts.map +1 -1
- package/out/candidate-schema.js +8 -0
- package/out/candidate-schema.js.map +1 -1
- package/out/candidate-scoring.d.ts +34 -0
- package/out/candidate-scoring.d.ts.map +1 -0
- package/out/candidate-scoring.js +200 -0
- package/out/candidate-scoring.js.map +1 -0
- package/out/capital-schema.d.ts +51 -0
- package/out/capital-schema.d.ts.map +1 -0
- package/out/capital-schema.js +63 -0
- package/out/capital-schema.js.map +1 -0
- package/out/capitals.d.ts +69 -0
- package/out/capitals.d.ts.map +1 -0
- package/out/capitals.js +98 -0
- package/out/capitals.js.map +1 -0
- package/out/coincident-roles.d.ts +7 -0
- package/out/coincident-roles.d.ts.map +1 -1
- package/out/coincident-roles.js +42 -8
- package/out/coincident-roles.js.map +1 -1
- package/out/convention-schema.d.ts +51 -0
- package/out/convention-schema.d.ts.map +1 -0
- package/out/convention-schema.js +34 -0
- package/out/convention-schema.js.map +1 -0
- package/out/convention.d.ts +1 -1
- package/out/convention.js +2 -2
- package/out/coverage-manifest-schema.js +3 -7
- package/out/coverage-manifest-schema.js.map +1 -1
- package/out/currency-backfill.d.ts +46 -0
- package/out/currency-backfill.d.ts.map +1 -0
- package/out/currency-backfill.js +180 -0
- package/out/currency-backfill.js.map +1 -0
- package/out/exact-match.d.ts +25 -0
- package/out/exact-match.d.ts.map +1 -0
- package/out/exact-match.js +89 -0
- package/out/exact-match.js.map +1 -0
- package/out/fst-autocomplete.d.ts +24 -14
- package/out/fst-autocomplete.d.ts.map +1 -1
- package/out/fst-autocomplete.js +84 -100
- package/out/fst-autocomplete.js.map +1 -1
- package/out/fst-builder.d.ts.map +1 -1
- package/out/fst-builder.js +32 -40
- package/out/fst-builder.js.map +1 -1
- package/out/fst-deserialize-web.d.ts.map +1 -1
- package/out/fst-deserialize-web.js +36 -7
- package/out/fst-deserialize-web.js.map +1 -1
- package/out/fst-freshness.d.ts +2 -2
- package/out/fst-freshness.js +2 -2
- package/out/fst-serialize.d.ts +14 -4
- package/out/fst-serialize.d.ts.map +1 -1
- package/out/fst-serialize.js +60 -12
- package/out/fst-serialize.js.map +1 -1
- package/out/fst-types.d.ts +35 -1
- package/out/fst-types.d.ts.map +1 -1
- package/out/fts-query.js +1 -1
- package/out/fts-query.js.map +1 -1
- package/out/fts.d.ts +15 -4
- package/out/fts.d.ts.map +1 -1
- package/out/fts.js +15 -4
- package/out/fts.js.map +1 -1
- package/out/geonames-postal.d.ts +2 -2
- package/out/geonames-postal.js +2 -2
- package/out/index.d.ts +4 -2
- package/out/index.d.ts.map +1 -1
- package/out/index.js +3 -2
- package/out/index.js.map +1 -1
- package/out/interpolation.d.ts +8 -0
- package/out/interpolation.d.ts.map +1 -1
- package/out/interpolation.js +91 -19
- package/out/interpolation.js.map +1 -1
- package/out/lookup.d.ts +4 -5
- package/out/lookup.d.ts.map +1 -1
- package/out/lookup.js +102 -444
- package/out/lookup.js.map +1 -1
- package/out/name-score.d.ts +0 -10
- package/out/name-score.d.ts.map +1 -1
- package/out/name-score.js +6 -4
- package/out/name-score.js.map +1 -1
- package/out/place-importance-schema.d.ts +226 -0
- package/out/place-importance-schema.d.ts.map +1 -0
- package/out/place-importance-schema.js +288 -0
- package/out/place-importance-schema.js.map +1 -0
- package/out/poi-lookup.d.ts +1 -1
- package/out/poi-lookup.d.ts.map +1 -1
- package/out/poi-lookup.js +12 -13
- package/out/poi-lookup.js.map +1 -1
- package/out/poi-schema.d.ts +7 -3
- package/out/poi-schema.d.ts.map +1 -1
- package/out/poi-schema.js.map +1 -1
- package/out/polygon-schema.d.ts +37 -0
- package/out/polygon-schema.d.ts.map +1 -0
- package/out/polygon-schema.js +23 -0
- package/out/polygon-schema.js.map +1 -0
- package/out/postal-city-alias-lookup.d.ts +1 -1
- package/out/postal-city-alias-lookup.js +1 -1
- package/out/postal-city-candidate-schema.d.ts +2 -1
- package/out/postal-city-candidate-schema.d.ts.map +1 -1
- package/out/postal-city-candidate-schema.js.map +1 -1
- package/out/postcode-point-lookup.d.ts +1 -1
- package/out/postcode-point-lookup.js +1 -1
- package/out/primary-preference.d.ts +125 -0
- package/out/primary-preference.d.ts.map +1 -0
- package/out/primary-preference.js +138 -0
- package/out/primary-preference.js.map +1 -0
- package/out/proximity-rerank.d.ts +77 -0
- package/out/proximity-rerank.d.ts.map +1 -0
- package/out/proximity-rerank.js +86 -0
- package/out/proximity-rerank.js.map +1 -0
- package/out/region-keys.d.ts +47 -0
- package/out/region-keys.d.ts.map +1 -0
- package/out/region-keys.js +121 -0
- package/out/region-keys.js.map +1 -0
- package/out/reverse.d.ts.map +1 -1
- package/out/reverse.js +6 -9
- package/out/reverse.js.map +1 -1
- package/out/schema.d.ts +1 -1
- package/out/search-fetch.d.ts +57 -0
- package/out/search-fetch.d.ts.map +1 -0
- package/out/search-fetch.js +183 -0
- package/out/search-fetch.js.map +1 -0
- package/out/sharding.d.ts +3 -3
- package/out/sharding.js +1 -1
- package/out/sqlite-convention-source.d.ts +1 -1
- package/out/sqlite-convention-source.js +1 -1
- package/out/sqlite-utils.d.ts +31 -1
- package/out/sqlite-utils.d.ts.map +1 -1
- package/out/sqlite-utils.js +38 -0
- package/out/sqlite-utils.js.map +1 -1
- package/out/street-centroid-schema.d.ts +7 -2
- package/out/street-centroid-schema.d.ts.map +1 -1
- package/out/street-centroid-schema.js.map +1 -1
- package/out/street-centroid.d.ts.map +1 -1
- package/out/street-centroid.js +7 -7
- package/out/street-centroid.js.map +1 -1
- package/out/street-morphology-fst-builder.d.ts.map +1 -1
- package/out/street-morphology-fst-builder.js +5 -4
- package/out/street-morphology-fst-builder.js.map +1 -1
- package/out/street-normalize.d.ts +83 -9
- package/out/street-normalize.d.ts.map +1 -1
- package/out/street-normalize.js +177 -10
- package/out/street-normalize.js.map +1 -1
- package/out/street-segment-schema.d.ts +6 -2
- package/out/street-segment-schema.d.ts.map +1 -1
- package/out/street-segment-schema.js.map +1 -1
- package/out/types.d.ts +74 -1
- package/out/types.d.ts.map +1 -1
- package/out/unified-schema.d.ts +1 -1
- package/out/unified-schema.js +1 -1
- package/out/uprn-lookup.d.ts +85 -0
- package/out/uprn-lookup.d.ts.map +1 -0
- package/out/uprn-lookup.js +152 -0
- package/out/uprn-lookup.js.map +1 -0
- package/out/uprn-schema.d.ts +93 -0
- package/out/uprn-schema.d.ts.map +1 -0
- package/out/uprn-schema.js +78 -0
- package/out/uprn-schema.js.map +1 -0
- package/out/weights-overlay-linker.d.ts +141 -0
- package/out/weights-overlay-linker.d.ts.map +1 -0
- package/out/weights-overlay-linker.js +259 -0
- package/out/weights-overlay-linker.js.map +1 -0
- package/package.json +296 -16
- package/place-importance-schema.ts +402 -0
- package/poi-lookup.ts +12 -13
- package/poi-schema.ts +8 -3
- package/polygon-schema.ts +47 -0
- package/postal-city-alias-lookup.ts +1 -1
- package/postal-city-candidate-schema.ts +3 -1
- package/postcode-point-lookup.ts +1 -1
- package/primary-preference.ts +207 -0
- package/proximity-rerank.ts +120 -0
- package/region-keys.ts +144 -0
- package/reverse.ts +17 -16
- package/schema.ts +1 -1
- package/search-fetch.ts +256 -0
- package/sharding.ts +3 -3
- package/sqlite-convention-source.ts +1 -1
- package/sqlite-utils.ts +63 -1
- package/street-centroid-schema.ts +8 -2
- package/street-centroid.ts +13 -8
- package/street-morphology-fst-builder.ts +5 -4
- package/street-normalize.ts +254 -24
- package/street-segment-schema.ts +7 -2
- package/types.ts +74 -1
- package/unified-schema.ts +1 -1
- package/uprn-lookup.ts +210 -0
- package/uprn-schema.ts +124 -0
- package/weights-overlay-linker.ts +377 -0
- package/geo.ts +0 -121
- package/out/geo.d.ts +0 -74
- package/out/geo.d.ts.map +0 -1
- package/out/geo.js +0 -71
- package/out/geo.js.map +0 -1
package/build-candidate.ts
CHANGED
|
@@ -26,26 +26,49 @@
|
|
|
26
26
|
*
|
|
27
27
|
* Measured (2026-06-20, vs the 2.6 GB full-DB FTS): ~5 M rows; ~12 range fetches per 8-query
|
|
28
28
|
* session (the full DB needs 243); US locality 96.8% (region bbox), EU coord parity 88.6%.
|
|
29
|
+
*
|
|
30
|
+
* #28 adds one more denormalized field, `importance` — the toponym-fame prior that decides a BARE
|
|
31
|
+
* city name, joined in from a separate score source by name rather than by id (see
|
|
32
|
+
* `candidate-importance.ts`, which owns that join and explains why the id would be wrong). It is
|
|
33
|
+
* optional: without {@link BuildCandidateOptions.importance} the column is NULL on every row, which
|
|
34
|
+
* the consumer reads as unmeasured and ignores.
|
|
35
|
+
*
|
|
36
|
+
* The build also materializes the ANCESTORS SIDECAR (`candidate_ancestor` closure rows +
|
|
37
|
+
* `candidate_interval` pre/post labels) from the source `ancestors` table — the containment
|
|
38
|
+
* lineage behind {@link WOFCandidateTableLookup.ancestors} and the admin-coherence check.
|
|
39
|
+
* `candidate-ancestors-schema.ts` owns the encoding decision and the DAG/absence semantics.
|
|
29
40
|
*/
|
|
30
41
|
|
|
31
42
|
import { existsSync, rmSync } from "node:fs"
|
|
32
43
|
import { DatabaseSync } from "node:sqlite"
|
|
33
44
|
|
|
45
|
+
import { COUNTRY_POPULATION } from "@mailwoman/codex/country"
|
|
34
46
|
import { DatabaseClient } from "@mailwoman/core/kysley/client"
|
|
35
47
|
|
|
36
48
|
import { createCandidateFTS } from "./candidate-fts.ts"
|
|
49
|
+
import { IMPORTANCE_JOIN_GATE_KM, loadImportanceIndex } from "./candidate-importance.ts"
|
|
37
50
|
import {
|
|
38
51
|
CANDIDATE_COLUMNS,
|
|
39
52
|
createCandidateStagingTables,
|
|
40
53
|
createCandidateTable,
|
|
41
54
|
type CandidateDatabase,
|
|
42
55
|
} from "./candidate-schema.ts"
|
|
56
|
+
import { explodeAliasBags } from "./candidate/alias-bags.ts"
|
|
57
|
+
import { buildAncestorsSidecar } from "./candidate/ancestors-sidecar.ts"
|
|
58
|
+
import { stageCountryDisplayNames } from "./candidate/country-display-names.ts"
|
|
59
|
+
import { GLOSS_KEY_THRESHOLD, stampNameRoles } from "./candidate/name-roles.ts"
|
|
60
|
+
import type { PlaceAttrs } from "./candidate/place-attrs.ts"
|
|
61
|
+
import { foldShard } from "./candidate/shard-fold.ts"
|
|
62
|
+
import { createCapitalTable } from "./capital-schema.ts"
|
|
63
|
+
import type { CapitalPoint } from "./capitals.ts"
|
|
64
|
+
import { resurrectCurrencyHoles } from "./currency-backfill.ts"
|
|
43
65
|
import { normalizeLocalityForKey } from "./street-normalize.ts"
|
|
44
66
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
67
|
+
// The build's contract is this module path; the passes behind it live in `./candidate/`. Re-exported
|
|
68
|
+
// here so a consumer never has to know which pass owns which name.
|
|
69
|
+
export { stageCountryDisplayNames } from "./candidate/country-display-names.ts"
|
|
70
|
+
export { GLOSS_EXCLUDED_PLACETYPES, GLOSS_KEY_THRESHOLD } from "./candidate/name-roles.ts"
|
|
71
|
+
export type { PlaceAttrs } from "./candidate/place-attrs.ts"
|
|
49
72
|
|
|
50
73
|
export interface BuildCandidateOptions {
|
|
51
74
|
/**
|
|
@@ -56,6 +79,12 @@ export interface BuildCandidateOptions {
|
|
|
56
79
|
* Output candidate DB path (overwritten if present).
|
|
57
80
|
*/
|
|
58
81
|
output: string
|
|
82
|
+
/**
|
|
83
|
+
* The capital-status reference entries (#1880) to carry in-artifact — the parsed `data/gazetteer/capitals-v1.json`
|
|
84
|
+
* entries, passed by the CALLER because this module publishes to npm and must not read repo-root paths. Absent → the
|
|
85
|
+
* `capital` table is not created, and the session loader falls back to the repo file where one exists.
|
|
86
|
+
*/
|
|
87
|
+
capitals?: readonly CapitalPoint[]
|
|
59
88
|
/**
|
|
60
89
|
* Optional postcode shards (`spr` rows with `placetype='postalcode'` + real coords, e.g. postalcode-us.db) — folded
|
|
61
90
|
* in as `postalcode` candidate rows so `findPlace(postalcode)` resolves a ZIP directly (the demo's primary postcode
|
|
@@ -65,10 +94,47 @@ export interface BuildCandidateOptions {
|
|
|
65
94
|
* ("Brooklyn" for 11201), and they were previously reachable only through FTS.
|
|
66
95
|
*/
|
|
67
96
|
postcodes?: string[]
|
|
97
|
+
/**
|
|
98
|
+
* Optional LOCALITY shards (`spr` rows with `placetype='locality'` + real coords, e.g. localities-nz-linz.db — the
|
|
99
|
+
* #1564 NZ suburb tier) — folded through the same shard loop as the postcode shards, staged as `locality` candidate
|
|
100
|
+
* rows with no region scope and UNMEASURED population (`neg_rank 0`: a shard row ranks behind any populated namesake
|
|
101
|
+
* and wins only where its key is the answer). Each shard's `names` table folds as aliases, `is_primary = 0`, same as
|
|
102
|
+
* the delivery-city pass.
|
|
103
|
+
*/
|
|
104
|
+
localities?: string[]
|
|
105
|
+
/**
|
|
106
|
+
* Optional WOF admin database carrying a `place_importance` table — the source of the `importance` column (#28), the
|
|
107
|
+
* toponym-fame prior that decides the bare-city-name class. Joined by `(name_key, country, placetype)` + nearest
|
|
108
|
+
* centroid, NOT by id; see `candidate-importance.ts` for why the id join silently drops the foreign homonyms the
|
|
109
|
+
* prior exists to demote.
|
|
110
|
+
*
|
|
111
|
+
* Omit it and every row's `importance` is NULL — unmeasured, which is what the consumer's positive-evidence-only rule
|
|
112
|
+
* already treats as "do not participate", so the artifact is byte-identical to a pre-#28 build except for the empty
|
|
113
|
+
* column. That is the honest degradation and it is the DEFAULT: a caller with no score source must not get a
|
|
114
|
+
* population-derived stand-in written into a column that means fame.
|
|
115
|
+
*/
|
|
116
|
+
importance?: string
|
|
117
|
+
/**
|
|
118
|
+
* Cross-source currency backfill (#1737). WOF carries deprecated-with-no-successor records for real, populated
|
|
119
|
+
* settlements (Rochester Kent, Aldershot, Telford — 120 GB localities alone), and the currency filter correctly drops
|
|
120
|
+
* them, leaving holes no ranking can fill. When this option is set, pass 1c resurrects a dead locality ONLY under
|
|
121
|
+
* three gates, positive evidence throughout: no live same-name row of any placetype near the dead record (a distant
|
|
122
|
+
* same-name row is a NAMESAKE and does not block — Rochester, Northumberland pop 318 must not veto Rochester, Kent);
|
|
123
|
+
* an independent GeoNames P-class attestation of the same folded name within {@link CURRENCY_BACKFILL_RADIUS_KM}; and
|
|
124
|
+
* the attestor at or above {@link CURRENCY_BACKFILL_POP_FLOOR}. The staged row keeps the real WOF id, name, centroid
|
|
125
|
+
* and bbox — GeoNames only ATTESTS the place and supplies the population that lets it stand in prominence races.
|
|
126
|
+
* `countries` are judged only where `<cc>.txt` exists under `geonamesDir`; absent dumps are skipped loudly.
|
|
127
|
+
*/
|
|
128
|
+
currencyBackfill?: { geonamesDir: string; countries: readonly string[] }
|
|
68
129
|
/**
|
|
69
130
|
* Optional progress callback for CLI / test introspection.
|
|
70
131
|
*/
|
|
71
132
|
onProgress?: (phase: string, message: string) => void
|
|
133
|
+
/**
|
|
134
|
+
* Key-count threshold for the gloss anomaly detector — {@link GLOSS_KEY_THRESHOLD} unless a test passes a
|
|
135
|
+
* fixture-scale value. The production number is the #1730 sweep's own cut (4,000 places at >= 50 keys).
|
|
136
|
+
*/
|
|
137
|
+
glossKeyThreshold?: number
|
|
72
138
|
}
|
|
73
139
|
|
|
74
140
|
export interface BuildCandidateResult {
|
|
@@ -84,22 +150,53 @@ export interface BuildCandidateResult {
|
|
|
84
150
|
* through `onProgress`.
|
|
85
151
|
*/
|
|
86
152
|
postcodeAliases: number
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
153
|
+
/**
|
|
154
|
+
* Closure rows in the `candidate_ancestor` sidecar (see candidate-ancestors-schema.ts). Zero means the source
|
|
155
|
+
* `ancestors` table contributed nothing within the resolvable placetypes — a finding, reported rather than implied.
|
|
156
|
+
*/
|
|
157
|
+
ancestorRows: number
|
|
158
|
+
/**
|
|
159
|
+
* Places carrying at least one closure row.
|
|
160
|
+
*/
|
|
161
|
+
ancestorPlaces: number
|
|
162
|
+
/**
|
|
163
|
+
* Places that received a pre/post interval label — the canonical-parent forest's node count. Places outside it (no
|
|
164
|
+
* recorded ancestry, shard rows, cycle-skipped) have NO label: containment against them is unverifiable, never
|
|
165
|
+
* false.
|
|
166
|
+
*/
|
|
167
|
+
intervalPlaces: number
|
|
168
|
+
/**
|
|
169
|
+
* Places that took an `importance` score from the join (#28).
|
|
170
|
+
*
|
|
171
|
+
* `undefined` and `0` mean different things. `undefined` is "the pass did not run" — no score source was given. A `0`
|
|
172
|
+
* would be the source matching NOTHING, which is a finding. Never collapse the two.
|
|
173
|
+
*/
|
|
174
|
+
importanceScored?: number
|
|
175
|
+
/**
|
|
176
|
+
* Places whose `(name_key, country, placetype)` matched a scored group but whose nearest scored centroid was outside
|
|
177
|
+
* {@link IMPORTANCE_JOIN_GATE_KM} — a different town wearing the same name, refused rather than scored.
|
|
178
|
+
*
|
|
179
|
+
* Worth watching across rebuilds: a jump here means the score source and the admin source have drifted apart, and the
|
|
180
|
+
* join is being asked to guess.
|
|
181
|
+
*/
|
|
182
|
+
importanceGated?: number
|
|
183
|
+
/**
|
|
184
|
+
* Alias rows stamped `name_role = 'gloss'` — the anomaly detector's certain core (#1730).
|
|
185
|
+
*/
|
|
186
|
+
roleGloss: number
|
|
187
|
+
/**
|
|
188
|
+
* Alias rows stamped `name_role = 'abbr'` — the variant-in-official-language provenance signal (#1730/#936).
|
|
189
|
+
*/
|
|
190
|
+
roleAbbr: number
|
|
191
|
+
/**
|
|
192
|
+
* Admin places whose staged key count reached {@link GLOSS_KEY_THRESHOLD} — the sweep's tail, reported so the stamped
|
|
193
|
+
* fraction has its denominator.
|
|
194
|
+
*/
|
|
195
|
+
keyTailPlaces: number
|
|
196
|
+
/**
|
|
197
|
+
* Of {@link BuildCandidateResult.keyTailPlaces}, how many carry at least one stamped role row.
|
|
198
|
+
*/
|
|
199
|
+
keyTailWithRole: number
|
|
103
200
|
}
|
|
104
201
|
|
|
105
202
|
export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<BuildCandidateResult> {
|
|
@@ -146,6 +243,25 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
146
243
|
return id
|
|
147
244
|
}
|
|
148
245
|
|
|
246
|
+
// --- importance source (#28): loaded BEFORE pass 1, which is the only pass that sees a place's
|
|
247
|
+
// name/country/placetype/centroid together. Absent → every row's `importance` stays NULL. ---
|
|
248
|
+
let importance: ReturnType<typeof loadImportanceIndex> | undefined
|
|
249
|
+
|
|
250
|
+
if (opts.importance) {
|
|
251
|
+
progress("importance", `loading place_importance from ${opts.importance}`)
|
|
252
|
+
importance = loadImportanceIndex(opts.importance)
|
|
253
|
+
|
|
254
|
+
progress(
|
|
255
|
+
"importance",
|
|
256
|
+
`${importance.stats.places.toLocaleString()} scored places in ${importance.stats.keys.toLocaleString()} (name, country, placetype) groups` +
|
|
257
|
+
(importance.stats.unkeyable ? `; ${importance.stats.unkeyable.toLocaleString()} unkeyable names skipped` : "")
|
|
258
|
+
)
|
|
259
|
+
} else {
|
|
260
|
+
// Never a silent column of nulls: a build without a score source produces one, and the reason has
|
|
261
|
+
// to be visible in the log rather than inferred from the artifact.
|
|
262
|
+
progress("importance", "no score source given — `importance` will be NULL on every row")
|
|
263
|
+
}
|
|
264
|
+
|
|
149
265
|
// --- region_id per place (its region-tier ancestor) for same-name disambiguation ---
|
|
150
266
|
progress("region", "loading region ancestry")
|
|
151
267
|
const regionOf = new Map<number, number>()
|
|
@@ -202,18 +318,25 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
202
318
|
const cid = ccID(r.country as string | null)
|
|
203
319
|
const ptid = ptID(r.placetype as string | null)
|
|
204
320
|
const rid = regionOf.get(sid) ?? 0
|
|
205
|
-
|
|
321
|
+
// A zero population on a COUNTRY row is a WOF absence artifact, never a real zero — 147 of 237
|
|
322
|
+
// primary country records carried none (measured 2026-08-18, #1650), which ranked those nations
|
|
323
|
+
// below any namesake hamlet in every prominence race ("Georgia" → Georgia VT). The codex table is
|
|
324
|
+
// the secondary source; a country absent from it too stays at zero honestly.
|
|
325
|
+
const wofPop = Number(r.pop) || 0
|
|
326
|
+
const pop = wofPop === 0 && r.placetype === "country" ? (COUNTRY_POPULATION[String(r.country ?? "")] ?? 0) : wofPop
|
|
206
327
|
const neg = -Math.log10(pop + 1)
|
|
207
328
|
const name = String(r.name ?? "")
|
|
208
329
|
const pkey = normalizeLocalityForKey(name)
|
|
330
|
+
const lat = r.lat as number
|
|
331
|
+
const lon = r.lon as number
|
|
209
332
|
|
|
210
333
|
const a: PlaceAttrs = {
|
|
211
334
|
cid,
|
|
212
335
|
rid,
|
|
213
336
|
ptid,
|
|
214
337
|
name,
|
|
215
|
-
lat
|
|
216
|
-
lon
|
|
338
|
+
lat,
|
|
339
|
+
lon,
|
|
217
340
|
mnLat: r.mnlat as number,
|
|
218
341
|
mnLon: r.mnlon as number,
|
|
219
342
|
mxLat: r.mxlat as number,
|
|
@@ -221,12 +344,31 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
221
344
|
pop,
|
|
222
345
|
neg,
|
|
223
346
|
pkey,
|
|
347
|
+
imp: importance?.find(name, r.country as string | null, r.placetype as string | null, lat, lon) ?? null,
|
|
224
348
|
}
|
|
225
349
|
|
|
226
350
|
attrs.set(sid, a)
|
|
227
351
|
|
|
228
352
|
if (pkey) {
|
|
229
|
-
insStage.run(
|
|
353
|
+
insStage.run(
|
|
354
|
+
pkey,
|
|
355
|
+
cid,
|
|
356
|
+
rid,
|
|
357
|
+
ptid,
|
|
358
|
+
neg,
|
|
359
|
+
sid,
|
|
360
|
+
name,
|
|
361
|
+
a.lat,
|
|
362
|
+
a.lon,
|
|
363
|
+
a.mnLat,
|
|
364
|
+
a.mnLon,
|
|
365
|
+
a.mxLat,
|
|
366
|
+
a.mxLon,
|
|
367
|
+
pop,
|
|
368
|
+
1,
|
|
369
|
+
a.imp,
|
|
370
|
+
null
|
|
371
|
+
)
|
|
230
372
|
|
|
231
373
|
nPrim++
|
|
232
374
|
}
|
|
@@ -235,6 +377,14 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
235
377
|
out.exec("COMMIT")
|
|
236
378
|
progress("primaries", `${nPrim.toLocaleString()} primaries; ${attrs.size.toLocaleString()} places`)
|
|
237
379
|
|
|
380
|
+
if (importance) {
|
|
381
|
+
progress(
|
|
382
|
+
"importance",
|
|
383
|
+
`${importance.matched.toLocaleString()} places scored; ` +
|
|
384
|
+
`${importance.gated.toLocaleString()} refused (nearest same-name place > ${IMPORTANCE_JOIN_GATE_KM} km away)`
|
|
385
|
+
)
|
|
386
|
+
}
|
|
387
|
+
|
|
238
388
|
const stageRow = (k: string, a: PlaceAttrs, sid: number, isPrimary: number): void => {
|
|
239
389
|
insStage.run(
|
|
240
390
|
k,
|
|
@@ -251,34 +401,53 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
251
401
|
a.mxLat,
|
|
252
402
|
a.mxLon,
|
|
253
403
|
a.pop,
|
|
254
|
-
isPrimary
|
|
404
|
+
isPrimary,
|
|
405
|
+
a.imp,
|
|
406
|
+
null
|
|
255
407
|
)
|
|
256
408
|
}
|
|
257
409
|
|
|
258
|
-
// --- pass
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
const k = normalizeLocalityForKey(piece)
|
|
272
|
-
|
|
273
|
-
if (!k || seen.has(k)) continue
|
|
274
|
-
seen.add(k)
|
|
275
|
-
stageRow(k, a, Number(r.wof_id), 0)
|
|
410
|
+
// --- pass 1b: country surfaces across scripts (#1678 thread 1) — see stageCountryDisplayNames ---
|
|
411
|
+
// Never a silent zero: a runtime whose ICU lacks these locales degrades to fewer surfaces, and the count is how a
|
|
412
|
+
// reader tells that apart from the pass not having run.
|
|
413
|
+
progress(
|
|
414
|
+
"country-display-names",
|
|
415
|
+
`${stageCountryDisplayNames({
|
|
416
|
+
attrs,
|
|
417
|
+
iso2ByID: new Map([...ccodes].map(([code, id]) => [id, code])),
|
|
418
|
+
countryPtID: ptID("country"),
|
|
419
|
+
stageRow,
|
|
420
|
+
tx: out,
|
|
421
|
+
}).toLocaleString()} country surfaces`
|
|
422
|
+
)
|
|
276
423
|
|
|
277
|
-
|
|
278
|
-
|
|
424
|
+
// --- pass 1c: cross-source currency backfill (#1737 — resurrectCurrencyHoles owns the gates). Runs BEFORE
|
|
425
|
+
// the alias pass so a resurrected place's alt names explode like any primary's. ---
|
|
426
|
+
if (opts.currencyBackfill) {
|
|
427
|
+
const nBackfill = await resurrectCurrencyHoles({
|
|
428
|
+
src,
|
|
429
|
+
tx: out,
|
|
430
|
+
geonamesDir: opts.currencyBackfill.geonamesDir,
|
|
431
|
+
countries: opts.currencyBackfill.countries,
|
|
432
|
+
attrs,
|
|
433
|
+
ccID,
|
|
434
|
+
ptID,
|
|
435
|
+
regionOf,
|
|
436
|
+
importance,
|
|
437
|
+
stageRow,
|
|
438
|
+
progress,
|
|
439
|
+
})
|
|
440
|
+
|
|
441
|
+
progress("currency-backfill", `${nBackfill.toLocaleString()} resurrections staged`)
|
|
442
|
+
} else {
|
|
443
|
+
// Never a silent absence: a build without the option leaves the deprecated-no-successor holes dead,
|
|
444
|
+
// and the log must say so rather than leave it inferable only from a missing row.
|
|
445
|
+
progress("currency-backfill", "not configured — deprecated-no-successor holes stay dead (#1737)")
|
|
279
446
|
}
|
|
280
447
|
|
|
281
|
-
|
|
448
|
+
// --- pass 2: distinct normalized aliases from place_search.alt_names (explodeAliasBags owns the loop) ---
|
|
449
|
+
progress("aliases", "exploding alias bags")
|
|
450
|
+
const { nAlias, keyCounts } = explodeAliasBags(src, out, attrs, stageRow)
|
|
282
451
|
progress("aliases", `${nAlias.toLocaleString()} aliases`)
|
|
283
452
|
|
|
284
453
|
// --- pass 3: region abbreviations (place_abbr) ---
|
|
@@ -300,121 +469,37 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
300
469
|
out.exec("COMMIT")
|
|
301
470
|
progress("abbrevs", `${nAbbr.toLocaleString()} abbrevs`)
|
|
302
471
|
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
out.exec("BEGIN")
|
|
323
|
-
|
|
324
|
-
for (const r of pc
|
|
325
|
-
.prepare(
|
|
326
|
-
`SELECT id, name, country, latitude, longitude,
|
|
327
|
-
min_latitude AS mnlat, min_longitude AS mnlon, max_latitude AS mxlat, max_longitude AS mxlon
|
|
328
|
-
FROM spr WHERE placetype='postalcode' AND latitude != 0 AND longitude != 0`
|
|
329
|
-
)
|
|
330
|
-
.iterate()) {
|
|
331
|
-
const name = String(r.name ?? "")
|
|
332
|
-
const key = normalizeLocalityForKey(name)
|
|
333
|
-
|
|
334
|
-
if (!key) continue
|
|
335
|
-
|
|
336
|
-
const lat = r.latitude as number
|
|
337
|
-
const lon = r.longitude as number
|
|
338
|
-
|
|
339
|
-
// region_id 0 (a postcode is unique by name+country — no same-name disambiguation); neg_rank 0
|
|
340
|
-
// (no population). bbox = the postcode's own min/max (falls back to the centroid point).
|
|
341
|
-
const a: PlaceAttrs = {
|
|
342
|
-
cid: ccID(r.country as string | null),
|
|
343
|
-
rid: 0,
|
|
344
|
-
ptid: pcPtid,
|
|
345
|
-
name,
|
|
346
|
-
lat,
|
|
347
|
-
lon,
|
|
348
|
-
mnLat: (r.mnlat as number) || lat,
|
|
349
|
-
mnLon: (r.mnlon as number) || lon,
|
|
350
|
-
mxLat: (r.mxlat as number) || lat,
|
|
351
|
-
mxLon: (r.mxlon as number) || lon,
|
|
352
|
-
pop: 0,
|
|
353
|
-
neg: 0,
|
|
354
|
-
pkey: key,
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
pcAttrs.set(Number(r.id), a)
|
|
358
|
-
stageRow(key, a, Number(r.id), 1)
|
|
359
|
-
|
|
360
|
-
primaries++
|
|
361
|
-
}
|
|
362
|
-
|
|
363
|
-
out.exec("COMMIT")
|
|
364
|
-
|
|
365
|
-
// --- pass 4b: postcode ALIAS names (#1495) ---
|
|
366
|
-
//
|
|
367
|
-
// The delivery-city names GeoNames supplies for a ZIP ("Brooklyn" for 11201) are written into
|
|
368
|
-
// the shard's `names` table by `postcode/centroid-fills.ts`'s `geonamesNameFill`. Everything
|
|
369
|
-
// downstream of `names` picked them up EXCEPT this build: `fts.ts` unions `spr.name` with every
|
|
370
|
-
// `names` row into `place_search.alt_names`, so the FTS backend resolved "Brooklyn" → 11201
|
|
371
|
-
// while the candidate backend — whose every row IS an exact-tier row — had no key for it at
|
|
372
|
-
// all. Pass 2 does the equivalent fold for admin places, but reads the ADMIN `place_search`,
|
|
373
|
-
// and `attrs` holds admin ids only, so a postcode shard could never reach it.
|
|
374
|
-
//
|
|
375
|
-
// Same discipline as pass 2: `is_primary = 0` (so `rankByPrimaryPreference` treats it as an
|
|
376
|
-
// alias, not a canonical postcode name), the row stays denormalized onto the POSTCODE's own
|
|
377
|
-
// spr_id/coords/bbox, and the display `name` stays the postcode — resolving "brooklyn" answers
|
|
378
|
-
// with place 11201, it does not rename the place to its delivery city.
|
|
379
|
-
const hasNames = pc.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='names'").get() !== undefined
|
|
380
|
-
|
|
381
|
-
if (hasNames) {
|
|
382
|
-
out.exec("BEGIN")
|
|
383
|
-
|
|
384
|
-
for (const r of pc.prepare("SELECT id, name FROM names").iterate()) {
|
|
385
|
-
const a = pcAttrs.get(Number(r.id))
|
|
386
|
-
|
|
387
|
-
if (!a) continue
|
|
388
|
-
|
|
389
|
-
const k = normalizeLocalityForKey(String(r.name ?? ""))
|
|
390
|
-
|
|
391
|
-
// The postcode's own key is already staged as the primary; `INSERT OR IGNORE` at
|
|
392
|
-
// materialization dedupes repeats, so this only skips the obvious self-alias.
|
|
393
|
-
if (!k || k === a.pkey) continue
|
|
394
|
-
|
|
395
|
-
stageRow(k, a, Number(r.id), 0)
|
|
396
|
-
|
|
397
|
-
aliases++
|
|
398
|
-
}
|
|
399
|
-
|
|
400
|
-
out.exec("COMMIT")
|
|
401
|
-
} else {
|
|
402
|
-
// Never a silent zero: real shards come from `createUnifiedSchema`, which always creates
|
|
403
|
-
// `names`. A shard without it has no alias surface to lose, but say so rather than reporting
|
|
404
|
-
// "0 aliases" from a table that was never read.
|
|
405
|
-
progress("postcode-aliases", `${pcDB} has no \`names\` table — no delivery-city aliases to fold`)
|
|
406
|
-
}
|
|
407
|
-
|
|
408
|
-
pc.close()
|
|
409
|
-
|
|
410
|
-
return { primaries, aliases }
|
|
411
|
-
}
|
|
412
|
-
|
|
472
|
+
// --- pass 3b: name roles (#1730 prototype — stampNameRoles owns the detectors) ---
|
|
473
|
+
// Independent of the sidecar below: this writes `cand_stage.name_role`, that writes the ancestor and
|
|
474
|
+
// interval tables, and neither reads the other's output. Ordered by label only.
|
|
475
|
+
const roles = stampNameRoles({
|
|
476
|
+
src,
|
|
477
|
+
out,
|
|
478
|
+
attrs,
|
|
479
|
+
keyCounts,
|
|
480
|
+
glossThreshold: opts.glossKeyThreshold ?? GLOSS_KEY_THRESHOLD,
|
|
481
|
+
ptcodes,
|
|
482
|
+
ccodes,
|
|
483
|
+
progress,
|
|
484
|
+
})
|
|
485
|
+
|
|
486
|
+
// --- pass 3c: the ancestors sidecar (candidate-ancestors-schema.ts owns the encoding decision) ---
|
|
487
|
+
const sidecar = await buildAncestorsSidecar({ src, out, kdb, attrs, ptID, progress })
|
|
488
|
+
|
|
489
|
+
// --- pass 4 + 4b: postcode and locality shards (foldShard owns the per-shard loop) ---
|
|
413
490
|
let nPostcode = 0
|
|
414
491
|
let nPostcodeAlias = 0
|
|
415
492
|
|
|
416
493
|
for (const pcDB of opts.postcodes ?? []) {
|
|
417
|
-
const folded =
|
|
494
|
+
const folded = foldShard({
|
|
495
|
+
out,
|
|
496
|
+
shardPath: pcDB,
|
|
497
|
+
shardPlacetype: "postalcode",
|
|
498
|
+
ccID,
|
|
499
|
+
ptID,
|
|
500
|
+
stageRow,
|
|
501
|
+
progress,
|
|
502
|
+
})
|
|
418
503
|
|
|
419
504
|
nPostcode += folded.primaries
|
|
420
505
|
nPostcodeAlias += folded.aliases
|
|
@@ -424,6 +509,26 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
424
509
|
progress("postcodes", `${nPostcode.toLocaleString()} postcodes; ${nPostcodeAlias.toLocaleString()} aliases`)
|
|
425
510
|
}
|
|
426
511
|
|
|
512
|
+
let nLocality = 0
|
|
513
|
+
|
|
514
|
+
for (const locDB of opts.localities ?? []) {
|
|
515
|
+
const folded = foldShard({
|
|
516
|
+
out,
|
|
517
|
+
shardPath: locDB,
|
|
518
|
+
shardPlacetype: "locality",
|
|
519
|
+
ccID,
|
|
520
|
+
ptID,
|
|
521
|
+
stageRow,
|
|
522
|
+
progress,
|
|
523
|
+
})
|
|
524
|
+
|
|
525
|
+
nLocality += folded.primaries
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
if (nLocality > 0) {
|
|
529
|
+
progress("localities", `${nLocality.toLocaleString()} shard localities folded`)
|
|
530
|
+
}
|
|
531
|
+
|
|
427
532
|
// --- code dictionaries: typed batch inserts via kdb (a few hundred rows — Kysely is clean here) ---
|
|
428
533
|
if (ccodes.size) {
|
|
429
534
|
await kdb
|
|
@@ -439,6 +544,26 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
439
544
|
.execute()
|
|
440
545
|
}
|
|
441
546
|
|
|
547
|
+
// --- capital-status reference (#1880's distribution home) — cold, small (~3.7k rows), typed ---
|
|
548
|
+
if (opts.capitals?.length) {
|
|
549
|
+
await createCapitalTable<CandidateDatabase>(kdb)
|
|
550
|
+
|
|
551
|
+
await kdb
|
|
552
|
+
.insertInto("capital")
|
|
553
|
+
.values(
|
|
554
|
+
opts.capitals.map((entry) => ({
|
|
555
|
+
country: entry.country,
|
|
556
|
+
latitude: entry.latitude,
|
|
557
|
+
longitude: entry.longitude,
|
|
558
|
+
level: entry.level,
|
|
559
|
+
keys: JSON.stringify(entry.k),
|
|
560
|
+
}))
|
|
561
|
+
)
|
|
562
|
+
.execute()
|
|
563
|
+
|
|
564
|
+
progress("capitals", `${opts.capitals.length.toLocaleString()} capital-reference rows carried in-artifact`)
|
|
565
|
+
}
|
|
566
|
+
|
|
442
567
|
// --- materialize the clustered WITHOUT ROWID table (sorted insert → contiguous leaves) ---
|
|
443
568
|
progress("cluster", "building clustered candidate table + VACUUM")
|
|
444
569
|
// Column list + clustered-key order are sourced from CANDIDATE_COLUMNS (the first 6 ARE the PRIMARY
|
|
@@ -478,5 +603,10 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
478
603
|
abbrevs: nAbbr,
|
|
479
604
|
postcodes: nPostcode,
|
|
480
605
|
postcodeAliases: nPostcodeAlias,
|
|
606
|
+
ancestorRows: sidecar.ancestorRows,
|
|
607
|
+
ancestorPlaces: sidecar.ancestorPlaces,
|
|
608
|
+
intervalPlaces: sidecar.intervalPlaces,
|
|
609
|
+
...roles,
|
|
610
|
+
...(importance ? { importanceScored: importance.matched, importanceGated: importance.gated } : {}),
|
|
481
611
|
}
|
|
482
612
|
}
|
package/build-slim.ts
CHANGED
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
*
|
|
27
27
|
* The output DB has the resolver-facing schema: `spr`, `names`, `place_population`, plus the
|
|
28
28
|
* `place_search` FTS5 / `place_bbox` R*Tree virtual tables rebuilt against the trimmed row set
|
|
29
|
-
* (both derive purely from `spr` + `names` — see `fts.ts`). That means `
|
|
29
|
+
* (both derive purely from `spr` + `names` — see `fts.ts`). That means `WOFSQLitePlaceLookup`
|
|
30
30
|
* opens the slim DB without any code change — it sees a smaller universe, nothing more.
|
|
31
31
|
*
|
|
32
32
|
* Multi-shard inputs (e.g. admin + postcode) are processed in sequence; selected rows accumulate
|
|
@@ -223,10 +223,10 @@ export async function buildSlimWOFDatabase(opts: BuildSlimOptions): Promise<Buil
|
|
|
223
223
|
|
|
224
224
|
// Materialize region/state ABBREVIATIONS into a standalone `place_abbr (id, abbr)` table BEFORE
|
|
225
225
|
// `names` is (optionally) dropped. The full DB lets the resolver tier an exact-abbrev match by
|
|
226
|
-
// querying `names` (`#
|
|
226
|
+
// querying `names` (`#exactMatchIDs`), but the slim DB drops `names` for size — so the
|
|
227
227
|
// browser resolver gets its own tiny lookup (~hundreds of rows) to do the same data-driven
|
|
228
228
|
// exact-abbrev tiering ("VT" → Vermont, not a token-matching foreign region) instead of the
|
|
229
|
-
// demo's hardcoded
|
|
229
|
+
// demo's hardcoded region-abbreviation map (since deleted). Sourced from the `language='abbr'` rows
|
|
230
230
|
// `add-region-abbrevs.ts` wrote, already filtered to surviving spr ids via the names copy. The
|
|
231
231
|
// table is always created (empty when the source predates the abbrev enrichment) so the
|
|
232
232
|
// resolver can query it unconditionally.
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
* @file Pass 2 of the candidate build — explode `place_search.alt_names` into distinct alias rows.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type { DatabaseSync } from "node:sqlite"
|
|
9
|
+
|
|
10
|
+
import { ALIAS_SEPARATOR } from "../fts.ts"
|
|
11
|
+
import { normalizeLocalityForKey } from "../street-normalize.ts"
|
|
12
|
+
import type { PlaceAttrs, StageRow } from "./place-attrs.ts"
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Pass 2 — explode each place's `place_search.alt_names` bag into distinct-key alias rows (`is_primary = 0`), and count
|
|
16
|
+
* each place's distinct staged keys (primary included) — the gloss detector's key-count signal (#1730).
|
|
17
|
+
*/
|
|
18
|
+
export function explodeAliasBags(
|
|
19
|
+
src: DatabaseSync,
|
|
20
|
+
out: DatabaseSync,
|
|
21
|
+
attrs: Map<number, PlaceAttrs>,
|
|
22
|
+
stageRow: StageRow
|
|
23
|
+
): { nAlias: number; keyCounts: Map<number, number> } {
|
|
24
|
+
let nAlias = 0
|
|
25
|
+
const keyCounts = new Map<number, number>()
|
|
26
|
+
out.exec("BEGIN")
|
|
27
|
+
|
|
28
|
+
for (const r of src.prepare("SELECT wof_id, alt_names FROM place_search").iterate()) {
|
|
29
|
+
const a = attrs.get(Number(r.wof_id))
|
|
30
|
+
const alt = r.alt_names as string | null
|
|
31
|
+
|
|
32
|
+
if (!a || !alt) continue
|
|
33
|
+
const seen = new Set<string>([a.pkey])
|
|
34
|
+
|
|
35
|
+
// The writer space-pads each separator and appends a trailing one, so every piece arrives with
|
|
36
|
+
// surrounding whitespace and the last one is empty. `normalizeLocalityForKey` folds both away,
|
|
37
|
+
// and the empty tail falls out at the `!k` guard below.
|
|
38
|
+
for (const piece of alt.split(ALIAS_SEPARATOR)) {
|
|
39
|
+
const k = normalizeLocalityForKey(piece)
|
|
40
|
+
|
|
41
|
+
if (!k || seen.has(k)) continue
|
|
42
|
+
seen.add(k)
|
|
43
|
+
stageRow(k, a, Number(r.wof_id), 0)
|
|
44
|
+
|
|
45
|
+
nAlias++
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
keyCounts.set(Number(r.wof_id), seen.size)
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
out.exec("COMMIT")
|
|
52
|
+
|
|
53
|
+
return { nAlias, keyCounts }
|
|
54
|
+
}
|