@mailwoman/resolver-wof-sqlite 9.0.0 → 9.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -9
- package/address-point-interpolation.ts +18 -8
- package/address-point-schema.ts +18 -6
- package/address-point.ts +111 -18
- package/ancestry.ts +9 -6
- package/build-candidate.ts +287 -157
- package/build-slim.ts +3 -3
- package/candidate/alias-bags.ts +54 -0
- package/candidate/ancestors-sidecar.ts +206 -0
- package/candidate/country-display-names.ts +79 -0
- package/candidate/name-roles.ts +237 -0
- package/candidate/own-name.ts +146 -0
- package/candidate/place-attrs.ts +44 -0
- package/candidate/shard-fold.ts +137 -0
- package/candidate-ancestors-schema.ts +195 -0
- package/candidate-fts.ts +4 -2
- package/candidate-importance.ts +228 -0
- package/candidate-lookup.ts +564 -174
- package/candidate-schema.ts +60 -3
- package/candidate-scoring.ts +268 -0
- package/capital-schema.ts +90 -0
- package/capitals.ts +148 -0
- package/coincident-roles.ts +69 -10
- package/convention-schema.ts +72 -0
- package/convention.ts +2 -2
- package/coverage-manifest-schema.ts +7 -7
- package/currency-backfill.ts +249 -0
- package/exact-match.ts +104 -0
- package/fst-autocomplete.ts +105 -122
- package/fst-builder.ts +39 -47
- package/fst-deserialize-web.ts +43 -7
- package/fst-freshness.ts +2 -2
- package/fst-serialize.ts +68 -12
- package/fst-types.ts +35 -1
- package/fts-query.ts +1 -1
- package/fts.ts +16 -4
- package/geonames-postal.ts +2 -2
- package/index.ts +26 -14
- package/interpolation.ts +113 -19
- package/lookup.ts +118 -560
- package/name-score.ts +6 -4
- package/out/address-point-interpolation.d.ts.map +1 -1
- package/out/address-point-interpolation.js +13 -7
- package/out/address-point-interpolation.js.map +1 -1
- package/out/address-point-schema.d.ts +16 -6
- package/out/address-point-schema.d.ts.map +1 -1
- package/out/address-point-schema.js.map +1 -1
- package/out/address-point.d.ts.map +1 -1
- package/out/address-point.js +70 -14
- package/out/address-point.js.map +1 -1
- package/out/ancestry.d.ts +2 -2
- package/out/ancestry.d.ts.map +1 -1
- package/out/ancestry.js +5 -6
- package/out/ancestry.js.map +1 -1
- package/out/build-candidate.d.ts +108 -0
- package/out/build-candidate.d.ts.map +1 -1
- package/out/build-candidate.js +151 -120
- package/out/build-candidate.js.map +1 -1
- package/out/build-slim.d.ts +1 -1
- package/out/build-slim.js +3 -3
- package/out/build-slim.js.map +1 -1
- package/out/candidate/alias-bags.d.ts +17 -0
- package/out/candidate/alias-bags.d.ts.map +1 -0
- package/out/candidate/alias-bags.js +39 -0
- package/out/candidate/alias-bags.js.map +1 -0
- package/out/candidate/ancestors-sidecar.d.ts +33 -0
- package/out/candidate/ancestors-sidecar.d.ts.map +1 -0
- package/out/candidate/ancestors-sidecar.js +140 -0
- package/out/candidate/ancestors-sidecar.js.map +1 -0
- package/out/candidate/country-display-names.d.ts +35 -0
- package/out/candidate/country-display-names.d.ts.map +1 -0
- package/out/candidate/country-display-names.js +59 -0
- package/out/candidate/country-display-names.js.map +1 -0
- package/out/candidate/name-roles.d.ts +55 -0
- package/out/candidate/name-roles.d.ts.map +1 -0
- package/out/candidate/name-roles.js +165 -0
- package/out/candidate/name-roles.js.map +1 -0
- package/out/candidate/own-name.d.ts +50 -0
- package/out/candidate/own-name.d.ts.map +1 -0
- package/out/candidate/own-name.js +132 -0
- package/out/candidate/own-name.js.map +1 -0
- package/out/candidate/place-attrs.d.ts +43 -0
- package/out/candidate/place-attrs.d.ts.map +1 -0
- package/out/candidate/place-attrs.js +15 -0
- package/out/candidate/place-attrs.js.map +1 -0
- package/out/candidate/shard-fold.d.ts +31 -0
- package/out/candidate/shard-fold.d.ts.map +1 -0
- package/out/candidate/shard-fold.js +104 -0
- package/out/candidate/shard-fold.js.map +1 -0
- package/out/candidate-ancestors-schema.d.ts +150 -0
- package/out/candidate-ancestors-schema.d.ts.map +1 -0
- package/out/candidate-ancestors-schema.js +123 -0
- package/out/candidate-ancestors-schema.js.map +1 -0
- package/out/candidate-fts.d.ts +4 -2
- package/out/candidate-fts.d.ts.map +1 -1
- package/out/candidate-fts.js +4 -2
- package/out/candidate-fts.js.map +1 -1
- package/out/candidate-importance.d.ts +132 -0
- package/out/candidate-importance.d.ts.map +1 -0
- package/out/candidate-importance.js +174 -0
- package/out/candidate-importance.js.map +1 -0
- package/out/candidate-lookup.d.ts +22 -37
- package/out/candidate-lookup.d.ts.map +1 -1
- package/out/candidate-lookup.js +446 -132
- package/out/candidate-lookup.js.map +1 -1
- package/out/candidate-schema.d.ts +52 -4
- package/out/candidate-schema.d.ts.map +1 -1
- package/out/candidate-schema.js +8 -0
- package/out/candidate-schema.js.map +1 -1
- package/out/candidate-scoring.d.ts +34 -0
- package/out/candidate-scoring.d.ts.map +1 -0
- package/out/candidate-scoring.js +200 -0
- package/out/candidate-scoring.js.map +1 -0
- package/out/capital-schema.d.ts +51 -0
- package/out/capital-schema.d.ts.map +1 -0
- package/out/capital-schema.js +63 -0
- package/out/capital-schema.js.map +1 -0
- package/out/capitals.d.ts +69 -0
- package/out/capitals.d.ts.map +1 -0
- package/out/capitals.js +98 -0
- package/out/capitals.js.map +1 -0
- package/out/coincident-roles.d.ts +7 -0
- package/out/coincident-roles.d.ts.map +1 -1
- package/out/coincident-roles.js +42 -8
- package/out/coincident-roles.js.map +1 -1
- package/out/convention-schema.d.ts +51 -0
- package/out/convention-schema.d.ts.map +1 -0
- package/out/convention-schema.js +34 -0
- package/out/convention-schema.js.map +1 -0
- package/out/convention.d.ts +1 -1
- package/out/convention.js +2 -2
- package/out/coverage-manifest-schema.js +3 -7
- package/out/coverage-manifest-schema.js.map +1 -1
- package/out/currency-backfill.d.ts +46 -0
- package/out/currency-backfill.d.ts.map +1 -0
- package/out/currency-backfill.js +180 -0
- package/out/currency-backfill.js.map +1 -0
- package/out/exact-match.d.ts +25 -0
- package/out/exact-match.d.ts.map +1 -0
- package/out/exact-match.js +89 -0
- package/out/exact-match.js.map +1 -0
- package/out/fst-autocomplete.d.ts +24 -14
- package/out/fst-autocomplete.d.ts.map +1 -1
- package/out/fst-autocomplete.js +84 -100
- package/out/fst-autocomplete.js.map +1 -1
- package/out/fst-builder.d.ts.map +1 -1
- package/out/fst-builder.js +32 -40
- package/out/fst-builder.js.map +1 -1
- package/out/fst-deserialize-web.d.ts.map +1 -1
- package/out/fst-deserialize-web.js +36 -7
- package/out/fst-deserialize-web.js.map +1 -1
- package/out/fst-freshness.d.ts +2 -2
- package/out/fst-freshness.js +2 -2
- package/out/fst-serialize.d.ts +14 -4
- package/out/fst-serialize.d.ts.map +1 -1
- package/out/fst-serialize.js +60 -12
- package/out/fst-serialize.js.map +1 -1
- package/out/fst-types.d.ts +35 -1
- package/out/fst-types.d.ts.map +1 -1
- package/out/fts-query.js +1 -1
- package/out/fts-query.js.map +1 -1
- package/out/fts.d.ts +15 -4
- package/out/fts.d.ts.map +1 -1
- package/out/fts.js +15 -4
- package/out/fts.js.map +1 -1
- package/out/geonames-postal.d.ts +2 -2
- package/out/geonames-postal.js +2 -2
- package/out/index.d.ts +4 -2
- package/out/index.d.ts.map +1 -1
- package/out/index.js +3 -2
- package/out/index.js.map +1 -1
- package/out/interpolation.d.ts +8 -0
- package/out/interpolation.d.ts.map +1 -1
- package/out/interpolation.js +91 -19
- package/out/interpolation.js.map +1 -1
- package/out/lookup.d.ts +4 -5
- package/out/lookup.d.ts.map +1 -1
- package/out/lookup.js +102 -444
- package/out/lookup.js.map +1 -1
- package/out/name-score.d.ts +0 -10
- package/out/name-score.d.ts.map +1 -1
- package/out/name-score.js +6 -4
- package/out/name-score.js.map +1 -1
- package/out/place-importance-schema.d.ts +226 -0
- package/out/place-importance-schema.d.ts.map +1 -0
- package/out/place-importance-schema.js +288 -0
- package/out/place-importance-schema.js.map +1 -0
- package/out/poi-lookup.d.ts +1 -1
- package/out/poi-lookup.d.ts.map +1 -1
- package/out/poi-lookup.js +12 -13
- package/out/poi-lookup.js.map +1 -1
- package/out/poi-schema.d.ts +7 -3
- package/out/poi-schema.d.ts.map +1 -1
- package/out/poi-schema.js.map +1 -1
- package/out/polygon-schema.d.ts +37 -0
- package/out/polygon-schema.d.ts.map +1 -0
- package/out/polygon-schema.js +23 -0
- package/out/polygon-schema.js.map +1 -0
- package/out/postal-city-alias-lookup.d.ts +1 -1
- package/out/postal-city-alias-lookup.js +1 -1
- package/out/postal-city-candidate-schema.d.ts +2 -1
- package/out/postal-city-candidate-schema.d.ts.map +1 -1
- package/out/postal-city-candidate-schema.js.map +1 -1
- package/out/postcode-point-lookup.d.ts +1 -1
- package/out/postcode-point-lookup.js +1 -1
- package/out/primary-preference.d.ts +125 -0
- package/out/primary-preference.d.ts.map +1 -0
- package/out/primary-preference.js +138 -0
- package/out/primary-preference.js.map +1 -0
- package/out/proximity-rerank.d.ts +77 -0
- package/out/proximity-rerank.d.ts.map +1 -0
- package/out/proximity-rerank.js +86 -0
- package/out/proximity-rerank.js.map +1 -0
- package/out/region-keys.d.ts +47 -0
- package/out/region-keys.d.ts.map +1 -0
- package/out/region-keys.js +121 -0
- package/out/region-keys.js.map +1 -0
- package/out/reverse.d.ts.map +1 -1
- package/out/reverse.js +6 -9
- package/out/reverse.js.map +1 -1
- package/out/schema.d.ts +1 -1
- package/out/search-fetch.d.ts +57 -0
- package/out/search-fetch.d.ts.map +1 -0
- package/out/search-fetch.js +183 -0
- package/out/search-fetch.js.map +1 -0
- package/out/sharding.d.ts +3 -3
- package/out/sharding.js +1 -1
- package/out/sqlite-convention-source.d.ts +1 -1
- package/out/sqlite-convention-source.js +1 -1
- package/out/sqlite-utils.d.ts +31 -1
- package/out/sqlite-utils.d.ts.map +1 -1
- package/out/sqlite-utils.js +38 -0
- package/out/sqlite-utils.js.map +1 -1
- package/out/street-centroid-schema.d.ts +7 -2
- package/out/street-centroid-schema.d.ts.map +1 -1
- package/out/street-centroid-schema.js.map +1 -1
- package/out/street-centroid.d.ts.map +1 -1
- package/out/street-centroid.js +7 -7
- package/out/street-centroid.js.map +1 -1
- package/out/street-morphology-fst-builder.d.ts.map +1 -1
- package/out/street-morphology-fst-builder.js +5 -4
- package/out/street-morphology-fst-builder.js.map +1 -1
- package/out/street-normalize.d.ts +83 -9
- package/out/street-normalize.d.ts.map +1 -1
- package/out/street-normalize.js +177 -10
- package/out/street-normalize.js.map +1 -1
- package/out/street-segment-schema.d.ts +6 -2
- package/out/street-segment-schema.d.ts.map +1 -1
- package/out/street-segment-schema.js.map +1 -1
- package/out/types.d.ts +74 -1
- package/out/types.d.ts.map +1 -1
- package/out/unified-schema.d.ts +1 -1
- package/out/unified-schema.js +1 -1
- package/out/uprn-lookup.d.ts +85 -0
- package/out/uprn-lookup.d.ts.map +1 -0
- package/out/uprn-lookup.js +152 -0
- package/out/uprn-lookup.js.map +1 -0
- package/out/uprn-schema.d.ts +93 -0
- package/out/uprn-schema.d.ts.map +1 -0
- package/out/uprn-schema.js +78 -0
- package/out/uprn-schema.js.map +1 -0
- package/out/weights-overlay-linker.d.ts +141 -0
- package/out/weights-overlay-linker.d.ts.map +1 -0
- package/out/weights-overlay-linker.js +259 -0
- package/out/weights-overlay-linker.js.map +1 -0
- package/package.json +296 -16
- package/place-importance-schema.ts +402 -0
- package/poi-lookup.ts +12 -13
- package/poi-schema.ts +8 -3
- package/polygon-schema.ts +47 -0
- package/postal-city-alias-lookup.ts +1 -1
- package/postal-city-candidate-schema.ts +3 -1
- package/postcode-point-lookup.ts +1 -1
- package/primary-preference.ts +207 -0
- package/proximity-rerank.ts +120 -0
- package/region-keys.ts +144 -0
- package/reverse.ts +17 -16
- package/schema.ts +1 -1
- package/search-fetch.ts +256 -0
- package/sharding.ts +3 -3
- package/sqlite-convention-source.ts +1 -1
- package/sqlite-utils.ts +63 -1
- package/street-centroid-schema.ts +8 -2
- package/street-centroid.ts +13 -8
- package/street-morphology-fst-builder.ts +5 -4
- package/street-normalize.ts +254 -24
- package/street-segment-schema.ts +7 -2
- package/types.ts +74 -1
- package/unified-schema.ts +1 -1
- package/uprn-lookup.ts +210 -0
- package/uprn-schema.ts +124 -0
- package/weights-overlay-linker.ts +377 -0
- package/geo.ts +0 -121
- package/out/geo.d.ts +0 -74
- package/out/geo.d.ts.map +0 -1
- package/out/geo.js +0 -71
- package/out/geo.js.map +0 -1
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Pass 1c of the candidate build (#1737): resurrect deprecated-with-no-successor WOF localities that a second
|
|
7
|
+
* source independently attests. Extracted from `build-candidate.ts` as a cohesive unit — the gates, their measured
|
|
8
|
+
* constants, and the GeoNames dump reader live together here; the build calls {@link resurrectCurrencyHoles} once,
|
|
9
|
+
* between the primaries pass and the alias pass.
|
|
10
|
+
*/
|
|
11
|
+
import { existsSync } from "node:fs";
|
|
12
|
+
import { resolve } from "node:path";
|
|
13
|
+
import { isStrictlyFiner } from "@mailwoman/core/resources/whosonfirst";
|
|
14
|
+
import { haversineKm } from "@mailwoman/spatial";
|
|
15
|
+
import { TSVSpliterator } from "spliterator";
|
|
16
|
+
import { normalizeLocalityForKey } from "./street-normalize.js";
|
|
17
|
+
/**
|
|
18
|
+
* Corroboration radius for the currency backfill (#1737), km — both for the live-near blocker and the GeoNames
|
|
19
|
+
* attestation. Measured basis (2026-08-19 prototype, GB locality slice): at 10 km, 20 of 108 dead names resurrect —
|
|
20
|
+
* Rochester (Kent), Aldershot, Staines, Telford, Ebbw Vale among them — while the `Birmingham/Wolverhampton/…`
|
|
21
|
+
* conurbation blobs stay dead (no attestation) and Swansea/Wrexham stay out because a live same-name row already serves
|
|
22
|
+
* them within the radius.
|
|
23
|
+
*/
|
|
24
|
+
const CURRENCY_BACKFILL_RADIUS_KM = 10;
|
|
25
|
+
/**
|
|
26
|
+
* Minimum attestor population for a resurrection. The same prototype measured 44 of 108 dead GB names attested but
|
|
27
|
+
* under this floor — hamlet-scale ghosts whose absence from the index nobody has reported. A floor keeps the pass
|
|
28
|
+
* answering the measured defect (real settlements) rather than re-importing the tail WOF chose to prune.
|
|
29
|
+
*/
|
|
30
|
+
const CURRENCY_BACKFILL_POP_FLOOR = 1000;
|
|
31
|
+
/**
|
|
32
|
+
* Pass 1c (#1737): resurrect deprecated-with-no-successor localities that a second source independently attests.
|
|
33
|
+
*
|
|
34
|
+
* WOF deprecations WITH a successor need nothing — the successor indexes. A deprecation with `is_superseded = 0` on a
|
|
35
|
+
* populated place is the shape of an upstream mistake (Rochester Kent, Aldershot, Telford; 120 GB localities alone),
|
|
36
|
+
* and it is indistinguishable from a correct pruning at this layer without outside evidence. So every resurrection
|
|
37
|
+
* requires all three gates, positive evidence only:
|
|
38
|
+
*
|
|
39
|
+
* 1. NO live same-name spr row of any placetype within {@link CURRENCY_BACKFILL_RADIUS_KM} of the dead record — a live row
|
|
40
|
+
* means the place is alive (possibly under another placetype) and there is no hole. A DISTANT same-name row is a
|
|
41
|
+
* namesake and does not block.
|
|
42
|
+
* 2. A GeoNames feature-class-P attestation of the same folded name within the radius.
|
|
43
|
+
* 3. The attestor at or above {@link CURRENCY_BACKFILL_POP_FLOOR}.
|
|
44
|
+
*
|
|
45
|
+
* The staged row keeps the WOF identity — id, name, centroid, bbox, region ancestry — because the dead record's own
|
|
46
|
+
* data is not what is wrong with it. GeoNames contributes exactly two things: the attestation, and the population that
|
|
47
|
+
* lets the row stand in prominence races (the dead record's own population is absent). Each name is judged once per
|
|
48
|
+
* country; the resurrected place joins `attrs`, so the alias pass explodes its alt names like any primary's.
|
|
49
|
+
*/
|
|
50
|
+
export async function resurrectCurrencyHoles(ctx) {
|
|
51
|
+
const deadStmt = ctx.src.prepare(`SELECT id, name, placetype, latitude, longitude, min_latitude, min_longitude, max_latitude, max_longitude
|
|
52
|
+
FROM spr
|
|
53
|
+
WHERE country = ? AND placetype = 'locality'
|
|
54
|
+
AND is_current = 0 AND is_deprecated = 1 AND is_superseded = 0`);
|
|
55
|
+
const liveStmt = ctx.src.prepare(`SELECT latitude, longitude, placetype FROM spr WHERE country = ? AND name = ? AND is_current != 0`);
|
|
56
|
+
let total = 0;
|
|
57
|
+
for (const country of ctx.countries) {
|
|
58
|
+
const cc = country.toUpperCase();
|
|
59
|
+
const dumpPath = resolve(ctx.geonamesDir, `${cc}.txt`);
|
|
60
|
+
if (!existsSync(dumpPath)) {
|
|
61
|
+
ctx.progress("currency-backfill", `${cc}: no GeoNames dump at ${dumpPath} — holes stay dead`);
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
// Dead rows FIRST: a country with no deprecated-no-successor localities needs no attestors at all, and
|
|
65
|
+
// loading a national dump to judge zero rows is pure heap pressure on a build already near its ceiling
|
|
66
|
+
// (the first live run OOM'd in a later pass with JP/KR dumps loaded for 0 dead names each).
|
|
67
|
+
const dead = deadStmt.all(cc);
|
|
68
|
+
if (!dead.length) {
|
|
69
|
+
ctx.progress("currency-backfill", `${cc}: 0 dead names — dump not loaded`);
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
// Only the dead names' own folded keys can ever be probed, so only those keys are worth holding —
|
|
73
|
+
// the rest of the national dump streams through without residency.
|
|
74
|
+
const deadKeys = new Set();
|
|
75
|
+
for (const d of dead) {
|
|
76
|
+
const k = normalizeLocalityForKey(String(d.name ?? ""));
|
|
77
|
+
if (k) {
|
|
78
|
+
deadKeys.add(k);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
// Folded name → P-class attestors. GeoNames columns by index: 1 name, 2 ascii, 4 lat, 5 lon,
|
|
82
|
+
// 6 feature_class, 14 population.
|
|
83
|
+
const attestors = new Map();
|
|
84
|
+
for await (const f of TSVSpliterator.fromAsync(dumpPath, { header: false })) {
|
|
85
|
+
if (f[6] !== "P")
|
|
86
|
+
continue;
|
|
87
|
+
const keys = [normalizeLocalityForKey(String(f[1] ?? "")), normalizeLocalityForKey(String(f[2] ?? ""))].filter((key) => key && deadKeys.has(key));
|
|
88
|
+
if (!keys.length)
|
|
89
|
+
continue;
|
|
90
|
+
const row = { lat: Number(f[4]), lon: Number(f[5]), pop: Number(f[14]) || 0 };
|
|
91
|
+
for (const key of new Set(keys)) {
|
|
92
|
+
const bag = attestors.get(key);
|
|
93
|
+
if (bag) {
|
|
94
|
+
bag.push(row);
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
attestors.set(key, [row]);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
let judged = 0;
|
|
102
|
+
let blocked = 0;
|
|
103
|
+
let unattested = 0;
|
|
104
|
+
let floored = 0;
|
|
105
|
+
let resurrected = 0;
|
|
106
|
+
const seen = new Set();
|
|
107
|
+
ctx.tx.exec("BEGIN");
|
|
108
|
+
for (const d of dead) {
|
|
109
|
+
const name = String(d.name ?? "");
|
|
110
|
+
const pkey = normalizeLocalityForKey(name);
|
|
111
|
+
if (!pkey || seen.has(pkey))
|
|
112
|
+
continue;
|
|
113
|
+
seen.add(pkey);
|
|
114
|
+
judged++;
|
|
115
|
+
const dLat = Number(d.latitude);
|
|
116
|
+
const dLon = Number(d.longitude);
|
|
117
|
+
// The dead-row query is scoped to `locality` today; read it from the row anyway so widening that query
|
|
118
|
+
// cannot silently start comparing every candidate against a hardcoded rung.
|
|
119
|
+
const deadPlacetype = String(d.placetype ?? "locality");
|
|
120
|
+
// A live row blocks only when it is AT LEAST AS COARSE as the dead one. The original gate compared name and
|
|
121
|
+
// distance alone, on the premise that a nearby same-name row means "the place is alive under another
|
|
122
|
+
// placetype" — true for a place recorded twice, false for a placetype DEMOTION, which is the shape that
|
|
123
|
+
// actually occurs: WOF retired `Gillingham` the locality (pop 101,187) and kept `Gillingham` the
|
|
124
|
+
// neighbourhood 3.2 km away, and the gate read the surviving CHILD as covering its own dead parent.
|
|
125
|
+
// Sixteen of seventeen GB refusals had exactly that shape (#1746).
|
|
126
|
+
//
|
|
127
|
+
// An UNRANKED placetype blocks, which is the conservative direction: this gate's failure mode is inventing
|
|
128
|
+
// a place, so a row we cannot rank is treated as covering rather than waved through.
|
|
129
|
+
const liveNear = liveStmt.all(cc, name).some((row) => {
|
|
130
|
+
if (haversineKm(dLat, dLon, Number(row.latitude), Number(row.longitude)) > CURRENCY_BACKFILL_RADIUS_KM) {
|
|
131
|
+
return false;
|
|
132
|
+
}
|
|
133
|
+
// Blocks UNLESS the live row is strictly finer. The equal rung must still block — a live `locality`
|
|
134
|
+
// covers a dead `locality` — and an unranked placetype blocks too, since this gate's failure mode
|
|
135
|
+
// is inventing a place.
|
|
136
|
+
return isStrictlyFiner(String(row.placetype ?? ""), deadPlacetype) !== true;
|
|
137
|
+
});
|
|
138
|
+
if (liveNear) {
|
|
139
|
+
blocked++;
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
const near = (attestors.get(pkey) ?? []).filter((g) => haversineKm(dLat, dLon, g.lat, g.lon) <= CURRENCY_BACKFILL_RADIUS_KM);
|
|
143
|
+
if (!near.length) {
|
|
144
|
+
unattested++;
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
147
|
+
const pop = Math.max(...near.map((g) => g.pop));
|
|
148
|
+
if (pop < CURRENCY_BACKFILL_POP_FLOOR) {
|
|
149
|
+
floored++;
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
const sid = Number(d.id);
|
|
153
|
+
const a = {
|
|
154
|
+
cid: ctx.ccID(cc),
|
|
155
|
+
rid: ctx.regionOf.get(sid) ?? 0,
|
|
156
|
+
ptid: ctx.ptID("locality"),
|
|
157
|
+
name,
|
|
158
|
+
lat: dLat,
|
|
159
|
+
lon: dLon,
|
|
160
|
+
mnLat: Number(d.min_latitude),
|
|
161
|
+
mnLon: Number(d.min_longitude),
|
|
162
|
+
mxLat: Number(d.max_latitude),
|
|
163
|
+
mxLon: Number(d.max_longitude),
|
|
164
|
+
pop,
|
|
165
|
+
neg: -Math.log10(pop + 1),
|
|
166
|
+
pkey,
|
|
167
|
+
imp: ctx.importance?.find(name, cc, "locality", dLat, dLon) ?? null,
|
|
168
|
+
};
|
|
169
|
+
ctx.attrs.set(sid, a);
|
|
170
|
+
ctx.stageRow(pkey, a, sid, 1);
|
|
171
|
+
resurrected++;
|
|
172
|
+
}
|
|
173
|
+
ctx.tx.exec("COMMIT");
|
|
174
|
+
ctx.progress("currency-backfill", `${cc}: ${resurrected} resurrected of ${judged} dead names ` +
|
|
175
|
+
`(${blocked} blocked by a live near row, ${unattested} unattested, ${floored} under the population floor)`);
|
|
176
|
+
total += resurrected;
|
|
177
|
+
}
|
|
178
|
+
return total;
|
|
179
|
+
}
|
|
180
|
+
//# sourceMappingURL=currency-backfill.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"currency-backfill.js","sourceRoot":"","sources":["../currency-backfill.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAGnC,OAAO,EAAE,eAAe,EAAE,MAAM,uCAAuC,CAAA;AACvE,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAI5C,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAA;AAE/D;;;;;;GAMG;AACH,MAAM,2BAA2B,GAAG,EAAE,CAAA;AAEtC;;;;GAIG;AACH,MAAM,2BAA2B,GAAG,IAAI,CAAA;AAExC;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,GAY5C;IACA,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAC/B;;;oEAGkE,CAClE,CAAA;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAC/B,mGAAmG,CACnG,CAAA;IAED,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,KAAK,MAAM,OAAO,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;QACrC,MAAM,EAAE,GAAG,OAAO,CAAC,WAAW,EAAE,CAAA;QAChC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,EAAE,MAAM,CAAC,CAAA;QAEtD,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,GAAG,CAAC,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE,yBAAyB,QAAQ,oBAAoB,CAAC,CAAA;YAE7F,SAAQ;QACT,CAAC;QAED,uGAAuG;QACvG,uGAAuG;QACvG,4FAA4F;QAC5F,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAE7B,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAClB,GAAG,CAAC,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE,kCAAkC,CAAC,CAAA;YAE1E,SAAQ;QACT,CAAC;QAED,kGAAkG;QAClG,mEAAmE;QACnE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;QAElC,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACtB,MAAM,CAAC,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAA;YAEvD,IAAI,CAAC,EAAE,CAAC;gBACP,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAChB,CAAC;QACF,CAAC;QAED,6FAA6F;QAC7F,kCAAkC;QAClC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAuD,CAAA;QAEhF,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,cAAc,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YAC7E,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG;gBAAE,SAAQ;YAE1B,MAAM,IAAI,GAAG,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAC7G,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CACjC,CAAA;YAED,IAAI,CAAC,IAAI,CAAC,MAAM;gBAAE,SAAQ;YAE1B,MAAM,GAAG,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAA;YAE7E,KAAK,MAAM,GAAG,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjC,MAAM,GAAG,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;gBAE9B,IAAI,GAAG,EAAE,CAAC;oBACT,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACd,CAAC;qBAAM,CAAC;oBACP,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC1B,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,MAAM,GAAG,CAAC,CAAA;QACd,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,WAAW,GAAG,CAAC,CAAA;QACnB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;QAE9B,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAEpB,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACtB,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAA;YACjC,MAAM,IAAI,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAA;YAE1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,SAAQ;YACrC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YAEd,MAAM,EAAE,CAAA;YAER,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAA;YAC/B,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAA;YAChC,uGAAuG;YACvG,4EAA4E;YAC5E,MAAM,aAAa,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,IAAI,UAAU,CAAC,CAAA;YAEvD,4GAA4G;YAC5G,qGAAqG;YACrG,wGAAwG;YACxG,iGAAiG;YACjG,oGAAoG;YACpG,mEAAmE;YACnE,EAAE;YACF,2GAA2G;YAC3G,qFAAqF;YACrF,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;gBACpD,IAAI,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,2BAA2B,EAAE,CAAC;oBACxG,OAAO,KAAK,CAAA;gBACb,CAAC;gBAED,oGAAoG;gBACpG,kGAAkG;gBAClG,wBAAwB;gBACxB,OAAO,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,EAAE,aAAa,CAAC,KAAK,IAAI,CAAA;YAC5E,CAAC,CAAC,CAAA;YAEF,IAAI,QAAQ,EAAE,CAAC;gBACd,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAC9C,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,2BAA2B,CAC3E,CAAA;YAED,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBAClB,UAAU,EAAE,CAAA;gBAEZ,SAAQ;YACT,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAE/C,IAAI,GAAG,GAAG,2BAA2B,EAAE,CAAC;gBACvC,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAExB,MAAM,CAAC,GAAe;gBACrB,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjB,GAAG,EAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;gBAC/B,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC;gBAC1B,IAAI;gBACJ,GAAG,EAAE,IAAI;gBACT,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC;gBAC7B,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC;gBAC9B,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC;gBAC7B,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC;gBAC9B,GAAG;gBACH,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;gBACzB,IAAI;gBACJ,GAAG,EAAE,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI;aACnE,CAAA;YAED,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;YACrB,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAA;YAE7B,WAAW,EAAE,CAAA;QACd,CAAC;QAED,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAErB,GAAG,CAAC,QAAQ,CACX,mBAAmB,EACnB,GAAG,EAAE,KAAK,WAAW,mBAAmB,MAAM,cAAc;YAC3D,IAAI,OAAO,gCAAgC,UAAU,gBAAgB,OAAO,8BAA8B,CAC3G,CAAA;QAED,KAAK,IAAI,WAAW,CAAA;IACrB,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* The exact-match tier's name probes — the two case-folded equality lookups that decide whether a
|
|
7
|
+
* candidate holds the query text as its own name, an alias, or an official name.
|
|
8
|
+
*/
|
|
9
|
+
import type { DatabaseSync } from "node:sqlite";
|
|
10
|
+
/**
|
|
11
|
+
* Among `ids`, return the subset whose name OR any alias equals `text` case-insensitively — the exact-match tier for
|
|
12
|
+
* ranking. One indexed query over `<schema>.names`. When the shard has no `names` table (a slim DB built with
|
|
13
|
+
* `dropNames`, or a postcode-only shard), fall back to the self-contained `place_search` FTS content: its `alt_names`
|
|
14
|
+
* column is the same alias set joined on the boundary-preserving `ALIAS_SEPARATOR` (#523), so `aliasBagExactMatch`
|
|
15
|
+
* recovers the exact alias tier ("New York City" → New York) that the dropped `names` table used to provide.
|
|
16
|
+
*/
|
|
17
|
+
export declare function exactMatchIDs(db: DatabaseSync, schemaName: string, ids: number[], text: string): Set<number>;
|
|
18
|
+
/**
|
|
19
|
+
* Among `ids` (already known exact matches), the subset holding `text` as an OFFICIAL name (`names.official = 1`, the
|
|
20
|
+
* #940 ingest bit). Same COLLATE NOCASE semantics as {@link WOFSQLitePlaceLookup.#exactMatchIDs} so the two probes
|
|
21
|
+
* agree on what "equals the query" means. Fails soft on gazetteers built before #940 (no `official` column) — the
|
|
22
|
+
* sub-tier then behaves exactly as if `officialNameExact` were off.
|
|
23
|
+
*/
|
|
24
|
+
export declare function officialNameIDs(db: DatabaseSync, schemaName: string, ids: number[], text: string): Set<number>;
|
|
25
|
+
//# sourceMappingURL=exact-match.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"exact-match.d.ts","sourceRoot":"","sources":["../exact-match.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAI/C;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,EAAE,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAqD5G;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,EAAE,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAsB9G"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* The exact-match tier's name probes — the two case-folded equality lookups that decide whether a
|
|
7
|
+
* candidate holds the query text as its own name, an alias, or an official name.
|
|
8
|
+
*/
|
|
9
|
+
import { aliasBagExactMatch } from "./fts.js";
|
|
10
|
+
/**
|
|
11
|
+
* Among `ids`, return the subset whose name OR any alias equals `text` case-insensitively — the exact-match tier for
|
|
12
|
+
* ranking. One indexed query over `<schema>.names`. When the shard has no `names` table (a slim DB built with
|
|
13
|
+
* `dropNames`, or a postcode-only shard), fall back to the self-contained `place_search` FTS content: its `alt_names`
|
|
14
|
+
* column is the same alias set joined on the boundary-preserving `ALIAS_SEPARATOR` (#523), so `aliasBagExactMatch`
|
|
15
|
+
* recovers the exact alias tier ("New York City" → New York) that the dropped `names` table used to provide.
|
|
16
|
+
*/
|
|
17
|
+
export function exactMatchIDs(db, schemaName, ids, text) {
|
|
18
|
+
const out = new Set();
|
|
19
|
+
const trimmed = text.trim();
|
|
20
|
+
if (!ids.length || !trimmed)
|
|
21
|
+
return out;
|
|
22
|
+
const placeholders = ids.map(() => "?").join(", ");
|
|
23
|
+
try {
|
|
24
|
+
const rows = db
|
|
25
|
+
.prepare(`SELECT DISTINCT id FROM ${schemaName}.names WHERE id IN (${placeholders}) AND name = ? COLLATE NOCASE`)
|
|
26
|
+
.all(...ids, trimmed);
|
|
27
|
+
for (const r of rows) {
|
|
28
|
+
out.add(r.id);
|
|
29
|
+
}
|
|
30
|
+
return out;
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
// No `names` table on this shard — fall through to the place_search alias bag.
|
|
34
|
+
}
|
|
35
|
+
try {
|
|
36
|
+
const rows = db
|
|
37
|
+
.prepare(`SELECT wof_id AS id, name, alt_names FROM ${schemaName}.place_search WHERE wof_id IN (${placeholders})`)
|
|
38
|
+
.all(...ids);
|
|
39
|
+
const norm = (s) => s.toLowerCase().trim().replaceAll(/\s+/g, " ");
|
|
40
|
+
const needle = norm(trimmed);
|
|
41
|
+
for (const r of rows) {
|
|
42
|
+
if (r.name !== null && norm(r.name) === needle) {
|
|
43
|
+
out.add(r.id);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
// Alias pass via the shared bag parser (#523). Separated bags (built since #523) get a true
|
|
47
|
+
// per-alias equality check, ungated — matching the `names`-table branch above, where an
|
|
48
|
+
// alias match counts as exact regardless of other candidates. Legacy bags (no separator)
|
|
49
|
+
// fall back to padded containment, gated on "no canonical exact in the pool" because their
|
|
50
|
+
// lost boundaries would otherwise false-promote interior fragments ("York" inside the alias
|
|
51
|
+
// "New York City") or cross-alias fragments ("York New" across "…York" + "New City…").
|
|
52
|
+
const anyCanonicalExact = out.size > 0;
|
|
53
|
+
for (const r of rows) {
|
|
54
|
+
if (aliasBagExactMatch(r.alt_names, needle, anyCanonicalExact)) {
|
|
55
|
+
out.add(r.id);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
// Shard without place_search either → no exact-match tier. Falls back to weighted-sum order.
|
|
61
|
+
}
|
|
62
|
+
return out;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Among `ids` (already known exact matches), the subset holding `text` as an OFFICIAL name (`names.official = 1`, the
|
|
66
|
+
* #940 ingest bit). Same COLLATE NOCASE semantics as {@link WOFSQLitePlaceLookup.#exactMatchIDs} so the two probes
|
|
67
|
+
* agree on what "equals the query" means. Fails soft on gazetteers built before #940 (no `official` column) — the
|
|
68
|
+
* sub-tier then behaves exactly as if `officialNameExact` were off.
|
|
69
|
+
*/
|
|
70
|
+
export function officialNameIDs(db, schemaName, ids, text) {
|
|
71
|
+
const out = new Set();
|
|
72
|
+
const trimmed = text.trim();
|
|
73
|
+
if (!ids.length || !trimmed)
|
|
74
|
+
return out;
|
|
75
|
+
const placeholders = ids.map(() => "?").join(", ");
|
|
76
|
+
try {
|
|
77
|
+
const rows = db
|
|
78
|
+
.prepare(`SELECT DISTINCT id FROM ${schemaName}.names WHERE id IN (${placeholders}) AND official = 1 AND name = ? COLLATE NOCASE`)
|
|
79
|
+
.all(...ids, trimmed);
|
|
80
|
+
for (const r of rows) {
|
|
81
|
+
out.add(r.id);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
catch {
|
|
85
|
+
// Pre-#940 gazetteer (no `official` column) or a names-less slim shard — feature inert.
|
|
86
|
+
}
|
|
87
|
+
return out;
|
|
88
|
+
}
|
|
89
|
+
//# sourceMappingURL=exact-match.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"exact-match.js","sourceRoot":"","sources":["../exact-match.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAA;AAE7C;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,EAAgB,EAAE,UAAkB,EAAE,GAAa,EAAE,IAAY;IAC9F,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAA;IAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;IAE3B,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO;QAAE,OAAO,GAAG,CAAA;IACvC,MAAM,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAElD,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,EAAE;aACb,OAAO,CAAC,2BAA2B,UAAU,uBAAuB,YAAY,+BAA+B,CAAC;aAChH,GAAG,CAAC,GAAG,GAAG,EAAE,OAAO,CAA0B,CAAA;QAE/C,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACtB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;QACd,CAAC;QAED,OAAO,GAAG,CAAA;IACX,CAAC;IAAC,MAAM,CAAC;QACR,+EAA+E;IAChF,CAAC;IAED,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,EAAE;aACb,OAAO,CAAC,6CAA6C,UAAU,kCAAkC,YAAY,GAAG,CAAC;aACjH,GAAG,CAAC,GAAG,GAAG,CAAyE,CAAA;QAErF,MAAM,IAAI,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QAClF,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,CAAA;QAE5B,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACtB,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,MAAM,EAAE,CAAC;gBAChD,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YACd,CAAC;QACF,CAAC;QAED,4FAA4F;QAC5F,wFAAwF;QACxF,yFAAyF;QACzF,2FAA2F;QAC3F,4FAA4F;QAC5F,uFAAuF;QACvF,MAAM,iBAAiB,GAAG,GAAG,CAAC,IAAI,GAAG,CAAC,CAAA;QAEtC,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACtB,IAAI,kBAAkB,CAAC,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,iBAAiB,CAAC,EAAE,CAAC;gBAChE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YACd,CAAC;QACF,CAAC;IACF,CAAC;IAAC,MAAM,CAAC;QACR,6FAA6F;IAC9F,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,EAAgB,EAAE,UAAkB,EAAE,GAAa,EAAE,IAAY;IAChG,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAA;IAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;IAE3B,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO;QAAE,OAAO,GAAG,CAAA;IACvC,MAAM,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAElD,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,EAAE;aACb,OAAO,CACP,2BAA2B,UAAU,uBAAuB,YAAY,gDAAgD,CACxH;aACA,GAAG,CAAC,GAAG,GAAG,EAAE,OAAO,CAA0B,CAAA;QAE/C,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACtB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;QACd,CAAC;IACF,CAAC;IAAC,MAAM,CAAC;QACR,wFAAwF;IACzF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}
|
|
@@ -3,18 +3,18 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
* FST-based autocomplete
|
|
7
|
-
*
|
|
6
|
+
* FST-based autocomplete — mailwoman vocabulary over `@mailwoman/ancestrie`'s generic algorithm
|
|
7
|
+
* (#1728 phase 2). The #587 behavior — prefix walk + BFS expansion, partial-last-token completion,
|
|
8
|
+
* per-branch capping, dedupe — lives in ancestrie's `autocomplete`; this module contributes only
|
|
9
|
+
* the storage adapter ({@link FSTMatcher} → `AncestrieReaderLike`) and the mapping back to
|
|
10
|
+
* mailwoman's suggestion shape (name, placetype, referential/encyclopedic, WOF ids).
|
|
8
11
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* -
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
* walk the complete prefix, then complete the partial token by prefix-filtering the
|
|
16
|
-
* continuation edges (`token.startsWith(partial)`). This is what a char-level typeahead
|
|
17
|
-
* needs; without it "new yor" returns nothing useful. (#587)
|
|
12
|
+
* THE BYTES DO NOT MIGRATE. The shipped artifacts are `FST\0` v1–v5 (`fst-serialize.ts`), not
|
|
13
|
+
* ancestrie's `ANCT`: ancestrie entries are id-keyed with one record per id, while an FST place row
|
|
14
|
+
* is per-(surface, place) — `crossCountryBranches` is a property of the SURFACE, so the same wofID
|
|
15
|
+
* legitimately carries different values under different aliases and cannot be represented id-keyed.
|
|
16
|
+
* The matcher, both deserializers, and the serializer therefore stay here; what migrated is the
|
|
17
|
+
* ALGORITHM, which is the half that drifts (the #861 share-the-function rule).
|
|
18
18
|
*/
|
|
19
19
|
import type { FSTMatcher } from "./fst-matcher.ts";
|
|
20
20
|
export interface AutocompleteResult {
|
|
@@ -26,7 +26,17 @@ export interface AutocompleteResult {
|
|
|
26
26
|
export interface AutocompleteSuggestion {
|
|
27
27
|
name: string;
|
|
28
28
|
placetype: string;
|
|
29
|
-
|
|
29
|
+
/**
|
|
30
|
+
* The REFERENTIAL likelihood the suggestion is ranked by (ROAD_TO_V9 §2). Autocomplete answers "which place does the
|
|
31
|
+
* user mean", so it ranks referentially like everything else; encyclopedic importance rides along on
|
|
32
|
+
* {@link AutocompleteSuggestion.encyclopedic} for display and never enters the order.
|
|
33
|
+
*/
|
|
34
|
+
referential: number;
|
|
35
|
+
/**
|
|
36
|
+
* Encyclopedic (Wikipedia) importance, when the FST artifact carries one for this place. `undefined` = no article, or
|
|
37
|
+
* a pre-v5 binary — never 0.
|
|
38
|
+
*/
|
|
39
|
+
encyclopedic?: number;
|
|
30
40
|
wofID: number;
|
|
31
41
|
parentChain: number[];
|
|
32
42
|
matchDepth: number;
|
|
@@ -36,14 +46,14 @@ export interface AutocompleteOpts {
|
|
|
36
46
|
maxSuggestions?: number;
|
|
37
47
|
maxExpansionDepth?: number;
|
|
38
48
|
/**
|
|
39
|
-
* Collapse same-name suggestions to the single highest-
|
|
49
|
+
* Collapse same-name suggestions to the single highest-referential one. Off by default (the CLI surfaces distinct
|
|
40
50
|
* same-name places — New York the city vs the county); a typeahead wants it ON so the dropdown isn't four "New
|
|
41
51
|
* London"s. (#587)
|
|
42
52
|
*/
|
|
43
53
|
dedupeByName?: boolean;
|
|
44
54
|
}
|
|
45
55
|
/**
|
|
46
|
-
* Autocomplete from the current prefix. Returns suggestions ranked
|
|
56
|
+
* Autocomplete from the current prefix. Returns suggestions ranked referential-descending.
|
|
47
57
|
*/
|
|
48
58
|
export declare function autocomplete(fst: FSTMatcher, query: string, opts?: AutocompleteOpts): AutocompleteResult;
|
|
49
59
|
//# sourceMappingURL=fst-autocomplete.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fst-autocomplete.d.ts","sourceRoot":"","sources":["../fst-autocomplete.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;
|
|
1
|
+
{"version":3,"file":"fst-autocomplete.d.ts","sourceRoot":"","sources":["../fst-autocomplete.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAWH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAIlD,MAAM,WAAW,kBAAkB;IAClC,KAAK,EAAE,MAAM,CAAA;IACb,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,EAAE,sBAAsB,EAAE,CAAA;CACrC;AAED,MAAM,WAAW,sBAAsB;IACtC,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,MAAM,CAAA;IACjB;;;;OAIG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,UAAU,EAAE,MAAM,CAAA;IAClB,gBAAgB,EAAE,MAAM,EAAE,CAAA;CAC1B;AAED,MAAM,WAAW,gBAAgB;IAChC,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B;;;;OAIG;IACH,YAAY,CAAC,EAAE,OAAO,CAAA;CACtB;AAyED;;GAEG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,gBAAqB,GAAG,kBAAkB,CA+B5G"}
|
package/out/fst-autocomplete.js
CHANGED
|
@@ -3,127 +3,111 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
* FST-based autocomplete
|
|
7
|
-
*
|
|
6
|
+
* FST-based autocomplete — mailwoman vocabulary over `@mailwoman/ancestrie`'s generic algorithm
|
|
7
|
+
* (#1728 phase 2). The #587 behavior — prefix walk + BFS expansion, partial-last-token completion,
|
|
8
|
+
* per-branch capping, dedupe — lives in ancestrie's `autocomplete`; this module contributes only
|
|
9
|
+
* the storage adapter ({@link FSTMatcher} → `AncestrieReaderLike`) and the mapping back to
|
|
10
|
+
* mailwoman's suggestion shape (name, placetype, referential/encyclopedic, WOF ids).
|
|
8
11
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* -
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
* walk the complete prefix, then complete the partial token by prefix-filtering the
|
|
16
|
-
* continuation edges (`token.startsWith(partial)`). This is what a char-level typeahead
|
|
17
|
-
* needs; without it "new yor" returns nothing useful. (#587)
|
|
12
|
+
* THE BYTES DO NOT MIGRATE. The shipped artifacts are `FST\0` v1–v5 (`fst-serialize.ts`), not
|
|
13
|
+
* ancestrie's `ANCT`: ancestrie entries are id-keyed with one record per id, while an FST place row
|
|
14
|
+
* is per-(surface, place) — `crossCountryBranches` is a property of the SURFACE, so the same wofID
|
|
15
|
+
* legitimately carries different values under different aliases and cannot be represented id-keyed.
|
|
16
|
+
* The matcher, both deserializers, and the serializer therefore stay here; what migrated is the
|
|
17
|
+
* ALGORITHM, which is the half that drifts (the #861 share-the-function rule).
|
|
18
18
|
*/
|
|
19
|
+
import { autocomplete as ancestrieAutocomplete } from "@mailwoman/ancestrie";
|
|
19
20
|
import { normalizeTokens } from "./fst-matcher.js";
|
|
20
21
|
/**
|
|
21
22
|
* Max accepting entries collected per BFS branch — keeps one dense branch from starving the search.
|
|
22
23
|
*/
|
|
23
24
|
const PER_BRANCH = 4;
|
|
24
25
|
/**
|
|
25
|
-
* The top-`k` entries by
|
|
26
|
+
* The top-`k` entries by REFERENTIAL likelihood (descending). Avoids sorting/allocating when `entries` is small — and
|
|
27
|
+
* that shortcut is part of the observable contract: at or under `k` the INSERTION order is served, which decides
|
|
28
|
+
* suggestion order among referential ties.
|
|
26
29
|
*/
|
|
27
|
-
function
|
|
30
|
+
function topByReferential(entries, k) {
|
|
28
31
|
if (entries.length <= k)
|
|
29
32
|
return [...entries];
|
|
30
|
-
return [...entries].toSorted((a, b) => b.
|
|
33
|
+
return [...entries].toSorted((a, b) => b.referential - a.referential).slice(0, k);
|
|
31
34
|
}
|
|
32
35
|
/**
|
|
33
|
-
*
|
|
36
|
+
* {@link FSTMatcher} presented through ancestrie's storage seam. Records carry the {@link PlaceEntry} itself as the
|
|
37
|
+
* payload, so the entry that WINS the algorithm's shallowest-depth rule is the entry whose fields the suggestion
|
|
38
|
+
* reports — a side lookup keyed on id could pick a different surface's row (`crossCountryBranches` differs per
|
|
39
|
+
* surface).
|
|
34
40
|
*/
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
+
class FSTReader {
|
|
42
|
+
#fst;
|
|
43
|
+
/**
|
|
44
|
+
* Parent chains of the entries this reader has served, id-keyed. A place's chain is identical across its surfaces (it
|
|
45
|
+
* is place-row data), so last-write-wins is safe. The algorithm asks {@link FSTReader.ancestorsOf} only for ids it
|
|
46
|
+
* just received from {@link FSTReader.entriesAt}, so serving from this memo answers every real call without an
|
|
47
|
+
* artifact-wide id index.
|
|
48
|
+
*/
|
|
49
|
+
#chains = new Map();
|
|
50
|
+
constructor(fst) {
|
|
51
|
+
this.#fst = fst;
|
|
41
52
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
let depth;
|
|
45
|
-
const match = fst.walk(normalizedTokens);
|
|
46
|
-
if (match) {
|
|
47
|
-
// COMPLETE-token prefix landed on a state. Seed at the match state (accepting + continuations).
|
|
48
|
-
depth = match.depth;
|
|
49
|
-
for (const entry of fst.accepting(match.stateID)) {
|
|
50
|
-
addSuggestion(seen, entry, match.depth, []);
|
|
51
|
-
}
|
|
52
|
-
for (const cont of fst.continuations(match.stateID)) {
|
|
53
|
-
queue.push({ stateID: cont.targetState, depth: 1, tokens: [cont.token] });
|
|
54
|
-
}
|
|
53
|
+
walk(tokens) {
|
|
54
|
+
return this.#fst.walk([...tokens]);
|
|
55
55
|
}
|
|
56
|
-
|
|
57
|
-
//
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
}
|
|
64
|
-
depth = complete.length;
|
|
65
|
-
for (const cont of fst.continuations(prefixState)) {
|
|
66
|
-
if (!cont.token.startsWith(partial))
|
|
67
|
-
continue;
|
|
68
|
-
// This edge completes the typed partial token — its target is a real match at depth+1.
|
|
69
|
-
for (const entry of topByImportance(fst.accepting(cont.targetState), PER_BRANCH)) {
|
|
70
|
-
addSuggestion(seen, entry, complete.length + 1, [cont.token]);
|
|
71
|
-
}
|
|
72
|
-
// BFS a little past it too (multi-token completions: "new yor" → "New York Mills").
|
|
73
|
-
queue.push({ stateID: cont.targetState, depth: 1, tokens: [cont.token] });
|
|
74
|
-
}
|
|
56
|
+
continuations(stateID) {
|
|
57
|
+
// Insertion order, verbatim — BFS visit order under the suggestion budget depends on it.
|
|
58
|
+
return this.#fst.continuations(stateID).map((c) => ({
|
|
59
|
+
token: c.token,
|
|
60
|
+
targetState: c.targetState,
|
|
61
|
+
entryCount: c.acceptingCount,
|
|
62
|
+
}));
|
|
75
63
|
}
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
for (const cont of fst.continuations(item.stateID)) {
|
|
89
|
-
queue.push({ stateID: cont.targetState, depth: item.depth + 1, tokens: [...item.tokens, cont.token] });
|
|
90
|
-
}
|
|
91
|
-
}
|
|
64
|
+
entriesAt(stateID, limit) {
|
|
65
|
+
const places = this.#fst.accepting(stateID);
|
|
66
|
+
const selected = limit === undefined ? places : topByReferential(places, limit);
|
|
67
|
+
return selected.map((entry) => {
|
|
68
|
+
this.#chains.set(entry.wofID, entry.parentChain);
|
|
69
|
+
return {
|
|
70
|
+
id: entry.wofID,
|
|
71
|
+
rank: entry.referential,
|
|
72
|
+
parentIDs: entry.parentChain,
|
|
73
|
+
payload: entry,
|
|
74
|
+
};
|
|
75
|
+
});
|
|
92
76
|
}
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
suggestions = dedupeByName(suggestions);
|
|
77
|
+
ancestorsOf(id) {
|
|
78
|
+
return this.#chains.get(id) ?? [];
|
|
96
79
|
}
|
|
97
|
-
return { query, normalizedTokens, depth, suggestions: suggestions.slice(0, maxSuggestions) };
|
|
98
|
-
}
|
|
99
|
-
function addSuggestion(seen, entry, matchDepth, completionTokens) {
|
|
100
|
-
const existing = seen.get(entry.wofID);
|
|
101
|
-
if (existing && existing.matchDepth <= matchDepth)
|
|
102
|
-
return;
|
|
103
|
-
seen.set(entry.wofID, {
|
|
104
|
-
name: entry.name,
|
|
105
|
-
placetype: entry.placetype,
|
|
106
|
-
importance: entry.importance,
|
|
107
|
-
wofID: entry.wofID,
|
|
108
|
-
parentChain: entry.parentChain,
|
|
109
|
-
matchDepth,
|
|
110
|
-
completionTokens: [...completionTokens],
|
|
111
|
-
});
|
|
112
80
|
}
|
|
113
81
|
/**
|
|
114
|
-
*
|
|
115
|
-
* per name wins; order is preserved.
|
|
82
|
+
* Autocomplete from the current prefix. Returns suggestions ranked referential-descending.
|
|
116
83
|
*/
|
|
117
|
-
function
|
|
118
|
-
const
|
|
119
|
-
const
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
84
|
+
export function autocomplete(fst, query, opts = {}) {
|
|
85
|
+
const normalizedTokens = normalizeTokens(query);
|
|
86
|
+
const result = ancestrieAutocomplete(new FSTReader(fst), normalizedTokens, {
|
|
87
|
+
...(opts.maxSuggestions === undefined ? {} : { maxSuggestions: opts.maxSuggestions }),
|
|
88
|
+
...(opts.maxExpansionDepth === undefined ? {} : { maxExpansionDepth: opts.maxExpansionDepth }),
|
|
89
|
+
perBranchLimit: PER_BRANCH,
|
|
90
|
+
// The dedupe key is the DISPLAY name, not the token path: two surfaces of one name must still collapse. (#587)
|
|
91
|
+
...(opts.dedupeByName ? { dedupe: (s) => s.payload.name.toLowerCase() } : {}),
|
|
92
|
+
});
|
|
93
|
+
return {
|
|
94
|
+
query,
|
|
95
|
+
normalizedTokens,
|
|
96
|
+
depth: result.depth,
|
|
97
|
+
suggestions: result.suggestions.map((s) => {
|
|
98
|
+
// Every record this adapter serves carries its entry; the assertion documents the invariant.
|
|
99
|
+
const entry = s.payload;
|
|
100
|
+
return {
|
|
101
|
+
name: entry.name,
|
|
102
|
+
placetype: entry.placetype,
|
|
103
|
+
referential: entry.referential,
|
|
104
|
+
...(entry.encyclopedic === undefined ? {} : { encyclopedic: entry.encyclopedic }),
|
|
105
|
+
wofID: s.id,
|
|
106
|
+
parentChain: s.parentIDs,
|
|
107
|
+
matchDepth: s.matchDepth,
|
|
108
|
+
completionTokens: s.completionTokens,
|
|
109
|
+
};
|
|
110
|
+
}),
|
|
111
|
+
};
|
|
128
112
|
}
|
|
129
113
|
//# sourceMappingURL=fst-autocomplete.js.map
|