@mailwoman/resolver-wof-sqlite 8.1.0 → 8.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/address-point-interpolation.ts +9 -3
- package/address-point-schema.ts +32 -10
- package/address-point.ts +3 -0
- package/ancestry-backfill.ts +18 -5
- package/ancestry.ts +7 -2
- package/build-candidate.ts +29 -6
- package/build-slim.ts +40 -11
- package/candidate-fts.ts +1 -0
- package/candidate-lookup.ts +36 -14
- package/candidate-schema.ts +35 -11
- package/coincident-roles.ts +28 -6
- package/convention.ts +3 -1
- package/coverage-manifest-schema.ts +49 -16
- package/fst-autocomplete.ts +15 -9
- package/fst-builder.ts +29 -5
- package/fst-deserialize-web.ts +46 -9
- package/fst-matcher.ts +12 -5
- package/fst-serialize.ts +83 -9
- package/fst-types.ts +26 -3
- package/fts-query.ts +84 -0
- package/fts.ts +35 -9
- package/geo.ts +9 -3
- package/geonames-aliases.ts +116 -79
- package/geonames-postal.ts +25 -5
- package/index.ts +10 -0
- package/interpolation.ts +33 -55
- package/lookup.ts +103 -292
- package/name-score.ts +76 -0
- package/out/address-point-interpolation.d.ts.map +1 -1
- package/out/address-point-interpolation.js +4 -2
- package/out/address-point-interpolation.js.map +1 -1
- package/out/address-point-schema.d.ts +30 -10
- package/out/address-point-schema.d.ts.map +1 -1
- package/out/address-point-schema.js +6 -2
- package/out/address-point-schema.js.map +1 -1
- package/out/address-point.d.ts.map +1 -1
- package/out/address-point.js.map +1 -1
- package/out/ancestry-backfill.d.ts +6 -2
- package/out/ancestry-backfill.d.ts.map +1 -1
- package/out/ancestry-backfill.js +7 -3
- package/out/ancestry-backfill.js.map +1 -1
- package/out/ancestry.d.ts +6 -2
- package/out/ancestry.d.ts.map +1 -1
- package/out/ancestry.js +3 -1
- package/out/ancestry.js.map +1 -1
- package/out/build-candidate.d.ts +9 -3
- package/out/build-candidate.d.ts.map +1 -1
- package/out/build-candidate.js +5 -3
- package/out/build-candidate.js.map +1 -1
- package/out/build-slim.d.ts +15 -5
- package/out/build-slim.d.ts.map +1 -1
- package/out/build-slim.js +11 -5
- package/out/build-slim.js.map +1 -1
- package/out/candidate-fts.d.ts.map +1 -1
- package/out/candidate-fts.js.map +1 -1
- package/out/candidate-lookup.d.ts +9 -3
- package/out/candidate-lookup.d.ts.map +1 -1
- package/out/candidate-lookup.js +16 -11
- package/out/candidate-lookup.js.map +1 -1
- package/out/candidate-schema.d.ts +33 -11
- package/out/candidate-schema.d.ts.map +1 -1
- package/out/candidate-schema.js.map +1 -1
- package/out/coincident-roles.d.ts +16 -4
- package/out/coincident-roles.d.ts.map +1 -1
- package/out/coincident-roles.js +9 -3
- package/out/coincident-roles.js.map +1 -1
- package/out/convention.d.ts +3 -1
- package/out/convention.d.ts.map +1 -1
- package/out/convention.js.map +1 -1
- package/out/coverage-manifest-schema.d.ts +45 -14
- package/out/coverage-manifest-schema.d.ts.map +1 -1
- package/out/coverage-manifest-schema.js +14 -5
- package/out/coverage-manifest-schema.js.map +1 -1
- package/out/fst-autocomplete.d.ts +1 -1
- package/out/fst-autocomplete.d.ts.map +1 -1
- package/out/fst-autocomplete.js +11 -9
- package/out/fst-autocomplete.js.map +1 -1
- package/out/fst-builder.d.ts.map +1 -1
- package/out/fst-builder.js +15 -5
- package/out/fst-builder.js.map +1 -1
- package/out/fst-deserialize-web.d.ts.map +1 -1
- package/out/fst-deserialize-web.js +34 -9
- package/out/fst-deserialize-web.js.map +1 -1
- package/out/fst-matcher.d.ts +6 -2
- package/out/fst-matcher.d.ts.map +1 -1
- package/out/fst-matcher.js +9 -5
- package/out/fst-matcher.js.map +1 -1
- package/out/fst-serialize.d.ts.map +1 -1
- package/out/fst-serialize.js +62 -9
- package/out/fst-serialize.js.map +1 -1
- package/out/fst-types.d.ts +26 -3
- package/out/fst-types.d.ts.map +1 -1
- package/out/fts-query.d.ts +41 -0
- package/out/fts-query.d.ts.map +1 -0
- package/out/fts-query.js +75 -0
- package/out/fts-query.js.map +1 -0
- package/out/fts.d.ts +21 -7
- package/out/fts.d.ts.map +1 -1
- package/out/fts.js +10 -4
- package/out/fts.js.map +1 -1
- package/out/geo.d.ts +6 -2
- package/out/geo.d.ts.map +1 -1
- package/out/geo.js +3 -1
- package/out/geo.js.map +1 -1
- package/out/geonames-aliases.d.ts +12 -4
- package/out/geonames-aliases.d.ts.map +1 -1
- package/out/geonames-aliases.js +72 -67
- package/out/geonames-aliases.js.map +1 -1
- package/out/geonames-postal.d.ts +9 -3
- package/out/geonames-postal.d.ts.map +1 -1
- package/out/geonames-postal.js +7 -2
- package/out/geonames-postal.js.map +1 -1
- package/out/index.d.ts.map +1 -1
- package/out/index.js.map +1 -1
- package/out/interpolation.d.ts +18 -6
- package/out/interpolation.d.ts.map +1 -1
- package/out/interpolation.js +11 -40
- package/out/interpolation.js.map +1 -1
- package/out/lookup.d.ts +3 -97
- package/out/lookup.d.ts.map +1 -1
- package/out/lookup.js +52 -184
- package/out/lookup.js.map +1 -1
- package/out/name-score.d.ts +28 -0
- package/out/name-score.d.ts.map +1 -0
- package/out/name-score.js +67 -0
- package/out/name-score.js.map +1 -0
- package/out/poi-lookup.d.ts +24 -8
- package/out/poi-lookup.d.ts.map +1 -1
- package/out/poi-lookup.js +27 -13
- package/out/poi-lookup.js.map +1 -1
- package/out/poi-schema.d.ts +42 -13
- package/out/poi-schema.d.ts.map +1 -1
- package/out/poi-schema.js +12 -3
- package/out/poi-schema.js.map +1 -1
- package/out/postal-city-alias-lookup.d.ts +18 -6
- package/out/postal-city-alias-lookup.d.ts.map +1 -1
- package/out/postal-city-alias-lookup.js.map +1 -1
- package/out/postal-city-alias-schema.d.ts +27 -9
- package/out/postal-city-alias-schema.d.ts.map +1 -1
- package/out/postal-city-alias-schema.js +3 -1
- package/out/postal-city-alias-schema.js.map +1 -1
- package/out/postal-city-candidate-schema.d.ts +15 -5
- package/out/postal-city-candidate-schema.d.ts.map +1 -1
- package/out/postal-city-candidate-schema.js +3 -1
- package/out/postal-city-candidate-schema.js.map +1 -1
- package/out/postcode-point-lookup.d.ts +6 -2
- package/out/postcode-point-lookup.d.ts.map +1 -1
- package/out/postcode-point-lookup.js +6 -2
- package/out/postcode-point-lookup.js.map +1 -1
- package/out/ranking-weights.d.ts +118 -0
- package/out/ranking-weights.d.ts.map +1 -0
- package/out/ranking-weights.js +44 -0
- package/out/ranking-weights.js.map +1 -0
- package/out/reverse.d.ts +9 -3
- package/out/reverse.d.ts.map +1 -1
- package/out/reverse.js +20 -6
- package/out/reverse.js.map +1 -1
- package/out/sharding.d.ts +3 -1
- package/out/sharding.d.ts.map +1 -1
- package/out/sharding.js +7 -5
- package/out/sharding.js.map +1 -1
- package/out/sqlite-convention-source.d.ts.map +1 -1
- package/out/sqlite-convention-source.js +3 -1
- package/out/sqlite-convention-source.js.map +1 -1
- package/out/street-centroid-schema.d.ts +33 -11
- package/out/street-centroid-schema.d.ts.map +1 -1
- package/out/street-centroid-schema.js +3 -1
- package/out/street-centroid-schema.js.map +1 -1
- package/out/street-centroid.d.ts.map +1 -1
- package/out/street-centroid.js +6 -2
- package/out/street-centroid.js.map +1 -1
- package/out/street-morphology-fst-builder.d.ts +6 -2
- package/out/street-morphology-fst-builder.d.ts.map +1 -1
- package/out/street-morphology-fst-builder.js +8 -7
- package/out/street-morphology-fst-builder.js.map +1 -1
- package/out/street-morphology-fst-loader.d.ts +24 -8
- package/out/street-morphology-fst-loader.d.ts.map +1 -1
- package/out/street-morphology-fst-loader.js +11 -5
- package/out/street-morphology-fst-loader.js.map +1 -1
- package/out/street-name-lookup.d.ts +9 -3
- package/out/street-name-lookup.d.ts.map +1 -1
- package/out/street-name-lookup.js +9 -7
- package/out/street-name-lookup.js.map +1 -1
- package/out/street-normalize.d.ts +3 -1
- package/out/street-normalize.d.ts.map +1 -1
- package/out/street-normalize.js +23 -13
- package/out/street-normalize.js.map +1 -1
- package/out/street-segment-schema.d.ts +48 -16
- package/out/street-segment-schema.d.ts.map +1 -1
- package/out/street-segment-schema.js +6 -2
- package/out/street-segment-schema.js.map +1 -1
- package/out/types.d.ts +18 -6
- package/out/types.d.ts.map +1 -1
- package/out/unified-schema.d.ts +1 -1
- package/out/unified-schema.d.ts.map +1 -1
- package/out/unified-schema.js +2 -2
- package/out/unified-schema.js.map +1 -1
- package/package.json +5 -5
- package/poi-lookup.ts +53 -21
- package/poi-schema.ts +43 -13
- package/postal-city-alias-lookup.ts +20 -6
- package/postal-city-alias-schema.ts +28 -9
- package/postal-city-candidate-schema.ts +15 -5
- package/postcode-point-lookup.ts +6 -2
- package/ranking-weights.ts +148 -0
- package/reverse.ts +47 -10
- package/sharding.ts +13 -6
- package/sqlite-convention-source.ts +4 -1
- package/street-centroid-schema.ts +35 -11
- package/street-centroid.ts +10 -3
- package/street-morphology-fst-builder.ts +25 -9
- package/street-morphology-fst-loader.ts +26 -10
- package/street-name-lookup.ts +19 -7
- package/street-normalize.ts +28 -13
- package/street-segment-schema.ts +50 -16
- package/types.ts +18 -6
- package/unified-schema.ts +11 -2
package/fts.ts
CHANGED
|
@@ -66,7 +66,9 @@ export const PLACE_SEARCH_TABLE = "place_search"
|
|
|
66
66
|
*/
|
|
67
67
|
export const ALIAS_SEPARATOR = "\uE000"
|
|
68
68
|
|
|
69
|
-
/**
|
|
69
|
+
/**
|
|
70
|
+
* `char()` argument for {@link ALIAS_SEPARATOR} in SQL — keeps the SQL text plain ASCII.
|
|
71
|
+
*/
|
|
70
72
|
const ALIAS_SEPARATOR_CODEPOINT = ALIAS_SEPARATOR.codePointAt(0) as number
|
|
71
73
|
|
|
72
74
|
/**
|
|
@@ -93,7 +95,7 @@ const ALIAS_SEPARATOR_CODEPOINT = ALIAS_SEPARATOR.codePointAt(0) as number
|
|
|
93
95
|
*/
|
|
94
96
|
export function aliasBagExactMatch(altNames: string | null, normalizedQuery: string, anyStrictExact: boolean): boolean {
|
|
95
97
|
if (altNames === null || altNames === "" || !normalizedQuery) return false
|
|
96
|
-
const norm = (s: string): string => s.toLowerCase().trim().
|
|
98
|
+
const norm = (s: string): string => s.toLowerCase().trim().replaceAll(/\s+/g, " ")
|
|
97
99
|
|
|
98
100
|
if (altNames.includes(ALIAS_SEPARATOR)) {
|
|
99
101
|
return altNames.split(ALIAS_SEPARATOR).some((alias) => norm(alias) === normalizedQuery)
|
|
@@ -126,15 +128,25 @@ export const PLACE_POPULATION_TABLE = "place_population"
|
|
|
126
128
|
* user.
|
|
127
129
|
*/
|
|
128
130
|
export interface BuildPlaceSearchFTSResult {
|
|
129
|
-
/**
|
|
131
|
+
/**
|
|
132
|
+
* Whether the FTS5 index was created (true) or already existed and was left alone (false).
|
|
133
|
+
*/
|
|
130
134
|
created: boolean
|
|
131
|
-
/**
|
|
135
|
+
/**
|
|
136
|
+
* Number of rows in the `place_search` table after the call.
|
|
137
|
+
*/
|
|
132
138
|
indexedRows: number
|
|
133
|
-
/**
|
|
139
|
+
/**
|
|
140
|
+
* Whether the R*Tree bbox index was created (true) or already existed (false).
|
|
141
|
+
*/
|
|
134
142
|
bboxCreated: boolean
|
|
135
|
-
/**
|
|
143
|
+
/**
|
|
144
|
+
* Number of rows in the `place_bbox` R*Tree after the call.
|
|
145
|
+
*/
|
|
136
146
|
bboxIndexedRows: number
|
|
137
|
-
/**
|
|
147
|
+
/**
|
|
148
|
+
* Wall-clock duration of the build step, in milliseconds.
|
|
149
|
+
*/
|
|
138
150
|
durationMs: number
|
|
139
151
|
}
|
|
140
152
|
|
|
@@ -183,6 +195,7 @@ export function buildPlaceSearchFTS(db: DatabaseSync, opts: BuildPlaceSearchFTSO
|
|
|
183
195
|
|
|
184
196
|
if (!ftsExisting || opts.drop) {
|
|
185
197
|
onProgress("creating")
|
|
198
|
+
|
|
186
199
|
db.exec(`
|
|
187
200
|
CREATE VIRTUAL TABLE ${PLACE_SEARCH_TABLE} USING fts5(
|
|
188
201
|
wof_id UNINDEXED,
|
|
@@ -191,7 +204,9 @@ export function buildPlaceSearchFTS(db: DatabaseSync, opts: BuildPlaceSearchFTSO
|
|
|
191
204
|
tokenize = 'unicode61 remove_diacritics 2'
|
|
192
205
|
);
|
|
193
206
|
`)
|
|
207
|
+
|
|
194
208
|
onProgress("populating")
|
|
209
|
+
|
|
195
210
|
// Excludes only definitively-not-current places. WOF's `is_current` carries TWO conventions:
|
|
196
211
|
// `-1` (modern Who's On First) and `1` (legacy Mapzen-era), both meaning "currently valid".
|
|
197
212
|
// Only `0` means "no longer current". Filtering on `= -1` strict (as Phase 4.2 did) excluded
|
|
@@ -220,8 +235,10 @@ export function buildPlaceSearchFTS(db: DatabaseSync, opts: BuildPlaceSearchFTSO
|
|
|
220
235
|
AND spr.is_deprecated = 0
|
|
221
236
|
AND spr.name IS NOT NULL;
|
|
222
237
|
`)
|
|
238
|
+
|
|
223
239
|
ftsCreated = true
|
|
224
240
|
}
|
|
241
|
+
|
|
225
242
|
const ftsCountRow = db.prepare(`SELECT COUNT(*) AS n FROM ${PLACE_SEARCH_TABLE}`).get() as { n: number }
|
|
226
243
|
|
|
227
244
|
// ─── R*Tree phase ────────────────────────────────────────────────
|
|
@@ -234,6 +251,7 @@ export function buildPlaceSearchFTS(db: DatabaseSync, opts: BuildPlaceSearchFTSO
|
|
|
234
251
|
|
|
235
252
|
if (!bboxExisting || opts.drop) {
|
|
236
253
|
onProgress("creating-bbox")
|
|
254
|
+
|
|
237
255
|
// R*Tree requires INTEGER PRIMARY KEY (id) + paired min/max for each indexed dimension.
|
|
238
256
|
// `rtree` (not `rtree_i32`) keeps coordinates as REAL — what we want for WGS-84.
|
|
239
257
|
db.exec(`
|
|
@@ -243,7 +261,9 @@ export function buildPlaceSearchFTS(db: DatabaseSync, opts: BuildPlaceSearchFTSO
|
|
|
243
261
|
min_lon, max_lon
|
|
244
262
|
);
|
|
245
263
|
`)
|
|
264
|
+
|
|
246
265
|
onProgress("populating-bbox")
|
|
266
|
+
|
|
247
267
|
// Only index places that have non-zero coordinates AND a real bbox. WOF stores both the
|
|
248
268
|
// centroid (latitude/longitude) and the bounding box (min_*/max_*). A subset of rows have
|
|
249
269
|
// all-zero coordinates — likely placeholders for deprecated / unmapped entries; the
|
|
@@ -266,8 +286,10 @@ export function buildPlaceSearchFTS(db: DatabaseSync, opts: BuildPlaceSearchFTSO
|
|
|
266
286
|
AND NOT (spr.min_latitude = 0 AND spr.max_latitude = 0
|
|
267
287
|
AND spr.min_longitude = 0 AND spr.max_longitude = 0);
|
|
268
288
|
`)
|
|
289
|
+
|
|
269
290
|
bboxCreated = true
|
|
270
291
|
}
|
|
292
|
+
|
|
271
293
|
const bboxCountRow = db.prepare(`SELECT COUNT(*) AS n FROM ${PLACE_BBOX_TABLE}`).get() as { n: number }
|
|
272
294
|
|
|
273
295
|
// NOTE: `place_population` is NOT built here. `scripts/build-unified-wof.ts` extracts
|
|
@@ -299,12 +321,16 @@ export function placeSearchFTSExists(db: DatabaseSync): boolean {
|
|
|
299
321
|
return tableExists(db, PLACE_SEARCH_TABLE)
|
|
300
322
|
}
|
|
301
323
|
|
|
302
|
-
/**
|
|
324
|
+
/**
|
|
325
|
+
* Returns true iff the `place_bbox` R*Tree table exists. Used for opt-in proximity lookup checks.
|
|
326
|
+
*/
|
|
303
327
|
export function placeBboxExists(db: DatabaseSync): boolean {
|
|
304
328
|
return tableExists(db, PLACE_BBOX_TABLE)
|
|
305
329
|
}
|
|
306
330
|
|
|
307
|
-
/**
|
|
331
|
+
/**
|
|
332
|
+
* Returns true iff the `place_population` table exists. Used for opt-in population-ranking checks.
|
|
333
|
+
*/
|
|
308
334
|
export function placePopulationExists(db: DatabaseSync): boolean {
|
|
309
335
|
return tableExists(db, PLACE_POPULATION_TABLE)
|
|
310
336
|
}
|
package/geo.ts
CHANGED
|
@@ -22,7 +22,9 @@
|
|
|
22
22
|
// readers keep importing it from "./geo.ts" (the spatial dep is transitive via @mailwoman/resolver).
|
|
23
23
|
export { haversineKm } from "@mailwoman/spatial"
|
|
24
24
|
|
|
25
|
-
/**
|
|
25
|
+
/**
|
|
26
|
+
* WGS-84 degrees → radians.
|
|
27
|
+
*/
|
|
26
28
|
function toRad(deg: number): number {
|
|
27
29
|
return (deg * Math.PI) / 180
|
|
28
30
|
}
|
|
@@ -61,10 +63,14 @@ export function bboxAround(lat: number, lon: number, radiusKm: number): Bbox {
|
|
|
61
63
|
*/
|
|
62
64
|
export type GeojsonPosition = [number, number, ...number[]]
|
|
63
65
|
|
|
64
|
-
/**
|
|
66
|
+
/**
|
|
67
|
+
* The two areal GeoJSON geometry types PIP can test, plus an open fallback (Point etc.).
|
|
68
|
+
*/
|
|
65
69
|
export interface GeojsonPolygon {
|
|
66
70
|
type: "Polygon"
|
|
67
|
-
/**
|
|
71
|
+
/**
|
|
72
|
+
* `[outerRing, hole1, hole2, …]` — each ring a closed list of positions.
|
|
73
|
+
*/
|
|
68
74
|
coordinates: GeojsonPosition[][]
|
|
69
75
|
}
|
|
70
76
|
|
package/geonames-aliases.ts
CHANGED
|
@@ -36,16 +36,115 @@ import { isOfficialLanguage } from "@mailwoman/codex/country"
|
|
|
36
36
|
*/
|
|
37
37
|
export const GEONAMES_ID_BASE = 9_000_000_000_000
|
|
38
38
|
|
|
39
|
-
/**
|
|
39
|
+
/**
|
|
40
|
+
* Per-country progress for the ingest — one event per country dump processed (or skipped).
|
|
41
|
+
*/
|
|
40
42
|
export interface GeonamesIngestProgress {
|
|
41
|
-
/**
|
|
43
|
+
/**
|
|
44
|
+
* ISO 3166-1 alpha-2 code.
|
|
45
|
+
*/
|
|
42
46
|
country: string
|
|
43
|
-
/**
|
|
47
|
+
/**
|
|
48
|
+
* Populated places ingested from this country's dump (0 when skipped).
|
|
49
|
+
*/
|
|
44
50
|
places: number
|
|
45
|
-
/**
|
|
51
|
+
/**
|
|
52
|
+
* True when the country's `<CC>.txt` dump was missing — the country is skipped, not fatal.
|
|
53
|
+
*/
|
|
46
54
|
skipped: boolean
|
|
47
55
|
}
|
|
48
56
|
|
|
57
|
+
/**
|
|
58
|
+
* One spelling's language attribution, as reconciled across every alternate-names row that carries it.
|
|
59
|
+
*/
|
|
60
|
+
interface V2Alias {
|
|
61
|
+
language: string
|
|
62
|
+
privateuse: string
|
|
63
|
+
official: number
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Parse a country's alternateNamesV2 file into `geonameid -> spelling -> attribution`, restricted to the populated
|
|
68
|
+
* places (`P`) present in `lines`.
|
|
69
|
+
*/
|
|
70
|
+
function parseAlternateNamesV2(v2File: string, cc: string, lines: string[]): Map<number, Map<string, V2Alias>> {
|
|
71
|
+
const wanted = new Set<number>()
|
|
72
|
+
|
|
73
|
+
for (const line of lines) {
|
|
74
|
+
const f = line.split("\t")
|
|
75
|
+
|
|
76
|
+
if (f[6] === "P") {
|
|
77
|
+
wanted.add(Number(f[0]))
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const v2 = new Map<number, Map<string, V2Alias>>()
|
|
82
|
+
|
|
83
|
+
// V2 columns (0-indexed): 1 geonameid, 2 isolanguage, 3 name, 4 isPreferredName, 5 isShortName,
|
|
84
|
+
// 6 isColloquial, 7 isHistoric, 8 from, 9 to.
|
|
85
|
+
//
|
|
86
|
+
// Two passes, because historic-ness is a fact about the NAME, not the row: GeoNames splits one
|
|
87
|
+
// spelling across rows — Malabo carries "Santa Isabel" as (es, unflagged) AND as (no-language,
|
|
88
|
+
// isHistoric=1, to=1973). Officialness must see the flags from EVERY row for the spelling, or the
|
|
89
|
+
// colonial-era name sails through on the language-tagged row (the #936 review's Malabo finding).
|
|
90
|
+
// Do NOT gate on isPreferredName instead — it's sparse annotation, not a signal (Turku's sv "Åbo"
|
|
91
|
+
// is unflagged; FI has 1,746 flags across the whole dump).
|
|
92
|
+
const v2Lines = readFileSync(v2File, "utf8").split("\n")
|
|
93
|
+
const historicNames = new Set<string>()
|
|
94
|
+
|
|
95
|
+
for (const line of v2Lines) {
|
|
96
|
+
if (!line) continue
|
|
97
|
+
const f = line.split("\t")
|
|
98
|
+
|
|
99
|
+
if (f[6] === "1" || f[7] === "1" || (f[9] ?? "").trim() !== "") {
|
|
100
|
+
const alt = (f[3] ?? "").trim()
|
|
101
|
+
|
|
102
|
+
if (alt && wanted.has(Number(f[1]))) {
|
|
103
|
+
historicNames.add(`${f[1]}|${alt}`)
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
for (const line of v2Lines) {
|
|
109
|
+
if (!line) continue
|
|
110
|
+
const f = line.split("\t")
|
|
111
|
+
const gid = Number(f[1])
|
|
112
|
+
|
|
113
|
+
if (!wanted.has(gid)) continue
|
|
114
|
+
const lang = f[2] ?? ""
|
|
115
|
+
|
|
116
|
+
// ISO 639 codes are 2-3 letters; GeoNames' pseudo-codes (post, link, iata, wkdt, …) are 4+.
|
|
117
|
+
if (!/^[a-z]{2,3}$/.test(lang)) continue
|
|
118
|
+
const alt = (f[3] ?? "").trim()
|
|
119
|
+
|
|
120
|
+
if (!alt) continue
|
|
121
|
+
const preferred = f[4] === "1"
|
|
122
|
+
const official = !historicNames.has(`${gid}|${alt}`) && isOfficialLanguage(cc, lang) ? 1 : 0
|
|
123
|
+
let byName = v2.get(gid)
|
|
124
|
+
|
|
125
|
+
if (!byName) {
|
|
126
|
+
v2.set(gid, (byName = new Map()))
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const prev = byName.get(alt)
|
|
130
|
+
|
|
131
|
+
if (!prev) {
|
|
132
|
+
byName.set(alt, { language: lang, privateuse: preferred ? "preferred" : "", official })
|
|
133
|
+
} else {
|
|
134
|
+
if (official && !prev.official) {
|
|
135
|
+
prev.language = lang
|
|
136
|
+
prev.official = 1
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
if (preferred && !prev.privateuse) {
|
|
140
|
+
prev.privateuse = "preferred"
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
return v2
|
|
146
|
+
}
|
|
147
|
+
|
|
49
148
|
/**
|
|
50
149
|
* Fold the GeoNames `P`-class places (+ their Latin alt-names) for `countries` into `db`'s `spr` / `names` /
|
|
51
150
|
* `place_population` tables. Returns the total places ingested.
|
|
@@ -82,18 +181,23 @@ export function ingestGeonamesAliases(
|
|
|
82
181
|
// Latin-only, no bracket/paren noise GeoNames packs into `alternatenames` ("(( Karis Landskommun ))",
|
|
83
182
|
// airport codes), 2–60 chars, at least one letter (drops bare postcodes/numbers).
|
|
84
183
|
const LATIN_NAME = /^[\p{Script=Latin}\p{M}\s\-'.]{2,60}$/u
|
|
184
|
+
|
|
85
185
|
const clean = (s: string): string | null => {
|
|
86
186
|
const t = s.trim()
|
|
87
187
|
|
|
88
188
|
return t && LATIN_NAME.test(t) && /\p{L}/u.test(t) ? t : null
|
|
89
189
|
}
|
|
190
|
+
|
|
90
191
|
const sprInsert = db.prepare(
|
|
91
192
|
`INSERT OR REPLACE INTO spr (id, parent_id, name, placetype, country, latitude, longitude, min_latitude, min_longitude, max_latitude, max_longitude, is_current, is_deprecated, is_ceased, is_superseded, is_superseding, lastmodified) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
92
193
|
)
|
|
194
|
+
|
|
93
195
|
const namesInsert = db.prepare(
|
|
94
196
|
`INSERT INTO names (id, name, placetype, country, language, privateuse, official, lastmodified) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
|
95
197
|
)
|
|
198
|
+
|
|
96
199
|
const populationInsert = db.prepare(`INSERT OR REPLACE INTO place_population (id, population) VALUES (?, ?)`)
|
|
200
|
+
|
|
97
201
|
// #267 admin linkage: ancestor rows (locality→region→country) so parentID scoping + adminCoherence reach
|
|
98
202
|
// the gap countries. Only used for a country in opts.adminForCountries.
|
|
99
203
|
const ancestorInsert = db.prepare(
|
|
@@ -123,8 +227,10 @@ export function ingestGeonamesAliases(
|
|
|
123
227
|
|
|
124
228
|
if (!existsSync(file)) {
|
|
125
229
|
report({ country: cc, places: 0, skipped: true }, file)
|
|
230
|
+
|
|
126
231
|
continue
|
|
127
232
|
}
|
|
233
|
+
|
|
128
234
|
let nc = 0
|
|
129
235
|
// #267: add A-class admin + ancestry only for the gap countries this country is in (never the EU set).
|
|
130
236
|
const addAdmin = opts?.adminForCountries?.has(cc) ?? false
|
|
@@ -136,81 +242,7 @@ export function ingestGeonamesAliases(
|
|
|
136
242
|
// dump repeats one spelling under several languages ("Åbo" sv/da/no); the merged tag is official /
|
|
137
243
|
// preferred if ANY qualifying row is.
|
|
138
244
|
const v2File = opts?.alternateDir ? join(opts.alternateDir, `${cc}.txt`) : undefined
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
if (v2File && existsSync(v2File)) {
|
|
142
|
-
const wanted = new Set<number>()
|
|
143
|
-
|
|
144
|
-
for (const line of lines) {
|
|
145
|
-
const f = line.split("\t")
|
|
146
|
-
|
|
147
|
-
if (f[6] === "P") {
|
|
148
|
-
wanted.add(Number(f[0]))
|
|
149
|
-
}
|
|
150
|
-
}
|
|
151
|
-
v2 = new Map()
|
|
152
|
-
|
|
153
|
-
// V2 columns (0-indexed): 1 geonameid, 2 isolanguage, 3 name, 4 isPreferredName, 5 isShortName,
|
|
154
|
-
// 6 isColloquial, 7 isHistoric, 8 from, 9 to.
|
|
155
|
-
//
|
|
156
|
-
// Two passes, because historic-ness is a fact about the NAME, not the row: GeoNames splits one
|
|
157
|
-
// spelling across rows — Malabo carries "Santa Isabel" as (es, unflagged) AND as (no-language,
|
|
158
|
-
// isHistoric=1, to=1973). Officialness must see the flags from EVERY row for the spelling, or the
|
|
159
|
-
// colonial-era name sails through on the language-tagged row (the #936 review's Malabo finding).
|
|
160
|
-
// Do NOT gate on isPreferredName instead — it's sparse annotation, not a signal (Turku's sv "Åbo"
|
|
161
|
-
// is unflagged; FI has 1,746 flags across the whole dump).
|
|
162
|
-
const v2Lines = readFileSync(v2File, "utf8").split("\n")
|
|
163
|
-
const historicNames = new Set<string>()
|
|
164
|
-
|
|
165
|
-
for (const line of v2Lines) {
|
|
166
|
-
if (!line) continue
|
|
167
|
-
const f = line.split("\t")
|
|
168
|
-
|
|
169
|
-
if (f[6] === "1" || f[7] === "1" || (f[9] ?? "").trim() !== "") {
|
|
170
|
-
const alt = (f[3] ?? "").trim()
|
|
171
|
-
|
|
172
|
-
if (alt && wanted.has(Number(f[1]))) {
|
|
173
|
-
historicNames.add(`${f[1]}|${alt}`)
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
for (const line of v2Lines) {
|
|
179
|
-
if (!line) continue
|
|
180
|
-
const f = line.split("\t")
|
|
181
|
-
const gid = Number(f[1])
|
|
182
|
-
|
|
183
|
-
if (!wanted.has(gid)) continue
|
|
184
|
-
const lang = f[2] ?? ""
|
|
185
|
-
|
|
186
|
-
// ISO 639 codes are 2-3 letters; GeoNames' pseudo-codes (post, link, iata, wkdt, …) are 4+.
|
|
187
|
-
if (!/^[a-z]{2,3}$/.test(lang)) continue
|
|
188
|
-
const alt = (f[3] ?? "").trim()
|
|
189
|
-
|
|
190
|
-
if (!alt) continue
|
|
191
|
-
const preferred = f[4] === "1"
|
|
192
|
-
const official = !historicNames.has(`${gid}|${alt}`) && isOfficialLanguage(cc, lang) ? 1 : 0
|
|
193
|
-
let byName = v2.get(gid)
|
|
194
|
-
|
|
195
|
-
if (!byName) {
|
|
196
|
-
v2.set(gid, (byName = new Map()))
|
|
197
|
-
}
|
|
198
|
-
const prev = byName.get(alt)
|
|
199
|
-
|
|
200
|
-
if (!prev) {
|
|
201
|
-
byName.set(alt, { language: lang, privateuse: preferred ? "preferred" : "", official })
|
|
202
|
-
} else {
|
|
203
|
-
if (official && !prev.official) {
|
|
204
|
-
prev.language = lang
|
|
205
|
-
prev.official = 1
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
if (preferred && !prev.privateuse) {
|
|
209
|
-
prev.privateuse = "preferred"
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
|
-
}
|
|
245
|
+
const v2 = v2File && existsSync(v2File) ? parseAlternateNamesV2(v2File, cc, lines) : undefined
|
|
214
246
|
|
|
215
247
|
// #267 admin pre-pass (gap countries): fold the country (PCLI) + regions (ADM1), self+ancestry them, and
|
|
216
248
|
// build the admin1→region map the localities link through. Point bbox (GeoNames gives a centroid only).
|
|
@@ -287,6 +319,7 @@ export function ingestGeonamesAliases(
|
|
|
287
319
|
ancestorInsert.run(nid, countryID, "country")
|
|
288
320
|
}
|
|
289
321
|
}
|
|
322
|
+
|
|
290
323
|
const seen = new Set([name])
|
|
291
324
|
|
|
292
325
|
const tags = v2?.get(Number(f[0]))
|
|
@@ -301,16 +334,20 @@ export function ingestGeonamesAliases(
|
|
|
301
334
|
namesInsert.run(nid, alt, "locality", cc, tag?.language ?? "", tag?.privateuse ?? "", tag?.official ?? 0, 0)
|
|
302
335
|
}
|
|
303
336
|
}
|
|
337
|
+
|
|
304
338
|
const pop = Number(f[14]) || 0
|
|
305
339
|
|
|
306
340
|
if (pop > 0) {
|
|
307
341
|
populationInsert.run(nid, pop)
|
|
308
342
|
}
|
|
343
|
+
|
|
309
344
|
nc++
|
|
310
345
|
}
|
|
346
|
+
|
|
311
347
|
report({ country: cc, places: nc, skipped: false })
|
|
312
348
|
total += nc
|
|
313
349
|
}
|
|
350
|
+
|
|
314
351
|
db.exec("COMMIT")
|
|
315
352
|
|
|
316
353
|
return total
|
package/geonames-postal.ts
CHANGED
|
@@ -38,6 +38,12 @@ import { existsSync, readFileSync } from "node:fs"
|
|
|
38
38
|
import { join } from "node:path"
|
|
39
39
|
import type { DatabaseSync } from "node:sqlite"
|
|
40
40
|
|
|
41
|
+
/**
|
|
42
|
+
* Column count of a GeoNames postal-code TSV row. Short rows are truncated or blank and are skipped. See the
|
|
43
|
+
* allCountries.zip readme for the field list.
|
|
44
|
+
*/
|
|
45
|
+
const GEONAMES_POSTAL_COLUMNS = 11
|
|
46
|
+
|
|
41
47
|
/**
|
|
42
48
|
* Synthetic id base for GeoNames-POSTAL rows — its own namespace above the alias fold's {@link GEONAMES_ID_BASE} (9e12)
|
|
43
49
|
* allocation so all four sources (WOF, Overture, GeoNames-alias, GeoNames-postal) coexist collision-free in a combined
|
|
@@ -51,15 +57,21 @@ export const GEONAMES_POSTAL_ID_BASE = 9_500_000_000_000
|
|
|
51
57
|
* `"11-041"` → `"11041"`, `"AD500"` → `"AD500"`.
|
|
52
58
|
*/
|
|
53
59
|
export function normalizePostcodeName(raw: string): string {
|
|
54
|
-
return raw.
|
|
60
|
+
return raw.replaceAll(/[^\p{L}\p{N}]/gu, "")
|
|
55
61
|
}
|
|
56
62
|
|
|
57
63
|
export interface GeonamesPostalIngestResult {
|
|
58
|
-
/**
|
|
64
|
+
/**
|
|
65
|
+
* Distinct postcodes inserted across all countries.
|
|
66
|
+
*/
|
|
59
67
|
inserted: number
|
|
60
|
-
/**
|
|
68
|
+
/**
|
|
69
|
+
* Per-country distinct-postcode counts.
|
|
70
|
+
*/
|
|
61
71
|
byCountry: Record<string, number>
|
|
62
|
-
/**
|
|
72
|
+
/**
|
|
73
|
+
* Countries whose `<CC>.txt` was missing under the postal dir (skipped, reported).
|
|
74
|
+
*/
|
|
63
75
|
missing: string[]
|
|
64
76
|
}
|
|
65
77
|
|
|
@@ -77,6 +89,7 @@ export function ingestGeonamesPostal(
|
|
|
77
89
|
const sprInsert = db.prepare(
|
|
78
90
|
`INSERT OR REPLACE INTO spr (id, parent_id, name, placetype, country, latitude, longitude, min_latitude, min_longitude, max_latitude, max_longitude, is_current, is_deprecated, is_ceased, is_superseded, is_superseding, lastmodified) VALUES (?, -1, ?, 'postalcode', ?, ?, ?, ?, ?, ?, ?, 1, 0, 0, 0, 0, 0)`
|
|
79
91
|
)
|
|
92
|
+
|
|
80
93
|
const namesInsert = db.prepare(
|
|
81
94
|
`INSERT INTO names (id, name, placetype, country, language, lastmodified) VALUES (?, ?, 'postalcode', ?, '', 0)`
|
|
82
95
|
)
|
|
@@ -92,18 +105,21 @@ export function ingestGeonamesPostal(
|
|
|
92
105
|
|
|
93
106
|
if (!existsSync(file)) {
|
|
94
107
|
missing.push(cc)
|
|
108
|
+
|
|
95
109
|
console.error(
|
|
96
110
|
` GeoNames postal ${cc}: ${file} missing — download from download.geonames.org/export/zip/${cc}.zip; skipped`
|
|
97
111
|
)
|
|
112
|
+
|
|
98
113
|
continue
|
|
99
114
|
}
|
|
115
|
+
|
|
100
116
|
// Group member settlement points per NORMALIZED code; remember one display form.
|
|
101
117
|
const members = new Map<string, { display: string; pts: Array<[number, number]> }>()
|
|
102
118
|
|
|
103
119
|
for (const line of readFileSync(file, "utf8").split("\n")) {
|
|
104
120
|
const cols = line.split("\t")
|
|
105
121
|
|
|
106
|
-
if (cols.length <
|
|
122
|
+
if (cols.length < GEONAMES_POSTAL_COLUMNS) continue
|
|
107
123
|
const display = cols[1]!.trim()
|
|
108
124
|
const name = normalizePostcodeName(display)
|
|
109
125
|
const lat = Number(cols[9])
|
|
@@ -132,6 +148,7 @@ export function ingestGeonamesPostal(
|
|
|
132
148
|
best = p
|
|
133
149
|
}
|
|
134
150
|
}
|
|
151
|
+
|
|
135
152
|
const id = nextID++
|
|
136
153
|
sprInsert.run(id, name, cc, best[0], best[1], best[0], best[1], best[0], best[1])
|
|
137
154
|
namesInsert.run(id, name, cc)
|
|
@@ -139,10 +156,13 @@ export function ingestGeonamesPostal(
|
|
|
139
156
|
if (m.display !== name) {
|
|
140
157
|
namesInsert.run(id, m.display, cc)
|
|
141
158
|
}
|
|
159
|
+
|
|
142
160
|
inserted++
|
|
143
161
|
}
|
|
162
|
+
|
|
144
163
|
db.exec("COMMIT")
|
|
145
164
|
byCountry[cc] = members.size
|
|
165
|
+
|
|
146
166
|
console.error(` GeoNames postal ${cc}: ${members.size.toLocaleString()} distinct codes (medoid centroids)`)
|
|
147
167
|
}
|
|
148
168
|
|
package/index.ts
CHANGED
|
@@ -23,6 +23,7 @@ export { WOFSqlitePlaceLookup, type RankingWeights, type WOFSqlitePlaceLookupOpt
|
|
|
23
23
|
|
|
24
24
|
export { CANDIDATE_FTS_TABLE, createCandidateFTS } from "./candidate-fts.ts"
|
|
25
25
|
export { WOFCandidateTableLookup, type WOFCandidateTableLookupOpts } from "./candidate-lookup.ts"
|
|
26
|
+
|
|
26
27
|
export {
|
|
27
28
|
COUNTRY_BBOX_TABLE,
|
|
28
29
|
COUNTRY_COVERAGE_TABLE,
|
|
@@ -31,6 +32,7 @@ export {
|
|
|
31
32
|
readGazetteerCoverageManifest,
|
|
32
33
|
writeGazetteerCoverageManifest,
|
|
33
34
|
} from "./coverage-manifest-schema.ts"
|
|
35
|
+
|
|
34
36
|
export type { CountryBBoxTable, CountryCoverageTable, GazetteerCoverageDatabase } from "./coverage-manifest-schema.ts"
|
|
35
37
|
export { SQLiteStreetNameLookup, type SQLiteStreetNameLookupOpts } from "./street-name-lookup.ts"
|
|
36
38
|
|
|
@@ -38,17 +40,21 @@ export { GEONAMES_ID_BASE, ingestGeonamesAliases, type GeonamesIngestProgress }
|
|
|
38
40
|
|
|
39
41
|
export { ADDRESS_POINT_COLUMNS, createAddressPointIndexes, createAddressPointTable } from "./address-point-schema.ts"
|
|
40
42
|
export type { AddressPointDatabase, AddressPointTable } from "./address-point-schema.ts"
|
|
43
|
+
|
|
41
44
|
export {
|
|
42
45
|
WOFPostalCityAliasLookup,
|
|
43
46
|
type PostalCityAlias,
|
|
44
47
|
type WOFPostalCityAliasLookupOpts,
|
|
45
48
|
} from "./postal-city-alias-lookup.ts"
|
|
49
|
+
|
|
46
50
|
export type { PostalCityAliasDatabase, PostalCityAliasTable } from "./postal-city-alias-schema.ts"
|
|
51
|
+
|
|
47
52
|
export {
|
|
48
53
|
POSTAL_CITY_CANDIDATE_COLUMNS,
|
|
49
54
|
POSTAL_CITY_CANDIDATE_TABLE,
|
|
50
55
|
createPostalCityCandidateTable,
|
|
51
56
|
} from "./postal-city-candidate-schema.ts"
|
|
57
|
+
|
|
52
58
|
export type { PostalCityCandidateDatabase, PostalCityCandidateTable } from "./postal-city-candidate-schema.ts"
|
|
53
59
|
|
|
54
60
|
export {
|
|
@@ -104,19 +110,23 @@ export {
|
|
|
104
110
|
|
|
105
111
|
export { AddressPointInterpolator } from "./address-point-interpolation.ts"
|
|
106
112
|
export { AddressPointSqliteLookup } from "./address-point.ts"
|
|
113
|
+
|
|
107
114
|
export {
|
|
108
115
|
STREET_CENTROID_COLUMNS,
|
|
109
116
|
createStreetCentroidIndexes,
|
|
110
117
|
createStreetCentroidTable,
|
|
111
118
|
} from "./street-centroid-schema.ts"
|
|
119
|
+
|
|
112
120
|
export type { StreetCentroidDatabase, StreetCentroidTable } from "./street-centroid-schema.ts"
|
|
113
121
|
export { StreetCentroidSqliteLookup } from "./street-centroid.ts"
|
|
122
|
+
|
|
114
123
|
export {
|
|
115
124
|
StreetInterpolator,
|
|
116
125
|
type InterpolatedHit,
|
|
117
126
|
type InterpolationMethod,
|
|
118
127
|
type InterpolationQuery,
|
|
119
128
|
} from "./interpolation.ts"
|
|
129
|
+
|
|
120
130
|
export {
|
|
121
131
|
deriveSchemaName,
|
|
122
132
|
pickShardForPlacetype,
|