@mailwoman/resolver-wof-sqlite 7.2.0 → 7.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/address-point-interpolation.ts +207 -0
- package/address-point-schema.ts +107 -0
- package/address-point.ts +122 -0
- package/ancestry-backfill.ts +205 -0
- package/ancestry.ts +70 -0
- package/build-candidate.ts +351 -0
- package/build-slim.ts +394 -0
- package/candidate-fts.ts +43 -0
- package/candidate-lookup.ts +382 -0
- package/candidate-schema.ts +166 -0
- package/coincident-roles.ts +240 -0
- package/convention.ts +152 -0
- package/fst-autocomplete.ts +187 -0
- package/fst-builder.ts +291 -0
- package/fst-deserialize-web.ts +164 -0
- package/fst-matcher.ts +150 -0
- package/fst-serialize.ts +311 -0
- package/fst-types.ts +78 -0
- package/fts.ts +318 -0
- package/geo.ts +140 -0
- package/geonames-aliases.ts +317 -0
- package/geonames-postal.ts +150 -0
- package/index.ts +117 -0
- package/interpolation.ts +232 -0
- package/lookup.ts +1498 -0
- package/out/poi-lookup.d.ts +14 -2
- package/out/poi-lookup.d.ts.map +1 -1
- package/out/poi-lookup.js +55 -21
- package/out/poi-lookup.js.map +1 -1
- package/out/poi-schema.d.ts +9 -0
- package/out/poi-schema.d.ts.map +1 -1
- package/out/poi-schema.js +16 -0
- package/out/poi-schema.js.map +1 -1
- package/out/reverse.d.ts +8 -1
- package/out/reverse.d.ts.map +1 -1
- package/out/reverse.js +10 -1
- package/out/reverse.js.map +1 -1
- package/package.json +168 -82
- package/poi-lookup.ts +375 -0
- package/poi-schema.ts +164 -0
- package/postal-city-alias-lookup.ts +89 -0
- package/postal-city-alias-schema.ts +75 -0
- package/postal-city-candidate-schema.ts +81 -0
- package/postcode-point-lookup.ts +64 -0
- package/reverse.ts +439 -0
- package/schema.ts +176 -0
- package/sharding.ts +235 -0
- package/sqlite-convention-source.ts +61 -0
- package/sqlite-utils.ts +25 -0
- package/street-centroid-schema.ts +124 -0
- package/street-centroid.ts +124 -0
- package/street-morphology-fst-builder.ts +230 -0
- package/street-name-lookup.ts +101 -0
- package/street-normalize.ts +302 -0
- package/street-segment-schema.ts +104 -0
- package/types.ts +164 -0
- package/unified-schema.ts +171 -0
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* #743/#193 — fold GeoNames bilingual / alt-language place-names into a WOF/unified admin DB as
|
|
7
|
+
* first-class places. The hard-filter recall gap on bilingual countries (the address says
|
|
8
|
+
* "Karjaa" but the table holds the Swedish "Karis") is missing alt-LANGUAGE names, not missing
|
|
9
|
+
* places: the WOF/Overture `names` carried only the primary, so the candidate build's Latin-alias
|
|
10
|
+
* explode (build-candidate pass 2) had nothing to widen. GeoNames' per-country dump carries the
|
|
11
|
+
* variants inline (the Karis row's `alternatenames` includes "Karjaa").
|
|
12
|
+
*
|
|
13
|
+
* For each POPULATED place (feature class `P`) this writes an `spr` row + `names` rows (primary +
|
|
14
|
+
* Latin alt-names) + population into the SAME tables the WOF/Overture paths use — synthetic ids
|
|
15
|
+
* based at {@link GEONAMES_ID_BASE} so the three sources never collide. The caller then rebuilds
|
|
16
|
+
* `place_search` ({@link buildPlaceSearchFTS} with `drop: true`) so the candidate build carries
|
|
17
|
+
* Karjaa↔Karis. Proven (FI hard-resolve 69.5 → 85.8 %, coverage 74.4 → 94.0 %); duplicating a
|
|
18
|
+
* place already held under another source is benign — the rows share name_key+coord and the
|
|
19
|
+
* candidate ranking dedupes by score.
|
|
20
|
+
*
|
|
21
|
+
* This is the package home so the canonical `build-unified-wof --geonames-countries`, the
|
|
22
|
+
* pipeline fold (`gazetteer-pipeline/admin/fold-geonames`), AND the `mailwoman gazetteer` commands all share
|
|
23
|
+
* ONE implementation. GeoNames dump = `download.geonames.org/export/dump/<CC>.zip` → `<CC>.txt`
|
|
24
|
+
* (TSV).
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { existsSync, readFileSync } from "node:fs"
|
|
28
|
+
import { join } from "node:path"
|
|
29
|
+
import type { DatabaseSync } from "node:sqlite"
|
|
30
|
+
|
|
31
|
+
import { isOfficialLanguage } from "@mailwoman/codex/country"
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Synthetic id base for GeoNames-sourced rows (#743/#193) — above Overture's 8e12 so the three sources (WOF real ids,
|
|
35
|
+
* Overture, GeoNames) never collide in a combined DB.
|
|
36
|
+
*/
|
|
37
|
+
export const GEONAMES_ID_BASE = 9_000_000_000_000
|
|
38
|
+
|
|
39
|
+
/** Per-country progress for the ingest — one event per country dump processed (or skipped). */
|
|
40
|
+
export interface GeonamesIngestProgress {
|
|
41
|
+
/** ISO 3166-1 alpha-2 code. */
|
|
42
|
+
country: string
|
|
43
|
+
/** Populated places ingested from this country's dump (0 when skipped). */
|
|
44
|
+
places: number
|
|
45
|
+
/** True when the country's `<CC>.txt` dump was missing — the country is skipped, not fatal. */
|
|
46
|
+
skipped: boolean
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Fold the GeoNames `P`-class places (+ their Latin alt-names) for `countries` into `db`'s `spr` / `names` /
|
|
51
|
+
* `place_population` tables. Returns the total places ingested.
|
|
52
|
+
*
|
|
53
|
+
* `onProgress` receives one event per country (default: a stderr line, matching the build scripts' legacy output). The
|
|
54
|
+
* caller MUST rebuild `place_search` afterward (`buildPlaceSearchFTS(db, { drop: true })`) for the new names to reach
|
|
55
|
+
* the candidate build's alias pass.
|
|
56
|
+
*/
|
|
57
|
+
export function ingestGeonamesAliases(
|
|
58
|
+
db: DatabaseSync,
|
|
59
|
+
countries: string[],
|
|
60
|
+
geonamesDir: string,
|
|
61
|
+
onProgress?: (event: GeonamesIngestProgress) => void,
|
|
62
|
+
opts?: {
|
|
63
|
+
/**
|
|
64
|
+
* #267: the countries for which to ALSO fold the GeoNames A-class admin (PCLI country + ADM1 regions) and link each
|
|
65
|
+
* locality's `parent_id` + ancestry chain (locality → region → country). PER-COUNTRY because a country that already
|
|
66
|
+
* carries WOF admin would double up — pass only the ZERO-COVERAGE gap countries (the coverage-expansion targets),
|
|
67
|
+
* never the EU alias set. Without admin, a gap country's localities are orphans (`parent_id=-1`, no ancestors), so
|
|
68
|
+
* `parentID` scoping and adminCoherence can't reach them and "Tbilisi, GE" can't resolve.
|
|
69
|
+
*/
|
|
70
|
+
adminForCountries?: ReadonlySet<string>
|
|
71
|
+
/**
|
|
72
|
+
* #936: directory of per-country alternateNamesV2 dumps
|
|
73
|
+
* (`download.geonames.org/export/dump/alternatenames/<CC>.zip` → `<CC>.txt`). When a country's file is present,
|
|
74
|
+
* alias rows gain their language tag, `privateuse` ("preferred" from `isPreferredName`), and the `official` bit
|
|
75
|
+
* (language is CLDR-official for the country, colloquial/historic excluded — the rule the #936 risk probe measured
|
|
76
|
+
* at 7 new name-exact collisions globally). The main dump's bare `alternatenames` list still decides WHICH rows
|
|
77
|
+
* exist; V2 only decorates them. Missing file = the pre-#936 untagged behavior, not an error.
|
|
78
|
+
*/
|
|
79
|
+
alternateDir?: string
|
|
80
|
+
}
|
|
81
|
+
): number {
|
|
82
|
+
// Latin-only, no bracket/paren noise GeoNames packs into `alternatenames` ("(( Karis Landskommun ))",
|
|
83
|
+
// airport codes), 2–60 chars, at least one letter (drops bare postcodes/numbers).
|
|
84
|
+
const LATIN_NAME = /^[\p{Script=Latin}\p{M}\s\-'.]{2,60}$/u
|
|
85
|
+
const clean = (s: string): string | null => {
|
|
86
|
+
const t = s.trim()
|
|
87
|
+
|
|
88
|
+
return t && LATIN_NAME.test(t) && /\p{L}/u.test(t) ? t : null
|
|
89
|
+
}
|
|
90
|
+
const sprInsert = db.prepare(
|
|
91
|
+
`INSERT OR REPLACE INTO spr (id, parent_id, name, placetype, country, latitude, longitude, min_latitude, min_longitude, max_latitude, max_longitude, is_current, is_deprecated, is_ceased, is_superseded, is_superseding, lastmodified) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
92
|
+
)
|
|
93
|
+
const namesInsert = db.prepare(
|
|
94
|
+
`INSERT INTO names (id, name, placetype, country, language, privateuse, official, lastmodified) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
|
95
|
+
)
|
|
96
|
+
const populationInsert = db.prepare(`INSERT OR REPLACE INTO place_population (id, population) VALUES (?, ?)`)
|
|
97
|
+
// #267 admin linkage: ancestor rows (locality→region→country) so parentID scoping + adminCoherence reach
|
|
98
|
+
// the gap countries. Only used for a country in opts.adminForCountries.
|
|
99
|
+
const ancestorInsert = db.prepare(
|
|
100
|
+
`INSERT INTO ancestors (id, ancestor_id, ancestor_placetype, lastmodified) VALUES (?, ?, ?, 0)`
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
const report = (event: GeonamesIngestProgress, missingFile?: string): void => {
|
|
104
|
+
if (onProgress) {
|
|
105
|
+
onProgress(event)
|
|
106
|
+
} else if (event.skipped) {
|
|
107
|
+
console.error(
|
|
108
|
+
` GeoNames ${event.country}: ${missingFile} missing — download from download.geonames.org/export/dump/${event.country}.zip; skipped`
|
|
109
|
+
)
|
|
110
|
+
} else {
|
|
111
|
+
console.error(
|
|
112
|
+
` GeoNames ${event.country}: ${event.places.toLocaleString()} populated places (+ Latin alt-names)`
|
|
113
|
+
)
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
let id = GEONAMES_ID_BASE
|
|
118
|
+
let total = 0
|
|
119
|
+
db.exec("BEGIN")
|
|
120
|
+
|
|
121
|
+
for (const cc of countries) {
|
|
122
|
+
const file = join(geonamesDir, `${cc}.txt`)
|
|
123
|
+
|
|
124
|
+
if (!existsSync(file)) {
|
|
125
|
+
report({ country: cc, places: 0, skipped: true }, file)
|
|
126
|
+
continue
|
|
127
|
+
}
|
|
128
|
+
let nc = 0
|
|
129
|
+
// #267: add A-class admin + ancestry only for the gap countries this country is in (never the EU set).
|
|
130
|
+
const addAdmin = opts?.adminForCountries?.has(cc) ?? false
|
|
131
|
+
// GeoNames dump columns (0-indexed): 0 geonameid, 1 name, 2 asciiname, 3 alternatenames, 4 lat, 5 lon,
|
|
132
|
+
// 6 feature_class, 7 feature_code, 10 admin1 code, 14 pop.
|
|
133
|
+
const lines = readFileSync(file, "utf8").split("\n")
|
|
134
|
+
|
|
135
|
+
// #936: V2 tags for this country's P-class rows — geonameid → exact alias spelling → tag. The V2
|
|
136
|
+
// dump repeats one spelling under several languages ("Åbo" sv/da/no); the merged tag is official /
|
|
137
|
+
// preferred if ANY qualifying row is.
|
|
138
|
+
const v2File = opts?.alternateDir ? join(opts.alternateDir, `${cc}.txt`) : undefined
|
|
139
|
+
let v2: Map<number, Map<string, { language: string; privateuse: string; official: number }>> | undefined
|
|
140
|
+
|
|
141
|
+
if (v2File && existsSync(v2File)) {
|
|
142
|
+
const wanted = new Set<number>()
|
|
143
|
+
|
|
144
|
+
for (const line of lines) {
|
|
145
|
+
const f = line.split("\t")
|
|
146
|
+
|
|
147
|
+
if (f[6] === "P") {
|
|
148
|
+
wanted.add(Number(f[0]))
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
v2 = new Map()
|
|
152
|
+
|
|
153
|
+
// V2 columns (0-indexed): 1 geonameid, 2 isolanguage, 3 name, 4 isPreferredName, 5 isShortName,
|
|
154
|
+
// 6 isColloquial, 7 isHistoric, 8 from, 9 to.
|
|
155
|
+
//
|
|
156
|
+
// Two passes, because historic-ness is a fact about the NAME, not the row: GeoNames splits one
|
|
157
|
+
// spelling across rows — Malabo carries "Santa Isabel" as (es, unflagged) AND as (no-language,
|
|
158
|
+
// isHistoric=1, to=1973). Officialness must see the flags from EVERY row for the spelling, or the
|
|
159
|
+
// colonial-era name sails through on the language-tagged row (the #936 review's Malabo finding).
|
|
160
|
+
// Do NOT gate on isPreferredName instead — it's sparse annotation, not a signal (Turku's sv "Åbo"
|
|
161
|
+
// is unflagged; FI has 1,746 flags across the whole dump).
|
|
162
|
+
const v2Lines = readFileSync(v2File, "utf8").split("\n")
|
|
163
|
+
const historicNames = new Set<string>()
|
|
164
|
+
|
|
165
|
+
for (const line of v2Lines) {
|
|
166
|
+
if (!line) continue
|
|
167
|
+
const f = line.split("\t")
|
|
168
|
+
|
|
169
|
+
if (f[6] === "1" || f[7] === "1" || (f[9] ?? "").trim() !== "") {
|
|
170
|
+
const alt = (f[3] ?? "").trim()
|
|
171
|
+
|
|
172
|
+
if (alt && wanted.has(Number(f[1]))) {
|
|
173
|
+
historicNames.add(`${f[1]}|${alt}`)
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
for (const line of v2Lines) {
|
|
179
|
+
if (!line) continue
|
|
180
|
+
const f = line.split("\t")
|
|
181
|
+
const gid = Number(f[1])
|
|
182
|
+
|
|
183
|
+
if (!wanted.has(gid)) continue
|
|
184
|
+
const lang = f[2] ?? ""
|
|
185
|
+
|
|
186
|
+
// ISO 639 codes are 2-3 letters; GeoNames' pseudo-codes (post, link, iata, wkdt, …) are 4+.
|
|
187
|
+
if (!/^[a-z]{2,3}$/.test(lang)) continue
|
|
188
|
+
const alt = (f[3] ?? "").trim()
|
|
189
|
+
|
|
190
|
+
if (!alt) continue
|
|
191
|
+
const preferred = f[4] === "1"
|
|
192
|
+
const official = !historicNames.has(`${gid}|${alt}`) && isOfficialLanguage(cc, lang) ? 1 : 0
|
|
193
|
+
let byName = v2.get(gid)
|
|
194
|
+
|
|
195
|
+
if (!byName) {
|
|
196
|
+
v2.set(gid, (byName = new Map()))
|
|
197
|
+
}
|
|
198
|
+
const prev = byName.get(alt)
|
|
199
|
+
|
|
200
|
+
if (!prev) {
|
|
201
|
+
byName.set(alt, { language: lang, privateuse: preferred ? "preferred" : "", official })
|
|
202
|
+
} else {
|
|
203
|
+
if (official && !prev.official) {
|
|
204
|
+
prev.language = lang
|
|
205
|
+
prev.official = 1
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
if (preferred && !prev.privateuse) {
|
|
209
|
+
prev.privateuse = "preferred"
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// #267 admin pre-pass (gap countries): fold the country (PCLI) + regions (ADM1), self+ancestry them, and
|
|
216
|
+
// build the admin1→region map the localities link through. Point bbox (GeoNames gives a centroid only).
|
|
217
|
+
let countryID = -1
|
|
218
|
+
const adminMap = new Map<string, number>()
|
|
219
|
+
|
|
220
|
+
if (addAdmin) {
|
|
221
|
+
for (const line of lines) {
|
|
222
|
+
const f = line.split("\t")
|
|
223
|
+
|
|
224
|
+
if (f[6] !== "A") continue
|
|
225
|
+
const aname = clean(f[2] ?? "") ?? clean(f[1] ?? "")
|
|
226
|
+
|
|
227
|
+
if (!aname) continue
|
|
228
|
+
const lat = Number(f[4]) || 0
|
|
229
|
+
const lon = Number(f[5]) || 0
|
|
230
|
+
|
|
231
|
+
if (f[7]?.startsWith("PCL")) {
|
|
232
|
+
// Any country-level political entity — PCLI (independent), PCLD (dependent territory),
|
|
233
|
+
// PCLF (freely associated), PCLS (special administrative region: HK/MO/PS). All are the
|
|
234
|
+
// country tier; restricting to PCLI left those ~17 territories without a country row.
|
|
235
|
+
if (countryID >= 0) continue // one country row
|
|
236
|
+
countryID = id++
|
|
237
|
+
sprInsert.run(countryID, -1, aname, "country", cc, lat, lon, lat, lon, lat, lon, 1, 0, 0, 0, 0, 0)
|
|
238
|
+
namesInsert.run(countryID, aname, "country", cc, "", "", 0, 0)
|
|
239
|
+
ancestorInsert.run(countryID, countryID, "country")
|
|
240
|
+
} else if (f[7] === "ADM1" && f[10]) {
|
|
241
|
+
const rid = id++
|
|
242
|
+
sprInsert.run(rid, -1, aname, "region", cc, lat, lon, lat, lon, lat, lon, 1, 0, 0, 0, 0, 0)
|
|
243
|
+
namesInsert.run(rid, aname, "region", cc, "", "", 0, 0)
|
|
244
|
+
ancestorInsert.run(rid, rid, "region")
|
|
245
|
+
adminMap.set(f[10], rid)
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// Re-parent regions + ancestor them to the (now-known) country.
|
|
250
|
+
if (countryID >= 0) {
|
|
251
|
+
for (const rid of adminMap.values()) {
|
|
252
|
+
db.prepare("UPDATE spr SET parent_id = ? WHERE id = ?").run(countryID, rid)
|
|
253
|
+
ancestorInsert.run(rid, countryID, "country")
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
for (const line of lines) {
|
|
259
|
+
if (!line) continue
|
|
260
|
+
const f = line.split("\t")
|
|
261
|
+
|
|
262
|
+
if (f[6] !== "P") continue // populated places only
|
|
263
|
+
const lat = Number(f[4])
|
|
264
|
+
const lon = Number(f[5])
|
|
265
|
+
|
|
266
|
+
if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue
|
|
267
|
+
const name = clean(f[1] ?? "")
|
|
268
|
+
|
|
269
|
+
if (!name) continue
|
|
270
|
+
const nid = id++
|
|
271
|
+
// #267: link to the locality's region (else country) for gap countries; -1 (orphan) otherwise.
|
|
272
|
+
const regionID = addAdmin ? (adminMap.get(f[10] ?? "") ?? -1) : -1
|
|
273
|
+
const parentID = regionID >= 0 ? regionID : addAdmin && countryID >= 0 ? countryID : -1
|
|
274
|
+
// Point bbox — a GeoNames row is a centroid; the candidate's region-bbox disambiguation just
|
|
275
|
+
// sees it as contained in itself, fine for a locality.
|
|
276
|
+
sprInsert.run(nid, parentID, name, "locality", cc, lat, lon, lat, lon, lat, lon, 1, 0, 0, 0, 0, 0)
|
|
277
|
+
namesInsert.run(nid, name, "locality", cc, "", "", 0, 0)
|
|
278
|
+
|
|
279
|
+
if (addAdmin) {
|
|
280
|
+
ancestorInsert.run(nid, nid, "locality")
|
|
281
|
+
|
|
282
|
+
if (regionID >= 0) {
|
|
283
|
+
ancestorInsert.run(nid, regionID, "region")
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
if (countryID >= 0) {
|
|
287
|
+
ancestorInsert.run(nid, countryID, "country")
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
const seen = new Set([name])
|
|
291
|
+
|
|
292
|
+
const tags = v2?.get(Number(f[0]))
|
|
293
|
+
|
|
294
|
+
for (const raw of [f[2] ?? "", ...(f[3] ? f[3].split(",") : [])]) {
|
|
295
|
+
const alt = clean(raw)
|
|
296
|
+
|
|
297
|
+
if (alt && !seen.has(alt)) {
|
|
298
|
+
seen.add(alt)
|
|
299
|
+
const tag = tags?.get(alt)
|
|
300
|
+
|
|
301
|
+
namesInsert.run(nid, alt, "locality", cc, tag?.language ?? "", tag?.privateuse ?? "", tag?.official ?? 0, 0)
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
const pop = Number(f[14]) || 0
|
|
305
|
+
|
|
306
|
+
if (pop > 0) {
|
|
307
|
+
populationInsert.run(nid, pop)
|
|
308
|
+
}
|
|
309
|
+
nc++
|
|
310
|
+
}
|
|
311
|
+
report({ country: cc, places: nc, skipped: false })
|
|
312
|
+
total += nc
|
|
313
|
+
}
|
|
314
|
+
db.exec("COMMIT")
|
|
315
|
+
|
|
316
|
+
return total
|
|
317
|
+
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* #920 — fold GeoNames POSTAL codes into a WOF/unified postcode shard as first-class
|
|
7
|
+
* `postalcode` places, for the countries whose WOF postalcode repos don't exist (the
|
|
8
|
+
* namesake-tail locales: FI/CZ/SK/SI/DK/NO/HR/PL and any future gap).
|
|
9
|
+
*
|
|
10
|
+
* Why: the night-31 taxonomy measured the cross-locale resolve tail as NAMESAKE COLLISION
|
|
11
|
+
* (FI 300/1k … PL 75/1k offender rows), and the controlled experiment showed postcode-shard
|
|
12
|
+
* coverage alone collapses it (FI 300→1, CZ 131→4): a resolvable postcode node feeds the
|
|
13
|
+
* resolver's coordinate-first sibling-postcode candidate injection, which binds the locality
|
|
14
|
+
* pick to its postcode neighborhood. The machinery already ships; it was coverage-starved.
|
|
15
|
+
*
|
|
16
|
+
* Two hard-won laws from the experiment are enforced HERE, in code, not in a runbook:
|
|
17
|
+
*
|
|
18
|
+
* 1. **The name law (#920 format law):** a postcode row's `name` is stored in the
|
|
19
|
+
* SANITIZED-QUERY token shape — every non-letter/number stripped — because that is what
|
|
20
|
+
* `sanitizeFTSQuery` reduces the parsed token to at lookup time. Stored `"110 00"` (CZ) or
|
|
21
|
+
* `"11-041"` (PL) can never match the query `"11000"`/`"11041"`; the spaced CZ build
|
|
22
|
+
* measured WORSE than no coverage (+13 namesake rows) because its bigrams partial-matched
|
|
23
|
+
* WRONG codes. The display form is preserved as an alt row in `names`.
|
|
24
|
+
* 2. **Medoid centroids:** GeoNames postal is one row per (postcode, settlement); the naive
|
|
25
|
+
* mean-of-members centroid displaced tighter village coordinates on already-correct rows
|
|
26
|
+
* (the p50-tax that ni-failed SK/SI/HR at 1.10–1.94 km CI). The MEDOID — the member point
|
|
27
|
+
* nearest the mean — keeps the coordinate on a real settlement.
|
|
28
|
+
*
|
|
29
|
+
* Package home for the same reason as `geonames-aliases.ts`: `build-unified-wof
|
|
30
|
+
* --geonames-postal-countries`, any standalone fold, and the `mailwoman gazetteer` commands
|
|
31
|
+
* share ONE implementation. GeoNames postal dump = `download.geonames.org/export/zip/<CC>.zip`
|
|
32
|
+
* → `<CC>.txt` (TSV: country, postcode, place, admin1, code1, admin2, code2, admin3, code3,
|
|
33
|
+
* lat, lon, accuracy). License CC BY 4.0 — attribution rides the shard's `meta` provenance and
|
|
34
|
+
* the model card like the existing GeoNames alias fold.
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
import { existsSync, readFileSync } from "node:fs"
|
|
38
|
+
import { join } from "node:path"
|
|
39
|
+
import type { DatabaseSync } from "node:sqlite"
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Synthetic id base for GeoNames-POSTAL rows — its own namespace above the alias fold's {@link GEONAMES_ID_BASE} (9e12)
|
|
43
|
+
* allocation so all four sources (WOF, Overture, GeoNames-alias, GeoNames-postal) coexist collision-free in a combined
|
|
44
|
+
* DB.
|
|
45
|
+
*/
|
|
46
|
+
export const GEONAMES_POSTAL_ID_BASE = 9_500_000_000_000
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* The #920 name law: reduce a postal code to the sanitized-query token shape — strip every non-letter/number — so the
|
|
50
|
+
* stored name matches what `sanitizeFTSQuery` produces from the parsed postcode token. `"110 00"` → `"11000"`,
|
|
51
|
+
* `"11-041"` → `"11041"`, `"AD500"` → `"AD500"`.
|
|
52
|
+
*/
|
|
53
|
+
export function normalizePostcodeName(raw: string): string {
|
|
54
|
+
return raw.replace(/[^\p{L}\p{N}]/gu, "")
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export interface GeonamesPostalIngestResult {
|
|
58
|
+
/** Distinct postcodes inserted across all countries. */
|
|
59
|
+
inserted: number
|
|
60
|
+
/** Per-country distinct-postcode counts. */
|
|
61
|
+
byCountry: Record<string, number>
|
|
62
|
+
/** Countries whose `<CC>.txt` was missing under the postal dir (skipped, reported). */
|
|
63
|
+
missing: string[]
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Fold GeoNames postal codes for `countries` into an open unified/postcode ingest DB: one `spr` row per distinct
|
|
68
|
+
* normalized postcode (placetype `postalcode`, medoid centroid, degenerate bbox), the normalized form as `name`, and
|
|
69
|
+
* the display form as an extra `names` row when it differs. The caller owns the FTS rebuild (rows ride the standard
|
|
70
|
+
* freeze phase).
|
|
71
|
+
*/
|
|
72
|
+
export function ingestGeonamesPostal(
|
|
73
|
+
db: DatabaseSync,
|
|
74
|
+
countries: readonly string[],
|
|
75
|
+
postalDir: string
|
|
76
|
+
): GeonamesPostalIngestResult {
|
|
77
|
+
const sprInsert = db.prepare(
|
|
78
|
+
`INSERT OR REPLACE INTO spr (id, parent_id, name, placetype, country, latitude, longitude, min_latitude, min_longitude, max_latitude, max_longitude, is_current, is_deprecated, is_ceased, is_superseded, is_superseding, lastmodified) VALUES (?, -1, ?, 'postalcode', ?, ?, ?, ?, ?, ?, ?, 1, 0, 0, 0, 0, 0)`
|
|
79
|
+
)
|
|
80
|
+
const namesInsert = db.prepare(
|
|
81
|
+
`INSERT INTO names (id, name, placetype, country, language, lastmodified) VALUES (?, ?, 'postalcode', ?, '', 0)`
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
let nextID = GEONAMES_POSTAL_ID_BASE
|
|
85
|
+
const byCountry: Record<string, number> = {}
|
|
86
|
+
const missing: string[] = []
|
|
87
|
+
let inserted = 0
|
|
88
|
+
|
|
89
|
+
for (const country of countries) {
|
|
90
|
+
const cc = country.toUpperCase()
|
|
91
|
+
const file = join(postalDir, `${cc}.txt`)
|
|
92
|
+
|
|
93
|
+
if (!existsSync(file)) {
|
|
94
|
+
missing.push(cc)
|
|
95
|
+
console.error(
|
|
96
|
+
` GeoNames postal ${cc}: ${file} missing — download from download.geonames.org/export/zip/${cc}.zip; skipped`
|
|
97
|
+
)
|
|
98
|
+
continue
|
|
99
|
+
}
|
|
100
|
+
// Group member settlement points per NORMALIZED code; remember one display form.
|
|
101
|
+
const members = new Map<string, { display: string; pts: Array<[number, number]> }>()
|
|
102
|
+
|
|
103
|
+
for (const line of readFileSync(file, "utf8").split("\n")) {
|
|
104
|
+
const cols = line.split("\t")
|
|
105
|
+
|
|
106
|
+
if (cols.length < 11) continue
|
|
107
|
+
const display = cols[1]!.trim()
|
|
108
|
+
const name = normalizePostcodeName(display)
|
|
109
|
+
const lat = Number(cols[9])
|
|
110
|
+
const lon = Number(cols[10])
|
|
111
|
+
|
|
112
|
+
if (!name || !Number.isFinite(lat) || !Number.isFinite(lon)) continue
|
|
113
|
+
const m = members.get(name) ?? { display, pts: [] }
|
|
114
|
+
m.pts.push([lat, lon])
|
|
115
|
+
members.set(name, m)
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
db.exec("BEGIN")
|
|
119
|
+
|
|
120
|
+
for (const [name, m] of members) {
|
|
121
|
+
// Medoid: the member point nearest the mean — stays on a real settlement (the p50-tax law).
|
|
122
|
+
const meanLat = m.pts.reduce((s, p) => s + p[0], 0) / m.pts.length
|
|
123
|
+
const meanLon = m.pts.reduce((s, p) => s + p[1], 0) / m.pts.length
|
|
124
|
+
let best = m.pts[0]!
|
|
125
|
+
let bestD = Infinity
|
|
126
|
+
|
|
127
|
+
for (const p of m.pts) {
|
|
128
|
+
const d = (p[0] - meanLat) ** 2 + (p[1] - meanLon) ** 2
|
|
129
|
+
|
|
130
|
+
if (d < bestD) {
|
|
131
|
+
bestD = d
|
|
132
|
+
best = p
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
const id = nextID++
|
|
136
|
+
sprInsert.run(id, name, cc, best[0], best[1], best[0], best[1], best[0], best[1])
|
|
137
|
+
namesInsert.run(id, name, cc)
|
|
138
|
+
|
|
139
|
+
if (m.display !== name) {
|
|
140
|
+
namesInsert.run(id, m.display, cc)
|
|
141
|
+
}
|
|
142
|
+
inserted++
|
|
143
|
+
}
|
|
144
|
+
db.exec("COMMIT")
|
|
145
|
+
byCountry[cc] = members.size
|
|
146
|
+
console.error(` GeoNames postal ${cc}: ${members.size.toLocaleString()} distinct codes (medoid centroids)`)
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
return { inserted, byCountry, missing }
|
|
150
|
+
}
|
package/index.ts
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
export type { FindPlaceQuery, GeoBbox, GeoPoint, PlaceCandidate, PlaceLookup, WOFPlacetype } from "./types.ts"
|
|
8
|
+
|
|
9
|
+
export type {
|
|
10
|
+
AncestorsTable,
|
|
11
|
+
CoincidentRolesTable,
|
|
12
|
+
ConcordancesTable,
|
|
13
|
+
GeojsonTable,
|
|
14
|
+
NamesTable,
|
|
15
|
+
PlaceAbbrTable,
|
|
16
|
+
PlacePopulationTable,
|
|
17
|
+
PlaceSearchTable,
|
|
18
|
+
SprTable,
|
|
19
|
+
WOFDatabase,
|
|
20
|
+
} from "./schema.ts"
|
|
21
|
+
|
|
22
|
+
export { WOFSqlitePlaceLookup, type RankingWeights, type WOFSqlitePlaceLookupOpts } from "./lookup.ts"
|
|
23
|
+
|
|
24
|
+
export { CANDIDATE_FTS_TABLE, createCandidateFTS } from "./candidate-fts.ts"
|
|
25
|
+
export { WOFCandidateTableLookup, type WOFCandidateTableLookupOpts } from "./candidate-lookup.ts"
|
|
26
|
+
export { SQLiteStreetNameLookup, type SQLiteStreetNameLookupOpts } from "./street-name-lookup.ts"
|
|
27
|
+
|
|
28
|
+
export { GEONAMES_ID_BASE, ingestGeonamesAliases, type GeonamesIngestProgress } from "./geonames-aliases.ts"
|
|
29
|
+
|
|
30
|
+
export { ADDRESS_POINT_COLUMNS, createAddressPointIndexes, createAddressPointTable } from "./address-point-schema.ts"
|
|
31
|
+
export type { AddressPointDatabase, AddressPointTable } from "./address-point-schema.ts"
|
|
32
|
+
export {
|
|
33
|
+
WOFPostalCityAliasLookup,
|
|
34
|
+
type PostalCityAlias,
|
|
35
|
+
type WOFPostalCityAliasLookupOpts,
|
|
36
|
+
} from "./postal-city-alias-lookup.ts"
|
|
37
|
+
export type { PostalCityAliasDatabase, PostalCityAliasTable } from "./postal-city-alias-schema.ts"
|
|
38
|
+
export {
|
|
39
|
+
POSTAL_CITY_CANDIDATE_COLUMNS,
|
|
40
|
+
POSTAL_CITY_CANDIDATE_TABLE,
|
|
41
|
+
createPostalCityCandidateTable,
|
|
42
|
+
} from "./postal-city-candidate-schema.ts"
|
|
43
|
+
export type { PostalCityCandidateDatabase, PostalCityCandidateTable } from "./postal-city-candidate-schema.ts"
|
|
44
|
+
|
|
45
|
+
export {
|
|
46
|
+
ADDRESS_CONVENTION_TABLE,
|
|
47
|
+
BUILTIN_STRATEGY_NAMES,
|
|
48
|
+
SeedConventionSource,
|
|
49
|
+
WORLD_DEFAULT,
|
|
50
|
+
mergeConventions,
|
|
51
|
+
resolveConvention,
|
|
52
|
+
type Convention,
|
|
53
|
+
type ConventionSource,
|
|
54
|
+
type ResolvedConvention,
|
|
55
|
+
type ScoringWeights,
|
|
56
|
+
type Strategy,
|
|
57
|
+
} from "./convention.ts"
|
|
58
|
+
|
|
59
|
+
export { SqliteConventionSource } from "./sqlite-convention-source.ts"
|
|
60
|
+
|
|
61
|
+
export { WOFPostcodeLookup, type PostcodePlace } from "./postcode-point-lookup.ts"
|
|
62
|
+
|
|
63
|
+
export {
|
|
64
|
+
PLACE_BBOX_TABLE,
|
|
65
|
+
PLACE_SEARCH_TABLE,
|
|
66
|
+
buildPlaceSearchFTS,
|
|
67
|
+
placeBboxExists,
|
|
68
|
+
placeSearchFTSExists,
|
|
69
|
+
type BuildPlaceSearchFTSOpts,
|
|
70
|
+
type BuildPlaceSearchFTSResult,
|
|
71
|
+
} from "./fts.ts"
|
|
72
|
+
|
|
73
|
+
export {
|
|
74
|
+
bboxAround,
|
|
75
|
+
geometryContains,
|
|
76
|
+
haversineKm,
|
|
77
|
+
pointInPolygonRings,
|
|
78
|
+
pointInRing,
|
|
79
|
+
type Bbox,
|
|
80
|
+
type GeojsonGeometry,
|
|
81
|
+
type GeojsonMultiPolygon,
|
|
82
|
+
type GeojsonPolygon,
|
|
83
|
+
type GeojsonPosition,
|
|
84
|
+
} from "./geo.ts"
|
|
85
|
+
|
|
86
|
+
export { PLACETYPE_DEPTH, ancestorLineage, placetypeDepth, type AncestorPlaceRow } from "./ancestry.ts"
|
|
87
|
+
|
|
88
|
+
export {
|
|
89
|
+
WOFReverseGeocoder,
|
|
90
|
+
type ContainmentKind,
|
|
91
|
+
type ReverseGeocodeOpts,
|
|
92
|
+
type ReverseGeocodeResult,
|
|
93
|
+
type WOFReverseGeocoderOpts,
|
|
94
|
+
} from "./reverse.ts"
|
|
95
|
+
|
|
96
|
+
export { AddressPointInterpolator } from "./address-point-interpolation.ts"
|
|
97
|
+
export { AddressPointSqliteLookup } from "./address-point.ts"
|
|
98
|
+
export {
|
|
99
|
+
STREET_CENTROID_COLUMNS,
|
|
100
|
+
createStreetCentroidIndexes,
|
|
101
|
+
createStreetCentroidTable,
|
|
102
|
+
} from "./street-centroid-schema.ts"
|
|
103
|
+
export type { StreetCentroidDatabase, StreetCentroidTable } from "./street-centroid-schema.ts"
|
|
104
|
+
export { StreetCentroidSqliteLookup } from "./street-centroid.ts"
|
|
105
|
+
export {
|
|
106
|
+
StreetInterpolator,
|
|
107
|
+
type InterpolatedHit,
|
|
108
|
+
type InterpolationMethod,
|
|
109
|
+
type InterpolationQuery,
|
|
110
|
+
} from "./interpolation.ts"
|
|
111
|
+
export {
|
|
112
|
+
deriveSchemaName,
|
|
113
|
+
pickShardForPlacetype,
|
|
114
|
+
resolveShards,
|
|
115
|
+
type ResolvedShard,
|
|
116
|
+
type ShardConfig,
|
|
117
|
+
} from "./sharding.ts"
|