@mailwoman/resolver-wof-sqlite 10.7.0 → 10.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/address/point/interpolation.ts +4 -4
- package/lib/address/point/spine-schema.ts +59 -0
- package/lib/ancestry/backfill.ts +3 -25
- package/lib/build/candidate.ts +22 -5
- package/lib/candidate/alias-bags.ts +32 -14
- package/lib/candidate/bare-admin-names.ts +54 -0
- package/lib/candidate/importance.ts +2 -2
- package/lib/candidate/lookup.ts +54 -26
- package/lib/candidate/own-name.ts +1 -5
- package/lib/candidate/scoring.ts +4 -4
- package/lib/capitals.ts +2 -2
- package/lib/coincident-roles.ts +3 -3
- package/lib/currency-backfill.ts +4 -4
- package/lib/fst/autocomplete.ts +1 -1
- package/lib/fst/binary-matcher.ts +379 -0
- package/lib/fst/builder.ts +1 -1
- package/lib/fst/deserialize-web.ts +14 -143
- package/lib/fst/matcher.ts +3 -3
- package/lib/fst/serialize.ts +1 -1
- package/lib/fst.ts +1 -0
- package/lib/geonames/aliases.ts +2 -2
- package/lib/interpolation.ts +3 -2
- package/lib/lookup.ts +6 -6
- package/lib/nsul/lookup.ts +2 -1
- package/lib/nsul/schema.ts +2 -8
- package/lib/place-importance-schema.ts +1 -1
- package/lib/poi/lookup.ts +14 -14
- package/lib/proximity-rerank.ts +2 -2
- package/lib/reverse.ts +8 -5
- package/lib/search-fetch.ts +3 -3
- package/lib/sqlite/convention-source.ts +1 -1
- package/lib/street/centroid.ts +2 -2
- package/lib/street/morphology/fst/builder.ts +3 -3
- package/lib/street/morphology/fst/loader.ts +3 -3
- package/lib/street/normalize.ts +59 -1
- package/lib/types.ts +8 -0
- package/lib/uprn/existence.ts +0 -1
- package/lib/uprn/lookup.ts +5 -5
- package/lib/uprn/schema.ts +3 -4
- package/lib/weights-overlay-linker.ts +4 -4
- package/out/address/point/interpolation.js +4 -4
- package/out/address/point/interpolation.js.map +1 -1
- package/out/address/point/spine-schema.d.ts +40 -0
- package/out/address/point/spine-schema.d.ts.map +1 -0
- package/out/address/point/spine-schema.js +40 -0
- package/out/address/point/spine-schema.js.map +1 -0
- package/out/ancestry/backfill.d.ts.map +1 -1
- package/out/ancestry/backfill.js +3 -22
- package/out/ancestry/backfill.js.map +1 -1
- package/out/build/candidate.d.ts.map +1 -1
- package/out/build/candidate.js +17 -5
- package/out/build/candidate.js.map +1 -1
- package/out/candidate/alias-bags.d.ts +8 -0
- package/out/candidate/alias-bags.d.ts.map +1 -1
- package/out/candidate/alias-bags.js +20 -11
- package/out/candidate/alias-bags.js.map +1 -1
- package/out/candidate/bare-admin-names.d.ts +26 -0
- package/out/candidate/bare-admin-names.d.ts.map +1 -0
- package/out/candidate/bare-admin-names.js +38 -0
- package/out/candidate/bare-admin-names.js.map +1 -0
- package/out/candidate/importance.js +2 -2
- package/out/candidate/importance.js.map +1 -1
- package/out/candidate/lookup.d.ts.map +1 -1
- package/out/candidate/lookup.js +35 -19
- package/out/candidate/lookup.js.map +1 -1
- package/out/candidate/own-name.d.ts.map +1 -1
- package/out/candidate/own-name.js +1 -5
- package/out/candidate/own-name.js.map +1 -1
- package/out/candidate/scoring.js +4 -4
- package/out/candidate/scoring.js.map +1 -1
- package/out/capitals.js +2 -2
- package/out/capitals.js.map +1 -1
- package/out/coincident-roles.js +3 -3
- package/out/coincident-roles.js.map +1 -1
- package/out/currency-backfill.js +4 -4
- package/out/currency-backfill.js.map +1 -1
- package/out/env.d.ts +3 -0
- package/out/env.d.ts.map +1 -1
- package/out/fst/autocomplete.js +1 -1
- package/out/fst/autocomplete.js.map +1 -1
- package/out/fst/binary-matcher.d.ts +34 -0
- package/out/fst/binary-matcher.d.ts.map +1 -0
- package/out/fst/binary-matcher.js +294 -0
- package/out/fst/binary-matcher.js.map +1 -0
- package/out/fst/builder.js +1 -1
- package/out/fst/builder.js.map +1 -1
- package/out/fst/deserialize-web.d.ts +11 -6
- package/out/fst/deserialize-web.d.ts.map +1 -1
- package/out/fst/deserialize-web.js +12 -94
- package/out/fst/deserialize-web.js.map +1 -1
- package/out/fst/matcher.js +3 -3
- package/out/fst/matcher.js.map +1 -1
- package/out/fst/serialize.js +1 -1
- package/out/fst/serialize.js.map +1 -1
- package/out/fst.d.ts +1 -0
- package/out/fst.d.ts.map +1 -1
- package/out/fst.js +1 -0
- package/out/fst.js.map +1 -1
- package/out/geonames/aliases.js +2 -2
- package/out/geonames/aliases.js.map +1 -1
- package/out/interpolation.d.ts.map +1 -1
- package/out/interpolation.js +3 -2
- package/out/interpolation.js.map +1 -1
- package/out/lookup.d.ts +2 -2
- package/out/lookup.d.ts.map +1 -1
- package/out/lookup.js +4 -4
- package/out/lookup.js.map +1 -1
- package/out/nsul/lookup.d.ts +1 -1
- package/out/nsul/lookup.d.ts.map +1 -1
- package/out/nsul/lookup.js +1 -1
- package/out/nsul/lookup.js.map +1 -1
- package/out/nsul/schema.d.ts +2 -5
- package/out/nsul/schema.d.ts.map +1 -1
- package/out/nsul/schema.js +2 -7
- package/out/nsul/schema.js.map +1 -1
- package/out/place-importance-schema.js +1 -1
- package/out/place-importance-schema.js.map +1 -1
- package/out/poi/lookup.d.ts +3 -3
- package/out/poi/lookup.d.ts.map +1 -1
- package/out/poi/lookup.js +9 -9
- package/out/poi/lookup.js.map +1 -1
- package/out/proximity-rerank.js +2 -2
- package/out/proximity-rerank.js.map +1 -1
- package/out/reverse.d.ts.map +1 -1
- package/out/reverse.js +7 -5
- package/out/reverse.js.map +1 -1
- package/out/search-fetch.js +3 -3
- package/out/search-fetch.js.map +1 -1
- package/out/sqlite/convention-source.js +1 -1
- package/out/sqlite/convention-source.js.map +1 -1
- package/out/street/centroid.js +2 -2
- package/out/street/centroid.js.map +1 -1
- package/out/street/morphology/fst/builder.d.ts +2 -2
- package/out/street/morphology/fst/builder.js +2 -2
- package/out/street/morphology/fst/builder.js.map +1 -1
- package/out/street/morphology/fst/loader.d.ts +2 -2
- package/out/street/morphology/fst/loader.js +2 -2
- package/out/street/morphology/fst/loader.js.map +1 -1
- package/out/street/normalize.d.ts +16 -0
- package/out/street/normalize.d.ts.map +1 -1
- package/out/street/normalize.js +51 -1
- package/out/street/normalize.js.map +1 -1
- package/out/types.d.ts +7 -0
- package/out/types.d.ts.map +1 -1
- package/out/uprn/existence.d.ts.map +1 -1
- package/out/uprn/existence.js.map +1 -1
- package/out/uprn/lookup.d.ts.map +1 -1
- package/out/uprn/lookup.js +3 -3
- package/out/uprn/lookup.js.map +1 -1
- package/out/uprn/schema.d.ts +1 -1
- package/out/uprn/schema.d.ts.map +1 -1
- package/out/uprn/schema.js +1 -1
- package/out/uprn/schema.js.map +1 -1
- package/out/weights-overlay-linker.js +4 -4
- package/out/weights-overlay-linker.js.map +1 -1
- package/package.json +16 -13
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
*/
|
|
31
31
|
|
|
32
32
|
import type { InterpolationLookup } from "@mailwoman/core/resolver"
|
|
33
|
-
import {
|
|
33
|
+
import { haversine } from "@mailwoman/spatial/geometries/point/distance"
|
|
34
34
|
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
35
35
|
|
|
36
36
|
import type { AddressPointDatabase } from "#address/point/schema"
|
|
@@ -185,7 +185,7 @@ function interpolateFromNeighbors(rows: readonly PointRow[], n: number): Interpo
|
|
|
185
185
|
|
|
186
186
|
if (below && above) {
|
|
187
187
|
const t = (n - below.n) / (above.n - below.n)
|
|
188
|
-
const spanM =
|
|
188
|
+
const spanM = haversine(below.lat, below.lon, above.lat, above.lon) * 1000
|
|
189
189
|
|
|
190
190
|
return {
|
|
191
191
|
lat: below.lat + (above.lat - below.lat) * t,
|
|
@@ -212,8 +212,8 @@ function interpolateFromNeighbors(rows: readonly PointRow[], n: number): Interpo
|
|
|
212
212
|
|
|
213
213
|
const lat = far.lat + (near.lat - far.lat) * t
|
|
214
214
|
const lon = far.lon + (near.lon - far.lon) * t
|
|
215
|
-
const pairM =
|
|
216
|
-
const overshootM =
|
|
215
|
+
const pairM = haversine(near.lat, near.lon, far.lat, far.lon) * 1000
|
|
216
|
+
const overshootM = haversine(lat, lon, near.lat, near.lon) * 1000
|
|
217
217
|
|
|
218
218
|
return {
|
|
219
219
|
lat,
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software.
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
* @file The shared rooftop schema with an H3 spine column and the layer-interface tables.
|
|
6
|
+
*
|
|
7
|
+
* The OpenStreetMap and G-NAF rooftop builders write this shape. Readers of existing address points
|
|
8
|
+
* project only the columns of `address_point` in `#address/point/schema`, so the H3 spine and the
|
|
9
|
+
* layer-interface tables leave their query path unchanged.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { createLayerCoverageTable, createLayerManifestTable, type layerschemadatabase } from "@mailwoman/core/layers"
|
|
13
|
+
import type { Kysely } from "kysely"
|
|
14
|
+
|
|
15
|
+
import { ADDRESS_POINT_COLUMNS, type AddressPointTable, createAddressPointTable } from "#address/point/schema"
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Jitter-stable address/POI join cell.
|
|
19
|
+
* Matches @mailwoman/address-id.
|
|
20
|
+
*/
|
|
21
|
+
export const ADDRESS_POINT_H3_RESOLUTION = 9
|
|
22
|
+
|
|
23
|
+
export interface SpineAddressPointTable extends AddressPointTable {
|
|
24
|
+
/**
|
|
25
|
+
* 48-bit shortened H3 cell stored as a safe SQLite integer.
|
|
26
|
+
*/
|
|
27
|
+
h3_cell: number
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface SpineAddressPointDatabase extends layerschemadatabase {
|
|
31
|
+
address_point: SpineAddressPointTable
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Positional insert order: the shared address-point fields followed by the H3 spine.
|
|
36
|
+
*/
|
|
37
|
+
export const SPINE_ADDRESS_POINT_COLUMNS = [...ADDRESS_POINT_COLUMNS, "h3_cell"] as const
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Create the legacy-compatible domain table plus the H3 spine and the layer-interface tables.
|
|
41
|
+
*/
|
|
42
|
+
export async function createSpineAddressPointTables(db: Kysely<SpineAddressPointDatabase>): Promise<void> {
|
|
43
|
+
await createAddressPointTable(db)
|
|
44
|
+
|
|
45
|
+
await db.schema
|
|
46
|
+
.alterTable("address_point")
|
|
47
|
+
.addColumn("h3_cell", "integer", (column) => column.notNull())
|
|
48
|
+
.execute()
|
|
49
|
+
|
|
50
|
+
await createLayerManifestTable(db)
|
|
51
|
+
await createLayerCoverageTable(db)
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Build after the bulk load so the spine index is not updated on every streaming insert.
|
|
56
|
+
*/
|
|
57
|
+
export async function createSpineAddressPointIndexes(db: Kysely<SpineAddressPointDatabase>): Promise<void> {
|
|
58
|
+
await db.schema.createIndex("idx_ap_h3").on("address_point").column("h3_cell").execute()
|
|
59
|
+
}
|
package/lib/ancestry/backfill.ts
CHANGED
|
@@ -6,7 +6,8 @@
|
|
|
6
6
|
* Repair the truncated ancestry left by the `parent_id` closure. Must run after `populateAncestors` and before the build freezes.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
-
import { readWOFFeature } from "@mailwoman/
|
|
9
|
+
import { readWOFFeature } from "@mailwoman/gazetteer/whosonfirst"
|
|
10
|
+
import { hierarchyAncestorsInAnyBranch } from "@mailwoman/gazetteer/whosonfirst/hierarchy"
|
|
10
11
|
import type { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
11
12
|
import { PathBuilder, type PathBuilderLike } from "path-ts"
|
|
12
13
|
import { Globerator } from "spliterator/node/fs"
|
|
@@ -61,14 +62,6 @@ export async function discoverAdminDataRoots(reposRoot: PathBuilderLike): Promis
|
|
|
61
62
|
return roots
|
|
62
63
|
}
|
|
63
64
|
|
|
64
|
-
// The `aid === id` check filters out a place's own `locality_id`.
|
|
65
|
-
// For a neighborhood, that id identifies a real ancestor.
|
|
66
|
-
function placetypeFromKey(key: string): string | null {
|
|
67
|
-
if (!key.endsWith("_id")) return null
|
|
68
|
-
|
|
69
|
-
return key.slice(0, -3)
|
|
70
|
-
}
|
|
71
|
-
|
|
72
65
|
/**
|
|
73
66
|
* Insert missing ancestor rows for places whose ancestry chain ended before reaching a country.
|
|
74
67
|
*
|
|
@@ -141,22 +134,7 @@ export async function backfillAncestorsFromHierarchy(
|
|
|
141
134
|
continue
|
|
142
135
|
}
|
|
143
136
|
|
|
144
|
-
const seen =
|
|
145
|
-
|
|
146
|
-
for (const branch of hierarchy) {
|
|
147
|
-
for (const [key, val] of Object.entries(branch)) {
|
|
148
|
-
const pt = placetypeFromKey(key)
|
|
149
|
-
|
|
150
|
-
if (!pt) continue
|
|
151
|
-
const aid = Number(val)
|
|
152
|
-
|
|
153
|
-
if (!Number.isFinite(aid) || aid <= 0 || aid === id) continue
|
|
154
|
-
|
|
155
|
-
if (!seen.has(aid)) {
|
|
156
|
-
seen.set(aid, pt)
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
}
|
|
137
|
+
const seen = hierarchyAncestorsInAnyBranch(id, hierarchy)
|
|
160
138
|
|
|
161
139
|
let present = alreadyPresent.get(id)
|
|
162
140
|
|
package/lib/build/candidate.ts
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import { COUNTRY_POPULATION } from "@mailwoman/codex/country"
|
|
8
|
+
import { bareAdminName } from "@mailwoman/codex/country/bare-admin-names"
|
|
8
9
|
import { pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
9
10
|
import { removePath } from "@mailwoman/core/fs/writers"
|
|
10
11
|
import { stringifyJSON } from "@mailwoman/core/json"
|
|
@@ -14,6 +15,7 @@ import type { PathBuilderLike } from "path-ts"
|
|
|
14
15
|
import { POPULATION_CORRECTIONS } from "#build/population-corrections"
|
|
15
16
|
import { explodeAliasBags } from "#candidate/alias-bags"
|
|
16
17
|
import { buildAncestorsSidecar } from "#candidate/ancestors/sidecar"
|
|
18
|
+
import { stageBareAdminNames } from "#candidate/bare-admin-names"
|
|
17
19
|
import { stageCountryDisplayNames } from "#candidate/country-display-names"
|
|
18
20
|
import { foldExtract } from "#candidate/extract-fold"
|
|
19
21
|
import { createCandidateFTS } from "#candidate/fts"
|
|
@@ -205,6 +207,8 @@ export interface BuildCandidateResult {
|
|
|
205
207
|
* Builds the candidate lookup database from a WOF admin database and replaces any existing output.
|
|
206
208
|
* The builder adds an FTS5 trigram index for fuzzy lookup.
|
|
207
209
|
*/
|
|
210
|
+
// repo-health-ignore export-name-affix -- writes candidate.db, and the gazetteer
|
|
211
|
+
// pipeline's buildCandidate runs a whole build around it
|
|
208
212
|
export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<BuildCandidateResult> {
|
|
209
213
|
const progress = opts.onProgress ?? (() => {})
|
|
210
214
|
|
|
@@ -226,7 +230,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
226
230
|
const c = (code || "??").toUpperCase()
|
|
227
231
|
let id = ccodes.get(c)
|
|
228
232
|
|
|
229
|
-
if (id
|
|
233
|
+
if (typeof id !== "number") {
|
|
230
234
|
id = ccodes.size
|
|
231
235
|
ccodes.set(c, id)
|
|
232
236
|
}
|
|
@@ -238,7 +242,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
238
242
|
const p = pt || ""
|
|
239
243
|
let id = ptcodes.get(p)
|
|
240
244
|
|
|
241
|
-
if (id
|
|
245
|
+
if (typeof id !== "number") {
|
|
242
246
|
id = ptcodes.size
|
|
243
247
|
ptcodes.set(p, id)
|
|
244
248
|
}
|
|
@@ -306,7 +310,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
306
310
|
const ptid = ptID(r.placetype as string | null)
|
|
307
311
|
const rid = regionOf.get(sid) ?? 0
|
|
308
312
|
|
|
309
|
-
const wofPop = r.pop === null || r.pop
|
|
313
|
+
const wofPop = r.pop === null || !r.pop ? null : Number(r.pop)
|
|
310
314
|
|
|
311
315
|
const pop =
|
|
312
316
|
POPULATION_CORRECTIONS[sid]?.population ??
|
|
@@ -406,6 +410,16 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
406
410
|
}).toLocaleString()} country surfaces`
|
|
407
411
|
)
|
|
408
412
|
|
|
413
|
+
progress(
|
|
414
|
+
"bare-admin-names",
|
|
415
|
+
`${stageBareAdminNames({
|
|
416
|
+
attrs,
|
|
417
|
+
iso2ByID: new Map([...ccodes].map(([code, id]) => [id, code])),
|
|
418
|
+
stageRow,
|
|
419
|
+
tx: kdb,
|
|
420
|
+
}).toLocaleString()} bare admin names`
|
|
421
|
+
)
|
|
422
|
+
|
|
409
423
|
if (opts.currencyBackfill) {
|
|
410
424
|
const nBackfill = await resurrectCurrencyHoles({
|
|
411
425
|
src,
|
|
@@ -429,14 +443,17 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
429
443
|
|
|
430
444
|
progress("aliases", "exploding alias bags")
|
|
431
445
|
|
|
432
|
-
const
|
|
446
|
+
const iso2ByCountryID = new Map([...ccodes].map(([code, id]) => [id, code]))
|
|
447
|
+
|
|
448
|
+
const { nAlias, nBare, keyCounts, regionOfficialRefused } = explodeAliasBags(src, kdb, attrs, stageRow, {
|
|
433
449
|
regionPlacetypeID: ptID("region"),
|
|
434
450
|
ccID,
|
|
451
|
+
bareName: (countryID, alias) => bareAdminName(iso2ByCountryID.get(countryID) ?? "", alias),
|
|
435
452
|
})
|
|
436
453
|
|
|
437
454
|
progress(
|
|
438
455
|
"aliases",
|
|
439
|
-
`${nAlias.toLocaleString()} aliases` +
|
|
456
|
+
`${nAlias.toLocaleString()} aliases; ${nBare.toLocaleString()} bare admin aliases` +
|
|
440
457
|
(regionOfficialRefused
|
|
441
458
|
? `; ${regionOfficialRefused.toLocaleString()} region aliases refused as another region's official name`
|
|
442
459
|
: "")
|
|
@@ -61,9 +61,20 @@ export function explodeAliasBags(
|
|
|
61
61
|
out: DatabaseClient<CandidateDatabase>,
|
|
62
62
|
attrs: Map<number, PlaceAttrs>,
|
|
63
63
|
stageRow: StageRow,
|
|
64
|
-
opts: {
|
|
65
|
-
|
|
64
|
+
opts: {
|
|
65
|
+
regionPlacetypeID?: number
|
|
66
|
+
ccID?: (code: string | null) => number
|
|
67
|
+
/**
|
|
68
|
+
* The bare form of an alias in its place's country, staged as one more alias: `渋谷` for the alias `渋谷区`.
|
|
69
|
+
*
|
|
70
|
+
* A Japanese place's primary name is romanized, so its suffixed name reaches
|
|
71
|
+
* the candidate only as an alias.
|
|
72
|
+
*/
|
|
73
|
+
bareName?: (countryID: number, alias: string) => string | null
|
|
74
|
+
} = {}
|
|
75
|
+
): { nAlias: number; nBare: number; keyCounts: Map<number, number>; regionOfficialRefused: number } {
|
|
66
76
|
let nAlias = 0
|
|
77
|
+
let nBare = 0
|
|
67
78
|
let regionOfficialRefused = 0
|
|
68
79
|
const keyCounts = new Map<number, number>()
|
|
69
80
|
|
|
@@ -87,24 +98,31 @@ export function explodeAliasBags(
|
|
|
87
98
|
// includes surrounding whitespace and the last one is empty.
|
|
88
99
|
// `normalizeLocalityForKey` folds both away.
|
|
89
100
|
for (const piece of alt.split(ALIAS_SEPARATOR)) {
|
|
90
|
-
const
|
|
101
|
+
const bare = opts.bareName?.(a.cid, piece.trim())
|
|
102
|
+
const keys = [normalizeLocalityForKey(piece), bare ? normalizeLocalityForKey(bare) : ""]
|
|
91
103
|
|
|
92
|
-
|
|
93
|
-
|
|
104
|
+
for (const [position, k] of keys.entries()) {
|
|
105
|
+
if (!k || seen.has(k)) continue
|
|
106
|
+
seen.add(k)
|
|
94
107
|
|
|
95
|
-
|
|
96
|
-
|
|
108
|
+
if (isRegion) {
|
|
109
|
+
const holder = official.get(`${a.cid}\0${k}`)
|
|
97
110
|
|
|
98
|
-
|
|
99
|
-
|
|
111
|
+
if (typeof holder === "number" && holder !== id) {
|
|
112
|
+
regionOfficialRefused++
|
|
100
113
|
|
|
101
|
-
|
|
114
|
+
continue
|
|
115
|
+
}
|
|
102
116
|
}
|
|
103
|
-
}
|
|
104
117
|
|
|
105
|
-
|
|
118
|
+
stageRow(k, a, id, 0)
|
|
106
119
|
|
|
107
|
-
|
|
120
|
+
if (position) {
|
|
121
|
+
nBare++
|
|
122
|
+
} else {
|
|
123
|
+
nAlias++
|
|
124
|
+
}
|
|
125
|
+
}
|
|
108
126
|
}
|
|
109
127
|
|
|
110
128
|
keyCounts.set(id, seen.size)
|
|
@@ -112,5 +130,5 @@ export function explodeAliasBags(
|
|
|
112
130
|
|
|
113
131
|
out.exec("COMMIT")
|
|
114
132
|
|
|
115
|
-
return { nAlias, keyCounts, regionOfficialRefused }
|
|
133
|
+
return { nAlias, nBare, keyCounts, regionOfficialRefused }
|
|
116
134
|
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
* @file A candidate build pass that keys each admin place under its bare name as well as its full one.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { bareAdminName } from "@mailwoman/codex/country/bare-admin-names"
|
|
9
|
+
|
|
10
|
+
import type { PlaceAttrs, StageRow } from "#candidate/place-attrs"
|
|
11
|
+
import { normalizeLocalityForKey } from "#street/normalize"
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Stages an alias row under the bare form of every place name a country writes with an admin-unit suffix.
|
|
15
|
+
*
|
|
16
|
+
* A bare `渋谷` otherwise reaches only the places whose full name is `渋谷`, a Hyogo
|
|
17
|
+
* and a Chiba locality with no recorded population, and never Tokyo's `渋谷区`.
|
|
18
|
+
* The alias row includes the ward's own population, so the candidate ordering
|
|
19
|
+
* ranks it above the unmeasured exact matches.
|
|
20
|
+
*
|
|
21
|
+
* @returns The number of alias rows staged.
|
|
22
|
+
*/
|
|
23
|
+
export function stageBareAdminNames(ctx: {
|
|
24
|
+
attrs: Map<number, PlaceAttrs>
|
|
25
|
+
iso2ByID: Map<number, string>
|
|
26
|
+
stageRow: StageRow
|
|
27
|
+
tx: { exec(sql: string): void }
|
|
28
|
+
}): number {
|
|
29
|
+
let staged = 0
|
|
30
|
+
|
|
31
|
+
ctx.tx.exec("BEGIN")
|
|
32
|
+
|
|
33
|
+
for (const [sid, a] of ctx.attrs) {
|
|
34
|
+
const iso2 = ctx.iso2ByID.get(a.cid)
|
|
35
|
+
|
|
36
|
+
if (!iso2) continue
|
|
37
|
+
|
|
38
|
+
const bare = bareAdminName(iso2, a.name)
|
|
39
|
+
|
|
40
|
+
if (!bare) continue
|
|
41
|
+
|
|
42
|
+
const key = normalizeLocalityForKey(bare)
|
|
43
|
+
|
|
44
|
+
if (!key || key === a.pkey) continue
|
|
45
|
+
|
|
46
|
+
ctx.stageRow(key, a, sid, 0)
|
|
47
|
+
|
|
48
|
+
staged++
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
ctx.tx.exec("COMMIT")
|
|
52
|
+
|
|
53
|
+
return staged
|
|
54
|
+
}
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
* population assigned it.
|
|
30
30
|
*/
|
|
31
31
|
|
|
32
|
-
import {
|
|
32
|
+
import { haversine } from "@mailwoman/spatial/geometries/point/distance"
|
|
33
33
|
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
34
34
|
import type { PathBuilderLike } from "path-ts"
|
|
35
35
|
|
|
@@ -144,7 +144,7 @@ export class ImportanceIndex {
|
|
|
144
144
|
let bestKm = Infinity
|
|
145
145
|
|
|
146
146
|
for (const place of group) {
|
|
147
|
-
const km =
|
|
147
|
+
const km = haversine(lat, lon, place.lat, place.lon)
|
|
148
148
|
|
|
149
149
|
if (km < bestKm) {
|
|
150
150
|
bestKm = km
|
package/lib/candidate/lookup.ts
CHANGED
|
@@ -10,7 +10,7 @@ import { type Ancestor, type GazetteerArtifactCoverage, referentialFromPopulatio
|
|
|
10
10
|
import { allRows } from "@mailwoman/core/utils"
|
|
11
11
|
import { jaroWinkler, levenshteinSimilarity } from "@mailwoman/match/comparators"
|
|
12
12
|
import { partitionByContainment } from "@mailwoman/resolver"
|
|
13
|
-
import {
|
|
13
|
+
import { haversine } from "@mailwoman/spatial/geometries/point/distance"
|
|
14
14
|
import type { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
15
15
|
import { SQLiteLookup, type SQLiteLookupOptions } from "@mailwoman/sqlite/lookup"
|
|
16
16
|
|
|
@@ -67,6 +67,18 @@ type CandidateRow = Pick<
|
|
|
67
67
|
// Optional because older artifacts may not include `importance`.
|
|
68
68
|
Partial<Pick<CandidateTable, "importance">>
|
|
69
69
|
|
|
70
|
+
/**
|
|
71
|
+
* What a containment re-rank needs beside the rows: the keys to inject contained
|
|
72
|
+
* misses from and the shape filters the initial probe applied.
|
|
73
|
+
*/
|
|
74
|
+
interface ContainmentOptions {
|
|
75
|
+
nameKey: NameKey
|
|
76
|
+
strippedKey: NameKey
|
|
77
|
+
shapeFilters: string[]
|
|
78
|
+
shapeParams: Array<string | number>
|
|
79
|
+
limit: number
|
|
80
|
+
}
|
|
81
|
+
|
|
70
82
|
const FUZZY_FETCH = 40
|
|
71
83
|
|
|
72
84
|
const WORD_FUZZY_MIN = 0.85
|
|
@@ -177,7 +189,7 @@ export class WOFCandidateTableLookup extends SQLiteLookup<CandidateDatabase> imp
|
|
|
177
189
|
|
|
178
190
|
const bandIDs = [...REGION_CLASS_PLACETYPES, "country"]
|
|
179
191
|
.map((placetype) => this.#placetypeToID.get(placetype))
|
|
180
|
-
.filter((id)
|
|
192
|
+
.filter((id) => typeof id === "number")
|
|
181
193
|
|
|
182
194
|
if (bandIDs.length) {
|
|
183
195
|
this.#qualifierProbe = this.database.prepare(
|
|
@@ -219,7 +231,7 @@ export class WOFCandidateTableLookup extends SQLiteLookup<CandidateDatabase> imp
|
|
|
219
231
|
|
|
220
232
|
const cached = this.#intervalCache.get(sprID)
|
|
221
233
|
|
|
222
|
-
if (cached
|
|
234
|
+
if (cached) return cached
|
|
223
235
|
|
|
224
236
|
const row = this.#intervalProbe.get(sprID) as { pre: number; post: number } | undefined
|
|
225
237
|
const label = row ? { pre: Number(row.pre), post: Number(row.post) } : null
|
|
@@ -260,23 +272,30 @@ export class WOFCandidateTableLookup extends SQLiteLookup<CandidateDatabase> imp
|
|
|
260
272
|
}
|
|
261
273
|
|
|
262
274
|
/**
|
|
263
|
-
* Apply admin-containment re-rank
|
|
264
|
-
* then
|
|
275
|
+
* Apply admin-containment re-rank for a parsed region qualifier: resolve the
|
|
276
|
+
* qualifier's region rows, then {@linkcode #applyContainment}.
|
|
265
277
|
*/
|
|
266
278
|
#applyAdminContainment(
|
|
267
279
|
rows: Array<RankedRow<CandidateRow>>,
|
|
268
280
|
qualifier: string,
|
|
269
281
|
country: string | null,
|
|
270
|
-
opts:
|
|
271
|
-
nameKey: NameKey
|
|
272
|
-
strippedKey: NameKey
|
|
273
|
-
shapeFilters: string[]
|
|
274
|
-
shapeParams: Array<string | number>
|
|
275
|
-
limit: number
|
|
276
|
-
}
|
|
282
|
+
opts: ContainmentOptions
|
|
277
283
|
): Array<RankedRow<CandidateRow>> {
|
|
278
|
-
|
|
284
|
+
return this.#applyContainment(rows, this.#qualifierRegionIDs(qualifier, country), opts)
|
|
285
|
+
}
|
|
279
286
|
|
|
287
|
+
/**
|
|
288
|
+
* Stamp containment in the qualifier rows, inject contained misses, then partition contained-first.
|
|
289
|
+
*
|
|
290
|
+
* The qualifier rows are a parsed region's rows or one resolved parent place (#2804).
|
|
291
|
+
* The injection reads the same name key under the shape filters and under the sub-locality placetypes,
|
|
292
|
+
* so a dependent locality the locality band excludes re-enters the list when its parent contains it.
|
|
293
|
+
*/
|
|
294
|
+
#applyContainment(
|
|
295
|
+
rows: Array<RankedRow<CandidateRow>>,
|
|
296
|
+
qualifierIDs: ReadonlySet<number>,
|
|
297
|
+
opts: ContainmentOptions
|
|
298
|
+
): Array<RankedRow<CandidateRow>> {
|
|
280
299
|
if (!qualifierIDs.size) {
|
|
281
300
|
for (const row of rows) {
|
|
282
301
|
row.containedByQualifier = false
|
|
@@ -326,7 +345,7 @@ export class WOFCandidateTableLookup extends SQLiteLookup<CandidateDatabase> imp
|
|
|
326
345
|
// Also inject contained neighborhood-band namesakes (primary-keyed only).
|
|
327
346
|
const bandIDs = ["neighbourhood", "macrohood", "microhood"]
|
|
328
347
|
.map((placetype) => this.#placetypeToID.get(placetype))
|
|
329
|
-
.filter((id)
|
|
348
|
+
.filter((id) => typeof id === "number")
|
|
330
349
|
|
|
331
350
|
if (bandIDs.length) {
|
|
332
351
|
const bandSQL =
|
|
@@ -375,7 +394,7 @@ export class WOFCandidateTableLookup extends SQLiteLookup<CandidateDatabase> imp
|
|
|
375
394
|
#postcodeAnchor(postcode: string, country?: string): { lat: number; lon: number } | null {
|
|
376
395
|
const placetypeID = this.#placetypeToID.get("postalcode")
|
|
377
396
|
|
|
378
|
-
if (placetypeID
|
|
397
|
+
if (typeof placetypeID !== "number") return null
|
|
379
398
|
|
|
380
399
|
const conds = ["name_key = ?", "placetype_id = ?"]
|
|
381
400
|
const params: Array<string | number> = [postcode.replaceAll(/\s+/g, ""), placetypeID]
|
|
@@ -383,7 +402,7 @@ export class WOFCandidateTableLookup extends SQLiteLookup<CandidateDatabase> imp
|
|
|
383
402
|
if (country) {
|
|
384
403
|
const countryID = this.#countryToID.get(country.toUpperCase())
|
|
385
404
|
|
|
386
|
-
if (countryID
|
|
405
|
+
if (typeof countryID !== "number") return null // Unknown country in this artifact.
|
|
387
406
|
conds.push("country_id = ?")
|
|
388
407
|
params.push(countryID)
|
|
389
408
|
}
|
|
@@ -457,7 +476,7 @@ export class WOFCandidateTableLookup extends SQLiteLookup<CandidateDatabase> imp
|
|
|
457
476
|
if (query.country) {
|
|
458
477
|
const cid = this.#countryToID.get(query.country.toUpperCase())
|
|
459
478
|
|
|
460
|
-
if (cid
|
|
479
|
+
if (typeof cid !== "number") return [] // Unknown country in this artifact.
|
|
461
480
|
filters.push("country_id = ?")
|
|
462
481
|
filterParams.push(cid)
|
|
463
482
|
}
|
|
@@ -468,7 +487,7 @@ export class WOFCandidateTableLookup extends SQLiteLookup<CandidateDatabase> imp
|
|
|
468
487
|
|
|
469
488
|
const ids = expandPlacetypeFilter(want as readonly string[])
|
|
470
489
|
.map((t) => this.#placetypeToID.get(t))
|
|
471
|
-
.filter((v)
|
|
490
|
+
.filter((v) => typeof v === "number")
|
|
472
491
|
|
|
473
492
|
if (!ids.length) return []
|
|
474
493
|
shapeFilters.push(`placetype_id IN (${ids.map(() => "?").join(",")})`)
|
|
@@ -599,7 +618,7 @@ export class WOFCandidateTableLookup extends SQLiteLookup<CandidateDatabase> imp
|
|
|
599
618
|
const outsideThreshold: RankedRow<CandidateRow>[] = []
|
|
600
619
|
|
|
601
620
|
for (const row of rows) {
|
|
602
|
-
const distanceKm =
|
|
621
|
+
const distanceKm = haversine(anchor.lat, anchor.lon, Number(row.latitude), Number(row.longitude))
|
|
603
622
|
|
|
604
623
|
if (distanceKm <= POSTCODE_CONTAINMENT_THRESHOLD_KM) {
|
|
605
624
|
insideThreshold.push({ row, distanceKm })
|
|
@@ -617,14 +636,23 @@ export class WOFCandidateTableLookup extends SQLiteLookup<CandidateDatabase> imp
|
|
|
617
636
|
}
|
|
618
637
|
|
|
619
638
|
// Admin-containment re-rank runs last.
|
|
639
|
+
const containmentOptions: ContainmentOptions = {
|
|
640
|
+
nameKey,
|
|
641
|
+
strippedKey: normalizeLocalityForKey(stripLocalityQualifier(text)),
|
|
642
|
+
shapeFilters,
|
|
643
|
+
shapeParams,
|
|
644
|
+
limit,
|
|
645
|
+
}
|
|
646
|
+
|
|
620
647
|
if (query.regionQualifier?.trim() && this.#qualifierProbe && this.#wantsLocality(query.placetype)) {
|
|
621
|
-
rows = this.#applyAdminContainment(rows, query.regionQualifier.trim(), query.country ?? null,
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
648
|
+
rows = this.#applyAdminContainment(rows, query.regionQualifier.trim(), query.country ?? null, containmentOptions)
|
|
649
|
+
} else if (
|
|
650
|
+
typeof query.parentQualifierID === "number" &&
|
|
651
|
+
this.#ancestorsProbe &&
|
|
652
|
+
this.#wantsLocality(query.placetype)
|
|
653
|
+
) {
|
|
654
|
+
// A resolved parent qualifies by id, so no name probe runs (#2804).
|
|
655
|
+
rows = this.#applyContainment(rows, new Set([query.parentQualifierID]), containmentOptions)
|
|
628
656
|
}
|
|
629
657
|
|
|
630
658
|
const candidates = rows.map((row): PlaceCandidate => {
|
|
@@ -114,11 +114,7 @@ export function romanizeNameKey(key: string): string | null {
|
|
|
114
114
|
out += CYRILLIC_TO_LATIN[ch] ?? ch
|
|
115
115
|
}
|
|
116
116
|
|
|
117
|
-
out = out
|
|
118
|
-
.normalize("NFD")
|
|
119
|
-
.replaceAll(/[̀-ͯ]/g, "")
|
|
120
|
-
.replaceAll(/\s+/g, " ")
|
|
121
|
-
.trim()
|
|
117
|
+
out = out.normalize("NFD").replaceAll(/[̀-ͯ]/g, "").replaceAll(/\s+/g, " ").trim()
|
|
122
118
|
|
|
123
119
|
return /^[a-z0-9 ]*$/.test(out) ? out : null
|
|
124
120
|
}
|
package/lib/candidate/scoring.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import { compareReferential, referentialFromPopulation } from "@mailwoman/core/resolver"
|
|
8
|
-
import {
|
|
8
|
+
import { haversine } from "@mailwoman/spatial/geometries/point/distance"
|
|
9
9
|
import type { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
10
10
|
|
|
11
11
|
import { exactMatchIDs, officialNameIDs } from "#exact-match"
|
|
@@ -47,7 +47,7 @@ export function candidateFromSearchRow(
|
|
|
47
47
|
score += weights.countryMatchBoost
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
-
if (query.parentID
|
|
50
|
+
if (query.parentID) {
|
|
51
51
|
score += row.parent_id === query.parentID ? weights.directChildBoost : weights.descendantBoost
|
|
52
52
|
}
|
|
53
53
|
|
|
@@ -74,7 +74,7 @@ export function candidateFromSearchRow(
|
|
|
74
74
|
let scoreTerm = 0
|
|
75
75
|
|
|
76
76
|
for (const h of hints) {
|
|
77
|
-
const d =
|
|
77
|
+
const d = haversine(h.lat, h.lon, row.lat, row.lon)
|
|
78
78
|
const decay = h.weight / (1 + d / weights.proximityScaleKm)
|
|
79
79
|
const prom = decay * weights.biasBoost
|
|
80
80
|
|
|
@@ -184,7 +184,7 @@ export function rankCandidates<DB>(
|
|
|
184
184
|
.map((c) => c.id as number),
|
|
185
185
|
query.text
|
|
186
186
|
)
|
|
187
|
-
:
|
|
187
|
+
: null
|
|
188
188
|
|
|
189
189
|
const kind = (c: PlaceCandidate): number => {
|
|
190
190
|
if (!exactIDs.has(c.id as number)) return 0
|
package/lib/capitals.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* radius a centroid-drift allowance rather than a catchment.
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import {
|
|
18
|
+
import { haversine } from "@mailwoman/spatial/geometries/point/distance"
|
|
19
19
|
|
|
20
20
|
import { normalizeLocalityForKey } from "#street/normalize"
|
|
21
21
|
|
|
@@ -129,7 +129,7 @@ export class CapitalIndex {
|
|
|
129
129
|
if (
|
|
130
130
|
point.level > best &&
|
|
131
131
|
point.keys.has(key) &&
|
|
132
|
-
|
|
132
|
+
haversine(latitude, longitude, point.latitude, point.longitude) <= CAPITAL_MATCH_RADIUS_KM
|
|
133
133
|
) {
|
|
134
134
|
best = point.level
|
|
135
135
|
}
|
package/lib/coincident-roles.ts
CHANGED
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
|
|
39
39
|
import type { CoincidentLocality } from "@mailwoman/core/resolver"
|
|
40
40
|
import { allRows } from "@mailwoman/core/utils"
|
|
41
|
-
import {
|
|
41
|
+
import { haversine } from "@mailwoman/spatial/geometries/point/distance"
|
|
42
42
|
import type { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
43
43
|
import { tableExists } from "@mailwoman/sqlite/introspection"
|
|
44
44
|
|
|
@@ -190,8 +190,8 @@ export function buildCoincidentRoles(
|
|
|
190
190
|
|
|
191
191
|
try {
|
|
192
192
|
for (const c of candidates) {
|
|
193
|
-
const dist =
|
|
194
|
-
const diag =
|
|
193
|
+
const dist = haversine(c.rlat, c.rlon, c.llat, c.llon)
|
|
194
|
+
const diag = haversine(c.min_latitude, c.min_longitude, c.max_latitude, c.max_longitude)
|
|
195
195
|
const tolerance = Math.max(toleranceFraction * diag, minToleranceKm)
|
|
196
196
|
|
|
197
197
|
if (dist > tolerance) continue
|
package/lib/currency-backfill.ts
CHANGED
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
10
|
-
import { isStrictlyFiner } from "@mailwoman/
|
|
11
|
-
import {
|
|
10
|
+
import { isStrictlyFiner } from "@mailwoman/gazetteer/whosonfirst"
|
|
11
|
+
import { haversine } from "@mailwoman/spatial/geometries/point/distance"
|
|
12
12
|
import type { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
13
13
|
import { type PathBuilderLike, resolvePath } from "path-ts"
|
|
14
14
|
import { TSVSpliterator } from "spliterator"
|
|
@@ -208,7 +208,7 @@ export async function resurrectCurrencyHoles(ctx: {
|
|
|
208
208
|
// A live row blocks unless it is strictly finer than the dead one.
|
|
209
209
|
// An unranked placetype also blocks because the failure mode is inventing a place.
|
|
210
210
|
const liveNear = liveStmt.all(cc, name).some((row) => {
|
|
211
|
-
if (
|
|
211
|
+
if (haversine(dLat, dLon, Number(row.latitude), Number(row.longitude)) > CURRENCY_BACKFILL_RADIUS_KM) {
|
|
212
212
|
return false
|
|
213
213
|
}
|
|
214
214
|
|
|
@@ -222,7 +222,7 @@ export async function resurrectCurrencyHoles(ctx: {
|
|
|
222
222
|
}
|
|
223
223
|
|
|
224
224
|
const near = (attestors.get(pkey) ?? []).filter(
|
|
225
|
-
(g) =>
|
|
225
|
+
(g) => haversine(dLat, dLon, g.lat, g.lon) <= CURRENCY_BACKFILL_RADIUS_KM
|
|
226
226
|
)
|
|
227
227
|
|
|
228
228
|
if (!near.length) {
|
package/lib/fst/autocomplete.ts
CHANGED
|
@@ -135,7 +135,7 @@ class FSTReader implements AncestrieReaderLike<PlaceEntry> {
|
|
|
135
135
|
|
|
136
136
|
entriesAt(stateID: number, limit?: number): AncestrieRecord<PlaceEntry>[] {
|
|
137
137
|
const places = this.#fst.accepting(stateID)
|
|
138
|
-
const selected = limit
|
|
138
|
+
const selected = typeof limit !== "number" ? places : topByReferential(places, limit)
|
|
139
139
|
|
|
140
140
|
return selected.map((entry) => {
|
|
141
141
|
this.#chains.set(entry.wofID, entry.parentChain)
|