@mailwoman/resolver-wof-sqlite 9.4.0 → 10.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/address/index.ts +3 -3
- package/lib/address/{point.ts → point/index.ts} +1 -1
- package/lib/address/{point-interpolation.ts → point/interpolation.ts} +1 -1
- package/lib/ancestry/backfill.ts +4 -2
- package/lib/{build-candidate.ts → build/candidate.ts} +43 -25
- package/lib/candidate/alias-bags.ts +68 -7
- package/lib/{candidate-ancestors-schema.ts → candidate/ancestors/schema.ts} +1 -1
- package/lib/candidate/{ancestors-sidecar.ts → ancestors/sidecar.ts} +2 -2
- package/lib/candidate/extract-fold.ts +3 -3
- package/lib/{candidate-importance.ts → candidate/importance.ts} +1 -1
- package/lib/{candidate-lookup.ts → candidate/lookup.ts} +6 -10
- package/lib/candidate/name-roles.ts +2 -2
- package/lib/{candidate-schema.ts → candidate/schema.ts} +1 -1
- package/lib/currency-backfill.ts +2 -2
- package/lib/fst/deserialize-web.ts +2 -2
- package/lib/fst/serialize.ts +9 -9
- package/lib/geonames/aliases.ts +1 -7
- package/lib/geonames/postal.ts +1 -7
- package/lib/index.ts +18 -25
- package/lib/lookup.ts +5 -4
- package/lib/postal/{city-alias-lookup.ts → city/alias/lookup.ts} +1 -1
- package/lib/postal/index.ts +3 -3
- package/lib/primary-preference.ts +1 -1
- package/lib/street/{centroid.ts → centroid/index.ts} +1 -1
- package/lib/street/index.ts +4 -4
- package/lib/street/{morphology-fst-builder.ts → morphology/fst/builder.ts} +8 -2
- package/lib/street/{morphology-fst-loader.ts → morphology/fst/loader.ts} +1 -1
- package/lib/weights-overlay-linker.ts +4 -4
- package/out/address/index.d.ts +3 -3
- package/out/address/index.d.ts.map +1 -1
- package/out/address/index.js +3 -3
- package/out/address/index.js.map +1 -1
- package/out/address/{point.d.ts → point/index.d.ts} +2 -2
- package/out/address/point/index.d.ts.map +1 -0
- package/out/address/{point.js → point/index.js} +1 -1
- package/out/address/point/index.js.map +1 -0
- package/out/address/{point-interpolation.d.ts → point/interpolation.d.ts} +2 -2
- package/out/address/point/interpolation.d.ts.map +1 -0
- package/out/address/{point-interpolation.js → point/interpolation.js} +1 -1
- package/out/address/point/interpolation.js.map +1 -0
- package/out/address/{point-schema.d.ts → point/schema.d.ts} +1 -1
- package/out/address/point/schema.d.ts.map +1 -0
- package/out/address/{point-schema.js → point/schema.js} +1 -1
- package/out/address/point/schema.js.map +1 -0
- package/out/ancestry/backfill.d.ts.map +1 -1
- package/out/ancestry/backfill.js +4 -2
- package/out/ancestry/backfill.js.map +1 -1
- package/out/{build-candidate.d.ts → build/candidate.d.ts} +7 -1
- package/out/build/candidate.d.ts.map +1 -0
- package/out/{build-candidate.js → build/candidate.js} +28 -21
- package/out/build/candidate.js.map +1 -0
- package/out/{build-slim.d.ts → build/slim.d.ts} +1 -1
- package/out/build/slim.d.ts.map +1 -0
- package/out/{build-slim.js → build/slim.js} +1 -1
- package/out/build/slim.js.map +1 -0
- package/out/candidate/alias-bags.d.ts +10 -2
- package/out/candidate/alias-bags.d.ts.map +1 -1
- package/out/candidate/alias-bags.js +49 -5
- package/out/candidate/alias-bags.js.map +1 -1
- package/out/{candidate-ancestors-schema.d.ts → candidate/ancestors/schema.d.ts} +2 -2
- package/out/candidate/ancestors/schema.d.ts.map +1 -0
- package/out/{candidate-ancestors-schema.js → candidate/ancestors/schema.js} +1 -1
- package/out/candidate/ancestors/schema.js.map +1 -0
- package/out/candidate/{ancestors-sidecar.d.ts → ancestors/sidecar.d.ts} +2 -2
- package/out/candidate/ancestors/sidecar.d.ts.map +1 -0
- package/out/candidate/{ancestors-sidecar.js → ancestors/sidecar.js} +2 -2
- package/out/candidate/ancestors/sidecar.js.map +1 -0
- package/out/candidate/extract-fold.d.ts +1 -1
- package/out/candidate/extract-fold.d.ts.map +1 -1
- package/out/candidate/extract-fold.js +2 -2
- package/out/candidate/extract-fold.js.map +1 -1
- package/out/{candidate-fts.d.ts → candidate/fts.d.ts} +1 -1
- package/out/candidate/fts.d.ts.map +1 -0
- package/out/{candidate-fts.js → candidate/fts.js} +1 -1
- package/out/candidate/fts.js.map +1 -0
- package/out/{candidate-importance.d.ts → candidate/importance.d.ts} +1 -1
- package/out/candidate/importance.d.ts.map +1 -0
- package/out/{candidate-importance.js → candidate/importance.js} +1 -1
- package/out/candidate/importance.js.map +1 -0
- package/out/{candidate-lookup.d.ts → candidate/lookup.d.ts} +2 -2
- package/out/candidate/lookup.d.ts.map +1 -0
- package/out/{candidate-lookup.js → candidate/lookup.js} +6 -5
- package/out/candidate/lookup.js.map +1 -0
- package/out/candidate/name-roles.d.ts +1 -1
- package/out/candidate/name-roles.d.ts.map +1 -1
- package/out/candidate/name-roles.js +1 -1
- package/out/candidate/name-roles.js.map +1 -1
- package/out/{candidate-schema.d.ts → candidate/schema.d.ts} +2 -2
- package/out/candidate/schema.d.ts.map +1 -0
- package/out/{candidate-schema.js → candidate/schema.js} +1 -1
- package/out/candidate/schema.js.map +1 -0
- package/out/{candidate-scoring.d.ts → candidate/scoring.d.ts} +1 -1
- package/out/candidate/scoring.d.ts.map +1 -0
- package/out/{candidate-scoring.js → candidate/scoring.js} +1 -1
- package/out/candidate/scoring.js.map +1 -0
- package/out/capitals.d.ts.map +1 -1
- package/out/coincident-roles.js.map +1 -1
- package/out/convention/index.d.ts.map +1 -1
- package/out/convention/index.js.map +1 -1
- package/out/currency-backfill.d.ts +2 -2
- package/out/currency-backfill.d.ts.map +1 -1
- package/out/env.d.ts +1 -1
- package/out/fst/autocomplete.js.map +1 -1
- package/out/fst/deserialize-web.js +2 -2
- package/out/fst/deserialize-web.js.map +1 -1
- package/out/fst/matcher.d.ts.map +1 -1
- package/out/fst/serialize.js +9 -9
- package/out/fst/serialize.js.map +1 -1
- package/out/fts/index.js.map +1 -1
- package/out/geonames/aliases.d.ts +0 -5
- package/out/geonames/aliases.d.ts.map +1 -1
- package/out/geonames/aliases.js +1 -6
- package/out/geonames/aliases.js.map +1 -1
- package/out/geonames/postal.d.ts +0 -6
- package/out/geonames/postal.d.ts.map +1 -1
- package/out/geonames/postal.js +1 -6
- package/out/geonames/postal.js.map +1 -1
- package/out/index.d.ts +18 -19
- package/out/index.d.ts.map +1 -1
- package/out/index.js +13 -14
- package/out/index.js.map +1 -1
- package/out/interpolation.d.ts.map +1 -1
- package/out/lookup.d.ts +2 -2
- package/out/lookup.d.ts.map +1 -1
- package/out/lookup.js +3 -3
- package/out/lookup.js.map +1 -1
- package/out/nsul/lookup.d.ts.map +1 -1
- package/out/place-importance-schema.d.ts.map +1 -1
- package/out/poi/lookup.d.ts.map +1 -1
- package/out/poi/schema.d.ts.map +1 -1
- package/out/postal/{city-alias-lookup.d.ts → city/alias/lookup.d.ts} +2 -2
- package/out/postal/city/alias/lookup.d.ts.map +1 -0
- package/out/postal/{city-alias-lookup.js → city/alias/lookup.js} +1 -1
- package/out/postal/city/alias/lookup.js.map +1 -0
- package/out/postal/{city-alias-schema.d.ts → city/alias/schema.d.ts} +1 -1
- package/out/postal/city/alias/schema.d.ts.map +1 -0
- package/out/postal/{city-alias-schema.js → city/alias/schema.js} +1 -1
- package/out/postal/city/alias/schema.js.map +1 -0
- package/out/postal/{city-candidate-schema.d.ts → city/candidate-schema.d.ts} +1 -1
- package/out/postal/city/candidate-schema.d.ts.map +1 -0
- package/out/postal/{city-candidate-schema.js → city/candidate-schema.js} +1 -1
- package/out/postal/city/candidate-schema.js.map +1 -0
- package/out/postal/index.d.ts +3 -3
- package/out/postal/index.js +3 -3
- package/out/postcode-point-lookup.d.ts.map +1 -1
- package/out/primary-preference.d.ts +1 -1
- package/out/reverse.d.ts.map +1 -1
- package/out/reverse.js.map +1 -1
- package/out/{sqlite-convention-source.d.ts → sqlite/convention-source.d.ts} +1 -1
- package/out/sqlite/convention-source.d.ts.map +1 -0
- package/out/{sqlite-convention-source.js → sqlite/convention-source.js} +1 -1
- package/out/sqlite/convention-source.js.map +1 -0
- package/out/{sqlite-utils.d.ts → sqlite/utils.d.ts} +1 -1
- package/out/sqlite/utils.d.ts.map +1 -0
- package/out/{sqlite-utils.js → sqlite/utils.js} +1 -1
- package/out/sqlite/utils.js.map +1 -0
- package/out/street/{centroid.d.ts → centroid/index.d.ts} +1 -1
- package/out/street/centroid/index.d.ts.map +1 -0
- package/out/street/{centroid.js → centroid/index.js} +1 -1
- package/out/street/centroid/index.js.map +1 -0
- package/out/street/{centroid-schema.d.ts → centroid/schema.d.ts} +1 -1
- package/out/street/centroid/schema.d.ts.map +1 -0
- package/out/street/{centroid-schema.js → centroid/schema.js} +1 -1
- package/out/street/centroid/schema.js.map +1 -0
- package/out/street/index.d.ts +4 -4
- package/out/street/index.d.ts.map +1 -1
- package/out/street/index.js +4 -4
- package/out/street/index.js.map +1 -1
- package/out/street/{morphology-fst-builder.d.ts → morphology/fst/builder.d.ts} +1 -1
- package/out/street/morphology/fst/builder.d.ts.map +1 -0
- package/out/street/{morphology-fst-builder.js → morphology/fst/builder.js} +8 -3
- package/out/street/morphology/fst/builder.js.map +1 -0
- package/out/street/{morphology-fst-loader.d.ts → morphology/fst/loader.d.ts} +1 -1
- package/out/street/morphology/fst/loader.d.ts.map +1 -0
- package/out/street/{morphology-fst-loader.js → morphology/fst/loader.js} +2 -2
- package/out/street/morphology/fst/loader.js.map +1 -0
- package/out/street/name-lookup.d.ts.map +1 -1
- package/out/street/name-lookup.js.map +1 -1
- package/out/street/normalize.js.map +1 -1
- package/out/street/segment-schema.d.ts.map +1 -1
- package/out/uprn/lookup.d.ts.map +1 -1
- package/out/weights-overlay-linker.js +3 -3
- package/out/weights-overlay-linker.js.map +1 -1
- package/package.json +64 -64
- package/out/address/point-interpolation.d.ts.map +0 -1
- package/out/address/point-interpolation.js.map +0 -1
- package/out/address/point-schema.d.ts.map +0 -1
- package/out/address/point-schema.js.map +0 -1
- package/out/address/point.d.ts.map +0 -1
- package/out/address/point.js.map +0 -1
- package/out/build-candidate.d.ts.map +0 -1
- package/out/build-candidate.js.map +0 -1
- package/out/build-slim.d.ts.map +0 -1
- package/out/build-slim.js.map +0 -1
- package/out/candidate/ancestors-sidecar.d.ts.map +0 -1
- package/out/candidate/ancestors-sidecar.js.map +0 -1
- package/out/candidate-ancestors-schema.d.ts.map +0 -1
- package/out/candidate-ancestors-schema.js.map +0 -1
- package/out/candidate-fts.d.ts.map +0 -1
- package/out/candidate-fts.js.map +0 -1
- package/out/candidate-importance.d.ts.map +0 -1
- package/out/candidate-importance.js.map +0 -1
- package/out/candidate-lookup.d.ts.map +0 -1
- package/out/candidate-lookup.js.map +0 -1
- package/out/candidate-schema.d.ts.map +0 -1
- package/out/candidate-schema.js.map +0 -1
- package/out/candidate-scoring.d.ts.map +0 -1
- package/out/candidate-scoring.js.map +0 -1
- package/out/postal/city-alias-lookup.d.ts.map +0 -1
- package/out/postal/city-alias-lookup.js.map +0 -1
- package/out/postal/city-alias-schema.d.ts.map +0 -1
- package/out/postal/city-alias-schema.js.map +0 -1
- package/out/postal/city-candidate-schema.d.ts.map +0 -1
- package/out/postal/city-candidate-schema.js.map +0 -1
- package/out/sqlite-convention-source.d.ts.map +0 -1
- package/out/sqlite-convention-source.js.map +0 -1
- package/out/sqlite-utils.d.ts.map +0 -1
- package/out/sqlite-utils.js.map +0 -1
- package/out/street/centroid-schema.d.ts.map +0 -1
- package/out/street/centroid-schema.js.map +0 -1
- package/out/street/centroid.d.ts.map +0 -1
- package/out/street/centroid.js.map +0 -1
- package/out/street/morphology-fst-builder.d.ts.map +0 -1
- package/out/street/morphology-fst-builder.js.map +0 -1
- package/out/street/morphology-fst-loader.d.ts.map +0 -1
- package/out/street/morphology-fst-loader.js.map +0 -1
- /package/lib/address/{point-schema.ts → point/schema.ts} +0 -0
- /package/lib/{build-slim.ts → build/slim.ts} +0 -0
- /package/lib/{candidate-fts.ts → candidate/fts.ts} +0 -0
- /package/lib/{candidate-scoring.ts → candidate/scoring.ts} +0 -0
- /package/lib/postal/{city-alias-schema.ts → city/alias/schema.ts} +0 -0
- /package/lib/postal/{city-candidate-schema.ts → city/candidate-schema.ts} +0 -0
- /package/lib/{sqlite-convention-source.ts → sqlite/convention-source.ts} +0 -0
- /package/lib/{sqlite-utils.ts → sqlite/utils.ts} +0 -0
- /package/lib/street/{centroid-schema.ts → centroid/schema.ts} +0 -0
package/lib/address/index.ts
CHANGED
|
@@ -4,6 +4,6 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
export * from "#address/point"
|
|
8
|
-
export * from "#address/point
|
|
9
|
-
export * from "#address/point
|
|
7
|
+
export * from "#address/point/index"
|
|
8
|
+
export * from "#address/point/interpolation"
|
|
9
|
+
export * from "#address/point/schema"
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
import type { AddressPointHit, AddressPointLookup } from "@mailwoman/core/resolver"
|
|
19
19
|
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
20
20
|
|
|
21
|
-
import type { AddressPointDatabase, AddressPointTable } from "#address/point
|
|
21
|
+
import type { AddressPointDatabase, AddressPointTable } from "#address/point/schema"
|
|
22
22
|
import { hasTable, prepareGet, type PreparedGet } from "#sqlite-utils"
|
|
23
23
|
import {
|
|
24
24
|
normalizeHouseNumberForKey,
|
|
@@ -37,7 +37,7 @@ import type { InterpolationLookup } from "@mailwoman/core/resolver"
|
|
|
37
37
|
import { haversineKm } from "@mailwoman/spatial"
|
|
38
38
|
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
39
39
|
|
|
40
|
-
import type { AddressPointDatabase } from "#address/point
|
|
40
|
+
import type { AddressPointDatabase } from "#address/point/schema"
|
|
41
41
|
import type { InterpolatedHit, InterpolationQuery, StreetInterpolator } from "#interpolation"
|
|
42
42
|
import { hasTable, prepareAll, type PreparedAll } from "#sqlite-utils"
|
|
43
43
|
import { canonicalizeRouteKey, type RouteKey, streetKeyVariants } from "#street/normalize"
|
package/lib/ancestry/backfill.ts
CHANGED
|
@@ -45,10 +45,10 @@
|
|
|
45
45
|
* nothing.
|
|
46
46
|
*/
|
|
47
47
|
|
|
48
|
-
import { readDirectoryEntries } from "@mailwoman/core/fs/readers"
|
|
49
48
|
import { readWOFFeature } from "@mailwoman/core/resources/whosonfirst"
|
|
50
49
|
import type { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
51
50
|
import { join, resolvePath, type PathBuilderLike } from "path-ts"
|
|
51
|
+
import { Globerator } from "spliterator/node/fs"
|
|
52
52
|
|
|
53
53
|
import type { WOFDatabase } from "#schema"
|
|
54
54
|
|
|
@@ -88,7 +88,9 @@ export async function discoverAdminDataRoots(reposRoot: PathBuilderLike): Promis
|
|
|
88
88
|
let names: string[]
|
|
89
89
|
|
|
90
90
|
try {
|
|
91
|
-
names = (await
|
|
91
|
+
names = (await Globerator.from("*", { cwd: dir, withFileTypes: true, onlyFiles: false }).toArray())
|
|
92
|
+
.filter((entry) => entry.isDirectory())
|
|
93
|
+
.map((entry) => entry.name)
|
|
92
94
|
} catch {
|
|
93
95
|
return
|
|
94
96
|
}
|
|
@@ -44,20 +44,20 @@ import { pathExists } from "@mailwoman/core/fs/readers"
|
|
|
44
44
|
import { removePath } from "@mailwoman/core/fs/writers"
|
|
45
45
|
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
46
46
|
|
|
47
|
-
import { createCandidateFTS } from "#candidate-fts"
|
|
48
|
-
import { IMPORTANCE_JOIN_RADIUS_KM, loadImportanceIndex } from "#candidate-importance"
|
|
49
|
-
import {
|
|
50
|
-
CANDIDATE_COLUMNS,
|
|
51
|
-
createCandidateStagingTables,
|
|
52
|
-
createCandidateTable,
|
|
53
|
-
type CandidateDatabase,
|
|
54
|
-
} from "#candidate-schema"
|
|
55
47
|
import { explodeAliasBags } from "#candidate/alias-bags"
|
|
56
|
-
import { buildAncestorsSidecar } from "#candidate/ancestors
|
|
48
|
+
import { buildAncestorsSidecar } from "#candidate/ancestors/sidecar"
|
|
57
49
|
import { stageCountryDisplayNames } from "#candidate/country-display-names"
|
|
58
50
|
import { foldExtract } from "#candidate/extract-fold"
|
|
51
|
+
import { createCandidateFTS } from "#candidate/fts"
|
|
52
|
+
import { IMPORTANCE_JOIN_RADIUS_KM, loadImportanceIndex } from "#candidate/importance"
|
|
59
53
|
import { GLOSS_KEY_THRESHOLD, stampNameRoles } from "#candidate/name-roles"
|
|
60
54
|
import type { PlaceAttrs } from "#candidate/place-attrs"
|
|
55
|
+
import {
|
|
56
|
+
CANDIDATE_COLUMNS,
|
|
57
|
+
createCandidateStagingTables,
|
|
58
|
+
createCandidateTable,
|
|
59
|
+
type CandidateDatabase,
|
|
60
|
+
} from "#candidate/schema"
|
|
61
61
|
import { createCapitalTable } from "#capital-schema"
|
|
62
62
|
import type { CapitalPoint } from "#capitals"
|
|
63
63
|
import { resurrectCurrencyHoles } from "#currency-backfill"
|
|
@@ -153,6 +153,12 @@ export interface BuildCandidateResult {
|
|
|
153
153
|
places: number
|
|
154
154
|
primaries: number
|
|
155
155
|
aliases: number
|
|
156
|
+
/**
|
|
157
|
+
* Region aliases refused because another same-country region holds the name as its official name (three pairs in the
|
|
158
|
+
* admin artifact: `新竹市`, `嘉義市`, `충청남도`). Zero on a source without a `names` table means the rule had nothing to read,
|
|
159
|
+
* not that no pair exists.
|
|
160
|
+
*/
|
|
161
|
+
regionOfficialRefused: number
|
|
156
162
|
abbrevs: number
|
|
157
163
|
postcodes: number
|
|
158
164
|
/**
|
|
@@ -225,7 +231,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
225
231
|
|
|
226
232
|
await createCandidateStagingTables(kdb)
|
|
227
233
|
|
|
228
|
-
//
|
|
234
|
+
// Build compact country and placetype code maps for clustered keys.
|
|
229
235
|
// assigned here; the rows are bulk-inserted via kdb once the passes have discovered every code. ---
|
|
230
236
|
const ccodes = new Map<string, number>()
|
|
231
237
|
const ptcodes = new Map<string, number>()
|
|
@@ -254,7 +260,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
254
260
|
return id
|
|
255
261
|
}
|
|
256
262
|
|
|
257
|
-
//
|
|
263
|
+
// Load the importance source before the only pass that sees each place.
|
|
258
264
|
// name/country/placetype/centroid together. Absent → every row's `importance` stays NULL. ---
|
|
259
265
|
let importance: ReturnType<typeof loadImportanceIndex> | undefined
|
|
260
266
|
|
|
@@ -273,7 +279,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
273
279
|
progress("importance", "no score source given — `importance` will be NULL on every row")
|
|
274
280
|
}
|
|
275
281
|
|
|
276
|
-
//
|
|
282
|
+
// Derive each place's region ancestor for same-name disambiguation.
|
|
277
283
|
progress("region", "loading region ancestry")
|
|
278
284
|
const regionOf = new Map<number, number>()
|
|
279
285
|
let multiRegion = 0
|
|
@@ -309,7 +315,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
309
315
|
// the column COUNT can't drift; the positional run() args below MUST stay in CANDIDATE_COLUMNS order.
|
|
310
316
|
const insStage = kdb.prepare(`INSERT INTO cand_stage VALUES (${CANDIDATE_COLUMNS.map(() => "?").join(", ")})`)
|
|
311
317
|
|
|
312
|
-
//
|
|
318
|
+
// Stage primary names and attributes reused by later alias passes.
|
|
313
319
|
progress("primaries", "indexing place names")
|
|
314
320
|
const attrs = new Map<number, PlaceAttrs>()
|
|
315
321
|
let nPrim = 0
|
|
@@ -418,7 +424,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
418
424
|
)
|
|
419
425
|
}
|
|
420
426
|
|
|
421
|
-
//
|
|
427
|
+
// Stage country display names across scripts.
|
|
422
428
|
// Never a silent zero: a runtime whose ICU lacks these locales degrades to fewer surfaces, and the count is how a
|
|
423
429
|
// reader tells that apart from the pass not having run.
|
|
424
430
|
progress(
|
|
@@ -432,7 +438,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
432
438
|
}).toLocaleString()} country surfaces`
|
|
433
439
|
)
|
|
434
440
|
|
|
435
|
-
//
|
|
441
|
+
// Backfill cross-source currency before later candidate passes.
|
|
436
442
|
// the alias pass so a resurrected place's alt names explode like any primary's. ---
|
|
437
443
|
if (opts.currencyBackfill) {
|
|
438
444
|
const nBackfill = await resurrectCurrencyHoles({
|
|
@@ -457,12 +463,23 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
457
463
|
progress("currency-backfill", "not configured — deprecated-no-successor holes stay dead (#1737)")
|
|
458
464
|
}
|
|
459
465
|
|
|
460
|
-
//
|
|
466
|
+
// Stage distinct normalized aliases from alternate names.
|
|
461
467
|
progress("aliases", "exploding alias bags")
|
|
462
|
-
const { nAlias, keyCounts } = explodeAliasBags(src, kdb, attrs, stageRow)
|
|
463
|
-
progress("aliases", `${nAlias.toLocaleString()} aliases`)
|
|
464
468
|
|
|
465
|
-
|
|
469
|
+
const { nAlias, keyCounts, regionOfficialRefused } = explodeAliasBags(src, kdb, attrs, stageRow, {
|
|
470
|
+
regionPlacetypeID: ptID("region"),
|
|
471
|
+
ccID,
|
|
472
|
+
})
|
|
473
|
+
|
|
474
|
+
progress(
|
|
475
|
+
"aliases",
|
|
476
|
+
`${nAlias.toLocaleString()} aliases` +
|
|
477
|
+
(regionOfficialRefused
|
|
478
|
+
? `; ${regionOfficialRefused.toLocaleString()} region aliases refused as another region's official name`
|
|
479
|
+
: "")
|
|
480
|
+
)
|
|
481
|
+
|
|
482
|
+
// Stage region abbreviations.
|
|
466
483
|
let nAbbr = 0
|
|
467
484
|
kdb.exec("BEGIN")
|
|
468
485
|
|
|
@@ -481,7 +498,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
481
498
|
kdb.exec("COMMIT")
|
|
482
499
|
progress("abbrevs", `${nAbbr.toLocaleString()} abbrevs`)
|
|
483
500
|
|
|
484
|
-
//
|
|
501
|
+
// Stamp each name's semantic role.
|
|
485
502
|
// Independent of the sidecar below: this writes `cand_stage.name_role`, that writes the ancestor and
|
|
486
503
|
// interval tables, and neither reads the other's output. Ordered by label only.
|
|
487
504
|
const roles = stampNameRoles({
|
|
@@ -495,10 +512,10 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
495
512
|
progress,
|
|
496
513
|
})
|
|
497
514
|
|
|
498
|
-
//
|
|
515
|
+
// Stage the encoded ancestors sidecar.
|
|
499
516
|
const sidecar = await buildAncestorsSidecar({ src, out: kdb, attrs, ptID, progress })
|
|
500
517
|
|
|
501
|
-
//
|
|
518
|
+
// Fold postcode and locality extracts into candidate records.
|
|
502
519
|
let nPostcode = 0
|
|
503
520
|
let nPostcodeAlias = 0
|
|
504
521
|
|
|
@@ -552,7 +569,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
552
569
|
)
|
|
553
570
|
}
|
|
554
571
|
|
|
555
|
-
//
|
|
572
|
+
// Insert the small typed code dictionaries through Kysely.
|
|
556
573
|
if (ccodes.size) {
|
|
557
574
|
await kdb
|
|
558
575
|
.insertInto("country_codes")
|
|
@@ -567,7 +584,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
567
584
|
.execute()
|
|
568
585
|
}
|
|
569
586
|
|
|
570
|
-
//
|
|
587
|
+
// Insert the small typed capital-status reference table.
|
|
571
588
|
if (opts.capitals?.length) {
|
|
572
589
|
await createCapitalTable<CandidateDatabase>(kdb)
|
|
573
590
|
|
|
@@ -587,7 +604,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
587
604
|
progress("capitals", `${opts.capitals.length.toLocaleString()} capital-reference rows carried in-artifact`)
|
|
588
605
|
}
|
|
589
606
|
|
|
590
|
-
//
|
|
607
|
+
// Materialize the clustered WITHOUT ROWID table in sorted order.
|
|
591
608
|
progress("cluster", "building clustered candidate table + VACUUM")
|
|
592
609
|
// Column list + clustered-key order are sourced from CANDIDATE_COLUMNS (the first 6 ARE the PRIMARY
|
|
593
610
|
// KEY) so the SELECT, the ORDER BY, and the table can't drift. The table comes from the shared
|
|
@@ -620,6 +637,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
620
637
|
places: attrs.size,
|
|
621
638
|
primaries: nPrim,
|
|
622
639
|
aliases: nAlias,
|
|
640
|
+
regionOfficialRefused,
|
|
623
641
|
abbrevs: nAbbr,
|
|
624
642
|
postcodes: nPostcode,
|
|
625
643
|
postcodeAliases: nPostcodeAlias,
|
|
@@ -6,33 +6,83 @@
|
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import type { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
9
|
+
import { tableExists } from "@mailwoman/sqlite/introspection"
|
|
9
10
|
|
|
10
|
-
import type { CandidateDatabase } from "#candidate-schema"
|
|
11
11
|
import type { PlaceAttrs, StageRow } from "#candidate/place-attrs"
|
|
12
|
+
import type { CandidateDatabase } from "#candidate/schema"
|
|
12
13
|
import { ALIAS_SEPARATOR } from "#fts/index"
|
|
13
14
|
import type { WOFDatabase } from "#schema"
|
|
14
15
|
import { normalizeLocalityForKey } from "#street/normalize"
|
|
15
16
|
|
|
17
|
+
/**
|
|
18
|
+
* The official name of every current region, keyed `${country id}\0${name key}` → region id, from the source `names`
|
|
19
|
+
* table's `official` bit. Empty when the source carries no `names` table (a fixture, or an extract shape).
|
|
20
|
+
*
|
|
21
|
+
* This is the positive evidence behind the one alias refusal below: a name is refused from a region's bag ONLY when
|
|
22
|
+
* another region of the same country holds it as its official name. Three such pairs exist in the admin artifact —
|
|
23
|
+
* `新竹市` on Hsinchu County (the city's official name), `嘉義市` on Chiayi County, `충청남도` on Sejong — and each let the more
|
|
24
|
+
* populous holder of the variant outrank the place the name officially is, so a `新竹市` region node resolved the county
|
|
25
|
+
* and every district scoped to the city contradicted it.
|
|
26
|
+
*/
|
|
27
|
+
function officialNameHoldersByRegion(
|
|
28
|
+
src: DatabaseClient<WOFDatabase>,
|
|
29
|
+
ccID: (code: string | null) => number
|
|
30
|
+
): Map<string, number> {
|
|
31
|
+
const keys = new Map<string, number>()
|
|
32
|
+
|
|
33
|
+
if (!tableExists(src, "names")) return keys
|
|
34
|
+
|
|
35
|
+
for (const r of src
|
|
36
|
+
.prepare(
|
|
37
|
+
`SELECT n.id AS id, n.name AS name, s.country AS country
|
|
38
|
+
FROM names n JOIN spr s ON s.id = n.id
|
|
39
|
+
WHERE s.placetype = 'region' AND s.is_current != 0 AND s.is_deprecated = 0 AND n.official = 1`
|
|
40
|
+
)
|
|
41
|
+
.iterate()) {
|
|
42
|
+
const k = normalizeLocalityForKey(String(r.name ?? ""))
|
|
43
|
+
|
|
44
|
+
if (k) {
|
|
45
|
+
keys.set(`${ccID(r.country as string | null)}\0${k}`, Number(r.id))
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
return keys
|
|
50
|
+
}
|
|
51
|
+
|
|
16
52
|
/**
|
|
17
53
|
* Pass 2 — explode each place's `place_search.alt_names` bag into distinct-key alias rows (`is_primary = 0`), and count
|
|
18
54
|
* each place's distinct staged keys (primary included) — the gloss detector's key-count signal (#1730).
|
|
55
|
+
*
|
|
56
|
+
* With `regionPlacetypeID` and `ccID` given, a REGION's alias that is another same-country region's official name is
|
|
57
|
+
* refused and counted rather than staged (see {@link officialNameHoldersByRegion}); without them the pass stages every
|
|
58
|
+
* alias.
|
|
19
59
|
*/
|
|
20
60
|
export function explodeAliasBags(
|
|
21
61
|
src: DatabaseClient<WOFDatabase>,
|
|
22
62
|
out: DatabaseClient<CandidateDatabase>,
|
|
23
63
|
attrs: Map<number, PlaceAttrs>,
|
|
24
|
-
stageRow: StageRow
|
|
25
|
-
|
|
64
|
+
stageRow: StageRow,
|
|
65
|
+
opts: { regionPlacetypeID?: number; ccID?: (code: string | null) => number } = {}
|
|
66
|
+
): { nAlias: number; keyCounts: Map<number, number>; regionOfficialRefused: number } {
|
|
26
67
|
let nAlias = 0
|
|
68
|
+
let regionOfficialRefused = 0
|
|
27
69
|
const keyCounts = new Map<number, number>()
|
|
70
|
+
|
|
71
|
+
const official =
|
|
72
|
+
typeof opts.regionPlacetypeID === "number" && opts.ccID
|
|
73
|
+
? officialNameHoldersByRegion(src, opts.ccID)
|
|
74
|
+
: new Map<string, number>()
|
|
75
|
+
|
|
28
76
|
out.exec("BEGIN")
|
|
29
77
|
|
|
30
78
|
for (const r of src.prepare("SELECT wof_id, alt_names FROM place_search").iterate()) {
|
|
31
|
-
const
|
|
79
|
+
const id = Number(r.wof_id)
|
|
80
|
+
const a = attrs.get(id)
|
|
32
81
|
const alt = r.alt_names as string | null
|
|
33
82
|
|
|
34
83
|
if (!a || !alt) continue
|
|
35
84
|
const seen = new Set<string>([a.pkey])
|
|
85
|
+
const isRegion = a.ptid === opts.regionPlacetypeID
|
|
36
86
|
|
|
37
87
|
// The writer space-pads each separator and appends a trailing one, so every piece arrives with
|
|
38
88
|
// surrounding whitespace and the last one is empty. `normalizeLocalityForKey` folds both away,
|
|
@@ -42,15 +92,26 @@ export function explodeAliasBags(
|
|
|
42
92
|
|
|
43
93
|
if (!k || seen.has(k)) continue
|
|
44
94
|
seen.add(k)
|
|
45
|
-
|
|
95
|
+
|
|
96
|
+
if (isRegion) {
|
|
97
|
+
const holder = official.get(`${a.cid}\0${k}`)
|
|
98
|
+
|
|
99
|
+
if (typeof holder === "number" && holder !== id) {
|
|
100
|
+
regionOfficialRefused++
|
|
101
|
+
|
|
102
|
+
continue
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
stageRow(k, a, id, 0)
|
|
46
107
|
|
|
47
108
|
nAlias++
|
|
48
109
|
}
|
|
49
110
|
|
|
50
|
-
keyCounts.set(
|
|
111
|
+
keyCounts.set(id, seen.size)
|
|
51
112
|
}
|
|
52
113
|
|
|
53
114
|
out.exec("COMMIT")
|
|
54
115
|
|
|
55
|
-
return { nAlias, keyCounts }
|
|
116
|
+
return { nAlias, keyCounts, regionOfficialRefused }
|
|
56
117
|
}
|
|
@@ -59,7 +59,7 @@ import { sql, type Kysely } from "kysely"
|
|
|
59
59
|
// Type-only and circular on purpose (candidate-schema extends CandidateAncestorsDatabase): Kysely's
|
|
60
60
|
// DB parameter is invariant, so the DDL functions must take the FULL database type their caller
|
|
61
61
|
// holds. Erased at runtime.
|
|
62
|
-
import type { CandidateDatabase } from "#candidate
|
|
62
|
+
import type { CandidateDatabase } from "#candidate/schema"
|
|
63
63
|
import type { NameKey } from "#street/normalize"
|
|
64
64
|
|
|
65
65
|
/**
|
|
@@ -15,9 +15,9 @@ import {
|
|
|
15
15
|
createCandidateAncestorTable,
|
|
16
16
|
createCandidateIntervalTable,
|
|
17
17
|
MAX_ANCESTOR_DEPTH,
|
|
18
|
-
} from "#candidate
|
|
19
|
-
import type { CandidateDatabase } from "#candidate-schema"
|
|
18
|
+
} from "#candidate/ancestors/schema"
|
|
20
19
|
import type { PlaceAttrs } from "#candidate/place-attrs"
|
|
20
|
+
import type { CandidateDatabase } from "#candidate/schema"
|
|
21
21
|
import type { WOFDatabase } from "#schema"
|
|
22
22
|
|
|
23
23
|
/**
|
|
@@ -8,9 +8,9 @@
|
|
|
8
8
|
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
9
9
|
import { tableExists } from "@mailwoman/sqlite/introspection"
|
|
10
10
|
|
|
11
|
-
import { CANDIDATE_ANCESTOR_COLUMNS, CANDIDATE_ANCESTOR_TABLE } from "#candidate
|
|
12
|
-
import type { CandidateDatabase } from "#candidate-schema"
|
|
11
|
+
import { CANDIDATE_ANCESTOR_COLUMNS, CANDIDATE_ANCESTOR_TABLE } from "#candidate/ancestors/schema"
|
|
13
12
|
import type { PlaceAttrs, StageRow } from "#candidate/place-attrs"
|
|
13
|
+
import type { CandidateDatabase } from "#candidate/schema"
|
|
14
14
|
import type { WOFDatabase } from "#schema"
|
|
15
15
|
import { normalizeLocalityForKey } from "#street/normalize"
|
|
16
16
|
|
|
@@ -172,7 +172,7 @@ export function foldExtract(ctx: {
|
|
|
172
172
|
|
|
173
173
|
out.exec("COMMIT")
|
|
174
174
|
|
|
175
|
-
//
|
|
175
|
+
// Fold postcode aliases into the staged candidate names.
|
|
176
176
|
//
|
|
177
177
|
// The delivery-city names GeoNames supplies for a ZIP ("Brooklyn" for 11201) are written into
|
|
178
178
|
// the extract's `names` table by `postcode/centroid-fills.ts`'s `geonamesNameFill`. Everything
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
import { haversineKm } from "@mailwoman/spatial"
|
|
45
45
|
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
46
46
|
|
|
47
|
-
import type { CandidateDatabase } from "#candidate
|
|
47
|
+
import type { CandidateDatabase } from "#candidate/schema"
|
|
48
48
|
import { normalizeLocalityForKey } from "#street/normalize"
|
|
49
49
|
|
|
50
50
|
/**
|
|
@@ -23,12 +23,8 @@
|
|
|
23
23
|
* `bbox` field on {@link FindPlaceQuery}).
|
|
24
24
|
*/
|
|
25
25
|
|
|
26
|
-
import {
|
|
27
|
-
|
|
28
|
-
type Ancestor,
|
|
29
|
-
type GazetteerArtifactCoverage,
|
|
30
|
-
referentialFromPopulation,
|
|
31
|
-
} from "@mailwoman/core/resolver"
|
|
26
|
+
import { expandPlacetypeFilter } from "@mailwoman/codex/placetype-map"
|
|
27
|
+
import { type Ancestor, type GazetteerArtifactCoverage, referentialFromPopulation } from "@mailwoman/core/resolver"
|
|
32
28
|
import { allRows } from "@mailwoman/core/utils"
|
|
33
29
|
import { jaroWinkler, levenshteinSimilarity } from "@mailwoman/match/comparators"
|
|
34
30
|
import { partitionByContainment } from "@mailwoman/resolver"
|
|
@@ -42,11 +38,11 @@ import {
|
|
|
42
38
|
intervalContains,
|
|
43
39
|
type CandidateAncestorTable,
|
|
44
40
|
type IntervalLabel,
|
|
45
|
-
} from "#candidate
|
|
46
|
-
import { CANDIDATE_FTS_TABLE } from "#candidate
|
|
47
|
-
import type { CandidateDatabase, CandidateTable, CountryCodeTable, PlacetypeCodeTable } from "#candidate
|
|
41
|
+
} from "#candidate/ancestors/schema"
|
|
42
|
+
import { CANDIDATE_FTS_TABLE } from "#candidate/fts"
|
|
43
|
+
import type { CandidateDatabase, CandidateTable, CountryCodeTable, PlacetypeCodeTable } from "#candidate/schema"
|
|
48
44
|
import { readGazetteerCoverageManifest } from "#coverage-manifest-schema"
|
|
49
|
-
import { POSTAL_CITY_CANDIDATE_TABLE, type PostalCityCandidateTable } from "#postal/city
|
|
45
|
+
import { POSTAL_CITY_CANDIDATE_TABLE, type PostalCityCandidateTable } from "#postal/city/candidate-schema"
|
|
50
46
|
import { rankByPrimaryPreference, type RankedRow, RERANK_FETCH } from "#primary-preference"
|
|
51
47
|
import { applyProximityRerank } from "#proximity-rerank"
|
|
52
48
|
import { REGION_CLASS_PLACETYPES, regionQualifierProbeKeys } from "#region-keys"
|
|
@@ -9,9 +9,9 @@ import { isOfficialLanguage } from "@mailwoman/codex/country"
|
|
|
9
9
|
import type { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
10
10
|
import { tableExists } from "@mailwoman/sqlite/introspection"
|
|
11
11
|
|
|
12
|
-
import type { CandidateDatabase } from "#candidate-schema"
|
|
13
12
|
import { isOwnNameVariant } from "#candidate/own-name"
|
|
14
13
|
import type { PlaceAttrs } from "#candidate/place-attrs"
|
|
14
|
+
import type { CandidateDatabase } from "#candidate/schema"
|
|
15
15
|
import type { WOFDatabase } from "#schema"
|
|
16
16
|
import { normalizeLocalityForKey } from "#street/normalize"
|
|
17
17
|
|
|
@@ -141,7 +141,7 @@ export function stampNameRoles(ctx: {
|
|
|
141
141
|
.run().changes
|
|
142
142
|
)
|
|
143
143
|
|
|
144
|
-
//
|
|
144
|
+
// Detect aliases that are the holder's own primary name in another variant.
|
|
145
145
|
// orthography — romanization, spacing/diacritic variant, or abbreviation expansion. The verdict
|
|
146
146
|
// is per (alias key, primary key) pair, so it runs in JS over the still-unstamped alias rows;
|
|
147
147
|
// an uncovered script answers no-verdict and stamps nothing (own-name.ts owns the predicate and
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
|
|
18
18
|
import { sql, type Kysely } from "kysely"
|
|
19
19
|
|
|
20
|
-
import type { CandidateAncestorsDatabase } from "#candidate
|
|
20
|
+
import type { CandidateAncestorsDatabase } from "#candidate/ancestors/schema"
|
|
21
21
|
import type { CapitalTable } from "#capital-schema"
|
|
22
22
|
import type { NameKey } from "#street/normalize"
|
|
23
23
|
|
package/lib/currency-backfill.ts
CHANGED
|
@@ -16,9 +16,9 @@ import type { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
|
16
16
|
import { resolvePath } from "path-ts"
|
|
17
17
|
import { TSVSpliterator } from "spliterator"
|
|
18
18
|
|
|
19
|
-
import type { loadImportanceIndex } from "#candidate
|
|
20
|
-
import type { CandidateDatabase } from "#candidate-schema"
|
|
19
|
+
import type { loadImportanceIndex } from "#candidate/importance"
|
|
21
20
|
import type { PlaceAttrs, StageRow } from "#candidate/place-attrs"
|
|
21
|
+
import type { CandidateDatabase } from "#candidate/schema"
|
|
22
22
|
import type { WOFDatabase } from "#schema"
|
|
23
23
|
import { normalizeLocalityForKey } from "#street/normalize"
|
|
24
24
|
|
|
@@ -67,7 +67,7 @@ export function deserializeFSTWeb(input: ArrayBuffer | Uint8Array): FSTMatcher {
|
|
|
67
67
|
|
|
68
68
|
let pos = HEADER_SIZE
|
|
69
69
|
|
|
70
|
-
//
|
|
70
|
+
// Decode the interned string table.
|
|
71
71
|
const strOffsets = new Uint32Array(stringCount + 1)
|
|
72
72
|
|
|
73
73
|
for (let i = 0; i <= stringCount; i++) {
|
|
@@ -86,7 +86,7 @@ export function deserializeFSTWeb(input: ArrayBuffer | Uint8Array): FSTMatcher {
|
|
|
86
86
|
|
|
87
87
|
pos += stringBytes
|
|
88
88
|
|
|
89
|
-
//
|
|
89
|
+
// Decode the state table and its transitions.
|
|
90
90
|
const stateEntrySize = version >= VERSION_WIDE_STATE_COUNTERS ? WIDE_STATE_ENTRY_SIZE : NARROW_STATE_ENTRY_SIZE
|
|
91
91
|
// v5 grew the place entry by the encyclopedic float; v4-and-below files are read at the old stride.
|
|
92
92
|
const placeEntrySize = isSplit ? SPLIT_PLACE_ENTRY_SIZE : LEGACY_PLACE_ENTRY_SIZE
|
package/lib/fst/serialize.ts
CHANGED
|
@@ -77,7 +77,7 @@ for (let i = 0; i < PLACETYPE_ORDER.length; i++) {
|
|
|
77
77
|
export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): Buffer {
|
|
78
78
|
const nodes = matcher.toNodes() as FSTNode[]
|
|
79
79
|
|
|
80
|
-
//
|
|
80
|
+
// Intern state-edge strings before serializing their table.
|
|
81
81
|
const stringMap = new Map<string, number>()
|
|
82
82
|
const strings: string[] = []
|
|
83
83
|
|
|
@@ -106,7 +106,7 @@ export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): B
|
|
|
106
106
|
const encodedStrings = strings.map((s) => Buffer.from(s, "utf8"))
|
|
107
107
|
const stringBytes = encodedStrings.reduce((sum, b) => sum + b.length, 0)
|
|
108
108
|
|
|
109
|
-
//
|
|
109
|
+
// Count each serialized record class.
|
|
110
110
|
let totalEdges = 0
|
|
111
111
|
let totalPlaces = 0
|
|
112
112
|
|
|
@@ -115,7 +115,7 @@ export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): B
|
|
|
115
115
|
totalPlaces += node.places.length
|
|
116
116
|
}
|
|
117
117
|
|
|
118
|
-
//
|
|
118
|
+
// Allocate the complete binary buffer.
|
|
119
119
|
const stringTableSize = (strings.length + 1) * 4 + stringBytes
|
|
120
120
|
const stateTableSize = nodes.length * WIDE_STATE_ENTRY_SIZE
|
|
121
121
|
const edgeTableSize = totalEdges * EDGE_ENTRY_SIZE
|
|
@@ -127,7 +127,7 @@ export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): B
|
|
|
127
127
|
const buf = Buffer.alloc(totalSize)
|
|
128
128
|
let pos = 0
|
|
129
129
|
|
|
130
|
-
//
|
|
130
|
+
// Read the versioned binary header.
|
|
131
131
|
// flags bit0 (survey #4, 2026-07-27): place rows carry surface-ambiguity data in the former _pad
|
|
132
132
|
// byte (pp+6 = crossCountryBranches u8, pp+7 reserved). Presence-signaled here so VERSION stays
|
|
133
133
|
// put: pre-ambiguity artifacts read flags=0 → readers expose `undefined`, never a fake 0.
|
|
@@ -151,7 +151,7 @@ export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): B
|
|
|
151
151
|
buf.writeUInt32LE(provenanceJson ? binarySize : 0, pos)
|
|
152
152
|
pos += 4
|
|
153
153
|
|
|
154
|
-
//
|
|
154
|
+
// Read the interned string table.
|
|
155
155
|
let strOffset = 0
|
|
156
156
|
|
|
157
157
|
for (const encoded of encodedStrings) {
|
|
@@ -170,7 +170,7 @@ export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): B
|
|
|
170
170
|
pos += encoded.length
|
|
171
171
|
}
|
|
172
172
|
|
|
173
|
-
//
|
|
173
|
+
// Read state, edge, and place tables.
|
|
174
174
|
const stateTableStart = pos
|
|
175
175
|
const edgeTableStart = stateTableStart + stateTableSize
|
|
176
176
|
const placeTableStart = edgeTableStart + edgeTableSize
|
|
@@ -234,7 +234,7 @@ export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): B
|
|
|
234
234
|
}
|
|
235
235
|
|
|
236
236
|
export function deserializeFST(buf: Buffer): FSTMatcher {
|
|
237
|
-
//
|
|
237
|
+
// Verify the versioned binary header.
|
|
238
238
|
if (buf.length < HEADER_SIZE) throw new Error("FST buffer too small for header")
|
|
239
239
|
|
|
240
240
|
if (!buf.subarray(0, 4).equals(MAGIC)) throw new Error("FST magic mismatch")
|
|
@@ -257,7 +257,7 @@ export function deserializeFST(buf: Buffer): FSTMatcher {
|
|
|
257
257
|
|
|
258
258
|
let pos = HEADER_SIZE
|
|
259
259
|
|
|
260
|
-
//
|
|
260
|
+
// Verify string-table offsets and contents.
|
|
261
261
|
const strOffsets = new Uint32Array(stringCount + 1)
|
|
262
262
|
|
|
263
263
|
for (let i = 0; i <= stringCount; i++) {
|
|
@@ -276,7 +276,7 @@ export function deserializeFST(buf: Buffer): FSTMatcher {
|
|
|
276
276
|
|
|
277
277
|
pos += stringBytes
|
|
278
278
|
|
|
279
|
-
//
|
|
279
|
+
// Verify state-table offsets and transitions.
|
|
280
280
|
const stateEntrySize = version >= VERSION_WIDE_STATE_COUNTERS ? WIDE_STATE_ENTRY_SIZE : NARROW_STATE_ENTRY_SIZE
|
|
281
281
|
// v5 grew the place entry by the encyclopedic float; v4-and-below files are read at the old stride.
|
|
282
282
|
const placeEntrySize = version >= VERSION_TWO_SCORE_SPLIT ? SPLIT_PLACE_ENTRY_SIZE : LEGACY_PLACE_ENTRY_SIZE
|
package/lib/geonames/aliases.ts
CHANGED
|
@@ -26,19 +26,13 @@
|
|
|
26
26
|
|
|
27
27
|
import { isOfficialLanguage } from "@mailwoman/codex/country"
|
|
28
28
|
import { pathExists } from "@mailwoman/core/fs/readers"
|
|
29
|
+
import { GEONAMES_ID_BASE, GEONAMES_POSTAL_ID_BASE } from "@mailwoman/core/resolver/synthetic-id-ranges"
|
|
29
30
|
import type { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
30
31
|
import { join, type PathBuilderLike } from "path-ts"
|
|
31
32
|
import { TSVSpliterator } from "spliterator"
|
|
32
33
|
|
|
33
|
-
import { GEONAMES_POSTAL_ID_BASE } from "#geonames/postal"
|
|
34
34
|
import type { WOFDatabase } from "#schema"
|
|
35
35
|
|
|
36
|
-
/**
|
|
37
|
-
* Synthetic id base for GeoNames-sourced rows (#743/#193) — above Overture's 8e12 so the three sources (WOF real ids,
|
|
38
|
-
* Overture, GeoNames) never collide in a combined DB.
|
|
39
|
-
*/
|
|
40
|
-
export const GEONAMES_ID_BASE = 9_000_000_000_000
|
|
41
|
-
|
|
42
36
|
/**
|
|
43
37
|
* The four tables the alias fold writes into the range it owns. The purge and the ingest must agree on this list — a
|
|
44
38
|
* table written but not purged is exactly the #1514 defect.
|
package/lib/geonames/postal.ts
CHANGED
|
@@ -35,6 +35,7 @@
|
|
|
35
35
|
*/
|
|
36
36
|
|
|
37
37
|
import { pathExists } from "@mailwoman/core/fs/readers"
|
|
38
|
+
import { GEONAMES_POSTAL_ID_BASE } from "@mailwoman/core/resolver/synthetic-id-ranges"
|
|
38
39
|
import type { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
39
40
|
import { join, type PathBuilderLike } from "path-ts"
|
|
40
41
|
import { TSVSpliterator } from "spliterator"
|
|
@@ -47,13 +48,6 @@ import type { WOFDatabase } from "#schema"
|
|
|
47
48
|
*/
|
|
48
49
|
const GEONAMES_POSTAL_COLUMNS = 11
|
|
49
50
|
|
|
50
|
-
/**
|
|
51
|
-
* Synthetic id base for GeoNames-POSTAL rows — its own namespace above the alias fold's {@link GEONAMES_ID_BASE} (9e12)
|
|
52
|
-
* allocation so all four sources (WOF, Overture, GeoNames-alias, GeoNames-postal) coexist collision-free in a combined
|
|
53
|
-
* DB.
|
|
54
|
-
*/
|
|
55
|
-
export const GEONAMES_POSTAL_ID_BASE = 9_500_000_000_000
|
|
56
|
-
|
|
57
51
|
/**
|
|
58
52
|
* The #920 name law: reduce a postcode to the sanitized-query token shape — strip every non-letter/number — so the
|
|
59
53
|
* stored name matches what `sanitizeFTSQuery` produces from the parsed postcode token. `"110 00"` → `"11000"`,
|