@mailwoman/resolver-wof-sqlite 8.1.0 → 8.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/address-point-interpolation.ts +9 -3
- package/address-point-schema.ts +32 -10
- package/address-point.ts +3 -0
- package/ancestry-backfill.ts +18 -5
- package/ancestry.ts +7 -2
- package/build-candidate.ts +29 -6
- package/build-slim.ts +40 -11
- package/candidate-fts.ts +1 -0
- package/candidate-lookup.ts +36 -14
- package/candidate-schema.ts +35 -11
- package/coincident-roles.ts +28 -6
- package/convention.ts +3 -1
- package/coverage-manifest-schema.ts +49 -16
- package/fst-autocomplete.ts +15 -9
- package/fst-builder.ts +29 -5
- package/fst-deserialize-web.ts +46 -9
- package/fst-matcher.ts +12 -5
- package/fst-serialize.ts +83 -9
- package/fst-types.ts +26 -3
- package/fts-query.ts +84 -0
- package/fts.ts +35 -9
- package/geo.ts +9 -3
- package/geonames-aliases.ts +116 -79
- package/geonames-postal.ts +25 -5
- package/index.ts +10 -0
- package/interpolation.ts +33 -55
- package/lookup.ts +103 -292
- package/name-score.ts +76 -0
- package/out/address-point-interpolation.d.ts.map +1 -1
- package/out/address-point-interpolation.js +4 -2
- package/out/address-point-interpolation.js.map +1 -1
- package/out/address-point-schema.d.ts +30 -10
- package/out/address-point-schema.d.ts.map +1 -1
- package/out/address-point-schema.js +6 -2
- package/out/address-point-schema.js.map +1 -1
- package/out/address-point.d.ts.map +1 -1
- package/out/address-point.js.map +1 -1
- package/out/ancestry-backfill.d.ts +6 -2
- package/out/ancestry-backfill.d.ts.map +1 -1
- package/out/ancestry-backfill.js +7 -3
- package/out/ancestry-backfill.js.map +1 -1
- package/out/ancestry.d.ts +6 -2
- package/out/ancestry.d.ts.map +1 -1
- package/out/ancestry.js +3 -1
- package/out/ancestry.js.map +1 -1
- package/out/build-candidate.d.ts +9 -3
- package/out/build-candidate.d.ts.map +1 -1
- package/out/build-candidate.js +5 -3
- package/out/build-candidate.js.map +1 -1
- package/out/build-slim.d.ts +15 -5
- package/out/build-slim.d.ts.map +1 -1
- package/out/build-slim.js +11 -5
- package/out/build-slim.js.map +1 -1
- package/out/candidate-fts.d.ts.map +1 -1
- package/out/candidate-fts.js.map +1 -1
- package/out/candidate-lookup.d.ts +9 -3
- package/out/candidate-lookup.d.ts.map +1 -1
- package/out/candidate-lookup.js +16 -11
- package/out/candidate-lookup.js.map +1 -1
- package/out/candidate-schema.d.ts +33 -11
- package/out/candidate-schema.d.ts.map +1 -1
- package/out/candidate-schema.js.map +1 -1
- package/out/coincident-roles.d.ts +16 -4
- package/out/coincident-roles.d.ts.map +1 -1
- package/out/coincident-roles.js +9 -3
- package/out/coincident-roles.js.map +1 -1
- package/out/convention.d.ts +3 -1
- package/out/convention.d.ts.map +1 -1
- package/out/convention.js.map +1 -1
- package/out/coverage-manifest-schema.d.ts +45 -14
- package/out/coverage-manifest-schema.d.ts.map +1 -1
- package/out/coverage-manifest-schema.js +14 -5
- package/out/coverage-manifest-schema.js.map +1 -1
- package/out/fst-autocomplete.d.ts +1 -1
- package/out/fst-autocomplete.d.ts.map +1 -1
- package/out/fst-autocomplete.js +11 -9
- package/out/fst-autocomplete.js.map +1 -1
- package/out/fst-builder.d.ts.map +1 -1
- package/out/fst-builder.js +15 -5
- package/out/fst-builder.js.map +1 -1
- package/out/fst-deserialize-web.d.ts.map +1 -1
- package/out/fst-deserialize-web.js +34 -9
- package/out/fst-deserialize-web.js.map +1 -1
- package/out/fst-matcher.d.ts +6 -2
- package/out/fst-matcher.d.ts.map +1 -1
- package/out/fst-matcher.js +9 -5
- package/out/fst-matcher.js.map +1 -1
- package/out/fst-serialize.d.ts.map +1 -1
- package/out/fst-serialize.js +62 -9
- package/out/fst-serialize.js.map +1 -1
- package/out/fst-types.d.ts +26 -3
- package/out/fst-types.d.ts.map +1 -1
- package/out/fts-query.d.ts +41 -0
- package/out/fts-query.d.ts.map +1 -0
- package/out/fts-query.js +75 -0
- package/out/fts-query.js.map +1 -0
- package/out/fts.d.ts +21 -7
- package/out/fts.d.ts.map +1 -1
- package/out/fts.js +10 -4
- package/out/fts.js.map +1 -1
- package/out/geo.d.ts +6 -2
- package/out/geo.d.ts.map +1 -1
- package/out/geo.js +3 -1
- package/out/geo.js.map +1 -1
- package/out/geonames-aliases.d.ts +12 -4
- package/out/geonames-aliases.d.ts.map +1 -1
- package/out/geonames-aliases.js +72 -67
- package/out/geonames-aliases.js.map +1 -1
- package/out/geonames-postal.d.ts +9 -3
- package/out/geonames-postal.d.ts.map +1 -1
- package/out/geonames-postal.js +7 -2
- package/out/geonames-postal.js.map +1 -1
- package/out/index.d.ts.map +1 -1
- package/out/index.js.map +1 -1
- package/out/interpolation.d.ts +18 -6
- package/out/interpolation.d.ts.map +1 -1
- package/out/interpolation.js +11 -40
- package/out/interpolation.js.map +1 -1
- package/out/lookup.d.ts +3 -97
- package/out/lookup.d.ts.map +1 -1
- package/out/lookup.js +52 -184
- package/out/lookup.js.map +1 -1
- package/out/name-score.d.ts +28 -0
- package/out/name-score.d.ts.map +1 -0
- package/out/name-score.js +67 -0
- package/out/name-score.js.map +1 -0
- package/out/poi-lookup.d.ts +24 -8
- package/out/poi-lookup.d.ts.map +1 -1
- package/out/poi-lookup.js +27 -13
- package/out/poi-lookup.js.map +1 -1
- package/out/poi-schema.d.ts +42 -13
- package/out/poi-schema.d.ts.map +1 -1
- package/out/poi-schema.js +12 -3
- package/out/poi-schema.js.map +1 -1
- package/out/postal-city-alias-lookup.d.ts +18 -6
- package/out/postal-city-alias-lookup.d.ts.map +1 -1
- package/out/postal-city-alias-lookup.js.map +1 -1
- package/out/postal-city-alias-schema.d.ts +27 -9
- package/out/postal-city-alias-schema.d.ts.map +1 -1
- package/out/postal-city-alias-schema.js +3 -1
- package/out/postal-city-alias-schema.js.map +1 -1
- package/out/postal-city-candidate-schema.d.ts +15 -5
- package/out/postal-city-candidate-schema.d.ts.map +1 -1
- package/out/postal-city-candidate-schema.js +3 -1
- package/out/postal-city-candidate-schema.js.map +1 -1
- package/out/postcode-point-lookup.d.ts +6 -2
- package/out/postcode-point-lookup.d.ts.map +1 -1
- package/out/postcode-point-lookup.js +6 -2
- package/out/postcode-point-lookup.js.map +1 -1
- package/out/ranking-weights.d.ts +118 -0
- package/out/ranking-weights.d.ts.map +1 -0
- package/out/ranking-weights.js +44 -0
- package/out/ranking-weights.js.map +1 -0
- package/out/reverse.d.ts +9 -3
- package/out/reverse.d.ts.map +1 -1
- package/out/reverse.js +20 -6
- package/out/reverse.js.map +1 -1
- package/out/sharding.d.ts +3 -1
- package/out/sharding.d.ts.map +1 -1
- package/out/sharding.js +7 -5
- package/out/sharding.js.map +1 -1
- package/out/sqlite-convention-source.d.ts.map +1 -1
- package/out/sqlite-convention-source.js +3 -1
- package/out/sqlite-convention-source.js.map +1 -1
- package/out/street-centroid-schema.d.ts +33 -11
- package/out/street-centroid-schema.d.ts.map +1 -1
- package/out/street-centroid-schema.js +3 -1
- package/out/street-centroid-schema.js.map +1 -1
- package/out/street-centroid.d.ts.map +1 -1
- package/out/street-centroid.js +6 -2
- package/out/street-centroid.js.map +1 -1
- package/out/street-morphology-fst-builder.d.ts +6 -2
- package/out/street-morphology-fst-builder.d.ts.map +1 -1
- package/out/street-morphology-fst-builder.js +8 -7
- package/out/street-morphology-fst-builder.js.map +1 -1
- package/out/street-morphology-fst-loader.d.ts +24 -8
- package/out/street-morphology-fst-loader.d.ts.map +1 -1
- package/out/street-morphology-fst-loader.js +11 -5
- package/out/street-morphology-fst-loader.js.map +1 -1
- package/out/street-name-lookup.d.ts +9 -3
- package/out/street-name-lookup.d.ts.map +1 -1
- package/out/street-name-lookup.js +9 -7
- package/out/street-name-lookup.js.map +1 -1
- package/out/street-normalize.d.ts +3 -1
- package/out/street-normalize.d.ts.map +1 -1
- package/out/street-normalize.js +23 -13
- package/out/street-normalize.js.map +1 -1
- package/out/street-segment-schema.d.ts +48 -16
- package/out/street-segment-schema.d.ts.map +1 -1
- package/out/street-segment-schema.js +6 -2
- package/out/street-segment-schema.js.map +1 -1
- package/out/types.d.ts +18 -6
- package/out/types.d.ts.map +1 -1
- package/out/unified-schema.d.ts +1 -1
- package/out/unified-schema.d.ts.map +1 -1
- package/out/unified-schema.js +2 -2
- package/out/unified-schema.js.map +1 -1
- package/package.json +5 -5
- package/poi-lookup.ts +53 -21
- package/poi-schema.ts +43 -13
- package/postal-city-alias-lookup.ts +20 -6
- package/postal-city-alias-schema.ts +28 -9
- package/postal-city-candidate-schema.ts +15 -5
- package/postcode-point-lookup.ts +6 -2
- package/ranking-weights.ts +148 -0
- package/reverse.ts +47 -10
- package/sharding.ts +13 -6
- package/sqlite-convention-source.ts +4 -1
- package/street-centroid-schema.ts +35 -11
- package/street-centroid.ts +10 -3
- package/street-morphology-fst-builder.ts +25 -9
- package/street-morphology-fst-loader.ts +26 -10
- package/street-name-lookup.ts +19 -7
- package/street-normalize.ts +28 -13
- package/street-segment-schema.ts +50 -16
- package/types.ts +18 -6
- package/unified-schema.ts +11 -2
|
@@ -56,7 +56,9 @@ interface PointRow {
|
|
|
56
56
|
release: string
|
|
57
57
|
}
|
|
58
58
|
|
|
59
|
-
/**
|
|
59
|
+
/**
|
|
60
|
+
* One known house number on the street: the centroid of its rows (unit siblings collapse).
|
|
61
|
+
*/
|
|
60
62
|
interface NumberAnchor {
|
|
61
63
|
n: number
|
|
62
64
|
lat: number
|
|
@@ -81,6 +83,7 @@ export class AddressPointInterpolator implements InterpolationLookup {
|
|
|
81
83
|
} else {
|
|
82
84
|
throw new Error("AddressPointInterpolator: one of dbPath or database is required")
|
|
83
85
|
}
|
|
86
|
+
|
|
84
87
|
this.#fallback = opts.fallback
|
|
85
88
|
|
|
86
89
|
// Degrade gracefully on an empty/tableless shard (#568): with no `address_point` table this tier
|
|
@@ -121,7 +124,9 @@ export class AddressPointInterpolator implements InterpolationLookup {
|
|
|
121
124
|
}
|
|
122
125
|
}
|
|
123
126
|
|
|
124
|
-
/**
|
|
127
|
+
/**
|
|
128
|
+
* Collapse rows to one centroid anchor per distinct house number, sorted ascending.
|
|
129
|
+
*/
|
|
125
130
|
function anchorsByNumber(rows: readonly PointRow[]): NumberAnchor[] {
|
|
126
131
|
const byN = new Map<number, PointRow[]>()
|
|
127
132
|
|
|
@@ -143,7 +148,7 @@ function anchorsByNumber(rows: readonly PointRow[]): NumberAnchor[] {
|
|
|
143
148
|
source: group[0]!.source,
|
|
144
149
|
release: group[0]!.release,
|
|
145
150
|
}))
|
|
146
|
-
.
|
|
151
|
+
.toSorted((a, b) => a.n - b.n)
|
|
147
152
|
}
|
|
148
153
|
|
|
149
154
|
function interpolateFromNeighbors(rows: readonly PointRow[], n: number): InterpolatedHit | null {
|
|
@@ -158,6 +163,7 @@ function interpolateFromNeighbors(rows: readonly PointRow[], n: number): Interpo
|
|
|
158
163
|
below = anchor
|
|
159
164
|
} else {
|
|
160
165
|
above = anchor
|
|
166
|
+
|
|
161
167
|
break
|
|
162
168
|
}
|
|
163
169
|
}
|
package/address-point-schema.ts
CHANGED
|
@@ -24,27 +24,43 @@ import type { Kysely } from "kysely"
|
|
|
24
24
|
* non-finite coords). `unit`/`postcode`/`locality_norm` are nullable (not every source carries all three).
|
|
25
25
|
*/
|
|
26
26
|
export interface AddressPointTable {
|
|
27
|
-
/**
|
|
27
|
+
/**
|
|
28
|
+
* Shared {@link normalizeStreetForKey} of the street — the build/query-consistent probe key.
|
|
29
|
+
*/
|
|
28
30
|
street_norm: string
|
|
29
|
-
/**
|
|
31
|
+
/**
|
|
32
|
+
* `canonicalizeRouteKey(street_norm)` — the route-fold key (#483 Method 2).
|
|
33
|
+
*/
|
|
30
34
|
street_key: string
|
|
31
|
-
/**
|
|
35
|
+
/**
|
|
36
|
+
* House number, normalized lower-case (kept TEXT — "123-A", "12 1/2" must survive).
|
|
37
|
+
*/
|
|
32
38
|
number: string
|
|
33
39
|
unit: string | null
|
|
34
40
|
postcode: string | null
|
|
35
|
-
/**
|
|
41
|
+
/**
|
|
42
|
+
* Shared {@link normalizeLocalityForKey} of the locality — the fallback scope.
|
|
43
|
+
*/
|
|
36
44
|
locality_norm: string | null
|
|
37
|
-
/**
|
|
45
|
+
/**
|
|
46
|
+
* The street as it appeared in the source (kept for display / debugging).
|
|
47
|
+
*/
|
|
38
48
|
street_raw: string
|
|
39
49
|
lat: number
|
|
40
50
|
lon: number
|
|
41
|
-
/**
|
|
51
|
+
/**
|
|
52
|
+
* Provenance: the dataset this point came from (e.g. `overture:us`, `openaddresses`).
|
|
53
|
+
*/
|
|
42
54
|
source: string
|
|
43
|
-
/**
|
|
55
|
+
/**
|
|
56
|
+
* The pinned data release the point was ingested from.
|
|
57
|
+
*/
|
|
44
58
|
release: string
|
|
45
59
|
}
|
|
46
60
|
|
|
47
|
-
/**
|
|
61
|
+
/**
|
|
62
|
+
* The address-point database schema for `new DatabaseClient<AddressPointDatabase>(...)`.
|
|
63
|
+
*/
|
|
48
64
|
export interface AddressPointDatabase {
|
|
49
65
|
address_point: AddressPointTable
|
|
50
66
|
}
|
|
@@ -67,7 +83,9 @@ export const ADDRESS_POINT_COLUMNS = [
|
|
|
67
83
|
"release",
|
|
68
84
|
] as const
|
|
69
85
|
|
|
70
|
-
/**
|
|
86
|
+
/**
|
|
87
|
+
* Create the `address_point` table — called before the streaming bulk load.
|
|
88
|
+
*/
|
|
71
89
|
export async function createAddressPointTable(db: Kysely<AddressPointDatabase>): Promise<void> {
|
|
72
90
|
await db.schema
|
|
73
91
|
.createTable("address_point")
|
|
@@ -86,18 +104,22 @@ export async function createAddressPointTable(db: Kysely<AddressPointDatabase>):
|
|
|
86
104
|
.execute()
|
|
87
105
|
}
|
|
88
106
|
|
|
89
|
-
/**
|
|
107
|
+
/**
|
|
108
|
+
* Create the three probe indexes the reader relies on (postcode-scope, locality-scope, route-key).
|
|
109
|
+
*/
|
|
90
110
|
export async function createAddressPointIndexes(db: Kysely<AddressPointDatabase>): Promise<void> {
|
|
91
111
|
await db.schema
|
|
92
112
|
.createIndex("idx_ap_postcode")
|
|
93
113
|
.on("address_point")
|
|
94
114
|
.columns(["postcode", "street_norm", "number"])
|
|
95
115
|
.execute()
|
|
116
|
+
|
|
96
117
|
await db.schema
|
|
97
118
|
.createIndex("idx_ap_locality")
|
|
98
119
|
.on("address_point")
|
|
99
120
|
.columns(["locality_norm", "street_norm", "number"])
|
|
100
121
|
.execute()
|
|
122
|
+
|
|
101
123
|
await db.schema.createIndex("idx_ap_streetkey").on("address_point").columns(["postcode", "street_key"]).execute()
|
|
102
124
|
// Street-first index for the BBOX scope (#247): OSM points often carry no postcode/locality, so the
|
|
103
125
|
// reader scopes a `(street_norm, number)` probe by the resolved locality's bbox (lat/lon BETWEEN). The
|
package/address-point.ts
CHANGED
|
@@ -63,10 +63,12 @@ export class AddressPointSqliteLookup implements AddressPointLookup {
|
|
|
63
63
|
`SELECT ${SELECT_COLS} FROM address_point
|
|
64
64
|
WHERE postcode = ? AND street_norm = ? AND number = ? LIMIT 1`
|
|
65
65
|
)
|
|
66
|
+
|
|
66
67
|
this.#byLocality = this.#db.prepare(
|
|
67
68
|
`SELECT ${SELECT_COLS} FROM address_point
|
|
68
69
|
WHERE locality_norm = ? AND street_norm = ? AND number = ? LIMIT 1`
|
|
69
70
|
)
|
|
71
|
+
|
|
70
72
|
this.#byBbox = this.#db.prepare(
|
|
71
73
|
`SELECT ${SELECT_COLS} FROM address_point
|
|
72
74
|
WHERE street_norm = ? AND number = ? AND lat BETWEEN ? AND ? AND lon BETWEEN ? AND ? LIMIT 1`
|
|
@@ -101,6 +103,7 @@ export class AddressPointSqliteLookup implements AddressPointLookup {
|
|
|
101
103
|
this.#locale === "fr"
|
|
102
104
|
? stripArrondissement(normalizeLocalityForKey(query.locality))
|
|
103
105
|
: normalizeLocalityForKey(query.locality)
|
|
106
|
+
|
|
104
107
|
row = this.#byLocality.get(localityKey, streetNorm, number) as AddressPointRow | undefined
|
|
105
108
|
}
|
|
106
109
|
|
package/ancestry-backfill.ts
CHANGED
|
@@ -30,13 +30,19 @@ import { existsSync, readdirSync, readFileSync } from "node:fs"
|
|
|
30
30
|
import { join } from "node:path"
|
|
31
31
|
import type { DatabaseSync } from "node:sqlite"
|
|
32
32
|
|
|
33
|
-
/**
|
|
33
|
+
/**
|
|
34
|
+
* Genuinely top-level placetypes — they never have (or need) an ancestor, so skip them.
|
|
35
|
+
*/
|
|
34
36
|
const TOP_PLACETYPES = new Set(["country", "continent", "empire", "ocean", "marinearea", "planet"])
|
|
35
37
|
|
|
36
38
|
export interface AncestryBackfillResult {
|
|
37
|
-
/**
|
|
39
|
+
/**
|
|
40
|
+
* Places that gained at least one ancestor row.
|
|
41
|
+
*/
|
|
38
42
|
placesFixed: number
|
|
39
|
-
/**
|
|
43
|
+
/**
|
|
44
|
+
* Total ancestor rows inserted.
|
|
45
|
+
*/
|
|
40
46
|
rowsAdded: number
|
|
41
47
|
/**
|
|
42
48
|
* Only-self candidates whose source geojson could not be found (non-WOF backfilled places, or repos not present
|
|
@@ -83,7 +89,9 @@ export function discoverAdminDataRoots(reposRoot: string): string[] {
|
|
|
83
89
|
return roots
|
|
84
90
|
}
|
|
85
91
|
|
|
86
|
-
/**
|
|
92
|
+
/**
|
|
93
|
+
* WOF geojson lives sharded: an id resolves to `<3-char chunks>/<id>.geojson` under each data root.
|
|
94
|
+
*/
|
|
87
95
|
function geojsonForID(id: number, roots: readonly string[]): Record<string, unknown> | null {
|
|
88
96
|
const s = String(id)
|
|
89
97
|
const chunks: string[] = []
|
|
@@ -91,6 +99,7 @@ function geojsonForID(id: number, roots: readonly string[]): Record<string, unkn
|
|
|
91
99
|
for (let i = 0; i < s.length; i += 3) {
|
|
92
100
|
chunks.push(s.slice(i, i + 3))
|
|
93
101
|
}
|
|
102
|
+
|
|
94
103
|
const rel = join(chunks.join("/"), `${s}.geojson`)
|
|
95
104
|
|
|
96
105
|
for (const root of roots) {
|
|
@@ -136,6 +145,7 @@ export function backfillAncestorsFromHierarchy(
|
|
|
136
145
|
opts: { maxId?: number } = {}
|
|
137
146
|
): AncestryBackfillResult {
|
|
138
147
|
const maxId = opts.maxId ?? Number.MAX_SAFE_INTEGER
|
|
148
|
+
|
|
139
149
|
// `s.id < ?` first lets SQLite prune by the PK index before the correlated only-self subquery runs at all.
|
|
140
150
|
const candidates = db
|
|
141
151
|
.prepare(
|
|
@@ -147,6 +157,7 @@ export function backfillAncestorsFromHierarchy(
|
|
|
147
157
|
const insert = db.prepare(
|
|
148
158
|
"INSERT INTO ancestors (id, ancestor_id, ancestor_placetype, lastmodified) VALUES (?, ?, ?, 0)"
|
|
149
159
|
)
|
|
160
|
+
|
|
150
161
|
const hasRow = db.prepare("SELECT 1 FROM ancestors WHERE id = ? AND ancestor_id = ? LIMIT 1")
|
|
151
162
|
|
|
152
163
|
let placesFixed = 0
|
|
@@ -160,10 +171,11 @@ export function backfillAncestorsFromHierarchy(
|
|
|
160
171
|
const props = (gj?.["properties"] ?? null) as Record<string, unknown> | null
|
|
161
172
|
const hierarchy = (props?.["wof:hierarchy"] ?? null) as Array<Record<string, number>> | null
|
|
162
173
|
|
|
163
|
-
if (!hierarchy || hierarchy.length
|
|
174
|
+
if (!hierarchy || !hierarchy.length) {
|
|
164
175
|
if (!gj) {
|
|
165
176
|
noGeojson++
|
|
166
177
|
}
|
|
178
|
+
|
|
167
179
|
continue
|
|
168
180
|
}
|
|
169
181
|
|
|
@@ -190,6 +202,7 @@ export function backfillAncestorsFromHierarchy(
|
|
|
190
202
|
for (const [aid, pt] of seen) {
|
|
191
203
|
if (hasRow.get(id, aid)) continue
|
|
192
204
|
insert.run(id, aid, pt)
|
|
205
|
+
|
|
193
206
|
added++
|
|
194
207
|
}
|
|
195
208
|
|
package/ancestry.ts
CHANGED
|
@@ -36,12 +36,16 @@ export const PLACETYPE_DEPTH: Readonly<Record<string, number>> = {
|
|
|
36
36
|
microhood: 11,
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
/**
|
|
39
|
+
/**
|
|
40
|
+
* Containment depth for a placetype — 0 (sorts coarsest) when unknown.
|
|
41
|
+
*/
|
|
40
42
|
export function placetypeDepth(placetype: string): number {
|
|
41
43
|
return PLACETYPE_DEPTH[placetype] ?? 0
|
|
42
44
|
}
|
|
43
45
|
|
|
44
|
-
/**
|
|
46
|
+
/**
|
|
47
|
+
* One ancestor row, enriched with the `spr` columns both consumers need.
|
|
48
|
+
*/
|
|
45
49
|
export interface AncestorPlaceRow {
|
|
46
50
|
id: number
|
|
47
51
|
placetype: string
|
|
@@ -64,6 +68,7 @@ export function ancestorLineage(db: DatabaseSync, id: number, schemaName = "main
|
|
|
64
68
|
WHERE a.id = ? AND a.ancestor_id != a.id`
|
|
65
69
|
)
|
|
66
70
|
.all(id) as unknown as AncestorPlaceRow[]
|
|
71
|
+
|
|
67
72
|
rows.sort((a, b) => placetypeDepth(b.placetype) - placetypeDepth(a.placetype))
|
|
68
73
|
|
|
69
74
|
return rows
|
package/build-candidate.ts
CHANGED
|
@@ -42,13 +42,19 @@ import {
|
|
|
42
42
|
} from "./candidate-schema.ts"
|
|
43
43
|
import { normalizeLocalityForKey } from "./street-normalize.ts"
|
|
44
44
|
|
|
45
|
-
/**
|
|
45
|
+
/**
|
|
46
|
+
* Boundary-preserving alias-bag separator (#523, U+E000).
|
|
47
|
+
*/
|
|
46
48
|
const ALIAS_SEP = "\u{E000}"
|
|
47
49
|
|
|
48
50
|
export interface BuildCandidateOptions {
|
|
49
|
-
/**
|
|
51
|
+
/**
|
|
52
|
+
* Source unified admin DB — needs spr, place_population, place_search, place_abbr, ancestors.
|
|
53
|
+
*/
|
|
50
54
|
input: string
|
|
51
|
-
/**
|
|
55
|
+
/**
|
|
56
|
+
* Output candidate DB path (overwritten if present).
|
|
57
|
+
*/
|
|
52
58
|
output: string
|
|
53
59
|
/**
|
|
54
60
|
* Optional postcode shards (`spr` rows with `placetype='postalcode'` + real coords, e.g. postalcode-us.db) — folded
|
|
@@ -56,7 +62,9 @@ export interface BuildCandidateOptions {
|
|
|
56
62
|
* path; the postcode-*.bin anchor stays the fallback). Matches the slim wof-hot.db, which took one such postcode DB.
|
|
57
63
|
*/
|
|
58
64
|
postcodes?: string[]
|
|
59
|
-
/**
|
|
65
|
+
/**
|
|
66
|
+
* Optional progress callback for CLI / test introspection.
|
|
67
|
+
*/
|
|
60
68
|
onProgress?: (phase: string, message: string) => void
|
|
61
69
|
}
|
|
62
70
|
|
|
@@ -104,6 +112,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
104
112
|
// assigned here; the rows are bulk-inserted via kdb once the passes have discovered every code. ---
|
|
105
113
|
const ccodes = new Map<string, number>()
|
|
106
114
|
const ptcodes = new Map<string, number>()
|
|
115
|
+
|
|
107
116
|
const ccID = (code: string | null): number => {
|
|
108
117
|
const c = (code || "??").toUpperCase()
|
|
109
118
|
let id = ccodes.get(c)
|
|
@@ -115,6 +124,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
115
124
|
|
|
116
125
|
return id
|
|
117
126
|
}
|
|
127
|
+
|
|
118
128
|
const ptID = (pt: string | null): number => {
|
|
119
129
|
const p = pt || ""
|
|
120
130
|
let id = ptcodes.get(p)
|
|
@@ -134,6 +144,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
134
144
|
for (const r of src.prepare("SELECT id, ancestor_id FROM ancestors WHERE ancestor_placetype='region'").iterate()) {
|
|
135
145
|
regionOf.set(Number(r.id), Number(r.ancestor_id))
|
|
136
146
|
}
|
|
147
|
+
|
|
137
148
|
progress("region", `${regionOf.size.toLocaleString()} places carry a region`)
|
|
138
149
|
|
|
139
150
|
// The hot path — millions of clustered rows. Kept a single positional prepared statement (the fastest
|
|
@@ -165,6 +176,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
165
176
|
const neg = -Math.log10(pop + 1)
|
|
166
177
|
const name = String(r.name ?? "")
|
|
167
178
|
const pkey = normalizeLocalityForKey(name)
|
|
179
|
+
|
|
168
180
|
const a: PlaceAttrs = {
|
|
169
181
|
cid,
|
|
170
182
|
rid,
|
|
@@ -180,13 +192,16 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
180
192
|
neg,
|
|
181
193
|
pkey,
|
|
182
194
|
}
|
|
195
|
+
|
|
183
196
|
attrs.set(sid, a)
|
|
184
197
|
|
|
185
198
|
if (pkey) {
|
|
186
199
|
insStage.run(pkey, cid, rid, ptid, neg, sid, name, a.lat, a.lon, a.mnLat, a.mnLon, a.mxLat, a.mxLon, pop, 1)
|
|
200
|
+
|
|
187
201
|
nPrim++
|
|
188
202
|
}
|
|
189
203
|
}
|
|
204
|
+
|
|
190
205
|
out.exec("COMMIT")
|
|
191
206
|
progress("primaries", `${nPrim.toLocaleString()} primaries; ${attrs.size.toLocaleString()} places`)
|
|
192
207
|
|
|
@@ -228,9 +243,11 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
228
243
|
if (!k || seen.has(k)) continue
|
|
229
244
|
seen.add(k)
|
|
230
245
|
stageRow(k, a, Number(r.wof_id), 0)
|
|
246
|
+
|
|
231
247
|
nAlias++
|
|
232
248
|
}
|
|
233
249
|
}
|
|
250
|
+
|
|
234
251
|
out.exec("COMMIT")
|
|
235
252
|
progress("aliases", `${nAlias.toLocaleString()} aliases`)
|
|
236
253
|
|
|
@@ -246,8 +263,10 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
246
263
|
|
|
247
264
|
if (!k) continue
|
|
248
265
|
stageRow(k, a, Number(r.id), 1)
|
|
266
|
+
|
|
249
267
|
nAbbr++
|
|
250
268
|
}
|
|
269
|
+
|
|
251
270
|
out.exec("COMMIT")
|
|
252
271
|
progress("abbrevs", `${nAbbr.toLocaleString()} abbrevs`)
|
|
253
272
|
|
|
@@ -273,6 +292,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
273
292
|
if (!key) continue
|
|
274
293
|
const lat = r.latitude as number
|
|
275
294
|
const lon = r.longitude as number
|
|
295
|
+
|
|
276
296
|
// region_id 0 (a postcode is unique by name+country — no same-name disambiguation); neg_rank 0
|
|
277
297
|
// (no population). bbox = the postcode's own min/max (falls back to the centroid point).
|
|
278
298
|
insStage.run(
|
|
@@ -292,8 +312,10 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
292
312
|
0,
|
|
293
313
|
1
|
|
294
314
|
)
|
|
315
|
+
|
|
295
316
|
nPostcode++
|
|
296
317
|
}
|
|
318
|
+
|
|
297
319
|
out.exec("COMMIT")
|
|
298
320
|
pc.close()
|
|
299
321
|
}
|
|
@@ -303,14 +325,14 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
303
325
|
}
|
|
304
326
|
|
|
305
327
|
// --- code dictionaries: typed batch inserts via kdb (a few hundred rows — Kysely is clean here) ---
|
|
306
|
-
if (ccodes.size
|
|
328
|
+
if (ccodes.size) {
|
|
307
329
|
await kdb
|
|
308
330
|
.insertInto("country_codes")
|
|
309
331
|
.values([...ccodes].map(([code, id]) => ({ id, code })))
|
|
310
332
|
.execute()
|
|
311
333
|
}
|
|
312
334
|
|
|
313
|
-
if (ptcodes.size
|
|
335
|
+
if (ptcodes.size) {
|
|
314
336
|
await kdb
|
|
315
337
|
.insertInto("placetype_codes")
|
|
316
338
|
.values([...ptcodes].map(([placetype, id]) => ({ id, placetype })))
|
|
@@ -343,6 +365,7 @@ export async function buildCandidateTable(opts: BuildCandidateOptions): Promise<
|
|
|
343
365
|
.selectFrom("candidate")
|
|
344
366
|
.select((eb) => eb.fn.countAll<number>().as("n"))
|
|
345
367
|
.executeTakeFirstOrThrow()
|
|
368
|
+
|
|
346
369
|
src.close()
|
|
347
370
|
await kdb.destroy()
|
|
348
371
|
|
package/build-slim.ts
CHANGED
|
@@ -48,13 +48,21 @@ import { buildPlaceSearchFTS, PLACE_BBOX_TABLE, PLACE_POPULATION_TABLE, PLACE_SE
|
|
|
48
48
|
import type { NamesTable, SprTable } from "./schema.ts"
|
|
49
49
|
|
|
50
50
|
export interface BuildSlimOptions {
|
|
51
|
-
/**
|
|
51
|
+
/**
|
|
52
|
+
* Input WOF SQLite distributions. Each should already have spr / names / place_population tables.
|
|
53
|
+
*/
|
|
52
54
|
inputs: string[]
|
|
53
|
-
/**
|
|
55
|
+
/**
|
|
56
|
+
* Output path for the slim DB. Will be overwritten if it exists.
|
|
57
|
+
*/
|
|
54
58
|
output: string
|
|
55
|
-
/**
|
|
59
|
+
/**
|
|
60
|
+
* Country codes to keep (ISO 2-letter). Defaults to `["US"]`.
|
|
61
|
+
*/
|
|
56
62
|
countries?: string[]
|
|
57
|
-
/**
|
|
63
|
+
/**
|
|
64
|
+
* Cap on the number of localities to keep per country, by descending population.
|
|
65
|
+
*/
|
|
58
66
|
topLocalitiesPerCountry?: number
|
|
59
67
|
/**
|
|
60
68
|
* Drop the `names` table after the FTS index is built (default false). `place_search` is a self-contained FTS5 (no
|
|
@@ -64,7 +72,9 @@ export interface BuildSlimOptions {
|
|
|
64
72
|
* alt-names at runtime should ship a SEPARATE shard rather than re-bloat the hot DB.
|
|
65
73
|
*/
|
|
66
74
|
dropNames?: boolean
|
|
67
|
-
/**
|
|
75
|
+
/**
|
|
76
|
+
* Optional progress callback for CLI / test introspection.
|
|
77
|
+
*/
|
|
68
78
|
onProgress?: (phase: SlimBuildPhase, detail: string) => void
|
|
69
79
|
}
|
|
70
80
|
|
|
@@ -96,16 +106,24 @@ export interface BuildSlimResult {
|
|
|
96
106
|
}
|
|
97
107
|
}
|
|
98
108
|
|
|
99
|
-
/**
|
|
109
|
+
/**
|
|
110
|
+
* Placetypes that we always keep so the ancestor chain a selected locality reports stays valid.
|
|
111
|
+
*/
|
|
100
112
|
const ANCESTOR_PLACETYPES = ["country", "region", "county", "borough", "macroregion"] as const
|
|
101
113
|
|
|
102
|
-
/**
|
|
114
|
+
/**
|
|
115
|
+
* Tables copied verbatim (schema + filtered rows) from each source DB. Anything else is dropped.
|
|
116
|
+
*/
|
|
103
117
|
const COPIED_TABLES = ["spr", "names", PLACE_POPULATION_TABLE] as const
|
|
104
118
|
|
|
105
|
-
/**
|
|
119
|
+
/**
|
|
120
|
+
* Fallback DDL for `place_population` when the first source predates the aux table (defensive).
|
|
121
|
+
*/
|
|
106
122
|
const PLACE_POPULATION_DDL = `CREATE TABLE ${PLACE_POPULATION_TABLE} (id INTEGER PRIMARY KEY, population INTEGER NOT NULL DEFAULT 0)`
|
|
107
123
|
|
|
108
|
-
/**
|
|
124
|
+
/**
|
|
125
|
+
* Minimal row shape for the population aux table — id + population, nothing else.
|
|
126
|
+
*/
|
|
109
127
|
interface PlacePopulationTable {
|
|
110
128
|
id: number
|
|
111
129
|
population: number
|
|
@@ -135,7 +153,7 @@ export async function buildSlimWOFDatabase(opts: BuildSlimOptions): Promise<Buil
|
|
|
135
153
|
// Skip empties up front; require every remaining path to exist.
|
|
136
154
|
const inputs = opts.inputs.filter((p) => p.length > 0)
|
|
137
155
|
|
|
138
|
-
if (inputs.length
|
|
156
|
+
if (!inputs.length) throw new Error("no input WOF dbs provided")
|
|
139
157
|
|
|
140
158
|
for (const input of inputs) {
|
|
141
159
|
if (!existsSync(input)) throw new Error(`input WOF db not found: ${input}`)
|
|
@@ -177,6 +195,7 @@ export async function buildSlimWOFDatabase(opts: BuildSlimOptions): Promise<Buil
|
|
|
177
195
|
throw new Error(`source DB ${inputs[0]} is missing required table '${table}'`)
|
|
178
196
|
}
|
|
179
197
|
}
|
|
198
|
+
|
|
180
199
|
// PRIMARY KEY on spr.id + place_population.id come from the schemas we copied; an explicit
|
|
181
200
|
// index on names.id helps the per-id INSERT SELECT later.
|
|
182
201
|
out.exec(`CREATE INDEX IF NOT EXISTS names_id_idx ON names(id);`)
|
|
@@ -196,6 +215,7 @@ export async function buildSlimWOFDatabase(opts: BuildSlimOptions): Promise<Buil
|
|
|
196
215
|
// population aux table is NOT rebuilt here: it was copied verbatim above, and fts.ts only
|
|
197
216
|
// (re)builds it when a `geojson` table is present, which the slim DB intentionally has not.
|
|
198
217
|
progress("fts", "building place_search / place_bbox on slim DB")
|
|
218
|
+
|
|
199
219
|
buildPlaceSearchFTS(out, {
|
|
200
220
|
drop: true, // schema we copied had no FTS tables, but be explicit
|
|
201
221
|
onProgress: (phase, name) => progress("fts", `${phase} ${name}`),
|
|
@@ -239,6 +259,7 @@ export async function buildSlimWOFDatabase(opts: BuildSlimOptions): Promise<Buil
|
|
|
239
259
|
placeBbox: countRows(out, PLACE_BBOX_TABLE),
|
|
240
260
|
placePopulation: countRows(out, PLACE_POPULATION_TABLE),
|
|
241
261
|
}
|
|
262
|
+
|
|
242
263
|
progress("done", JSON.stringify(rowCounts))
|
|
243
264
|
|
|
244
265
|
result = {
|
|
@@ -249,6 +270,7 @@ export async function buildSlimWOFDatabase(opts: BuildSlimOptions): Promise<Buil
|
|
|
249
270
|
} finally {
|
|
250
271
|
out.close()
|
|
251
272
|
}
|
|
273
|
+
|
|
252
274
|
// The sealed-artifact invariant: a built DB is a read-only asset from the moment it exists.
|
|
253
275
|
sealDatabase(opts.output)
|
|
254
276
|
|
|
@@ -273,7 +295,7 @@ async function copyFromSource(
|
|
|
273
295
|
copyFileSync(inputPath, scratchPath)
|
|
274
296
|
|
|
275
297
|
try {
|
|
276
|
-
out.exec(`ATTACH DATABASE '${scratchPath.
|
|
298
|
+
out.exec(`ATTACH DATABASE '${scratchPath.replaceAll("'", "''")}' AS src;`)
|
|
277
299
|
|
|
278
300
|
try {
|
|
279
301
|
// Does this shard carry the pre-built population aux table? The admin source does; a bare
|
|
@@ -290,6 +312,7 @@ async function copyFromSource(
|
|
|
290
312
|
|
|
291
313
|
// 1. Ancestor placetypes (country / region / county / etc.) — always-kept.
|
|
292
314
|
progress("country", `${inputPath}: ancestor placetypes in (${countries.join(",")})`)
|
|
315
|
+
|
|
293
316
|
await kysely
|
|
294
317
|
.insertInto("spr")
|
|
295
318
|
.expression((eb) =>
|
|
@@ -308,6 +331,7 @@ async function copyFromSource(
|
|
|
308
331
|
// aux table — left-join it so localities without a population row still qualify (sorted
|
|
309
332
|
// last). If the shard has no population table, fall back to a deterministic id ordering.
|
|
310
333
|
progress("locality", `${inputPath}: top-${topLocalities} localities by population`)
|
|
334
|
+
|
|
311
335
|
await kysely
|
|
312
336
|
.insertInto("spr")
|
|
313
337
|
.expression((eb) =>
|
|
@@ -327,6 +351,7 @@ async function copyFromSource(
|
|
|
327
351
|
|
|
328
352
|
// 3. All postcodes in scope.
|
|
329
353
|
progress("postcode", `${inputPath}: all postcodes`)
|
|
354
|
+
|
|
330
355
|
await kysely
|
|
331
356
|
.insertInto("spr")
|
|
332
357
|
.expression((eb) =>
|
|
@@ -343,6 +368,7 @@ async function copyFromSource(
|
|
|
343
368
|
|
|
344
369
|
// 4. Pull names for the IDs we just selected.
|
|
345
370
|
progress("names", `${inputPath}: names rows for selected IDs`)
|
|
371
|
+
|
|
346
372
|
await kysely
|
|
347
373
|
.insertInto("names")
|
|
348
374
|
.expression((eb) => eb.selectFrom("src.names").selectAll().where("id", "in", eb.selectFrom("spr").select("id")))
|
|
@@ -352,6 +378,7 @@ async function copyFromSource(
|
|
|
352
378
|
// 5. Pull population rows for the selected IDs (sparse — only the places WOF has a count for).
|
|
353
379
|
if (srcHasPopulation) {
|
|
354
380
|
progress("place_population", `${inputPath}: population rows for selected IDs`)
|
|
381
|
+
|
|
355
382
|
await kysely
|
|
356
383
|
.insertInto("place_population")
|
|
357
384
|
.expression((eb) =>
|
|
@@ -373,10 +400,12 @@ async function copyFromSource(
|
|
|
373
400
|
if (relationSchema?.sql) {
|
|
374
401
|
progress("coincident_roles", `${inputPath}: copying dual-role relation`)
|
|
375
402
|
out.exec(relationSchema.sql.replace(/CREATE TABLE/i, "CREATE TABLE IF NOT EXISTS"))
|
|
403
|
+
|
|
376
404
|
out.exec(
|
|
377
405
|
`INSERT OR IGNORE INTO coincident_roles SELECT * FROM src.coincident_roles
|
|
378
406
|
WHERE admin_id IN (SELECT id FROM spr) AND locality_id IN (SELECT id FROM spr)`
|
|
379
407
|
)
|
|
408
|
+
|
|
380
409
|
out.exec(`CREATE INDEX IF NOT EXISTS coincident_roles_by_admin ON coincident_roles (admin_id)`)
|
|
381
410
|
}
|
|
382
411
|
} finally {
|
package/candidate-fts.ts
CHANGED
|
@@ -37,6 +37,7 @@ export const CANDIDATE_FTS_TABLE = "candidate_fts"
|
|
|
37
37
|
export function createCandidateFTS(db: DatabaseSync): void {
|
|
38
38
|
db.exec(`DROP TABLE IF EXISTS ${CANDIDATE_FTS_TABLE}`)
|
|
39
39
|
db.exec(`CREATE VIRTUAL TABLE ${CANDIDATE_FTS_TABLE} USING fts5(name_key, tokenize='trigram')`)
|
|
40
|
+
|
|
40
41
|
db.exec(
|
|
41
42
|
`INSERT INTO ${CANDIDATE_FTS_TABLE}(name_key) SELECT DISTINCT name_key FROM candidate WHERE name_key IS NOT NULL`
|
|
42
43
|
)
|