@mailwoman/resolver-wof-sqlite 7.2.0 → 7.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/address-point-interpolation.ts +207 -0
- package/address-point-schema.ts +107 -0
- package/address-point.ts +122 -0
- package/ancestry-backfill.ts +205 -0
- package/ancestry.ts +70 -0
- package/build-candidate.ts +351 -0
- package/build-slim.ts +394 -0
- package/candidate-fts.ts +43 -0
- package/candidate-lookup.ts +382 -0
- package/candidate-schema.ts +166 -0
- package/coincident-roles.ts +240 -0
- package/convention.ts +152 -0
- package/fst-autocomplete.ts +187 -0
- package/fst-builder.ts +291 -0
- package/fst-deserialize-web.ts +164 -0
- package/fst-matcher.ts +150 -0
- package/fst-serialize.ts +311 -0
- package/fst-types.ts +78 -0
- package/fts.ts +318 -0
- package/geo.ts +140 -0
- package/geonames-aliases.ts +317 -0
- package/geonames-postal.ts +150 -0
- package/index.ts +117 -0
- package/interpolation.ts +232 -0
- package/lookup.ts +1498 -0
- package/out/poi-lookup.d.ts +14 -2
- package/out/poi-lookup.d.ts.map +1 -1
- package/out/poi-lookup.js +55 -21
- package/out/poi-lookup.js.map +1 -1
- package/out/poi-schema.d.ts +9 -0
- package/out/poi-schema.d.ts.map +1 -1
- package/out/poi-schema.js +16 -0
- package/out/poi-schema.js.map +1 -1
- package/out/reverse.d.ts +8 -1
- package/out/reverse.d.ts.map +1 -1
- package/out/reverse.js +10 -1
- package/out/reverse.js.map +1 -1
- package/package.json +168 -82
- package/poi-lookup.ts +375 -0
- package/poi-schema.ts +164 -0
- package/postal-city-alias-lookup.ts +89 -0
- package/postal-city-alias-schema.ts +75 -0
- package/postal-city-candidate-schema.ts +81 -0
- package/postcode-point-lookup.ts +64 -0
- package/reverse.ts +439 -0
- package/schema.ts +176 -0
- package/sharding.ts +235 -0
- package/sqlite-convention-source.ts +61 -0
- package/sqlite-utils.ts +25 -0
- package/street-centroid-schema.ts +124 -0
- package/street-centroid.ts +124 -0
- package/street-morphology-fst-builder.ts +230 -0
- package/street-name-lookup.ts +101 -0
- package/street-normalize.ts +302 -0
- package/street-segment-schema.ts +104 -0
- package/types.ts +164 -0
- package/unified-schema.ts +171 -0
package/schema.ts
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Kysely table types for the subset of the Who's On First SQLite schema we touch in Phase 4.2.
|
|
7
|
+
*
|
|
8
|
+
* The full upstream distribution at data.geocode.earth/wof/dist/sqlite/ ships ~7 tables; this file
|
|
9
|
+
* models only the ones we read. Pretending to model the others would be misleading — we haven't
|
|
10
|
+
* verified their shapes and they're not part of the resolver's contract.
|
|
11
|
+
*
|
|
12
|
+
* Authoritative schema docs:
|
|
13
|
+
*
|
|
14
|
+
* - Whosonfirst SQLite README: https://github.com/whosonfirst/go-whosonfirst-sqlite
|
|
15
|
+
* - Per-table sources under https://github.com/whosonfirst/go-whosonfirst-sqlite-features
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* The FTS5 virtual table built by this package on first open (NOT shipped by upstream WOF).
|
|
20
|
+
*
|
|
21
|
+
* `content` is unindexed — it's there so we can roundtrip the original name back to the caller without a second SELECT.
|
|
22
|
+
* The actual FTS rebuild happens in `fts.ts::buildPlaceSearchFTS`.
|
|
23
|
+
*/
|
|
24
|
+
export interface PlaceSearchTable {
|
|
25
|
+
rowid: number
|
|
26
|
+
wof_id: number
|
|
27
|
+
name: string
|
|
28
|
+
alt_names: string | null
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* `spr` — the Who's On First "Standard Places Response": a denormalized lightweight summary of one row per place. The
|
|
33
|
+
* resolver's main lookup table.
|
|
34
|
+
*
|
|
35
|
+
* Lifecycle flags carry TWO conventions, both meaning "currently valid": `is_current = -1` (modern Who's On First) and
|
|
36
|
+
* `is_current = 1` (legacy Mapzen-era). Only `is_current = 0` means "not current". Filters in `lookup.ts` and `fts.ts`
|
|
37
|
+
* use `is_current != 0 AND is_deprecated = 0` — see #91 for the diagnostic that uncovered the mixed-convention
|
|
38
|
+
* reality.
|
|
39
|
+
*
|
|
40
|
+
* Lat/lon live directly on this row — no GeoJSON extraction needed for centroid resolution. `min_*` / `max_*` form a
|
|
41
|
+
* bounding box if callers want one (Phase 4.3 candidate).
|
|
42
|
+
*/
|
|
43
|
+
export interface SprTable {
|
|
44
|
+
id: number
|
|
45
|
+
parent_id: number | null
|
|
46
|
+
name: string | null
|
|
47
|
+
placetype: string | null
|
|
48
|
+
country: string | null
|
|
49
|
+
latitude: number
|
|
50
|
+
longitude: number
|
|
51
|
+
min_latitude: number
|
|
52
|
+
min_longitude: number
|
|
53
|
+
max_latitude: number
|
|
54
|
+
max_longitude: number
|
|
55
|
+
is_current: number
|
|
56
|
+
is_deprecated: number
|
|
57
|
+
is_ceased: number
|
|
58
|
+
is_superseded: number
|
|
59
|
+
is_superseding: number
|
|
60
|
+
superseded_by: string | null
|
|
61
|
+
supersedes: string | null
|
|
62
|
+
lastmodified: number
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Alternate names per place, keyed by language tag subfields (BCP-47 components). Joins back to `spr.id` via `id` (NOT
|
|
67
|
+
* `place_id` — the real WOF schema uses the same column name as the spr primary key; this is a normal join across two
|
|
68
|
+
* tables with the same FK column name).
|
|
69
|
+
*
|
|
70
|
+
* No `kind` column in real WOF — the FTS build just concatenates ALL names per id.
|
|
71
|
+
*
|
|
72
|
+
* `official` (#936 ingest bit, our unified builds only; absent in real WOF dumps) marks a PREFERRED-form name in an
|
|
73
|
+
* official language of the place's country — the aliases eligible to join the name-exact tier under the option-3 rule.
|
|
74
|
+
* See `unified-schema.ts` for the full contract.
|
|
75
|
+
*/
|
|
76
|
+
export interface NamesTable {
|
|
77
|
+
id: number
|
|
78
|
+
placetype: string | null
|
|
79
|
+
country: string | null
|
|
80
|
+
language: string | null // ISO-639 alpha-3
|
|
81
|
+
extlang: string | null
|
|
82
|
+
script: string | null
|
|
83
|
+
region: string | null
|
|
84
|
+
variant: string | null
|
|
85
|
+
extension: string | null
|
|
86
|
+
privateuse: string | null
|
|
87
|
+
official: number | null
|
|
88
|
+
name: string
|
|
89
|
+
lastmodified: number
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Per-place GeoJSON blob. Centroid lat/lon are already exposed via `spr.{latitude,longitude}` so the resolver doesn't
|
|
94
|
+
* need to parse this; we keep the table modeled in case Phase 4.3 wants the full geometry for bbox / polygon work.
|
|
95
|
+
*/
|
|
96
|
+
export interface GeojsonTable {
|
|
97
|
+
id: number
|
|
98
|
+
body: string
|
|
99
|
+
source: string | null
|
|
100
|
+
alt_label: string | null
|
|
101
|
+
is_alt: number
|
|
102
|
+
lastmodified: number
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Adjacency table for ancestor relationships. One row per (place, ancestor) pair, including transitive ancestors. Used
|
|
107
|
+
* to implement `FindPlaceQuery.parentID` (descendant lookup).
|
|
108
|
+
*/
|
|
109
|
+
export interface AncestorsTable {
|
|
110
|
+
id: number
|
|
111
|
+
ancestor_id: number
|
|
112
|
+
ancestor_placetype: string | null
|
|
113
|
+
lastmodified: number
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* `place_population` — `id → wof:population`, split off `spr` so a population-rank join is a single indexed probe.
|
|
118
|
+
* Written by the build/augment ingest + the GeoNames backfill; read by the candidate build's `neg_rank`. WOF carries
|
|
119
|
+
* population for ~15% of localities; absent = unknown, not zero.
|
|
120
|
+
*/
|
|
121
|
+
export interface PlacePopulationTable {
|
|
122
|
+
id: number
|
|
123
|
+
population: number
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* `place_abbr` — `id → abbreviation` (e.g. `IL → Illinois`), derived from `names` rows whose `language = 'abbr'`. Lets
|
|
128
|
+
* the resolver accept a 2-letter region abbreviation as an exact match.
|
|
129
|
+
*/
|
|
130
|
+
export interface PlaceAbbrTable {
|
|
131
|
+
id: number
|
|
132
|
+
abbr: string
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* `concordances` — external-id cross-references per place (`id → (other_source, other_id)`), e.g. a GeoNames or
|
|
137
|
+
* Overture GERS id. Metadata only; not part of the resolve path.
|
|
138
|
+
*/
|
|
139
|
+
export interface ConcordancesTable {
|
|
140
|
+
id: number
|
|
141
|
+
other_id: string
|
|
142
|
+
other_source: string
|
|
143
|
+
lastmodified: number
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* `coincident_roles` (#402) — the dual-role relation: a place that is BOTH an admin region AND a locality (Berlin the
|
|
148
|
+
* city-state). One row per (admin, locality) pair the resolver can complete a hierarchy with. Surfaced by
|
|
149
|
+
* {@link MailwomanLookupLike.coincidentRolesFor}.
|
|
150
|
+
*/
|
|
151
|
+
export interface CoincidentRolesTable {
|
|
152
|
+
admin_id: number
|
|
153
|
+
locality_id: number
|
|
154
|
+
relationship_type: string
|
|
155
|
+
admin_placetype: string
|
|
156
|
+
distance_km: number
|
|
157
|
+
locality_population: number
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* The full schema we hand to `Kysely<WOFDatabase>` / `new DatabaseClient<WOFDatabase>(...)`. Tables not listed here
|
|
162
|
+
* will fail type-checked queries — by design. The reader ({@link WOFSqlitePlaceLookup}) already consumes this; the
|
|
163
|
+
* build/augment WRITERS adopt it so a column rename is a compile error on both sides (the drift that bit the corpus
|
|
164
|
+
* TIGER adapter).
|
|
165
|
+
*/
|
|
166
|
+
export interface WOFDatabase {
|
|
167
|
+
place_search: PlaceSearchTable
|
|
168
|
+
spr: SprTable
|
|
169
|
+
names: NamesTable
|
|
170
|
+
geojson: GeojsonTable
|
|
171
|
+
ancestors: AncestorsTable
|
|
172
|
+
place_population: PlacePopulationTable
|
|
173
|
+
place_abbr: PlaceAbbrTable
|
|
174
|
+
concordances: ConcordancesTable
|
|
175
|
+
coincident_roles: CoincidentRolesTable
|
|
176
|
+
}
|
package/sharding.ts
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Multi-shard support for `WOFSqlitePlaceLookup` — opens multiple WOF SQLite distributions on one
|
|
7
|
+
* connection via `ATTACH DATABASE`, and routes queries to the right shard based on placetype.
|
|
8
|
+
*
|
|
9
|
+
* ## The FTS5 syntax rule that drove this design
|
|
10
|
+
*
|
|
11
|
+
* The naive `SELECT … FROM pc.place_search WHERE pc.place_search MATCH ?` fails — SQLite parses the
|
|
12
|
+
* schema-qualified table on the left of MATCH as "column place_search of table pc". Discovered in
|
|
13
|
+
* the spike at PR review time; documented as `_SHARD_RULE.md` should it ever bite again.
|
|
14
|
+
*
|
|
15
|
+
* The working form: schema-qualified in FROM, bare table name in MATCH:
|
|
16
|
+
*
|
|
17
|
+
* ```sql
|
|
18
|
+
* SELECT … FROM pc.place_search WHERE place_search MATCH ?
|
|
19
|
+
* ```
|
|
20
|
+
*
|
|
21
|
+
* Identical table names across attached shards (which is what we have — every shard ships its own
|
|
22
|
+
* `place_search` + `place_bbox`) are fine because the bare-name MATCH resolves against FROM
|
|
23
|
+
* scope.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { basename } from "node:path"
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Derive a SQL-safe schema name from a WOF distribution filename. Used by `ATTACH DATABASE … AS <name>` so each shard
|
|
30
|
+
* gets a stable, predictable handle.
|
|
31
|
+
*
|
|
32
|
+
* Convention strips the `whosonfirst-data-` prefix and the `-latest.db` (or just `.db`) suffix, then replaces `-` with
|
|
33
|
+
* `_` for SQL identifier safety.
|
|
34
|
+
*
|
|
35
|
+
* Examples:
|
|
36
|
+
*
|
|
37
|
+
* - `whosonfirst-data-admin-us-latest.db` → `admin_us`
|
|
38
|
+
* - `whosonfirst-data-postalcode-us-latest.db` → `postalcode_us`
|
|
39
|
+
* - `whosonfirst-data-admin-latest.db` → `admin`
|
|
40
|
+
* - `my-custom.db` → `my_custom`
|
|
41
|
+
*
|
|
42
|
+
* Callers can override the derived name explicitly via `ShardConfig.schemaName` when the filename doesn't follow WOF
|
|
43
|
+
* convention.
|
|
44
|
+
*/
|
|
45
|
+
export function deriveSchemaName(path: string): string {
|
|
46
|
+
const stem = basename(path)
|
|
47
|
+
.replace(/^whosonfirst-data-/u, "")
|
|
48
|
+
.replace(/-latest\.db$/u, "")
|
|
49
|
+
.replace(/\.db$/u, "")
|
|
50
|
+
.replace(/[^a-zA-Z0-9_]/g, "_")
|
|
51
|
+
|
|
52
|
+
if (!stem) {
|
|
53
|
+
throw new Error(`deriveSchemaName: could not derive a SQL schema name from path ${JSON.stringify(path)}`)
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
return stem
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Per-shard configuration. The simple form is just a path string — the schema name is derived from it. The object form
|
|
61
|
+
* lets callers override the derived schema name (useful when a filename doesn't follow WOF convention) or attach an
|
|
62
|
+
* extra hint about which placetypes route here.
|
|
63
|
+
*/
|
|
64
|
+
export interface ShardConfig {
|
|
65
|
+
path: string
|
|
66
|
+
/**
|
|
67
|
+
* Override the auto-derived schema name. Useful when the filename doesn't match WOF convention or when you want a
|
|
68
|
+
* memorable handle. Must be a valid SQLite identifier — `[a-zA-Z_][a-zA-Z0-9_]*`.
|
|
69
|
+
*/
|
|
70
|
+
schemaName?: string
|
|
71
|
+
/**
|
|
72
|
+
* Optional explicit list of placetypes this shard serves. When set, queries against any listed placetype are routed
|
|
73
|
+
* to this shard. When omitted, routing falls back to a name-match heuristic: a shard whose `schemaName` contains the
|
|
74
|
+
* placetype as a substring (e.g. `postalcode_us` for `postalcode` queries) is preferred for that placetype.
|
|
75
|
+
*/
|
|
76
|
+
placetypes?: readonly string[]
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Resolved post-derivation: paired path + chosen schema name + (possibly empty) placetypes hint. Used internally by
|
|
81
|
+
* `WOFSqlitePlaceLookup` so the routing logic operates on uniform structures.
|
|
82
|
+
*/
|
|
83
|
+
export interface ResolvedShard {
|
|
84
|
+
path: string
|
|
85
|
+
schemaName: string
|
|
86
|
+
placetypes: readonly string[]
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** SQLite identifier regex — `[A-Za-z_][A-Za-z0-9_]*`. */
|
|
90
|
+
const SQLITE_IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]*$/u
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Normalize the user-provided `databasePath` opt (which may be a single string, an array of strings, or an array of
|
|
94
|
+
* `ShardConfig` objects) into a uniform `ResolvedShard[]`.
|
|
95
|
+
*
|
|
96
|
+
* The first shard becomes `main` regardless of its derived schema name — that's the SQLite convention. Subsequent
|
|
97
|
+
* shards keep their derived (or override) schema name.
|
|
98
|
+
*/
|
|
99
|
+
export function resolveShards(input: string | ReadonlyArray<string | ShardConfig>): ResolvedShard[] {
|
|
100
|
+
const list = typeof input === "string" ? [input] : input
|
|
101
|
+
|
|
102
|
+
if (list.length === 0) throw new Error("resolveShards: at least one shard is required")
|
|
103
|
+
|
|
104
|
+
const seen = new Set<string>()
|
|
105
|
+
const out: ResolvedShard[] = []
|
|
106
|
+
|
|
107
|
+
for (let i = 0; i < list.length; i++) {
|
|
108
|
+
const entry = list[i]!
|
|
109
|
+
const cfg: ShardConfig = typeof entry === "string" ? { path: entry } : entry
|
|
110
|
+
const derived = cfg.schemaName ?? deriveSchemaName(cfg.path)
|
|
111
|
+
|
|
112
|
+
if (!SQLITE_IDENT_RE.test(derived)) {
|
|
113
|
+
throw new Error(
|
|
114
|
+
`resolveShards: schema name ${JSON.stringify(derived)} is not a valid SQLite identifier ` +
|
|
115
|
+
`(derived from path ${JSON.stringify(cfg.path)}). Pass an explicit ` +
|
|
116
|
+
`{ path, schemaName } to override.`
|
|
117
|
+
)
|
|
118
|
+
}
|
|
119
|
+
// The first shard is always main per SQLite semantics — its derived name is informational
|
|
120
|
+
// only. Subsequent shards must have unique non-main names.
|
|
121
|
+
const schemaName = i === 0 ? "main" : derived
|
|
122
|
+
|
|
123
|
+
if (i > 0 && (schemaName === "main" || seen.has(schemaName))) {
|
|
124
|
+
throw new Error(
|
|
125
|
+
`resolveShards: schema name ${JSON.stringify(schemaName)} collides ` +
|
|
126
|
+
`(either with "main" or another shard). Pass an explicit { path, schemaName }.`
|
|
127
|
+
)
|
|
128
|
+
}
|
|
129
|
+
seen.add(schemaName)
|
|
130
|
+
out.push({
|
|
131
|
+
path: cfg.path,
|
|
132
|
+
schemaName,
|
|
133
|
+
placetypes: cfg.placetypes ?? [],
|
|
134
|
+
})
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
return out
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Pick the shard to route a query to given the requested placetype(s).
|
|
142
|
+
*
|
|
143
|
+
* Routing rules, in order:
|
|
144
|
+
*
|
|
145
|
+
* 1. If any shard has explicit `placetypes` that includes the requested placetype, use it.
|
|
146
|
+
* 2. Otherwise, if a non-main shard's `schemaName` matches the placetype (e.g. `postalcode_us` matches `postalcode`), use
|
|
147
|
+
* it.
|
|
148
|
+
* 3. Otherwise, fall back to `main`.
|
|
149
|
+
*
|
|
150
|
+
* This deliberately doesn't UNION across shards — BM25 scores aren't comparable across separately- indexed corpora, and
|
|
151
|
+
* the typical mailwoman query has a single placetype anyway. If a caller needs cross-shard results they can issue two
|
|
152
|
+
* `findPlace` calls.
|
|
153
|
+
*/
|
|
154
|
+
/**
|
|
155
|
+
* All placetype-matching shards, in routing order (the country-aware pick chooses among these). Used by the bias path:
|
|
156
|
+
* a country-less postcode query with proximity hints fans out across every matching shard and merges, because
|
|
157
|
+
* single-shard routing would hide the cross-country ambiguity the hints exist to resolve ("48026" lives in
|
|
158
|
+
* postalcode-us AND postalcode-intl).
|
|
159
|
+
*/
|
|
160
|
+
export function pickShardsForPlacetype(shards: ResolvedShard[], placetype: string | undefined): ResolvedShard[] {
|
|
161
|
+
if (!placetype) return [shards[0]!]
|
|
162
|
+
const matches: ResolvedShard[] = []
|
|
163
|
+
|
|
164
|
+
for (const s of shards) {
|
|
165
|
+
if (s.placetypes.includes(placetype)) {
|
|
166
|
+
matches.push(s)
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
for (const s of shards) {
|
|
171
|
+
if (s.schemaName === "main" || matches.includes(s)) continue
|
|
172
|
+
|
|
173
|
+
if (
|
|
174
|
+
s.schemaName === placetype ||
|
|
175
|
+
s.schemaName.startsWith(`${placetype}_`) ||
|
|
176
|
+
s.schemaName.endsWith(`_${placetype}`)
|
|
177
|
+
) {
|
|
178
|
+
matches.push(s)
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
return matches.length > 0 ? matches : [shards[0]!]
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
export function pickShardForPlacetype(
|
|
186
|
+
shards: ResolvedShard[],
|
|
187
|
+
placetype: string | undefined,
|
|
188
|
+
opts?: {
|
|
189
|
+
/**
|
|
190
|
+
* #920: the query's country constraint, when the caller has one. With MULTIPLE shards matching a placetype
|
|
191
|
+
* (postalcode-us + postalcode-geonames-tail), first-match routing sent every postcode query to the first shard and
|
|
192
|
+
* starved the rest — a FI postcode could never reach the tail shard. When `country` is given and a matching shard's
|
|
193
|
+
* probed country set contains it, that shard wins; shards without the country are skipped; the placetype-match
|
|
194
|
+
* order remains the tiebreak when no shard claims the country (or none was probed).
|
|
195
|
+
*/
|
|
196
|
+
country?: string
|
|
197
|
+
/** Per-schema probed country sets (see `WOFSqlitePlaceLookup`'s construction probe). */
|
|
198
|
+
countriesBySchema?: ReadonlyMap<string, ReadonlySet<string>>
|
|
199
|
+
}
|
|
200
|
+
): ResolvedShard {
|
|
201
|
+
if (!placetype) return shards[0]!
|
|
202
|
+
|
|
203
|
+
const matches: ResolvedShard[] = []
|
|
204
|
+
|
|
205
|
+
for (const s of shards) {
|
|
206
|
+
if (s.placetypes.includes(placetype)) {
|
|
207
|
+
matches.push(s)
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
for (const s of shards) {
|
|
212
|
+
if (s.schemaName === "main" || matches.includes(s)) continue
|
|
213
|
+
|
|
214
|
+
// Substring match: `postalcode_us` matches `postalcode`. Conservative — requires the
|
|
215
|
+
// placetype to appear at a word boundary in the schema name to avoid false hits like
|
|
216
|
+
// `region` matching `arboregion`.
|
|
217
|
+
if (
|
|
218
|
+
s.schemaName === placetype ||
|
|
219
|
+
s.schemaName.startsWith(`${placetype}_`) ||
|
|
220
|
+
s.schemaName.endsWith(`_${placetype}`)
|
|
221
|
+
) {
|
|
222
|
+
matches.push(s)
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
if (matches.length === 0) return shards[0]!
|
|
227
|
+
|
|
228
|
+
if (opts?.country && opts.countriesBySchema) {
|
|
229
|
+
for (const s of matches) {
|
|
230
|
+
if (opts.countriesBySchema.get(s.schemaName)?.has(opts.country)) return s
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
return matches[0]!
|
|
235
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `SqliteConventionSource` — a `ConventionSource` backed by the build-from-source convention asset
|
|
7
|
+
* (#290, Direction E). Conventions live in a read-only, provenance-stamped `address_convention`
|
|
8
|
+
* table keyed by WOF polygon id; this source queries them ON DEMAND by id (one indexed lookup,
|
|
9
|
+
* memoized) rather than paging the whole table into memory as a code constant — the deliberate
|
|
10
|
+
* counter to the Pelias "giant dictionary in RAM, no provenance" pattern (see the operator design
|
|
11
|
+
* value in memory `feedback-no-load-bearing-trivia`).
|
|
12
|
+
*
|
|
13
|
+
* The asset is the queryable, distributable artifact; the strategy IMPLEMENTATIONS stay in code. An
|
|
14
|
+
* unknown strategy NAME is surfaced loudly at dispatch (see `lookup.ts`), not silently
|
|
15
|
+
* swallowed.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import type { DatabaseSync } from "node:sqlite"
|
|
19
|
+
|
|
20
|
+
import { ADDRESS_CONVENTION_TABLE, type Convention, type ConventionSource } from "./convention.ts"
|
|
21
|
+
|
|
22
|
+
export class SqliteConventionSource implements ConventionSource {
|
|
23
|
+
readonly #db: DatabaseSync
|
|
24
|
+
readonly #schema: string
|
|
25
|
+
/** Memoize per-id lookups (including misses, as `null`) so a hot ancestor chain is queried once. */
|
|
26
|
+
readonly #cache = new Map<number, Convention | null>()
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* @param db An open handle to a DB that has the convention asset attached (or is it).
|
|
30
|
+
* @param schema The schema name the `address_convention` table lives under (`main` or an ATTACHed shard name —
|
|
31
|
+
* `WOFSqlitePlaceLookup` auto-detects which shard carries the table).
|
|
32
|
+
*/
|
|
33
|
+
constructor(db: DatabaseSync, schema: string) {
|
|
34
|
+
this.#db = db
|
|
35
|
+
this.#schema = schema
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
get(wofID: number): Convention | undefined {
|
|
39
|
+
const cached = this.#cache.get(wofID)
|
|
40
|
+
|
|
41
|
+
if (cached !== undefined) return cached ?? undefined
|
|
42
|
+
let value: Convention | null = null
|
|
43
|
+
|
|
44
|
+
try {
|
|
45
|
+
const row = this.#db
|
|
46
|
+
.prepare(`SELECT convention FROM ${this.#schema}.${ADDRESS_CONVENTION_TABLE} WHERE wof_id = ?`)
|
|
47
|
+
.get(wofID) as { convention: string } | undefined
|
|
48
|
+
|
|
49
|
+
if (row?.convention) {
|
|
50
|
+
value = JSON.parse(row.convention) as Convention
|
|
51
|
+
}
|
|
52
|
+
} catch {
|
|
53
|
+
// Malformed JSON or a missing table → treat as no override (the chain falls back to
|
|
54
|
+
// WORLD_DEFAULT). The build script validates structure, so this is purely defensive.
|
|
55
|
+
value = null
|
|
56
|
+
}
|
|
57
|
+
this.#cache.set(wofID, value)
|
|
58
|
+
|
|
59
|
+
return value ?? undefined
|
|
60
|
+
}
|
|
61
|
+
}
|
package/sqlite-utils.ts
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Small shared helpers for the SQLite-backed lookups.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import type { DatabaseSync } from "node:sqlite"
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* True when `name` is a table in the open database. The street-level lookups use this to degrade gracefully on an
|
|
13
|
+
* empty/tableless shard — an interrupted `build-*-shard.ts`, or a stray 0-byte file (e.g. `sqlite3 <missing>.db "…"`
|
|
14
|
+
* CREATES one) — rather than throwing `no such table` at construction and taking down a whole state's geocode (#568). A
|
|
15
|
+
* missing table makes the lookup a no-op miss.
|
|
16
|
+
*/
|
|
17
|
+
export function hasTable(db: DatabaseSync, name: string): boolean {
|
|
18
|
+
try {
|
|
19
|
+
const row = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ? LIMIT 1").get(name)
|
|
20
|
+
|
|
21
|
+
return row !== undefined
|
|
22
|
+
} catch {
|
|
23
|
+
return false
|
|
24
|
+
}
|
|
25
|
+
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Typed schema for the DERIVED STREET-CENTROID shard (`street-centroids-<cc>.db`, built by
|
|
7
|
+
* `ban/scripts/build-street-centroid-shard.ts` — the #1042 street-level tier behind "street-only
|
|
8
|
+
* FR queries deserve a street-level answer"). The shard is a `GROUP BY street` roll-up of the
|
|
9
|
+
* sealed rooftop address-point shard: one row per (street, postcode, commune) carrying the street's
|
|
10
|
+
* CENTROID + bounding-box EXTENT + member-point count. No new data source — a derived artifact.
|
|
11
|
+
*
|
|
12
|
+
* Single source of truth for the columns shared by the BUILDER (a positional prepared INSERT for
|
|
13
|
+
* throughput) and the READER ({@link StreetCentroidSqliteLookup}), so a column rename in one is a
|
|
14
|
+
* compile error in the other — the same discipline as `address-point-schema.ts`.
|
|
15
|
+
*
|
|
16
|
+
* Probe scopes (most-selective first): by `postcode`, else by `locality_base` (the
|
|
17
|
+
* arrondissement-stripped commune — see `stripArrondissement`; BAN names Paris/Lyon/Marseille rows
|
|
18
|
+
* per arrondissement, but a query names the base commune). The reader WEIGHTED-aggregates across the
|
|
19
|
+
* matched rows (by `point_count`) so a locality-scope probe returns the street's grand centroid over
|
|
20
|
+
* every postcode/arrondissement it spans.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import type { Kysely } from "kysely"
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* One street roll-up. `(street_norm, postcode, locality_base)` is unique. `lat`/`lon` are the UNWEIGHTED mean of the
|
|
27
|
+
* group's member address points (each source row = one point), so a cross-group weighted mean (`SUM(lat*point_count) /
|
|
28
|
+
* SUM(point_count)`) reconstructs the grand centroid. `min_/max_lat/lon` are the group's extent (the reader turns the
|
|
29
|
+
* bbox diagonal into an honest `uncertainty_m`).
|
|
30
|
+
*/
|
|
31
|
+
export interface StreetCentroidTable {
|
|
32
|
+
/** Shared `normalizeStreetForKeyLocale` of the street — the build/query-consistent probe key. */
|
|
33
|
+
street_norm: string
|
|
34
|
+
/** The 5-digit postcode of this group, or null when the source row carried none. */
|
|
35
|
+
postcode: string | null
|
|
36
|
+
/** Arrondissement-stripped commune (`stripArrondissement(normalizeLocalityForKey(commune))`) — the fallback scope. */
|
|
37
|
+
locality_base: string
|
|
38
|
+
/** Weighted-mean centroid latitude of the street's member points. */
|
|
39
|
+
lat: number
|
|
40
|
+
/** Weighted-mean centroid longitude of the street's member points. */
|
|
41
|
+
lon: number
|
|
42
|
+
min_lat: number
|
|
43
|
+
max_lat: number
|
|
44
|
+
min_lon: number
|
|
45
|
+
max_lon: number
|
|
46
|
+
/** Member address-point count — the weight for a cross-group centroid aggregate. */
|
|
47
|
+
point_count: number
|
|
48
|
+
/** A representative street name as it appeared in the source (display / debugging). */
|
|
49
|
+
street_raw: string
|
|
50
|
+
/** Provenance: the register this street was derived from (e.g. `ban:fr`). */
|
|
51
|
+
source: string
|
|
52
|
+
/** The pinned data release the underlying points came from. */
|
|
53
|
+
release: string
|
|
54
|
+
/**
|
|
55
|
+
* #727 phase-4c: `foldStreetSurface(street_raw)` — the contract-fold street-NAME existence key for
|
|
56
|
+
* {@link StreetLocalityEvidence}. Distinct from `street_norm` (the `street-normalize` geocoding key): the
|
|
57
|
+
* name-evidence rerank folds the model's street surface with the SAME `foldStreetSurface` used to build this column
|
|
58
|
+
* (the fold-parity contract), so it must not drift from `street_norm`'s richer normalizer. Indexed (`idx_sc_name`)
|
|
59
|
+
* for a direct seek.
|
|
60
|
+
*/
|
|
61
|
+
name_key: string
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** The street-centroid database schema for `new DatabaseClient<StreetCentroidDatabase>(...)`. */
|
|
65
|
+
export interface StreetCentroidDatabase {
|
|
66
|
+
street_centroid: StreetCentroidTable
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* The `street_centroid` columns in INSERT order. The builder's positional prepared statement derives its placeholder
|
|
71
|
+
* list from this, so the positional order can't drift from the DDL / the reader.
|
|
72
|
+
*/
|
|
73
|
+
export const STREET_CENTROID_COLUMNS = [
|
|
74
|
+
"street_norm",
|
|
75
|
+
"postcode",
|
|
76
|
+
"locality_base",
|
|
77
|
+
"lat",
|
|
78
|
+
"lon",
|
|
79
|
+
"min_lat",
|
|
80
|
+
"max_lat",
|
|
81
|
+
"min_lon",
|
|
82
|
+
"max_lon",
|
|
83
|
+
"point_count",
|
|
84
|
+
"street_raw",
|
|
85
|
+
"source",
|
|
86
|
+
"release",
|
|
87
|
+
"name_key",
|
|
88
|
+
] as const
|
|
89
|
+
|
|
90
|
+
/** Create the `street_centroid` table — called before the streaming bulk load. */
|
|
91
|
+
export async function createStreetCentroidTable(db: Kysely<StreetCentroidDatabase>): Promise<void> {
|
|
92
|
+
await db.schema
|
|
93
|
+
.createTable("street_centroid")
|
|
94
|
+
.addColumn("street_norm", "text", (c) => c.notNull())
|
|
95
|
+
.addColumn("postcode", "text")
|
|
96
|
+
.addColumn("locality_base", "text", (c) => c.notNull())
|
|
97
|
+
.addColumn("lat", "real", (c) => c.notNull())
|
|
98
|
+
.addColumn("lon", "real", (c) => c.notNull())
|
|
99
|
+
.addColumn("min_lat", "real", (c) => c.notNull())
|
|
100
|
+
.addColumn("max_lat", "real", (c) => c.notNull())
|
|
101
|
+
.addColumn("min_lon", "real", (c) => c.notNull())
|
|
102
|
+
.addColumn("max_lon", "real", (c) => c.notNull())
|
|
103
|
+
.addColumn("point_count", "integer", (c) => c.notNull())
|
|
104
|
+
.addColumn("street_raw", "text", (c) => c.notNull())
|
|
105
|
+
.addColumn("source", "text", (c) => c.notNull())
|
|
106
|
+
.addColumn("release", "text", (c) => c.notNull())
|
|
107
|
+
.addColumn("name_key", "text", (c) => c.notNull())
|
|
108
|
+
.execute()
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Create the probe indexes: the two geocoding-scope indexes (postcode, locality-base) the resolver reader relies on,
|
|
113
|
+
* plus `idx_sc_name` — the #727 phase-4c name-existence key for a direct `name_key = ?` seek (the unscoped fragment
|
|
114
|
+
* lookup; without it that query skip-scans `idx_sc_postcode` at ~5 ms/probe).
|
|
115
|
+
*/
|
|
116
|
+
export async function createStreetCentroidIndexes(db: Kysely<StreetCentroidDatabase>): Promise<void> {
|
|
117
|
+
await db.schema.createIndex("idx_sc_postcode").on("street_centroid").columns(["postcode", "street_norm"]).execute()
|
|
118
|
+
await db.schema
|
|
119
|
+
.createIndex("idx_sc_locality")
|
|
120
|
+
.on("street_centroid")
|
|
121
|
+
.columns(["locality_base", "street_norm"])
|
|
122
|
+
.execute()
|
|
123
|
+
await db.schema.createIndex("idx_sc_name").on("street_centroid").columns(["name_key"]).execute()
|
|
124
|
+
}
|