@mailwoman/resolver-wof-sqlite 9.4.0 → 10.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/lib/address/{point-interpolation.ts → point/interpolation.ts} +40 -38
- package/lib/address/point/schema.ts +132 -0
- package/lib/address/point.ts +68 -81
- package/lib/{address/index.ts → address.ts} +2 -2
- package/lib/ancestry/backfill.ts +24 -88
- package/lib/{ancestry/index.ts → ancestry.ts} +19 -15
- package/lib/build/candidate.ts +608 -0
- package/lib/build/population-corrections.ts +64 -0
- package/lib/build/slim.ts +338 -0
- package/lib/candidate/alias-bags.ts +74 -14
- package/lib/candidate/ancestors/schema.ts +211 -0
- package/lib/candidate/{ancestors-sidecar.ts → ancestors/sidecar.ts} +17 -24
- package/lib/candidate/country-display-names.ts +6 -20
- package/lib/candidate/extract-fold.ts +12 -57
- package/lib/candidate/fts.ts +28 -0
- package/lib/candidate/importance.ts +221 -0
- package/lib/candidate/lookup.ts +669 -0
- package/lib/candidate/name-roles.ts +16 -34
- package/lib/candidate/own-name.ts +34 -31
- package/lib/candidate/place-attrs.ts +20 -13
- package/lib/candidate/schema.ts +211 -0
- package/lib/candidate/scoring.ts +222 -0
- package/lib/capital-schema.ts +21 -17
- package/lib/capitals.ts +26 -24
- package/lib/coincident-roles.ts +74 -55
- package/lib/convention/schema.ts +13 -7
- package/lib/convention.ts +144 -0
- package/lib/coverage-manifest-schema.ts +34 -39
- package/lib/currency-backfill.ts +24 -69
- package/lib/env.ts +1 -1
- package/lib/exact-match.ts +31 -20
- package/lib/extracts.ts +96 -73
- package/lib/fst/autocomplete.ts +52 -35
- package/lib/fst/builder.ts +30 -36
- package/lib/fst/deserialize-web.ts +8 -9
- package/lib/fst/format.ts +41 -23
- package/lib/fst/freshness.ts +17 -95
- package/lib/fst/matcher.ts +2 -2
- package/lib/fst/serialize.ts +45 -60
- package/lib/fst/types.ts +71 -59
- package/lib/fts/query.ts +5 -36
- package/lib/fts.ts +280 -0
- package/lib/geonames/aliases.ts +72 -149
- package/lib/geonames/postal.ts +162 -77
- package/lib/index.ts +20 -27
- package/lib/interpolation.ts +43 -96
- package/lib/lookup.ts +137 -257
- package/lib/name-score.ts +9 -7
- package/lib/nsul/lookup.ts +36 -53
- package/lib/nsul/schema.ts +16 -43
- package/lib/paths.ts +159 -0
- package/lib/place-importance-schema.ts +57 -164
- package/lib/poi/lookup.ts +100 -109
- package/lib/poi/schema.ts +42 -26
- package/lib/polygon-schema.ts +10 -6
- package/lib/postal/city/alias/lookup.ts +79 -0
- package/lib/postal/{city-alias-schema.ts → city/alias/schema.ts} +14 -10
- package/lib/postal/{city-candidate-schema.ts → city/candidate-schema.ts} +26 -20
- package/lib/postal.ts +9 -0
- package/lib/postcode-point-lookup.ts +9 -19
- package/lib/primary-preference.ts +45 -125
- package/lib/proximity-rerank.ts +43 -32
- package/lib/ranking-weights.ts +98 -76
- package/lib/region-keys.ts +68 -40
- package/lib/reverse.ts +108 -72
- package/lib/schema.ts +74 -42
- package/lib/search-fetch.ts +90 -68
- package/lib/sqlite/convention-source.ts +57 -0
- package/lib/sqlite/utils.ts +79 -0
- package/lib/street/centroid/schema.ts +149 -0
- package/lib/street/centroid.ts +16 -25
- package/lib/street/{morphology-fst-builder.ts → morphology/fst/builder.ts} +56 -41
- package/lib/street/{morphology-fst-loader.ts → morphology/fst/loader.ts} +36 -29
- package/lib/street/name-lookup.ts +43 -35
- package/lib/street/normalize.ts +85 -216
- package/lib/street/segment-schema.ts +116 -43
- package/lib/{street/index.ts → street.ts} +3 -3
- package/lib/types.ts +94 -140
- package/lib/unified-schema.ts +29 -29
- package/lib/uprn/existence.ts +30 -19
- package/lib/uprn/lookup.ts +28 -67
- package/lib/uprn/schema.ts +18 -40
- package/lib/weights-overlay-linker.ts +204 -232
- package/out/address/point/interpolation.d.ts +45 -0
- package/out/address/point/interpolation.d.ts.map +1 -0
- package/out/address/{point-interpolation.js → point/interpolation.js} +38 -38
- package/out/address/point/interpolation.js.map +1 -0
- package/out/address/point/schema.d.ts +74 -0
- package/out/address/point/schema.d.ts.map +1 -0
- package/out/address/{point-schema.js → point/schema.js} +4 -20
- package/out/address/point/schema.js.map +1 -0
- package/out/address/point.d.ts +24 -21
- package/out/address/point.d.ts.map +1 -1
- package/out/address/point.js +60 -77
- package/out/address/point.js.map +1 -1
- package/out/{address/index.js → address.d.ts} +3 -3
- package/out/address.d.ts.map +1 -0
- package/out/{address/index.d.ts → address.js} +3 -3
- package/out/address.js.map +1 -0
- package/out/ancestry/backfill.d.ts +11 -64
- package/out/ancestry/backfill.d.ts.map +1 -1
- package/out/ancestry/backfill.js +18 -76
- package/out/ancestry/backfill.js.map +1 -1
- package/out/ancestry.d.ts +49 -0
- package/out/ancestry.d.ts.map +1 -0
- package/out/ancestry.js +59 -0
- package/out/ancestry.js.map +1 -0
- package/out/build/candidate.d.ts +170 -0
- package/out/build/candidate.d.ts.map +1 -0
- package/out/{build-candidate.js → build/candidate.js} +33 -102
- package/out/build/candidate.js.map +1 -0
- package/out/build/population-corrections.d.ts +52 -0
- package/out/build/population-corrections.d.ts.map +1 -0
- package/out/build/population-corrections.js +40 -0
- package/out/build/population-corrections.js.map +1 -0
- package/out/build/slim.d.ts +44 -0
- package/out/build/slim.d.ts.map +1 -0
- package/out/{build-slim.js → build/slim.js} +33 -94
- package/out/build/slim.js.map +1 -0
- package/out/candidate/alias-bags.d.ts +13 -5
- package/out/candidate/alias-bags.d.ts.map +1 -1
- package/out/candidate/alias-bags.js +55 -12
- package/out/candidate/alias-bags.js.map +1 -1
- package/out/candidate/ancestors/schema.d.ts +166 -0
- package/out/candidate/ancestors/schema.d.ts.map +1 -0
- package/out/candidate/ancestors/schema.js +131 -0
- package/out/candidate/ancestors/schema.js.map +1 -0
- package/out/candidate/ancestors/sidecar.d.ts +27 -0
- package/out/candidate/ancestors/sidecar.d.ts.map +1 -0
- package/out/candidate/{ancestors-sidecar.js → ancestors/sidecar.js} +17 -24
- package/out/candidate/ancestors/sidecar.js.map +1 -0
- package/out/candidate/country-display-names.d.ts +5 -15
- package/out/candidate/country-display-names.d.ts.map +1 -1
- package/out/candidate/country-display-names.js +6 -20
- package/out/candidate/country-display-names.js.map +1 -1
- package/out/candidate/extract-fold.d.ts +8 -16
- package/out/candidate/extract-fold.d.ts.map +1 -1
- package/out/candidate/extract-fold.js +6 -48
- package/out/candidate/extract-fold.js.map +1 -1
- package/out/candidate/fts.d.ts +19 -0
- package/out/candidate/fts.d.ts.map +1 -0
- package/out/candidate/fts.js +22 -0
- package/out/candidate/fts.js.map +1 -0
- package/out/candidate/importance.d.ts +127 -0
- package/out/candidate/importance.d.ts.map +1 -0
- package/out/candidate/importance.js +162 -0
- package/out/candidate/importance.js.map +1 -0
- package/out/candidate/lookup.d.ts +36 -0
- package/out/candidate/lookup.d.ts.map +1 -0
- package/out/candidate/lookup.js +456 -0
- package/out/candidate/lookup.js.map +1 -0
- package/out/candidate/name-roles.d.ts +10 -22
- package/out/candidate/name-roles.d.ts.map +1 -1
- package/out/candidate/name-roles.js +15 -33
- package/out/candidate/name-roles.js.map +1 -1
- package/out/candidate/own-name.d.ts +23 -25
- package/out/candidate/own-name.d.ts.map +1 -1
- package/out/candidate/own-name.js +34 -31
- package/out/candidate/own-name.js.map +1 -1
- package/out/candidate/place-attrs.d.ts +20 -13
- package/out/candidate/place-attrs.d.ts.map +1 -1
- package/out/candidate/place-attrs.js +4 -5
- package/out/candidate/place-attrs.js.map +1 -1
- package/out/candidate/schema.d.ts +120 -0
- package/out/candidate/schema.d.ts.map +1 -0
- package/out/{candidate-schema.js → candidate/schema.js} +11 -23
- package/out/candidate/schema.js.map +1 -0
- package/out/{candidate-scoring.d.ts → candidate/scoring.d.ts} +8 -9
- package/out/candidate/scoring.d.ts.map +1 -0
- package/out/candidate/scoring.js +157 -0
- package/out/candidate/scoring.js.map +1 -0
- package/out/capital-schema.d.ts +21 -17
- package/out/capital-schema.d.ts.map +1 -1
- package/out/capital-schema.js +17 -15
- package/out/capital-schema.js.map +1 -1
- package/out/capitals.d.ts +26 -24
- package/out/capitals.d.ts.map +1 -1
- package/out/capitals.js +23 -21
- package/out/capitals.js.map +1 -1
- package/out/coincident-roles.d.ts +60 -43
- package/out/coincident-roles.d.ts.map +1 -1
- package/out/coincident-roles.js +61 -50
- package/out/coincident-roles.js.map +1 -1
- package/out/convention/schema.d.ts +12 -6
- package/out/convention/schema.d.ts.map +1 -1
- package/out/convention/schema.js +6 -4
- package/out/convention/schema.js.map +1 -1
- package/out/convention.d.ts +97 -0
- package/out/convention.d.ts.map +1 -0
- package/out/convention.js +81 -0
- package/out/convention.js.map +1 -0
- package/out/coverage-manifest-schema.d.ts +33 -38
- package/out/coverage-manifest-schema.d.ts.map +1 -1
- package/out/coverage-manifest-schema.js +20 -29
- package/out/coverage-manifest-schema.js.map +1 -1
- package/out/currency-backfill.d.ts +13 -35
- package/out/currency-backfill.d.ts.map +1 -1
- package/out/currency-backfill.js +12 -39
- package/out/currency-backfill.js.map +1 -1
- package/out/env.d.ts +2 -2
- package/out/env.js +1 -1
- package/out/exact-match.d.ts +19 -10
- package/out/exact-match.d.ts.map +1 -1
- package/out/exact-match.js +31 -20
- package/out/exact-match.js.map +1 -1
- package/out/extracts.d.ts +65 -54
- package/out/extracts.d.ts.map +1 -1
- package/out/extracts.js +63 -55
- package/out/extracts.js.map +1 -1
- package/out/fst/autocomplete.d.ts +28 -20
- package/out/fst/autocomplete.d.ts.map +1 -1
- package/out/fst/autocomplete.js +38 -27
- package/out/fst/autocomplete.js.map +1 -1
- package/out/fst/builder.d.ts.map +1 -1
- package/out/fst/builder.js +30 -36
- package/out/fst/builder.js.map +1 -1
- package/out/fst/deserialize-web.d.ts +3 -3
- package/out/fst/deserialize-web.d.ts.map +1 -1
- package/out/fst/deserialize-web.js +8 -9
- package/out/fst/deserialize-web.js.map +1 -1
- package/out/fst/format.d.ts +41 -23
- package/out/fst/format.d.ts.map +1 -1
- package/out/fst/format.js +41 -23
- package/out/fst/format.js.map +1 -1
- package/out/fst/freshness.d.ts +11 -73
- package/out/fst/freshness.d.ts.map +1 -1
- package/out/fst/freshness.js +12 -76
- package/out/fst/freshness.js.map +1 -1
- package/out/fst/matcher.d.ts +1 -1
- package/out/fst/matcher.d.ts.map +1 -1
- package/out/fst/matcher.js +2 -2
- package/out/fst/matcher.js.map +1 -1
- package/out/fst/serialize.d.ts +30 -26
- package/out/fst/serialize.d.ts.map +1 -1
- package/out/fst/serialize.js +44 -58
- package/out/fst/serialize.js.map +1 -1
- package/out/fst/types.d.ts +71 -59
- package/out/fst/types.d.ts.map +1 -1
- package/out/fst/types.js +1 -4
- package/out/fst/types.js.map +1 -1
- package/out/{fst/index.js → fst.d.ts} +1 -1
- package/out/fst.d.ts.map +1 -0
- package/out/{fst/index.d.ts → fst.js} +1 -1
- package/out/fst.js.map +1 -0
- package/out/fts/query.d.ts +2 -27
- package/out/fts/query.d.ts.map +1 -1
- package/out/fts/query.js +5 -36
- package/out/fts/query.js.map +1 -1
- package/out/fts.d.ts +126 -0
- package/out/fts.d.ts.map +1 -0
- package/out/fts.js +188 -0
- package/out/fts.js.map +1 -0
- package/out/geonames/aliases.d.ts +26 -67
- package/out/geonames/aliases.d.ts.map +1 -1
- package/out/geonames/aliases.js +44 -121
- package/out/geonames/aliases.js.map +1 -1
- package/out/geonames/postal.d.ts +84 -50
- package/out/geonames/postal.d.ts.map +1 -1
- package/out/geonames/postal.js +115 -78
- package/out/geonames/postal.js.map +1 -1
- package/out/{geonames/index.js → geonames.d.ts} +1 -1
- package/out/geonames.d.ts.map +1 -0
- package/out/{geonames/index.d.ts → geonames.js} +1 -1
- package/out/geonames.js.map +1 -0
- package/out/index.d.ts +20 -21
- package/out/index.d.ts.map +1 -1
- package/out/index.js +14 -15
- package/out/index.js.map +1 -1
- package/out/interpolation.d.ts +15 -52
- package/out/interpolation.d.ts.map +1 -1
- package/out/interpolation.js +25 -68
- package/out/interpolation.js.map +1 -1
- package/out/lookup.d.ts +24 -53
- package/out/lookup.d.ts.map +1 -1
- package/out/lookup.js +117 -223
- package/out/lookup.js.map +1 -1
- package/out/name-score.d.ts +2 -2
- package/out/name-score.d.ts.map +1 -1
- package/out/name-score.js +9 -7
- package/out/name-score.js.map +1 -1
- package/out/nsul/lookup.d.ts +33 -34
- package/out/nsul/lookup.d.ts.map +1 -1
- package/out/nsul/lookup.js +26 -40
- package/out/nsul/lookup.js.map +1 -1
- package/out/nsul/schema.d.ts +16 -43
- package/out/nsul/schema.d.ts.map +1 -1
- package/out/nsul/schema.js +9 -35
- package/out/nsul/schema.js.map +1 -1
- package/out/{nsul/index.d.ts → nsul.d.ts} +1 -1
- package/out/nsul.d.ts.map +1 -0
- package/out/{nsul/index.js → nsul.js} +1 -1
- package/out/nsul.js.map +1 -0
- package/out/paths.d.ts +124 -0
- package/out/paths.d.ts.map +1 -0
- package/out/paths.js +104 -0
- package/out/paths.js.map +1 -0
- package/out/place-importance-schema.d.ts +49 -141
- package/out/place-importance-schema.d.ts.map +1 -1
- package/out/place-importance-schema.js +25 -140
- package/out/place-importance-schema.js.map +1 -1
- package/out/poi/lookup.d.ts +38 -42
- package/out/poi/lookup.d.ts.map +1 -1
- package/out/poi/lookup.js +67 -90
- package/out/poi/lookup.js.map +1 -1
- package/out/poi/schema.d.ts +41 -25
- package/out/poi/schema.d.ts.map +1 -1
- package/out/poi/schema.js +25 -17
- package/out/poi/schema.js.map +1 -1
- package/out/{poi/index.js → poi.d.ts} +1 -1
- package/out/poi.d.ts.map +1 -0
- package/out/{poi/index.d.ts → poi.js} +1 -1
- package/out/poi.js.map +1 -0
- package/out/polygon-schema.d.ts +10 -6
- package/out/polygon-schema.d.ts.map +1 -1
- package/out/polygon-schema.js +2 -2
- package/out/polygon-schema.js.map +1 -1
- package/out/postal/{city-alias-lookup.d.ts → city/alias/lookup.d.ts} +22 -25
- package/out/postal/city/alias/lookup.d.ts.map +1 -0
- package/out/postal/city/alias/lookup.js +49 -0
- package/out/postal/city/alias/lookup.js.map +1 -0
- package/out/postal/{city-alias-schema.d.ts → city/alias/schema.d.ts} +15 -11
- package/out/postal/city/alias/schema.d.ts.map +1 -0
- package/out/postal/{city-alias-schema.js → city/alias/schema.js} +11 -9
- package/out/postal/city/alias/schema.js.map +1 -0
- package/out/postal/city/candidate-schema.d.ts +72 -0
- package/out/postal/city/candidate-schema.d.ts.map +1 -0
- package/out/postal/city/candidate-schema.js +60 -0
- package/out/postal/city/candidate-schema.js.map +1 -0
- package/out/postal.d.ts +9 -0
- package/out/postal.d.ts.map +1 -0
- package/out/postal.js +9 -0
- package/out/postal.js.map +1 -0
- package/out/postcode-point-lookup.d.ts +9 -19
- package/out/postcode-point-lookup.d.ts.map +1 -1
- package/out/postcode-point-lookup.js +5 -17
- package/out/postcode-point-lookup.js.map +1 -1
- package/out/primary-preference.d.ts +39 -93
- package/out/primary-preference.d.ts.map +1 -1
- package/out/primary-preference.js +13 -97
- package/out/primary-preference.js.map +1 -1
- package/out/proximity-rerank.d.ts +40 -29
- package/out/proximity-rerank.d.ts.map +1 -1
- package/out/proximity-rerank.js +38 -29
- package/out/proximity-rerank.js.map +1 -1
- package/out/ranking-weights.d.ts +91 -62
- package/out/ranking-weights.d.ts.map +1 -1
- package/out/ranking-weights.js +26 -25
- package/out/ranking-weights.js.map +1 -1
- package/out/region-keys.d.ts +48 -29
- package/out/region-keys.d.ts.map +1 -1
- package/out/region-keys.js +68 -40
- package/out/region-keys.js.map +1 -1
- package/out/reverse.d.ts +75 -50
- package/out/reverse.d.ts.map +1 -1
- package/out/reverse.js +61 -49
- package/out/reverse.js.map +1 -1
- package/out/schema.d.ts +74 -42
- package/out/schema.d.ts.map +1 -1
- package/out/schema.js +4 -5
- package/out/schema.js.map +1 -1
- package/out/search-fetch.d.ts +16 -12
- package/out/search-fetch.d.ts.map +1 -1
- package/out/search-fetch.js +83 -63
- package/out/search-fetch.js.map +1 -1
- package/out/sqlite/convention-source.d.ts +25 -0
- package/out/sqlite/convention-source.d.ts.map +1 -0
- package/out/sqlite/convention-source.js +48 -0
- package/out/sqlite/convention-source.js.map +1 -0
- package/out/sqlite/utils.d.ts +45 -0
- package/out/sqlite/utils.d.ts.map +1 -0
- package/out/sqlite/utils.js +56 -0
- package/out/sqlite/utils.js.map +1 -0
- package/out/street/centroid/schema.d.ts +100 -0
- package/out/street/centroid/schema.d.ts.map +1 -0
- package/out/street/centroid/schema.js +66 -0
- package/out/street/centroid/schema.js.map +1 -0
- package/out/street/centroid.d.ts +9 -19
- package/out/street/centroid.d.ts.map +1 -1
- package/out/street/centroid.js +11 -22
- package/out/street/centroid.js.map +1 -1
- package/out/street/{morphology-fst-builder.d.ts → morphology/fst/builder.d.ts} +21 -11
- package/out/street/morphology/fst/builder.d.ts.map +1 -0
- package/out/street/{morphology-fst-builder.js → morphology/fst/builder.js} +40 -34
- package/out/street/morphology/fst/builder.js.map +1 -0
- package/out/street/morphology/fst/loader.d.ts +74 -0
- package/out/street/morphology/fst/loader.d.ts.map +1 -0
- package/out/street/{morphology-fst-loader.js → morphology/fst/loader.js} +21 -21
- package/out/street/morphology/fst/loader.js.map +1 -0
- package/out/street/name-lookup.d.ts +24 -18
- package/out/street/name-lookup.d.ts.map +1 -1
- package/out/street/name-lookup.js +32 -31
- package/out/street/name-lookup.js.map +1 -1
- package/out/street/normalize.d.ts +35 -136
- package/out/street/normalize.d.ts.map +1 -1
- package/out/street/normalize.js +74 -191
- package/out/street/normalize.js.map +1 -1
- package/out/street/segment-schema.d.ts +79 -39
- package/out/street/segment-schema.d.ts.map +1 -1
- package/out/street/segment-schema.js +71 -21
- package/out/street/segment-schema.js.map +1 -1
- package/out/{street/index.d.ts → street.d.ts} +4 -4
- package/out/street.d.ts.map +1 -0
- package/out/{street/index.js → street.js} +4 -4
- package/out/street.js.map +1 -0
- package/out/types.d.ts +69 -138
- package/out/types.d.ts.map +1 -1
- package/out/types.js +0 -6
- package/out/types.js.map +1 -1
- package/out/unified-schema.d.ts +8 -12
- package/out/unified-schema.d.ts.map +1 -1
- package/out/unified-schema.js +29 -29
- package/out/unified-schema.js.map +1 -1
- package/out/uprn/existence.d.ts +28 -17
- package/out/uprn/existence.d.ts.map +1 -1
- package/out/uprn/existence.js +26 -16
- package/out/uprn/existence.js.map +1 -1
- package/out/uprn/lookup.d.ts +14 -44
- package/out/uprn/lookup.d.ts.map +1 -1
- package/out/uprn/lookup.js +20 -56
- package/out/uprn/lookup.js.map +1 -1
- package/out/uprn/schema.d.ts +18 -40
- package/out/uprn/schema.d.ts.map +1 -1
- package/out/uprn/schema.js +10 -32
- package/out/uprn/schema.js.map +1 -1
- package/out/{uprn/index.d.ts → uprn.d.ts} +1 -1
- package/out/uprn.d.ts.map +1 -0
- package/out/{uprn/index.js → uprn.js} +1 -1
- package/out/uprn.js.map +1 -0
- package/out/weights-overlay-linker.d.ts +132 -141
- package/out/weights-overlay-linker.d.ts.map +1 -1
- package/out/weights-overlay-linker.js +86 -154
- package/out/weights-overlay-linker.js.map +1 -1
- package/package.json +94 -599
- package/lib/address/point-schema.ts +0 -155
- package/lib/build-candidate.ts +0 -632
- package/lib/build-slim.ts +0 -408
- package/lib/candidate-ancestors-schema.ts +0 -195
- package/lib/candidate-fts.ts +0 -46
- package/lib/candidate-importance.ts +0 -224
- package/lib/candidate-lookup.ts +0 -977
- package/lib/candidate-schema.ts +0 -247
- package/lib/candidate-scoring.ts +0 -261
- package/lib/convention/index.ts +0 -154
- package/lib/fts/index.ts +0 -361
- package/lib/postal/city-alias-lookup.ts +0 -97
- package/lib/postal/index.ts +0 -9
- package/lib/sqlite-convention-source.ts +0 -65
- package/lib/sqlite-utils.ts +0 -82
- package/lib/street/centroid-schema.ts +0 -154
- package/out/address/index.d.ts.map +0 -1
- package/out/address/index.js.map +0 -1
- package/out/address/point-interpolation.d.ts +0 -49
- package/out/address/point-interpolation.d.ts.map +0 -1
- package/out/address/point-interpolation.js.map +0 -1
- package/out/address/point-schema.d.ts +0 -97
- package/out/address/point-schema.d.ts.map +0 -1
- package/out/address/point-schema.js.map +0 -1
- package/out/ancestry/index.d.ts +0 -45
- package/out/ancestry/index.d.ts.map +0 -1
- package/out/ancestry/index.js +0 -55
- package/out/ancestry/index.js.map +0 -1
- package/out/build-candidate.d.ts +0 -184
- package/out/build-candidate.d.ts.map +0 -1
- package/out/build-candidate.js.map +0 -1
- package/out/build-slim.d.ts +0 -80
- package/out/build-slim.d.ts.map +0 -1
- package/out/build-slim.js.map +0 -1
- package/out/candidate/ancestors-sidecar.d.ts +0 -32
- package/out/candidate/ancestors-sidecar.d.ts.map +0 -1
- package/out/candidate/ancestors-sidecar.js.map +0 -1
- package/out/candidate-ancestors-schema.d.ts +0 -150
- package/out/candidate-ancestors-schema.d.ts.map +0 -1
- package/out/candidate-ancestors-schema.js +0 -123
- package/out/candidate-ancestors-schema.js.map +0 -1
- package/out/candidate-fts.d.ts +0 -37
- package/out/candidate-fts.d.ts.map +0 -1
- package/out/candidate-fts.js +0 -40
- package/out/candidate-fts.js.map +0 -1
- package/out/candidate-importance.d.ts +0 -132
- package/out/candidate-importance.d.ts.map +0 -1
- package/out/candidate-importance.js +0 -169
- package/out/candidate-importance.js.map +0 -1
- package/out/candidate-lookup.d.ts +0 -76
- package/out/candidate-lookup.d.ts.map +0 -1
- package/out/candidate-lookup.js +0 -768
- package/out/candidate-lookup.js.map +0 -1
- package/out/candidate-schema.d.ts +0 -155
- package/out/candidate-schema.d.ts.map +0 -1
- package/out/candidate-schema.js.map +0 -1
- package/out/candidate-scoring.d.ts.map +0 -1
- package/out/candidate-scoring.js +0 -196
- package/out/candidate-scoring.js.map +0 -1
- package/out/convention/index.d.ts +0 -106
- package/out/convention/index.d.ts.map +0 -1
- package/out/convention/index.js +0 -93
- package/out/convention/index.js.map +0 -1
- package/out/fst/index.d.ts.map +0 -1
- package/out/fst/index.js.map +0 -1
- package/out/fts/index.d.ts +0 -184
- package/out/fts/index.d.ts.map +0 -1
- package/out/fts/index.js +0 -277
- package/out/fts/index.js.map +0 -1
- package/out/geonames/index.d.ts.map +0 -1
- package/out/geonames/index.js.map +0 -1
- package/out/nsul/index.d.ts.map +0 -1
- package/out/nsul/index.js.map +0 -1
- package/out/poi/index.d.ts.map +0 -1
- package/out/poi/index.js.map +0 -1
- package/out/postal/city-alias-lookup.d.ts.map +0 -1
- package/out/postal/city-alias-lookup.js +0 -62
- package/out/postal/city-alias-lookup.js.map +0 -1
- package/out/postal/city-alias-schema.d.ts.map +0 -1
- package/out/postal/city-alias-schema.js.map +0 -1
- package/out/postal/city-candidate-schema.d.ts +0 -66
- package/out/postal/city-candidate-schema.d.ts.map +0 -1
- package/out/postal/city-candidate-schema.js +0 -56
- package/out/postal/city-candidate-schema.js.map +0 -1
- package/out/postal/index.d.ts +0 -9
- package/out/postal/index.d.ts.map +0 -1
- package/out/postal/index.js +0 -9
- package/out/postal/index.js.map +0 -1
- package/out/sqlite-convention-source.d.ts +0 -29
- package/out/sqlite-convention-source.d.ts.map +0 -1
- package/out/sqlite-convention-source.js +0 -57
- package/out/sqlite-convention-source.js.map +0 -1
- package/out/sqlite-utils.d.ts +0 -45
- package/out/sqlite-utils.d.ts.map +0 -1
- package/out/sqlite-utils.js +0 -57
- package/out/sqlite-utils.js.map +0 -1
- package/out/street/centroid-schema.d.ts +0 -105
- package/out/street/centroid-schema.d.ts.map +0 -1
- package/out/street/centroid-schema.js +0 -78
- package/out/street/centroid-schema.js.map +0 -1
- package/out/street/index.d.ts.map +0 -1
- package/out/street/index.js.map +0 -1
- package/out/street/morphology-fst-builder.d.ts.map +0 -1
- package/out/street/morphology-fst-builder.js.map +0 -1
- package/out/street/morphology-fst-loader.d.ts +0 -67
- package/out/street/morphology-fst-loader.d.ts.map +0 -1
- package/out/street/morphology-fst-loader.js.map +0 -1
- package/out/uprn/index.d.ts.map +0 -1
- package/out/uprn/index.js.map +0 -1
- /package/lib/{fst/index.ts → fst.ts} +0 -0
- /package/lib/{geonames/index.ts → geonames.ts} +0 -0
- /package/lib/{nsul/index.ts → nsul.ts} +0 -0
- /package/lib/{poi/index.ts → poi.ts} +0 -0
- /package/lib/{uprn/index.ts → uprn.ts} +0 -0
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { statPath, pathExists } from "@mailwoman/core/fs/readers"
|
|
8
|
+
import { temporaryDirectory } from "@mailwoman/core/fs/temporary"
|
|
9
|
+
import { copyFileTo, removePath } from "@mailwoman/core/fs/writers"
|
|
10
|
+
import { stringifyJSON } from "@mailwoman/core/json"
|
|
11
|
+
import { countRows } from "@mailwoman/sqlite"
|
|
12
|
+
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
13
|
+
import { sealDatabase } from "@mailwoman/sqlite/sealed/db"
|
|
14
|
+
import { sql } from "kysely"
|
|
15
|
+
import type { Kysely } from "kysely"
|
|
16
|
+
import type { PathBuilderLike } from "path-ts"
|
|
17
|
+
|
|
18
|
+
import { buildPlaceSearchFTS, PLACE_BBOX_TABLE, PLACE_POPULATION_TABLE, PLACE_SEARCH_TABLE } from "#fts"
|
|
19
|
+
import type { NamesTable, SprTable, WOFDatabase } from "#schema"
|
|
20
|
+
|
|
21
|
+
export interface BuildSlimOptions {
|
|
22
|
+
/**
|
|
23
|
+
* Input WOF SQLite distributions, each already carrying spr / names / place_population tables.
|
|
24
|
+
*/
|
|
25
|
+
inputs: readonly PathBuilderLike[]
|
|
26
|
+
/**
|
|
27
|
+
* Output path for the slim DB, overwritten if it exists.
|
|
28
|
+
*/
|
|
29
|
+
output: PathBuilderLike
|
|
30
|
+
/**
|
|
31
|
+
* Country codes to keep (ISO 2-letter); defaults to `["US"]`.
|
|
32
|
+
*/
|
|
33
|
+
countries?: string[]
|
|
34
|
+
/**
|
|
35
|
+
* Cap on the number of localities to keep per country, by descending population.
|
|
36
|
+
*/
|
|
37
|
+
topLocalitiesPerCountry?: number
|
|
38
|
+
/**
|
|
39
|
+
* Drop the `names` table after the FTS index is built; `place_search` is self-contained
|
|
40
|
+
* and the resolver never reads `names` at runtime.
|
|
41
|
+
*/
|
|
42
|
+
dropNames?: boolean
|
|
43
|
+
|
|
44
|
+
onProgress?: (phase: SlimBuildPhase, detail: string) => void
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export type SlimBuildPhase =
|
|
48
|
+
| "init"
|
|
49
|
+
| "schema"
|
|
50
|
+
| "country"
|
|
51
|
+
| "region"
|
|
52
|
+
| "county"
|
|
53
|
+
| "locality"
|
|
54
|
+
| "postcode"
|
|
55
|
+
| "names"
|
|
56
|
+
| "place_population"
|
|
57
|
+
| "coincident_roles"
|
|
58
|
+
| "place_abbr"
|
|
59
|
+
| "fts"
|
|
60
|
+
| "vacuum"
|
|
61
|
+
| "done"
|
|
62
|
+
|
|
63
|
+
export interface BuildSlimResult {
|
|
64
|
+
outputPath: string
|
|
65
|
+
outputBytes: number
|
|
66
|
+
rowCounts: {
|
|
67
|
+
spr: number
|
|
68
|
+
names: number
|
|
69
|
+
placeSearch: number
|
|
70
|
+
placeBbox: number
|
|
71
|
+
placePopulation: number
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Placetypes that we always keep so the ancestor chain a selected locality reports stays valid.
|
|
77
|
+
*/
|
|
78
|
+
const ANCESTOR_PLACETYPES = ["country", "region", "county", "borough", "macroregion"] as const
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Tables copied verbatim (schema + filtered rows) from each source DB.
|
|
82
|
+
* Anything else is dropped.
|
|
83
|
+
*/
|
|
84
|
+
const COPIED_TABLES = ["spr", "names", PLACE_POPULATION_TABLE] as const
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Fallback DDL for `place_population` when the first source predates the aux table (defensive).
|
|
88
|
+
*/
|
|
89
|
+
const PLACE_POPULATION_DDL = `CREATE TABLE ${PLACE_POPULATION_TABLE} (id INTEGER PRIMARY KEY, population INTEGER NOT NULL DEFAULT 0)`
|
|
90
|
+
|
|
91
|
+
interface PlacePopulationTable {
|
|
92
|
+
id: number
|
|
93
|
+
population: number
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Kysely schema for the build phase, including the ATTACHed `src.*` tables
|
|
98
|
+
* so row-copying queries can name the source schema.
|
|
99
|
+
*/
|
|
100
|
+
interface BuildSchema {
|
|
101
|
+
spr: SprTable
|
|
102
|
+
names: NamesTable
|
|
103
|
+
place_population: PlacePopulationTable
|
|
104
|
+
"src.spr": SprTable
|
|
105
|
+
"src.names": NamesTable
|
|
106
|
+
"src.place_population": PlacePopulationTable
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export async function buildSlimWOFDatabase(opts: BuildSlimOptions): Promise<BuildSlimResult> {
|
|
110
|
+
const countries = (opts.countries ?? ["US"]).map((c) => c.toUpperCase())
|
|
111
|
+
const topLocalities = opts.topLocalitiesPerCountry ?? 1000
|
|
112
|
+
const progress = opts.onProgress ?? (() => {})
|
|
113
|
+
|
|
114
|
+
// An empty string marks an input that isn't built yet, so empties are skipped and every remaining path must exist.
|
|
115
|
+
const inputs = opts.inputs.map((input) => input.toString()).filter((p) => p.length)
|
|
116
|
+
|
|
117
|
+
if (!inputs.length) throw new Error("no input WOF dbs provided")
|
|
118
|
+
|
|
119
|
+
for (const input of inputs) {
|
|
120
|
+
if (!(await pathExists(input))) throw new Error(`input WOF db not found: ${input}`)
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
progress("init", `${inputs.length} input(s) → ${opts.output}`)
|
|
124
|
+
|
|
125
|
+
if (await pathExists(opts.output)) {
|
|
126
|
+
await removePath(opts.output)
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// The output schema is replayed from the first input's sqlite_master so column ordering
|
|
130
|
+
// and affinity survive; `create table AS select` would flatten types to dynamic.
|
|
131
|
+
const out = new DatabaseClient<BuildSchema>(opts.output)
|
|
132
|
+
let result: BuildSlimResult
|
|
133
|
+
|
|
134
|
+
try {
|
|
135
|
+
using firstSource = new DatabaseClient<WOFDatabase>(inputs[0]!, { readOnly: true })
|
|
136
|
+
|
|
137
|
+
progress("schema", "copying spr / names / place_population schemas from first input")
|
|
138
|
+
|
|
139
|
+
for (const table of COPIED_TABLES) {
|
|
140
|
+
const createSQL = firstSource
|
|
141
|
+
.prepare(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`)
|
|
142
|
+
.get(table) as { sql?: string } | undefined
|
|
143
|
+
|
|
144
|
+
if (createSQL?.sql) {
|
|
145
|
+
// Raw DDL by design: the source DB's own create-table string cannot be expressed by a static Kysely builder.
|
|
146
|
+
out.exec(createSQL.sql)
|
|
147
|
+
} else if (table === PLACE_POPULATION_TABLE) {
|
|
148
|
+
// Older source builds may predate the aux table, so create it empty for the copy and ranking.
|
|
149
|
+
out.exec(PLACE_POPULATION_DDL)
|
|
150
|
+
} else {
|
|
151
|
+
throw new Error(`source DB ${inputs[0]} is missing required table '${table}'`)
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// The copied schemas define the primary keys.
|
|
156
|
+
// This index on `names.id` helps the per-id insert select.
|
|
157
|
+
out.exec(`CREATE INDEX IF NOT EXISTS names_id_idx ON names(id);`)
|
|
158
|
+
|
|
159
|
+
// Pull rows from each input.
|
|
160
|
+
for (const inputPath of inputs) {
|
|
161
|
+
await copyFromSource(out, out, inputPath, countries, topLocalities, progress)
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// `place_search` and `place_bbox` derive purely from `spr` + `names`, and the
|
|
165
|
+
// population aux table is not rebuilt because it was copied verbatim.
|
|
166
|
+
progress("fts", "building place_search / place_bbox on slim DB")
|
|
167
|
+
|
|
168
|
+
buildPlaceSearchFTS(out, {
|
|
169
|
+
drop: true, // schema we copied had no FTS tables, but be explicit
|
|
170
|
+
onProgress: (phase, name) => progress("fts", `${phase} ${name}`),
|
|
171
|
+
})
|
|
172
|
+
|
|
173
|
+
// `place_abbr` is materialized before `names` is dropped and always created,
|
|
174
|
+
// even empty, so the resolver can query it unconditionally.
|
|
175
|
+
progress("place_abbr", "materializing region abbreviations")
|
|
176
|
+
out.exec(`CREATE TABLE IF NOT EXISTS place_abbr (id INTEGER NOT NULL, abbr TEXT NOT NULL)`)
|
|
177
|
+
out.exec(`INSERT INTO place_abbr (id, abbr) SELECT id, name FROM names WHERE language = 'abbr'`)
|
|
178
|
+
out.exec(`CREATE INDEX IF NOT EXISTS place_abbr_by_abbr ON place_abbr (abbr COLLATE NOCASE)`)
|
|
179
|
+
out.exec(`CREATE INDEX IF NOT EXISTS place_abbr_by_id ON place_abbr (id)`)
|
|
180
|
+
|
|
181
|
+
const namesRows = countRows(out, "names")
|
|
182
|
+
|
|
183
|
+
// The resolver never reads `names` at query time, so dropping it and its index is pure size reduction.
|
|
184
|
+
if (opts.dropNames) {
|
|
185
|
+
progress("vacuum", `dropping names table (${namesRows} rows; FTS5 is self-contained)`)
|
|
186
|
+
out.exec(`DROP INDEX IF EXISTS names_id_idx;`)
|
|
187
|
+
out.exec(`DROP TABLE IF EXISTS names;`)
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
// Vacuum so the on-disk file reflects the trimmed rows rather than the in-flight insert churn.
|
|
191
|
+
progress("vacuum", "VACUUM (final size reduction)")
|
|
192
|
+
out.exec("VACUUM;")
|
|
193
|
+
|
|
194
|
+
const rowCounts = {
|
|
195
|
+
spr: countRows(out, "spr"),
|
|
196
|
+
names: namesRows,
|
|
197
|
+
placeSearch: countRows(out, PLACE_SEARCH_TABLE),
|
|
198
|
+
placeBbox: countRows(out, PLACE_BBOX_TABLE),
|
|
199
|
+
placePopulation: countRows(out, PLACE_POPULATION_TABLE),
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
progress("done", stringifyJSON(rowCounts))
|
|
203
|
+
|
|
204
|
+
result = {
|
|
205
|
+
outputPath: opts.output.toString(),
|
|
206
|
+
outputBytes: (await statPath(opts.output)).size,
|
|
207
|
+
rowCounts,
|
|
208
|
+
}
|
|
209
|
+
} finally {
|
|
210
|
+
await out.destroy()
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// The sealed-artifact invariant: a built DB is a read-only asset from the moment it exists.
|
|
214
|
+
await sealDatabase(opts.output)
|
|
215
|
+
|
|
216
|
+
return result
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
async function copyFromSource(
|
|
220
|
+
out: DatabaseClient<BuildSchema>,
|
|
221
|
+
kysely: Kysely<BuildSchema>,
|
|
222
|
+
inputPath: string,
|
|
223
|
+
countries: string[],
|
|
224
|
+
topLocalities: number,
|
|
225
|
+
progress: NonNullable<BuildSlimOptions["onProgress"]>
|
|
226
|
+
): Promise<void> {
|
|
227
|
+
// A fresh scratch copy is attached so read-only WOF distributions get the writable
|
|
228
|
+
// journal they need without mutating the canonical files.
|
|
229
|
+
await using tmpScratch = await temporaryDirectory("mailwoman-slim-src-")
|
|
230
|
+
const scratchPath = tmpScratch.path("src.db")
|
|
231
|
+
|
|
232
|
+
await copyFileTo(inputPath, scratchPath)
|
|
233
|
+
|
|
234
|
+
out.exec(`ATTACH DATABASE '${scratchPath.toString().replaceAll("'", "''")}' AS src;`)
|
|
235
|
+
|
|
236
|
+
try {
|
|
237
|
+
const srcHasPopulation = Boolean(
|
|
238
|
+
out.prepare(`SELECT 1 FROM src.sqlite_master WHERE type = 'table' AND name = '${PLACE_POPULATION_TABLE}'`).get()
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
// The `src.spr` declaration and its peers in `BuildSchema` let Kysely
|
|
242
|
+
// column-check the cross-schema select.
|
|
243
|
+
// SQLite reads the dotted identifier as a schema qualifier.
|
|
244
|
+
|
|
245
|
+
progress("country", `${inputPath}: ancestor placetypes in (${countries.join(",")})`)
|
|
246
|
+
|
|
247
|
+
await kysely
|
|
248
|
+
.insertInto("spr")
|
|
249
|
+
.expression((eb) =>
|
|
250
|
+
eb
|
|
251
|
+
.selectFrom("src.spr")
|
|
252
|
+
.selectAll()
|
|
253
|
+
.where("is_current", "!=", 0)
|
|
254
|
+
.where("is_deprecated", "=", 0)
|
|
255
|
+
.where("country", "in", countries)
|
|
256
|
+
.where("placetype", "in", [...ANCESTOR_PLACETYPES])
|
|
257
|
+
)
|
|
258
|
+
.onConflict((oc) => oc.doNothing())
|
|
259
|
+
.execute()
|
|
260
|
+
|
|
261
|
+
// Localities without a population row still qualify.
|
|
262
|
+
// Without the aux table, fall back to a deterministic id ordering.
|
|
263
|
+
progress("locality", `${inputPath}: top-${topLocalities} localities by population`)
|
|
264
|
+
|
|
265
|
+
await kysely
|
|
266
|
+
.insertInto("spr")
|
|
267
|
+
.expression((eb) =>
|
|
268
|
+
eb
|
|
269
|
+
.selectFrom("src.spr as s")
|
|
270
|
+
.$if(srcHasPopulation, (qb) => qb.leftJoin("src.place_population as p", "p.id", "s.id"))
|
|
271
|
+
.selectAll("s")
|
|
272
|
+
.where("s.is_current", "!=", 0)
|
|
273
|
+
.where("s.is_deprecated", "=", 0)
|
|
274
|
+
.where("s.country", "in", countries)
|
|
275
|
+
.where("s.placetype", "=", "locality")
|
|
276
|
+
.orderBy(srcHasPopulation ? sql<number>`COALESCE(p.population, 0)` : sql<number>`s.id`, "desc")
|
|
277
|
+
.limit(topLocalities)
|
|
278
|
+
)
|
|
279
|
+
.onConflict((oc) => oc.doNothing())
|
|
280
|
+
.execute()
|
|
281
|
+
|
|
282
|
+
progress("postcode", `${inputPath}: all postcodes`)
|
|
283
|
+
|
|
284
|
+
await kysely
|
|
285
|
+
.insertInto("spr")
|
|
286
|
+
.expression((eb) =>
|
|
287
|
+
eb
|
|
288
|
+
.selectFrom("src.spr")
|
|
289
|
+
.selectAll()
|
|
290
|
+
.where("is_current", "!=", 0)
|
|
291
|
+
.where("is_deprecated", "=", 0)
|
|
292
|
+
.where("country", "in", countries)
|
|
293
|
+
.where("placetype", "=", "postalcode")
|
|
294
|
+
)
|
|
295
|
+
.onConflict((oc) => oc.doNothing())
|
|
296
|
+
.execute()
|
|
297
|
+
|
|
298
|
+
progress("names", `${inputPath}: names rows for selected IDs`)
|
|
299
|
+
|
|
300
|
+
await kysely
|
|
301
|
+
.insertInto("names")
|
|
302
|
+
.expression((eb) => eb.selectFrom("src.names").selectAll().where("id", "in", eb.selectFrom("spr").select("id")))
|
|
303
|
+
.onConflict((oc) => oc.doNothing())
|
|
304
|
+
.execute()
|
|
305
|
+
|
|
306
|
+
if (srcHasPopulation) {
|
|
307
|
+
progress("place_population", `${inputPath}: population rows for selected IDs`)
|
|
308
|
+
|
|
309
|
+
await kysely
|
|
310
|
+
.insertInto("place_population")
|
|
311
|
+
.expression((eb) =>
|
|
312
|
+
eb.selectFrom("src.place_population").selectAll().where("id", "in", eb.selectFrom("spr").select("id"))
|
|
313
|
+
)
|
|
314
|
+
.onConflict((oc) => oc.doNothing())
|
|
315
|
+
.execute()
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
// `ancestors` is intentionally not copied, so the derived `coincident_roles`
|
|
319
|
+
// table is copied and filtered to surviving spr ids.
|
|
320
|
+
const relationSchema = out
|
|
321
|
+
.prepare(`SELECT sql FROM src.sqlite_master WHERE type = 'table' AND name = 'coincident_roles'`)
|
|
322
|
+
.get() as { sql?: string } | undefined
|
|
323
|
+
|
|
324
|
+
if (relationSchema?.sql) {
|
|
325
|
+
progress("coincident_roles", `${inputPath}: copying dual-role relation`)
|
|
326
|
+
out.exec(relationSchema.sql.replace(/CREATE TABLE/i, "CREATE TABLE IF NOT EXISTS"))
|
|
327
|
+
|
|
328
|
+
out.exec(
|
|
329
|
+
`INSERT OR IGNORE INTO coincident_roles SELECT * FROM src.coincident_roles
|
|
330
|
+
WHERE admin_id IN (SELECT id FROM spr) AND locality_id IN (SELECT id FROM spr)`
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
out.exec(`CREATE INDEX IF NOT EXISTS coincident_roles_by_admin ON coincident_roles (admin_id)`)
|
|
334
|
+
}
|
|
335
|
+
} finally {
|
|
336
|
+
out.exec(`DETACH DATABASE src;`)
|
|
337
|
+
}
|
|
338
|
+
}
|
|
@@ -2,55 +2,115 @@
|
|
|
2
2
|
* @copyright Sister Software
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
|
-
* @file Pass 2 of the candidate build
|
|
5
|
+
* @file Pass 2 of the candidate build: explode `place_search.alt_names` into distinct alias rows.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import type { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
9
|
+
import { tableExists } from "@mailwoman/sqlite/introspection"
|
|
9
10
|
|
|
10
|
-
import type { CandidateDatabase } from "#candidate-schema"
|
|
11
11
|
import type { PlaceAttrs, StageRow } from "#candidate/place-attrs"
|
|
12
|
-
import {
|
|
12
|
+
import type { CandidateDatabase } from "#candidate/schema"
|
|
13
|
+
import { ALIAS_SEPARATOR } from "#fts"
|
|
13
14
|
import type { WOFDatabase } from "#schema"
|
|
14
15
|
import { normalizeLocalityForKey } from "#street/normalize"
|
|
15
16
|
|
|
16
17
|
/**
|
|
17
|
-
*
|
|
18
|
-
*
|
|
18
|
+
* The official name of every current region, keyed `${country id}\0${name key}` →
|
|
19
|
+
* region id, from the source `names` table's `official` bit.
|
|
20
|
+
*
|
|
21
|
+
* Empty when the source has no `names` table (a fixture, or an extract shape).
|
|
22
|
+
*
|
|
23
|
+
* The positive evidence behind the alias refusal below: a name is refused from a region's
|
|
24
|
+
* bag only when another region of the same country holds it as its official name.
|
|
25
|
+
*/
|
|
26
|
+
function officialNameHoldersByRegion(
|
|
27
|
+
src: DatabaseClient<WOFDatabase>,
|
|
28
|
+
ccID: (code: string | null) => number
|
|
29
|
+
): Map<string, number> {
|
|
30
|
+
const keys = new Map<string, number>()
|
|
31
|
+
|
|
32
|
+
if (!tableExists(src, "names")) return keys
|
|
33
|
+
|
|
34
|
+
for (const r of src
|
|
35
|
+
.prepare(
|
|
36
|
+
`SELECT n.id AS id, n.name AS name, s.country AS country
|
|
37
|
+
FROM names n JOIN spr s ON s.id = n.id
|
|
38
|
+
WHERE s.placetype = 'region' AND s.is_current != 0 AND s.is_deprecated = 0 AND n.official = 1`
|
|
39
|
+
)
|
|
40
|
+
.iterate()) {
|
|
41
|
+
const k = normalizeLocalityForKey(String(r.name ?? ""))
|
|
42
|
+
|
|
43
|
+
if (k) {
|
|
44
|
+
keys.set(`${ccID(r.country as string | null)}\0${k}`, Number(r.id))
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
return keys
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Pass 2: explode each place's `place_search.alt_names` bag into distinct-key alias rows
|
|
53
|
+
* (`is_primary = 0`), and count each place's distinct staged keys with the primary included.
|
|
54
|
+
*
|
|
55
|
+
* With `regionPlacetypeID` and `ccID` given, a region's alias that is another same-country region's
|
|
56
|
+
* official name is refused and counted rather than staged (see {@link officialNameHoldersByRegion}).
|
|
57
|
+
* Without them the pass stages every alias.
|
|
19
58
|
*/
|
|
20
59
|
export function explodeAliasBags(
|
|
21
60
|
src: DatabaseClient<WOFDatabase>,
|
|
22
61
|
out: DatabaseClient<CandidateDatabase>,
|
|
23
62
|
attrs: Map<number, PlaceAttrs>,
|
|
24
|
-
stageRow: StageRow
|
|
25
|
-
|
|
63
|
+
stageRow: StageRow,
|
|
64
|
+
opts: { regionPlacetypeID?: number; ccID?: (code: string | null) => number } = {}
|
|
65
|
+
): { nAlias: number; keyCounts: Map<number, number>; regionOfficialRefused: number } {
|
|
26
66
|
let nAlias = 0
|
|
67
|
+
let regionOfficialRefused = 0
|
|
27
68
|
const keyCounts = new Map<number, number>()
|
|
69
|
+
|
|
70
|
+
const official =
|
|
71
|
+
typeof opts.regionPlacetypeID === "number" && opts.ccID
|
|
72
|
+
? officialNameHoldersByRegion(src, opts.ccID)
|
|
73
|
+
: new Map<string, number>()
|
|
74
|
+
|
|
28
75
|
out.exec("BEGIN")
|
|
29
76
|
|
|
30
77
|
for (const r of src.prepare("SELECT wof_id, alt_names FROM place_search").iterate()) {
|
|
31
|
-
const
|
|
78
|
+
const id = Number(r.wof_id)
|
|
79
|
+
const a = attrs.get(id)
|
|
32
80
|
const alt = r.alt_names as string | null
|
|
33
81
|
|
|
34
82
|
if (!a || !alt) continue
|
|
35
83
|
const seen = new Set<string>([a.pkey])
|
|
84
|
+
const isRegion = a.ptid === opts.regionPlacetypeID
|
|
36
85
|
|
|
37
|
-
// The writer space-pads each separator and appends a trailing one, so every piece
|
|
38
|
-
// surrounding whitespace and the last one is empty.
|
|
39
|
-
//
|
|
86
|
+
// The writer space-pads each separator and appends a trailing one, so every piece
|
|
87
|
+
// includes surrounding whitespace and the last one is empty.
|
|
88
|
+
// `normalizeLocalityForKey` folds both away.
|
|
40
89
|
for (const piece of alt.split(ALIAS_SEPARATOR)) {
|
|
41
90
|
const k = normalizeLocalityForKey(piece)
|
|
42
91
|
|
|
43
92
|
if (!k || seen.has(k)) continue
|
|
44
93
|
seen.add(k)
|
|
45
|
-
|
|
94
|
+
|
|
95
|
+
if (isRegion) {
|
|
96
|
+
const holder = official.get(`${a.cid}\0${k}`)
|
|
97
|
+
|
|
98
|
+
if (typeof holder === "number" && holder !== id) {
|
|
99
|
+
regionOfficialRefused++
|
|
100
|
+
|
|
101
|
+
continue
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
stageRow(k, a, id, 0)
|
|
46
106
|
|
|
47
107
|
nAlias++
|
|
48
108
|
}
|
|
49
109
|
|
|
50
|
-
keyCounts.set(
|
|
110
|
+
keyCounts.set(id, seen.size)
|
|
51
111
|
}
|
|
52
112
|
|
|
53
113
|
out.exec("COMMIT")
|
|
54
114
|
|
|
55
|
-
return { nAlias, keyCounts }
|
|
115
|
+
return { nAlias, keyCounts, regionOfficialRefused }
|
|
56
116
|
}
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Typed schema for the candidate gazetteer's ancestors sidecar (`candidate_ancestor` +
|
|
7
|
+
* `candidate_interval`) — the containment lineage the candidate table itself cannot answer, built
|
|
8
|
+
* into the same `candidate.db` by `build-candidate.ts` and read by
|
|
9
|
+
* {@link WOFCandidateTableLookup.ancestors}.
|
|
10
|
+
*
|
|
11
|
+
* The sidecar stores denormalized closure rows. Each row represents one (place, ancestor) edge. The row includes the
|
|
12
|
+
* parent's placetype, display name and folded key. The two consumers determine this layout:
|
|
13
|
+
*
|
|
14
|
+
* 1. The admin-coherence check needs the winner's chain as (placetype, name) pairs in one probe.
|
|
15
|
+
* A fixed-slot `[id.8]` chain returns ids. Each id would then need a name lookup, but the artifact has no
|
|
16
|
+
* per-id table. That design requires up to 8 indirections. The closure row requires none.
|
|
17
|
+
* 2. The account layer needs every candidate under a `name_key` enumerable with its chain from
|
|
18
|
+
* one artifact probe ("present-but-outranked, discriminated by containment"). That is the
|
|
19
|
+
* candidate probe (contiguous) followed by one `spr_id`-clustered closure probe per candidate
|
|
20
|
+
* — each a handful of adjacent pages.
|
|
21
|
+
*
|
|
22
|
+
* The artifact is read over HTTP byte ranges. A join to a dimension table scatters page fetches. A `without rowid`
|
|
23
|
+
* B-tree clustered on `(spr_id, depth)` keeps a whole chain in 1–2 pages. The builder repeats parent strings to
|
|
24
|
+
* avoid joins during reads.
|
|
25
|
+
*
|
|
26
|
+
* `depth` is 1 for the nearest ancestor (deepest containment tier) and increases outward to the
|
|
27
|
+
* country. This matches the nearest-first order that `ancestry.ts` serves for the FTS backend. Both backends'
|
|
28
|
+
* `ancestors()` results therefore use the same order. The order within a place is deterministic:
|
|
29
|
+
* containment depth descending (`placetypeDepth`), then ancestor id ascending, capped at
|
|
30
|
+
* {@link MAX_ANCESTOR_DEPTH}. `parent_name_key` is the shared {@link normalizeLocalityForKey}
|
|
31
|
+
* fold — the same fold `candidate.name_key` is built with, so a chain entry and a candidate key
|
|
32
|
+
* compare under one normalizer.
|
|
33
|
+
*
|
|
34
|
+
* `candidate_interval` stores pre/post-order labels over the canonical-parent forest. The builder assigns them
|
|
35
|
+
* during construction. The predicate `a.pre <= d.pre and d.post <= a.post` tests whether `a` contains `d` in O(1)
|
|
36
|
+
* time. It needs no chain scan or tier information. Descendants of `a` occupy the contiguous range
|
|
37
|
+
* `pre between a.pre and a.post`. Interval labels add relabeling work to updates. This database is sealed and
|
|
38
|
+
* read-only, so the builder can regenerate all labels together.
|
|
39
|
+
*
|
|
40
|
+
* WOF places can have multiple parents because boundaries can be ambiguous or participate in multiple
|
|
41
|
+
* hierarchies. `candidate_ancestor` stores every parent. Its closure rows hold the complete containment record.
|
|
42
|
+
* One interval pair encodes only a tree. The interval forest therefore links each place to one canonical parent:
|
|
43
|
+
* the depth-1 edge with the finest containment tier and lowest ancestor id. This follows the same MIN-based
|
|
44
|
+
* stability convention as the candidate table's `region_id` stamp. A containment query about another hierarchy
|
|
45
|
+
* must consult the closure rows. The interval result answers only whether a place is contained along the canonical
|
|
46
|
+
* hierarchy.
|
|
47
|
+
*
|
|
48
|
+
* absence semantics (meaning-of-zero): a place with no `candidate_interval` row has no recorded
|
|
49
|
+
* ancestry in the source (extract-fed postcodes and localities, isolated places, cycle-skipped
|
|
50
|
+
* rows). Absence is unverifiable, never a containment verdict.
|
|
51
|
+
*/
|
|
52
|
+
|
|
53
|
+
import { sql, type Kysely } from "kysely"
|
|
54
|
+
|
|
55
|
+
// Type-only and circular on purpose (candidate-schema extends CandidateAncestorsDatabase): Kysely's DB
|
|
56
|
+
// parameter is invariant, so the DDL functions must take the full database type their caller holds.
|
|
57
|
+
// Erased at runtime.
|
|
58
|
+
import type { CandidateDatabase } from "#candidate/schema"
|
|
59
|
+
import type { NameKey } from "#street/normalize"
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* The deepest chain the sidecar stores per place.
|
|
63
|
+
*
|
|
64
|
+
* WOF containment within the resolvable placetypes (country … microhood) never legitimately exceeds this.
|
|
65
|
+
* The build drops and counts anything past it as source noise rather than storing it.
|
|
66
|
+
*/
|
|
67
|
+
export const MAX_ANCESTOR_DEPTH = 8
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* The closure-row table's name — probed with `hasTable` by the reader, so an artifact
|
|
71
|
+
* predating the sidecar degrades to "no ancestors capability" rather than `no such table`.
|
|
72
|
+
*/
|
|
73
|
+
export const CANDIDATE_ANCESTOR_TABLE = "candidate_ancestor"
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* The interval-label table's name — the closure table's seal-time sibling.
|
|
77
|
+
* Existence-restricted the same way.
|
|
78
|
+
*/
|
|
79
|
+
export const CANDIDATE_INTERVAL_TABLE = "candidate_interval"
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* One (place, ancestor) edge, denormalized so a chain read is a single clustered probe (no join).
|
|
83
|
+
*/
|
|
84
|
+
export interface CandidateAncestorTable {
|
|
85
|
+
/**
|
|
86
|
+
* WOF id of the place whose chain this row belongs to — the same `spr_id` the candidate table resolves to.
|
|
87
|
+
*/
|
|
88
|
+
spr_id: number
|
|
89
|
+
/**
|
|
90
|
+
* 1 = nearest ancestor, increasing outward to the country. Deterministic within a place (containment depth
|
|
91
|
+
* descending, then ancestor id ascending), so `(spr_id, depth)` is a stable primary key across rebuilds.
|
|
92
|
+
*/
|
|
93
|
+
depth: number
|
|
94
|
+
/**
|
|
95
|
+
* WOF id of the ancestor.
|
|
96
|
+
*/
|
|
97
|
+
parent_spr_id: number
|
|
98
|
+
/**
|
|
99
|
+
* Small int from the shared `placetype_codes` dictionary (the same one the candidate table uses).
|
|
100
|
+
*/
|
|
101
|
+
parent_placetype_id: number
|
|
102
|
+
/**
|
|
103
|
+
* The ancestor's canonical display name.
|
|
104
|
+
*
|
|
105
|
+
* What {@link Ancestor.name} serves, matching the FTS backend's register.
|
|
106
|
+
*/
|
|
107
|
+
parent_name: string
|
|
108
|
+
/**
|
|
109
|
+
* The shared {@link normalizeLocalityForKey} fold of `parent_name` — comparable against
|
|
110
|
+
* `candidate.name_key` and against a query-side fold under one normalizer, by construction.
|
|
111
|
+
*/
|
|
112
|
+
parent_name_key: NameKey
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Pre/post-order labels over the canonical-parent forest — one row per place with
|
|
117
|
+
* recorded ancestry (see the module docstring for absence semantics).
|
|
118
|
+
*
|
|
119
|
+
* `pre < post` always.
|
|
120
|
+
* Labels are unique across the artifact.
|
|
121
|
+
*/
|
|
122
|
+
export interface CandidateIntervalTable {
|
|
123
|
+
spr_id: number
|
|
124
|
+
pre: number
|
|
125
|
+
post: number
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* The sidecar tables, for a `Kysely` view over the candidate DB.
|
|
130
|
+
*
|
|
131
|
+
* `CandidateDatabase` (candidate-schema.ts) extends this, so the builder's
|
|
132
|
+
* one typed client sees both families.
|
|
133
|
+
*/
|
|
134
|
+
export interface CandidateAncestorsDatabase {
|
|
135
|
+
candidate_ancestor: CandidateAncestorTable
|
|
136
|
+
candidate_interval: CandidateIntervalTable
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* The `candidate_ancestor` columns in clustered-key order.
|
|
141
|
+
*
|
|
142
|
+
* The first two columns form the primary key.
|
|
143
|
+
* The builder's positional `insert` binds values in this order.
|
|
144
|
+
*
|
|
145
|
+
* Keep in sync with {@link CandidateAncestorTable}.
|
|
146
|
+
*/
|
|
147
|
+
export const CANDIDATE_ANCESTOR_COLUMNS = [
|
|
148
|
+
"spr_id",
|
|
149
|
+
"depth",
|
|
150
|
+
"parent_spr_id",
|
|
151
|
+
"parent_placetype_id",
|
|
152
|
+
"parent_name",
|
|
153
|
+
"parent_name_key",
|
|
154
|
+
] as const
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Create the clustered closure table.
|
|
158
|
+
*
|
|
159
|
+
* The builder inserts in `(spr_id, depth)` order so the B-tree leaves are contiguous
|
|
160
|
+
* per place — the byte-range read discipline.
|
|
161
|
+
*/
|
|
162
|
+
export async function createCandidateAncestorTable(db: Kysely<CandidateDatabase>): Promise<void> {
|
|
163
|
+
await db.schema
|
|
164
|
+
.createTable(CANDIDATE_ANCESTOR_TABLE)
|
|
165
|
+
.addColumn("spr_id", "integer", (c) => c.notNull())
|
|
166
|
+
.addColumn("depth", "integer", (c) => c.notNull())
|
|
167
|
+
.addColumn("parent_spr_id", "integer", (c) => c.notNull())
|
|
168
|
+
.addColumn("parent_placetype_id", "integer", (c) => c.notNull())
|
|
169
|
+
.addColumn("parent_name", "text", (c) => c.notNull())
|
|
170
|
+
.addColumn("parent_name_key", "text", (c) => c.notNull())
|
|
171
|
+
.addPrimaryKeyConstraint("candidate_ancestor_pk", ["spr_id", "depth"])
|
|
172
|
+
// `without rowid` has no first-class builder. The raw modifier is the idiomatic fallback.
|
|
173
|
+
.modifyEnd(sql`without rowid`)
|
|
174
|
+
.execute()
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Create the interval-label table.
|
|
179
|
+
*
|
|
180
|
+
* Small rows probed by primary key — the `without rowid` win case.
|
|
181
|
+
*/
|
|
182
|
+
export async function createCandidateIntervalTable(db: Kysely<CandidateDatabase>): Promise<void> {
|
|
183
|
+
await db.schema
|
|
184
|
+
.createTable(CANDIDATE_INTERVAL_TABLE)
|
|
185
|
+
.addColumn("spr_id", "integer", (c) => c.primaryKey())
|
|
186
|
+
.addColumn("pre", "integer", (c) => c.notNull())
|
|
187
|
+
.addColumn("post", "integer", (c) => c.notNull())
|
|
188
|
+
.modifyEnd(sql`without rowid`)
|
|
189
|
+
.execute()
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* One place's interval label.
|
|
194
|
+
* The shape both sides of a containment comparison read.
|
|
195
|
+
*/
|
|
196
|
+
export interface IntervalLabel {
|
|
197
|
+
pre: number
|
|
198
|
+
post: number
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* Does `outer` contain `inner` along the canonical hierarchy?
|
|
203
|
+
*
|
|
204
|
+
* Reflexive: a place contains itself (containment degenerates to identity,
|
|
205
|
+
* the same reading the admin-coherence check gives a self-match).
|
|
206
|
+
* The shared function for every consumer of the labels — a re-derived comparison risks
|
|
207
|
+
* disagreeing at exactly the strict/inclusive boundary this line settles.
|
|
208
|
+
*/
|
|
209
|
+
export function intervalContains(outer: IntervalLabel, inner: IntervalLabel): boolean {
|
|
210
|
+
return outer.pre <= inner.pre && inner.post <= outer.post
|
|
211
|
+
}
|