@mailwoman/resolver-wof-sqlite 9.4.0 → 10.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/lib/address/{point-interpolation.ts → point/interpolation.ts} +40 -38
- package/lib/address/point/schema.ts +132 -0
- package/lib/address/point.ts +68 -81
- package/lib/{address/index.ts → address.ts} +2 -2
- package/lib/ancestry/backfill.ts +24 -88
- package/lib/{ancestry/index.ts → ancestry.ts} +19 -15
- package/lib/build/candidate.ts +608 -0
- package/lib/build/population-corrections.ts +64 -0
- package/lib/build/slim.ts +338 -0
- package/lib/candidate/alias-bags.ts +74 -14
- package/lib/candidate/ancestors/schema.ts +211 -0
- package/lib/candidate/{ancestors-sidecar.ts → ancestors/sidecar.ts} +17 -24
- package/lib/candidate/country-display-names.ts +6 -20
- package/lib/candidate/extract-fold.ts +12 -57
- package/lib/candidate/fts.ts +28 -0
- package/lib/candidate/importance.ts +221 -0
- package/lib/candidate/lookup.ts +669 -0
- package/lib/candidate/name-roles.ts +16 -34
- package/lib/candidate/own-name.ts +34 -31
- package/lib/candidate/place-attrs.ts +20 -13
- package/lib/candidate/schema.ts +211 -0
- package/lib/candidate/scoring.ts +222 -0
- package/lib/capital-schema.ts +21 -17
- package/lib/capitals.ts +26 -24
- package/lib/coincident-roles.ts +74 -55
- package/lib/convention/schema.ts +13 -7
- package/lib/convention.ts +144 -0
- package/lib/coverage-manifest-schema.ts +34 -39
- package/lib/currency-backfill.ts +24 -69
- package/lib/env.ts +1 -1
- package/lib/exact-match.ts +31 -20
- package/lib/extracts.ts +96 -73
- package/lib/fst/autocomplete.ts +52 -35
- package/lib/fst/builder.ts +30 -36
- package/lib/fst/deserialize-web.ts +8 -9
- package/lib/fst/format.ts +41 -23
- package/lib/fst/freshness.ts +17 -95
- package/lib/fst/matcher.ts +2 -2
- package/lib/fst/serialize.ts +45 -60
- package/lib/fst/types.ts +71 -59
- package/lib/fts/query.ts +5 -36
- package/lib/fts.ts +280 -0
- package/lib/geonames/aliases.ts +72 -149
- package/lib/geonames/postal.ts +162 -77
- package/lib/index.ts +20 -27
- package/lib/interpolation.ts +43 -96
- package/lib/lookup.ts +137 -257
- package/lib/name-score.ts +9 -7
- package/lib/nsul/lookup.ts +36 -53
- package/lib/nsul/schema.ts +16 -43
- package/lib/paths.ts +159 -0
- package/lib/place-importance-schema.ts +57 -164
- package/lib/poi/lookup.ts +100 -109
- package/lib/poi/schema.ts +42 -26
- package/lib/polygon-schema.ts +10 -6
- package/lib/postal/city/alias/lookup.ts +79 -0
- package/lib/postal/{city-alias-schema.ts → city/alias/schema.ts} +14 -10
- package/lib/postal/{city-candidate-schema.ts → city/candidate-schema.ts} +26 -20
- package/lib/postal.ts +9 -0
- package/lib/postcode-point-lookup.ts +9 -19
- package/lib/primary-preference.ts +45 -125
- package/lib/proximity-rerank.ts +43 -32
- package/lib/ranking-weights.ts +98 -76
- package/lib/region-keys.ts +68 -40
- package/lib/reverse.ts +108 -72
- package/lib/schema.ts +74 -42
- package/lib/search-fetch.ts +90 -68
- package/lib/sqlite/convention-source.ts +57 -0
- package/lib/sqlite/utils.ts +79 -0
- package/lib/street/centroid/schema.ts +149 -0
- package/lib/street/centroid.ts +16 -25
- package/lib/street/{morphology-fst-builder.ts → morphology/fst/builder.ts} +56 -41
- package/lib/street/{morphology-fst-loader.ts → morphology/fst/loader.ts} +36 -29
- package/lib/street/name-lookup.ts +43 -35
- package/lib/street/normalize.ts +85 -216
- package/lib/street/segment-schema.ts +116 -43
- package/lib/{street/index.ts → street.ts} +3 -3
- package/lib/types.ts +94 -140
- package/lib/unified-schema.ts +29 -29
- package/lib/uprn/existence.ts +30 -19
- package/lib/uprn/lookup.ts +28 -67
- package/lib/uprn/schema.ts +18 -40
- package/lib/weights-overlay-linker.ts +204 -232
- package/out/address/point/interpolation.d.ts +45 -0
- package/out/address/point/interpolation.d.ts.map +1 -0
- package/out/address/{point-interpolation.js → point/interpolation.js} +38 -38
- package/out/address/point/interpolation.js.map +1 -0
- package/out/address/point/schema.d.ts +74 -0
- package/out/address/point/schema.d.ts.map +1 -0
- package/out/address/{point-schema.js → point/schema.js} +4 -20
- package/out/address/point/schema.js.map +1 -0
- package/out/address/point.d.ts +24 -21
- package/out/address/point.d.ts.map +1 -1
- package/out/address/point.js +60 -77
- package/out/address/point.js.map +1 -1
- package/out/{address/index.js → address.d.ts} +3 -3
- package/out/address.d.ts.map +1 -0
- package/out/{address/index.d.ts → address.js} +3 -3
- package/out/address.js.map +1 -0
- package/out/ancestry/backfill.d.ts +11 -64
- package/out/ancestry/backfill.d.ts.map +1 -1
- package/out/ancestry/backfill.js +18 -76
- package/out/ancestry/backfill.js.map +1 -1
- package/out/ancestry.d.ts +49 -0
- package/out/ancestry.d.ts.map +1 -0
- package/out/ancestry.js +59 -0
- package/out/ancestry.js.map +1 -0
- package/out/build/candidate.d.ts +170 -0
- package/out/build/candidate.d.ts.map +1 -0
- package/out/{build-candidate.js → build/candidate.js} +33 -102
- package/out/build/candidate.js.map +1 -0
- package/out/build/population-corrections.d.ts +52 -0
- package/out/build/population-corrections.d.ts.map +1 -0
- package/out/build/population-corrections.js +40 -0
- package/out/build/population-corrections.js.map +1 -0
- package/out/build/slim.d.ts +44 -0
- package/out/build/slim.d.ts.map +1 -0
- package/out/{build-slim.js → build/slim.js} +33 -94
- package/out/build/slim.js.map +1 -0
- package/out/candidate/alias-bags.d.ts +13 -5
- package/out/candidate/alias-bags.d.ts.map +1 -1
- package/out/candidate/alias-bags.js +55 -12
- package/out/candidate/alias-bags.js.map +1 -1
- package/out/candidate/ancestors/schema.d.ts +166 -0
- package/out/candidate/ancestors/schema.d.ts.map +1 -0
- package/out/candidate/ancestors/schema.js +131 -0
- package/out/candidate/ancestors/schema.js.map +1 -0
- package/out/candidate/ancestors/sidecar.d.ts +27 -0
- package/out/candidate/ancestors/sidecar.d.ts.map +1 -0
- package/out/candidate/{ancestors-sidecar.js → ancestors/sidecar.js} +17 -24
- package/out/candidate/ancestors/sidecar.js.map +1 -0
- package/out/candidate/country-display-names.d.ts +5 -15
- package/out/candidate/country-display-names.d.ts.map +1 -1
- package/out/candidate/country-display-names.js +6 -20
- package/out/candidate/country-display-names.js.map +1 -1
- package/out/candidate/extract-fold.d.ts +8 -16
- package/out/candidate/extract-fold.d.ts.map +1 -1
- package/out/candidate/extract-fold.js +6 -48
- package/out/candidate/extract-fold.js.map +1 -1
- package/out/candidate/fts.d.ts +19 -0
- package/out/candidate/fts.d.ts.map +1 -0
- package/out/candidate/fts.js +22 -0
- package/out/candidate/fts.js.map +1 -0
- package/out/candidate/importance.d.ts +127 -0
- package/out/candidate/importance.d.ts.map +1 -0
- package/out/candidate/importance.js +162 -0
- package/out/candidate/importance.js.map +1 -0
- package/out/candidate/lookup.d.ts +36 -0
- package/out/candidate/lookup.d.ts.map +1 -0
- package/out/candidate/lookup.js +456 -0
- package/out/candidate/lookup.js.map +1 -0
- package/out/candidate/name-roles.d.ts +10 -22
- package/out/candidate/name-roles.d.ts.map +1 -1
- package/out/candidate/name-roles.js +15 -33
- package/out/candidate/name-roles.js.map +1 -1
- package/out/candidate/own-name.d.ts +23 -25
- package/out/candidate/own-name.d.ts.map +1 -1
- package/out/candidate/own-name.js +34 -31
- package/out/candidate/own-name.js.map +1 -1
- package/out/candidate/place-attrs.d.ts +20 -13
- package/out/candidate/place-attrs.d.ts.map +1 -1
- package/out/candidate/place-attrs.js +4 -5
- package/out/candidate/place-attrs.js.map +1 -1
- package/out/candidate/schema.d.ts +120 -0
- package/out/candidate/schema.d.ts.map +1 -0
- package/out/{candidate-schema.js → candidate/schema.js} +11 -23
- package/out/candidate/schema.js.map +1 -0
- package/out/{candidate-scoring.d.ts → candidate/scoring.d.ts} +8 -9
- package/out/candidate/scoring.d.ts.map +1 -0
- package/out/candidate/scoring.js +157 -0
- package/out/candidate/scoring.js.map +1 -0
- package/out/capital-schema.d.ts +21 -17
- package/out/capital-schema.d.ts.map +1 -1
- package/out/capital-schema.js +17 -15
- package/out/capital-schema.js.map +1 -1
- package/out/capitals.d.ts +26 -24
- package/out/capitals.d.ts.map +1 -1
- package/out/capitals.js +23 -21
- package/out/capitals.js.map +1 -1
- package/out/coincident-roles.d.ts +60 -43
- package/out/coincident-roles.d.ts.map +1 -1
- package/out/coincident-roles.js +61 -50
- package/out/coincident-roles.js.map +1 -1
- package/out/convention/schema.d.ts +12 -6
- package/out/convention/schema.d.ts.map +1 -1
- package/out/convention/schema.js +6 -4
- package/out/convention/schema.js.map +1 -1
- package/out/convention.d.ts +97 -0
- package/out/convention.d.ts.map +1 -0
- package/out/convention.js +81 -0
- package/out/convention.js.map +1 -0
- package/out/coverage-manifest-schema.d.ts +33 -38
- package/out/coverage-manifest-schema.d.ts.map +1 -1
- package/out/coverage-manifest-schema.js +20 -29
- package/out/coverage-manifest-schema.js.map +1 -1
- package/out/currency-backfill.d.ts +13 -35
- package/out/currency-backfill.d.ts.map +1 -1
- package/out/currency-backfill.js +12 -39
- package/out/currency-backfill.js.map +1 -1
- package/out/env.d.ts +2 -2
- package/out/env.js +1 -1
- package/out/exact-match.d.ts +19 -10
- package/out/exact-match.d.ts.map +1 -1
- package/out/exact-match.js +31 -20
- package/out/exact-match.js.map +1 -1
- package/out/extracts.d.ts +65 -54
- package/out/extracts.d.ts.map +1 -1
- package/out/extracts.js +63 -55
- package/out/extracts.js.map +1 -1
- package/out/fst/autocomplete.d.ts +28 -20
- package/out/fst/autocomplete.d.ts.map +1 -1
- package/out/fst/autocomplete.js +38 -27
- package/out/fst/autocomplete.js.map +1 -1
- package/out/fst/builder.d.ts.map +1 -1
- package/out/fst/builder.js +30 -36
- package/out/fst/builder.js.map +1 -1
- package/out/fst/deserialize-web.d.ts +3 -3
- package/out/fst/deserialize-web.d.ts.map +1 -1
- package/out/fst/deserialize-web.js +8 -9
- package/out/fst/deserialize-web.js.map +1 -1
- package/out/fst/format.d.ts +41 -23
- package/out/fst/format.d.ts.map +1 -1
- package/out/fst/format.js +41 -23
- package/out/fst/format.js.map +1 -1
- package/out/fst/freshness.d.ts +11 -73
- package/out/fst/freshness.d.ts.map +1 -1
- package/out/fst/freshness.js +12 -76
- package/out/fst/freshness.js.map +1 -1
- package/out/fst/matcher.d.ts +1 -1
- package/out/fst/matcher.d.ts.map +1 -1
- package/out/fst/matcher.js +2 -2
- package/out/fst/matcher.js.map +1 -1
- package/out/fst/serialize.d.ts +30 -26
- package/out/fst/serialize.d.ts.map +1 -1
- package/out/fst/serialize.js +44 -58
- package/out/fst/serialize.js.map +1 -1
- package/out/fst/types.d.ts +71 -59
- package/out/fst/types.d.ts.map +1 -1
- package/out/fst/types.js +1 -4
- package/out/fst/types.js.map +1 -1
- package/out/{fst/index.js → fst.d.ts} +1 -1
- package/out/fst.d.ts.map +1 -0
- package/out/{fst/index.d.ts → fst.js} +1 -1
- package/out/fst.js.map +1 -0
- package/out/fts/query.d.ts +2 -27
- package/out/fts/query.d.ts.map +1 -1
- package/out/fts/query.js +5 -36
- package/out/fts/query.js.map +1 -1
- package/out/fts.d.ts +126 -0
- package/out/fts.d.ts.map +1 -0
- package/out/fts.js +188 -0
- package/out/fts.js.map +1 -0
- package/out/geonames/aliases.d.ts +26 -67
- package/out/geonames/aliases.d.ts.map +1 -1
- package/out/geonames/aliases.js +44 -121
- package/out/geonames/aliases.js.map +1 -1
- package/out/geonames/postal.d.ts +84 -50
- package/out/geonames/postal.d.ts.map +1 -1
- package/out/geonames/postal.js +115 -78
- package/out/geonames/postal.js.map +1 -1
- package/out/{geonames/index.js → geonames.d.ts} +1 -1
- package/out/geonames.d.ts.map +1 -0
- package/out/{geonames/index.d.ts → geonames.js} +1 -1
- package/out/geonames.js.map +1 -0
- package/out/index.d.ts +20 -21
- package/out/index.d.ts.map +1 -1
- package/out/index.js +14 -15
- package/out/index.js.map +1 -1
- package/out/interpolation.d.ts +15 -52
- package/out/interpolation.d.ts.map +1 -1
- package/out/interpolation.js +25 -68
- package/out/interpolation.js.map +1 -1
- package/out/lookup.d.ts +24 -53
- package/out/lookup.d.ts.map +1 -1
- package/out/lookup.js +117 -223
- package/out/lookup.js.map +1 -1
- package/out/name-score.d.ts +2 -2
- package/out/name-score.d.ts.map +1 -1
- package/out/name-score.js +9 -7
- package/out/name-score.js.map +1 -1
- package/out/nsul/lookup.d.ts +33 -34
- package/out/nsul/lookup.d.ts.map +1 -1
- package/out/nsul/lookup.js +26 -40
- package/out/nsul/lookup.js.map +1 -1
- package/out/nsul/schema.d.ts +16 -43
- package/out/nsul/schema.d.ts.map +1 -1
- package/out/nsul/schema.js +9 -35
- package/out/nsul/schema.js.map +1 -1
- package/out/{nsul/index.d.ts → nsul.d.ts} +1 -1
- package/out/nsul.d.ts.map +1 -0
- package/out/{nsul/index.js → nsul.js} +1 -1
- package/out/nsul.js.map +1 -0
- package/out/paths.d.ts +124 -0
- package/out/paths.d.ts.map +1 -0
- package/out/paths.js +104 -0
- package/out/paths.js.map +1 -0
- package/out/place-importance-schema.d.ts +49 -141
- package/out/place-importance-schema.d.ts.map +1 -1
- package/out/place-importance-schema.js +25 -140
- package/out/place-importance-schema.js.map +1 -1
- package/out/poi/lookup.d.ts +38 -42
- package/out/poi/lookup.d.ts.map +1 -1
- package/out/poi/lookup.js +67 -90
- package/out/poi/lookup.js.map +1 -1
- package/out/poi/schema.d.ts +41 -25
- package/out/poi/schema.d.ts.map +1 -1
- package/out/poi/schema.js +25 -17
- package/out/poi/schema.js.map +1 -1
- package/out/{poi/index.js → poi.d.ts} +1 -1
- package/out/poi.d.ts.map +1 -0
- package/out/{poi/index.d.ts → poi.js} +1 -1
- package/out/poi.js.map +1 -0
- package/out/polygon-schema.d.ts +10 -6
- package/out/polygon-schema.d.ts.map +1 -1
- package/out/polygon-schema.js +2 -2
- package/out/polygon-schema.js.map +1 -1
- package/out/postal/{city-alias-lookup.d.ts → city/alias/lookup.d.ts} +22 -25
- package/out/postal/city/alias/lookup.d.ts.map +1 -0
- package/out/postal/city/alias/lookup.js +49 -0
- package/out/postal/city/alias/lookup.js.map +1 -0
- package/out/postal/{city-alias-schema.d.ts → city/alias/schema.d.ts} +15 -11
- package/out/postal/city/alias/schema.d.ts.map +1 -0
- package/out/postal/{city-alias-schema.js → city/alias/schema.js} +11 -9
- package/out/postal/city/alias/schema.js.map +1 -0
- package/out/postal/city/candidate-schema.d.ts +72 -0
- package/out/postal/city/candidate-schema.d.ts.map +1 -0
- package/out/postal/city/candidate-schema.js +60 -0
- package/out/postal/city/candidate-schema.js.map +1 -0
- package/out/postal.d.ts +9 -0
- package/out/postal.d.ts.map +1 -0
- package/out/postal.js +9 -0
- package/out/postal.js.map +1 -0
- package/out/postcode-point-lookup.d.ts +9 -19
- package/out/postcode-point-lookup.d.ts.map +1 -1
- package/out/postcode-point-lookup.js +5 -17
- package/out/postcode-point-lookup.js.map +1 -1
- package/out/primary-preference.d.ts +39 -93
- package/out/primary-preference.d.ts.map +1 -1
- package/out/primary-preference.js +13 -97
- package/out/primary-preference.js.map +1 -1
- package/out/proximity-rerank.d.ts +40 -29
- package/out/proximity-rerank.d.ts.map +1 -1
- package/out/proximity-rerank.js +38 -29
- package/out/proximity-rerank.js.map +1 -1
- package/out/ranking-weights.d.ts +91 -62
- package/out/ranking-weights.d.ts.map +1 -1
- package/out/ranking-weights.js +26 -25
- package/out/ranking-weights.js.map +1 -1
- package/out/region-keys.d.ts +48 -29
- package/out/region-keys.d.ts.map +1 -1
- package/out/region-keys.js +68 -40
- package/out/region-keys.js.map +1 -1
- package/out/reverse.d.ts +75 -50
- package/out/reverse.d.ts.map +1 -1
- package/out/reverse.js +61 -49
- package/out/reverse.js.map +1 -1
- package/out/schema.d.ts +74 -42
- package/out/schema.d.ts.map +1 -1
- package/out/schema.js +4 -5
- package/out/schema.js.map +1 -1
- package/out/search-fetch.d.ts +16 -12
- package/out/search-fetch.d.ts.map +1 -1
- package/out/search-fetch.js +83 -63
- package/out/search-fetch.js.map +1 -1
- package/out/sqlite/convention-source.d.ts +25 -0
- package/out/sqlite/convention-source.d.ts.map +1 -0
- package/out/sqlite/convention-source.js +48 -0
- package/out/sqlite/convention-source.js.map +1 -0
- package/out/sqlite/utils.d.ts +45 -0
- package/out/sqlite/utils.d.ts.map +1 -0
- package/out/sqlite/utils.js +56 -0
- package/out/sqlite/utils.js.map +1 -0
- package/out/street/centroid/schema.d.ts +100 -0
- package/out/street/centroid/schema.d.ts.map +1 -0
- package/out/street/centroid/schema.js +66 -0
- package/out/street/centroid/schema.js.map +1 -0
- package/out/street/centroid.d.ts +9 -19
- package/out/street/centroid.d.ts.map +1 -1
- package/out/street/centroid.js +11 -22
- package/out/street/centroid.js.map +1 -1
- package/out/street/{morphology-fst-builder.d.ts → morphology/fst/builder.d.ts} +21 -11
- package/out/street/morphology/fst/builder.d.ts.map +1 -0
- package/out/street/{morphology-fst-builder.js → morphology/fst/builder.js} +40 -34
- package/out/street/morphology/fst/builder.js.map +1 -0
- package/out/street/morphology/fst/loader.d.ts +74 -0
- package/out/street/morphology/fst/loader.d.ts.map +1 -0
- package/out/street/{morphology-fst-loader.js → morphology/fst/loader.js} +21 -21
- package/out/street/morphology/fst/loader.js.map +1 -0
- package/out/street/name-lookup.d.ts +24 -18
- package/out/street/name-lookup.d.ts.map +1 -1
- package/out/street/name-lookup.js +32 -31
- package/out/street/name-lookup.js.map +1 -1
- package/out/street/normalize.d.ts +35 -136
- package/out/street/normalize.d.ts.map +1 -1
- package/out/street/normalize.js +74 -191
- package/out/street/normalize.js.map +1 -1
- package/out/street/segment-schema.d.ts +79 -39
- package/out/street/segment-schema.d.ts.map +1 -1
- package/out/street/segment-schema.js +71 -21
- package/out/street/segment-schema.js.map +1 -1
- package/out/{street/index.d.ts → street.d.ts} +4 -4
- package/out/street.d.ts.map +1 -0
- package/out/{street/index.js → street.js} +4 -4
- package/out/street.js.map +1 -0
- package/out/types.d.ts +69 -138
- package/out/types.d.ts.map +1 -1
- package/out/types.js +0 -6
- package/out/types.js.map +1 -1
- package/out/unified-schema.d.ts +8 -12
- package/out/unified-schema.d.ts.map +1 -1
- package/out/unified-schema.js +29 -29
- package/out/unified-schema.js.map +1 -1
- package/out/uprn/existence.d.ts +28 -17
- package/out/uprn/existence.d.ts.map +1 -1
- package/out/uprn/existence.js +26 -16
- package/out/uprn/existence.js.map +1 -1
- package/out/uprn/lookup.d.ts +14 -44
- package/out/uprn/lookup.d.ts.map +1 -1
- package/out/uprn/lookup.js +20 -56
- package/out/uprn/lookup.js.map +1 -1
- package/out/uprn/schema.d.ts +18 -40
- package/out/uprn/schema.d.ts.map +1 -1
- package/out/uprn/schema.js +10 -32
- package/out/uprn/schema.js.map +1 -1
- package/out/{uprn/index.d.ts → uprn.d.ts} +1 -1
- package/out/uprn.d.ts.map +1 -0
- package/out/{uprn/index.js → uprn.js} +1 -1
- package/out/uprn.js.map +1 -0
- package/out/weights-overlay-linker.d.ts +132 -141
- package/out/weights-overlay-linker.d.ts.map +1 -1
- package/out/weights-overlay-linker.js +86 -154
- package/out/weights-overlay-linker.js.map +1 -1
- package/package.json +94 -599
- package/lib/address/point-schema.ts +0 -155
- package/lib/build-candidate.ts +0 -632
- package/lib/build-slim.ts +0 -408
- package/lib/candidate-ancestors-schema.ts +0 -195
- package/lib/candidate-fts.ts +0 -46
- package/lib/candidate-importance.ts +0 -224
- package/lib/candidate-lookup.ts +0 -977
- package/lib/candidate-schema.ts +0 -247
- package/lib/candidate-scoring.ts +0 -261
- package/lib/convention/index.ts +0 -154
- package/lib/fts/index.ts +0 -361
- package/lib/postal/city-alias-lookup.ts +0 -97
- package/lib/postal/index.ts +0 -9
- package/lib/sqlite-convention-source.ts +0 -65
- package/lib/sqlite-utils.ts +0 -82
- package/lib/street/centroid-schema.ts +0 -154
- package/out/address/index.d.ts.map +0 -1
- package/out/address/index.js.map +0 -1
- package/out/address/point-interpolation.d.ts +0 -49
- package/out/address/point-interpolation.d.ts.map +0 -1
- package/out/address/point-interpolation.js.map +0 -1
- package/out/address/point-schema.d.ts +0 -97
- package/out/address/point-schema.d.ts.map +0 -1
- package/out/address/point-schema.js.map +0 -1
- package/out/ancestry/index.d.ts +0 -45
- package/out/ancestry/index.d.ts.map +0 -1
- package/out/ancestry/index.js +0 -55
- package/out/ancestry/index.js.map +0 -1
- package/out/build-candidate.d.ts +0 -184
- package/out/build-candidate.d.ts.map +0 -1
- package/out/build-candidate.js.map +0 -1
- package/out/build-slim.d.ts +0 -80
- package/out/build-slim.d.ts.map +0 -1
- package/out/build-slim.js.map +0 -1
- package/out/candidate/ancestors-sidecar.d.ts +0 -32
- package/out/candidate/ancestors-sidecar.d.ts.map +0 -1
- package/out/candidate/ancestors-sidecar.js.map +0 -1
- package/out/candidate-ancestors-schema.d.ts +0 -150
- package/out/candidate-ancestors-schema.d.ts.map +0 -1
- package/out/candidate-ancestors-schema.js +0 -123
- package/out/candidate-ancestors-schema.js.map +0 -1
- package/out/candidate-fts.d.ts +0 -37
- package/out/candidate-fts.d.ts.map +0 -1
- package/out/candidate-fts.js +0 -40
- package/out/candidate-fts.js.map +0 -1
- package/out/candidate-importance.d.ts +0 -132
- package/out/candidate-importance.d.ts.map +0 -1
- package/out/candidate-importance.js +0 -169
- package/out/candidate-importance.js.map +0 -1
- package/out/candidate-lookup.d.ts +0 -76
- package/out/candidate-lookup.d.ts.map +0 -1
- package/out/candidate-lookup.js +0 -768
- package/out/candidate-lookup.js.map +0 -1
- package/out/candidate-schema.d.ts +0 -155
- package/out/candidate-schema.d.ts.map +0 -1
- package/out/candidate-schema.js.map +0 -1
- package/out/candidate-scoring.d.ts.map +0 -1
- package/out/candidate-scoring.js +0 -196
- package/out/candidate-scoring.js.map +0 -1
- package/out/convention/index.d.ts +0 -106
- package/out/convention/index.d.ts.map +0 -1
- package/out/convention/index.js +0 -93
- package/out/convention/index.js.map +0 -1
- package/out/fst/index.d.ts.map +0 -1
- package/out/fst/index.js.map +0 -1
- package/out/fts/index.d.ts +0 -184
- package/out/fts/index.d.ts.map +0 -1
- package/out/fts/index.js +0 -277
- package/out/fts/index.js.map +0 -1
- package/out/geonames/index.d.ts.map +0 -1
- package/out/geonames/index.js.map +0 -1
- package/out/nsul/index.d.ts.map +0 -1
- package/out/nsul/index.js.map +0 -1
- package/out/poi/index.d.ts.map +0 -1
- package/out/poi/index.js.map +0 -1
- package/out/postal/city-alias-lookup.d.ts.map +0 -1
- package/out/postal/city-alias-lookup.js +0 -62
- package/out/postal/city-alias-lookup.js.map +0 -1
- package/out/postal/city-alias-schema.d.ts.map +0 -1
- package/out/postal/city-alias-schema.js.map +0 -1
- package/out/postal/city-candidate-schema.d.ts +0 -66
- package/out/postal/city-candidate-schema.d.ts.map +0 -1
- package/out/postal/city-candidate-schema.js +0 -56
- package/out/postal/city-candidate-schema.js.map +0 -1
- package/out/postal/index.d.ts +0 -9
- package/out/postal/index.d.ts.map +0 -1
- package/out/postal/index.js +0 -9
- package/out/postal/index.js.map +0 -1
- package/out/sqlite-convention-source.d.ts +0 -29
- package/out/sqlite-convention-source.d.ts.map +0 -1
- package/out/sqlite-convention-source.js +0 -57
- package/out/sqlite-convention-source.js.map +0 -1
- package/out/sqlite-utils.d.ts +0 -45
- package/out/sqlite-utils.d.ts.map +0 -1
- package/out/sqlite-utils.js +0 -57
- package/out/sqlite-utils.js.map +0 -1
- package/out/street/centroid-schema.d.ts +0 -105
- package/out/street/centroid-schema.d.ts.map +0 -1
- package/out/street/centroid-schema.js +0 -78
- package/out/street/centroid-schema.js.map +0 -1
- package/out/street/index.d.ts.map +0 -1
- package/out/street/index.js.map +0 -1
- package/out/street/morphology-fst-builder.d.ts.map +0 -1
- package/out/street/morphology-fst-builder.js.map +0 -1
- package/out/street/morphology-fst-loader.d.ts +0 -67
- package/out/street/morphology-fst-loader.d.ts.map +0 -1
- package/out/street/morphology-fst-loader.js.map +0 -1
- package/out/uprn/index.d.ts.map +0 -1
- package/out/uprn/index.js.map +0 -1
- /package/lib/{fst/index.ts → fst.ts} +0 -0
- /package/lib/{geonames/index.ts → geonames.ts} +0 -0
- /package/lib/{nsul/index.ts → nsul.ts} +0 -0
- /package/lib/{poi/index.ts → poi.ts} +0 -0
- /package/lib/{uprn/index.ts → uprn.ts} +0 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
* @file Populations the gazetteer records at a fraction of their size, corrected against the record's own concordance.
|
|
6
|
+
*
|
|
7
|
+
* This file records corrections for review. The build must not rewrite populations from a
|
|
8
|
+
* same-name-parent ratio. The detector cannot distinguish a district's mis-recorded namesake
|
|
9
|
+
* city from a small village with the district name. Both places are real.
|
|
10
|
+
*
|
|
11
|
+
* Each entry identifies one row that a reviewer checked against a second source.
|
|
12
|
+
* The row's own `gn:id` must agree with the coordinates. The GeoNames record must identify the same place.
|
|
13
|
+
*
|
|
14
|
+
* The correction applies to the candidate artifact rather than the admin gazetteer.
|
|
15
|
+
* `place_population` in `admin-global-priority.db` still stores the WOF figure, so a reader going
|
|
16
|
+
* there sees WOF's value and a reader going to `candidate.db` sees the corroborated number. The
|
|
17
|
+
* admin database is a transcription of its source, so correcting it there would make the
|
|
18
|
+
* transcription disagree with what it transcribes.
|
|
19
|
+
*/
|
|
20
|
+
/**
|
|
21
|
+
* One corrected population, with what corroborates it.
|
|
22
|
+
*/
|
|
23
|
+
export interface PopulationCorrection {
|
|
24
|
+
/**
|
|
25
|
+
* The GeoNames id from the WOF record's own `concordances` row makes this a
|
|
26
|
+
* correction rather than a substitution.
|
|
27
|
+
*
|
|
28
|
+
* The two sources describe the same place according to WOF.
|
|
29
|
+
*/
|
|
30
|
+
geonamesID: string;
|
|
31
|
+
population: number;
|
|
32
|
+
/**
|
|
33
|
+
* What the gazetteer recorded, kept so a reader can see the size of the
|
|
34
|
+
* disagreement without opening the artifact.
|
|
35
|
+
*/
|
|
36
|
+
recorded: number;
|
|
37
|
+
/**
|
|
38
|
+
* The place and the reviewer's reading.
|
|
39
|
+
*/
|
|
40
|
+
note: string;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* WOF id → the corrected population.
|
|
44
|
+
*
|
|
45
|
+
* One entry, because one row has been followed.
|
|
46
|
+
* A second correction requires the same review.
|
|
47
|
+
*
|
|
48
|
+
* Read the row's `gn:id` and confirm the coordinates agree.
|
|
49
|
+
* Then record the second source's value.
|
|
50
|
+
*/
|
|
51
|
+
export declare const POPULATION_CORRECTIONS: Readonly<Record<number, PopulationCorrection>>;
|
|
52
|
+
//# sourceMappingURL=population-corrections.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"population-corrections.d.ts","sourceRoot":"","sources":["../../lib/build/population-corrections.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC;;;;;OAKG;IACH,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;CACZ;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,sBAAsB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAWjF,CAAA"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
* @file Populations the gazetteer records at a fraction of their size, corrected against the record's own concordance.
|
|
6
|
+
*
|
|
7
|
+
* This file records corrections for review. The build must not rewrite populations from a
|
|
8
|
+
* same-name-parent ratio. The detector cannot distinguish a district's mis-recorded namesake
|
|
9
|
+
* city from a small village with the district name. Both places are real.
|
|
10
|
+
*
|
|
11
|
+
* Each entry identifies one row that a reviewer checked against a second source.
|
|
12
|
+
* The row's own `gn:id` must agree with the coordinates. The GeoNames record must identify the same place.
|
|
13
|
+
*
|
|
14
|
+
* The correction applies to the candidate artifact rather than the admin gazetteer.
|
|
15
|
+
* `place_population` in `admin-global-priority.db` still stores the WOF figure, so a reader going
|
|
16
|
+
* there sees WOF's value and a reader going to `candidate.db` sees the corroborated number. The
|
|
17
|
+
* admin database is a transcription of its source, so correcting it there would make the
|
|
18
|
+
* transcription disagree with what it transcribes.
|
|
19
|
+
*/
|
|
20
|
+
/**
|
|
21
|
+
* WOF id → the corrected population.
|
|
22
|
+
*
|
|
23
|
+
* One entry, because one row has been followed.
|
|
24
|
+
* A second correction requires the same review.
|
|
25
|
+
*
|
|
26
|
+
* Read the row's `gn:id` and confirm the coordinates agree.
|
|
27
|
+
* Then record the second source's value.
|
|
28
|
+
*/
|
|
29
|
+
export const POPULATION_CORRECTIONS = {
|
|
30
|
+
102_030_887: {
|
|
31
|
+
geonamesID: "1278149",
|
|
32
|
+
population: 1_175_116,
|
|
33
|
+
recorded: 19_172,
|
|
34
|
+
note: "Aurangabad (Chhatrapati Sambhaji Nagar), Maharashtra. The WOF record sits at 19.870 / 75.349 and GeoNames " +
|
|
35
|
+
"1278149 — the id that record's own concordance names — sits at 19.878 / 75.342, about 1 km apart, classed " +
|
|
36
|
+
"`PPLA2`. WOF's 19,172 against its parent district's 3,701,282 is the 193x ratio that surfaced it; the " +
|
|
37
|
+
"corroborated figure makes the city 32% of its district rather than 0.5% of it.",
|
|
38
|
+
},
|
|
39
|
+
};
|
|
40
|
+
//# sourceMappingURL=population-corrections.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"population-corrections.js","sourceRoot":"","sources":["../../lib/build/population-corrections.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAyBH;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAmD;IACrF,WAAW,EAAE;QACZ,UAAU,EAAE,SAAS;QACrB,UAAU,EAAE,SAAS;QACrB,QAAQ,EAAE,MAAM;QAChB,IAAI,EACH,4GAA4G;YAC5G,4GAA4G;YAC5G,wGAAwG;YACxG,gFAAgF;KACjF;CACD,CAAA"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
import type { PathBuilderLike } from "path-ts";
|
|
7
|
+
export interface BuildSlimOptions {
|
|
8
|
+
/**
|
|
9
|
+
* Input WOF SQLite distributions, each already carrying spr / names / place_population tables.
|
|
10
|
+
*/
|
|
11
|
+
inputs: readonly PathBuilderLike[];
|
|
12
|
+
/**
|
|
13
|
+
* Output path for the slim DB, overwritten if it exists.
|
|
14
|
+
*/
|
|
15
|
+
output: PathBuilderLike;
|
|
16
|
+
/**
|
|
17
|
+
* Country codes to keep (ISO 2-letter); defaults to `["US"]`.
|
|
18
|
+
*/
|
|
19
|
+
countries?: string[];
|
|
20
|
+
/**
|
|
21
|
+
* Cap on the number of localities to keep per country, by descending population.
|
|
22
|
+
*/
|
|
23
|
+
topLocalitiesPerCountry?: number;
|
|
24
|
+
/**
|
|
25
|
+
* Drop the `names` table after the FTS index is built; `place_search` is self-contained
|
|
26
|
+
* and the resolver never reads `names` at runtime.
|
|
27
|
+
*/
|
|
28
|
+
dropNames?: boolean;
|
|
29
|
+
onProgress?: (phase: SlimBuildPhase, detail: string) => void;
|
|
30
|
+
}
|
|
31
|
+
export type SlimBuildPhase = "init" | "schema" | "country" | "region" | "county" | "locality" | "postcode" | "names" | "place_population" | "coincident_roles" | "place_abbr" | "fts" | "vacuum" | "done";
|
|
32
|
+
export interface BuildSlimResult {
|
|
33
|
+
outputPath: string;
|
|
34
|
+
outputBytes: number;
|
|
35
|
+
rowCounts: {
|
|
36
|
+
spr: number;
|
|
37
|
+
names: number;
|
|
38
|
+
placeSearch: number;
|
|
39
|
+
placeBbox: number;
|
|
40
|
+
placePopulation: number;
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
export declare function buildSlimWOFDatabase(opts: BuildSlimOptions): Promise<BuildSlimResult>;
|
|
44
|
+
//# sourceMappingURL=slim.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"slim.d.ts","sourceRoot":"","sources":["../../lib/build/slim.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAWH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAK9C,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,MAAM,EAAE,SAAS,eAAe,EAAE,CAAA;IAClC;;OAEG;IACH,MAAM,EAAE,eAAe,CAAA;IACvB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,EAAE,CAAA;IACpB;;OAEG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAA;IAChC;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;IAEnB,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI,CAAA;CAC5D;AAED,MAAM,MAAM,cAAc,GACvB,MAAM,GACN,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,QAAQ,GACR,UAAU,GACV,UAAU,GACV,OAAO,GACP,kBAAkB,GAClB,kBAAkB,GAClB,YAAY,GACZ,KAAK,GACL,QAAQ,GACR,MAAM,CAAA;AAET,MAAM,WAAW,eAAe;IAC/B,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE;QACV,GAAG,EAAE,MAAM,CAAA;QACX,KAAK,EAAE,MAAM,CAAA;QACb,WAAW,EAAE,MAAM,CAAA;QACnB,SAAS,EAAE,MAAM,CAAA;QACjB,eAAe,EAAE,MAAM,CAAA;KACvB,CAAA;CACD;AAoCD,wBAAsB,oBAAoB,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CA4G3F"}
|
|
@@ -2,52 +2,23 @@
|
|
|
2
2
|
* @copyright Sister Software
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
|
-
*
|
|
6
|
-
* Build a "slim" Who's On First SQLite distribution that's small enough to ship as a static asset
|
|
7
|
-
* for the browser-side mailwoman demo (Path B of the demo plan). The full admin distribution is
|
|
8
|
-
* ~2 GB; the slim variant aims for the ~50–100 MB range by keeping only the places a public demo
|
|
9
|
-
* will actually query for.
|
|
10
|
-
*
|
|
11
|
-
* Selection policy (v1, US-focused):
|
|
12
|
-
*
|
|
13
|
-
* - All countries / regions / counties / boroughs in the configured `countries` set, so the ancestor
|
|
14
|
-
* chain a locality / postcode reports through `parent_id` stays intact.
|
|
15
|
-
* - Top-K localities by population (read from the source's pre-built `place_population` aux table) in
|
|
16
|
-
* those countries.
|
|
17
|
-
* - All postcodes in those countries — they're small and addressing-relevant.
|
|
18
|
-
* - All `names` + `place_population` rows for selected place IDs.
|
|
19
|
-
* - The `coincident_roles` dual-role relation (#402), filtered to surviving spr ids.
|
|
20
|
-
*
|
|
21
|
-
* No geojson, by design. The upstream WOF GeoJSON bodies live ONLY in the raw `whosonfirst-data-*`
|
|
22
|
-
* repos; `scripts/build-unified-wof.ts` extracts `wof:population` straight into
|
|
23
|
-
* `place_population` (and the bbox into `spr`) at ingest and never persists a `geojson` table. So
|
|
24
|
-
* the source admin DB carries population in `place_population`, and this builder consumes it
|
|
25
|
-
* directly — there is nothing to extract from, and nothing to drop.
|
|
26
|
-
*
|
|
27
|
-
* The output DB has the resolver-facing schema: `spr`, `names`, `place_population`, plus the
|
|
28
|
-
* `place_search` FTS5 / `place_bbox` R*Tree virtual tables rebuilt against the trimmed row set
|
|
29
|
-
* (both derive purely from `spr` + `names` — see `fts.ts`). That means `WOFSQLitePlaceLookup`
|
|
30
|
-
* opens the slim DB without any code change — it sees a smaller universe, nothing more.
|
|
31
|
-
*
|
|
32
|
-
* Multi-extract inputs (e.g. admin + postcode) are processed in sequence; selected rows accumulate
|
|
33
|
-
* into the single output DB. The postcode extract contributes only postcodes; admin contributes
|
|
34
|
-
* everything else. Empty / missing input paths are skipped (callers pass `""` when a extract, such
|
|
35
|
-
* as a custom postcode DB, isn't built yet).
|
|
36
5
|
*/
|
|
37
6
|
import { statPath, pathExists } from "@mailwoman/core/fs/readers";
|
|
38
7
|
import { temporaryDirectory } from "@mailwoman/core/fs/temporary";
|
|
39
8
|
import { copyFileTo, removePath } from "@mailwoman/core/fs/writers";
|
|
9
|
+
import { stringifyJSON } from "@mailwoman/core/json";
|
|
40
10
|
import { countRows } from "@mailwoman/sqlite";
|
|
41
11
|
import { DatabaseClient } from "@mailwoman/sqlite/client";
|
|
42
|
-
import { sealDatabase } from "@mailwoman/sqlite/sealed
|
|
12
|
+
import { sealDatabase } from "@mailwoman/sqlite/sealed/db";
|
|
43
13
|
import { sql } from "kysely";
|
|
44
|
-
import { buildPlaceSearchFTS, PLACE_BBOX_TABLE, PLACE_POPULATION_TABLE, PLACE_SEARCH_TABLE } from "#fts
|
|
14
|
+
import { buildPlaceSearchFTS, PLACE_BBOX_TABLE, PLACE_POPULATION_TABLE, PLACE_SEARCH_TABLE } from "#fts";
|
|
45
15
|
/**
|
|
46
16
|
* Placetypes that we always keep so the ancestor chain a selected locality reports stays valid.
|
|
47
17
|
*/
|
|
48
18
|
const ANCESTOR_PLACETYPES = ["country", "region", "county", "borough", "macroregion"];
|
|
49
19
|
/**
|
|
50
|
-
* Tables copied verbatim (schema + filtered rows) from each source DB.
|
|
20
|
+
* Tables copied verbatim (schema + filtered rows) from each source DB.
|
|
21
|
+
* Anything else is dropped.
|
|
51
22
|
*/
|
|
52
23
|
const COPIED_TABLES = ["spr", "names", PLACE_POPULATION_TABLE];
|
|
53
24
|
/**
|
|
@@ -58,9 +29,8 @@ export async function buildSlimWOFDatabase(opts) {
|
|
|
58
29
|
const countries = (opts.countries ?? ["US"]).map((c) => c.toUpperCase());
|
|
59
30
|
const topLocalities = opts.topLocalitiesPerCountry ?? 1000;
|
|
60
31
|
const progress = opts.onProgress ?? (() => { });
|
|
61
|
-
//
|
|
62
|
-
|
|
63
|
-
const inputs = opts.inputs.filter((p) => p.length > 0);
|
|
32
|
+
// An empty string marks an input that isn't built yet, so empties are skipped and every remaining path must exist.
|
|
33
|
+
const inputs = opts.inputs.map((input) => input.toString()).filter((p) => p.length);
|
|
64
34
|
if (!inputs.length)
|
|
65
35
|
throw new Error("no input WOF dbs provided");
|
|
66
36
|
for (const input of inputs) {
|
|
@@ -71,10 +41,8 @@ export async function buildSlimWOFDatabase(opts) {
|
|
|
71
41
|
if (await pathExists(opts.output)) {
|
|
72
42
|
await removePath(opts.output);
|
|
73
43
|
}
|
|
74
|
-
//
|
|
75
|
-
//
|
|
76
|
-
// ordering / types. `CREATE TABLE AS SELECT` flattens types to dynamic, which would break
|
|
77
|
-
// callers that rely on column-affinity behavior.
|
|
44
|
+
// The output schema is replayed from the first input's sqlite_master so column ordering
|
|
45
|
+
// and affinity survive; `create table AS select` would flatten types to dynamic.
|
|
78
46
|
const out = new DatabaseClient(opts.output);
|
|
79
47
|
let result;
|
|
80
48
|
try {
|
|
@@ -85,60 +53,46 @@ export async function buildSlimWOFDatabase(opts) {
|
|
|
85
53
|
.prepare(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`)
|
|
86
54
|
.get(table);
|
|
87
55
|
if (createSQL?.sql) {
|
|
88
|
-
// Raw DDL by design
|
|
89
|
-
// string read from sqlite_master, so a static Kysely builder can't express it. See AGENTS.md.
|
|
56
|
+
// Raw DDL by design: the source DB's own create-table string cannot be expressed by a static Kysely builder.
|
|
90
57
|
out.exec(createSQL.sql);
|
|
91
58
|
}
|
|
92
59
|
else if (table === PLACE_POPULATION_TABLE) {
|
|
93
|
-
// Older source builds may predate the aux table
|
|
94
|
-
// copy + ranking have somewhere to land. Sparse-by-design; missing rows are fine.
|
|
60
|
+
// Older source builds may predate the aux table, so create it empty for the copy and ranking.
|
|
95
61
|
out.exec(PLACE_POPULATION_DDL);
|
|
96
62
|
}
|
|
97
63
|
else {
|
|
98
64
|
throw new Error(`source DB ${inputs[0]} is missing required table '${table}'`);
|
|
99
65
|
}
|
|
100
66
|
}
|
|
101
|
-
//
|
|
102
|
-
// index on names.id helps the per-id
|
|
67
|
+
// The copied schemas define the primary keys.
|
|
68
|
+
// This index on `names.id` helps the per-id insert select.
|
|
103
69
|
out.exec(`CREATE INDEX IF NOT EXISTS names_id_idx ON names(id);`);
|
|
104
70
|
// Pull rows from each input.
|
|
105
71
|
for (const inputPath of inputs) {
|
|
106
72
|
await copyFromSource(out, out, inputPath, countries, topLocalities, progress);
|
|
107
73
|
}
|
|
108
|
-
//
|
|
109
|
-
//
|
|
110
|
-
// population aux table is NOT rebuilt here: it was copied verbatim above, and fts.ts only
|
|
111
|
-
// (re)builds it when a `geojson` table is present, which the slim DB intentionally has not.
|
|
74
|
+
// `place_search` and `place_bbox` derive purely from `spr` + `names`, and the
|
|
75
|
+
// population aux table is not rebuilt because it was copied verbatim.
|
|
112
76
|
progress("fts", "building place_search / place_bbox on slim DB");
|
|
113
77
|
buildPlaceSearchFTS(out, {
|
|
114
78
|
drop: true, // schema we copied had no FTS tables, but be explicit
|
|
115
79
|
onProgress: (phase, name) => progress("fts", `${phase} ${name}`),
|
|
116
80
|
});
|
|
117
|
-
//
|
|
118
|
-
//
|
|
119
|
-
// querying `names` (`#exactMatchIDs`), but the slim DB drops `names` for size — so the
|
|
120
|
-
// browser resolver gets its own tiny lookup (~hundreds of rows) to do the same data-driven
|
|
121
|
-
// exact-abbrev tiering ("VT" → Vermont, not a token-matching foreign region) instead of the
|
|
122
|
-
// demo's hardcoded region-abbreviation map (since deleted). Sourced from the `language='abbr'` rows
|
|
123
|
-
// `add-region-abbrevs.ts` wrote, already filtered to surviving spr ids via the names copy. The
|
|
124
|
-
// table is always created (empty when the source predates the abbrev enrichment) so the
|
|
125
|
-
// resolver can query it unconditionally.
|
|
81
|
+
// `place_abbr` is materialized before `names` is dropped and always created,
|
|
82
|
+
// even empty, so the resolver can query it unconditionally.
|
|
126
83
|
progress("place_abbr", "materializing region abbreviations");
|
|
127
84
|
out.exec(`CREATE TABLE IF NOT EXISTS place_abbr (id INTEGER NOT NULL, abbr TEXT NOT NULL)`);
|
|
128
85
|
out.exec(`INSERT INTO place_abbr (id, abbr) SELECT id, name FROM names WHERE language = 'abbr'`);
|
|
129
86
|
out.exec(`CREATE INDEX IF NOT EXISTS place_abbr_by_abbr ON place_abbr (abbr COLLATE NOCASE)`);
|
|
130
87
|
out.exec(`CREATE INDEX IF NOT EXISTS place_abbr_by_id ON place_abbr (id)`);
|
|
131
|
-
// Capture the names count BEFORE any drop so the build report stays informative.
|
|
132
88
|
const namesRows = countRows(out, "names");
|
|
133
|
-
//
|
|
134
|
-
// it. The resolver never reads `names` at query time, so this is pure size reduction.
|
|
89
|
+
// The resolver never reads `names` at query time, so dropping it and its index is pure size reduction.
|
|
135
90
|
if (opts.dropNames) {
|
|
136
91
|
progress("vacuum", `dropping names table (${namesRows} rows; FTS5 is self-contained)`);
|
|
137
92
|
out.exec(`DROP INDEX IF EXISTS names_id_idx;`);
|
|
138
93
|
out.exec(`DROP TABLE IF EXISTS names;`);
|
|
139
94
|
}
|
|
140
|
-
//
|
|
141
|
-
// file size stays inflated from the in-flight INSERT churn.
|
|
95
|
+
// Vacuum so the on-disk file reflects the trimmed rows rather than the in-flight insert churn.
|
|
142
96
|
progress("vacuum", "VACUUM (final size reduction)");
|
|
143
97
|
out.exec("VACUUM;");
|
|
144
98
|
const rowCounts = {
|
|
@@ -148,9 +102,9 @@ export async function buildSlimWOFDatabase(opts) {
|
|
|
148
102
|
placeBbox: countRows(out, PLACE_BBOX_TABLE),
|
|
149
103
|
placePopulation: countRows(out, PLACE_POPULATION_TABLE),
|
|
150
104
|
};
|
|
151
|
-
progress("done",
|
|
105
|
+
progress("done", stringifyJSON(rowCounts));
|
|
152
106
|
result = {
|
|
153
|
-
outputPath: opts.output,
|
|
107
|
+
outputPath: opts.output.toString(),
|
|
154
108
|
outputBytes: (await statPath(opts.output)).size,
|
|
155
109
|
rowCounts,
|
|
156
110
|
};
|
|
@@ -163,25 +117,17 @@ export async function buildSlimWOFDatabase(opts) {
|
|
|
163
117
|
return result;
|
|
164
118
|
}
|
|
165
119
|
async function copyFromSource(out, kysely, inputPath, countries, topLocalities, progress) {
|
|
166
|
-
//
|
|
167
|
-
//
|
|
168
|
-
// ATTACH will still want a writable journal on the side; copying to /tmp dodges that without
|
|
169
|
-
// mutating the canonical files in /mnt/playpen/mailwoman-data/wof/. ATTACH / DETACH stay raw
|
|
170
|
-
// — Kysely doesn't model them.
|
|
120
|
+
// A fresh scratch copy is attached so read-only WOF distributions get the writable
|
|
121
|
+
// journal they need without mutating the canonical files.
|
|
171
122
|
await using tmpScratch = await temporaryDirectory("mailwoman-slim-src-");
|
|
172
|
-
const scratchPath = tmpScratch.
|
|
123
|
+
const scratchPath = tmpScratch.path("src.db");
|
|
173
124
|
await copyFileTo(inputPath, scratchPath);
|
|
174
|
-
out.exec(`ATTACH DATABASE '${scratchPath.replaceAll("'", "''")}' AS src;`);
|
|
125
|
+
out.exec(`ATTACH DATABASE '${scratchPath.toString().replaceAll("'", "''")}' AS src;`);
|
|
175
126
|
try {
|
|
176
|
-
// Does this extract carry the pre-built population aux table? The admin source does; a bare
|
|
177
|
-
// postcode extract might not. The locality ranking + population copy below adapt accordingly.
|
|
178
127
|
const srcHasPopulation = Boolean(out.prepare(`SELECT 1 FROM src.sqlite_master WHERE type = 'table' AND name = '${PLACE_POPULATION_TABLE}'`).get());
|
|
179
|
-
// The
|
|
180
|
-
//
|
|
181
|
-
//
|
|
182
|
-
// checking as the regular schema. SQLite parses the dotted identifier as a schema-name
|
|
183
|
-
// qualifier, so this works directly without any aliasing trick.
|
|
184
|
-
// 1. Ancestor placetypes (country / region / county / etc.) — always-kept.
|
|
128
|
+
// The `src.spr` declaration and its peers in `BuildSchema` let Kysely
|
|
129
|
+
// column-check the cross-schema select.
|
|
130
|
+
// SQLite reads the dotted identifier as a schema qualifier.
|
|
185
131
|
progress("country", `${inputPath}: ancestor placetypes in (${countries.join(",")})`);
|
|
186
132
|
await kysely
|
|
187
133
|
.insertInto("spr")
|
|
@@ -194,9 +140,8 @@ async function copyFromSource(out, kysely, inputPath, countries, topLocalities,
|
|
|
194
140
|
.where("placetype", "in", [...ANCESTOR_PLACETYPES]))
|
|
195
141
|
.onConflict((oc) => oc.doNothing())
|
|
196
142
|
.execute();
|
|
197
|
-
//
|
|
198
|
-
// aux table
|
|
199
|
-
// last). If the extract has no population table, fall back to a deterministic id ordering.
|
|
143
|
+
// Localities without a population row still qualify.
|
|
144
|
+
// Without the aux table, fall back to a deterministic id ordering.
|
|
200
145
|
progress("locality", `${inputPath}: top-${topLocalities} localities by population`);
|
|
201
146
|
await kysely
|
|
202
147
|
.insertInto("spr")
|
|
@@ -212,7 +157,6 @@ async function copyFromSource(out, kysely, inputPath, countries, topLocalities,
|
|
|
212
157
|
.limit(topLocalities))
|
|
213
158
|
.onConflict((oc) => oc.doNothing())
|
|
214
159
|
.execute();
|
|
215
|
-
// 3. All postcodes in scope.
|
|
216
160
|
progress("postcode", `${inputPath}: all postcodes`);
|
|
217
161
|
await kysely
|
|
218
162
|
.insertInto("spr")
|
|
@@ -225,14 +169,12 @@ async function copyFromSource(out, kysely, inputPath, countries, topLocalities,
|
|
|
225
169
|
.where("placetype", "=", "postalcode"))
|
|
226
170
|
.onConflict((oc) => oc.doNothing())
|
|
227
171
|
.execute();
|
|
228
|
-
// 4. Pull names for the IDs we just selected.
|
|
229
172
|
progress("names", `${inputPath}: names rows for selected IDs`);
|
|
230
173
|
await kysely
|
|
231
174
|
.insertInto("names")
|
|
232
175
|
.expression((eb) => eb.selectFrom("src.names").selectAll().where("id", "in", eb.selectFrom("spr").select("id")))
|
|
233
176
|
.onConflict((oc) => oc.doNothing())
|
|
234
177
|
.execute();
|
|
235
|
-
// 5. Pull population rows for the selected IDs (sparse — only the places WOF has a count for).
|
|
236
178
|
if (srcHasPopulation) {
|
|
237
179
|
progress("place_population", `${inputPath}: population rows for selected IDs`);
|
|
238
180
|
await kysely
|
|
@@ -241,11 +183,8 @@ async function copyFromSource(out, kysely, inputPath, countries, topLocalities,
|
|
|
241
183
|
.onConflict((oc) => oc.doNothing())
|
|
242
184
|
.execute();
|
|
243
185
|
}
|
|
244
|
-
//
|
|
245
|
-
//
|
|
246
|
-
// spr ids → no orphans. Tiny (~hundreds of rows). `ancestors` is intentionally NOT copied (huge
|
|
247
|
-
// + build-only), so we copy the derived table rather than rebuild it. Raw SQL — conditional +
|
|
248
|
-
// not in the Kysely build schema.
|
|
186
|
+
// `ancestors` is intentionally not copied, so the derived `coincident_roles`
|
|
187
|
+
// table is copied and filtered to surviving spr ids.
|
|
249
188
|
const relationSchema = out
|
|
250
189
|
.prepare(`SELECT sql FROM src.sqlite_master WHERE type = 'table' AND name = 'coincident_roles'`)
|
|
251
190
|
.get();
|
|
@@ -261,4 +200,4 @@ async function copyFromSource(out, kysely, inputPath, countries, topLocalities,
|
|
|
261
200
|
out.exec(`DETACH DATABASE src;`);
|
|
262
201
|
}
|
|
263
202
|
}
|
|
264
|
-
//# sourceMappingURL=
|
|
203
|
+
//# sourceMappingURL=slim.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"slim.js","sourceRoot":"","sources":["../../lib/build/slim.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAA;AACjE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAA;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAA;AAC1D,OAAO,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAA;AAI5B,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,MAAM,MAAM,CAAA;AAyDxG;;GAEG;AACH,MAAM,mBAAmB,GAAG,CAAC,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,aAAa,CAAU,CAAA;AAE9F;;;GAGG;AACH,MAAM,aAAa,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,sBAAsB,CAAU,CAAA;AAEvE;;GAEG;AACH,MAAM,oBAAoB,GAAG,gBAAgB,sBAAsB,kEAAkE,CAAA;AAoBrI,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAsB;IAChE,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAA;IACxE,MAAM,aAAa,GAAG,IAAI,CAAC,uBAAuB,IAAI,IAAI,CAAA;IAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;IAE9C,mHAAmH;IACnH,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;IAEnF,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAA;IAEhE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,KAAK,EAAE,CAAC,CAAA;IACpF,CAAC;IAED,QAAQ,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,eAAe,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;IAE9D,IAAI,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACnC,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAC9B,CAAC;IAED,wFAAwF;IACxF,iFAAiF;IACjF,MAAM,GAAG,GAAG,IAAI,cAAc,CAAc,IAAI,CAAC,MAAM,CAAC,CAAA;IACxD,IAAI,MAAuB,CAAA;IAE3B,IAAI,CAAC;QACJ,MAAM,WAAW,GAAG,IAAI,cAAc,CAAc,MAAM,CAAC,CAAC,CAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;QAEnF,QAAQ,CAAC,QAAQ,EAAE,iEAAiE,CAAC,CAAA;QAErF,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,WAAW;iBAC3B,OAAO,CAAC,iEAAiE,CAAC;iBAC1E,GAAG,CAAC,KAAK,CAAiC,CAAA;YAE5C,IAAI,SAAS,EAAE,GAAG,EAAE,CAAC;gBACpB,6GAA6G;gBAC7G,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;YACxB,CAAC;iBAAM,IAAI,KAAK,KAAK,sBAAsB,EAAE,CAAC;gBAC7C,8FAA8F;gBAC9F,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAA;YAC/B,CAAC;iBAAM,CAAC;gBACP,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,CAAC,CAAC,CAAC,+BAA+B,KAAK,GAAG,CAAC,CAAA;YAC/E,CAAC;QACF,CAAC;QAED,8CAA8C;QAC9C,2DAA2D;QAC3D,GAAG,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAA;QAEjE,6BAA6B;QAC7B,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;YAChC,MAAM,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAA;QAC9E,CAAC;QAED,8EAA8E;QAC9E,sEAAsE;QACtE,QAAQ,CAAC,KAAK,EAAE,+CAA+C,CAAC,CAAA;QAEhE,mBAAmB,CAAC,GAAG,EAAE;YACxB,IAAI,EAAE,IAAI,EAAE,sDAAsD;YAClE,UAAU,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC;SAChE,CAAC,CAAA;QAEF,6EAA6E;QAC7E,4DAA4D;QAC5D,QAAQ,CAAC,YAAY,EAAE,oCAAoC,CAAC,CAAA;QAC5D,GAAG,CAAC,IAAI,CAAC,iFAAiF,CAAC,CAAA;QAC3F,GAAG,CAAC,IAAI,CAAC,sFAAsF,CAAC,CAAA;QAChG,GAAG,CAAC,IAAI,CAAC,mFAAmF,CAAC,CAAA;QAC7F,GAAG,CAAC,IAAI,CAAC,gEAAgE,CAAC,CAAA;QAE1E,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;QAEzC,uGAAuG;QACvG,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,QAAQ,CAAC,QAAQ,EAAE,yBAAyB,SAAS,gCAAgC,CAAC,CAAA;YACtF,GAAG,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAA;YAC9C,GAAG,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAA;QACxC,CAAC;QAED,+FAA+F;QAC/F,QAAQ,CAAC,QAAQ,EAAE,+BAA+B,CAAC,CAAA;QACnD,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAEnB,MAAM,SAAS,GAAG;YACjB,GAAG,EAAE,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC;YAC1B,KAAK,EAAE,SAAS;YAChB,WAAW,EAAE,SAAS,CAAC,GAAG,EAAE,kBAAkB,CAAC;YAC/C,SAAS,EAAE,SAAS,CAAC,GAAG,EAAE,gBAAgB,CAAC;YAC3C,eAAe,EAAE,SAAS,CAAC,GAAG,EAAE,sBAAsB,CAAC;SACvD,CAAA;QAED,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,SAAS,CAAC,CAAC,CAAA;QAE1C,MAAM,GAAG;YACR,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE;YAClC,WAAW,EAAE,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;YAC/C,SAAS;SACT,CAAA;IACF,CAAC;YAAS,CAAC;QACV,MAAM,GAAG,CAAC,OAAO,EAAE,CAAA;IACpB,CAAC;IAED,4FAA4F;IAC5F,MAAM,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAE/B,OAAO,MAAM,CAAA;AACd,CAAC;AAED,KAAK,UAAU,cAAc,CAC5B,GAAgC,EAChC,MAA2B,EAC3B,SAAiB,EACjB,SAAmB,EACnB,aAAqB,EACrB,QAAqD;IAErD,mFAAmF;IACnF,0DAA0D;IAC1D,YAAY,UAAU,GAAG,MAAM,kBAAkB,CAAC,qBAAqB,CAAC,CAAA;IACxE,MAAM,WAAW,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IAE7C,MAAM,UAAU,CAAC,SAAS,EAAE,WAAW,CAAC,CAAA;IAExC,GAAG,CAAC,IAAI,CAAC,oBAAoB,WAAW,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;IAErF,IAAI,CAAC;QACJ,MAAM,gBAAgB,GAAG,OAAO,CAC/B,GAAG,CAAC,OAAO,CAAC,oEAAoE,sBAAsB,GAAG,CAAC,CAAC,GAAG,EAAE,CAChH,CAAA;QAED,sEAAsE;QACtE,wCAAwC;QACxC,4DAA4D;QAE5D,QAAQ,CAAC,SAAS,EAAE,GAAG,SAAS,6BAA6B,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAEpF,MAAM,MAAM;aACV,UAAU,CAAC,KAAK,CAAC;aACjB,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,CAClB,EAAE;aACA,UAAU,CAAC,SAAS,CAAC;aACrB,SAAS,EAAE;aACX,KAAK,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;aAC5B,KAAK,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC;aAC9B,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC;aACjC,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,GAAG,mBAAmB,CAAC,CAAC,CACpD;aACA,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;aAClC,OAAO,EAAE,CAAA;QAEX,qDAAqD;QACrD,mEAAmE;QACnE,QAAQ,CAAC,UAAU,EAAE,GAAG,SAAS,SAAS,aAAa,2BAA2B,CAAC,CAAA;QAEnF,MAAM,MAAM;aACV,UAAU,CAAC,KAAK,CAAC;aACjB,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,CAClB,EAAE;aACA,UAAU,CAAC,cAAc,CAAC;aAC1B,GAAG,CAAC,gBAAgB,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,2BAA2B,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;aACvF,SAAS,CAAC,GAAG,CAAC;aACd,KAAK,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;aAC9B,KAAK,CAAC,iBAAiB,EAAE,GAAG,EAAE,CAAC,CAAC;aAChC,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,SAAS,CAAC;aACnC,KAAK,CAAC,aAAa,EAAE,GAAG,EAAE,UAAU,CAAC;aACrC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,GAAG,CAAQ,2BAA2B,CAAC,CAAC,CAAC,GAAG,CAAQ,MAAM,EAAE,MAAM,CAAC;aAC9F,KAAK,CAAC,aAAa,CAAC,CACtB;aACA,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;aAClC,OAAO,EAAE,CAAA;QAEX,QAAQ,CAAC,UAAU,EAAE,GAAG,SAAS,iBAAiB,CAAC,CAAA;QAEnD,MAAM,MAAM;aACV,UAAU,CAAC,KAAK,CAAC;aACjB,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,CAClB,EAAE;aACA,UAAU,CAAC,SAAS,CAAC;aACrB,SAAS,EAAE;aACX,KAAK,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;aAC5B,KAAK,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC;aAC9B,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC;aACjC,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,YAAY,CAAC,CACvC;aACA,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;aAClC,OAAO,EAAE,CAAA;QAEX,QAAQ,CAAC,OAAO,EAAE,GAAG,SAAS,+BAA+B,CAAC,CAAA;QAE9D,MAAM,MAAM;aACV,UAAU,CAAC,OAAO,CAAC;aACnB,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;aAC/G,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;aAClC,OAAO,EAAE,CAAA;QAEX,IAAI,gBAAgB,EAAE,CAAC;YACtB,QAAQ,CAAC,kBAAkB,EAAE,GAAG,SAAS,oCAAoC,CAAC,CAAA;YAE9E,MAAM,MAAM;iBACV,UAAU,CAAC,kBAAkB,CAAC;iBAC9B,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,CAClB,EAAE,CAAC,UAAU,CAAC,sBAAsB,CAAC,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CACtG;iBACA,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;iBAClC,OAAO,EAAE,CAAA;QACZ,CAAC;QAED,6EAA6E;QAC7E,qDAAqD;QACrD,MAAM,cAAc,GAAG,GAAG;aACxB,OAAO,CAAC,sFAAsF,CAAC;aAC/F,GAAG,EAAkC,CAAA;QAEvC,IAAI,cAAc,EAAE,GAAG,EAAE,CAAC;YACzB,QAAQ,CAAC,kBAAkB,EAAE,GAAG,SAAS,8BAA8B,CAAC,CAAA;YACxE,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,eAAe,EAAE,4BAA4B,CAAC,CAAC,CAAA;YAEnF,GAAG,CAAC,IAAI,CACP;oFACgF,CAChF,CAAA;YAED,GAAG,CAAC,IAAI,CAAC,qFAAqF,CAAC,CAAA;QAChG,CAAC;IACF,CAAC;YAAS,CAAC;QACV,GAAG,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAA;IACjC,CAAC;AACF,CAAC"}
|
|
@@ -2,18 +2,26 @@
|
|
|
2
2
|
* @copyright Sister Software
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
|
-
* @file Pass 2 of the candidate build
|
|
5
|
+
* @file Pass 2 of the candidate build: explode `place_search.alt_names` into distinct alias rows.
|
|
6
6
|
*/
|
|
7
7
|
import type { DatabaseClient } from "@mailwoman/sqlite/client";
|
|
8
|
-
import type { CandidateDatabase } from "#candidate-schema";
|
|
9
8
|
import type { PlaceAttrs, StageRow } from "#candidate/place-attrs";
|
|
9
|
+
import type { CandidateDatabase } from "#candidate/schema";
|
|
10
10
|
import type { WOFDatabase } from "#schema";
|
|
11
11
|
/**
|
|
12
|
-
* Pass 2
|
|
13
|
-
* each place's distinct staged keys
|
|
12
|
+
* Pass 2: explode each place's `place_search.alt_names` bag into distinct-key alias rows
|
|
13
|
+
* (`is_primary = 0`), and count each place's distinct staged keys with the primary included.
|
|
14
|
+
*
|
|
15
|
+
* With `regionPlacetypeID` and `ccID` given, a region's alias that is another same-country region's
|
|
16
|
+
* official name is refused and counted rather than staged (see {@link officialNameHoldersByRegion}).
|
|
17
|
+
* Without them the pass stages every alias.
|
|
14
18
|
*/
|
|
15
|
-
export declare function explodeAliasBags(src: DatabaseClient<WOFDatabase>, out: DatabaseClient<CandidateDatabase>, attrs: Map<number, PlaceAttrs>, stageRow: StageRow
|
|
19
|
+
export declare function explodeAliasBags(src: DatabaseClient<WOFDatabase>, out: DatabaseClient<CandidateDatabase>, attrs: Map<number, PlaceAttrs>, stageRow: StageRow, opts?: {
|
|
20
|
+
regionPlacetypeID?: number;
|
|
21
|
+
ccID?: (code: string | null) => number;
|
|
22
|
+
}): {
|
|
16
23
|
nAlias: number;
|
|
17
24
|
keyCounts: Map<number, number>;
|
|
25
|
+
regionOfficialRefused: number;
|
|
18
26
|
};
|
|
19
27
|
//# sourceMappingURL=alias-bags.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"alias-bags.d.ts","sourceRoot":"","sources":["../../lib/candidate/alias-bags.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;
|
|
1
|
+
{"version":3,"file":"alias-bags.d.ts","sourceRoot":"","sources":["../../lib/candidate/alias-bags.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAG9D,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AAClE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AAE1D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAqC1C;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAC/B,GAAG,EAAE,cAAc,CAAC,WAAW,CAAC,EAChC,GAAG,EAAE,cAAc,CAAC,iBAAiB,CAAC,EACtC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,EAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,GAAE;IAAE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,KAAK,MAAM,CAAA;CAAO,GAC/E;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,qBAAqB,EAAE,MAAM,CAAA;CAAE,CAmDnF"}
|
|
@@ -2,38 +2,81 @@
|
|
|
2
2
|
* @copyright Sister Software
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
|
-
* @file Pass 2 of the candidate build
|
|
5
|
+
* @file Pass 2 of the candidate build: explode `place_search.alt_names` into distinct alias rows.
|
|
6
6
|
*/
|
|
7
|
-
import {
|
|
7
|
+
import { tableExists } from "@mailwoman/sqlite/introspection";
|
|
8
|
+
import { ALIAS_SEPARATOR } from "#fts";
|
|
8
9
|
import { normalizeLocalityForKey } from "#street/normalize";
|
|
9
10
|
/**
|
|
10
|
-
*
|
|
11
|
-
*
|
|
11
|
+
* The official name of every current region, keyed `${country id}\0${name key}` →
|
|
12
|
+
* region id, from the source `names` table's `official` bit.
|
|
13
|
+
*
|
|
14
|
+
* Empty when the source has no `names` table (a fixture, or an extract shape).
|
|
15
|
+
*
|
|
16
|
+
* The positive evidence behind the alias refusal below: a name is refused from a region's
|
|
17
|
+
* bag only when another region of the same country holds it as its official name.
|
|
12
18
|
*/
|
|
13
|
-
|
|
19
|
+
function officialNameHoldersByRegion(src, ccID) {
|
|
20
|
+
const keys = new Map();
|
|
21
|
+
if (!tableExists(src, "names"))
|
|
22
|
+
return keys;
|
|
23
|
+
for (const r of src
|
|
24
|
+
.prepare(`SELECT n.id AS id, n.name AS name, s.country AS country
|
|
25
|
+
FROM names n JOIN spr s ON s.id = n.id
|
|
26
|
+
WHERE s.placetype = 'region' AND s.is_current != 0 AND s.is_deprecated = 0 AND n.official = 1`)
|
|
27
|
+
.iterate()) {
|
|
28
|
+
const k = normalizeLocalityForKey(String(r.name ?? ""));
|
|
29
|
+
if (k) {
|
|
30
|
+
keys.set(`${ccID(r.country)}\0${k}`, Number(r.id));
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
return keys;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Pass 2: explode each place's `place_search.alt_names` bag into distinct-key alias rows
|
|
37
|
+
* (`is_primary = 0`), and count each place's distinct staged keys with the primary included.
|
|
38
|
+
*
|
|
39
|
+
* With `regionPlacetypeID` and `ccID` given, a region's alias that is another same-country region's
|
|
40
|
+
* official name is refused and counted rather than staged (see {@link officialNameHoldersByRegion}).
|
|
41
|
+
* Without them the pass stages every alias.
|
|
42
|
+
*/
|
|
43
|
+
export function explodeAliasBags(src, out, attrs, stageRow, opts = {}) {
|
|
14
44
|
let nAlias = 0;
|
|
45
|
+
let regionOfficialRefused = 0;
|
|
15
46
|
const keyCounts = new Map();
|
|
47
|
+
const official = typeof opts.regionPlacetypeID === "number" && opts.ccID
|
|
48
|
+
? officialNameHoldersByRegion(src, opts.ccID)
|
|
49
|
+
: new Map();
|
|
16
50
|
out.exec("BEGIN");
|
|
17
51
|
for (const r of src.prepare("SELECT wof_id, alt_names FROM place_search").iterate()) {
|
|
18
|
-
const
|
|
52
|
+
const id = Number(r.wof_id);
|
|
53
|
+
const a = attrs.get(id);
|
|
19
54
|
const alt = r.alt_names;
|
|
20
55
|
if (!a || !alt)
|
|
21
56
|
continue;
|
|
22
57
|
const seen = new Set([a.pkey]);
|
|
23
|
-
|
|
24
|
-
//
|
|
25
|
-
//
|
|
58
|
+
const isRegion = a.ptid === opts.regionPlacetypeID;
|
|
59
|
+
// The writer space-pads each separator and appends a trailing one, so every piece
|
|
60
|
+
// includes surrounding whitespace and the last one is empty.
|
|
61
|
+
// `normalizeLocalityForKey` folds both away.
|
|
26
62
|
for (const piece of alt.split(ALIAS_SEPARATOR)) {
|
|
27
63
|
const k = normalizeLocalityForKey(piece);
|
|
28
64
|
if (!k || seen.has(k))
|
|
29
65
|
continue;
|
|
30
66
|
seen.add(k);
|
|
31
|
-
|
|
67
|
+
if (isRegion) {
|
|
68
|
+
const holder = official.get(`${a.cid}\0${k}`);
|
|
69
|
+
if (typeof holder === "number" && holder !== id) {
|
|
70
|
+
regionOfficialRefused++;
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
stageRow(k, a, id, 0);
|
|
32
75
|
nAlias++;
|
|
33
76
|
}
|
|
34
|
-
keyCounts.set(
|
|
77
|
+
keyCounts.set(id, seen.size);
|
|
35
78
|
}
|
|
36
79
|
out.exec("COMMIT");
|
|
37
|
-
return { nAlias, keyCounts };
|
|
80
|
+
return { nAlias, keyCounts, regionOfficialRefused };
|
|
38
81
|
}
|
|
39
82
|
//# sourceMappingURL=alias-bags.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"alias-bags.js","sourceRoot":"","sources":["../../lib/candidate/alias-bags.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;
|
|
1
|
+
{"version":3,"file":"alias-bags.js","sourceRoot":"","sources":["../../lib/candidate/alias-bags.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAA;AAI7D,OAAO,EAAE,eAAe,EAAE,MAAM,MAAM,CAAA;AAEtC,OAAO,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAA;AAE3D;;;;;;;;GAQG;AACH,SAAS,2BAA2B,CACnC,GAAgC,EAChC,IAAqC;IAErC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAA;IAEtC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC;QAAE,OAAO,IAAI,CAAA;IAE3C,KAAK,MAAM,CAAC,IAAI,GAAG;SACjB,OAAO,CACP;;kGAE+F,CAC/F;SACA,OAAO,EAAE,EAAE,CAAC;QACb,MAAM,CAAC,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAA;QAEvD,IAAI,CAAC,EAAE,CAAC;YACP,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,OAAwB,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QACpE,CAAC;IACF,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,gBAAgB,CAC/B,GAAgC,EAChC,GAAsC,EACtC,KAA8B,EAC9B,QAAkB,EAClB,IAAI,GAA2E,EAAE;IAEjF,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,qBAAqB,GAAG,CAAC,CAAA;IAC7B,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAA;IAE3C,MAAM,QAAQ,GACb,OAAO,IAAI,CAAC,iBAAiB,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI;QACtD,CAAC,CAAC,2BAA2B,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;QAC7C,CAAC,CAAC,IAAI,GAAG,EAAkB,CAAA;IAE7B,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IAEjB,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,4CAA4C,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QACrF,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;QAC3B,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QACvB,MAAM,GAAG,GAAG,CAAC,CAAC,SAA0B,CAAA;QAExC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG;YAAE,SAAQ;QACxB,MAAM,IAAI,GAAG,IAAI,GAAG,CAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;QACtC,MAAM,QAAQ,GAAG,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,iBAAiB,CAAA;QAElD,kFAAkF;QAClF,6DAA6D;QAC7D,6CAA6C;QAC7C,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,CAAC;YAChD,MAAM,CAAC,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAA;YAExC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,SAAQ;YAC/B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAEX,IAAI,QAAQ,EAAE,CAAC;gBACd,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,EAAE,CAAC,CAAA;gBAE7C,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;oBACjD,qBAAqB,EAAE,CAAA;oBAEvB,SAAQ;gBACT,CAAC;YACF,CAAC;YAED,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAA;YAErB,MAAM,EAAE,CAAA;QACT,CAAC;QAED,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;IAC7B,CAAC;IAED,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IAElB,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,qBAAqB,EAAE,CAAA;AACpD,CAAC"}
|