@mailwoman/resolver-wof-sqlite 9.2.0 → 9.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -19
- package/lib/address/index.ts +9 -0
- package/{address-point-interpolation.ts → lib/address/point-interpolation.ts} +24 -22
- package/{address-point-schema.ts → lib/address/point-schema.ts} +18 -4
- package/{address-point.ts → lib/address/point.ts} +89 -27
- package/{ancestry-backfill.ts → lib/ancestry/backfill.ts} +14 -19
- package/{ancestry.ts → lib/ancestry/index.ts} +5 -3
- package/{build-candidate.ts → lib/build-candidate.ts} +84 -77
- package/{build-slim.ts → lib/build-slim.ts} +150 -165
- package/{candidate → lib/candidate}/alias-bags.ts +8 -6
- package/{candidate → lib/candidate}/ancestors-sidecar.ts +13 -15
- package/{candidate → lib/candidate}/country-display-names.ts +2 -2
- package/{candidate/shard-fold.ts → lib/candidate/extract-fold.ts} +26 -25
- package/{candidate → lib/candidate}/name-roles.ts +17 -16
- package/{candidate → lib/candidate}/own-name.ts +1 -1
- package/{candidate → lib/candidate}/place-attrs.ts +3 -3
- package/{candidate-ancestors-schema.ts → lib/candidate-ancestors-schema.ts} +4 -4
- package/{candidate-fts.ts → lib/candidate-fts.ts} +3 -3
- package/{candidate-importance.ts → lib/candidate-importance.ts} +53 -57
- package/{candidate-lookup.ts → lib/candidate-lookup.ts} +76 -73
- package/{candidate-schema.ts → lib/candidate-schema.ts} +5 -5
- package/{candidate-scoring.ts → lib/candidate-scoring.ts} +15 -22
- package/{capital-schema.ts → lib/capital-schema.ts} +5 -6
- package/{capitals.ts → lib/capitals.ts} +2 -2
- package/{coincident-roles.ts → lib/coincident-roles.ts} +8 -11
- package/{convention.ts → lib/convention/index.ts} +3 -3
- package/{convention-schema.ts → lib/convention/schema.ts} +2 -2
- package/{coverage-manifest-schema.ts → lib/coverage-manifest-schema.ts} +9 -10
- package/{currency-backfill.ts → lib/currency-backfill.ts} +126 -41
- package/lib/env.ts +50 -0
- package/{exact-match.ts → lib/exact-match.ts} +23 -14
- package/{sharding.ts → lib/extracts.ts} +61 -58
- package/{fst-autocomplete.ts → lib/fst/autocomplete.ts} +4 -4
- package/{fst-builder.ts → lib/fst/builder.ts} +17 -16
- package/{fst-deserialize-web.ts → lib/fst/deserialize-web.ts} +27 -87
- package/lib/fst/format.ts +106 -0
- package/{fst-freshness.ts → lib/fst/freshness.ts} +41 -74
- package/lib/fst/index.ts +14 -0
- package/{fst-matcher.ts → lib/fst/matcher.ts} +1 -1
- package/{fst-serialize.ts → lib/fst/serialize.ts} +37 -115
- package/{fst-types.ts → lib/fst/types.ts} +4 -2
- package/{fts.ts → lib/fts/index.ts} +24 -19
- package/{fts-query.ts → lib/fts/query.ts} +1 -1
- package/{geonames-aliases.ts → lib/geonames/aliases.ts} +13 -13
- package/lib/geonames/index.ts +8 -0
- package/{geonames-postal.ts → lib/geonames/postal.ts} +14 -13
- package/{index.ts → lib/index.ts} +37 -37
- package/{interpolation.ts → lib/interpolation.ts} +27 -25
- package/{lookup.ts → lib/lookup.ts} +129 -141
- package/lib/nsul/index.ts +8 -0
- package/lib/nsul/lookup.ts +144 -0
- package/lib/nsul/schema.ts +130 -0
- package/{place-importance-schema.ts → lib/place-importance-schema.ts} +7 -8
- package/lib/poi/index.ts +8 -0
- package/{poi-lookup.ts → lib/poi/lookup.ts} +22 -28
- package/{poi-schema.ts → lib/poi/schema.ts} +3 -4
- package/{postal-city-alias-lookup.ts → lib/postal/city-alias-lookup.ts} +14 -20
- package/{postal-city-candidate-schema.ts → lib/postal/city-candidate-schema.ts} +2 -2
- package/lib/postal/index.ts +9 -0
- package/{postcode-point-lookup.ts → lib/postcode-point-lookup.ts} +17 -15
- package/{primary-preference.ts → lib/primary-preference.ts} +29 -10
- package/{proximity-rerank.ts → lib/proximity-rerank.ts} +2 -2
- package/{ranking-weights.ts → lib/ranking-weights.ts} +30 -4
- package/{region-keys.ts → lib/region-keys.ts} +9 -9
- package/{reverse.ts → lib/reverse.ts} +34 -46
- package/{schema.ts → lib/schema.ts} +10 -0
- package/{search-fetch.ts → lib/search-fetch.ts} +29 -28
- package/{sqlite-convention-source.ts → lib/sqlite-convention-source.ts} +9 -10
- package/{sqlite-utils.ts → lib/sqlite-utils.ts} +14 -18
- package/{street-centroid-schema.ts → lib/street/centroid-schema.ts} +5 -5
- package/{street-centroid.ts → lib/street/centroid.ts} +14 -21
- package/lib/street/index.ts +13 -0
- package/{street-morphology-fst-builder.ts → lib/street/morphology-fst-builder.ts} +24 -14
- package/{street-morphology-fst-loader.ts → lib/street/morphology-fst-loader.ts} +14 -12
- package/{street-name-lookup.ts → lib/street/name-lookup.ts} +14 -30
- package/{street-normalize.ts → lib/street/normalize.ts} +44 -19
- package/{street-segment-schema.ts → lib/street/segment-schema.ts} +7 -7
- package/{types.ts → lib/types.ts} +18 -12
- package/{unified-schema.ts → lib/unified-schema.ts} +20 -24
- package/lib/uprn/existence.ts +84 -0
- package/lib/uprn/index.ts +9 -0
- package/{uprn-lookup.ts → lib/uprn/lookup.ts} +14 -18
- package/{uprn-schema.ts → lib/uprn/schema.ts} +3 -3
- package/lib/weights-overlay-linker.ts +878 -0
- package/out/address/index.d.ts +9 -0
- package/out/address/index.d.ts.map +1 -0
- package/out/address/index.js +9 -0
- package/out/address/index.js.map +1 -0
- package/out/{address-point-interpolation.d.ts → address/point-interpolation.d.ts} +11 -10
- package/out/address/point-interpolation.d.ts.map +1 -0
- package/out/{address-point-interpolation.js → address/point-interpolation.js} +18 -18
- package/out/address/point-interpolation.js.map +1 -0
- package/out/{address-point-schema.d.ts → address/point-schema.d.ts} +16 -6
- package/out/address/point-schema.d.ts.map +1 -0
- package/out/{address-point-schema.js → address/point-schema.js} +7 -3
- package/out/address/point-schema.js.map +1 -0
- package/out/{address-point.d.ts → address/point.d.ts} +16 -8
- package/out/address/point.d.ts.map +1 -0
- package/out/{address-point.js → address/point.js} +71 -20
- package/out/address/point.js.map +1 -0
- package/out/{ancestry-backfill.d.ts → ancestry/backfill.d.ts} +7 -5
- package/out/ancestry/backfill.d.ts.map +1 -0
- package/out/{ancestry-backfill.js → ancestry/backfill.js} +12 -16
- package/out/ancestry/backfill.js.map +1 -0
- package/out/{ancestry.d.ts → ancestry/index.d.ts} +4 -3
- package/out/ancestry/index.d.ts.map +1 -0
- package/out/{ancestry.js → ancestry/index.js} +3 -2
- package/out/ancestry/index.js.map +1 -0
- package/out/build-candidate.d.ts +30 -24
- package/out/build-candidate.d.ts.map +1 -1
- package/out/build-candidate.js +50 -52
- package/out/build-candidate.js.map +1 -1
- package/out/build-slim.d.ts +4 -4
- package/out/build-slim.d.ts.map +1 -1
- package/out/build-slim.js +124 -139
- package/out/build-slim.js.map +1 -1
- package/out/candidate/alias-bags.d.ts +5 -3
- package/out/candidate/alias-bags.d.ts.map +1 -1
- package/out/candidate/alias-bags.js +2 -2
- package/out/candidate/alias-bags.js.map +1 -1
- package/out/candidate/ancestors-sidecar.d.ts +9 -10
- package/out/candidate/ancestors-sidecar.d.ts.map +1 -1
- package/out/candidate/ancestors-sidecar.js +7 -7
- package/out/candidate/ancestors-sidecar.js.map +1 -1
- package/out/candidate/country-display-names.d.ts +1 -1
- package/out/candidate/country-display-names.d.ts.map +1 -1
- package/out/candidate/country-display-names.js +1 -1
- package/out/candidate/country-display-names.js.map +1 -1
- package/out/candidate/extract-fold.d.ts +32 -0
- package/out/candidate/extract-fold.d.ts.map +1 -0
- package/out/candidate/{shard-fold.js → extract-fold.js} +20 -20
- package/out/candidate/extract-fold.js.map +1 -0
- package/out/candidate/name-roles.d.ts +13 -11
- package/out/candidate/name-roles.d.ts.map +1 -1
- package/out/candidate/name-roles.js +11 -10
- package/out/candidate/name-roles.js.map +1 -1
- package/out/candidate/own-name.d.ts +1 -1
- package/out/candidate/own-name.d.ts.map +1 -1
- package/out/candidate/own-name.js +1 -1
- package/out/candidate/own-name.js.map +1 -1
- package/out/candidate/place-attrs.d.ts +3 -3
- package/out/candidate/place-attrs.d.ts.map +1 -1
- package/out/candidate/place-attrs.js +1 -1
- package/out/candidate/place-attrs.js.map +1 -1
- package/out/candidate-ancestors-schema.d.ts +4 -4
- package/out/candidate-ancestors-schema.d.ts.map +1 -1
- package/out/candidate-ancestors-schema.js +2 -2
- package/out/candidate-ancestors-schema.js.map +1 -1
- package/out/candidate-fts.d.ts +3 -3
- package/out/candidate-fts.d.ts.map +1 -1
- package/out/candidate-fts.js +1 -1
- package/out/candidate-fts.js.map +1 -1
- package/out/candidate-importance.d.ts +16 -16
- package/out/candidate-importance.d.ts.map +1 -1
- package/out/candidate-importance.js +46 -51
- package/out/candidate-importance.js.map +1 -1
- package/out/candidate-lookup.d.ts +11 -9
- package/out/candidate-lookup.d.ts.map +1 -1
- package/out/candidate-lookup.js +61 -58
- package/out/candidate-lookup.js.map +1 -1
- package/out/candidate-schema.d.ts +5 -5
- package/out/candidate-schema.d.ts.map +1 -1
- package/out/candidate-schema.js.map +1 -1
- package/out/candidate-scoring.d.ts +7 -7
- package/out/candidate-scoring.d.ts.map +1 -1
- package/out/candidate-scoring.js +10 -14
- package/out/candidate-scoring.js.map +1 -1
- package/out/capital-schema.d.ts +3 -3
- package/out/capital-schema.d.ts.map +1 -1
- package/out/capital-schema.js +2 -2
- package/out/capital-schema.js.map +1 -1
- package/out/capitals.d.ts +1 -1
- package/out/capitals.d.ts.map +1 -1
- package/out/capitals.js +2 -2
- package/out/capitals.js.map +1 -1
- package/out/coincident-roles.d.ts +6 -5
- package/out/coincident-roles.d.ts.map +1 -1
- package/out/coincident-roles.js +2 -4
- package/out/coincident-roles.js.map +1 -1
- package/out/{convention.d.ts → convention/index.d.ts} +4 -4
- package/out/convention/index.d.ts.map +1 -0
- package/out/{convention.js → convention/index.js} +2 -2
- package/out/convention/index.js.map +1 -0
- package/out/{convention-schema.d.ts → convention/schema.d.ts} +2 -2
- package/out/convention/schema.d.ts.map +1 -0
- package/out/{convention-schema.js → convention/schema.js} +3 -3
- package/out/convention/schema.js.map +1 -0
- package/out/coverage-manifest-schema.d.ts +8 -8
- package/out/coverage-manifest-schema.d.ts.map +1 -1
- package/out/coverage-manifest-schema.js +4 -4
- package/out/coverage-manifest-schema.js.map +1 -1
- package/out/currency-backfill.d.ts +51 -7
- package/out/currency-backfill.d.ts.map +1 -1
- package/out/currency-backfill.js +65 -49
- package/out/currency-backfill.js.map +1 -1
- package/out/env.d.ts +43 -0
- package/out/env.d.ts.map +1 -0
- package/out/env.js +48 -0
- package/out/env.js.map +1 -0
- package/out/exact-match.d.ts +5 -5
- package/out/exact-match.d.ts.map +1 -1
- package/out/exact-match.js +10 -11
- package/out/exact-match.js.map +1 -1
- package/out/extracts.d.ts +114 -0
- package/out/extracts.d.ts.map +1 -0
- package/out/{sharding.js → extracts.js} +38 -38
- package/out/extracts.js.map +1 -0
- package/out/{fst-autocomplete.d.ts → fst/autocomplete.d.ts} +2 -2
- package/out/fst/autocomplete.d.ts.map +1 -0
- package/out/{fst-autocomplete.js → fst/autocomplete.js} +3 -3
- package/out/fst/autocomplete.js.map +1 -0
- package/out/{fst-builder.d.ts → fst/builder.d.ts} +5 -5
- package/out/fst/builder.d.ts.map +1 -0
- package/out/{fst-builder.js → fst/builder.js} +14 -13
- package/out/fst/builder.js.map +1 -0
- package/out/{fst-deserialize-web.d.ts → fst/deserialize-web.d.ts} +3 -3
- package/out/fst/deserialize-web.d.ts.map +1 -0
- package/out/{fst-deserialize-web.js → fst/deserialize-web.js} +10 -74
- package/out/fst/deserialize-web.js.map +1 -0
- package/out/fst/format.d.ts +80 -0
- package/out/fst/format.d.ts.map +1 -0
- package/out/fst/format.js +91 -0
- package/out/fst/format.js.map +1 -0
- package/out/{fst-freshness.d.ts → fst/freshness.d.ts} +10 -13
- package/out/fst/freshness.d.ts.map +1 -0
- package/out/{fst-freshness.js → fst/freshness.js} +41 -73
- package/out/fst/freshness.js.map +1 -0
- package/out/fst/index.d.ts +14 -0
- package/out/fst/index.d.ts.map +1 -0
- package/out/fst/index.js +14 -0
- package/out/fst/index.js.map +1 -0
- package/out/{fst-matcher.d.ts → fst/matcher.d.ts} +2 -2
- package/out/fst/matcher.d.ts.map +1 -0
- package/out/{fst-matcher.js → fst/matcher.js} +1 -1
- package/out/fst/matcher.js.map +1 -0
- package/out/{fst-serialize.d.ts → fst/serialize.d.ts} +5 -10
- package/out/fst/serialize.d.ts.map +1 -0
- package/out/{fst-serialize.js → fst/serialize.js} +18 -98
- package/out/fst/serialize.js.map +1 -0
- package/out/{fst-types.d.ts → fst/types.d.ts} +4 -3
- package/out/fst/types.d.ts.map +1 -0
- package/out/{fst-types.js → fst/types.js} +1 -1
- package/out/fst/types.js.map +1 -0
- package/out/{fts.d.ts → fts/index.d.ts} +16 -9
- package/out/fts/index.d.ts.map +1 -0
- package/out/{fts.js → fts/index.js} +16 -11
- package/out/fts/index.js.map +1 -0
- package/out/{fts-query.d.ts → fts/query.d.ts} +2 -2
- package/out/fts/query.d.ts.map +1 -0
- package/out/{fts-query.js → fts/query.js} +1 -1
- package/out/fts/query.js.map +1 -0
- package/out/{geonames-aliases.d.ts → geonames/aliases.d.ts} +8 -6
- package/out/geonames/aliases.d.ts.map +1 -0
- package/out/{geonames-aliases.js → geonames/aliases.js} +7 -7
- package/out/geonames/aliases.js.map +1 -0
- package/out/geonames/index.d.ts +8 -0
- package/out/geonames/index.d.ts.map +1 -0
- package/out/geonames/index.js +8 -0
- package/out/geonames/index.js.map +1 -0
- package/out/{geonames-postal.d.ts → geonames/postal.d.ts} +10 -8
- package/out/geonames/postal.d.ts.map +1 -0
- package/out/{geonames-postal.js → geonames/postal.js} +9 -9
- package/out/geonames/postal.js.map +1 -0
- package/out/index.d.ts +32 -32
- package/out/index.d.ts.map +1 -1
- package/out/index.js +24 -24
- package/out/index.js.map +1 -1
- package/out/interpolation.d.ts +10 -9
- package/out/interpolation.d.ts.map +1 -1
- package/out/interpolation.js +20 -20
- package/out/interpolation.js.map +1 -1
- package/out/lookup.d.ts +21 -20
- package/out/lookup.d.ts.map +1 -1
- package/out/lookup.js +102 -112
- package/out/lookup.js.map +1 -1
- package/out/name-score.d.ts.map +1 -1
- package/out/name-score.js.map +1 -1
- package/out/nsul/index.d.ts +8 -0
- package/out/nsul/index.d.ts.map +1 -0
- package/out/nsul/index.js +8 -0
- package/out/nsul/index.js.map +1 -0
- package/out/nsul/lookup.d.ts +83 -0
- package/out/nsul/lookup.d.ts.map +1 -0
- package/out/nsul/lookup.js +80 -0
- package/out/nsul/lookup.js.map +1 -0
- package/out/nsul/schema.d.ts +100 -0
- package/out/nsul/schema.d.ts.map +1 -0
- package/out/nsul/schema.js +78 -0
- package/out/nsul/schema.js.map +1 -0
- package/out/place-importance-schema.d.ts +5 -5
- package/out/place-importance-schema.d.ts.map +1 -1
- package/out/place-importance-schema.js +3 -3
- package/out/place-importance-schema.js.map +1 -1
- package/out/poi/index.d.ts +8 -0
- package/out/poi/index.d.ts.map +1 -0
- package/out/poi/index.js +8 -0
- package/out/poi/index.js.map +1 -0
- package/out/{poi-lookup.d.ts → poi/lookup.d.ts} +8 -10
- package/out/poi/lookup.d.ts.map +1 -0
- package/out/{poi-lookup.js → poi/lookup.js} +14 -18
- package/out/poi/lookup.js.map +1 -0
- package/out/{poi-schema.d.ts → poi/schema.d.ts} +4 -4
- package/out/poi/schema.d.ts.map +1 -0
- package/out/{poi-schema.js → poi/schema.js} +1 -1
- package/out/poi/schema.js.map +1 -0
- package/out/polygon-schema.d.ts.map +1 -1
- package/out/polygon-schema.js.map +1 -1
- package/out/{postal-city-alias-lookup.d.ts → postal/city-alias-lookup.d.ts} +6 -5
- package/out/postal/city-alias-lookup.d.ts.map +1 -0
- package/out/{postal-city-alias-lookup.js → postal/city-alias-lookup.js} +11 -15
- package/out/postal/city-alias-lookup.js.map +1 -0
- package/out/{postal-city-alias-schema.d.ts → postal/city-alias-schema.d.ts} +1 -1
- package/out/{postal-city-alias-schema.d.ts.map → postal/city-alias-schema.d.ts.map} +1 -1
- package/out/{postal-city-alias-schema.js → postal/city-alias-schema.js} +1 -1
- package/out/{postal-city-alias-schema.js.map → postal/city-alias-schema.js.map} +1 -1
- package/out/{postal-city-candidate-schema.d.ts → postal/city-candidate-schema.d.ts} +3 -3
- package/out/{postal-city-candidate-schema.d.ts.map → postal/city-candidate-schema.d.ts.map} +1 -1
- package/out/{postal-city-candidate-schema.js → postal/city-candidate-schema.js} +2 -2
- package/out/{postal-city-candidate-schema.js.map → postal/city-candidate-schema.js.map} +1 -1
- package/out/postal/index.d.ts +9 -0
- package/out/postal/index.d.ts.map +1 -0
- package/out/postal/index.js +9 -0
- package/out/postal/index.js.map +1 -0
- package/out/postcode-point-lookup.d.ts +10 -10
- package/out/postcode-point-lookup.d.ts.map +1 -1
- package/out/postcode-point-lookup.js +13 -13
- package/out/postcode-point-lookup.js.map +1 -1
- package/out/primary-preference.d.ts +13 -5
- package/out/primary-preference.d.ts.map +1 -1
- package/out/primary-preference.js +16 -8
- package/out/primary-preference.js.map +1 -1
- package/out/proximity-rerank.d.ts +2 -2
- package/out/proximity-rerank.d.ts.map +1 -1
- package/out/proximity-rerank.js +2 -2
- package/out/proximity-rerank.js.map +1 -1
- package/out/ranking-weights.d.ts +15 -3
- package/out/ranking-weights.d.ts.map +1 -1
- package/out/ranking-weights.js +21 -3
- package/out/ranking-weights.js.map +1 -1
- package/out/region-keys.d.ts +8 -8
- package/out/region-keys.d.ts.map +1 -1
- package/out/region-keys.js +9 -9
- package/out/region-keys.js.map +1 -1
- package/out/reverse.d.ts +6 -6
- package/out/reverse.d.ts.map +1 -1
- package/out/reverse.js +20 -29
- package/out/reverse.js.map +1 -1
- package/out/schema.d.ts +9 -0
- package/out/schema.d.ts.map +1 -1
- package/out/schema.js.map +1 -1
- package/out/search-fetch.d.ts +12 -10
- package/out/search-fetch.d.ts.map +1 -1
- package/out/search-fetch.js +21 -20
- package/out/search-fetch.js.map +1 -1
- package/out/sqlite-convention-source.d.ts +7 -7
- package/out/sqlite-convention-source.d.ts.map +1 -1
- package/out/sqlite-convention-source.js +5 -5
- package/out/sqlite-convention-source.js.map +1 -1
- package/out/sqlite-utils.d.ts +8 -8
- package/out/sqlite-utils.d.ts.map +1 -1
- package/out/sqlite-utils.js +7 -8
- package/out/sqlite-utils.js.map +1 -1
- package/out/{street-centroid-schema.d.ts → street/centroid-schema.d.ts} +6 -6
- package/out/{street-centroid-schema.d.ts.map → street/centroid-schema.d.ts.map} +1 -1
- package/out/{street-centroid-schema.js → street/centroid-schema.js} +5 -5
- package/out/{street-centroid-schema.js.map → street/centroid-schema.js.map} +1 -1
- package/out/{street-centroid.d.ts → street/centroid.d.ts} +6 -6
- package/out/street/centroid.d.ts.map +1 -0
- package/out/{street-centroid.js → street/centroid.js} +13 -19
- package/out/street/centroid.js.map +1 -0
- package/out/street/index.d.ts +13 -0
- package/out/street/index.d.ts.map +1 -0
- package/out/street/index.js +13 -0
- package/out/street/index.js.map +1 -0
- package/out/{street-morphology-fst-builder.d.ts → street/morphology-fst-builder.d.ts} +4 -4
- package/out/street/morphology-fst-builder.d.ts.map +1 -0
- package/out/{street-morphology-fst-builder.js → street/morphology-fst-builder.js} +18 -13
- package/out/street/morphology-fst-builder.js.map +1 -0
- package/out/{street-morphology-fst-loader.d.ts → street/morphology-fst-loader.d.ts} +5 -5
- package/out/street/morphology-fst-loader.d.ts.map +1 -0
- package/out/{street-morphology-fst-loader.js → street/morphology-fst-loader.js} +11 -10
- package/out/street/morphology-fst-loader.js.map +1 -0
- package/out/{street-name-lookup.d.ts → street/name-lookup.d.ts} +4 -7
- package/out/street/name-lookup.d.ts.map +1 -0
- package/out/{street-name-lookup.js → street/name-lookup.js} +9 -23
- package/out/street/name-lookup.js.map +1 -0
- package/out/{street-normalize.d.ts → street/normalize.d.ts} +25 -14
- package/out/street/normalize.d.ts.map +1 -0
- package/out/{street-normalize.js → street/normalize.js} +36 -17
- package/out/street/normalize.js.map +1 -0
- package/out/{street-segment-schema.d.ts → street/segment-schema.d.ts} +8 -8
- package/out/{street-segment-schema.d.ts.map → street/segment-schema.d.ts.map} +1 -1
- package/out/{street-segment-schema.js → street/segment-schema.js} +4 -4
- package/out/{street-segment-schema.js.map → street/segment-schema.js.map} +1 -1
- package/out/types.d.ts +18 -12
- package/out/types.d.ts.map +1 -1
- package/out/types.js.map +1 -1
- package/out/unified-schema.d.ts +5 -4
- package/out/unified-schema.d.ts.map +1 -1
- package/out/unified-schema.js +15 -18
- package/out/unified-schema.js.map +1 -1
- package/out/uprn/existence.d.ts +53 -0
- package/out/uprn/existence.d.ts.map +1 -0
- package/out/uprn/existence.js +61 -0
- package/out/uprn/existence.js.map +1 -0
- package/out/uprn/index.d.ts +9 -0
- package/out/uprn/index.d.ts.map +1 -0
- package/out/uprn/index.js +9 -0
- package/out/uprn/index.js.map +1 -0
- package/out/{uprn-lookup.d.ts → uprn/lookup.d.ts} +4 -4
- package/out/uprn/lookup.d.ts.map +1 -0
- package/out/{uprn-lookup.js → uprn/lookup.js} +11 -14
- package/out/uprn/lookup.js.map +1 -0
- package/out/{uprn-schema.d.ts → uprn/schema.d.ts} +4 -4
- package/out/uprn/schema.d.ts.map +1 -0
- package/out/{uprn-schema.js → uprn/schema.js} +4 -4
- package/out/uprn/schema.js.map +1 -0
- package/out/weights-overlay-linker.d.ts +184 -26
- package/out/weights-overlay-linker.d.ts.map +1 -1
- package/out/weights-overlay-linker.js +338 -86
- package/out/weights-overlay-linker.js.map +1 -1
- package/package.json +465 -269
- package/out/address-point-interpolation.d.ts.map +0 -1
- package/out/address-point-interpolation.js.map +0 -1
- package/out/address-point-schema.d.ts.map +0 -1
- package/out/address-point-schema.js.map +0 -1
- package/out/address-point.d.ts.map +0 -1
- package/out/address-point.js.map +0 -1
- package/out/ancestry-backfill.d.ts.map +0 -1
- package/out/ancestry-backfill.js.map +0 -1
- package/out/ancestry.d.ts.map +0 -1
- package/out/ancestry.js.map +0 -1
- package/out/candidate/shard-fold.d.ts +0 -31
- package/out/candidate/shard-fold.d.ts.map +0 -1
- package/out/candidate/shard-fold.js.map +0 -1
- package/out/convention-schema.d.ts.map +0 -1
- package/out/convention-schema.js.map +0 -1
- package/out/convention.d.ts.map +0 -1
- package/out/convention.js.map +0 -1
- package/out/fst-autocomplete.d.ts.map +0 -1
- package/out/fst-autocomplete.js.map +0 -1
- package/out/fst-builder.d.ts.map +0 -1
- package/out/fst-builder.js.map +0 -1
- package/out/fst-deserialize-web.d.ts.map +0 -1
- package/out/fst-deserialize-web.js.map +0 -1
- package/out/fst-freshness.d.ts.map +0 -1
- package/out/fst-freshness.js.map +0 -1
- package/out/fst-matcher.d.ts.map +0 -1
- package/out/fst-matcher.js.map +0 -1
- package/out/fst-serialize.d.ts.map +0 -1
- package/out/fst-serialize.js.map +0 -1
- package/out/fst-types.d.ts.map +0 -1
- package/out/fst-types.js.map +0 -1
- package/out/fts-query.d.ts.map +0 -1
- package/out/fts-query.js.map +0 -1
- package/out/fts.d.ts.map +0 -1
- package/out/fts.js.map +0 -1
- package/out/geonames-aliases.d.ts.map +0 -1
- package/out/geonames-aliases.js.map +0 -1
- package/out/geonames-postal.d.ts.map +0 -1
- package/out/geonames-postal.js.map +0 -1
- package/out/poi-lookup.d.ts.map +0 -1
- package/out/poi-lookup.js.map +0 -1
- package/out/poi-schema.d.ts.map +0 -1
- package/out/poi-schema.js.map +0 -1
- package/out/postal-city-alias-lookup.d.ts.map +0 -1
- package/out/postal-city-alias-lookup.js.map +0 -1
- package/out/sharding.d.ts +0 -114
- package/out/sharding.d.ts.map +0 -1
- package/out/sharding.js.map +0 -1
- package/out/street-centroid.d.ts.map +0 -1
- package/out/street-centroid.js.map +0 -1
- package/out/street-morphology-fst-builder.d.ts.map +0 -1
- package/out/street-morphology-fst-builder.js.map +0 -1
- package/out/street-morphology-fst-loader.d.ts.map +0 -1
- package/out/street-morphology-fst-loader.js.map +0 -1
- package/out/street-name-lookup.d.ts.map +0 -1
- package/out/street-name-lookup.js.map +0 -1
- package/out/street-normalize.d.ts.map +0 -1
- package/out/street-normalize.js.map +0 -1
- package/out/uprn-lookup.d.ts.map +0 -1
- package/out/uprn-lookup.js.map +0 -1
- package/out/uprn-schema.d.ts.map +0 -1
- package/out/uprn-schema.js.map +0 -1
- package/weights-overlay-linker.ts +0 -377
- /package/{name-score.ts → lib/name-score.ts} +0 -0
- /package/{polygon-schema.ts → lib/polygon-schema.ts} +0 -0
- /package/{postal-city-alias-schema.ts → lib/postal/city-alias-schema.ts} +0 -0
|
@@ -29,13 +29,14 @@
|
|
|
29
29
|
* and continue.
|
|
30
30
|
*/
|
|
31
31
|
|
|
32
|
-
import {
|
|
33
|
-
import {
|
|
32
|
+
import { pathExists, readFileRange, readLocalTextFile, statPath } from "@mailwoman/core/fs/readers"
|
|
33
|
+
import { writeLocalTextFile } from "@mailwoman/core/fs/writers"
|
|
34
|
+
import { md5File } from "@mailwoman/core/hash"
|
|
35
|
+
import { tryParsingJSON } from "@mailwoman/core/json"
|
|
36
|
+
import { basename, resolvePath, type PathBuilderLike } from "path-ts"
|
|
34
37
|
|
|
35
|
-
import {
|
|
36
|
-
|
|
37
|
-
import { FST_FORMAT_VERSION } from "./fst-serialize.ts"
|
|
38
|
-
import type { FSTProvenance } from "./fst-types.ts"
|
|
38
|
+
import { FST_FORMAT_VERSION } from "#fst/serialize"
|
|
39
|
+
import type { FSTProvenance } from "#fst/types"
|
|
39
40
|
|
|
40
41
|
/**
|
|
41
42
|
* Fixed header size in bytes — mirrors `fst-serialize.ts`'s `HEADER_SIZE`. Duplicated rather than exported across
|
|
@@ -65,12 +66,6 @@ export const MIN_STAMPED_FORMAT_VERSION = 3
|
|
|
65
66
|
*/
|
|
66
67
|
const MD5_HEX_LENGTH = 32
|
|
67
68
|
|
|
68
|
-
/**
|
|
69
|
-
* Bytes per read in {@link md5FileSync}. 8 MiB measured at 7.3 s for the 5.27 GB admin DB on the lab playpen — the same
|
|
70
|
-
* throughput as `md5sum(1)`, so the sidecar below is what makes the check cheap, not the chunk size.
|
|
71
|
-
*/
|
|
72
|
-
const MD5_CHUNK_BYTES = 8 * 1024 * 1024
|
|
73
|
-
|
|
74
69
|
/**
|
|
75
70
|
* What an FST was built from, recorded so a later reader can tell whether that thing still exists.
|
|
76
71
|
*
|
|
@@ -121,69 +116,37 @@ export interface FSTExpectation {
|
|
|
121
116
|
* Returns `undefined` for a file that is absent, too small, or not an FST at all — none of which is this function's
|
|
122
117
|
* business to diagnose.
|
|
123
118
|
*/
|
|
124
|
-
export function peekFSTStampFields(path: string): FSTStampFields | undefined {
|
|
125
|
-
if (!
|
|
126
|
-
const size =
|
|
119
|
+
export async function peekFSTStampFields(path: string): Promise<FSTStampFields | undefined> {
|
|
120
|
+
if (!(await pathExists(path))) return undefined
|
|
121
|
+
const size = (await statPath(path)).size
|
|
127
122
|
|
|
128
123
|
if (size < HEADER_SIZE) return undefined
|
|
129
|
-
const
|
|
124
|
+
const header = await readFileRange(path, 0, HEADER_SIZE)
|
|
130
125
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
readSync(fd, header, 0, HEADER_SIZE, 0)
|
|
126
|
+
if (header.readUInt32LE(0) !== MAGIC) return undefined
|
|
127
|
+
const formatVersion = header.readUInt16LE(4)
|
|
134
128
|
|
|
135
|
-
|
|
136
|
-
|
|
129
|
+
if (formatVersion < MIN_STAMPED_FORMAT_VERSION) return { formatVersion, provenance: undefined }
|
|
130
|
+
const trailerStart = header.readUInt32LE(PROVENANCE_OFFSET_FIELD)
|
|
137
131
|
|
|
138
|
-
|
|
139
|
-
|
|
132
|
+
// 0 = "this build wrote no trailer" (the serializer's own encoding); anything past EOF is a
|
|
133
|
+
// truncated file. Both read as "no stamp", which is what the caller does with them anyway.
|
|
134
|
+
if (trailerStart === 0 || trailerStart + 4 > size) return { formatVersion, provenance: undefined }
|
|
140
135
|
|
|
141
|
-
|
|
142
|
-
// truncated file. Both read as "no stamp", which is what the caller does with them anyway.
|
|
143
|
-
if (trailerStart === 0 || trailerStart + 4 > size) return { formatVersion, provenance: undefined }
|
|
136
|
+
const jsonLength = (await readFileRange(path, trailerStart, 4)).readUInt32LE(0)
|
|
144
137
|
|
|
145
|
-
|
|
146
|
-
readSync(fd, lengthBytes, 0, 4, trailerStart)
|
|
147
|
-
const jsonLength = lengthBytes.readUInt32LE(0)
|
|
138
|
+
if (jsonLength === 0 || trailerStart + 4 + jsonLength > size) return { formatVersion, provenance: undefined }
|
|
148
139
|
|
|
149
|
-
|
|
140
|
+
const json = await readFileRange(path, trailerStart + 4, jsonLength)
|
|
150
141
|
|
|
151
|
-
|
|
152
|
-
readSync(fd, json, 0, jsonLength, trailerStart + 4)
|
|
153
|
-
|
|
154
|
-
return { formatVersion, provenance: tryParsingJSON<FSTProvenance>(json.toString("utf8")) ?? undefined }
|
|
155
|
-
} finally {
|
|
156
|
-
closeSync(fd)
|
|
157
|
-
}
|
|
142
|
+
return { formatVersion, provenance: tryParsingJSON<FSTProvenance>(json.toString("utf8")) ?? undefined }
|
|
158
143
|
}
|
|
159
144
|
|
|
160
145
|
/**
|
|
161
|
-
*
|
|
162
|
-
*
|
|
163
|
-
* The async `md5File` in `@mailwoman/core/utils` is the one to reach for anywhere else. This exists because the FST
|
|
164
|
-
* builder and its whole call chain are synchronous by design (`buildFSTFromWOF` → `buildLocaleFSTs`), and making them
|
|
165
|
-
* async to stamp a checksum would cascade through command callers and tests for one hash. It reads in
|
|
166
|
-
* {@link MD5_CHUNK_BYTES} chunks rather than `readFileSync` — the source is a multi-gigabyte database.
|
|
146
|
+
* Re-exported from `@mailwoman/core/hash`. This package's own test imports it from here, and the sidecar convention
|
|
147
|
+
* below is what it is for.
|
|
167
148
|
*/
|
|
168
|
-
export
|
|
169
|
-
const hash = createHash("md5")
|
|
170
|
-
const fd = openSync(path, "r")
|
|
171
|
-
|
|
172
|
-
try {
|
|
173
|
-
const chunk = Buffer.alloc(MD5_CHUNK_BYTES)
|
|
174
|
-
|
|
175
|
-
for (;;) {
|
|
176
|
-
const read = readSync(fd, chunk, 0, MD5_CHUNK_BYTES, null)
|
|
177
|
-
|
|
178
|
-
if (read <= 0) break
|
|
179
|
-
hash.update(chunk.subarray(0, read))
|
|
180
|
-
}
|
|
181
|
-
} finally {
|
|
182
|
-
closeSync(fd)
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
return hash.digest("hex")
|
|
186
|
-
}
|
|
149
|
+
export { md5File } from "@mailwoman/core/hash"
|
|
187
150
|
|
|
188
151
|
/**
|
|
189
152
|
* The source identity an FST build should stamp, or a check should compare against.
|
|
@@ -197,8 +160,12 @@ export function md5FileSync(path: string): string {
|
|
|
197
160
|
* write and the caller still gets its answer, because refusing to check freshness on a read-only tree would be the
|
|
198
161
|
* wrong trade.
|
|
199
162
|
*/
|
|
200
|
-
export function readWOFSourceIdentity(
|
|
201
|
-
|
|
163
|
+
export async function readWOFSourceIdentity(
|
|
164
|
+
source: PathBuilderLike,
|
|
165
|
+
{ refreshSidecar = true } = {}
|
|
166
|
+
): Promise<FSTSourceIdentity> {
|
|
167
|
+
const path = resolvePath(source)
|
|
168
|
+
const stats = await statPath(path)
|
|
202
169
|
const memoKey = `${path}\0${stats.mtimeMs}\0${stats.size}`
|
|
203
170
|
const hit = sourceIdentityMemo.get(memoKey)
|
|
204
171
|
|
|
@@ -206,11 +173,11 @@ export function readWOFSourceIdentity(path: string, { refreshSidecar = true } =
|
|
|
206
173
|
const sidecarPath = `${path}.md5`
|
|
207
174
|
let md5: string | undefined
|
|
208
175
|
|
|
209
|
-
if (
|
|
210
|
-
const sidecarStats =
|
|
176
|
+
if (await pathExists(sidecarPath)) {
|
|
177
|
+
const sidecarStats = await statPath(sidecarPath)
|
|
211
178
|
|
|
212
179
|
if (sidecarStats.mtimeMs >= stats.mtimeMs) {
|
|
213
|
-
const [hash] =
|
|
180
|
+
const [hash] = (await readLocalTextFile(sidecarPath)).trim().split(/\s+/)
|
|
214
181
|
|
|
215
182
|
if (hash && hash.length === MD5_HEX_LENGTH) {
|
|
216
183
|
md5 = hash
|
|
@@ -219,11 +186,11 @@ export function readWOFSourceIdentity(path: string, { refreshSidecar = true } =
|
|
|
219
186
|
}
|
|
220
187
|
|
|
221
188
|
if (!md5) {
|
|
222
|
-
md5 =
|
|
189
|
+
md5 = await md5File(path)
|
|
223
190
|
|
|
224
191
|
if (refreshSidecar) {
|
|
225
192
|
try {
|
|
226
|
-
|
|
193
|
+
await writeLocalTextFile(`${md5} ${basename(path)}\n`, sidecarPath)
|
|
227
194
|
} catch {
|
|
228
195
|
// Read-only data root — the digest is still correct, it just isn't cached.
|
|
229
196
|
}
|
|
@@ -293,7 +260,7 @@ export function fstStaleReason(fields: FSTStampFields | undefined, expected: FST
|
|
|
293
260
|
* Returns `undefined` when the artifact is current OR when it is absent — an absent artifact is a different problem
|
|
294
261
|
* with a different message, and every existing caller already reports it in place.
|
|
295
262
|
*/
|
|
296
|
-
export function fstFreshnessWarning({
|
|
263
|
+
export async function fstFreshnessWarning({
|
|
297
264
|
fstPath,
|
|
298
265
|
sourceDBPath,
|
|
299
266
|
formatVersion,
|
|
@@ -305,11 +272,11 @@ export function fstFreshnessWarning({
|
|
|
305
272
|
formatVersion?: number
|
|
306
273
|
exclusionPolicy?: string
|
|
307
274
|
rebuildCommand: string
|
|
308
|
-
}): string | undefined {
|
|
309
|
-
if (!
|
|
275
|
+
}): Promise<string | undefined> {
|
|
276
|
+
if (!(await pathExists(fstPath)) || !(await pathExists(sourceDBPath))) return undefined
|
|
310
277
|
|
|
311
|
-
const reason = fstStaleReason(peekFSTStampFields(fstPath), {
|
|
312
|
-
source: readWOFSourceIdentity(sourceDBPath),
|
|
278
|
+
const reason = fstStaleReason(await peekFSTStampFields(fstPath), {
|
|
279
|
+
source: await readWOFSourceIdentity(sourceDBPath),
|
|
313
280
|
...(formatVersion === undefined ? {} : { formatVersion }),
|
|
314
281
|
...(exclusionPolicy === undefined ? {} : { exclusionPolicy }),
|
|
315
282
|
})
|
package/lib/fst/index.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
export * from "#fst/autocomplete"
|
|
8
|
+
export * from "#fst/builder"
|
|
9
|
+
export * from "#fst/deserialize-web"
|
|
10
|
+
export * from "#fst/format"
|
|
11
|
+
export * from "#fst/freshness"
|
|
12
|
+
export * from "#fst/matcher"
|
|
13
|
+
export * from "#fst/serialize"
|
|
14
|
+
export * from "#fst/types"
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* arrays at accepting states.
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
import type { FSTContinuation, FSTMatchResult, FSTQueryResult, PlaceEntry } from "
|
|
11
|
+
import type { FSTContinuation, FSTMatchResult, FSTQueryResult, PlaceEntry } from "#fst/types"
|
|
12
12
|
|
|
13
13
|
interface FSTNode {
|
|
14
14
|
edges: Map<string, number>
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
* EDGE TABLE [edgeCount × 8 bytes] stringIdx u32 index into string table targetState u32
|
|
20
20
|
*
|
|
21
21
|
* PLACE TABLE [placeCount × 60 bytes at V5, 56 below] wofID u32 placetypeIdx u8 index into
|
|
22
|
-
* PLACETYPE_ORDER chainLen u8 0..8 crossCountryBranches u8 (header flags bit0
|
|
22
|
+
* PLACETYPE_ORDER chainLen u8 0..8 crossCountryBranches u8 (header flags bit0 enables the read)
|
|
23
23
|
* placeFlags u8 (V5; bit0 = encyclopedic present) nameIdx u32 index into string table referential f32
|
|
24
24
|
* population-anchored likelihood [0,1] — was the conflated `importance` (V2–V4), was population u32
|
|
25
25
|
* (V1) lat f32 lon f32 chain [u32; 8] parent chain (unused slots = 0) encyclopedic f32 (V5 only;
|
|
@@ -33,97 +33,34 @@
|
|
|
33
33
|
* V4-and-below artifact as format-stale for exactly this reason: its single float is unattributable.
|
|
34
34
|
*/
|
|
35
35
|
|
|
36
|
-
import { tryParsingJSON } from "@mailwoman/core/
|
|
37
|
-
|
|
38
|
-
import
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
* First format version carrying the trailing metadata block; older files simply have none.
|
|
60
|
-
*/
|
|
61
|
-
const VERSION_WITH_METADATA = 3
|
|
36
|
+
import { tryParsingJSON } from "@mailwoman/core/json"
|
|
37
|
+
|
|
38
|
+
import {
|
|
39
|
+
EDGE_ENTRY_SIZE,
|
|
40
|
+
ENCYCLOPEDIC_OFFSET,
|
|
41
|
+
FST_FORMAT_VERSION,
|
|
42
|
+
FST_MAGIC_BYTES,
|
|
43
|
+
HEADER_SIZE,
|
|
44
|
+
LEGACY_PLACE_ENTRY_SIZE,
|
|
45
|
+
NARROW_STATE_ENTRY_SIZE,
|
|
46
|
+
PLACE_FLAG_HAS_ENCYCLOPEDIC,
|
|
47
|
+
PLACETYPE_ORDER,
|
|
48
|
+
SPLIT_PLACE_ENTRY_SIZE,
|
|
49
|
+
VERSION_TWO_SCORE_SPLIT,
|
|
50
|
+
VERSION_WIDE_STATE_COUNTERS,
|
|
51
|
+
VERSION_WITH_METADATA,
|
|
52
|
+
WIDE_STATE_ENTRY_SIZE,
|
|
53
|
+
} from "#fst/format"
|
|
54
|
+
import type { FSTNode } from "#fst/matcher"
|
|
55
|
+
import { FSTMatcher } from "#fst/matcher"
|
|
56
|
+
import type { FSTProvenance, PlaceEntry } from "#fst/types"
|
|
57
|
+
|
|
58
|
+
export { FST_FORMAT_VERSION } from "#fst/format"
|
|
62
59
|
|
|
63
60
|
/**
|
|
64
|
-
*
|
|
65
|
-
* from 56 to 60 bytes and claiming the previously-reserved `pp+7` byte as {@link PLACE_FLAG_HAS_ENCYCLOPEDIC}.
|
|
61
|
+
* File magic as a Buffer, for the Node-side prefix comparison.
|
|
66
62
|
*/
|
|
67
|
-
const
|
|
68
|
-
|
|
69
|
-
/**
|
|
70
|
-
* Place-entry size in bytes at or above {@link VERSION_TWO_SCORE_SPLIT}.
|
|
71
|
-
*/
|
|
72
|
-
const SPLIT_PLACE_ENTRY_SIZE = 60
|
|
73
|
-
|
|
74
|
-
/**
|
|
75
|
-
* Place-entry size in bytes below {@link VERSION_TWO_SCORE_SPLIT}.
|
|
76
|
-
*/
|
|
77
|
-
const LEGACY_PLACE_ENTRY_SIZE = 56
|
|
78
|
-
|
|
79
|
-
/**
|
|
80
|
-
* Byte offset of the encyclopedic float inside a v5 place entry — immediately after the 8-slot parent chain.
|
|
81
|
-
*/
|
|
82
|
-
const ENCYCLOPEDIC_OFFSET = 56
|
|
83
|
-
|
|
84
|
-
/**
|
|
85
|
-
* `placeFlags` bit 0 (byte `pp+7`, v5+): this place carries an encyclopedic score. Per-PLACE rather than per-file
|
|
86
|
-
* because absence is the common case — roughly 89% of the 2026-08-05 gazetteer has no Wikipedia article — and a
|
|
87
|
-
* file-level flag would force every one of those rows to claim a 0 it never had.
|
|
88
|
-
*/
|
|
89
|
-
const PLACE_FLAG_HAS_ENCYCLOPEDIC = 1
|
|
90
|
-
|
|
91
|
-
/**
|
|
92
|
-
* File magic. A reader rejects anything not starting with these four bytes before parsing further.
|
|
93
|
-
*/
|
|
94
|
-
const MAGIC = Buffer.from("FST\0", "ascii")
|
|
95
|
-
|
|
96
|
-
/**
|
|
97
|
-
* Format version this serializer emits. See {@link VERSION_WIDE_STATE_COUNTERS} for what each bump changed.
|
|
98
|
-
*/
|
|
99
|
-
const VERSION = 5
|
|
100
|
-
|
|
101
|
-
/**
|
|
102
|
-
* The format version this tree WRITES, published so a freshness guard can call an older artifact format-stale without
|
|
103
|
-
* re-typing the number. Mirrors `REQUIRED_PAIR_INDEX_SCHEMA`'s role for PIX1: one constant, so a version bump cannot be
|
|
104
|
-
* noticed by some checkers and not others. See `fst-freshness.ts`.
|
|
105
|
-
*/
|
|
106
|
-
export const FST_FORMAT_VERSION = VERSION
|
|
107
|
-
|
|
108
|
-
/**
|
|
109
|
-
* Fixed header size in bytes: magic, version, and the section offsets that follow it.
|
|
110
|
-
*/
|
|
111
|
-
const HEADER_SIZE = 32
|
|
112
|
-
|
|
113
|
-
/**
|
|
114
|
-
* State-table entry: edge offset, place offset, and the two 32-bit counters (v4 widths).
|
|
115
|
-
*/
|
|
116
|
-
const STATE_ENTRY_SIZE = 16
|
|
117
|
-
|
|
118
|
-
/**
|
|
119
|
-
* Edge-table entry: the transition label and the target state index.
|
|
120
|
-
*/
|
|
121
|
-
const EDGE_ENTRY_SIZE = 8
|
|
122
|
-
|
|
123
|
-
/**
|
|
124
|
-
* Place-table entry at the version this tree WRITES: the place id, its placetype, coordinates, and both scores.
|
|
125
|
-
*/
|
|
126
|
-
const PLACE_ENTRY_SIZE = SPLIT_PLACE_ENTRY_SIZE
|
|
63
|
+
const MAGIC = Buffer.from(FST_MAGIC_BYTES)
|
|
127
64
|
|
|
128
65
|
/**
|
|
129
66
|
* Longest ancestry chain stored per place. Deeper hierarchies are truncated at the leaf end, since the specific end of
|
|
@@ -131,24 +68,6 @@ const PLACE_ENTRY_SIZE = SPLIT_PLACE_ENTRY_SIZE
|
|
|
131
68
|
*/
|
|
132
69
|
const MAX_CHAIN_LEN = 8
|
|
133
70
|
|
|
134
|
-
/**
|
|
135
|
-
* Placetypes in hierarchy order, largest first. The index into this array is what gets written into a place entry, so
|
|
136
|
-
* REORDERING IT BREAKS EVERY EXISTING FILE — append instead, and bump the version.
|
|
137
|
-
*/
|
|
138
|
-
const PLACETYPE_ORDER: readonly PlacetypeID[] = [
|
|
139
|
-
"country",
|
|
140
|
-
"region",
|
|
141
|
-
"county",
|
|
142
|
-
"locality",
|
|
143
|
-
"localadmin",
|
|
144
|
-
"borough",
|
|
145
|
-
"neighbourhood",
|
|
146
|
-
"postalcode",
|
|
147
|
-
"campus",
|
|
148
|
-
"dependency",
|
|
149
|
-
"street_affix",
|
|
150
|
-
]
|
|
151
|
-
|
|
152
71
|
const placetypeToIdx = new Map<string, number>()
|
|
153
72
|
|
|
154
73
|
for (let i = 0; i < PLACETYPE_ORDER.length; i++) {
|
|
@@ -198,9 +117,9 @@ export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): B
|
|
|
198
117
|
|
|
199
118
|
// --- Allocate ---
|
|
200
119
|
const stringTableSize = (strings.length + 1) * 4 + stringBytes
|
|
201
|
-
const stateTableSize = nodes.length *
|
|
120
|
+
const stateTableSize = nodes.length * WIDE_STATE_ENTRY_SIZE
|
|
202
121
|
const edgeTableSize = totalEdges * EDGE_ENTRY_SIZE
|
|
203
|
-
const placeTableSize = totalPlaces *
|
|
122
|
+
const placeTableSize = totalPlaces * SPLIT_PLACE_ENTRY_SIZE
|
|
204
123
|
const provenanceJson = provenance ? Buffer.from(JSON.stringify(provenance), "utf8") : null
|
|
205
124
|
const provenanceSize = provenanceJson ? 4 + provenanceJson.length : 0
|
|
206
125
|
const binarySize = HEADER_SIZE + stringTableSize + stateTableSize + edgeTableSize + placeTableSize
|
|
@@ -215,7 +134,7 @@ export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): B
|
|
|
215
134
|
const hasAmbiguity = nodes.some((n) => n.places.some((p) => p.crossCountryBranches !== undefined))
|
|
216
135
|
MAGIC.copy(buf, pos)
|
|
217
136
|
pos += 4
|
|
218
|
-
buf.writeUInt16LE(
|
|
137
|
+
buf.writeUInt16LE(FST_FORMAT_VERSION, pos)
|
|
219
138
|
pos += 2
|
|
220
139
|
buf.writeUInt16LE(hasAmbiguity ? 1 : 0, pos)
|
|
221
140
|
pos += 2
|
|
@@ -261,7 +180,7 @@ export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): B
|
|
|
261
180
|
|
|
262
181
|
for (let si = 0; si < nodes.length; si++) {
|
|
263
182
|
const node = nodes[si]!
|
|
264
|
-
const sp = stateTableStart + si *
|
|
183
|
+
const sp = stateTableStart + si * WIDE_STATE_ENTRY_SIZE
|
|
265
184
|
|
|
266
185
|
buf.writeUInt32LE(edgeIdx, sp)
|
|
267
186
|
buf.writeUInt32LE(placeIdx, sp + 4)
|
|
@@ -277,14 +196,14 @@ export function serializeFST(matcher: FSTMatcher, provenance?: FSTProvenance): B
|
|
|
277
196
|
}
|
|
278
197
|
|
|
279
198
|
for (const place of node.places) {
|
|
280
|
-
const pp = placeTableStart + placeIdx *
|
|
199
|
+
const pp = placeTableStart + placeIdx * SPLIT_PLACE_ENTRY_SIZE
|
|
281
200
|
// Filter out WOF sentinel parent IDs (negative values like -1, -4).
|
|
282
201
|
const validChain = place.parentChain.filter((id) => id > 0)
|
|
283
202
|
const chainLen = Math.min(validChain.length, MAX_CHAIN_LEN)
|
|
284
203
|
buf.writeUInt32LE(place.wofID, pp)
|
|
285
204
|
buf.writeUInt8(placetypeToIdx.get(place.placetype) ?? 0, pp + 4)
|
|
286
205
|
buf.writeUInt8(chainLen, pp + 5)
|
|
287
|
-
// Former _pad: byte 0 = crossCountryBranches (header flags bit0
|
|
206
|
+
// Former _pad: byte 0 = crossCountryBranches (header flags bit0 enables the read), byte 1 = v5 placeFlags.
|
|
288
207
|
buf.writeUInt8(hasAmbiguity ? Math.min(place.crossCountryBranches ?? 0, 255) : 0, pp + 6)
|
|
289
208
|
// An absent encyclopedic score writes flag 0 AND a 0.0 float. The float is unread in that
|
|
290
209
|
// state, so absence can never surface as a score — the meaning-of-zero rule, in bytes.
|
|
@@ -321,7 +240,10 @@ export function deserializeFST(buf: Buffer): FSTMatcher {
|
|
|
321
240
|
if (!buf.subarray(0, 4).equals(MAGIC)) throw new Error("FST magic mismatch")
|
|
322
241
|
const version = buf.readUInt16LE(4)
|
|
323
242
|
|
|
324
|
-
if (version < 1 || version >
|
|
243
|
+
if (version < 1 || version > FST_FORMAT_VERSION) {
|
|
244
|
+
throw new Error(`FST version ${version} unsupported (expected 1..${FST_FORMAT_VERSION})`)
|
|
245
|
+
}
|
|
246
|
+
|
|
325
247
|
const isV2 = version >= 2
|
|
326
248
|
const isSplit = version >= VERSION_TWO_SCORE_SPLIT
|
|
327
249
|
// flags bit0 (survey #4): place rows carry surface-ambiguity data in the former _pad byte.
|
|
@@ -416,7 +338,7 @@ export function deserializeFST(buf: Buffer): FSTMatcher {
|
|
|
416
338
|
lat: buf.readFloatLE(pp + 16),
|
|
417
339
|
lon: buf.readFloatLE(pp + 20),
|
|
418
340
|
parentChain,
|
|
419
|
-
// Header flags bit0
|
|
341
|
+
// Header flags bit0 enables the read (survey #4): pre-ambiguity artifacts expose undefined.
|
|
420
342
|
...(hasAmbiguity ? { crossCountryBranches: buf.readUInt8(pp + 6) } : {}),
|
|
421
343
|
...(hasEncyclopedic ? { encyclopedic: buf.readFloatLE(pp + ENCYCLOPEDIC_OFFSET) } : {}),
|
|
422
344
|
}
|
|
@@ -8,6 +8,8 @@
|
|
|
8
8
|
* prefix of every place name in the gazetteer.
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
+
import type { PathBuilderLike } from "path-ts"
|
|
12
|
+
|
|
11
13
|
export interface PlaceEntry {
|
|
12
14
|
wofID: number
|
|
13
15
|
placetype: PlacetypeID
|
|
@@ -129,7 +131,7 @@ export interface FSTProvenance {
|
|
|
129
131
|
}
|
|
130
132
|
|
|
131
133
|
export interface BuildFSTOpts {
|
|
132
|
-
dbPath:
|
|
134
|
+
dbPath: PathBuilderLike
|
|
133
135
|
/**
|
|
134
136
|
* Pre-computed identity of `dbPath` to stamp into provenance. Omit and the builder reads it via
|
|
135
137
|
* `readWOFSourceIdentity` (sidecar-cached). Supply it when the digest is already in hand, or when the caller is
|
|
@@ -164,7 +166,7 @@ export interface BuildFSTOpts {
|
|
|
164
166
|
* accessible under several surfaces records each surface's own count. Serialized into the place row's former `_pad`
|
|
165
167
|
* byte with presence signaled by header flags bit0, so VERSION stays put and pre-ambiguity artifacts read as "no
|
|
166
168
|
* data" (never "0 branches" — the meaning-of-zero rule). No decoder consumes it yet; consumers (FST-prior tempering,
|
|
167
|
-
* the Option-A evidence channel) arrive behind their own measured
|
|
169
|
+
* the Option-A evidence channel) arrive behind their own measured checks.
|
|
168
170
|
*/
|
|
169
171
|
surfaceCountryCounts?: ReadonlyMap<string, number>
|
|
170
172
|
onProgress?: (phase: string, detail?: string) => void
|
|
@@ -14,7 +14,8 @@
|
|
|
14
14
|
* bag.
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
|
-
import type {
|
|
17
|
+
import type { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
18
|
+
import { tableExists } from "@mailwoman/sqlite/introspection"
|
|
18
19
|
|
|
19
20
|
/**
|
|
20
21
|
* Name of the FTS5 virtual table this module owns. Centralized so `WOFSQLitePlaceLookup` and the CLI can't drift apart.
|
|
@@ -71,6 +72,16 @@ export const ALIAS_SEPARATOR = "\uE000"
|
|
|
71
72
|
*/
|
|
72
73
|
const ALIAS_SEPARATOR_CODEPOINT = ALIAS_SEPARATOR.codePointAt(0) as number
|
|
73
74
|
|
|
75
|
+
/**
|
|
76
|
+
* The query/name fold every exact-tier comparison uses: lowercase, trim, collapse internal whitespace. The `alt_names`
|
|
77
|
+
* bag is compared alias-by-alias against a query folded THIS way, so the fold lives beside the bag parser — every
|
|
78
|
+
* consumer (the Node resolver's exact tier, the WASM resolver, the name-exact sub-tier sort) folds through this one
|
|
79
|
+
* function.
|
|
80
|
+
*/
|
|
81
|
+
export function foldQueryText(input: string): string {
|
|
82
|
+
return input.toLowerCase().trim().replaceAll(/\s+/g, " ")
|
|
83
|
+
}
|
|
84
|
+
|
|
74
85
|
/**
|
|
75
86
|
* Does any alias in an `alt_names` bag exactly equal the (already-normalized) query? The single shared implementation
|
|
76
87
|
* of the exact-tier alias check for every consumer of the bag — the Node resolver's `#exactMatchIDs` fallback, the WASM
|
|
@@ -80,10 +91,10 @@ const ALIAS_SEPARATOR_CODEPOINT = ALIAS_SEPARATOR.codePointAt(0) as number
|
|
|
80
91
|
*
|
|
81
92
|
* - **Separated bags** (built since #523): aliases joined with {@link ALIAS_SEPARATOR}, plus a trailing separator so even
|
|
82
93
|
* a single-alias bag self-identifies as separator-formatted. Split + per-alias equality — a true exact-alias check,
|
|
83
|
-
* matching the semantics of the full `names` table (`names.name = ? COLLATE NOCASE`), so it runs
|
|
84
|
-
* match is an exact match whether or not another candidate matched on its canonical name.
|
|
94
|
+
* matching the semantics of the full `names` table (`names.name = ? COLLATE NOCASE`), so it runs unrestricted: an
|
|
95
|
+
* alias match is an exact match whether or not another candidate matched on its canonical name.
|
|
85
96
|
* - **Legacy bags** (pre-#523 artifacts, e.g. an already-deployed slim DB): aliases space-joined, boundaries lost. Falls
|
|
86
|
-
* back to the historical padded-containment check,
|
|
97
|
+
* back to the historical padded-containment check, conditioned on `anyStrictExact` — unrestricted containment would
|
|
87
98
|
* false-promote interior fragments ("York" inside the alias "New York City") and cross-boundary fragments ("York New"
|
|
88
99
|
* across "…York" + "New…"). Delete this branch once every shipped artifact carries the separator.
|
|
89
100
|
*
|
|
@@ -95,15 +106,14 @@ const ALIAS_SEPARATOR_CODEPOINT = ALIAS_SEPARATOR.codePointAt(0) as number
|
|
|
95
106
|
*/
|
|
96
107
|
export function aliasBagExactMatch(altNames: string | null, normalizedQuery: string, anyStrictExact: boolean): boolean {
|
|
97
108
|
if (altNames === null || altNames === "" || !normalizedQuery) return false
|
|
98
|
-
const norm = (s: string): string => s.toLowerCase().trim().replaceAll(/\s+/g, " ")
|
|
99
109
|
|
|
100
110
|
if (altNames.includes(ALIAS_SEPARATOR)) {
|
|
101
|
-
return altNames.split(ALIAS_SEPARATOR).some((alias) =>
|
|
111
|
+
return altNames.split(ALIAS_SEPARATOR).some((alias) => foldQueryText(alias) === normalizedQuery)
|
|
102
112
|
}
|
|
103
113
|
|
|
104
114
|
if (anyStrictExact) return false
|
|
105
115
|
|
|
106
|
-
return ` ${
|
|
116
|
+
return ` ${foldQueryText(altNames)} `.includes(` ${normalizedQuery} `)
|
|
107
117
|
}
|
|
108
118
|
|
|
109
119
|
/**
|
|
@@ -189,7 +199,10 @@ export interface BuildPlaceSearchFTSOpts {
|
|
|
189
199
|
* Returns a `BuildPlaceSearchFTSResult` summary. Idempotent when `drop: false` — re-running against an already-indexed
|
|
190
200
|
* DB skips whichever indexes already exist.
|
|
191
201
|
*/
|
|
192
|
-
export function buildPlaceSearchFTS(
|
|
202
|
+
export function buildPlaceSearchFTS<DB>(
|
|
203
|
+
db: DatabaseClient<DB>,
|
|
204
|
+
opts: BuildPlaceSearchFTSOpts = {}
|
|
205
|
+
): BuildPlaceSearchFTSResult {
|
|
193
206
|
const start = Date.now()
|
|
194
207
|
const onProgress = opts.onProgress ?? (() => {})
|
|
195
208
|
|
|
@@ -329,28 +342,20 @@ export function buildPlaceSearchFTS(db: DatabaseSync, opts: BuildPlaceSearchFTSO
|
|
|
329
342
|
* Returns true iff the `place_search` table exists in the connected DB. Used by `WOFSQLitePlaceLookup` for its "FTS
|
|
330
343
|
* missing — pass buildFTS:true or run the CLI" guard.
|
|
331
344
|
*/
|
|
332
|
-
export function placeSearchFTSExists(db:
|
|
345
|
+
export function placeSearchFTSExists<DB>(db: DatabaseClient<DB>): boolean {
|
|
333
346
|
return tableExists(db, PLACE_SEARCH_TABLE)
|
|
334
347
|
}
|
|
335
348
|
|
|
336
349
|
/**
|
|
337
350
|
* Returns true iff the `place_bbox` R*Tree table exists. Used for opt-in proximity lookup checks.
|
|
338
351
|
*/
|
|
339
|
-
export function placeBboxExists(db:
|
|
352
|
+
export function placeBboxExists<DB>(db: DatabaseClient<DB>): boolean {
|
|
340
353
|
return tableExists(db, PLACE_BBOX_TABLE)
|
|
341
354
|
}
|
|
342
355
|
|
|
343
356
|
/**
|
|
344
357
|
* Returns true iff the `place_population` table exists. Used for opt-in population-ranking checks.
|
|
345
358
|
*/
|
|
346
|
-
export function placePopulationExists(db:
|
|
359
|
+
export function placePopulationExists<DB>(db: DatabaseClient<DB>): boolean {
|
|
347
360
|
return tableExists(db, PLACE_POPULATION_TABLE)
|
|
348
361
|
}
|
|
349
|
-
|
|
350
|
-
function tableExists(db: DatabaseSync, name: string): boolean {
|
|
351
|
-
const row = db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?`).get(name) as
|
|
352
|
-
| { name: string }
|
|
353
|
-
| undefined
|
|
354
|
-
|
|
355
|
-
return Boolean(row)
|
|
356
|
-
}
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* an unescaped quote or a bare `*` is a syntax error, not an empty result.
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
import type { FindPlaceQuery, WOFPlacetype } from "
|
|
11
|
+
import type { FindPlaceQuery, WOFPlacetype } from "#types"
|
|
12
12
|
|
|
13
13
|
export function normalizePlacetypes(p: FindPlaceQuery["placetype"]): WOFPlacetype[] | null {
|
|
14
14
|
if (!p) return null
|
|
@@ -24,14 +24,14 @@
|
|
|
24
24
|
* (TSV).
|
|
25
25
|
*/
|
|
26
26
|
|
|
27
|
-
import { existsSync } from "node:fs"
|
|
28
|
-
import type { DatabaseSync } from "node:sqlite"
|
|
29
|
-
|
|
30
27
|
import { isOfficialLanguage } from "@mailwoman/codex/country"
|
|
31
|
-
import {
|
|
28
|
+
import { pathExists } from "@mailwoman/core/fs/readers"
|
|
29
|
+
import type { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
30
|
+
import { join, type PathBuilderLike } from "path-ts"
|
|
32
31
|
import { TSVSpliterator } from "spliterator"
|
|
33
32
|
|
|
34
|
-
import { GEONAMES_POSTAL_ID_BASE } from "
|
|
33
|
+
import { GEONAMES_POSTAL_ID_BASE } from "#geonames/postal"
|
|
34
|
+
import type { WOFDatabase } from "#schema"
|
|
35
35
|
|
|
36
36
|
/**
|
|
37
37
|
* Synthetic id base for GeoNames-sourced rows (#743/#193) — above Overture's 8e12 so the three sources (WOF real ids,
|
|
@@ -62,11 +62,11 @@ const FOLD_OWNED_TABLES = ["spr", "names", "place_population", "ancestors"] as c
|
|
|
62
62
|
* country. Gaborone/BW at id 9000000121151 became Aichegg/AT and kept all 26 of its names; Kinshasa's 16,000,000
|
|
63
63
|
* population landed on a Lithuanian hamlet.
|
|
64
64
|
*
|
|
65
|
-
* The upper bound is NOT the end of the id space. Each later
|
|
65
|
+
* The upper bound is NOT the end of the id space. Each later extract owns a range above this one — GeoNames-postal @
|
|
66
66
|
* 9.5e12, NL-PC6 @ 9.6e12, Code-Point @ 9.7e12, NI @ 9.8e12 — and a purge that ran past {@link GEONAMES_POSTAL_ID_BASE}
|
|
67
67
|
* would delete them.
|
|
68
68
|
*/
|
|
69
|
-
export function purgeGeonamesAliasRange(db:
|
|
69
|
+
export function purgeGeonamesAliasRange<DB>(db: DatabaseClient<DB>): Record<string, number> {
|
|
70
70
|
const removed: Record<string, number> = {}
|
|
71
71
|
|
|
72
72
|
for (const table of FOLD_OWNED_TABLES) {
|
|
@@ -125,7 +125,7 @@ async function parseAlternateNamesV2(
|
|
|
125
125
|
// spelling across rows — Malabo carries "Santa Isabel" as (es, unflagged) AND as (no-language,
|
|
126
126
|
// isHistoric=1, to=1973). Officialness must see the flags from EVERY row for the spelling, or the
|
|
127
127
|
// colonial-era name sails through on the language-tagged row (the #936 review's Malabo finding).
|
|
128
|
-
// Do NOT
|
|
128
|
+
// Do NOT condition on isPreferredName instead — it's sparse annotation, not a signal (Turku's sv "Åbo"
|
|
129
129
|
// is unflagged; FI has 1,746 flags across the whole dump).
|
|
130
130
|
//
|
|
131
131
|
// Both passes STREAM the file rather than sharing one materialized array: NO's V2 dump is 33 MB,
|
|
@@ -190,9 +190,9 @@ async function parseAlternateNamesV2(
|
|
|
190
190
|
* the candidate build's alias pass.
|
|
191
191
|
*/
|
|
192
192
|
export async function ingestGeonamesAliases(
|
|
193
|
-
db:
|
|
193
|
+
db: DatabaseClient<WOFDatabase>,
|
|
194
194
|
countries: string[],
|
|
195
|
-
geonamesDir:
|
|
195
|
+
geonamesDir: PathBuilderLike,
|
|
196
196
|
onProgress?: (event: GeonamesIngestProgress) => void,
|
|
197
197
|
opts?: {
|
|
198
198
|
/**
|
|
@@ -211,7 +211,7 @@ export async function ingestGeonamesAliases(
|
|
|
211
211
|
* at 7 new name-exact collisions globally). The main dump's bare `alternatenames` list still decides WHICH rows
|
|
212
212
|
* exist; V2 only decorates them. Missing file = the pre-#936 untagged behavior, not an error.
|
|
213
213
|
*/
|
|
214
|
-
alternateDir?:
|
|
214
|
+
alternateDir?: PathBuilderLike
|
|
215
215
|
}
|
|
216
216
|
): Promise<number> {
|
|
217
217
|
// Latin-only, no bracket/paren noise GeoNames packs into `alternatenames` ("(( Karis Landskommun ))",
|
|
@@ -265,7 +265,7 @@ export async function ingestGeonamesAliases(
|
|
|
265
265
|
for (const cc of countries) {
|
|
266
266
|
const file = join(geonamesDir, `${cc}.txt`)
|
|
267
267
|
|
|
268
|
-
if (!
|
|
268
|
+
if (!(await pathExists(file))) {
|
|
269
269
|
report({ country: cc, places: 0, skipped: true }, file)
|
|
270
270
|
|
|
271
271
|
continue
|
|
@@ -275,7 +275,7 @@ export async function ingestGeonamesAliases(
|
|
|
275
275
|
// #267: add A-class admin + ancestry only for the gap countries this country is in (never the EU set).
|
|
276
276
|
const addAdmin = opts?.adminForCountries?.has(cc) ?? false
|
|
277
277
|
const v2File = opts?.alternateDir ? join(opts.alternateDir, `${cc}.txt`) : undefined
|
|
278
|
-
const readV2 = Boolean(v2File &&
|
|
278
|
+
const readV2 = Boolean(v2File && (await pathExists(v2File)))
|
|
279
279
|
|
|
280
280
|
// Survey pass. The dump is STREAMED — NO's is 71 MB and only the caller knows which country
|
|
281
281
|
// comes next — so what a later pass needs has to be collected here rather than re-scanned off a
|