@mailwoman/resolver-wof-sqlite 9.1.0 → 9.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -20
- package/lib/address/index.ts +9 -0
- package/{address-point-interpolation.ts → lib/address/point-interpolation.ts} +40 -28
- package/{address-point-schema.ts → lib/address/point-schema.ts} +34 -8
- package/lib/address/point.ts +280 -0
- package/{ancestry-backfill.ts → lib/ancestry/backfill.ts} +14 -19
- package/{ancestry.ts → lib/ancestry/index.ts} +13 -8
- package/{build-candidate.ts → lib/build-candidate.ts} +243 -199
- package/{build-slim.ts → lib/build-slim.ts} +153 -168
- package/lib/candidate/alias-bags.ts +56 -0
- package/lib/candidate/ancestors-sidecar.ts +204 -0
- package/lib/candidate/country-display-names.ts +79 -0
- package/lib/candidate/extract-fold.ts +138 -0
- package/lib/candidate/name-roles.ts +238 -0
- package/lib/candidate/own-name.ts +146 -0
- package/lib/candidate/place-attrs.ts +44 -0
- package/lib/candidate-ancestors-schema.ts +195 -0
- package/{candidate-fts.ts → lib/candidate-fts.ts} +7 -5
- package/{candidate-importance.ts → lib/candidate-importance.ts} +54 -57
- package/lib/candidate-lookup.ts +976 -0
- package/{candidate-schema.ts → lib/candidate-schema.ts} +34 -6
- package/lib/candidate-scoring.ts +261 -0
- package/lib/capital-schema.ts +89 -0
- package/lib/capitals.ts +148 -0
- package/{coincident-roles.ts → lib/coincident-roles.ts} +75 -19
- package/{convention.ts → lib/convention/index.ts} +5 -5
- package/lib/convention/schema.ts +72 -0
- package/{coverage-manifest-schema.ts → lib/coverage-manifest-schema.ts} +15 -16
- package/lib/currency-backfill.ts +334 -0
- package/lib/env.ts +50 -0
- package/lib/exact-match.ts +113 -0
- package/{sharding.ts → lib/extracts.ts} +63 -60
- package/lib/fst/autocomplete.ts +176 -0
- package/{fst-builder.ts → lib/fst/builder.ts} +30 -27
- package/{fst-deserialize-web.ts → lib/fst/deserialize-web.ts} +27 -87
- package/lib/fst/format.ts +106 -0
- package/{fst-freshness.ts → lib/fst/freshness.ts} +42 -75
- package/lib/fst/index.ts +14 -0
- package/{fst-matcher.ts → lib/fst/matcher.ts} +1 -1
- package/{fst-serialize.ts → lib/fst/serialize.ts} +37 -115
- package/{fst-types.ts → lib/fst/types.ts} +4 -2
- package/{fts.ts → lib/fts/index.ts} +28 -23
- package/{fts-query.ts → lib/fts/query.ts} +2 -2
- package/{geonames-aliases.ts → lib/geonames/aliases.ts} +13 -13
- package/lib/geonames/index.ts +8 -0
- package/{geonames-postal.ts → lib/geonames/postal.ts} +16 -15
- package/{index.ts → lib/index.ts} +52 -48
- package/{interpolation.ts → lib/interpolation.ts} +137 -41
- package/lib/lookup.ts +861 -0
- package/{name-score.ts → lib/name-score.ts} +6 -4
- package/lib/nsul/index.ts +8 -0
- package/lib/nsul/lookup.ts +144 -0
- package/lib/nsul/schema.ts +130 -0
- package/{place-importance-schema.ts → lib/place-importance-schema.ts} +70 -22
- package/lib/poi/index.ts +8 -0
- package/{poi-lookup.ts → lib/poi/lookup.ts} +32 -39
- package/{poi-schema.ts → lib/poi/schema.ts} +10 -6
- package/lib/polygon-schema.ts +47 -0
- package/{postal-city-alias-lookup.ts → lib/postal/city-alias-lookup.ts} +15 -21
- package/{postal-city-candidate-schema.ts → lib/postal/city-candidate-schema.ts} +4 -2
- package/lib/postal/index.ts +9 -0
- package/{postcode-point-lookup.ts → lib/postcode-point-lookup.ts} +17 -15
- package/lib/primary-preference.ts +226 -0
- package/lib/proximity-rerank.ts +120 -0
- package/{ranking-weights.ts → lib/ranking-weights.ts} +30 -4
- package/lib/region-keys.ts +144 -0
- package/{reverse.ts → lib/reverse.ts} +45 -56
- package/{schema.ts → lib/schema.ts} +11 -1
- package/lib/search-fetch.ts +257 -0
- package/{sqlite-convention-source.ts → lib/sqlite-convention-source.ts} +9 -10
- package/lib/sqlite-utils.ts +83 -0
- package/{street-centroid-schema.ts → lib/street/centroid-schema.ts} +12 -6
- package/{street-centroid.ts → lib/street/centroid.ts} +26 -28
- package/lib/street/index.ts +13 -0
- package/{street-morphology-fst-builder.ts → lib/street/morphology-fst-builder.ts} +24 -14
- package/{street-morphology-fst-loader.ts → lib/street/morphology-fst-loader.ts} +14 -12
- package/{street-name-lookup.ts → lib/street/name-lookup.ts} +14 -30
- package/lib/street/normalize.ts +572 -0
- package/{street-segment-schema.ts → lib/street/segment-schema.ts} +13 -8
- package/{types.ts → lib/types.ts} +49 -9
- package/{unified-schema.ts → lib/unified-schema.ts} +21 -25
- package/lib/uprn/existence.ts +84 -0
- package/lib/uprn/index.ts +9 -0
- package/lib/uprn/lookup.ts +206 -0
- package/lib/uprn/schema.ts +124 -0
- package/lib/weights-overlay-linker.ts +878 -0
- package/out/address/index.d.ts +9 -0
- package/out/address/index.d.ts.map +1 -0
- package/out/address/index.js +9 -0
- package/out/address/index.js.map +1 -0
- package/out/{address-point-interpolation.d.ts → address/point-interpolation.d.ts} +11 -10
- package/out/address/point-interpolation.d.ts.map +1 -0
- package/out/{address-point-interpolation.js → address/point-interpolation.js} +29 -23
- package/out/address/point-interpolation.js.map +1 -0
- package/out/{address-point-schema.d.ts → address/point-schema.d.ts} +30 -10
- package/out/address/point-schema.d.ts.map +1 -0
- package/out/{address-point-schema.js → address/point-schema.js} +7 -3
- package/out/address/point-schema.js.map +1 -0
- package/out/{address-point.d.ts → address/point.d.ts} +16 -8
- package/out/address/point.d.ts.map +1 -0
- package/out/address/point.js +191 -0
- package/out/address/point.js.map +1 -0
- package/out/{ancestry-backfill.d.ts → ancestry/backfill.d.ts} +7 -5
- package/out/ancestry/backfill.d.ts.map +1 -0
- package/out/{ancestry-backfill.js → ancestry/backfill.js} +12 -16
- package/out/ancestry/backfill.js.map +1 -0
- package/out/{ancestry.d.ts → ancestry/index.d.ts} +6 -5
- package/out/ancestry/index.d.ts.map +1 -0
- package/out/{ancestry.js → ancestry/index.js} +7 -7
- package/out/ancestry/index.js.map +1 -0
- package/out/build-candidate.d.ts +88 -7
- package/out/build-candidate.d.ts.map +1 -1
- package/out/build-candidate.js +145 -149
- package/out/build-candidate.js.map +1 -1
- package/out/build-slim.d.ts +5 -5
- package/out/build-slim.d.ts.map +1 -1
- package/out/build-slim.js +127 -142
- package/out/build-slim.js.map +1 -1
- package/out/candidate/alias-bags.d.ts +19 -0
- package/out/candidate/alias-bags.d.ts.map +1 -0
- package/out/candidate/alias-bags.js +39 -0
- package/out/candidate/alias-bags.js.map +1 -0
- package/out/candidate/ancestors-sidecar.d.ts +32 -0
- package/out/candidate/ancestors-sidecar.d.ts.map +1 -0
- package/out/candidate/ancestors-sidecar.js +140 -0
- package/out/candidate/ancestors-sidecar.js.map +1 -0
- package/out/candidate/country-display-names.d.ts +35 -0
- package/out/candidate/country-display-names.d.ts.map +1 -0
- package/out/candidate/country-display-names.js +59 -0
- package/out/candidate/country-display-names.js.map +1 -0
- package/out/candidate/extract-fold.d.ts +32 -0
- package/out/candidate/extract-fold.d.ts.map +1 -0
- package/out/candidate/extract-fold.js +104 -0
- package/out/candidate/extract-fold.js.map +1 -0
- package/out/candidate/name-roles.d.ts +57 -0
- package/out/candidate/name-roles.d.ts.map +1 -0
- package/out/candidate/name-roles.js +166 -0
- package/out/candidate/name-roles.js.map +1 -0
- package/out/candidate/own-name.d.ts +50 -0
- package/out/candidate/own-name.d.ts.map +1 -0
- package/out/candidate/own-name.js +132 -0
- package/out/candidate/own-name.js.map +1 -0
- package/out/candidate/place-attrs.d.ts +43 -0
- package/out/candidate/place-attrs.d.ts.map +1 -0
- package/out/candidate/place-attrs.js +15 -0
- package/out/candidate/place-attrs.js.map +1 -0
- package/out/candidate-ancestors-schema.d.ts +150 -0
- package/out/candidate-ancestors-schema.d.ts.map +1 -0
- package/out/candidate-ancestors-schema.js +123 -0
- package/out/candidate-ancestors-schema.js.map +1 -0
- package/out/candidate-fts.d.ts +7 -5
- package/out/candidate-fts.d.ts.map +1 -1
- package/out/candidate-fts.js +5 -3
- package/out/candidate-fts.js.map +1 -1
- package/out/candidate-importance.d.ts +16 -16
- package/out/candidate-importance.d.ts.map +1 -1
- package/out/candidate-importance.js +47 -52
- package/out/candidate-importance.js.map +1 -1
- package/out/candidate-lookup.d.ts +31 -52
- package/out/candidate-lookup.d.ts.map +1 -1
- package/out/candidate-lookup.js +380 -172
- package/out/candidate-lookup.js.map +1 -1
- package/out/candidate-schema.d.ts +31 -7
- package/out/candidate-schema.d.ts.map +1 -1
- package/out/candidate-schema.js +3 -0
- package/out/candidate-schema.js.map +1 -1
- package/out/candidate-scoring.d.ts +34 -0
- package/out/candidate-scoring.d.ts.map +1 -0
- package/out/candidate-scoring.js +196 -0
- package/out/candidate-scoring.js.map +1 -0
- package/out/capital-schema.d.ts +51 -0
- package/out/capital-schema.d.ts.map +1 -0
- package/out/capital-schema.js +63 -0
- package/out/capital-schema.js.map +1 -0
- package/out/capitals.d.ts +69 -0
- package/out/capitals.d.ts.map +1 -0
- package/out/capitals.js +98 -0
- package/out/capitals.js.map +1 -0
- package/out/coincident-roles.d.ts +12 -4
- package/out/coincident-roles.d.ts.map +1 -1
- package/out/coincident-roles.js +43 -11
- package/out/coincident-roles.js.map +1 -1
- package/out/{convention.d.ts → convention/index.d.ts} +5 -5
- package/out/convention/index.d.ts.map +1 -0
- package/out/{convention.js → convention/index.js} +4 -4
- package/out/convention/index.js.map +1 -0
- package/out/convention/schema.d.ts +51 -0
- package/out/convention/schema.d.ts.map +1 -0
- package/out/convention/schema.js +34 -0
- package/out/convention/schema.js.map +1 -0
- package/out/coverage-manifest-schema.d.ts +8 -8
- package/out/coverage-manifest-schema.d.ts.map +1 -1
- package/out/coverage-manifest-schema.js +6 -10
- package/out/coverage-manifest-schema.js.map +1 -1
- package/out/currency-backfill.d.ts +90 -0
- package/out/currency-backfill.d.ts.map +1 -0
- package/out/currency-backfill.js +196 -0
- package/out/currency-backfill.js.map +1 -0
- package/out/env.d.ts +43 -0
- package/out/env.d.ts.map +1 -0
- package/out/env.js +48 -0
- package/out/env.js.map +1 -0
- package/out/exact-match.d.ts +25 -0
- package/out/exact-match.d.ts.map +1 -0
- package/out/exact-match.js +88 -0
- package/out/exact-match.js.map +1 -0
- package/out/extracts.d.ts +114 -0
- package/out/extracts.d.ts.map +1 -0
- package/out/{sharding.js → extracts.js} +38 -38
- package/out/extracts.js.map +1 -0
- package/out/{fst-autocomplete.d.ts → fst/autocomplete.d.ts} +13 -13
- package/out/fst/autocomplete.d.ts.map +1 -0
- package/out/fst/autocomplete.js +113 -0
- package/out/fst/autocomplete.js.map +1 -0
- package/out/{fst-builder.d.ts → fst/builder.d.ts} +5 -5
- package/out/fst/builder.d.ts.map +1 -0
- package/out/{fst-builder.js → fst/builder.js} +24 -24
- package/out/fst/builder.js.map +1 -0
- package/out/{fst-deserialize-web.d.ts → fst/deserialize-web.d.ts} +3 -3
- package/out/fst/deserialize-web.d.ts.map +1 -0
- package/out/{fst-deserialize-web.js → fst/deserialize-web.js} +10 -74
- package/out/fst/deserialize-web.js.map +1 -0
- package/out/fst/format.d.ts +80 -0
- package/out/fst/format.d.ts.map +1 -0
- package/out/fst/format.js +91 -0
- package/out/fst/format.js.map +1 -0
- package/out/{fst-freshness.d.ts → fst/freshness.d.ts} +11 -14
- package/out/fst/freshness.d.ts.map +1 -0
- package/out/{fst-freshness.js → fst/freshness.js} +42 -74
- package/out/fst/freshness.js.map +1 -0
- package/out/fst/index.d.ts +14 -0
- package/out/fst/index.d.ts.map +1 -0
- package/out/fst/index.js +14 -0
- package/out/fst/index.js.map +1 -0
- package/out/{fst-matcher.d.ts → fst/matcher.d.ts} +2 -2
- package/out/fst/matcher.d.ts.map +1 -0
- package/out/{fst-matcher.js → fst/matcher.js} +1 -1
- package/out/fst/matcher.js.map +1 -0
- package/out/{fst-serialize.d.ts → fst/serialize.d.ts} +5 -10
- package/out/fst/serialize.d.ts.map +1 -0
- package/out/{fst-serialize.js → fst/serialize.js} +18 -98
- package/out/fst/serialize.js.map +1 -0
- package/out/{fst-types.d.ts → fst/types.d.ts} +4 -3
- package/out/fst/types.d.ts.map +1 -0
- package/out/{fst-types.js → fst/types.js} +1 -1
- package/out/fst/types.js.map +1 -0
- package/out/{fts.d.ts → fts/index.d.ts} +20 -13
- package/out/fts/index.d.ts.map +1 -0
- package/out/{fts.js → fts/index.js} +20 -15
- package/out/fts/index.js.map +1 -0
- package/out/{fts-query.d.ts → fts/query.d.ts} +2 -2
- package/out/fts/query.d.ts.map +1 -0
- package/out/{fts-query.js → fts/query.js} +2 -2
- package/out/fts/query.js.map +1 -0
- package/out/{geonames-aliases.d.ts → geonames/aliases.d.ts} +8 -6
- package/out/geonames/aliases.d.ts.map +1 -0
- package/out/{geonames-aliases.js → geonames/aliases.js} +7 -7
- package/out/geonames/aliases.js.map +1 -0
- package/out/geonames/index.d.ts +8 -0
- package/out/geonames/index.d.ts.map +1 -0
- package/out/geonames/index.js +8 -0
- package/out/geonames/index.js.map +1 -0
- package/out/{geonames-postal.d.ts → geonames/postal.d.ts} +12 -10
- package/out/geonames/postal.d.ts.map +1 -0
- package/out/{geonames-postal.js → geonames/postal.js} +11 -11
- package/out/geonames/postal.js.map +1 -0
- package/out/index.d.ts +32 -31
- package/out/index.d.ts.map +1 -1
- package/out/index.js +24 -24
- package/out/index.js.map +1 -1
- package/out/interpolation.d.ts +18 -9
- package/out/interpolation.d.ts.map +1 -1
- package/out/interpolation.js +108 -36
- package/out/interpolation.js.map +1 -1
- package/out/lookup.d.ts +25 -25
- package/out/lookup.d.ts.map +1 -1
- package/out/lookup.js +174 -558
- package/out/lookup.js.map +1 -1
- package/out/name-score.d.ts +0 -10
- package/out/name-score.d.ts.map +1 -1
- package/out/name-score.js +6 -4
- package/out/name-score.js.map +1 -1
- package/out/nsul/index.d.ts +8 -0
- package/out/nsul/index.d.ts.map +1 -0
- package/out/nsul/index.js +8 -0
- package/out/nsul/index.js.map +1 -0
- package/out/nsul/lookup.d.ts +83 -0
- package/out/nsul/lookup.d.ts.map +1 -0
- package/out/nsul/lookup.js +80 -0
- package/out/nsul/lookup.js.map +1 -0
- package/out/nsul/schema.d.ts +100 -0
- package/out/nsul/schema.d.ts.map +1 -0
- package/out/nsul/schema.js +78 -0
- package/out/nsul/schema.js.map +1 -0
- package/out/place-importance-schema.d.ts +47 -10
- package/out/place-importance-schema.d.ts.map +1 -1
- package/out/place-importance-schema.js +56 -10
- package/out/place-importance-schema.js.map +1 -1
- package/out/poi/index.d.ts +8 -0
- package/out/poi/index.d.ts.map +1 -0
- package/out/poi/index.js +8 -0
- package/out/poi/index.js.map +1 -0
- package/out/{poi-lookup.d.ts → poi/lookup.d.ts} +8 -10
- package/out/poi/lookup.d.ts.map +1 -0
- package/out/{poi-lookup.js → poi/lookup.js} +24 -29
- package/out/poi/lookup.js.map +1 -0
- package/out/{poi-schema.d.ts → poi/schema.d.ts} +10 -6
- package/out/poi/schema.d.ts.map +1 -0
- package/out/{poi-schema.js → poi/schema.js} +1 -1
- package/out/poi/schema.js.map +1 -0
- package/out/polygon-schema.d.ts +37 -0
- package/out/polygon-schema.d.ts.map +1 -0
- package/out/polygon-schema.js +23 -0
- package/out/polygon-schema.js.map +1 -0
- package/out/{postal-city-alias-lookup.d.ts → postal/city-alias-lookup.d.ts} +7 -6
- package/out/postal/city-alias-lookup.d.ts.map +1 -0
- package/out/{postal-city-alias-lookup.js → postal/city-alias-lookup.js} +12 -16
- package/out/postal/city-alias-lookup.js.map +1 -0
- package/out/{postal-city-alias-schema.d.ts → postal/city-alias-schema.d.ts} +1 -1
- package/out/{postal-city-alias-schema.d.ts.map → postal/city-alias-schema.d.ts.map} +1 -1
- package/out/{postal-city-alias-schema.js → postal/city-alias-schema.js} +1 -1
- package/out/{postal-city-alias-schema.js.map → postal/city-alias-schema.js.map} +1 -1
- package/out/{postal-city-candidate-schema.d.ts → postal/city-candidate-schema.d.ts} +4 -3
- package/out/postal/city-candidate-schema.d.ts.map +1 -0
- package/out/{postal-city-candidate-schema.js → postal/city-candidate-schema.js} +2 -2
- package/out/{postal-city-candidate-schema.js.map → postal/city-candidate-schema.js.map} +1 -1
- package/out/postal/index.d.ts +9 -0
- package/out/postal/index.d.ts.map +1 -0
- package/out/postal/index.js +9 -0
- package/out/postal/index.js.map +1 -0
- package/out/postcode-point-lookup.d.ts +10 -10
- package/out/postcode-point-lookup.d.ts.map +1 -1
- package/out/postcode-point-lookup.js +13 -13
- package/out/postcode-point-lookup.js.map +1 -1
- package/out/primary-preference.d.ts +133 -0
- package/out/primary-preference.d.ts.map +1 -0
- package/out/primary-preference.js +146 -0
- package/out/primary-preference.js.map +1 -0
- package/out/proximity-rerank.d.ts +77 -0
- package/out/proximity-rerank.d.ts.map +1 -0
- package/out/proximity-rerank.js +86 -0
- package/out/proximity-rerank.js.map +1 -0
- package/out/ranking-weights.d.ts +15 -3
- package/out/ranking-weights.d.ts.map +1 -1
- package/out/ranking-weights.js +21 -3
- package/out/ranking-weights.js.map +1 -1
- package/out/region-keys.d.ts +47 -0
- package/out/region-keys.d.ts.map +1 -0
- package/out/region-keys.js +121 -0
- package/out/region-keys.js.map +1 -0
- package/out/reverse.d.ts +6 -6
- package/out/reverse.d.ts.map +1 -1
- package/out/reverse.js +23 -35
- package/out/reverse.js.map +1 -1
- package/out/schema.d.ts +10 -1
- package/out/schema.d.ts.map +1 -1
- package/out/schema.js.map +1 -1
- package/out/search-fetch.d.ts +59 -0
- package/out/search-fetch.d.ts.map +1 -0
- package/out/search-fetch.js +184 -0
- package/out/search-fetch.js.map +1 -0
- package/out/sqlite-convention-source.d.ts +7 -7
- package/out/sqlite-convention-source.d.ts.map +1 -1
- package/out/sqlite-convention-source.js +5 -5
- package/out/sqlite-convention-source.js.map +1 -1
- package/out/sqlite-utils.d.ts +24 -6
- package/out/sqlite-utils.d.ts.map +1 -1
- package/out/sqlite-utils.js +24 -7
- package/out/sqlite-utils.js.map +1 -1
- package/out/{street-centroid-schema.d.ts → street/centroid-schema.d.ts} +12 -7
- package/out/street/centroid-schema.d.ts.map +1 -0
- package/out/{street-centroid-schema.js → street/centroid-schema.js} +5 -5
- package/out/{street-centroid-schema.js.map → street/centroid-schema.js.map} +1 -1
- package/out/{street-centroid.d.ts → street/centroid.d.ts} +6 -6
- package/out/street/centroid.d.ts.map +1 -0
- package/out/{street-centroid.js → street/centroid.js} +18 -24
- package/out/street/centroid.js.map +1 -0
- package/out/street/index.d.ts +13 -0
- package/out/street/index.d.ts.map +1 -0
- package/out/street/index.js +13 -0
- package/out/street/index.js.map +1 -0
- package/out/{street-morphology-fst-builder.d.ts → street/morphology-fst-builder.d.ts} +4 -4
- package/out/street/morphology-fst-builder.d.ts.map +1 -0
- package/out/{street-morphology-fst-builder.js → street/morphology-fst-builder.js} +18 -13
- package/out/street/morphology-fst-builder.js.map +1 -0
- package/out/{street-morphology-fst-loader.d.ts → street/morphology-fst-loader.d.ts} +5 -5
- package/out/street/morphology-fst-loader.d.ts.map +1 -0
- package/out/{street-morphology-fst-loader.js → street/morphology-fst-loader.js} +11 -10
- package/out/street/morphology-fst-loader.js.map +1 -0
- package/out/{street-name-lookup.d.ts → street/name-lookup.d.ts} +4 -7
- package/out/street/name-lookup.d.ts.map +1 -0
- package/out/{street-name-lookup.js → street/name-lookup.js} +9 -23
- package/out/street/name-lookup.js.map +1 -0
- package/out/street/normalize.d.ts +179 -0
- package/out/street/normalize.d.ts.map +1 -0
- package/out/street/normalize.js +457 -0
- package/out/street/normalize.js.map +1 -0
- package/out/{street-segment-schema.d.ts → street/segment-schema.d.ts} +13 -9
- package/out/street/segment-schema.d.ts.map +1 -0
- package/out/{street-segment-schema.js → street/segment-schema.js} +4 -4
- package/out/{street-segment-schema.js.map → street/segment-schema.js.map} +1 -1
- package/out/types.d.ts +49 -9
- package/out/types.d.ts.map +1 -1
- package/out/types.js.map +1 -1
- package/out/unified-schema.d.ts +6 -5
- package/out/unified-schema.d.ts.map +1 -1
- package/out/unified-schema.js +16 -19
- package/out/unified-schema.js.map +1 -1
- package/out/uprn/existence.d.ts +53 -0
- package/out/uprn/existence.d.ts.map +1 -0
- package/out/uprn/existence.js +61 -0
- package/out/uprn/existence.js.map +1 -0
- package/out/uprn/index.d.ts +9 -0
- package/out/uprn/index.d.ts.map +1 -0
- package/out/uprn/index.js +9 -0
- package/out/uprn/index.js.map +1 -0
- package/out/uprn/lookup.d.ts +85 -0
- package/out/uprn/lookup.d.ts.map +1 -0
- package/out/uprn/lookup.js +149 -0
- package/out/uprn/lookup.js.map +1 -0
- package/out/uprn/schema.d.ts +93 -0
- package/out/uprn/schema.d.ts.map +1 -0
- package/out/uprn/schema.js +78 -0
- package/out/uprn/schema.js.map +1 -0
- package/out/weights-overlay-linker.d.ts +299 -0
- package/out/weights-overlay-linker.d.ts.map +1 -0
- package/out/weights-overlay-linker.js +511 -0
- package/out/weights-overlay-linker.js.map +1 -0
- package/package.json +622 -154
- package/address-point.ts +0 -125
- package/candidate-lookup.ts +0 -720
- package/fst-autocomplete.ts +0 -204
- package/geo.ts +0 -121
- package/lookup.ts +0 -1354
- package/out/address-point-interpolation.d.ts.map +0 -1
- package/out/address-point-interpolation.js.map +0 -1
- package/out/address-point-schema.d.ts.map +0 -1
- package/out/address-point-schema.js.map +0 -1
- package/out/address-point.d.ts.map +0 -1
- package/out/address-point.js +0 -84
- package/out/address-point.js.map +0 -1
- package/out/ancestry-backfill.d.ts.map +0 -1
- package/out/ancestry-backfill.js.map +0 -1
- package/out/ancestry.d.ts.map +0 -1
- package/out/ancestry.js.map +0 -1
- package/out/convention.d.ts.map +0 -1
- package/out/convention.js.map +0 -1
- package/out/fst-autocomplete.d.ts.map +0 -1
- package/out/fst-autocomplete.js +0 -130
- package/out/fst-autocomplete.js.map +0 -1
- package/out/fst-builder.d.ts.map +0 -1
- package/out/fst-builder.js.map +0 -1
- package/out/fst-deserialize-web.d.ts.map +0 -1
- package/out/fst-deserialize-web.js.map +0 -1
- package/out/fst-freshness.d.ts.map +0 -1
- package/out/fst-freshness.js.map +0 -1
- package/out/fst-matcher.d.ts.map +0 -1
- package/out/fst-matcher.js.map +0 -1
- package/out/fst-serialize.d.ts.map +0 -1
- package/out/fst-serialize.js.map +0 -1
- package/out/fst-types.d.ts.map +0 -1
- package/out/fst-types.js.map +0 -1
- package/out/fts-query.d.ts.map +0 -1
- package/out/fts-query.js.map +0 -1
- package/out/fts.d.ts.map +0 -1
- package/out/fts.js.map +0 -1
- package/out/geo.d.ts +0 -74
- package/out/geo.d.ts.map +0 -1
- package/out/geo.js +0 -71
- package/out/geo.js.map +0 -1
- package/out/geonames-aliases.d.ts.map +0 -1
- package/out/geonames-aliases.js.map +0 -1
- package/out/geonames-postal.d.ts.map +0 -1
- package/out/geonames-postal.js.map +0 -1
- package/out/poi-lookup.d.ts.map +0 -1
- package/out/poi-lookup.js.map +0 -1
- package/out/poi-schema.d.ts.map +0 -1
- package/out/poi-schema.js.map +0 -1
- package/out/postal-city-alias-lookup.d.ts.map +0 -1
- package/out/postal-city-alias-lookup.js.map +0 -1
- package/out/postal-city-candidate-schema.d.ts.map +0 -1
- package/out/sharding.d.ts +0 -114
- package/out/sharding.d.ts.map +0 -1
- package/out/sharding.js.map +0 -1
- package/out/street-centroid-schema.d.ts.map +0 -1
- package/out/street-centroid.d.ts.map +0 -1
- package/out/street-centroid.js.map +0 -1
- package/out/street-morphology-fst-builder.d.ts.map +0 -1
- package/out/street-morphology-fst-builder.js.map +0 -1
- package/out/street-morphology-fst-loader.d.ts.map +0 -1
- package/out/street-morphology-fst-loader.js.map +0 -1
- package/out/street-name-lookup.d.ts.map +0 -1
- package/out/street-name-lookup.js.map +0 -1
- package/out/street-normalize.d.ts +0 -95
- package/out/street-normalize.d.ts.map +0 -1
- package/out/street-normalize.js +0 -272
- package/out/street-normalize.js.map +0 -1
- package/out/street-segment-schema.d.ts.map +0 -1
- package/sqlite-utils.ts +0 -46
- package/street-normalize.ts +0 -318
- /package/{postal-city-alias-schema.ts → lib/postal/city-alias-schema.ts} +0 -0
|
@@ -11,13 +11,16 @@
|
|
|
11
11
|
* PlaceEntry at terminals → return FSTMatcher.
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
|
-
import {
|
|
14
|
+
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
15
|
+
import { resolvePath } from "path-ts"
|
|
15
16
|
|
|
16
|
-
import { readWOFSourceIdentity } from "
|
|
17
|
-
import type { FSTNode } from "
|
|
18
|
-
import { FSTMatcher, normalizeTokens } from "
|
|
19
|
-
import type { BuildFSTOpts, BuildFSTResult, FSTProvenance, PlaceEntry, PlacetypeID } from "
|
|
20
|
-
import { loadImportanceSplit } from "
|
|
17
|
+
import { readWOFSourceIdentity } from "#fst/freshness"
|
|
18
|
+
import type { FSTNode } from "#fst/matcher"
|
|
19
|
+
import { FSTMatcher, normalizeTokens } from "#fst/matcher"
|
|
20
|
+
import type { BuildFSTOpts, BuildFSTResult, FSTProvenance, PlaceEntry, PlacetypeID } from "#fst/types"
|
|
21
|
+
import { loadImportanceSplit } from "#place-importance-schema"
|
|
22
|
+
import type { WOFDatabase } from "#schema"
|
|
23
|
+
import { allRows, getRow } from "#sqlite-utils"
|
|
21
24
|
|
|
22
25
|
const DEFAULT_PLACETYPES: PlacetypeID[] = [
|
|
23
26
|
"country",
|
|
@@ -53,18 +56,19 @@ interface NameRow {
|
|
|
53
56
|
privateuse: string
|
|
54
57
|
}
|
|
55
58
|
|
|
56
|
-
export function buildFSTFromWOF(opts: BuildFSTOpts): {
|
|
59
|
+
export async function buildFSTFromWOF(opts: BuildFSTOpts): Promise<{
|
|
57
60
|
matcher: FSTMatcher
|
|
58
61
|
provenance: FSTProvenance
|
|
59
62
|
result: BuildFSTResult
|
|
60
|
-
} {
|
|
63
|
+
}> {
|
|
61
64
|
const countries = opts.countries ?? DEFAULT_COUNTRIES
|
|
62
65
|
const placetypes = opts.placetypes ?? DEFAULT_PLACETYPES
|
|
63
66
|
const languages = opts.languages ?? DEFAULT_LANGUAGES
|
|
64
67
|
const progress = opts.onProgress ?? (() => {})
|
|
68
|
+
const dbPath = resolvePath(opts.dbPath)
|
|
65
69
|
|
|
66
|
-
progress("open",
|
|
67
|
-
|
|
70
|
+
progress("open", dbPath)
|
|
71
|
+
using db = new DatabaseClient<WOFDatabase>(dbPath, { open: true })
|
|
68
72
|
|
|
69
73
|
// Phase 1: Load all matching SPR rows.
|
|
70
74
|
progress("spr", `Loading places for countries=[${countries}], placetypes=[${placetypes}]`)
|
|
@@ -78,7 +82,7 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
|
|
|
78
82
|
AND placetype IN (${placeholders(placetypes)})`
|
|
79
83
|
)
|
|
80
84
|
|
|
81
|
-
const sprRows = sprStmt
|
|
85
|
+
const sprRows = allRows<SprRow>(sprStmt, ...countries, ...placetypes)
|
|
82
86
|
progress("spr", `Loaded ${sprRows.length} places`)
|
|
83
87
|
|
|
84
88
|
// Phase 2: Build a lookup for parent chain resolution.
|
|
@@ -103,8 +107,8 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
|
|
|
103
107
|
for (let i = 0; i < orphanIDs.length; i += ANCESTOR_CHUNK) {
|
|
104
108
|
const chunk = orphanIDs.slice(i, i + ANCESTOR_CHUNK)
|
|
105
109
|
|
|
106
|
-
const rows =
|
|
107
|
-
.prepare(
|
|
110
|
+
const rows = allRows<{ id: number; ancestor_id: number }>(
|
|
111
|
+
db.prepare(
|
|
108
112
|
`SELECT DISTINCT id, ancestor_id FROM ancestors
|
|
109
113
|
WHERE id IN (${chunk.map(() => "?").join(",")}) AND ancestor_placetype IN ('country', 'region', 'county')
|
|
110
114
|
ORDER BY id, CASE ancestor_placetype
|
|
@@ -112,8 +116,9 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
|
|
|
112
116
|
WHEN 'region' THEN 2
|
|
113
117
|
WHEN 'country' THEN 3
|
|
114
118
|
END`
|
|
115
|
-
)
|
|
116
|
-
|
|
119
|
+
),
|
|
120
|
+
...chunk
|
|
121
|
+
)
|
|
117
122
|
|
|
118
123
|
for (const row of rows) {
|
|
119
124
|
let chain = ancestorsByID.get(row.id)
|
|
@@ -150,7 +155,7 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
|
|
|
150
155
|
let parentRow = sprByID.get(current)
|
|
151
156
|
|
|
152
157
|
if (!parentRow) {
|
|
153
|
-
const fetched = parentStmt
|
|
158
|
+
const fetched = getRow<SprRow>(parentStmt, current)
|
|
154
159
|
|
|
155
160
|
if (!fetched) break
|
|
156
161
|
parentRow = fetched
|
|
@@ -186,13 +191,13 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
|
|
|
186
191
|
|
|
187
192
|
// Phase 4: Load names for matching places.
|
|
188
193
|
progress("names", "Loading name variants")
|
|
189
|
-
const
|
|
194
|
+
const placeIDs = sprRows.map((r) => r.id)
|
|
190
195
|
const namesByPlace = new Map<number, string[]>()
|
|
191
196
|
|
|
192
197
|
const allLanguages = languages.includes("*")
|
|
193
198
|
|
|
194
|
-
for (let i = 0; i <
|
|
195
|
-
const chunk =
|
|
199
|
+
for (let i = 0; i < placeIDs.length; i += 500) {
|
|
200
|
+
const chunk = placeIDs.slice(i, i + 500)
|
|
196
201
|
const idPlaceholders = chunk.map(() => "?").join(",")
|
|
197
202
|
|
|
198
203
|
const nameStmt = allLanguages
|
|
@@ -201,9 +206,9 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
|
|
|
201
206
|
`SELECT id, name, language, privateuse FROM names WHERE id IN (${idPlaceholders}) AND language IN (${languages.map(() => "?").join(",")})`
|
|
202
207
|
)
|
|
203
208
|
|
|
204
|
-
const nameRows =
|
|
205
|
-
? nameStmt
|
|
206
|
-
: nameStmt
|
|
209
|
+
const nameRows = allLanguages
|
|
210
|
+
? allRows<NameRow>(nameStmt, ...chunk)
|
|
211
|
+
: allRows<NameRow>(nameStmt, ...chunk, ...languages)
|
|
207
212
|
|
|
208
213
|
for (const row of nameRows) {
|
|
209
214
|
const existing = namesByPlace.get(row.id) ?? []
|
|
@@ -322,8 +327,6 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
|
|
|
322
327
|
}
|
|
323
328
|
}
|
|
324
329
|
|
|
325
|
-
db.close()
|
|
326
|
-
|
|
327
330
|
progress(
|
|
328
331
|
"done",
|
|
329
332
|
`Built trie: ${nodes.length} states, ${insertCount} name insertions` +
|
|
@@ -339,8 +342,8 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
|
|
|
339
342
|
// and is free whenever the `.md5` sidecar is current, which the admin build already writes.
|
|
340
343
|
// `sourceIdentity` lets a caller that already knows the digest (or is building from something that
|
|
341
344
|
// is not a file at all) supply it instead.
|
|
342
|
-
progress("stamp", `Reading source identity for ${
|
|
343
|
-
const source = opts.sourceIdentity ?? readWOFSourceIdentity(
|
|
345
|
+
progress("stamp", `Reading source identity for ${dbPath}`)
|
|
346
|
+
const source = opts.sourceIdentity ?? (await readWOFSourceIdentity(dbPath))
|
|
344
347
|
|
|
345
348
|
const provenance: FSTProvenance = {
|
|
346
349
|
builtAt: new Date().toISOString(),
|
|
@@ -352,7 +355,7 @@ export function buildFSTFromWOF(opts: BuildFSTOpts): {
|
|
|
352
355
|
importanceMatches: split.referential.size,
|
|
353
356
|
encyclopedicMatches: split.encyclopedic.size,
|
|
354
357
|
importanceSource: split.source,
|
|
355
|
-
sourceDB:
|
|
358
|
+
sourceDB: dbPath,
|
|
356
359
|
sourceDBMD5: source.md5,
|
|
357
360
|
sourceDBBytes: source.bytes,
|
|
358
361
|
...(excludeSurfaces !== undefined || excludeAllTokensOf !== undefined
|
|
@@ -10,87 +10,27 @@
|
|
|
10
10
|
* build-time operation).
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import { tryParsingJSON } from "@mailwoman/core/
|
|
14
|
-
|
|
15
|
-
import
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
/**
|
|
36
|
-
* First format version carrying the trailing metadata block; older files simply have none.
|
|
37
|
-
*/
|
|
38
|
-
const VERSION_WITH_METADATA = 3
|
|
39
|
-
|
|
40
|
-
const HEADER_SIZE = 32
|
|
41
|
-
const EDGE_ENTRY_SIZE = 8
|
|
42
|
-
|
|
43
|
-
/**
|
|
44
|
-
* Format version that split the single `importance` float into `referential` + `encyclopedic` (ROAD_TO_V9 §2 R1),
|
|
45
|
-
* growing the place entry from 56 to 60 bytes. Mirrors `fst-serialize.ts`'s constant of the same name.
|
|
46
|
-
*/
|
|
47
|
-
const VERSION_TWO_SCORE_SPLIT = 5
|
|
48
|
-
|
|
49
|
-
/**
|
|
50
|
-
* Place-entry size in bytes at or above {@link VERSION_TWO_SCORE_SPLIT}.
|
|
51
|
-
*/
|
|
52
|
-
const SPLIT_PLACE_ENTRY_SIZE = 60
|
|
53
|
-
|
|
54
|
-
/**
|
|
55
|
-
* Place-entry size in bytes below {@link VERSION_TWO_SCORE_SPLIT}.
|
|
56
|
-
*/
|
|
57
|
-
const LEGACY_PLACE_ENTRY_SIZE = 56
|
|
58
|
-
|
|
59
|
-
/**
|
|
60
|
-
* Byte offset of the encyclopedic float inside a v5 place entry — immediately after the 8-slot parent chain.
|
|
61
|
-
*/
|
|
62
|
-
const ENCYCLOPEDIC_OFFSET = 56
|
|
63
|
-
|
|
64
|
-
/**
|
|
65
|
-
* `placeFlags` bit 0 (byte `pp+7`, v5+): this place carries an encyclopedic score. Per-place because absence is the
|
|
66
|
-
* common case; see `fst-serialize.ts`.
|
|
67
|
-
*/
|
|
68
|
-
const PLACE_FLAG_HAS_ENCYCLOPEDIC = 1
|
|
69
|
-
/**
|
|
70
|
-
* "FST\0".
|
|
71
|
-
*/
|
|
72
|
-
const MAGIC_BYTES = [0x46, 0x53, 0x54, 0x00]
|
|
73
|
-
/**
|
|
74
|
-
* Must track the serializer's VERSION (fst-serialize.ts, currently 5). The v3 provenance + v4 16-byte-state/u32-count
|
|
75
|
-
* layout logic below already matches the Node deserializer; only this gate was left stale at 2, so the browser FST
|
|
76
|
-
* loader rejected every real (v4) artifact. It was left stale again at 4 by the v5 two-score split until this line
|
|
77
|
-
* moved with it — the gate is a SEPARATE number from the layout branches, which is exactly why it keeps drifting.
|
|
78
|
-
*/
|
|
79
|
-
const MAX_VERSION = 5
|
|
80
|
-
|
|
81
|
-
const PLACETYPE_ORDER: readonly PlacetypeID[] = [
|
|
82
|
-
"country",
|
|
83
|
-
"region",
|
|
84
|
-
"county",
|
|
85
|
-
"locality",
|
|
86
|
-
"localadmin",
|
|
87
|
-
"borough",
|
|
88
|
-
"neighbourhood",
|
|
89
|
-
"postalcode",
|
|
90
|
-
"campus",
|
|
91
|
-
"dependency",
|
|
92
|
-
"street_affix",
|
|
93
|
-
]
|
|
13
|
+
import { tryParsingJSON } from "@mailwoman/core/json"
|
|
14
|
+
|
|
15
|
+
import {
|
|
16
|
+
EDGE_ENTRY_SIZE,
|
|
17
|
+
ENCYCLOPEDIC_OFFSET,
|
|
18
|
+
FST_FORMAT_VERSION,
|
|
19
|
+
FST_MAGIC_BYTES,
|
|
20
|
+
HEADER_SIZE,
|
|
21
|
+
LEGACY_PLACE_ENTRY_SIZE,
|
|
22
|
+
NARROW_STATE_ENTRY_SIZE,
|
|
23
|
+
PLACE_FLAG_HAS_ENCYCLOPEDIC,
|
|
24
|
+
PLACETYPE_ORDER,
|
|
25
|
+
SPLIT_PLACE_ENTRY_SIZE,
|
|
26
|
+
VERSION_TWO_SCORE_SPLIT,
|
|
27
|
+
VERSION_WIDE_STATE_COUNTERS,
|
|
28
|
+
VERSION_WITH_METADATA,
|
|
29
|
+
WIDE_STATE_ENTRY_SIZE,
|
|
30
|
+
} from "#fst/format"
|
|
31
|
+
import type { FSTNode } from "#fst/matcher"
|
|
32
|
+
import { FSTMatcher } from "#fst/matcher"
|
|
33
|
+
import type { FSTProvenance, PlaceEntry } from "#fst/types"
|
|
94
34
|
|
|
95
35
|
export function deserializeFSTWeb(input: ArrayBuffer | Uint8Array): FSTMatcher {
|
|
96
36
|
const bytes = input instanceof ArrayBuffer ? new Uint8Array(input) : input
|
|
@@ -100,18 +40,18 @@ export function deserializeFSTWeb(input: ArrayBuffer | Uint8Array): FSTMatcher {
|
|
|
100
40
|
if (bytes.byteLength < HEADER_SIZE) throw new Error("FST buffer too small for header")
|
|
101
41
|
|
|
102
42
|
if (
|
|
103
|
-
bytes[0] !==
|
|
104
|
-
bytes[1] !==
|
|
105
|
-
bytes[2] !==
|
|
106
|
-
bytes[3] !==
|
|
43
|
+
bytes[0] !== FST_MAGIC_BYTES[0] ||
|
|
44
|
+
bytes[1] !== FST_MAGIC_BYTES[1] ||
|
|
45
|
+
bytes[2] !== FST_MAGIC_BYTES[2] ||
|
|
46
|
+
bytes[3] !== FST_MAGIC_BYTES[3]
|
|
107
47
|
) {
|
|
108
48
|
throw new Error("FST magic mismatch")
|
|
109
49
|
}
|
|
110
50
|
|
|
111
51
|
const version = view.getUint16(4, true)
|
|
112
52
|
|
|
113
|
-
if (version < 1 || version >
|
|
114
|
-
throw new Error(`FST version ${version} unsupported (expected 1..${
|
|
53
|
+
if (version < 1 || version > FST_FORMAT_VERSION) {
|
|
54
|
+
throw new Error(`FST version ${version} unsupported (expected 1..${FST_FORMAT_VERSION})`)
|
|
115
55
|
}
|
|
116
56
|
|
|
117
57
|
const isV2 = version >= 2
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* The FST binary-format constants, shared by the Node serializer/deserializer (`fst-serialize.ts`)
|
|
7
|
+
* and the browser deserializer (`fst-deserialize-web.ts`). Platform-free by design — no Buffer at
|
|
8
|
+
* module scope; both runtimes convert {@link FST_MAGIC_BYTES} themselves.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { PlacetypeID } from "#fst/types"
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Format version this tree WRITES and the maximum either deserializer accepts, published so a freshness guard can call
|
|
15
|
+
* an older artifact format-stale without re-typing the number (mirrors `REQUIRED_PAIR_INDEX_SCHEMA`'s role for PIX1 —
|
|
16
|
+
* see `fst-freshness.ts`).
|
|
17
|
+
*
|
|
18
|
+
* ONE constant for the writer and both readers, deliberately. The browser reader's acceptance criterion was a SEPARATE
|
|
19
|
+
* `MAX_VERSION` number from the layout branches, and it drifted twice: left stale at 2 when the v4 wide-state layout
|
|
20
|
+
* shipped (rejecting every real artifact), and stale again at 4 through the v5 two-score split until the line moved by
|
|
21
|
+
* hand. A check that is the writer's version cannot drift from it.
|
|
22
|
+
*/
|
|
23
|
+
export const FST_FORMAT_VERSION = 5
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Fixed header size in bytes: magic, version, and the section offsets that follow it.
|
|
27
|
+
*/
|
|
28
|
+
export const HEADER_SIZE = 32
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Edge-table entry: the transition label and the target state index.
|
|
32
|
+
*/
|
|
33
|
+
export const EDGE_ENTRY_SIZE = 8
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Format version that widened the per-state edge and place counters from 16 to 32 bits, growing the state entry from 12
|
|
37
|
+
* to 16 bytes. Readers branch on it to stay backward-compatible with v2/v3 files.
|
|
38
|
+
*/
|
|
39
|
+
export const VERSION_WIDE_STATE_COUNTERS = 4
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* State-table entry size in bytes at or above {@link VERSION_WIDE_STATE_COUNTERS}.
|
|
43
|
+
*/
|
|
44
|
+
export const WIDE_STATE_ENTRY_SIZE = 16
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* State-table entry size in bytes below {@link VERSION_WIDE_STATE_COUNTERS}.
|
|
48
|
+
*/
|
|
49
|
+
export const NARROW_STATE_ENTRY_SIZE = 12
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* First format version carrying the trailing metadata block; older files simply have none.
|
|
53
|
+
*/
|
|
54
|
+
export const VERSION_WITH_METADATA = 3
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Format version that split the single `importance` float into `referential` + `encyclopedic` (ROAD_TO_V9 §2 R1),
|
|
58
|
+
* growing the place entry from 56 to 60 bytes and claiming the previously-reserved `pp+7` byte as
|
|
59
|
+
* {@link PLACE_FLAG_HAS_ENCYCLOPEDIC}.
|
|
60
|
+
*/
|
|
61
|
+
export const VERSION_TWO_SCORE_SPLIT = 5
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Place-entry size in bytes at or above {@link VERSION_TWO_SCORE_SPLIT}.
|
|
65
|
+
*/
|
|
66
|
+
export const SPLIT_PLACE_ENTRY_SIZE = 60
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Place-entry size in bytes below {@link VERSION_TWO_SCORE_SPLIT}.
|
|
70
|
+
*/
|
|
71
|
+
export const LEGACY_PLACE_ENTRY_SIZE = 56
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Byte offset of the encyclopedic float inside a v5 place entry — immediately after the 8-slot parent chain.
|
|
75
|
+
*/
|
|
76
|
+
export const ENCYCLOPEDIC_OFFSET = 56
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* `placeFlags` bit 0 (byte `pp+7`, v5+): this place carries an encyclopedic score. Per-PLACE rather than per-file
|
|
80
|
+
* because absence is the common case — roughly 89% of the 2026-08-05 gazetteer has no Wikipedia article — and a
|
|
81
|
+
* file-level flag would force every one of those rows to claim a 0 it never had.
|
|
82
|
+
*/
|
|
83
|
+
export const PLACE_FLAG_HAS_ENCYCLOPEDIC = 1
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* File magic, "FST\0" as bytes. A reader rejects anything not starting with these four bytes before parsing further.
|
|
87
|
+
*/
|
|
88
|
+
export const FST_MAGIC_BYTES: readonly number[] = [0x46, 0x53, 0x54, 0x00]
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Placetypes in hierarchy order, largest first. The index into this array is what gets written into a place entry, so
|
|
92
|
+
* REORDERING IT BREAKS EVERY EXISTING FILE — append instead, and bump the version.
|
|
93
|
+
*/
|
|
94
|
+
export const PLACETYPE_ORDER: readonly PlacetypeID[] = [
|
|
95
|
+
"country",
|
|
96
|
+
"region",
|
|
97
|
+
"county",
|
|
98
|
+
"locality",
|
|
99
|
+
"localadmin",
|
|
100
|
+
"borough",
|
|
101
|
+
"neighbourhood",
|
|
102
|
+
"postalcode",
|
|
103
|
+
"campus",
|
|
104
|
+
"dependency",
|
|
105
|
+
"street_affix",
|
|
106
|
+
]
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* `FSTProvenance` already recorded `sourceDB` — the source's PATH, which is exactly the field that
|
|
16
16
|
* cannot change when the bytes behind it do. So the stamp gains the source's IDENTITY (md5 + byte
|
|
17
17
|
* size) and this module compares it. The shape deliberately mirrors
|
|
18
|
-
*
|
|
18
|
+
* `@mailwoman/resolver-wof-sqlite/weights-overlay-linker`'s `pairIndexStaleReason`: one function returning a reason
|
|
19
19
|
* string or `undefined`, so a fact added to the stamp cannot be checked by some callers and not
|
|
20
20
|
* others — which is how three of the four base linkers ended up unable to notice a PIX1 schema bump.
|
|
21
21
|
*
|
|
@@ -29,13 +29,14 @@
|
|
|
29
29
|
* and continue.
|
|
30
30
|
*/
|
|
31
31
|
|
|
32
|
-
import {
|
|
33
|
-
import {
|
|
32
|
+
import { pathExists, readFileRange, readLocalTextFile, statPath } from "@mailwoman/core/fs/readers"
|
|
33
|
+
import { writeLocalTextFile } from "@mailwoman/core/fs/writers"
|
|
34
|
+
import { md5File } from "@mailwoman/core/hash"
|
|
35
|
+
import { tryParsingJSON } from "@mailwoman/core/json"
|
|
36
|
+
import { basename, resolvePath, type PathBuilderLike } from "path-ts"
|
|
34
37
|
|
|
35
|
-
import {
|
|
36
|
-
|
|
37
|
-
import { FST_FORMAT_VERSION } from "./fst-serialize.ts"
|
|
38
|
-
import type { FSTProvenance } from "./fst-types.ts"
|
|
38
|
+
import { FST_FORMAT_VERSION } from "#fst/serialize"
|
|
39
|
+
import type { FSTProvenance } from "#fst/types"
|
|
39
40
|
|
|
40
41
|
/**
|
|
41
42
|
* Fixed header size in bytes — mirrors `fst-serialize.ts`'s `HEADER_SIZE`. Duplicated rather than exported across
|
|
@@ -65,12 +66,6 @@ export const MIN_STAMPED_FORMAT_VERSION = 3
|
|
|
65
66
|
*/
|
|
66
67
|
const MD5_HEX_LENGTH = 32
|
|
67
68
|
|
|
68
|
-
/**
|
|
69
|
-
* Bytes per read in {@link md5FileSync}. 8 MiB measured at 7.3 s for the 5.27 GB admin DB on the lab playpen — the same
|
|
70
|
-
* throughput as `md5sum(1)`, so the sidecar below is what makes the check cheap, not the chunk size.
|
|
71
|
-
*/
|
|
72
|
-
const MD5_CHUNK_BYTES = 8 * 1024 * 1024
|
|
73
|
-
|
|
74
69
|
/**
|
|
75
70
|
* What an FST was built from, recorded so a later reader can tell whether that thing still exists.
|
|
76
71
|
*
|
|
@@ -121,69 +116,37 @@ export interface FSTExpectation {
|
|
|
121
116
|
* Returns `undefined` for a file that is absent, too small, or not an FST at all — none of which is this function's
|
|
122
117
|
* business to diagnose.
|
|
123
118
|
*/
|
|
124
|
-
export function peekFSTStampFields(path: string): FSTStampFields | undefined {
|
|
125
|
-
if (!
|
|
126
|
-
const size =
|
|
119
|
+
export async function peekFSTStampFields(path: string): Promise<FSTStampFields | undefined> {
|
|
120
|
+
if (!(await pathExists(path))) return undefined
|
|
121
|
+
const size = (await statPath(path)).size
|
|
127
122
|
|
|
128
123
|
if (size < HEADER_SIZE) return undefined
|
|
129
|
-
const
|
|
124
|
+
const header = await readFileRange(path, 0, HEADER_SIZE)
|
|
130
125
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
readSync(fd, header, 0, HEADER_SIZE, 0)
|
|
126
|
+
if (header.readUInt32LE(0) !== MAGIC) return undefined
|
|
127
|
+
const formatVersion = header.readUInt16LE(4)
|
|
134
128
|
|
|
135
|
-
|
|
136
|
-
|
|
129
|
+
if (formatVersion < MIN_STAMPED_FORMAT_VERSION) return { formatVersion, provenance: undefined }
|
|
130
|
+
const trailerStart = header.readUInt32LE(PROVENANCE_OFFSET_FIELD)
|
|
137
131
|
|
|
138
|
-
|
|
139
|
-
|
|
132
|
+
// 0 = "this build wrote no trailer" (the serializer's own encoding); anything past EOF is a
|
|
133
|
+
// truncated file. Both read as "no stamp", which is what the caller does with them anyway.
|
|
134
|
+
if (trailerStart === 0 || trailerStart + 4 > size) return { formatVersion, provenance: undefined }
|
|
140
135
|
|
|
141
|
-
|
|
142
|
-
// truncated file. Both read as "no stamp", which is what the caller does with them anyway.
|
|
143
|
-
if (trailerStart === 0 || trailerStart + 4 > size) return { formatVersion, provenance: undefined }
|
|
136
|
+
const jsonLength = (await readFileRange(path, trailerStart, 4)).readUInt32LE(0)
|
|
144
137
|
|
|
145
|
-
|
|
146
|
-
readSync(fd, lengthBytes, 0, 4, trailerStart)
|
|
147
|
-
const jsonLength = lengthBytes.readUInt32LE(0)
|
|
138
|
+
if (jsonLength === 0 || trailerStart + 4 + jsonLength > size) return { formatVersion, provenance: undefined }
|
|
148
139
|
|
|
149
|
-
|
|
140
|
+
const json = await readFileRange(path, trailerStart + 4, jsonLength)
|
|
150
141
|
|
|
151
|
-
|
|
152
|
-
readSync(fd, json, 0, jsonLength, trailerStart + 4)
|
|
153
|
-
|
|
154
|
-
return { formatVersion, provenance: tryParsingJSON<FSTProvenance>(json.toString("utf8")) ?? undefined }
|
|
155
|
-
} finally {
|
|
156
|
-
closeSync(fd)
|
|
157
|
-
}
|
|
142
|
+
return { formatVersion, provenance: tryParsingJSON<FSTProvenance>(json.toString("utf8")) ?? undefined }
|
|
158
143
|
}
|
|
159
144
|
|
|
160
145
|
/**
|
|
161
|
-
*
|
|
162
|
-
*
|
|
163
|
-
* The async `md5File` in `@mailwoman/core/utils` is the one to reach for anywhere else. This exists because the FST
|
|
164
|
-
* builder and its whole call chain are synchronous by design (`buildFSTFromWOF` → `buildLocaleFSTs`), and making them
|
|
165
|
-
* async to stamp a checksum would cascade through the Pastel commands and the tests for one hash. It reads in
|
|
166
|
-
* {@link MD5_CHUNK_BYTES} chunks rather than `readFileSync` — the source is a multi-gigabyte database.
|
|
146
|
+
* Re-exported from `@mailwoman/core/hash`. This package's own test imports it from here, and the sidecar convention
|
|
147
|
+
* below is what it is for.
|
|
167
148
|
*/
|
|
168
|
-
export
|
|
169
|
-
const hash = createHash("md5")
|
|
170
|
-
const fd = openSync(path, "r")
|
|
171
|
-
|
|
172
|
-
try {
|
|
173
|
-
const chunk = Buffer.alloc(MD5_CHUNK_BYTES)
|
|
174
|
-
|
|
175
|
-
for (;;) {
|
|
176
|
-
const read = readSync(fd, chunk, 0, MD5_CHUNK_BYTES, null)
|
|
177
|
-
|
|
178
|
-
if (read <= 0) break
|
|
179
|
-
hash.update(chunk.subarray(0, read))
|
|
180
|
-
}
|
|
181
|
-
} finally {
|
|
182
|
-
closeSync(fd)
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
return hash.digest("hex")
|
|
186
|
-
}
|
|
149
|
+
export { md5File } from "@mailwoman/core/hash"
|
|
187
150
|
|
|
188
151
|
/**
|
|
189
152
|
* The source identity an FST build should stamp, or a check should compare against.
|
|
@@ -197,8 +160,12 @@ export function md5FileSync(path: string): string {
|
|
|
197
160
|
* write and the caller still gets its answer, because refusing to check freshness on a read-only tree would be the
|
|
198
161
|
* wrong trade.
|
|
199
162
|
*/
|
|
200
|
-
export function readWOFSourceIdentity(
|
|
201
|
-
|
|
163
|
+
export async function readWOFSourceIdentity(
|
|
164
|
+
source: PathBuilderLike,
|
|
165
|
+
{ refreshSidecar = true } = {}
|
|
166
|
+
): Promise<FSTSourceIdentity> {
|
|
167
|
+
const path = resolvePath(source)
|
|
168
|
+
const stats = await statPath(path)
|
|
202
169
|
const memoKey = `${path}\0${stats.mtimeMs}\0${stats.size}`
|
|
203
170
|
const hit = sourceIdentityMemo.get(memoKey)
|
|
204
171
|
|
|
@@ -206,11 +173,11 @@ export function readWOFSourceIdentity(path: string, { refreshSidecar = true } =
|
|
|
206
173
|
const sidecarPath = `${path}.md5`
|
|
207
174
|
let md5: string | undefined
|
|
208
175
|
|
|
209
|
-
if (
|
|
210
|
-
const sidecarStats =
|
|
176
|
+
if (await pathExists(sidecarPath)) {
|
|
177
|
+
const sidecarStats = await statPath(sidecarPath)
|
|
211
178
|
|
|
212
179
|
if (sidecarStats.mtimeMs >= stats.mtimeMs) {
|
|
213
|
-
const [hash] =
|
|
180
|
+
const [hash] = (await readLocalTextFile(sidecarPath)).trim().split(/\s+/)
|
|
214
181
|
|
|
215
182
|
if (hash && hash.length === MD5_HEX_LENGTH) {
|
|
216
183
|
md5 = hash
|
|
@@ -219,11 +186,11 @@ export function readWOFSourceIdentity(path: string, { refreshSidecar = true } =
|
|
|
219
186
|
}
|
|
220
187
|
|
|
221
188
|
if (!md5) {
|
|
222
|
-
md5 =
|
|
189
|
+
md5 = await md5File(path)
|
|
223
190
|
|
|
224
191
|
if (refreshSidecar) {
|
|
225
192
|
try {
|
|
226
|
-
|
|
193
|
+
await writeLocalTextFile(`${md5} ${basename(path)}\n`, sidecarPath)
|
|
227
194
|
} catch {
|
|
228
195
|
// Read-only data root — the digest is still correct, it just isn't cached.
|
|
229
196
|
}
|
|
@@ -293,7 +260,7 @@ export function fstStaleReason(fields: FSTStampFields | undefined, expected: FST
|
|
|
293
260
|
* Returns `undefined` when the artifact is current OR when it is absent — an absent artifact is a different problem
|
|
294
261
|
* with a different message, and every existing caller already reports it in place.
|
|
295
262
|
*/
|
|
296
|
-
export function fstFreshnessWarning({
|
|
263
|
+
export async function fstFreshnessWarning({
|
|
297
264
|
fstPath,
|
|
298
265
|
sourceDBPath,
|
|
299
266
|
formatVersion,
|
|
@@ -305,11 +272,11 @@ export function fstFreshnessWarning({
|
|
|
305
272
|
formatVersion?: number
|
|
306
273
|
exclusionPolicy?: string
|
|
307
274
|
rebuildCommand: string
|
|
308
|
-
}): string | undefined {
|
|
309
|
-
if (!
|
|
275
|
+
}): Promise<string | undefined> {
|
|
276
|
+
if (!(await pathExists(fstPath)) || !(await pathExists(sourceDBPath))) return undefined
|
|
310
277
|
|
|
311
|
-
const reason = fstStaleReason(peekFSTStampFields(fstPath), {
|
|
312
|
-
source: readWOFSourceIdentity(sourceDBPath),
|
|
278
|
+
const reason = fstStaleReason(await peekFSTStampFields(fstPath), {
|
|
279
|
+
source: await readWOFSourceIdentity(sourceDBPath),
|
|
313
280
|
...(formatVersion === undefined ? {} : { formatVersion }),
|
|
314
281
|
...(exclusionPolicy === undefined ? {} : { exclusionPolicy }),
|
|
315
282
|
})
|
package/lib/fst/index.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
export * from "#fst/autocomplete"
|
|
8
|
+
export * from "#fst/builder"
|
|
9
|
+
export * from "#fst/deserialize-web"
|
|
10
|
+
export * from "#fst/format"
|
|
11
|
+
export * from "#fst/freshness"
|
|
12
|
+
export * from "#fst/matcher"
|
|
13
|
+
export * from "#fst/serialize"
|
|
14
|
+
export * from "#fst/types"
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* arrays at accepting states.
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
import type { FSTContinuation, FSTMatchResult, FSTQueryResult, PlaceEntry } from "
|
|
11
|
+
import type { FSTContinuation, FSTMatchResult, FSTQueryResult, PlaceEntry } from "#fst/types"
|
|
12
12
|
|
|
13
13
|
interface FSTNode {
|
|
14
14
|
edges: Map<string, number>
|