@mailwoman/resolver-wof-sqlite 9.4.0 → 10.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/lib/address/{point-interpolation.ts → point/interpolation.ts} +40 -38
- package/lib/address/point/schema.ts +132 -0
- package/lib/address/point.ts +68 -81
- package/lib/{address/index.ts → address.ts} +2 -2
- package/lib/ancestry/backfill.ts +24 -88
- package/lib/{ancestry/index.ts → ancestry.ts} +19 -15
- package/lib/build/candidate.ts +608 -0
- package/lib/build/population-corrections.ts +64 -0
- package/lib/build/slim.ts +338 -0
- package/lib/candidate/alias-bags.ts +74 -14
- package/lib/candidate/ancestors/schema.ts +211 -0
- package/lib/candidate/{ancestors-sidecar.ts → ancestors/sidecar.ts} +17 -24
- package/lib/candidate/country-display-names.ts +6 -20
- package/lib/candidate/extract-fold.ts +12 -57
- package/lib/candidate/fts.ts +28 -0
- package/lib/candidate/importance.ts +221 -0
- package/lib/candidate/lookup.ts +669 -0
- package/lib/candidate/name-roles.ts +16 -34
- package/lib/candidate/own-name.ts +34 -31
- package/lib/candidate/place-attrs.ts +20 -13
- package/lib/candidate/schema.ts +211 -0
- package/lib/candidate/scoring.ts +222 -0
- package/lib/capital-schema.ts +21 -17
- package/lib/capitals.ts +26 -24
- package/lib/coincident-roles.ts +74 -55
- package/lib/convention/schema.ts +13 -7
- package/lib/convention.ts +144 -0
- package/lib/coverage-manifest-schema.ts +34 -39
- package/lib/currency-backfill.ts +24 -69
- package/lib/env.ts +1 -1
- package/lib/exact-match.ts +31 -20
- package/lib/extracts.ts +96 -73
- package/lib/fst/autocomplete.ts +52 -35
- package/lib/fst/builder.ts +30 -36
- package/lib/fst/deserialize-web.ts +8 -9
- package/lib/fst/format.ts +41 -23
- package/lib/fst/freshness.ts +17 -95
- package/lib/fst/matcher.ts +2 -2
- package/lib/fst/serialize.ts +45 -60
- package/lib/fst/types.ts +71 -59
- package/lib/fts/query.ts +5 -36
- package/lib/fts.ts +280 -0
- package/lib/geonames/aliases.ts +72 -149
- package/lib/geonames/postal.ts +162 -77
- package/lib/index.ts +20 -27
- package/lib/interpolation.ts +43 -96
- package/lib/lookup.ts +137 -257
- package/lib/name-score.ts +9 -7
- package/lib/nsul/lookup.ts +36 -53
- package/lib/nsul/schema.ts +16 -43
- package/lib/paths.ts +159 -0
- package/lib/place-importance-schema.ts +57 -164
- package/lib/poi/lookup.ts +100 -109
- package/lib/poi/schema.ts +42 -26
- package/lib/polygon-schema.ts +10 -6
- package/lib/postal/city/alias/lookup.ts +79 -0
- package/lib/postal/{city-alias-schema.ts → city/alias/schema.ts} +14 -10
- package/lib/postal/{city-candidate-schema.ts → city/candidate-schema.ts} +26 -20
- package/lib/postal.ts +9 -0
- package/lib/postcode-point-lookup.ts +9 -19
- package/lib/primary-preference.ts +45 -125
- package/lib/proximity-rerank.ts +43 -32
- package/lib/ranking-weights.ts +98 -76
- package/lib/region-keys.ts +68 -40
- package/lib/reverse.ts +108 -72
- package/lib/schema.ts +74 -42
- package/lib/search-fetch.ts +90 -68
- package/lib/sqlite/convention-source.ts +57 -0
- package/lib/sqlite/utils.ts +79 -0
- package/lib/street/centroid/schema.ts +149 -0
- package/lib/street/centroid.ts +16 -25
- package/lib/street/{morphology-fst-builder.ts → morphology/fst/builder.ts} +56 -41
- package/lib/street/{morphology-fst-loader.ts → morphology/fst/loader.ts} +36 -29
- package/lib/street/name-lookup.ts +43 -35
- package/lib/street/normalize.ts +85 -216
- package/lib/street/segment-schema.ts +116 -43
- package/lib/{street/index.ts → street.ts} +3 -3
- package/lib/types.ts +94 -140
- package/lib/unified-schema.ts +29 -29
- package/lib/uprn/existence.ts +30 -19
- package/lib/uprn/lookup.ts +28 -67
- package/lib/uprn/schema.ts +18 -40
- package/lib/weights-overlay-linker.ts +204 -232
- package/out/address/point/interpolation.d.ts +45 -0
- package/out/address/point/interpolation.d.ts.map +1 -0
- package/out/address/{point-interpolation.js → point/interpolation.js} +38 -38
- package/out/address/point/interpolation.js.map +1 -0
- package/out/address/point/schema.d.ts +74 -0
- package/out/address/point/schema.d.ts.map +1 -0
- package/out/address/{point-schema.js → point/schema.js} +4 -20
- package/out/address/point/schema.js.map +1 -0
- package/out/address/point.d.ts +24 -21
- package/out/address/point.d.ts.map +1 -1
- package/out/address/point.js +60 -77
- package/out/address/point.js.map +1 -1
- package/out/{address/index.js → address.d.ts} +3 -3
- package/out/address.d.ts.map +1 -0
- package/out/{address/index.d.ts → address.js} +3 -3
- package/out/address.js.map +1 -0
- package/out/ancestry/backfill.d.ts +11 -64
- package/out/ancestry/backfill.d.ts.map +1 -1
- package/out/ancestry/backfill.js +18 -76
- package/out/ancestry/backfill.js.map +1 -1
- package/out/ancestry.d.ts +49 -0
- package/out/ancestry.d.ts.map +1 -0
- package/out/ancestry.js +59 -0
- package/out/ancestry.js.map +1 -0
- package/out/build/candidate.d.ts +170 -0
- package/out/build/candidate.d.ts.map +1 -0
- package/out/{build-candidate.js → build/candidate.js} +33 -102
- package/out/build/candidate.js.map +1 -0
- package/out/build/population-corrections.d.ts +52 -0
- package/out/build/population-corrections.d.ts.map +1 -0
- package/out/build/population-corrections.js +40 -0
- package/out/build/population-corrections.js.map +1 -0
- package/out/build/slim.d.ts +44 -0
- package/out/build/slim.d.ts.map +1 -0
- package/out/{build-slim.js → build/slim.js} +33 -94
- package/out/build/slim.js.map +1 -0
- package/out/candidate/alias-bags.d.ts +13 -5
- package/out/candidate/alias-bags.d.ts.map +1 -1
- package/out/candidate/alias-bags.js +55 -12
- package/out/candidate/alias-bags.js.map +1 -1
- package/out/candidate/ancestors/schema.d.ts +166 -0
- package/out/candidate/ancestors/schema.d.ts.map +1 -0
- package/out/candidate/ancestors/schema.js +131 -0
- package/out/candidate/ancestors/schema.js.map +1 -0
- package/out/candidate/ancestors/sidecar.d.ts +27 -0
- package/out/candidate/ancestors/sidecar.d.ts.map +1 -0
- package/out/candidate/{ancestors-sidecar.js → ancestors/sidecar.js} +17 -24
- package/out/candidate/ancestors/sidecar.js.map +1 -0
- package/out/candidate/country-display-names.d.ts +5 -15
- package/out/candidate/country-display-names.d.ts.map +1 -1
- package/out/candidate/country-display-names.js +6 -20
- package/out/candidate/country-display-names.js.map +1 -1
- package/out/candidate/extract-fold.d.ts +8 -16
- package/out/candidate/extract-fold.d.ts.map +1 -1
- package/out/candidate/extract-fold.js +6 -48
- package/out/candidate/extract-fold.js.map +1 -1
- package/out/candidate/fts.d.ts +19 -0
- package/out/candidate/fts.d.ts.map +1 -0
- package/out/candidate/fts.js +22 -0
- package/out/candidate/fts.js.map +1 -0
- package/out/candidate/importance.d.ts +127 -0
- package/out/candidate/importance.d.ts.map +1 -0
- package/out/candidate/importance.js +162 -0
- package/out/candidate/importance.js.map +1 -0
- package/out/candidate/lookup.d.ts +36 -0
- package/out/candidate/lookup.d.ts.map +1 -0
- package/out/candidate/lookup.js +456 -0
- package/out/candidate/lookup.js.map +1 -0
- package/out/candidate/name-roles.d.ts +10 -22
- package/out/candidate/name-roles.d.ts.map +1 -1
- package/out/candidate/name-roles.js +15 -33
- package/out/candidate/name-roles.js.map +1 -1
- package/out/candidate/own-name.d.ts +23 -25
- package/out/candidate/own-name.d.ts.map +1 -1
- package/out/candidate/own-name.js +34 -31
- package/out/candidate/own-name.js.map +1 -1
- package/out/candidate/place-attrs.d.ts +20 -13
- package/out/candidate/place-attrs.d.ts.map +1 -1
- package/out/candidate/place-attrs.js +4 -5
- package/out/candidate/place-attrs.js.map +1 -1
- package/out/candidate/schema.d.ts +120 -0
- package/out/candidate/schema.d.ts.map +1 -0
- package/out/{candidate-schema.js → candidate/schema.js} +11 -23
- package/out/candidate/schema.js.map +1 -0
- package/out/{candidate-scoring.d.ts → candidate/scoring.d.ts} +8 -9
- package/out/candidate/scoring.d.ts.map +1 -0
- package/out/candidate/scoring.js +157 -0
- package/out/candidate/scoring.js.map +1 -0
- package/out/capital-schema.d.ts +21 -17
- package/out/capital-schema.d.ts.map +1 -1
- package/out/capital-schema.js +17 -15
- package/out/capital-schema.js.map +1 -1
- package/out/capitals.d.ts +26 -24
- package/out/capitals.d.ts.map +1 -1
- package/out/capitals.js +23 -21
- package/out/capitals.js.map +1 -1
- package/out/coincident-roles.d.ts +60 -43
- package/out/coincident-roles.d.ts.map +1 -1
- package/out/coincident-roles.js +61 -50
- package/out/coincident-roles.js.map +1 -1
- package/out/convention/schema.d.ts +12 -6
- package/out/convention/schema.d.ts.map +1 -1
- package/out/convention/schema.js +6 -4
- package/out/convention/schema.js.map +1 -1
- package/out/convention.d.ts +97 -0
- package/out/convention.d.ts.map +1 -0
- package/out/convention.js +81 -0
- package/out/convention.js.map +1 -0
- package/out/coverage-manifest-schema.d.ts +33 -38
- package/out/coverage-manifest-schema.d.ts.map +1 -1
- package/out/coverage-manifest-schema.js +20 -29
- package/out/coverage-manifest-schema.js.map +1 -1
- package/out/currency-backfill.d.ts +13 -35
- package/out/currency-backfill.d.ts.map +1 -1
- package/out/currency-backfill.js +12 -39
- package/out/currency-backfill.js.map +1 -1
- package/out/env.d.ts +2 -2
- package/out/env.js +1 -1
- package/out/exact-match.d.ts +19 -10
- package/out/exact-match.d.ts.map +1 -1
- package/out/exact-match.js +31 -20
- package/out/exact-match.js.map +1 -1
- package/out/extracts.d.ts +65 -54
- package/out/extracts.d.ts.map +1 -1
- package/out/extracts.js +63 -55
- package/out/extracts.js.map +1 -1
- package/out/fst/autocomplete.d.ts +28 -20
- package/out/fst/autocomplete.d.ts.map +1 -1
- package/out/fst/autocomplete.js +38 -27
- package/out/fst/autocomplete.js.map +1 -1
- package/out/fst/builder.d.ts.map +1 -1
- package/out/fst/builder.js +30 -36
- package/out/fst/builder.js.map +1 -1
- package/out/fst/deserialize-web.d.ts +3 -3
- package/out/fst/deserialize-web.d.ts.map +1 -1
- package/out/fst/deserialize-web.js +8 -9
- package/out/fst/deserialize-web.js.map +1 -1
- package/out/fst/format.d.ts +41 -23
- package/out/fst/format.d.ts.map +1 -1
- package/out/fst/format.js +41 -23
- package/out/fst/format.js.map +1 -1
- package/out/fst/freshness.d.ts +11 -73
- package/out/fst/freshness.d.ts.map +1 -1
- package/out/fst/freshness.js +12 -76
- package/out/fst/freshness.js.map +1 -1
- package/out/fst/matcher.d.ts +1 -1
- package/out/fst/matcher.d.ts.map +1 -1
- package/out/fst/matcher.js +2 -2
- package/out/fst/matcher.js.map +1 -1
- package/out/fst/serialize.d.ts +30 -26
- package/out/fst/serialize.d.ts.map +1 -1
- package/out/fst/serialize.js +44 -58
- package/out/fst/serialize.js.map +1 -1
- package/out/fst/types.d.ts +71 -59
- package/out/fst/types.d.ts.map +1 -1
- package/out/fst/types.js +1 -4
- package/out/fst/types.js.map +1 -1
- package/out/{fst/index.js → fst.d.ts} +1 -1
- package/out/fst.d.ts.map +1 -0
- package/out/{fst/index.d.ts → fst.js} +1 -1
- package/out/fst.js.map +1 -0
- package/out/fts/query.d.ts +2 -27
- package/out/fts/query.d.ts.map +1 -1
- package/out/fts/query.js +5 -36
- package/out/fts/query.js.map +1 -1
- package/out/fts.d.ts +126 -0
- package/out/fts.d.ts.map +1 -0
- package/out/fts.js +188 -0
- package/out/fts.js.map +1 -0
- package/out/geonames/aliases.d.ts +26 -67
- package/out/geonames/aliases.d.ts.map +1 -1
- package/out/geonames/aliases.js +44 -121
- package/out/geonames/aliases.js.map +1 -1
- package/out/geonames/postal.d.ts +84 -50
- package/out/geonames/postal.d.ts.map +1 -1
- package/out/geonames/postal.js +115 -78
- package/out/geonames/postal.js.map +1 -1
- package/out/{geonames/index.js → geonames.d.ts} +1 -1
- package/out/geonames.d.ts.map +1 -0
- package/out/{geonames/index.d.ts → geonames.js} +1 -1
- package/out/geonames.js.map +1 -0
- package/out/index.d.ts +20 -21
- package/out/index.d.ts.map +1 -1
- package/out/index.js +14 -15
- package/out/index.js.map +1 -1
- package/out/interpolation.d.ts +15 -52
- package/out/interpolation.d.ts.map +1 -1
- package/out/interpolation.js +25 -68
- package/out/interpolation.js.map +1 -1
- package/out/lookup.d.ts +24 -53
- package/out/lookup.d.ts.map +1 -1
- package/out/lookup.js +117 -223
- package/out/lookup.js.map +1 -1
- package/out/name-score.d.ts +2 -2
- package/out/name-score.d.ts.map +1 -1
- package/out/name-score.js +9 -7
- package/out/name-score.js.map +1 -1
- package/out/nsul/lookup.d.ts +33 -34
- package/out/nsul/lookup.d.ts.map +1 -1
- package/out/nsul/lookup.js +26 -40
- package/out/nsul/lookup.js.map +1 -1
- package/out/nsul/schema.d.ts +16 -43
- package/out/nsul/schema.d.ts.map +1 -1
- package/out/nsul/schema.js +9 -35
- package/out/nsul/schema.js.map +1 -1
- package/out/{nsul/index.d.ts → nsul.d.ts} +1 -1
- package/out/nsul.d.ts.map +1 -0
- package/out/{nsul/index.js → nsul.js} +1 -1
- package/out/nsul.js.map +1 -0
- package/out/paths.d.ts +124 -0
- package/out/paths.d.ts.map +1 -0
- package/out/paths.js +104 -0
- package/out/paths.js.map +1 -0
- package/out/place-importance-schema.d.ts +49 -141
- package/out/place-importance-schema.d.ts.map +1 -1
- package/out/place-importance-schema.js +25 -140
- package/out/place-importance-schema.js.map +1 -1
- package/out/poi/lookup.d.ts +38 -42
- package/out/poi/lookup.d.ts.map +1 -1
- package/out/poi/lookup.js +67 -90
- package/out/poi/lookup.js.map +1 -1
- package/out/poi/schema.d.ts +41 -25
- package/out/poi/schema.d.ts.map +1 -1
- package/out/poi/schema.js +25 -17
- package/out/poi/schema.js.map +1 -1
- package/out/{poi/index.js → poi.d.ts} +1 -1
- package/out/poi.d.ts.map +1 -0
- package/out/{poi/index.d.ts → poi.js} +1 -1
- package/out/poi.js.map +1 -0
- package/out/polygon-schema.d.ts +10 -6
- package/out/polygon-schema.d.ts.map +1 -1
- package/out/polygon-schema.js +2 -2
- package/out/polygon-schema.js.map +1 -1
- package/out/postal/{city-alias-lookup.d.ts → city/alias/lookup.d.ts} +22 -25
- package/out/postal/city/alias/lookup.d.ts.map +1 -0
- package/out/postal/city/alias/lookup.js +49 -0
- package/out/postal/city/alias/lookup.js.map +1 -0
- package/out/postal/{city-alias-schema.d.ts → city/alias/schema.d.ts} +15 -11
- package/out/postal/city/alias/schema.d.ts.map +1 -0
- package/out/postal/{city-alias-schema.js → city/alias/schema.js} +11 -9
- package/out/postal/city/alias/schema.js.map +1 -0
- package/out/postal/city/candidate-schema.d.ts +72 -0
- package/out/postal/city/candidate-schema.d.ts.map +1 -0
- package/out/postal/city/candidate-schema.js +60 -0
- package/out/postal/city/candidate-schema.js.map +1 -0
- package/out/postal.d.ts +9 -0
- package/out/postal.d.ts.map +1 -0
- package/out/postal.js +9 -0
- package/out/postal.js.map +1 -0
- package/out/postcode-point-lookup.d.ts +9 -19
- package/out/postcode-point-lookup.d.ts.map +1 -1
- package/out/postcode-point-lookup.js +5 -17
- package/out/postcode-point-lookup.js.map +1 -1
- package/out/primary-preference.d.ts +39 -93
- package/out/primary-preference.d.ts.map +1 -1
- package/out/primary-preference.js +13 -97
- package/out/primary-preference.js.map +1 -1
- package/out/proximity-rerank.d.ts +40 -29
- package/out/proximity-rerank.d.ts.map +1 -1
- package/out/proximity-rerank.js +38 -29
- package/out/proximity-rerank.js.map +1 -1
- package/out/ranking-weights.d.ts +91 -62
- package/out/ranking-weights.d.ts.map +1 -1
- package/out/ranking-weights.js +26 -25
- package/out/ranking-weights.js.map +1 -1
- package/out/region-keys.d.ts +48 -29
- package/out/region-keys.d.ts.map +1 -1
- package/out/region-keys.js +68 -40
- package/out/region-keys.js.map +1 -1
- package/out/reverse.d.ts +75 -50
- package/out/reverse.d.ts.map +1 -1
- package/out/reverse.js +61 -49
- package/out/reverse.js.map +1 -1
- package/out/schema.d.ts +74 -42
- package/out/schema.d.ts.map +1 -1
- package/out/schema.js +4 -5
- package/out/schema.js.map +1 -1
- package/out/search-fetch.d.ts +16 -12
- package/out/search-fetch.d.ts.map +1 -1
- package/out/search-fetch.js +83 -63
- package/out/search-fetch.js.map +1 -1
- package/out/sqlite/convention-source.d.ts +25 -0
- package/out/sqlite/convention-source.d.ts.map +1 -0
- package/out/sqlite/convention-source.js +48 -0
- package/out/sqlite/convention-source.js.map +1 -0
- package/out/sqlite/utils.d.ts +45 -0
- package/out/sqlite/utils.d.ts.map +1 -0
- package/out/sqlite/utils.js +56 -0
- package/out/sqlite/utils.js.map +1 -0
- package/out/street/centroid/schema.d.ts +100 -0
- package/out/street/centroid/schema.d.ts.map +1 -0
- package/out/street/centroid/schema.js +66 -0
- package/out/street/centroid/schema.js.map +1 -0
- package/out/street/centroid.d.ts +9 -19
- package/out/street/centroid.d.ts.map +1 -1
- package/out/street/centroid.js +11 -22
- package/out/street/centroid.js.map +1 -1
- package/out/street/{morphology-fst-builder.d.ts → morphology/fst/builder.d.ts} +21 -11
- package/out/street/morphology/fst/builder.d.ts.map +1 -0
- package/out/street/{morphology-fst-builder.js → morphology/fst/builder.js} +40 -34
- package/out/street/morphology/fst/builder.js.map +1 -0
- package/out/street/morphology/fst/loader.d.ts +74 -0
- package/out/street/morphology/fst/loader.d.ts.map +1 -0
- package/out/street/{morphology-fst-loader.js → morphology/fst/loader.js} +21 -21
- package/out/street/morphology/fst/loader.js.map +1 -0
- package/out/street/name-lookup.d.ts +24 -18
- package/out/street/name-lookup.d.ts.map +1 -1
- package/out/street/name-lookup.js +32 -31
- package/out/street/name-lookup.js.map +1 -1
- package/out/street/normalize.d.ts +35 -136
- package/out/street/normalize.d.ts.map +1 -1
- package/out/street/normalize.js +74 -191
- package/out/street/normalize.js.map +1 -1
- package/out/street/segment-schema.d.ts +79 -39
- package/out/street/segment-schema.d.ts.map +1 -1
- package/out/street/segment-schema.js +71 -21
- package/out/street/segment-schema.js.map +1 -1
- package/out/{street/index.d.ts → street.d.ts} +4 -4
- package/out/street.d.ts.map +1 -0
- package/out/{street/index.js → street.js} +4 -4
- package/out/street.js.map +1 -0
- package/out/types.d.ts +69 -138
- package/out/types.d.ts.map +1 -1
- package/out/types.js +0 -6
- package/out/types.js.map +1 -1
- package/out/unified-schema.d.ts +8 -12
- package/out/unified-schema.d.ts.map +1 -1
- package/out/unified-schema.js +29 -29
- package/out/unified-schema.js.map +1 -1
- package/out/uprn/existence.d.ts +28 -17
- package/out/uprn/existence.d.ts.map +1 -1
- package/out/uprn/existence.js +26 -16
- package/out/uprn/existence.js.map +1 -1
- package/out/uprn/lookup.d.ts +14 -44
- package/out/uprn/lookup.d.ts.map +1 -1
- package/out/uprn/lookup.js +20 -56
- package/out/uprn/lookup.js.map +1 -1
- package/out/uprn/schema.d.ts +18 -40
- package/out/uprn/schema.d.ts.map +1 -1
- package/out/uprn/schema.js +10 -32
- package/out/uprn/schema.js.map +1 -1
- package/out/{uprn/index.d.ts → uprn.d.ts} +1 -1
- package/out/uprn.d.ts.map +1 -0
- package/out/{uprn/index.js → uprn.js} +1 -1
- package/out/uprn.js.map +1 -0
- package/out/weights-overlay-linker.d.ts +132 -141
- package/out/weights-overlay-linker.d.ts.map +1 -1
- package/out/weights-overlay-linker.js +86 -154
- package/out/weights-overlay-linker.js.map +1 -1
- package/package.json +94 -599
- package/lib/address/point-schema.ts +0 -155
- package/lib/build-candidate.ts +0 -632
- package/lib/build-slim.ts +0 -408
- package/lib/candidate-ancestors-schema.ts +0 -195
- package/lib/candidate-fts.ts +0 -46
- package/lib/candidate-importance.ts +0 -224
- package/lib/candidate-lookup.ts +0 -977
- package/lib/candidate-schema.ts +0 -247
- package/lib/candidate-scoring.ts +0 -261
- package/lib/convention/index.ts +0 -154
- package/lib/fts/index.ts +0 -361
- package/lib/postal/city-alias-lookup.ts +0 -97
- package/lib/postal/index.ts +0 -9
- package/lib/sqlite-convention-source.ts +0 -65
- package/lib/sqlite-utils.ts +0 -82
- package/lib/street/centroid-schema.ts +0 -154
- package/out/address/index.d.ts.map +0 -1
- package/out/address/index.js.map +0 -1
- package/out/address/point-interpolation.d.ts +0 -49
- package/out/address/point-interpolation.d.ts.map +0 -1
- package/out/address/point-interpolation.js.map +0 -1
- package/out/address/point-schema.d.ts +0 -97
- package/out/address/point-schema.d.ts.map +0 -1
- package/out/address/point-schema.js.map +0 -1
- package/out/ancestry/index.d.ts +0 -45
- package/out/ancestry/index.d.ts.map +0 -1
- package/out/ancestry/index.js +0 -55
- package/out/ancestry/index.js.map +0 -1
- package/out/build-candidate.d.ts +0 -184
- package/out/build-candidate.d.ts.map +0 -1
- package/out/build-candidate.js.map +0 -1
- package/out/build-slim.d.ts +0 -80
- package/out/build-slim.d.ts.map +0 -1
- package/out/build-slim.js.map +0 -1
- package/out/candidate/ancestors-sidecar.d.ts +0 -32
- package/out/candidate/ancestors-sidecar.d.ts.map +0 -1
- package/out/candidate/ancestors-sidecar.js.map +0 -1
- package/out/candidate-ancestors-schema.d.ts +0 -150
- package/out/candidate-ancestors-schema.d.ts.map +0 -1
- package/out/candidate-ancestors-schema.js +0 -123
- package/out/candidate-ancestors-schema.js.map +0 -1
- package/out/candidate-fts.d.ts +0 -37
- package/out/candidate-fts.d.ts.map +0 -1
- package/out/candidate-fts.js +0 -40
- package/out/candidate-fts.js.map +0 -1
- package/out/candidate-importance.d.ts +0 -132
- package/out/candidate-importance.d.ts.map +0 -1
- package/out/candidate-importance.js +0 -169
- package/out/candidate-importance.js.map +0 -1
- package/out/candidate-lookup.d.ts +0 -76
- package/out/candidate-lookup.d.ts.map +0 -1
- package/out/candidate-lookup.js +0 -768
- package/out/candidate-lookup.js.map +0 -1
- package/out/candidate-schema.d.ts +0 -155
- package/out/candidate-schema.d.ts.map +0 -1
- package/out/candidate-schema.js.map +0 -1
- package/out/candidate-scoring.d.ts.map +0 -1
- package/out/candidate-scoring.js +0 -196
- package/out/candidate-scoring.js.map +0 -1
- package/out/convention/index.d.ts +0 -106
- package/out/convention/index.d.ts.map +0 -1
- package/out/convention/index.js +0 -93
- package/out/convention/index.js.map +0 -1
- package/out/fst/index.d.ts.map +0 -1
- package/out/fst/index.js.map +0 -1
- package/out/fts/index.d.ts +0 -184
- package/out/fts/index.d.ts.map +0 -1
- package/out/fts/index.js +0 -277
- package/out/fts/index.js.map +0 -1
- package/out/geonames/index.d.ts.map +0 -1
- package/out/geonames/index.js.map +0 -1
- package/out/nsul/index.d.ts.map +0 -1
- package/out/nsul/index.js.map +0 -1
- package/out/poi/index.d.ts.map +0 -1
- package/out/poi/index.js.map +0 -1
- package/out/postal/city-alias-lookup.d.ts.map +0 -1
- package/out/postal/city-alias-lookup.js +0 -62
- package/out/postal/city-alias-lookup.js.map +0 -1
- package/out/postal/city-alias-schema.d.ts.map +0 -1
- package/out/postal/city-alias-schema.js.map +0 -1
- package/out/postal/city-candidate-schema.d.ts +0 -66
- package/out/postal/city-candidate-schema.d.ts.map +0 -1
- package/out/postal/city-candidate-schema.js +0 -56
- package/out/postal/city-candidate-schema.js.map +0 -1
- package/out/postal/index.d.ts +0 -9
- package/out/postal/index.d.ts.map +0 -1
- package/out/postal/index.js +0 -9
- package/out/postal/index.js.map +0 -1
- package/out/sqlite-convention-source.d.ts +0 -29
- package/out/sqlite-convention-source.d.ts.map +0 -1
- package/out/sqlite-convention-source.js +0 -57
- package/out/sqlite-convention-source.js.map +0 -1
- package/out/sqlite-utils.d.ts +0 -45
- package/out/sqlite-utils.d.ts.map +0 -1
- package/out/sqlite-utils.js +0 -57
- package/out/sqlite-utils.js.map +0 -1
- package/out/street/centroid-schema.d.ts +0 -105
- package/out/street/centroid-schema.d.ts.map +0 -1
- package/out/street/centroid-schema.js +0 -78
- package/out/street/centroid-schema.js.map +0 -1
- package/out/street/index.d.ts.map +0 -1
- package/out/street/index.js.map +0 -1
- package/out/street/morphology-fst-builder.d.ts.map +0 -1
- package/out/street/morphology-fst-builder.js.map +0 -1
- package/out/street/morphology-fst-loader.d.ts +0 -67
- package/out/street/morphology-fst-loader.d.ts.map +0 -1
- package/out/street/morphology-fst-loader.js.map +0 -1
- package/out/uprn/index.d.ts.map +0 -1
- package/out/uprn/index.js.map +0 -1
- /package/lib/{fst/index.ts → fst.ts} +0 -0
- /package/lib/{geonames/index.ts → geonames.ts} +0 -0
- /package/lib/{nsul/index.ts → nsul.ts} +0 -0
- /package/lib/{poi/index.ts → poi.ts} +0 -0
- /package/lib/{uprn/index.ts → uprn.ts} +0 -0
|
@@ -2,12 +2,9 @@
|
|
|
2
2
|
* @copyright Sister Software
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
|
-
* Shared development tooling for weights overlays: the symlink primitives every overlay linker uses, the
|
|
6
|
-
* placetype-pair index build, and the manifest form ({@link materializeDevOverlay}) each locale's
|
|
7
|
-
* `scripts/link-dev-weights.ts` declares itself in.
|
|
8
5
|
*/
|
|
9
6
|
|
|
10
|
-
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
7
|
+
import { dataRootPath, weightsOverlayPath } from "@mailwoman/core/data-root"
|
|
11
8
|
import {
|
|
12
9
|
pathExists,
|
|
13
10
|
readLocalJSONFile,
|
|
@@ -25,22 +22,22 @@ import {
|
|
|
25
22
|
} from "@mailwoman/core/fs/writers"
|
|
26
23
|
import { md5File } from "@mailwoman/core/hash"
|
|
27
24
|
import { parseJSONStrict } from "@mailwoman/core/json"
|
|
28
|
-
import { repoRootPath,
|
|
25
|
+
import { repoRootPath, repoRootPathBuilder, workspacePathBuilder } from "@mailwoman/core/paths"
|
|
29
26
|
import { spawnProcessSync } from "@mailwoman/core/process"
|
|
30
27
|
import { readReleaseConfig, repoCommittedSoftFeedSources } from "@mailwoman/core/release-config"
|
|
31
|
-
import {
|
|
32
|
-
import { resolvePath } from "path-ts"
|
|
28
|
+
import type { PathBuilder, PathBuilderLike } from "path-ts"
|
|
33
29
|
|
|
34
30
|
import { $public } from "#env"
|
|
35
31
|
import { fstFreshnessWarning } from "#fst/freshness"
|
|
32
|
+
import { wofDatabasePath } from "#paths"
|
|
36
33
|
|
|
37
34
|
/**
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
* rename
|
|
35
|
+
* Replaces `dest` with a symlink to `src` by renaming a temporary link over it.
|
|
36
|
+
*
|
|
37
|
+
* The rename is atomic, so concurrent test workers never see `dest` missing.
|
|
41
38
|
*/
|
|
42
|
-
export async function linkForce(src:
|
|
43
|
-
const tmp = `${dest}.tmp-link`
|
|
39
|
+
export async function linkForce(src: PathBuilderLike, dest: PathBuilderLike): Promise<void> {
|
|
40
|
+
const tmp = `${dest.toString()}.tmp-link`
|
|
44
41
|
|
|
45
42
|
if (await pathExists(tmp)) {
|
|
46
43
|
await removePath(tmp)
|
|
@@ -51,9 +48,9 @@ export async function linkForce(src: string, dest: string): Promise<void> {
|
|
|
51
48
|
}
|
|
52
49
|
|
|
53
50
|
/**
|
|
54
|
-
*
|
|
51
|
+
* Removes a file or symlink at `dest` so that the runtime falls back to the base en-us weights.
|
|
55
52
|
*/
|
|
56
|
-
export async function removeIfPresent(dest:
|
|
53
|
+
export async function removeIfPresent(dest: PathBuilder): Promise<void> {
|
|
57
54
|
try {
|
|
58
55
|
await statLink(dest)
|
|
59
56
|
} catch {
|
|
@@ -66,15 +63,14 @@ export async function removeIfPresent(dest: string): Promise<void> {
|
|
|
66
63
|
}
|
|
67
64
|
|
|
68
65
|
/**
|
|
69
|
-
*
|
|
66
|
+
* Symlinks one optional soft-feed artifact into an overlay and returns whether the source existed.
|
|
70
67
|
*
|
|
71
|
-
*
|
|
72
|
-
* has
|
|
73
|
-
* operator needs to know which channel just resolved OFF, not to have the link step abort.
|
|
68
|
+
* A missing source prints `consequenceIfMissing` and does not throw, because the
|
|
69
|
+
* runtime has a fallback for every soft-feed artifact.
|
|
74
70
|
*/
|
|
75
71
|
export async function linkSoftFeedSibling(
|
|
76
|
-
source:
|
|
77
|
-
destination:
|
|
72
|
+
source: PathBuilderLike,
|
|
73
|
+
destination: PathBuilder,
|
|
78
74
|
consequenceIfMissing: string
|
|
79
75
|
): Promise<boolean> {
|
|
80
76
|
if (!(await pathExists(source))) {
|
|
@@ -91,100 +87,113 @@ export async function linkSoftFeedSibling(
|
|
|
91
87
|
}
|
|
92
88
|
|
|
93
89
|
/**
|
|
94
|
-
* The
|
|
95
|
-
* measured at δ=10 (each linker's own docstring carries its locale's receipt).
|
|
90
|
+
* The pair-index emission bias (δ) used by every shipped pair-index artifact.
|
|
96
91
|
*/
|
|
97
92
|
export const PAIR_INDEX_DELTA = 10
|
|
98
93
|
|
|
99
94
|
/**
|
|
100
|
-
* The decoder transition-entry bonus (
|
|
101
|
-
*
|
|
102
|
-
* independently, and a locale earning one says nothing about the other.
|
|
95
|
+
* The decoder transition-entry bonus (β).
|
|
96
|
+
* Only locales that were calibrated with it pass it.
|
|
103
97
|
*/
|
|
104
98
|
export const PAIR_INDEX_TRANSITION_BETA = 5
|
|
105
99
|
|
|
106
100
|
/**
|
|
107
|
-
* The
|
|
108
|
-
*
|
|
109
|
-
*
|
|
101
|
+
* The whole-edge parent-bias magnitude.
|
|
102
|
+
* Only locales that were calibrated with it pass it.
|
|
103
|
+
*
|
|
104
|
+
* Other locales omit the flag and do not inherit this value.
|
|
110
105
|
*/
|
|
111
106
|
export const PAIR_INDEX_PARENT_DELTA = 5
|
|
112
107
|
|
|
113
108
|
/**
|
|
114
|
-
*
|
|
109
|
+
* Describes one locale's pair-index build.
|
|
110
|
+
*
|
|
111
|
+
* When `sources`, `inputs` and `extraArgs` are omitted, the build reads the shared WOF admin database.
|
|
115
112
|
*/
|
|
116
113
|
export interface PairIndexOverlay {
|
|
117
114
|
/**
|
|
118
|
-
*
|
|
115
|
+
* The weights workspace directory name, such as `neural-weights-de-de`.
|
|
116
|
+
* The overlay locale is derived from it.
|
|
119
117
|
*/
|
|
120
118
|
packageDir: string
|
|
119
|
+
|
|
121
120
|
/**
|
|
122
|
-
* ISO country code passed to `
|
|
123
|
-
*
|
|
121
|
+
* The ISO country code passed to `--country` and used in `pair-index-<country>.bin`.
|
|
122
|
+
*
|
|
123
|
+
* It can differ from the locale tag.
|
|
124
|
+
* For example, `en-in` builds `pair-index-in.bin`.
|
|
124
125
|
*/
|
|
125
126
|
country: string
|
|
127
|
+
|
|
126
128
|
/**
|
|
127
|
-
*
|
|
128
|
-
*
|
|
129
|
-
*
|
|
130
|
-
* The
|
|
129
|
+
* The child bias magnitude.
|
|
130
|
+
*
|
|
131
|
+
* The build writes `delta`, `transitionBeta` and `parentDelta` into the PIX1 header.
|
|
132
|
+
* The freshness check compares them.
|
|
133
|
+
*
|
|
134
|
+
* An absent `transitionBeta` or `parentDelta` leaves its key out of the header.
|
|
135
|
+
* The freshness check treats an absent key as distinct from zero.
|
|
131
136
|
*/
|
|
132
137
|
delta: number
|
|
133
138
|
transitionBeta?: number
|
|
134
139
|
parentDelta?: number
|
|
140
|
+
|
|
135
141
|
/**
|
|
136
|
-
*
|
|
137
|
-
*
|
|
138
|
-
*
|
|
139
|
-
*
|
|
140
|
-
*
|
|
142
|
+
* The source files whose MD5s the build records, in the order that `gazetteer pair-index` records them.
|
|
143
|
+
* The default is the WOF admin database.
|
|
144
|
+
*
|
|
145
|
+
* The freshness check compares each MD5 with the header.
|
|
146
|
+
* An empty list makes the check compare the magnitudes only.
|
|
141
147
|
*/
|
|
142
|
-
sources?:
|
|
148
|
+
sources?: PathBuilder[]
|
|
149
|
+
|
|
143
150
|
/**
|
|
144
|
-
*
|
|
145
|
-
*
|
|
151
|
+
* Files that must exist before a build runs.
|
|
152
|
+
*
|
|
153
|
+
* The default is `sources`.
|
|
154
|
+
* A missing file prints a warning and skips the build.
|
|
146
155
|
*/
|
|
147
|
-
inputs?:
|
|
156
|
+
inputs?: PathBuilder[]
|
|
157
|
+
|
|
148
158
|
/**
|
|
149
|
-
* Extra CLI
|
|
150
|
-
* `--borough-db <admin db>`.
|
|
159
|
+
* Extra CLI arguments that pass the build's sources.
|
|
160
|
+
* The default is `--borough-db <admin db>`.
|
|
151
161
|
*/
|
|
152
|
-
extraArgs?:
|
|
162
|
+
extraArgs?: PathBuilderLike[]
|
|
163
|
+
|
|
153
164
|
/**
|
|
154
|
-
*
|
|
155
|
-
*
|
|
165
|
+
* The byte size below which an existing artifact is rebuilt.
|
|
166
|
+
*
|
|
167
|
+
* A build from the wrong source can have matching header magnitudes but a smaller size.
|
|
156
168
|
*/
|
|
157
169
|
minimumPlausibleBytes?: number
|
|
158
170
|
}
|
|
159
171
|
|
|
160
172
|
/**
|
|
161
|
-
* The header fields
|
|
173
|
+
* The pair-index header fields that the dev-weights freshness check reads.
|
|
162
174
|
*/
|
|
163
175
|
export interface PairIndexHeaderFields {
|
|
164
176
|
delta: number
|
|
165
177
|
transitionBeta: number | undefined
|
|
166
178
|
parentDelta: number | undefined
|
|
167
179
|
schemaVersion: number
|
|
180
|
+
|
|
168
181
|
/**
|
|
169
|
-
* One
|
|
182
|
+
* One MD5 per source the build read, in recorded order.
|
|
183
|
+
* The list is empty when the header has none.
|
|
170
184
|
*/
|
|
171
185
|
sourceMD5s: string[]
|
|
172
186
|
}
|
|
173
187
|
|
|
174
188
|
/**
|
|
175
|
-
*
|
|
176
|
-
* `@mailwoman/neural` (which pulls onnxruntime-node) to read a few fields. `neural/pair-index-resolver.ts`'s own header
|
|
177
|
-
* parse is the source of truth this must follow.
|
|
189
|
+
* Reads the calibration fields from a PIX1 pair-index header without depending on `@mailwoman/neural`.
|
|
178
190
|
*
|
|
179
|
-
*
|
|
180
|
-
* (`neural-weights-{en-us,en-gb,en-nz,fr-fr}/scripts/link-dev-weights.ts`), which each carried their own near-copy
|
|
181
|
-
* before 2026-08-04 — the ×5 clone the taste audit named, and the reason three of them were schema-blind while this one
|
|
182
|
-
* was not.
|
|
191
|
+
* The parse must match the reader in `neural/lib/pair/index/resolver.ts`.
|
|
183
192
|
*/
|
|
184
|
-
export async function peekPairIndexHeaderFields(path:
|
|
193
|
+
export async function peekPairIndexHeaderFields(path: PathBuilderLike): Promise<PairIndexHeaderFields> {
|
|
185
194
|
const bytes = await readLocalBuffer(path)
|
|
186
195
|
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
|
187
|
-
|
|
196
|
+
|
|
188
197
|
const MAGIC = 0x31_58_49_50
|
|
189
198
|
|
|
190
199
|
if (view.getUint32(0, true) !== MAGIC) {
|
|
@@ -210,21 +219,17 @@ export async function peekPairIndexHeaderFields(path: string): Promise<PairIndex
|
|
|
210
219
|
}
|
|
211
220
|
}
|
|
212
221
|
|
|
213
|
-
/**
|
|
214
|
-
* Hex characters in an md5 digest.
|
|
215
|
-
*/
|
|
216
222
|
const MD5_HEX_LENGTH = 32
|
|
217
223
|
|
|
224
|
+
// repo-health-ignore export-name-affix -- This variant caches the hash in a sidecar.
|
|
225
|
+
// `md5File` hashes every time.
|
|
218
226
|
/**
|
|
219
|
-
*
|
|
220
|
-
* source is hashed once per change rather than once per linker run. The sidecar is trusted only when at least as new as
|
|
221
|
-
* the source; a missing or stale sidecar recomputes and rewrites, so the cache self-heals.
|
|
227
|
+
* Returns the MD5 of `path`, cached in a `<path>.md5` sidecar in `md5sum` format.
|
|
222
228
|
*
|
|
223
|
-
* The
|
|
229
|
+
* The sidecar is used only when it is at least as new as the source.
|
|
224
230
|
*/
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
const sidecarPath = `${path}.md5`
|
|
231
|
+
export async function md5FileWithSidecar(path: PathBuilderLike): Promise<string> {
|
|
232
|
+
const sidecarPath = `${path.toString()}.md5`
|
|
228
233
|
const sourceStats = await statPath(path)
|
|
229
234
|
|
|
230
235
|
if (await pathExists(sidecarPath)) {
|
|
@@ -234,9 +239,7 @@ export async function md5FileWithSidecar(path: string): Promise<string> {
|
|
|
234
239
|
|
|
235
240
|
if (hash && hash.length === MD5_HEX_LENGTH) return hash
|
|
236
241
|
}
|
|
237
|
-
} catch {
|
|
238
|
-
// Unreadable sidecar — recompute below.
|
|
239
|
-
}
|
|
242
|
+
} catch {}
|
|
240
243
|
}
|
|
241
244
|
|
|
242
245
|
const hash = await md5File(path)
|
|
@@ -248,9 +251,10 @@ export async function md5FileWithSidecar(path: string): Promise<string> {
|
|
|
248
251
|
}
|
|
249
252
|
|
|
250
253
|
/**
|
|
251
|
-
* The calibrated magnitudes a linker
|
|
252
|
-
*
|
|
253
|
-
*
|
|
254
|
+
* The calibrated magnitudes that a linker writes into its pair-index artifact.
|
|
255
|
+
*
|
|
256
|
+
* An `undefined` magnitude means the header has no such key.
|
|
257
|
+
* It is distinct from zero.
|
|
254
258
|
*/
|
|
255
259
|
export interface PairIndexCalibration {
|
|
256
260
|
delta: number
|
|
@@ -259,12 +263,10 @@ export interface PairIndexCalibration {
|
|
|
259
263
|
}
|
|
260
264
|
|
|
261
265
|
/**
|
|
262
|
-
*
|
|
263
|
-
*
|
|
264
|
-
* linker passes a different set of sources and only it knows what they are.
|
|
266
|
+
* Returns why a pair-index header's schema or magnitudes differ from `expected`,
|
|
267
|
+
* or `undefined` when they match.
|
|
265
268
|
*
|
|
266
|
-
*
|
|
267
|
-
* three of the four base linkers ended up unable to notice a schema bump.
|
|
269
|
+
* The caller checks the source MD5s because only the caller knows its sources.
|
|
268
270
|
*/
|
|
269
271
|
export function pairIndexStaleReason(
|
|
270
272
|
header: PairIndexHeaderFields,
|
|
@@ -288,38 +290,24 @@ export function pairIndexStaleReason(
|
|
|
288
290
|
}
|
|
289
291
|
|
|
290
292
|
/**
|
|
291
|
-
* The PIX1 schema
|
|
292
|
-
* they are two ends of one fact, and this copy exists only because a data-only overlay must not gain a dependency on
|
|
293
|
-
* `@mailwoman/neural` (onnxruntime-node) to read one header field. Bump BOTH in the same commit; a schema bump that
|
|
294
|
-
* leaves this behind makes every dev checkout rebuild-loop or serve an artifact the runtime refuses.
|
|
293
|
+
* The PIX1 schema version the pair-index reader requires.
|
|
295
294
|
*
|
|
296
|
-
*
|
|
297
|
-
*
|
|
298
|
-
* edition. Also compared by the four hand-written base linkers (`neural-weights-{en-us,en-gb,en-nz,fr-fr}`), which
|
|
299
|
-
* import this constant rather than re-typing the number.
|
|
295
|
+
* It must equal `KNOWN_SCHEMA_VERSION` in `neural/lib/pair/index/resolver.ts`.
|
|
296
|
+
* A mismatch makes dev checkouts rebuild on every run or link an artifact that the runtime refuses.
|
|
300
297
|
*/
|
|
301
298
|
export const REQUIRED_PAIR_INDEX_SCHEMA = 3
|
|
302
299
|
|
|
303
300
|
/**
|
|
304
|
-
*
|
|
305
|
-
* now.
|
|
301
|
+
* Warns when a linked per-locale FST was built from a different admin database than the one on disk.
|
|
306
302
|
*
|
|
307
|
-
*
|
|
308
|
-
*
|
|
309
|
-
* the swap into `fst-per-locale/` is operator-approved after the battery, because an FST changes decoder behaviour and
|
|
310
|
-
* the D-rule does not let that land unmeasured. So the guard's job is to make the drift impossible to miss, and to name
|
|
311
|
-
* the command that starts fixing it. It is also why a stale FST is never fatal: the artifact is a decode-time bias
|
|
312
|
-
* list, and a dev tree must still run.
|
|
313
|
-
*
|
|
314
|
-
* The source-side half of the comparison lives here rather than in `fst-freshness.ts` for the same reason
|
|
315
|
-
* `pairIndexStaleReason` splits: only the caller knows which database it built against. All three FST-linking base
|
|
316
|
-
* packages build against the same one, so they share this rather than each pinning it.
|
|
303
|
+
* It does not rebuild, because an FST rebuild writes to a staging directory
|
|
304
|
+
* and an operator swaps it in after evaluation.
|
|
317
305
|
*/
|
|
318
|
-
export async function warnIfFSTStale(fstPath:
|
|
306
|
+
export async function warnIfFSTStale(fstPath: PathBuilder, locale: string): Promise<void> {
|
|
319
307
|
const warning = await fstFreshnessWarning({
|
|
320
308
|
fstPath,
|
|
321
|
-
sourceDBPath:
|
|
322
|
-
rebuildCommand: `node packages/mailwoman/out/cli.js gazetteer build fst --locales ${locale} (writes to a staging dir; swap is operator-conditional)`,
|
|
309
|
+
sourceDBPath: wofDatabasePath("admin-global-priority.db"),
|
|
310
|
+
rebuildCommand: `node packages/mailwoman/out/cli/main.js gazetteer build fst --locales ${locale} (writes to a staging dir; swap is operator-conditional)`,
|
|
323
311
|
})
|
|
324
312
|
|
|
325
313
|
if (warning) {
|
|
@@ -328,17 +316,15 @@ export async function warnIfFSTStale(fstPath: string, locale: string): Promise<v
|
|
|
328
316
|
}
|
|
329
317
|
|
|
330
318
|
/**
|
|
331
|
-
*
|
|
332
|
-
*
|
|
333
|
-
* runtime pipeline can auto-wire the gazetteer + street-context check, then run {@link warnIfFSTStale} on the linked
|
|
334
|
-
* artifact. The publish flow stages the real binary (release-sequenced).
|
|
319
|
+
* Symlinks the per-locale FST gazetteer `fst-<locale>.bin` into an overlay
|
|
320
|
+
* and checks it with {@link warnIfFSTStale}.
|
|
335
321
|
*/
|
|
336
|
-
export async function linkLocaleFST(destDir:
|
|
337
|
-
const source =
|
|
322
|
+
export async function linkLocaleFST(destDir: PathBuilder, locale: string): Promise<void> {
|
|
323
|
+
const source = wofDatabasePath("fst-per-locale", `fst-${locale}.bin`)
|
|
338
324
|
|
|
339
325
|
const linked = await linkSoftFeedSibling(
|
|
340
326
|
source,
|
|
341
|
-
|
|
327
|
+
destDir(`fst-${locale}.bin`),
|
|
342
328
|
"the FST gazetteer default will resolve OFF for this locale."
|
|
343
329
|
)
|
|
344
330
|
|
|
@@ -348,32 +334,22 @@ export async function linkLocaleFST(destDir: string, locale: string): Promise<vo
|
|
|
348
334
|
}
|
|
349
335
|
|
|
350
336
|
/**
|
|
351
|
-
*
|
|
352
|
-
*
|
|
353
|
-
* dev and the street-context check (#1315) deserializes the artifact instead of rebuilding from the libpostal
|
|
354
|
-
* dictionaries per process. Missing is non-fatal — the runtime loader's dictionary-build fallback covers it.
|
|
337
|
+
* Symlinks the prebuilt street-morphology FST into an overlay so that the
|
|
338
|
+
* street-context check does not rebuild it from dictionaries.
|
|
355
339
|
*/
|
|
356
|
-
export async function linkStreetMorphologyFST(destDir:
|
|
340
|
+
export async function linkStreetMorphologyFST(destDir: PathBuilder): Promise<void> {
|
|
357
341
|
await linkSoftFeedSibling(
|
|
358
|
-
|
|
359
|
-
|
|
342
|
+
wofDatabasePath("fst-street-morphology.bin"),
|
|
343
|
+
destDir("fst-street-morphology.bin"),
|
|
360
344
|
"the street-context check falls back to the per-process dictionary build."
|
|
361
345
|
)
|
|
362
346
|
}
|
|
363
347
|
|
|
364
|
-
/**
|
|
365
|
-
* Why an existing artifact may be TRUSTED without a rebuild, or `undefined` when it must be rebuilt (with the reason
|
|
366
|
-
* already printed). The skip requires both halves (#1734): the header magnitudes + format (`pairIndexStaleReason`) AND
|
|
367
|
-
* the header's `sourceMD5s` against the current sources' md5s. Magnitudes alone read a source change as "current"
|
|
368
|
-
* whenever pair counts happen not to move the calibrated numbers — the R5 freshness-guard lesson, which resurfaced in
|
|
369
|
-
* the 2026-08-18 admin swap. A source that is not on disk to re-hash leaves the magnitudes as the best available answer
|
|
370
|
-
* (the "missing source, can't build" branch below would fire anyway if a rebuild were needed).
|
|
371
|
-
*/
|
|
372
348
|
async function pairIndexIsFresh(
|
|
373
|
-
dest:
|
|
349
|
+
dest: PathBuilder,
|
|
374
350
|
artifact: string,
|
|
375
351
|
expected: PairIndexCalibration,
|
|
376
|
-
sources:
|
|
352
|
+
sources: PathBuilder[],
|
|
377
353
|
minimumPlausibleBytes: number | undefined
|
|
378
354
|
): Promise<boolean> {
|
|
379
355
|
try {
|
|
@@ -447,30 +423,23 @@ async function pairIndexIsFresh(
|
|
|
447
423
|
}
|
|
448
424
|
|
|
449
425
|
/**
|
|
450
|
-
*
|
|
451
|
-
* these magnitudes
|
|
452
|
-
* staleness expectation and the CLI flags, so a magnitude cannot be checked by the guard and dropped from the build.
|
|
426
|
+
* Builds `pair-index-<country>.bin` into the overlay unless the existing artifact
|
|
427
|
+
* already matches these magnitudes and sources.
|
|
453
428
|
*
|
|
454
|
-
*
|
|
455
|
-
*
|
|
456
|
-
* have run a build yet rather than a real fault.
|
|
429
|
+
* A failed build exits the process.
|
|
430
|
+
* A missing CLI or input only prints a warning, because a fresh clone has neither.
|
|
457
431
|
*/
|
|
458
432
|
export async function buildPairIndexOverlay(overlay: PairIndexOverlay): Promise<void> {
|
|
459
433
|
const { packageDir, country, delta, transitionBeta, parentDelta } = overlay
|
|
460
|
-
const CLI =
|
|
434
|
+
const CLI = workspacePathBuilder("mailwoman", "out", "cli", "main.js")
|
|
461
435
|
const ARTIFACT = `pair-index-${country}.bin`
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
const PKG_DIR = String(weightsOverlayPath(packageDir.replace(/^neural-weights-/, "")))
|
|
466
|
-
const DEST = resolvePath(PKG_DIR, ARTIFACT)
|
|
436
|
+
|
|
437
|
+
const PKG_DIR = weightsOverlayPath(packageDir.replace(/^neural-weights-/, ""))
|
|
438
|
+
const DEST = PKG_DIR(ARTIFACT)
|
|
467
439
|
|
|
468
440
|
await makeDirectories(PKG_DIR)
|
|
469
441
|
|
|
470
|
-
|
|
471
|
-
* Checked-in WOF-derived admin pairs — the default source, and the whole source list for the small overlays.
|
|
472
|
-
*/
|
|
473
|
-
const WOF_ADMIN_DB = String(dataRootPath("wof", "admin-global-priority.db"))
|
|
442
|
+
const WOF_ADMIN_DB = wofDatabasePath("admin-global-priority.db")
|
|
474
443
|
const sources = overlay.sources ?? [WOF_ADMIN_DB]
|
|
475
444
|
const inputs = overlay.inputs ?? sources
|
|
476
445
|
const extraArgs = overlay.extraArgs ?? ["--borough-db", WOF_ADMIN_DB]
|
|
@@ -532,11 +501,8 @@ export async function buildPairIndexOverlay(overlay: PairIndexOverlay): Promise<
|
|
|
532
501
|
console.log(`built ${ARTIFACT}`)
|
|
533
502
|
}
|
|
534
503
|
|
|
535
|
-
// --- the manifest form ----------------------------------------------------------------------------------------------
|
|
536
|
-
|
|
537
504
|
/**
|
|
538
|
-
* A soft-feed
|
|
539
|
-
* printed when the source is missing (the link is warn-and-continue; the channel resolves OFF).
|
|
505
|
+
* A soft-feed artifact that an overlay links, with the warning to print when its source is missing.
|
|
540
506
|
*/
|
|
541
507
|
export interface SoftFeedLink {
|
|
542
508
|
source: string
|
|
@@ -545,9 +511,9 @@ export interface SoftFeedLink {
|
|
|
545
511
|
}
|
|
546
512
|
|
|
547
513
|
/**
|
|
548
|
-
*
|
|
549
|
-
*
|
|
550
|
-
*
|
|
514
|
+
* Returns the committed soft-feed lexicons as links.
|
|
515
|
+
*
|
|
516
|
+
* The filenames come from `release.config.json` so that dev links match the release.
|
|
551
517
|
*/
|
|
552
518
|
export async function committedSoftFeedLinks(): Promise<{
|
|
553
519
|
anchor: SoftFeedLink
|
|
@@ -578,8 +544,7 @@ export async function committedSoftFeedLinks(): Promise<{
|
|
|
578
544
|
}
|
|
579
545
|
|
|
580
546
|
/**
|
|
581
|
-
* The
|
|
582
|
-
* release re-verifies against the published tarball, and the channel requirements that name evidence lexicons.
|
|
547
|
+
* The fields of a weights package's `model-card.json` that dev materialization reads.
|
|
583
548
|
*/
|
|
584
549
|
export interface WeightsCard {
|
|
585
550
|
files_md5?: Record<string, string>
|
|
@@ -587,134 +552,138 @@ export interface WeightsCard {
|
|
|
587
552
|
}
|
|
588
553
|
|
|
589
554
|
/**
|
|
590
|
-
*
|
|
591
|
-
*
|
|
592
|
-
* soft-feed siblings, card-named evidence lexicons, postcode binary, pair index, FSTs.
|
|
555
|
+
* Declares what one locale's dev overlay contains. {@link materializeDevOverlay}
|
|
556
|
+
* runs the steps in the order the fields are listed.
|
|
593
557
|
*/
|
|
594
558
|
export interface DevOverlayManifest {
|
|
595
559
|
/**
|
|
596
|
-
* The overlay's locale tag
|
|
597
|
-
*
|
|
598
|
-
*
|
|
599
|
-
*
|
|
560
|
+
* The overlay's lowercase locale tag.
|
|
561
|
+
* The workspace is `neural-weights-<locale>`.
|
|
562
|
+
*
|
|
563
|
+
* Artifacts go to the data root's `weights/<locale>/` overlay because the binaries are kept out of git.
|
|
600
564
|
*/
|
|
601
565
|
locale: string
|
|
566
|
+
|
|
602
567
|
/**
|
|
603
|
-
*
|
|
604
|
-
*
|
|
605
|
-
*
|
|
606
|
-
*
|
|
607
|
-
*
|
|
608
|
-
*
|
|
609
|
-
*
|
|
568
|
+
* How the overlay gets its model files.
|
|
569
|
+
*
|
|
570
|
+
* When it is omitted, `link-weights-overlay.ts` in release-kit handles them.
|
|
571
|
+
*
|
|
572
|
+
* The `link` kind symlinks `model.onnx` and `tokenizer.model` from the
|
|
573
|
+
* `release.config.json` weights paths under the data root.
|
|
574
|
+
* The `$MAILWOMAN_DEV_MODEL` and `$MAILWOMAN_DEV_TOKENIZER` variables override those paths.
|
|
575
|
+
*
|
|
576
|
+
* With `digestCard`, the linked default files must match the `files_md5` of that workspace's card.
|
|
577
|
+
* An override skips this check.
|
|
578
|
+
*
|
|
579
|
+
* The `inherit` kind removes any local pair, because the package declares
|
|
580
|
+
* `mailwoman.baseWeights` and a local file would shadow the base.
|
|
581
|
+
*
|
|
582
|
+
* The `char` kind links the `charWeights.<family>` model, character vocabulary and model card.
|
|
583
|
+
* It removes any tokenizer.
|
|
610
584
|
*/
|
|
611
585
|
model?: { kind: "link"; digestCard?: string } | { kind: "inherit" } | { kind: "char"; family: string }
|
|
586
|
+
|
|
612
587
|
/**
|
|
613
|
-
* Soft-feed siblings
|
|
588
|
+
* Soft-feed siblings to link in order.
|
|
589
|
+
* A missing source prints a warning and the step continues.
|
|
614
590
|
*/
|
|
615
591
|
softFeed?: ReadonlyArray<SoftFeedLink>
|
|
592
|
+
|
|
616
593
|
/**
|
|
617
|
-
*
|
|
618
|
-
*
|
|
619
|
-
*
|
|
594
|
+
* Whether to link the evidence lexicons listed under `requires.<channel>.lexicon`
|
|
595
|
+
* in the overlay's model card.
|
|
596
|
+
* A card update then changes the linked file too.
|
|
597
|
+
*
|
|
598
|
+
* The street-type lexicon comes from the repo's `data/gazetteer/`.
|
|
599
|
+
* The locality-surface lexicon comes from the data root.
|
|
620
600
|
*/
|
|
621
601
|
evidenceLexiconsFromCard?: boolean
|
|
602
|
+
|
|
622
603
|
/**
|
|
623
|
-
*
|
|
624
|
-
*
|
|
604
|
+
* Builds `postcode-<country>.bin` from a WOF postcode extract with the compiled CLI.
|
|
605
|
+
* The step is skipped when the file already exists.
|
|
625
606
|
*/
|
|
626
|
-
postcodeBinary?: { country: string; database:
|
|
607
|
+
postcodeBinary?: { country: string; database: PathBuilder }
|
|
608
|
+
|
|
627
609
|
/**
|
|
628
|
-
* The placetype-pair index build
|
|
610
|
+
* The placetype-pair index build.
|
|
611
|
+
* Its `packageDir` is derived from `locale`.
|
|
629
612
|
*/
|
|
630
613
|
pairIndex?: Omit<PairIndexOverlay, "packageDir">
|
|
614
|
+
|
|
631
615
|
/**
|
|
632
|
-
*
|
|
616
|
+
* Whether to link `fst-<locale>.bin` and warn when it is stale.
|
|
633
617
|
*/
|
|
634
618
|
localeFST?: boolean
|
|
619
|
+
|
|
635
620
|
/**
|
|
636
|
-
*
|
|
621
|
+
* Whether to link the shared street-morphology FST.
|
|
637
622
|
*/
|
|
638
623
|
streetMorphologyFST?: boolean
|
|
639
624
|
}
|
|
640
625
|
|
|
641
626
|
/**
|
|
642
|
-
*
|
|
643
|
-
*
|
|
627
|
+
* The overlay directory, CLI path and model card that {@link materializeDevOverlay} returns.
|
|
628
|
+
*
|
|
629
|
+
* Callers use them for locale-specific steps that a manifest cannot express.
|
|
644
630
|
*/
|
|
645
631
|
export interface DevOverlay {
|
|
646
|
-
destDir:
|
|
647
|
-
cli:
|
|
632
|
+
destDir: PathBuilder
|
|
633
|
+
cli: PathBuilder
|
|
648
634
|
card: WeightsCard | undefined
|
|
649
635
|
}
|
|
650
636
|
|
|
651
|
-
/**
|
|
652
|
-
* Where each evidence channel's lexicon is found, by the name the card declares.
|
|
653
|
-
*/
|
|
654
637
|
const EVIDENCE_LEXICON_SOURCES: ReadonlyArray<{
|
|
655
638
|
channel: "street_type" | "locality_surface"
|
|
656
|
-
source: (name: string) =>
|
|
639
|
+
source: (name: string) => PathBuilder
|
|
657
640
|
}> = [
|
|
658
|
-
{ channel: "street_type", source: (name) =>
|
|
659
|
-
{ channel: "locality_surface", source: (name) =>
|
|
641
|
+
{ channel: "street_type", source: (name) => repoRootPathBuilder("data", "gazetteer", name) },
|
|
642
|
+
{ channel: "locality_surface", source: (name) => dataRootPath("gazetteer", name) },
|
|
660
643
|
]
|
|
661
644
|
|
|
662
|
-
/**
|
|
663
|
-
* Read a weights workspace's committed card, or `undefined` when the workspace carries none.
|
|
664
|
-
*/
|
|
665
645
|
async function readWeightsCard(workspace: string): Promise<WeightsCard | undefined> {
|
|
666
|
-
const path =
|
|
646
|
+
const path = workspacePathBuilder(workspace, "model-card.json")
|
|
667
647
|
|
|
668
648
|
if (!(await pathExists(path))) return undefined
|
|
669
649
|
|
|
670
650
|
return readLocalJSONFile<WeightsCard>(path)
|
|
671
651
|
}
|
|
672
652
|
|
|
673
|
-
|
|
674
|
-
* Link the base model pair from the release recipe and, when a digest card is named, hold the linked default bytes to
|
|
675
|
-
* that card's `files_md5`. The recipe and the card are the two registers a ship bumps in lockstep; a path bumped
|
|
676
|
-
* without the card, or the reverse, fails here rather than after an eval shift graded against the wrong weights.
|
|
677
|
-
*/
|
|
678
|
-
/**
|
|
679
|
-
* A char-path base (#2164): the graph and its sealed vocabulary from `release.config.json`'s `charWeights[family]`, no
|
|
680
|
-
* tokenizer. The dev overlay carries both so `resolveWeights` finds the package whole.
|
|
681
|
-
*/
|
|
682
|
-
async function linkCharModel(destDir: string, family: string): Promise<void> {
|
|
653
|
+
async function linkCharModel(destDir: PathBuilder, family: string): Promise<void> {
|
|
683
654
|
const recipe = (await readReleaseConfig()).charWeights?.[family]
|
|
684
655
|
|
|
685
656
|
if (!recipe) {
|
|
686
657
|
throw new Error(`release.config.json declares no charWeights.${family} — nothing to link for the ${family} base`)
|
|
687
658
|
}
|
|
688
659
|
|
|
689
|
-
const dataRoot =
|
|
660
|
+
const dataRoot = dataRootPath()
|
|
690
661
|
|
|
691
662
|
for (const [name, relative] of [
|
|
692
663
|
["model.onnx", recipe.model],
|
|
693
664
|
["char-vocab.json", recipe.charVocab],
|
|
694
665
|
] as const) {
|
|
695
|
-
const source =
|
|
666
|
+
const source = dataRoot(relative)
|
|
696
667
|
|
|
697
668
|
if (!(await pathExists(source))) {
|
|
698
669
|
throw new Error(`missing char-path source ${name} for ${family}: ${source}`)
|
|
699
670
|
}
|
|
700
671
|
|
|
701
|
-
await linkForce(source,
|
|
672
|
+
await linkForce(source, destDir(name))
|
|
702
673
|
|
|
703
|
-
console.log(`linked ${
|
|
674
|
+
console.log(`linked ${destDir(name)} ← ${source}`)
|
|
704
675
|
}
|
|
705
676
|
|
|
706
|
-
|
|
707
|
-
// block), so the overlay carries the workspace's committed card beside the two binaries.
|
|
708
|
-
const card = resolvePath(String(workspacePath(`neural-weights-${family}`)), "model-card.json")
|
|
677
|
+
const card = workspacePathBuilder(`neural-weights-${family}`, "model-card.json")
|
|
709
678
|
|
|
710
|
-
await linkForce(card,
|
|
711
|
-
await removeIfPresent(
|
|
679
|
+
await linkForce(card, destDir("model-card.json"))
|
|
680
|
+
await removeIfPresent(destDir("tokenizer.model"))
|
|
712
681
|
}
|
|
713
682
|
|
|
714
|
-
async function linkBaseModelPair(destDir:
|
|
683
|
+
async function linkBaseModelPair(destDir: PathBuilder, digestCard: string | undefined): Promise<void> {
|
|
715
684
|
const recipe = await readReleaseConfig()
|
|
716
685
|
|
|
717
|
-
const dataRoot =
|
|
686
|
+
const dataRoot = dataRootPath()
|
|
718
687
|
const digests = digestCard ? (await readWeightsCard(digestCard))?.files_md5 : undefined
|
|
719
688
|
|
|
720
689
|
const pair = [
|
|
@@ -728,13 +697,13 @@ async function linkBaseModelPair(destDir: string, digestCard: string | undefined
|
|
|
728
697
|
]
|
|
729
698
|
|
|
730
699
|
for (const { label, name, override, recipe: recipePath } of pair) {
|
|
731
|
-
const source = override ||
|
|
700
|
+
const source = override || dataRoot(recipePath)
|
|
732
701
|
|
|
733
702
|
if (!(await pathExists(source))) {
|
|
734
703
|
throw new Error(`missing source ${label}: ${source} — set MAILWOMAN_DEV_${label.toUpperCase()} to override`)
|
|
735
704
|
}
|
|
736
705
|
|
|
737
|
-
const dest =
|
|
706
|
+
const dest = destDir(name)
|
|
738
707
|
|
|
739
708
|
await linkForce(source, dest)
|
|
740
709
|
|
|
@@ -770,17 +739,13 @@ async function linkBaseModelPair(destDir: string, digestCard: string | undefined
|
|
|
770
739
|
}
|
|
771
740
|
}
|
|
772
741
|
|
|
773
|
-
/**
|
|
774
|
-
* Build a postcode binary from a WOF postcode extract, skip-if-present. A missing CLI or extract is a warning (the
|
|
775
|
-
* anchor channel resolves OFF for that country); a build that runs and fails is an error.
|
|
776
|
-
*/
|
|
777
742
|
async function buildPostcodeBinary(
|
|
778
|
-
destDir:
|
|
779
|
-
cli:
|
|
780
|
-
{ country, database }: { country: string; database:
|
|
743
|
+
destDir: PathBuilder,
|
|
744
|
+
cli: PathBuilder,
|
|
745
|
+
{ country, database }: { country: string; database: PathBuilder }
|
|
781
746
|
): Promise<void> {
|
|
782
747
|
const artifact = `postcode-${country.toLowerCase()}.bin`
|
|
783
|
-
const dest =
|
|
748
|
+
const dest = destDir(artifact)
|
|
784
749
|
|
|
785
750
|
if (await pathExists(dest)) {
|
|
786
751
|
console.log(`skipped ${artifact} build — ${dest} already present`)
|
|
@@ -806,7 +771,15 @@ async function buildPostcodeBinary(
|
|
|
806
771
|
|
|
807
772
|
const result = spawnProcessSync(
|
|
808
773
|
process.execPath,
|
|
809
|
-
[
|
|
774
|
+
[
|
|
775
|
+
cli,
|
|
776
|
+
"gazetteer",
|
|
777
|
+
"postcode-binary",
|
|
778
|
+
"--out",
|
|
779
|
+
destDir,
|
|
780
|
+
"--locale",
|
|
781
|
+
`${country.toUpperCase()}:${database.toString()}`,
|
|
782
|
+
],
|
|
810
783
|
{ stdio: "inherit" }
|
|
811
784
|
)
|
|
812
785
|
|
|
@@ -818,19 +791,18 @@ async function buildPostcodeBinary(
|
|
|
818
791
|
}
|
|
819
792
|
|
|
820
793
|
/**
|
|
821
|
-
*
|
|
822
|
-
* documents, each independent on disk, and the result carries what a locale-specific step needs afterwards.
|
|
794
|
+
* Materializes one locale's dev weights overlay from its manifest.
|
|
823
795
|
*/
|
|
824
796
|
export async function materializeDevOverlay(manifest: DevOverlayManifest): Promise<DevOverlay> {
|
|
825
|
-
const destDir =
|
|
826
|
-
const cli =
|
|
797
|
+
const destDir = weightsOverlayPath(manifest.locale)
|
|
798
|
+
const cli = workspacePathBuilder("mailwoman", "out", "cli", "main.js")
|
|
827
799
|
const card = await readWeightsCard(`neural-weights-${manifest.locale}`)
|
|
828
800
|
|
|
829
801
|
await makeDirectories(destDir)
|
|
830
802
|
|
|
831
803
|
if (manifest.model?.kind === "inherit") {
|
|
832
|
-
await removeIfPresent(
|
|
833
|
-
await removeIfPresent(
|
|
804
|
+
await removeIfPresent(destDir("model.onnx"))
|
|
805
|
+
await removeIfPresent(destDir("tokenizer.model"))
|
|
834
806
|
} else if (manifest.model?.kind === "link") {
|
|
835
807
|
await linkBaseModelPair(destDir, manifest.model.digestCard)
|
|
836
808
|
} else if (manifest.model?.kind === "char") {
|
|
@@ -838,7 +810,7 @@ export async function materializeDevOverlay(manifest: DevOverlayManifest): Promi
|
|
|
838
810
|
}
|
|
839
811
|
|
|
840
812
|
for (const { source, name, consequenceIfMissing } of manifest.softFeed ?? []) {
|
|
841
|
-
await linkSoftFeedSibling(source,
|
|
813
|
+
await linkSoftFeedSibling(source, destDir(name), consequenceIfMissing)
|
|
842
814
|
}
|
|
843
815
|
|
|
844
816
|
if (manifest.evidenceLexiconsFromCard) {
|
|
@@ -853,7 +825,7 @@ export async function materializeDevOverlay(manifest: DevOverlayManifest): Promi
|
|
|
853
825
|
|
|
854
826
|
await linkSoftFeedSibling(
|
|
855
827
|
source(declared),
|
|
856
|
-
|
|
828
|
+
destDir(declared),
|
|
857
829
|
`the ${channel} channel will resolve OFF in this worktree.`
|
|
858
830
|
)
|
|
859
831
|
}
|