@mailwoman/resolver-wof-sqlite 10.0.0 → 10.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -8
- package/lib/address/point/interpolation.ts +45 -39
- package/lib/address/point/schema.ts +17 -40
- package/lib/address/point.ts +316 -0
- package/lib/{address/index.ts → address.ts} +1 -1
- package/lib/ancestry/backfill.ts +20 -86
- package/lib/{ancestry/index.ts → ancestry.ts} +19 -15
- package/lib/build/candidate.ts +122 -174
- package/lib/build/population-corrections.ts +64 -0
- package/lib/build/slim.ts +44 -114
- package/lib/candidate/alias-bags.ts +16 -17
- package/lib/candidate/ancestors/schema.ts +77 -61
- package/lib/candidate/ancestors/sidecar.ts +15 -22
- package/lib/candidate/country-display-names.ts +6 -20
- package/lib/candidate/extract-fold.ts +10 -55
- package/lib/candidate/fts.ts +5 -23
- package/lib/candidate/importance.ts +65 -68
- package/lib/candidate/lookup.ts +124 -415
- package/lib/candidate/name-roles.ts +15 -33
- package/lib/candidate/own-name.ts +35 -32
- package/lib/candidate/place-attrs.ts +20 -13
- package/lib/candidate/schema.ts +29 -65
- package/lib/candidate/scoring.ts +54 -93
- package/lib/capital-schema.ts +21 -17
- package/lib/capitals.ts +29 -27
- package/lib/coincident-roles.ts +85 -55
- package/lib/convention/schema.ts +15 -16
- package/lib/convention.ts +144 -0
- package/lib/coverage-manifest-schema.ts +40 -45
- package/lib/currency-backfill.ts +34 -79
- package/lib/env.ts +1 -1
- package/lib/exact-match.ts +32 -21
- package/lib/extracts.ts +97 -74
- package/lib/fst/autocomplete.ts +54 -37
- package/lib/fst/builder.ts +37 -41
- package/lib/fst/deserialize-web.ts +16 -17
- package/lib/fst/format.ts +41 -23
- package/lib/fst/freshness.ts +43 -120
- package/lib/fst/matcher.ts +7 -5
- package/lib/fst/serialize.ts +54 -74
- package/lib/fst/types.ts +78 -75
- package/lib/{fst/index.ts → fst.ts} +1 -0
- package/lib/fts/query.ts +5 -36
- package/lib/fts.ts +280 -0
- package/lib/geonames/aliases.ts +73 -144
- package/lib/geonames/postal.ts +162 -71
- package/lib/index.ts +9 -8
- package/lib/interpolation.ts +48 -100
- package/lib/lookup.ts +155 -270
- package/lib/name-score.ts +9 -7
- package/lib/nsul/lookup.ts +37 -54
- package/lib/nsul/schema.ts +16 -43
- package/lib/paths.ts +159 -0
- package/lib/place-importance-schema.ts +62 -169
- package/lib/poi/lookup.ts +104 -121
- package/lib/poi/schema.ts +42 -26
- package/lib/polygon-schema.ts +10 -6
- package/lib/postal/city/alias/lookup.ts +22 -40
- package/lib/postal/city/alias/schema.ts +14 -10
- package/lib/postal/city/candidate-schema.ts +26 -20
- package/lib/postcode-point-lookup.ts +8 -27
- package/lib/primary-preference.ts +46 -126
- package/lib/proximity-rerank.ts +44 -33
- package/lib/ranking-weights.ts +105 -78
- package/lib/region-keys.ts +70 -42
- package/lib/reverse.ts +149 -86
- package/lib/schema.ts +73 -50
- package/lib/search-fetch.ts +90 -68
- package/lib/sqlite/convention-source.ts +15 -23
- package/lib/sqlite/utils.ts +19 -22
- package/lib/street/centroid/schema.ts +37 -42
- package/lib/street/{centroid/index.ts → centroid.ts} +16 -25
- package/lib/street/morphology/fst/builder.ts +60 -42
- package/lib/street/morphology/fst/loader.ts +40 -32
- package/lib/street/name-lookup.ts +45 -41
- package/lib/street/normalize.ts +88 -219
- package/lib/street/segment-schema.ts +116 -43
- package/lib/{street/index.ts → street.ts} +1 -1
- package/lib/types.ts +109 -150
- package/lib/unified-schema.ts +29 -29
- package/lib/uprn/existence.ts +30 -19
- package/lib/uprn/lookup.ts +29 -68
- package/lib/uprn/schema.ts +18 -40
- package/lib/weights-overlay-linker.ts +236 -252
- package/out/address/point/interpolation.d.ts +20 -23
- package/out/address/point/interpolation.d.ts.map +1 -1
- package/out/address/point/interpolation.js +42 -39
- package/out/address/point/interpolation.js.map +1 -1
- package/out/address/point/schema.d.ts +12 -35
- package/out/address/point/schema.d.ts.map +1 -1
- package/out/address/point/schema.js +3 -19
- package/out/address/point/schema.js.map +1 -1
- package/out/address/point.d.ts +57 -0
- package/out/address/point.d.ts.map +1 -0
- package/out/address/point.js +206 -0
- package/out/address/point.js.map +1 -0
- package/out/{address/index.js → address.d.ts} +2 -2
- package/out/address.d.ts.map +1 -0
- package/out/{address/index.d.ts → address.js} +2 -2
- package/out/address.js.map +1 -0
- package/out/ancestry/backfill.d.ts +11 -64
- package/out/ancestry/backfill.d.ts.map +1 -1
- package/out/ancestry/backfill.js +14 -74
- package/out/ancestry/backfill.js.map +1 -1
- package/out/ancestry.d.ts +49 -0
- package/out/ancestry.d.ts.map +1 -0
- package/out/ancestry.js +59 -0
- package/out/ancestry.js.map +1 -0
- package/out/build/candidate.d.ts +82 -112
- package/out/build/candidate.d.ts.map +1 -1
- package/out/build/candidate.js +17 -100
- package/out/build/candidate.js.map +1 -1
- package/out/build/population-corrections.d.ts +52 -0
- package/out/build/population-corrections.d.ts.map +1 -0
- package/out/build/population-corrections.js +40 -0
- package/out/build/population-corrections.js.map +1 -0
- package/out/build/slim.d.ts +8 -44
- package/out/build/slim.d.ts.map +1 -1
- package/out/build/slim.js +33 -94
- package/out/build/slim.js.map +1 -1
- package/out/candidate/alias-bags.d.ts +6 -6
- package/out/candidate/alias-bags.d.ts.map +1 -1
- package/out/candidate/alias-bags.js +16 -17
- package/out/candidate/alias-bags.js.map +1 -1
- package/out/candidate/ancestors/schema.d.ts +73 -57
- package/out/candidate/ancestors/schema.d.ts.map +1 -1
- package/out/candidate/ancestors/schema.js +56 -48
- package/out/candidate/ancestors/schema.js.map +1 -1
- package/out/candidate/ancestors/sidecar.d.ts +3 -8
- package/out/candidate/ancestors/sidecar.d.ts.map +1 -1
- package/out/candidate/ancestors/sidecar.js +15 -22
- package/out/candidate/ancestors/sidecar.js.map +1 -1
- package/out/candidate/country-display-names.d.ts +5 -15
- package/out/candidate/country-display-names.d.ts.map +1 -1
- package/out/candidate/country-display-names.js +6 -20
- package/out/candidate/country-display-names.js.map +1 -1
- package/out/candidate/extract-fold.d.ts +7 -15
- package/out/candidate/extract-fold.d.ts.map +1 -1
- package/out/candidate/extract-fold.js +5 -47
- package/out/candidate/extract-fold.js.map +1 -1
- package/out/candidate/fts.d.ts +5 -23
- package/out/candidate/fts.d.ts.map +1 -1
- package/out/candidate/fts.js +5 -23
- package/out/candidate/fts.js.map +1 -1
- package/out/candidate/importance.d.ts +60 -65
- package/out/candidate/importance.d.ts.map +1 -1
- package/out/candidate/importance.js +56 -63
- package/out/candidate/importance.js.map +1 -1
- package/out/candidate/lookup.d.ts +18 -55
- package/out/candidate/lookup.d.ts.map +1 -1
- package/out/candidate/lookup.js +82 -385
- package/out/candidate/lookup.js.map +1 -1
- package/out/candidate/name-roles.d.ts +9 -21
- package/out/candidate/name-roles.d.ts.map +1 -1
- package/out/candidate/name-roles.js +15 -33
- package/out/candidate/name-roles.js.map +1 -1
- package/out/candidate/own-name.d.ts +23 -25
- package/out/candidate/own-name.d.ts.map +1 -1
- package/out/candidate/own-name.js +35 -32
- package/out/candidate/own-name.js.map +1 -1
- package/out/candidate/place-attrs.d.ts +20 -13
- package/out/candidate/place-attrs.d.ts.map +1 -1
- package/out/candidate/place-attrs.js +4 -5
- package/out/candidate/place-attrs.js.map +1 -1
- package/out/candidate/schema.d.ts +27 -62
- package/out/candidate/schema.d.ts.map +1 -1
- package/out/candidate/schema.js +10 -22
- package/out/candidate/schema.js.map +1 -1
- package/out/candidate/scoring.d.ts +7 -8
- package/out/candidate/scoring.d.ts.map +1 -1
- package/out/candidate/scoring.js +52 -89
- package/out/candidate/scoring.js.map +1 -1
- package/out/capital-schema.d.ts +21 -17
- package/out/capital-schema.d.ts.map +1 -1
- package/out/capital-schema.js +17 -15
- package/out/capital-schema.js.map +1 -1
- package/out/capitals.d.ts +27 -25
- package/out/capitals.d.ts.map +1 -1
- package/out/capitals.js +23 -21
- package/out/capitals.js.map +1 -1
- package/out/coincident-roles.d.ts +60 -43
- package/out/coincident-roles.d.ts.map +1 -1
- package/out/coincident-roles.js +72 -50
- package/out/coincident-roles.js.map +1 -1
- package/out/convention/schema.d.ts +14 -14
- package/out/convention/schema.d.ts.map +1 -1
- package/out/convention/schema.js +6 -4
- package/out/convention/schema.js.map +1 -1
- package/out/convention.d.ts +97 -0
- package/out/convention.d.ts.map +1 -0
- package/out/convention.js +81 -0
- package/out/convention.js.map +1 -0
- package/out/coverage-manifest-schema.d.ts +34 -39
- package/out/coverage-manifest-schema.d.ts.map +1 -1
- package/out/coverage-manifest-schema.js +25 -34
- package/out/coverage-manifest-schema.js.map +1 -1
- package/out/currency-backfill.d.ts +16 -38
- package/out/currency-backfill.d.ts.map +1 -1
- package/out/currency-backfill.js +19 -46
- package/out/currency-backfill.js.map +1 -1
- package/out/env.d.ts +1 -1
- package/out/env.js +1 -1
- package/out/exact-match.d.ts +19 -10
- package/out/exact-match.d.ts.map +1 -1
- package/out/exact-match.js +32 -21
- package/out/exact-match.js.map +1 -1
- package/out/extracts.d.ts +66 -55
- package/out/extracts.d.ts.map +1 -1
- package/out/extracts.js +63 -55
- package/out/extracts.js.map +1 -1
- package/out/fst/autocomplete.d.ts +29 -21
- package/out/fst/autocomplete.d.ts.map +1 -1
- package/out/fst/autocomplete.js +39 -28
- package/out/fst/autocomplete.js.map +1 -1
- package/out/fst/builder.d.ts.map +1 -1
- package/out/fst/builder.js +37 -41
- package/out/fst/builder.js.map +1 -1
- package/out/fst/deserialize-web.d.ts +4 -4
- package/out/fst/deserialize-web.d.ts.map +1 -1
- package/out/fst/deserialize-web.js +15 -16
- package/out/fst/deserialize-web.js.map +1 -1
- package/out/fst/format.d.ts +41 -23
- package/out/fst/format.d.ts.map +1 -1
- package/out/fst/format.js +41 -23
- package/out/fst/format.js.map +1 -1
- package/out/fst/freshness.d.ts +19 -81
- package/out/fst/freshness.d.ts.map +1 -1
- package/out/fst/freshness.js +31 -94
- package/out/fst/freshness.js.map +1 -1
- package/out/fst/matcher.d.ts +5 -4
- package/out/fst/matcher.d.ts.map +1 -1
- package/out/fst/matcher.js +2 -2
- package/out/fst/matcher.js.map +1 -1
- package/out/fst/serialize.d.ts +28 -28
- package/out/fst/serialize.d.ts.map +1 -1
- package/out/fst/serialize.js +53 -71
- package/out/fst/serialize.js.map +1 -1
- package/out/fst/types.d.ts +78 -74
- package/out/fst/types.d.ts.map +1 -1
- package/out/fst/types.js +1 -4
- package/out/fst/types.js.map +1 -1
- package/out/{fst/index.js → fst.d.ts} +2 -1
- package/out/fst.d.ts.map +1 -0
- package/out/{fst/index.d.ts → fst.js} +2 -1
- package/out/fst.js.map +1 -0
- package/out/fts/query.d.ts +2 -27
- package/out/fts/query.d.ts.map +1 -1
- package/out/fts/query.js +5 -36
- package/out/fts/query.js.map +1 -1
- package/out/fts.d.ts +126 -0
- package/out/fts.d.ts.map +1 -0
- package/out/fts.js +188 -0
- package/out/fts.js.map +1 -0
- package/out/geonames/aliases.d.ts +26 -62
- package/out/geonames/aliases.d.ts.map +1 -1
- package/out/geonames/aliases.js +45 -117
- package/out/geonames/aliases.js.map +1 -1
- package/out/geonames/postal.d.ts +84 -44
- package/out/geonames/postal.d.ts.map +1 -1
- package/out/geonames/postal.js +115 -73
- package/out/geonames/postal.js.map +1 -1
- package/out/{geonames/index.js → geonames.d.ts} +1 -1
- package/out/geonames.d.ts.map +1 -0
- package/out/{geonames/index.d.ts → geonames.js} +1 -1
- package/out/geonames.js.map +1 -0
- package/out/index.d.ts +9 -8
- package/out/index.d.ts.map +1 -1
- package/out/index.js +6 -5
- package/out/index.js.map +1 -1
- package/out/interpolation.d.ts +18 -55
- package/out/interpolation.d.ts.map +1 -1
- package/out/interpolation.js +27 -69
- package/out/interpolation.js.map +1 -1
- package/out/lookup.d.ts +23 -51
- package/out/lookup.d.ts.map +1 -1
- package/out/lookup.js +136 -234
- package/out/lookup.js.map +1 -1
- package/out/name-score.d.ts +2 -2
- package/out/name-score.d.ts.map +1 -1
- package/out/name-score.js +9 -7
- package/out/name-score.js.map +1 -1
- package/out/nsul/lookup.d.ts +33 -34
- package/out/nsul/lookup.d.ts.map +1 -1
- package/out/nsul/lookup.js +26 -40
- package/out/nsul/lookup.js.map +1 -1
- package/out/nsul/schema.d.ts +16 -43
- package/out/nsul/schema.d.ts.map +1 -1
- package/out/nsul/schema.js +9 -35
- package/out/nsul/schema.js.map +1 -1
- package/out/{nsul/index.d.ts → nsul.d.ts} +1 -1
- package/out/nsul.d.ts.map +1 -0
- package/out/{nsul/index.js → nsul.js} +1 -1
- package/out/nsul.js.map +1 -0
- package/out/paths.d.ts +124 -0
- package/out/paths.d.ts.map +1 -0
- package/out/paths.js +104 -0
- package/out/paths.js.map +1 -0
- package/out/place-importance-schema.d.ts +51 -143
- package/out/place-importance-schema.d.ts.map +1 -1
- package/out/place-importance-schema.js +28 -143
- package/out/place-importance-schema.js.map +1 -1
- package/out/poi/lookup.d.ts +40 -47
- package/out/poi/lookup.d.ts.map +1 -1
- package/out/poi/lookup.js +69 -96
- package/out/poi/lookup.js.map +1 -1
- package/out/poi/schema.d.ts +41 -25
- package/out/poi/schema.d.ts.map +1 -1
- package/out/poi/schema.js +25 -17
- package/out/poi/schema.js.map +1 -1
- package/out/{poi/index.js → poi.d.ts} +1 -1
- package/out/poi.d.ts.map +1 -0
- package/out/{poi/index.d.ts → poi.js} +1 -1
- package/out/poi.js.map +1 -0
- package/out/polygon-schema.d.ts +10 -6
- package/out/polygon-schema.d.ts.map +1 -1
- package/out/polygon-schema.js +2 -2
- package/out/polygon-schema.js.map +1 -1
- package/out/postal/city/alias/lookup.d.ts +20 -23
- package/out/postal/city/alias/lookup.d.ts.map +1 -1
- package/out/postal/city/alias/lookup.js +17 -30
- package/out/postal/city/alias/lookup.js.map +1 -1
- package/out/postal/city/alias/schema.d.ts +14 -10
- package/out/postal/city/alias/schema.d.ts.map +1 -1
- package/out/postal/city/alias/schema.js +10 -8
- package/out/postal/city/alias/schema.js.map +1 -1
- package/out/postal/city/candidate-schema.d.ts +25 -19
- package/out/postal/city/candidate-schema.d.ts.map +1 -1
- package/out/postal/city/candidate-schema.js +21 -17
- package/out/postal/city/candidate-schema.js.map +1 -1
- package/out/{postal/index.d.ts → postal.d.ts} +1 -1
- package/out/postal.d.ts.map +1 -0
- package/out/{postal/index.js → postal.js} +1 -1
- package/out/postal.js.map +1 -0
- package/out/postcode-point-lookup.d.ts +6 -24
- package/out/postcode-point-lookup.d.ts.map +1 -1
- package/out/postcode-point-lookup.js +5 -17
- package/out/postcode-point-lookup.js.map +1 -1
- package/out/primary-preference.d.ts +38 -92
- package/out/primary-preference.d.ts.map +1 -1
- package/out/primary-preference.js +15 -99
- package/out/primary-preference.js.map +1 -1
- package/out/proximity-rerank.d.ts +41 -30
- package/out/proximity-rerank.d.ts.map +1 -1
- package/out/proximity-rerank.js +38 -29
- package/out/proximity-rerank.js.map +1 -1
- package/out/ranking-weights.d.ts +98 -64
- package/out/ranking-weights.d.ts.map +1 -1
- package/out/ranking-weights.js +26 -25
- package/out/ranking-weights.js.map +1 -1
- package/out/region-keys.d.ts +50 -31
- package/out/region-keys.d.ts.map +1 -1
- package/out/region-keys.js +68 -40
- package/out/region-keys.js.map +1 -1
- package/out/reverse.d.ts +78 -50
- package/out/reverse.d.ts.map +1 -1
- package/out/reverse.js +98 -61
- package/out/reverse.js.map +1 -1
- package/out/schema.d.ts +72 -49
- package/out/schema.d.ts.map +1 -1
- package/out/schema.js +4 -5
- package/out/schema.js.map +1 -1
- package/out/search-fetch.d.ts +16 -12
- package/out/search-fetch.d.ts.map +1 -1
- package/out/search-fetch.js +83 -63
- package/out/search-fetch.js.map +1 -1
- package/out/sqlite/convention-source.d.ts +12 -16
- package/out/sqlite/convention-source.d.ts.map +1 -1
- package/out/sqlite/convention-source.js +13 -22
- package/out/sqlite/convention-source.js.map +1 -1
- package/out/sqlite/utils.d.ts +19 -19
- package/out/sqlite/utils.d.ts.map +1 -1
- package/out/sqlite/utils.js +14 -15
- package/out/sqlite/utils.js.map +1 -1
- package/out/street/centroid/schema.d.ts +37 -42
- package/out/street/centroid/schema.d.ts.map +1 -1
- package/out/street/centroid/schema.js +9 -21
- package/out/street/centroid/schema.js.map +1 -1
- package/out/street/centroid.d.ts +30 -0
- package/out/street/centroid.d.ts.map +1 -0
- package/out/street/{centroid/index.js → centroid.js} +13 -24
- package/out/street/centroid.js.map +1 -0
- package/out/street/morphology/fst/builder.d.ts +20 -10
- package/out/street/morphology/fst/builder.d.ts.map +1 -1
- package/out/street/morphology/fst/builder.js +44 -34
- package/out/street/morphology/fst/builder.js.map +1 -1
- package/out/street/morphology/fst/loader.d.ts +33 -26
- package/out/street/morphology/fst/loader.d.ts.map +1 -1
- package/out/street/morphology/fst/loader.js +23 -22
- package/out/street/morphology/fst/loader.js.map +1 -1
- package/out/street/name-lookup.d.ts +24 -18
- package/out/street/name-lookup.d.ts.map +1 -1
- package/out/street/name-lookup.js +34 -35
- package/out/street/name-lookup.js.map +1 -1
- package/out/street/normalize.d.ts +35 -136
- package/out/street/normalize.d.ts.map +1 -1
- package/out/street/normalize.js +77 -194
- package/out/street/normalize.js.map +1 -1
- package/out/street/segment-schema.d.ts +79 -39
- package/out/street/segment-schema.d.ts.map +1 -1
- package/out/street/segment-schema.js +71 -21
- package/out/street/segment-schema.js.map +1 -1
- package/out/{street/index.js → street.d.ts} +2 -2
- package/out/street.d.ts.map +1 -0
- package/out/{street/index.d.ts → street.js} +2 -2
- package/out/street.js.map +1 -0
- package/out/types.d.ts +84 -149
- package/out/types.d.ts.map +1 -1
- package/out/types.js +0 -6
- package/out/types.js.map +1 -1
- package/out/unified-schema.d.ts +8 -12
- package/out/unified-schema.d.ts.map +1 -1
- package/out/unified-schema.js +29 -29
- package/out/unified-schema.js.map +1 -1
- package/out/uprn/existence.d.ts +28 -17
- package/out/uprn/existence.d.ts.map +1 -1
- package/out/uprn/existence.js +26 -16
- package/out/uprn/existence.js.map +1 -1
- package/out/uprn/lookup.d.ts +14 -44
- package/out/uprn/lookup.d.ts.map +1 -1
- package/out/uprn/lookup.js +21 -57
- package/out/uprn/lookup.js.map +1 -1
- package/out/uprn/schema.d.ts +18 -40
- package/out/uprn/schema.d.ts.map +1 -1
- package/out/uprn/schema.js +10 -32
- package/out/uprn/schema.js.map +1 -1
- package/out/{uprn/index.d.ts → uprn.d.ts} +1 -1
- package/out/uprn.d.ts.map +1 -0
- package/out/{uprn/index.js → uprn.js} +1 -1
- package/out/uprn.js.map +1 -0
- package/out/weights-overlay-linker.d.ts +138 -144
- package/out/weights-overlay-linker.d.ts.map +1 -1
- package/out/weights-overlay-linker.js +105 -159
- package/out/weights-overlay-linker.js.map +1 -1
- package/package.json +91 -596
- package/lib/address/point/index.ts +0 -328
- package/lib/convention/index.ts +0 -154
- package/lib/fts/index.ts +0 -361
- package/out/address/index.d.ts.map +0 -1
- package/out/address/index.js.map +0 -1
- package/out/address/point/index.d.ts +0 -53
- package/out/address/point/index.d.ts.map +0 -1
- package/out/address/point/index.js +0 -222
- package/out/address/point/index.js.map +0 -1
- package/out/ancestry/index.d.ts +0 -45
- package/out/ancestry/index.d.ts.map +0 -1
- package/out/ancestry/index.js +0 -55
- package/out/ancestry/index.js.map +0 -1
- package/out/convention/index.d.ts +0 -106
- package/out/convention/index.d.ts.map +0 -1
- package/out/convention/index.js +0 -93
- package/out/convention/index.js.map +0 -1
- package/out/fst/index.d.ts.map +0 -1
- package/out/fst/index.js.map +0 -1
- package/out/fts/index.d.ts +0 -184
- package/out/fts/index.d.ts.map +0 -1
- package/out/fts/index.js +0 -277
- package/out/fts/index.js.map +0 -1
- package/out/geonames/index.d.ts.map +0 -1
- package/out/geonames/index.js.map +0 -1
- package/out/nsul/index.d.ts.map +0 -1
- package/out/nsul/index.js.map +0 -1
- package/out/poi/index.d.ts.map +0 -1
- package/out/poi/index.js.map +0 -1
- package/out/postal/index.d.ts.map +0 -1
- package/out/postal/index.js.map +0 -1
- package/out/street/centroid/index.d.ts +0 -40
- package/out/street/centroid/index.d.ts.map +0 -1
- package/out/street/centroid/index.js.map +0 -1
- package/out/street/index.d.ts.map +0 -1
- package/out/street/index.js.map +0 -1
- package/out/uprn/index.d.ts.map +0 -1
- package/out/uprn/index.js.map +0 -1
- /package/lib/{geonames/index.ts → geonames.ts} +0 -0
- /package/lib/{nsul/index.ts → nsul.ts} +0 -0
- /package/lib/{poi/index.ts → poi.ts} +0 -0
- /package/lib/{postal/index.ts → postal.ts} +0 -0
- /package/lib/{uprn/index.ts → uprn.ts} +0 -0
|
@@ -2,20 +2,11 @@
|
|
|
2
2
|
* @copyright Sister Software
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
|
-
* Shared development tooling for weights overlays: the symlink primitives every overlay linker uses, the
|
|
6
|
-
* placetype-pair index build, and the manifest form ({@link materializeDevOverlay}) each locale's
|
|
7
|
-
* `scripts/link-dev-weights.ts` declares itself in.
|
|
8
5
|
*/
|
|
9
6
|
|
|
10
|
-
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
11
|
-
import {
|
|
12
|
-
|
|
13
|
-
readLocalJSONFile,
|
|
14
|
-
readLocalTextFile,
|
|
15
|
-
statPath,
|
|
16
|
-
readLocalBuffer,
|
|
17
|
-
statLink,
|
|
18
|
-
} from "@mailwoman/core/fs/readers"
|
|
7
|
+
import { dataRootPath, weightsOverlayPath } from "@mailwoman/core/data-root"
|
|
8
|
+
import { readLocalJSONFile, readLocalTextFile, readLocalBuffer } from "@mailwoman/core/fs/readers"
|
|
9
|
+
import { pathExists, statPath, statLink } from "@mailwoman/core/fs/readers/stat"
|
|
19
10
|
import {
|
|
20
11
|
createSymbolicLink,
|
|
21
12
|
makeDirectories,
|
|
@@ -25,22 +16,22 @@ import {
|
|
|
25
16
|
} from "@mailwoman/core/fs/writers"
|
|
26
17
|
import { md5File } from "@mailwoman/core/hash"
|
|
27
18
|
import { parseJSONStrict } from "@mailwoman/core/json"
|
|
28
|
-
import { repoRootPath,
|
|
19
|
+
import { repoRootPath, repoRootPathBuilder, workspacePathBuilder } from "@mailwoman/core/paths"
|
|
29
20
|
import { spawnProcessSync } from "@mailwoman/core/process"
|
|
30
21
|
import { readReleaseConfig, repoCommittedSoftFeedSources } from "@mailwoman/core/release-config"
|
|
31
|
-
import {
|
|
32
|
-
import { resolvePath } from "path-ts"
|
|
22
|
+
import type { PathBuilder, PathBuilderLike } from "path-ts"
|
|
33
23
|
|
|
34
24
|
import { $public } from "#env"
|
|
35
25
|
import { fstFreshnessWarning } from "#fst/freshness"
|
|
26
|
+
import { wofDatabasePath } from "#paths"
|
|
36
27
|
|
|
37
28
|
/**
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
* rename
|
|
29
|
+
* Replaces `dest` with a symlink to `src` by renaming a temporary link over it.
|
|
30
|
+
*
|
|
31
|
+
* The rename is atomic, so concurrent test workers never see `dest` missing.
|
|
41
32
|
*/
|
|
42
|
-
export async function linkForce(src:
|
|
43
|
-
const tmp = `${dest}.tmp-link`
|
|
33
|
+
export async function linkForce(src: PathBuilderLike, dest: PathBuilderLike): Promise<void> {
|
|
34
|
+
const tmp = `${dest.toString()}.tmp-link`
|
|
44
35
|
|
|
45
36
|
if (await pathExists(tmp)) {
|
|
46
37
|
await removePath(tmp)
|
|
@@ -51,9 +42,9 @@ export async function linkForce(src: string, dest: string): Promise<void> {
|
|
|
51
42
|
}
|
|
52
43
|
|
|
53
44
|
/**
|
|
54
|
-
*
|
|
45
|
+
* Removes a file or symlink at `dest` so that the runtime falls back to the base en-us weights.
|
|
55
46
|
*/
|
|
56
|
-
export async function removeIfPresent(dest:
|
|
47
|
+
export async function removeIfPresent(dest: PathBuilder): Promise<void> {
|
|
57
48
|
try {
|
|
58
49
|
await statLink(dest)
|
|
59
50
|
} catch {
|
|
@@ -66,15 +57,14 @@ export async function removeIfPresent(dest: string): Promise<void> {
|
|
|
66
57
|
}
|
|
67
58
|
|
|
68
59
|
/**
|
|
69
|
-
*
|
|
60
|
+
* Symlinks one optional soft-feed artifact into an overlay and returns whether the source existed.
|
|
70
61
|
*
|
|
71
|
-
*
|
|
72
|
-
* has
|
|
73
|
-
* operator needs to know which channel just resolved OFF, not to have the link step abort.
|
|
62
|
+
* A missing source prints `consequenceIfMissing` and does not throw, because the
|
|
63
|
+
* runtime has a fallback for every soft-feed artifact.
|
|
74
64
|
*/
|
|
75
65
|
export async function linkSoftFeedSibling(
|
|
76
|
-
source:
|
|
77
|
-
destination:
|
|
66
|
+
source: PathBuilderLike,
|
|
67
|
+
destination: PathBuilder,
|
|
78
68
|
consequenceIfMissing: string
|
|
79
69
|
): Promise<boolean> {
|
|
80
70
|
if (!(await pathExists(source))) {
|
|
@@ -91,100 +81,116 @@ export async function linkSoftFeedSibling(
|
|
|
91
81
|
}
|
|
92
82
|
|
|
93
83
|
/**
|
|
94
|
-
* The
|
|
95
|
-
* measured at δ=10 (each linker's own docstring carries its locale's receipt).
|
|
84
|
+
* The pair-index emission bias (δ) used by every shipped pair-index artifact.
|
|
96
85
|
*/
|
|
97
86
|
export const PAIR_INDEX_DELTA = 10
|
|
98
87
|
|
|
99
88
|
/**
|
|
100
|
-
* The decoder transition-entry bonus (
|
|
101
|
-
*
|
|
102
|
-
* independently, and a locale earning one says nothing about the other.
|
|
89
|
+
* The decoder transition-entry bonus (β).
|
|
90
|
+
* Only locales that were calibrated with it pass it.
|
|
103
91
|
*/
|
|
104
92
|
export const PAIR_INDEX_TRANSITION_BETA = 5
|
|
105
93
|
|
|
106
94
|
/**
|
|
107
|
-
* The
|
|
108
|
-
*
|
|
109
|
-
*
|
|
95
|
+
* The whole-edge parent-bias magnitude.
|
|
96
|
+
* Only locales that were calibrated with it pass it.
|
|
97
|
+
*
|
|
98
|
+
* Other locales omit the flag and do not inherit this value.
|
|
110
99
|
*/
|
|
111
100
|
export const PAIR_INDEX_PARENT_DELTA = 5
|
|
112
101
|
|
|
113
102
|
/**
|
|
114
|
-
*
|
|
103
|
+
* Describes one locale's pair-index build.
|
|
104
|
+
*
|
|
105
|
+
* When `sources`, `inputs` and `extraArgs` are omitted, the build reads the shared WOF admin database.
|
|
115
106
|
*/
|
|
116
107
|
export interface PairIndexOverlay {
|
|
117
108
|
/**
|
|
118
|
-
*
|
|
109
|
+
* The weights workspace directory name, such as `neural-weights-de-de`.
|
|
110
|
+
* The overlay locale is derived from it.
|
|
119
111
|
*/
|
|
120
112
|
packageDir: string
|
|
113
|
+
|
|
121
114
|
/**
|
|
122
|
-
* ISO country code passed to `
|
|
123
|
-
*
|
|
115
|
+
* The ISO country code passed to `--country` and used in `pair-index-<country>.bin`.
|
|
116
|
+
*
|
|
117
|
+
* It can differ from the locale tag.
|
|
118
|
+
* For example, `en-in` builds `pair-index-in.bin`.
|
|
124
119
|
*/
|
|
125
120
|
country: string
|
|
121
|
+
|
|
126
122
|
/**
|
|
127
|
-
*
|
|
128
|
-
*
|
|
129
|
-
*
|
|
130
|
-
* The
|
|
123
|
+
* The child bias magnitude.
|
|
124
|
+
*
|
|
125
|
+
* The build writes `delta`, `transitionBeta` and `parentDelta` into the PIX1 header.
|
|
126
|
+
* The freshness check compares them.
|
|
127
|
+
*
|
|
128
|
+
* An absent `transitionBeta` or `parentDelta` leaves its key out of the header.
|
|
129
|
+
* The freshness check treats an absent key as distinct from zero.
|
|
131
130
|
*/
|
|
132
131
|
delta: number
|
|
133
132
|
transitionBeta?: number
|
|
134
133
|
parentDelta?: number
|
|
134
|
+
|
|
135
135
|
/**
|
|
136
|
-
*
|
|
137
|
-
*
|
|
138
|
-
*
|
|
139
|
-
*
|
|
140
|
-
*
|
|
136
|
+
* The source files whose MD5s the build records, in the order that `gazetteer pair-index` records them.
|
|
137
|
+
*
|
|
138
|
+
* The freshness check compares each MD5 with the header.
|
|
139
|
+
* An empty list makes the check compare the magnitudes only.
|
|
140
|
+
*
|
|
141
|
+
* @defaultValue the WOF admin database.
|
|
141
142
|
*/
|
|
142
|
-
sources?:
|
|
143
|
+
sources?: PathBuilder[]
|
|
144
|
+
|
|
143
145
|
/**
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
+
* Files that must exist before a build runs.
|
|
147
|
+
*
|
|
148
|
+
* A missing file prints a warning and skips the build.
|
|
149
|
+
*
|
|
150
|
+
* @defaultValue `sources`
|
|
146
151
|
*/
|
|
147
|
-
inputs?:
|
|
152
|
+
inputs?: PathBuilder[]
|
|
153
|
+
|
|
148
154
|
/**
|
|
149
|
-
* Extra CLI
|
|
150
|
-
*
|
|
155
|
+
* Extra CLI arguments that pass the build's sources.
|
|
156
|
+
*
|
|
157
|
+
* @defaultValue `--borough-db <admin db>`
|
|
151
158
|
*/
|
|
152
|
-
extraArgs?:
|
|
159
|
+
extraArgs?: PathBuilderLike[]
|
|
160
|
+
|
|
153
161
|
/**
|
|
154
|
-
*
|
|
155
|
-
*
|
|
162
|
+
* The byte size below which an existing artifact is rebuilt.
|
|
163
|
+
*
|
|
164
|
+
* A build from the wrong source can have matching header magnitudes but a smaller size.
|
|
156
165
|
*/
|
|
157
166
|
minimumPlausibleBytes?: number
|
|
158
167
|
}
|
|
159
168
|
|
|
160
169
|
/**
|
|
161
|
-
* The header fields
|
|
170
|
+
* The pair-index header fields that the dev-weights freshness check reads.
|
|
162
171
|
*/
|
|
163
172
|
export interface PairIndexHeaderFields {
|
|
164
173
|
delta: number
|
|
165
174
|
transitionBeta: number | undefined
|
|
166
175
|
parentDelta: number | undefined
|
|
167
176
|
schemaVersion: number
|
|
177
|
+
|
|
168
178
|
/**
|
|
169
|
-
* One
|
|
179
|
+
* One MD5 per source the build read, in recorded order.
|
|
180
|
+
* The list is empty when the header has none.
|
|
170
181
|
*/
|
|
171
182
|
sourceMD5s: string[]
|
|
172
183
|
}
|
|
173
184
|
|
|
174
185
|
/**
|
|
175
|
-
*
|
|
176
|
-
* `@mailwoman/neural` (which pulls onnxruntime-node) to read a few fields. `neural/pair-index-resolver.ts`'s own header
|
|
177
|
-
* parse is the source of truth this must follow.
|
|
186
|
+
* Reads the calibration fields from a PIX1 pair-index header without depending on `@mailwoman/neural`.
|
|
178
187
|
*
|
|
179
|
-
*
|
|
180
|
-
* (`neural-weights-{en-us,en-gb,en-nz,fr-fr}/scripts/link-dev-weights.ts`), which each carried their own near-copy
|
|
181
|
-
* before 2026-08-04 — the ×5 clone the taste audit named, and the reason three of them were schema-blind while this one
|
|
182
|
-
* was not.
|
|
188
|
+
* The parse must match the reader in `neural/lib/pair/index/resolver.ts`.
|
|
183
189
|
*/
|
|
184
|
-
export async function peekPairIndexHeaderFields(path:
|
|
190
|
+
export async function peekPairIndexHeaderFields(path: PathBuilderLike): Promise<PairIndexHeaderFields> {
|
|
185
191
|
const bytes = await readLocalBuffer(path)
|
|
186
192
|
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
|
187
|
-
|
|
193
|
+
|
|
188
194
|
const MAGIC = 0x31_58_49_50
|
|
189
195
|
|
|
190
196
|
if (view.getUint32(0, true) !== MAGIC) {
|
|
@@ -210,21 +216,17 @@ export async function peekPairIndexHeaderFields(path: string): Promise<PairIndex
|
|
|
210
216
|
}
|
|
211
217
|
}
|
|
212
218
|
|
|
213
|
-
/**
|
|
214
|
-
* Hex characters in an md5 digest.
|
|
215
|
-
*/
|
|
216
219
|
const MD5_HEX_LENGTH = 32
|
|
217
220
|
|
|
221
|
+
// repo-health-ignore export-name-affix -- This variant caches the hash in a sidecar.
|
|
222
|
+
// `md5File` hashes every time.
|
|
218
223
|
/**
|
|
219
|
-
*
|
|
220
|
-
* source is hashed once per change rather than once per linker run. The sidecar is trusted only when at least as new as
|
|
221
|
-
* the source; a missing or stale sidecar recomputes and rewrites, so the cache self-heals.
|
|
224
|
+
* Returns the MD5 of `path`, cached in a `<path>.md5` sidecar in `md5sum` format.
|
|
222
225
|
*
|
|
223
|
-
* The
|
|
226
|
+
* The sidecar is used only when it is at least as new as the source.
|
|
224
227
|
*/
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
const sidecarPath = `${path}.md5`
|
|
228
|
+
export async function md5FileWithSidecar(path: PathBuilderLike): Promise<string> {
|
|
229
|
+
const sidecarPath = `${path.toString()}.md5`
|
|
228
230
|
const sourceStats = await statPath(path)
|
|
229
231
|
|
|
230
232
|
if (await pathExists(sidecarPath)) {
|
|
@@ -234,9 +236,7 @@ export async function md5FileWithSidecar(path: string): Promise<string> {
|
|
|
234
236
|
|
|
235
237
|
if (hash && hash.length === MD5_HEX_LENGTH) return hash
|
|
236
238
|
}
|
|
237
|
-
} catch {
|
|
238
|
-
// Unreadable sidecar — recompute below.
|
|
239
|
-
}
|
|
239
|
+
} catch {}
|
|
240
240
|
}
|
|
241
241
|
|
|
242
242
|
const hash = await md5File(path)
|
|
@@ -248,9 +248,10 @@ export async function md5FileWithSidecar(path: string): Promise<string> {
|
|
|
248
248
|
}
|
|
249
249
|
|
|
250
250
|
/**
|
|
251
|
-
* The calibrated magnitudes a linker
|
|
252
|
-
*
|
|
253
|
-
*
|
|
251
|
+
* The calibrated magnitudes that a linker writes into its pair-index artifact.
|
|
252
|
+
*
|
|
253
|
+
* An `undefined` magnitude means the header has no such key.
|
|
254
|
+
* It is distinct from zero.
|
|
254
255
|
*/
|
|
255
256
|
export interface PairIndexCalibration {
|
|
256
257
|
delta: number
|
|
@@ -259,17 +260,12 @@ export interface PairIndexCalibration {
|
|
|
259
260
|
}
|
|
260
261
|
|
|
261
262
|
/**
|
|
262
|
-
*
|
|
263
|
-
*
|
|
264
|
-
* linker passes a different set of sources and only it knows what they are.
|
|
263
|
+
* Returns why a pair-index header's schema or magnitudes differ from `expected`,
|
|
264
|
+
* or `undefined` when they match.
|
|
265
265
|
*
|
|
266
|
-
*
|
|
267
|
-
* three of the four base linkers ended up unable to notice a schema bump.
|
|
266
|
+
* The caller checks the source MD5s because only the caller knows its sources.
|
|
268
267
|
*/
|
|
269
|
-
export function pairIndexStaleReason(
|
|
270
|
-
header: PairIndexHeaderFields,
|
|
271
|
-
expected: PairIndexCalibration
|
|
272
|
-
): string | undefined {
|
|
268
|
+
export function pairIndexStaleReason(header: PairIndexHeaderFields, expected: PairIndexCalibration): string | null {
|
|
273
269
|
if (header.schemaVersion !== REQUIRED_PAIR_INDEX_SCHEMA) {
|
|
274
270
|
return `schemaVersion ${header.schemaVersion} → ${REQUIRED_PAIR_INDEX_SCHEMA}`
|
|
275
271
|
}
|
|
@@ -284,42 +280,28 @@ export function pairIndexStaleReason(
|
|
|
284
280
|
return `parentDelta ${header.parentDelta ?? "(absent)"} → ${expected.parentDelta ?? "(absent)"}`
|
|
285
281
|
}
|
|
286
282
|
|
|
287
|
-
return
|
|
283
|
+
return null
|
|
288
284
|
}
|
|
289
285
|
|
|
290
286
|
/**
|
|
291
|
-
* The PIX1 schema
|
|
292
|
-
* they are two ends of one fact, and this copy exists only because a data-only overlay must not gain a dependency on
|
|
293
|
-
* `@mailwoman/neural` (onnxruntime-node) to read one header field. Bump BOTH in the same commit; a schema bump that
|
|
294
|
-
* leaves this behind makes every dev checkout rebuild-loop or serve an artifact the runtime refuses.
|
|
287
|
+
* The PIX1 schema version the pair-index reader requires.
|
|
295
288
|
*
|
|
296
|
-
*
|
|
297
|
-
*
|
|
298
|
-
* edition. Also compared by the four hand-written base linkers (`neural-weights-{en-us,en-gb,en-nz,fr-fr}`), which
|
|
299
|
-
* import this constant rather than re-typing the number.
|
|
289
|
+
* It must equal `KNOWN_SCHEMA_VERSION` in `neural/lib/pair/index/resolver.ts`.
|
|
290
|
+
* A mismatch makes dev checkouts rebuild on every run or link an artifact that the runtime refuses.
|
|
300
291
|
*/
|
|
301
292
|
export const REQUIRED_PAIR_INDEX_SCHEMA = 3
|
|
302
293
|
|
|
303
294
|
/**
|
|
304
|
-
*
|
|
305
|
-
* now.
|
|
295
|
+
* Warns when a linked per-locale FST was built from a different admin database than the one on disk.
|
|
306
296
|
*
|
|
307
|
-
*
|
|
308
|
-
*
|
|
309
|
-
* the swap into `fst-per-locale/` is operator-approved after the battery, because an FST changes decoder behaviour and
|
|
310
|
-
* the D-rule does not let that land unmeasured. So the guard's job is to make the drift impossible to miss, and to name
|
|
311
|
-
* the command that starts fixing it. It is also why a stale FST is never fatal: the artifact is a decode-time bias
|
|
312
|
-
* list, and a dev tree must still run.
|
|
313
|
-
*
|
|
314
|
-
* The source-side half of the comparison lives here rather than in `fst-freshness.ts` for the same reason
|
|
315
|
-
* `pairIndexStaleReason` splits: only the caller knows which database it built against. All three FST-linking base
|
|
316
|
-
* packages build against the same one, so they share this rather than each pinning it.
|
|
297
|
+
* It does not rebuild, because an FST rebuild writes to a staging directory
|
|
298
|
+
* and an operator swaps it in after evaluation.
|
|
317
299
|
*/
|
|
318
|
-
export async function warnIfFSTStale(fstPath:
|
|
300
|
+
export async function warnIfFSTStale(fstPath: PathBuilder, locale: string): Promise<void> {
|
|
319
301
|
const warning = await fstFreshnessWarning({
|
|
320
302
|
fstPath,
|
|
321
|
-
sourceDBPath:
|
|
322
|
-
rebuildCommand: `node packages/mailwoman/out/cli/
|
|
303
|
+
sourceDBPath: wofDatabasePath("admin-global-priority.db"),
|
|
304
|
+
rebuildCommand: `node packages/mailwoman/out/cli/main.js gazetteer build fst --locales ${locale} (writes to a staging dir; swap is operator-conditional)`,
|
|
323
305
|
})
|
|
324
306
|
|
|
325
307
|
if (warning) {
|
|
@@ -328,17 +310,15 @@ export async function warnIfFSTStale(fstPath: string, locale: string): Promise<v
|
|
|
328
310
|
}
|
|
329
311
|
|
|
330
312
|
/**
|
|
331
|
-
*
|
|
332
|
-
*
|
|
333
|
-
* runtime pipeline can auto-wire the gazetteer + street-context check, then run {@link warnIfFSTStale} on the linked
|
|
334
|
-
* artifact. The publish flow stages the real binary (release-sequenced).
|
|
313
|
+
* Symlinks the per-locale FST gazetteer `fst-<locale>.bin` into an overlay
|
|
314
|
+
* and checks it with {@link warnIfFSTStale}.
|
|
335
315
|
*/
|
|
336
|
-
export async function linkLocaleFST(destDir:
|
|
337
|
-
const source =
|
|
316
|
+
export async function linkLocaleFST(destDir: PathBuilder, locale: string): Promise<void> {
|
|
317
|
+
const source = wofDatabasePath("fst-per-locale", `fst-${locale}.bin`)
|
|
338
318
|
|
|
339
319
|
const linked = await linkSoftFeedSibling(
|
|
340
320
|
source,
|
|
341
|
-
|
|
321
|
+
destDir(`fst-${locale}.bin`),
|
|
342
322
|
"the FST gazetteer default will resolve OFF for this locale."
|
|
343
323
|
)
|
|
344
324
|
|
|
@@ -348,32 +328,22 @@ export async function linkLocaleFST(destDir: string, locale: string): Promise<vo
|
|
|
348
328
|
}
|
|
349
329
|
|
|
350
330
|
/**
|
|
351
|
-
*
|
|
352
|
-
*
|
|
353
|
-
* dev and the street-context check (#1315) deserializes the artifact instead of rebuilding from the libpostal
|
|
354
|
-
* dictionaries per process. Missing is non-fatal — the runtime loader's dictionary-build fallback covers it.
|
|
331
|
+
* Symlinks the prebuilt street-morphology FST into an overlay so that the
|
|
332
|
+
* street-context check does not rebuild it from dictionaries.
|
|
355
333
|
*/
|
|
356
|
-
export async function linkStreetMorphologyFST(destDir:
|
|
334
|
+
export async function linkStreetMorphologyFST(destDir: PathBuilder): Promise<void> {
|
|
357
335
|
await linkSoftFeedSibling(
|
|
358
|
-
|
|
359
|
-
|
|
336
|
+
wofDatabasePath("fst-street-morphology.bin"),
|
|
337
|
+
destDir("fst-street-morphology.bin"),
|
|
360
338
|
"the street-context check falls back to the per-process dictionary build."
|
|
361
339
|
)
|
|
362
340
|
}
|
|
363
341
|
|
|
364
|
-
/**
|
|
365
|
-
* Why an existing artifact may be TRUSTED without a rebuild, or `undefined` when it must be rebuilt (with the reason
|
|
366
|
-
* already printed). The skip requires both halves (#1734): the header magnitudes + format (`pairIndexStaleReason`) AND
|
|
367
|
-
* the header's `sourceMD5s` against the current sources' md5s. Magnitudes alone read a source change as "current"
|
|
368
|
-
* whenever pair counts happen not to move the calibrated numbers — the R5 freshness-guard lesson, which resurfaced in
|
|
369
|
-
* the 2026-08-18 admin swap. A source that is not on disk to re-hash leaves the magnitudes as the best available answer
|
|
370
|
-
* (the "missing source, can't build" branch below would fire anyway if a rebuild were needed).
|
|
371
|
-
*/
|
|
372
342
|
async function pairIndexIsFresh(
|
|
373
|
-
dest:
|
|
343
|
+
dest: PathBuilder,
|
|
374
344
|
artifact: string,
|
|
375
345
|
expected: PairIndexCalibration,
|
|
376
|
-
sources:
|
|
346
|
+
sources: PathBuilder[],
|
|
377
347
|
minimumPlausibleBytes: number | undefined
|
|
378
348
|
): Promise<boolean> {
|
|
379
349
|
try {
|
|
@@ -447,30 +417,23 @@ async function pairIndexIsFresh(
|
|
|
447
417
|
}
|
|
448
418
|
|
|
449
419
|
/**
|
|
450
|
-
*
|
|
451
|
-
* these magnitudes
|
|
452
|
-
* staleness expectation and the CLI flags, so a magnitude cannot be checked by the guard and dropped from the build.
|
|
420
|
+
* Builds `pair-index-<country>.bin` into the overlay unless the existing artifact
|
|
421
|
+
* already matches these magnitudes and sources.
|
|
453
422
|
*
|
|
454
|
-
*
|
|
455
|
-
*
|
|
456
|
-
* have run a build yet rather than a real fault.
|
|
423
|
+
* A failed build exits the process.
|
|
424
|
+
* A missing CLI or input only prints a warning, because a fresh clone has neither.
|
|
457
425
|
*/
|
|
458
426
|
export async function buildPairIndexOverlay(overlay: PairIndexOverlay): Promise<void> {
|
|
459
427
|
const { packageDir, country, delta, transitionBeta, parentDelta } = overlay
|
|
460
|
-
const CLI =
|
|
428
|
+
const CLI = workspacePathBuilder("mailwoman", "out", "cli", "main.js")
|
|
461
429
|
const ARTIFACT = `pair-index-${country}.bin`
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
const PKG_DIR = String(weightsOverlayPath(packageDir.replace(/^neural-weights-/, "")))
|
|
466
|
-
const DEST = resolvePath(PKG_DIR, ARTIFACT)
|
|
430
|
+
|
|
431
|
+
const PKG_DIR = weightsOverlayPath(packageDir.replace(/^neural-weights-/, ""))
|
|
432
|
+
const DEST = PKG_DIR(ARTIFACT)
|
|
467
433
|
|
|
468
434
|
await makeDirectories(PKG_DIR)
|
|
469
435
|
|
|
470
|
-
|
|
471
|
-
* Checked-in WOF-derived admin pairs — the default source, and the whole source list for the small overlays.
|
|
472
|
-
*/
|
|
473
|
-
const WOF_ADMIN_DB = String(dataRootPath("wof", "admin-global-priority.db"))
|
|
436
|
+
const WOF_ADMIN_DB = wofDatabasePath("admin-global-priority.db")
|
|
474
437
|
const sources = overlay.sources ?? [WOF_ADMIN_DB]
|
|
475
438
|
const inputs = overlay.inputs ?? sources
|
|
476
439
|
const extraArgs = overlay.extraArgs ?? ["--borough-db", WOF_ADMIN_DB]
|
|
@@ -532,11 +495,8 @@ export async function buildPairIndexOverlay(overlay: PairIndexOverlay): Promise<
|
|
|
532
495
|
console.log(`built ${ARTIFACT}`)
|
|
533
496
|
}
|
|
534
497
|
|
|
535
|
-
// The manifest form records the overlay's source and artifact identity.
|
|
536
|
-
|
|
537
498
|
/**
|
|
538
|
-
* A soft-feed
|
|
539
|
-
* printed when the source is missing (the link is warn-and-continue; the channel resolves OFF).
|
|
499
|
+
* A soft-feed artifact that an overlay links, with the warning to print when its source is missing.
|
|
540
500
|
*/
|
|
541
501
|
export interface SoftFeedLink {
|
|
542
502
|
source: string
|
|
@@ -545,22 +505,22 @@ export interface SoftFeedLink {
|
|
|
545
505
|
}
|
|
546
506
|
|
|
547
507
|
/**
|
|
548
|
-
*
|
|
549
|
-
*
|
|
550
|
-
*
|
|
508
|
+
* Returns the committed soft-feed lexicons as links.
|
|
509
|
+
*
|
|
510
|
+
* The filenames come from `release.config.json` so that dev links match the release.
|
|
551
511
|
*/
|
|
552
512
|
export async function committedSoftFeedLinks(): Promise<{
|
|
553
513
|
anchor: SoftFeedLink
|
|
554
514
|
country: SoftFeedLink
|
|
555
|
-
streetType
|
|
515
|
+
streetType: SoftFeedLink | null
|
|
556
516
|
}> {
|
|
557
517
|
const config = await readReleaseConfig()
|
|
558
518
|
const sources = repoCommittedSoftFeedSources(repoRootPath(), config.softFeed ?? {})
|
|
559
519
|
|
|
560
|
-
const link = (name: string, consequenceIfMissing: string): SoftFeedLink |
|
|
520
|
+
const link = (name: string, consequenceIfMissing: string): SoftFeedLink | null => {
|
|
561
521
|
const source = sources.get(name)
|
|
562
522
|
|
|
563
|
-
return source ? { source, name, consequenceIfMissing } :
|
|
523
|
+
return source ? { source, name, consequenceIfMissing } : null
|
|
564
524
|
}
|
|
565
525
|
|
|
566
526
|
const anchor = link("anchor-lexicon-v1.json", "gazetteer channel will resolve OFF in this worktree.")
|
|
@@ -574,12 +534,11 @@ export async function committedSoftFeedLinks(): Promise<{
|
|
|
574
534
|
|
|
575
535
|
const streetType = link("street-type-lexicon-v3.json", "the street_type channel will resolve OFF in this worktree.")
|
|
576
536
|
|
|
577
|
-
return { anchor, country,
|
|
537
|
+
return { anchor, country, streetType }
|
|
578
538
|
}
|
|
579
539
|
|
|
580
540
|
/**
|
|
581
|
-
* The
|
|
582
|
-
* release re-verifies against the published tarball, and the channel requirements that name evidence lexicons.
|
|
541
|
+
* The fields of a weights package's `model-card.json` that dev materialization reads.
|
|
583
542
|
*/
|
|
584
543
|
export interface WeightsCard {
|
|
585
544
|
files_md5?: Record<string, string>
|
|
@@ -587,134 +546,151 @@ export interface WeightsCard {
|
|
|
587
546
|
}
|
|
588
547
|
|
|
589
548
|
/**
|
|
590
|
-
*
|
|
591
|
-
*
|
|
592
|
-
* soft-feed siblings, card-named evidence lexicons, postcode binary, pair index, FSTs.
|
|
549
|
+
* Declares what one locale's dev overlay contains. {@link materializeDevOverlay}
|
|
550
|
+
* runs the steps in the order the fields are listed.
|
|
593
551
|
*/
|
|
594
552
|
export interface DevOverlayManifest {
|
|
595
553
|
/**
|
|
596
|
-
* The overlay's locale tag
|
|
597
|
-
*
|
|
598
|
-
*
|
|
599
|
-
*
|
|
554
|
+
* The overlay's lowercase locale tag.
|
|
555
|
+
* The workspace is `neural-weights-<locale>`.
|
|
556
|
+
*
|
|
557
|
+
* Artifacts go to the data root's `weights/<locale>/` overlay because the binaries are kept out of git.
|
|
600
558
|
*/
|
|
601
559
|
locale: string
|
|
560
|
+
|
|
602
561
|
/**
|
|
603
|
-
*
|
|
604
|
-
*
|
|
605
|
-
*
|
|
606
|
-
*
|
|
607
|
-
*
|
|
608
|
-
*
|
|
609
|
-
*
|
|
562
|
+
* How the overlay gets its model files.
|
|
563
|
+
*
|
|
564
|
+
* When it is omitted, `link-weights-overlay.ts` in release-kit handles them.
|
|
565
|
+
*
|
|
566
|
+
* The `link` kind symlinks `model.onnx` and `tokenizer.model` from the
|
|
567
|
+
* `release.config.json` weights paths under the data root.
|
|
568
|
+
* The `$MAILWOMAN_DEV_MODEL` and `$MAILWOMAN_DEV_TOKENIZER` variables override those paths.
|
|
569
|
+
*
|
|
570
|
+
* With `digestCard`, the linked default files must match the `files_md5` of that workspace's card.
|
|
571
|
+
* An override skips this check.
|
|
572
|
+
*
|
|
573
|
+
* The `inherit` kind removes any local pair, because the package declares
|
|
574
|
+
* `mailwoman.baseWeights` and a local file would shadow the base.
|
|
575
|
+
*
|
|
576
|
+
* The `char` kind links the `charWeights.<family>` model, character vocabulary and model card.
|
|
577
|
+
* It removes any tokenizer.
|
|
610
578
|
*/
|
|
611
579
|
model?: { kind: "link"; digestCard?: string } | { kind: "inherit" } | { kind: "char"; family: string }
|
|
580
|
+
|
|
612
581
|
/**
|
|
613
|
-
* Soft-feed siblings
|
|
582
|
+
* Soft-feed siblings to link in order.
|
|
583
|
+
* A missing source prints a warning and the step continues.
|
|
614
584
|
*/
|
|
615
585
|
softFeed?: ReadonlyArray<SoftFeedLink>
|
|
586
|
+
|
|
616
587
|
/**
|
|
617
|
-
*
|
|
618
|
-
*
|
|
619
|
-
*
|
|
588
|
+
* Whether to link the evidence lexicons listed under `requires.<channel>.lexicon`
|
|
589
|
+
* in the overlay's model card.
|
|
590
|
+
* A card update then changes the linked file too.
|
|
591
|
+
*
|
|
592
|
+
* The street-type lexicon comes from the repo's `data/gazetteer/`.
|
|
593
|
+
* The locality-surface lexicon comes from the data root.
|
|
620
594
|
*/
|
|
621
595
|
evidenceLexiconsFromCard?: boolean
|
|
596
|
+
|
|
622
597
|
/**
|
|
623
|
-
*
|
|
624
|
-
*
|
|
598
|
+
* Builds `postcode-<country>.bin` from a WOF postcode extract with the compiled CLI.
|
|
599
|
+
* The step is skipped when the file already exists.
|
|
625
600
|
*/
|
|
626
|
-
postcodeBinary?: { country: string; database:
|
|
601
|
+
postcodeBinary?: { country: string; database: PathBuilder }
|
|
602
|
+
|
|
627
603
|
/**
|
|
628
|
-
* The placetype-pair index build
|
|
604
|
+
* The placetype-pair index build.
|
|
605
|
+
* Its `packageDir` is derived from `locale`.
|
|
629
606
|
*/
|
|
630
607
|
pairIndex?: Omit<PairIndexOverlay, "packageDir">
|
|
608
|
+
|
|
631
609
|
/**
|
|
632
|
-
*
|
|
610
|
+
* Whether to link `fst-<locale>.bin` and warn when it is stale.
|
|
633
611
|
*/
|
|
634
612
|
localeFST?: boolean
|
|
613
|
+
|
|
635
614
|
/**
|
|
636
|
-
*
|
|
615
|
+
* Whether to link the shared street-morphology FST.
|
|
637
616
|
*/
|
|
638
617
|
streetMorphologyFST?: boolean
|
|
639
618
|
}
|
|
640
619
|
|
|
641
620
|
/**
|
|
642
|
-
*
|
|
643
|
-
*
|
|
621
|
+
* The overlay directory, CLI path and model card that {@link materializeDevOverlay} returns.
|
|
622
|
+
*
|
|
623
|
+
* Callers use them for locale-specific steps that a manifest cannot express.
|
|
644
624
|
*/
|
|
645
625
|
export interface DevOverlay {
|
|
646
|
-
destDir:
|
|
647
|
-
cli:
|
|
648
|
-
card: WeightsCard |
|
|
626
|
+
destDir: PathBuilder
|
|
627
|
+
cli: PathBuilder
|
|
628
|
+
card: WeightsCard | null
|
|
649
629
|
}
|
|
650
630
|
|
|
651
|
-
/**
|
|
652
|
-
* Where each evidence channel's lexicon is found, by the name the card declares.
|
|
653
|
-
*/
|
|
654
631
|
const EVIDENCE_LEXICON_SOURCES: ReadonlyArray<{
|
|
655
632
|
channel: "street_type" | "locality_surface"
|
|
656
|
-
source: (name: string) =>
|
|
633
|
+
source: (name: string) => PathBuilder
|
|
657
634
|
}> = [
|
|
658
|
-
{ channel: "street_type", source: (name) =>
|
|
659
|
-
{ channel: "locality_surface", source: (name) =>
|
|
635
|
+
{ channel: "street_type", source: (name) => repoRootPathBuilder("data", "gazetteer", name) },
|
|
636
|
+
{ channel: "locality_surface", source: (name) => dataRootPath("gazetteer", name) },
|
|
660
637
|
]
|
|
661
638
|
|
|
662
639
|
/**
|
|
663
|
-
*
|
|
640
|
+
* Whether the locale's workspace package declares `mailwoman.baseWeights`.
|
|
664
641
|
*/
|
|
665
|
-
async function
|
|
666
|
-
const path =
|
|
642
|
+
async function declaresBaseWeights(locale: string): Promise<boolean> {
|
|
643
|
+
const path = workspacePathBuilder(`neural-weights-${locale}`, "package.json")
|
|
644
|
+
|
|
645
|
+
if (!(await pathExists(path))) return false
|
|
646
|
+
|
|
647
|
+
const { mailwoman } = await readLocalJSONFile<{ mailwoman?: { baseWeights?: unknown } }>(path)
|
|
648
|
+
|
|
649
|
+
return typeof mailwoman?.baseWeights === "string" && mailwoman.baseWeights.length > 0
|
|
650
|
+
}
|
|
667
651
|
|
|
668
|
-
|
|
652
|
+
async function readWeightsCard(workspace: string): Promise<WeightsCard | null> {
|
|
653
|
+
const path = workspacePathBuilder(workspace, "model-card.json")
|
|
654
|
+
|
|
655
|
+
if (!(await pathExists(path))) return null
|
|
669
656
|
|
|
670
657
|
return readLocalJSONFile<WeightsCard>(path)
|
|
671
658
|
}
|
|
672
659
|
|
|
673
|
-
|
|
674
|
-
* Link the base model pair from the release recipe and, when a digest card is named, hold the linked default bytes to
|
|
675
|
-
* that card's `files_md5`. The recipe and the card are the two registers a ship bumps in lockstep; a path bumped
|
|
676
|
-
* without the card, or the reverse, fails here rather than after an eval shift graded against the wrong weights.
|
|
677
|
-
*/
|
|
678
|
-
/**
|
|
679
|
-
* A char-path base (#2164): the graph and its sealed vocabulary from `release.config.json`'s `charWeights[family]`, no
|
|
680
|
-
* tokenizer. The dev overlay carries both so `resolveWeights` finds the package whole.
|
|
681
|
-
*/
|
|
682
|
-
async function linkCharModel(destDir: string, family: string): Promise<void> {
|
|
660
|
+
async function linkCharModel(destDir: PathBuilder, family: string): Promise<void> {
|
|
683
661
|
const recipe = (await readReleaseConfig()).charWeights?.[family]
|
|
684
662
|
|
|
685
663
|
if (!recipe) {
|
|
686
664
|
throw new Error(`release.config.json declares no charWeights.${family} — nothing to link for the ${family} base`)
|
|
687
665
|
}
|
|
688
666
|
|
|
689
|
-
const dataRoot =
|
|
667
|
+
const dataRoot = dataRootPath()
|
|
690
668
|
|
|
691
669
|
for (const [name, relative] of [
|
|
692
670
|
["model.onnx", recipe.model],
|
|
693
671
|
["char-vocab.json", recipe.charVocab],
|
|
694
672
|
] as const) {
|
|
695
|
-
const source =
|
|
673
|
+
const source = dataRoot(relative)
|
|
696
674
|
|
|
697
675
|
if (!(await pathExists(source))) {
|
|
698
676
|
throw new Error(`missing char-path source ${name} for ${family}: ${source}`)
|
|
699
677
|
}
|
|
700
678
|
|
|
701
|
-
await linkForce(source,
|
|
679
|
+
await linkForce(source, destDir(name))
|
|
702
680
|
|
|
703
|
-
console.log(`linked ${
|
|
681
|
+
console.log(`linked ${destDir(name)} ← ${source}`)
|
|
704
682
|
}
|
|
705
683
|
|
|
706
|
-
|
|
707
|
-
// block), so the overlay carries the workspace's committed card beside the two binaries.
|
|
708
|
-
const card = resolvePath(String(workspacePath(`neural-weights-${family}`)), "model-card.json")
|
|
684
|
+
const card = workspacePathBuilder(`neural-weights-${family}`, "model-card.json")
|
|
709
685
|
|
|
710
|
-
await linkForce(card,
|
|
711
|
-
await removeIfPresent(
|
|
686
|
+
await linkForce(card, destDir("model-card.json"))
|
|
687
|
+
await removeIfPresent(destDir("tokenizer.model"))
|
|
712
688
|
}
|
|
713
689
|
|
|
714
|
-
async function linkBaseModelPair(destDir:
|
|
690
|
+
async function linkBaseModelPair(destDir: PathBuilder, digestCard: string | null): Promise<void> {
|
|
715
691
|
const recipe = await readReleaseConfig()
|
|
716
692
|
|
|
717
|
-
const dataRoot =
|
|
693
|
+
const dataRoot = dataRootPath()
|
|
718
694
|
const digests = digestCard ? (await readWeightsCard(digestCard))?.files_md5 : undefined
|
|
719
695
|
|
|
720
696
|
const pair = [
|
|
@@ -728,13 +704,13 @@ async function linkBaseModelPair(destDir: string, digestCard: string | undefined
|
|
|
728
704
|
]
|
|
729
705
|
|
|
730
706
|
for (const { label, name, override, recipe: recipePath } of pair) {
|
|
731
|
-
const source = override ||
|
|
707
|
+
const source = override || dataRoot(recipePath)
|
|
732
708
|
|
|
733
709
|
if (!(await pathExists(source))) {
|
|
734
710
|
throw new Error(`missing source ${label}: ${source} — set MAILWOMAN_DEV_${label.toUpperCase()} to override`)
|
|
735
711
|
}
|
|
736
712
|
|
|
737
|
-
const dest =
|
|
713
|
+
const dest = destDir(name)
|
|
738
714
|
|
|
739
715
|
await linkForce(source, dest)
|
|
740
716
|
|
|
@@ -770,17 +746,13 @@ async function linkBaseModelPair(destDir: string, digestCard: string | undefined
|
|
|
770
746
|
}
|
|
771
747
|
}
|
|
772
748
|
|
|
773
|
-
/**
|
|
774
|
-
* Build a postcode binary from a WOF postcode extract, skip-if-present. A missing CLI or extract is a warning (the
|
|
775
|
-
* anchor channel resolves OFF for that country); a build that runs and fails is an error.
|
|
776
|
-
*/
|
|
777
749
|
async function buildPostcodeBinary(
|
|
778
|
-
destDir:
|
|
779
|
-
cli:
|
|
780
|
-
{ country, database }: { country: string; database:
|
|
750
|
+
destDir: PathBuilder,
|
|
751
|
+
cli: PathBuilder,
|
|
752
|
+
{ country, database }: { country: string; database: PathBuilder }
|
|
781
753
|
): Promise<void> {
|
|
782
754
|
const artifact = `postcode-${country.toLowerCase()}.bin`
|
|
783
|
-
const dest =
|
|
755
|
+
const dest = destDir(artifact)
|
|
784
756
|
|
|
785
757
|
if (await pathExists(dest)) {
|
|
786
758
|
console.log(`skipped ${artifact} build — ${dest} already present`)
|
|
@@ -806,7 +778,15 @@ async function buildPostcodeBinary(
|
|
|
806
778
|
|
|
807
779
|
const result = spawnProcessSync(
|
|
808
780
|
process.execPath,
|
|
809
|
-
[
|
|
781
|
+
[
|
|
782
|
+
cli,
|
|
783
|
+
"gazetteer",
|
|
784
|
+
"postcode-binary",
|
|
785
|
+
"--out",
|
|
786
|
+
destDir,
|
|
787
|
+
"--locale",
|
|
788
|
+
`${country.toUpperCase()}:${database.toString()}`,
|
|
789
|
+
],
|
|
810
790
|
{ stdio: "inherit" }
|
|
811
791
|
)
|
|
812
792
|
|
|
@@ -818,27 +798,31 @@ async function buildPostcodeBinary(
|
|
|
818
798
|
}
|
|
819
799
|
|
|
820
800
|
/**
|
|
821
|
-
*
|
|
822
|
-
* documents, each independent on disk, and the result carries what a locale-specific step needs afterwards.
|
|
801
|
+
* Materializes one locale's dev weights overlay from its manifest.
|
|
823
802
|
*/
|
|
824
803
|
export async function materializeDevOverlay(manifest: DevOverlayManifest): Promise<DevOverlay> {
|
|
825
|
-
const destDir =
|
|
826
|
-
const cli =
|
|
804
|
+
const destDir = weightsOverlayPath(manifest.locale)
|
|
805
|
+
const cli = workspacePathBuilder("mailwoman", "out", "cli", "main.js")
|
|
827
806
|
const card = await readWeightsCard(`neural-weights-${manifest.locale}`)
|
|
828
807
|
|
|
829
808
|
await makeDirectories(destDir)
|
|
830
809
|
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
810
|
+
// A package that declares `mailwoman.baseWeights` inherits its graph whatever the manifest says,
|
|
811
|
+
// because `resolveWeights` takes the base's graph over a file in the overlay directory.
|
|
812
|
+
const inherits =
|
|
813
|
+
manifest.model?.kind === "inherit" || (!manifest.model && (await declaresBaseWeights(manifest.locale)))
|
|
814
|
+
|
|
815
|
+
if (inherits) {
|
|
816
|
+
await removeIfPresent(destDir("model.onnx"))
|
|
817
|
+
await removeIfPresent(destDir("tokenizer.model"))
|
|
834
818
|
} else if (manifest.model?.kind === "link") {
|
|
835
|
-
await linkBaseModelPair(destDir, manifest.model.digestCard)
|
|
819
|
+
await linkBaseModelPair(destDir, manifest.model.digestCard ?? null)
|
|
836
820
|
} else if (manifest.model?.kind === "char") {
|
|
837
821
|
await linkCharModel(destDir, manifest.model.family)
|
|
838
822
|
}
|
|
839
823
|
|
|
840
824
|
for (const { source, name, consequenceIfMissing } of manifest.softFeed ?? []) {
|
|
841
|
-
await linkSoftFeedSibling(source,
|
|
825
|
+
await linkSoftFeedSibling(source, destDir(name), consequenceIfMissing)
|
|
842
826
|
}
|
|
843
827
|
|
|
844
828
|
if (manifest.evidenceLexiconsFromCard) {
|
|
@@ -853,7 +837,7 @@ export async function materializeDevOverlay(manifest: DevOverlayManifest): Promi
|
|
|
853
837
|
|
|
854
838
|
await linkSoftFeedSibling(
|
|
855
839
|
source(declared),
|
|
856
|
-
|
|
840
|
+
destDir(declared),
|
|
857
841
|
`the ${channel} channel will resolve OFF in this worktree.`
|
|
858
842
|
)
|
|
859
843
|
}
|