@mailwoman/corpus 9.0.0 → 9.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/data/PROVENANCE.md +227 -0
- package/data/reviewed-ve-postcode-tuples.json +74 -0
- package/data/sub-venue-lexicon.json +18269 -0
- package/out/src/adapters/ban/adapter.d.ts +1 -1
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +5 -4
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +3 -3
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +1 -1
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +2 -2
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +3 -3
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +2 -2
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +1 -1
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +3 -3
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +1 -1
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +2 -2
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +7 -6
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +12 -7
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +12 -18
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +12 -7
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +3 -3
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +3 -3
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +1 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +2 -2
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +7 -6
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +11 -7
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +6 -5
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +8 -8
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +10 -10
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +7 -6
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/{adapter.d.ts → adapters/utils/index.d.ts} +8 -2
- package/out/src/adapters/utils/index.d.ts.map +1 -0
- package/out/src/{adapter.js → adapters/utils/index.js} +11 -3
- package/out/src/adapters/utils/index.js.map +1 -0
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +5 -5
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +4 -15
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +11 -35
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-json-rows.d.ts +32 -0
- package/out/src/adapters/wof-json-rows.d.ts.map +1 -0
- package/out/src/adapters/wof-json-rows.js +45 -0
- package/out/src/adapters/wof-json-rows.js.map +1 -0
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +4 -9
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +12 -37
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/build.d.ts +4 -4
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +7 -7
- package/out/src/build.js.map +1 -1
- package/out/src/index.d.ts +3 -17
- package/out/src/index.d.ts.map +1 -1
- package/out/src/index.js +3 -17
- package/out/src/index.js.map +1 -1
- package/out/src/name-prone-us-suffixes.d.ts +12 -0
- package/out/src/name-prone-us-suffixes.d.ts.map +1 -0
- package/out/src/name-prone-us-suffixes.js +12 -0
- package/out/src/name-prone-us-suffixes.js.map +1 -0
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +5 -0
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/runner.d.ts +2 -2
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +1 -1
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +4 -5
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/bare-country.d.ts +25 -0
- package/out/src/shard-recipes/bare-country.d.ts.map +1 -0
- package/out/src/shard-recipes/bare-country.js +87 -0
- package/out/src/shard-recipes/bare-country.js.map +1 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +2 -2
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +14 -21
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +28 -10
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +4 -3
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.js +72 -11
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +4 -3
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +11 -20
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +8 -14
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/house-venue.js +1 -1
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +8 -1
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +7 -13
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +5 -5
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +13 -20
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/no-street.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street.js +1 -1
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +60 -43
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +1 -1
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/reviewed-postcode-tail.d.ts +47 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.d.ts.map +1 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.js +125 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.js.map +1 -0
- package/out/src/shard-recipes/scaffold.d.ts +51 -23
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +59 -23
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +54 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +203 -35
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
- package/out/src/shard-recipes/street-bare.js +3 -3
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.d.ts.map +1 -1
- package/out/src/shard-recipes/street.js +2 -2
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.d.ts +3 -6
- package/out/src/shard-recipes/sub-venue-sources.d.ts.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.js +4 -9
- package/out/src/shard-recipes/sub-venue-sources.js.map +1 -1
- package/out/src/shard-recipes/sub-venue.d.ts +2 -2
- package/out/src/shard-recipes/sub-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/sub-venue.js +7 -6
- package/out/src/shard-recipes/sub-venue.js.map +1 -1
- package/out/src/shard-recipes/trailing-region.d.ts +74 -0
- package/out/src/shard-recipes/trailing-region.d.ts.map +1 -0
- package/out/src/shard-recipes/trailing-region.js +157 -0
- package/out/src/shard-recipes/trailing-region.js.map +1 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +6 -12
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/{synthesize-anchor-absorption.d.ts → synthesizers/anchor-absorption.d.ts} +1 -1
- package/out/src/synthesizers/anchor-absorption.d.ts.map +1 -0
- package/out/src/{synthesize-anchor-absorption.js → synthesizers/anchor-absorption.js} +1 -1
- package/out/src/synthesizers/anchor-absorption.js.map +1 -0
- package/out/src/{synthesize-boundary-stress.d.ts → synthesizers/boundary-stress.d.ts} +3 -3
- package/out/src/synthesizers/boundary-stress.d.ts.map +1 -0
- package/out/src/{synthesize-boundary-stress.js → synthesizers/boundary-stress.js} +2 -2
- package/out/src/synthesizers/boundary-stress.js.map +1 -0
- package/out/src/{synthesize-german.d.ts → synthesizers/german.d.ts} +2 -2
- package/out/src/synthesizers/german.d.ts.map +1 -0
- package/out/src/{synthesize-german.js → synthesizers/german.js} +2 -2
- package/out/src/synthesizers/german.js.map +1 -0
- package/out/src/{synthesize-house-venue.d.ts → synthesizers/house-venue.d.ts} +4 -4
- package/out/src/synthesizers/house-venue.d.ts.map +1 -0
- package/out/src/{synthesize-house-venue.js → synthesizers/house-venue.js} +20 -6
- package/out/src/synthesizers/house-venue.js.map +1 -0
- package/out/src/{synthesize-intersection.d.ts → synthesizers/intersection.d.ts} +2 -2
- package/out/src/synthesizers/intersection.d.ts.map +1 -0
- package/out/src/{synthesize-intersection.js → synthesizers/intersection.js} +1 -1
- package/out/src/synthesizers/intersection.js.map +1 -0
- package/out/src/{synthesize-no-street.d.ts → synthesizers/no-street.d.ts} +3 -3
- package/out/src/synthesizers/no-street.d.ts.map +1 -0
- package/out/src/{synthesize-no-street.js → synthesizers/no-street.js} +2 -2
- package/out/src/synthesizers/no-street.js.map +1 -0
- package/out/src/{synthesize-po-box.d.ts → synthesizers/po-box.d.ts} +2 -2
- package/out/src/synthesizers/po-box.d.ts.map +1 -0
- package/out/src/{synthesize-po-box.js → synthesizers/po-box.js} +1 -1
- package/out/src/synthesizers/po-box.js.map +1 -0
- package/out/src/{synthesize-street.d.ts → synthesizers/street.d.ts} +2 -2
- package/out/src/synthesizers/street.d.ts.map +1 -0
- package/out/src/{synthesize-street.js → synthesizers/street.js} +4 -3
- package/out/src/synthesizers/street.js.map +1 -0
- package/out/src/{synthesize.d.ts → synthesizers/utils.d.ts} +3 -3
- package/out/src/synthesizers/utils.d.ts.map +1 -0
- package/out/src/{synthesize.js → synthesizers/utils.js} +5 -4
- package/out/src/synthesizers/utils.js.map +1 -0
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +2 -2
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/fetch/ban.d.ts.map +1 -1
- package/out/src/tools/fetch/ban.js +2 -18
- package/out/src/tools/fetch/ban.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts +12 -3
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +20 -4
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/geonames-dump.d.ts +87 -0
- package/out/src/tools/fetch/geonames-dump.d.ts.map +1 -0
- package/out/src/tools/fetch/geonames-dump.js +178 -0
- package/out/src/tools/fetch/geonames-dump.js.map +1 -0
- package/out/src/tools/fetch/geonames-postal.d.ts +76 -0
- package/out/src/tools/fetch/geonames-postal.d.ts.map +1 -0
- package/out/src/tools/fetch/geonames-postal.js +125 -0
- package/out/src/tools/fetch/geonames-postal.js.map +1 -0
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +5 -17
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +11 -0
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +11 -0
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts.map +1 -1
- package/out/src/tools/fetch/nad.js +20 -11
- package/out/src/tools/fetch/nad.js.map +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +17 -22
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/src/tools/fetch/openaddresses.js +18 -17
- package/out/src/tools/fetch/openaddresses.js.map +1 -1
- package/out/src/tools/fetch/ourairports.d.ts.map +1 -1
- package/out/src/tools/fetch/ourairports.js +7 -2
- package/out/src/tools/fetch/ourairports.js.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +0 -4
- package/out/src/tools/fetch/ppd.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.js +1 -1
- package/out/src/tools/fetch/ppd.js.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.js +3 -19
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
- package/out/src/tools/fetch/state-sources.d.ts +4 -0
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
- package/out/src/tools/fetch/state-sources.js +1 -5
- package/out/src/tools/fetch/state-sources.js.map +1 -1
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
- package/out/src/tools/fetch/tiger-full.js +15 -21
- package/out/src/tools/fetch/tiger-full.js.map +1 -1
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +34 -25
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-relabel-street.d.ts.map +1 -1
- package/out/src/tools/golden-relabel-street.js +4 -77
- package/out/src/tools/golden-relabel-street.js.map +1 -1
- package/out/src/tools/index.d.ts +2 -2
- package/out/src/tools/index.d.ts.map +1 -1
- package/out/src/tools/index.js +2 -2
- package/out/src/tools/index.js.map +1 -1
- package/out/src/tools/ingest-csv.d.ts.map +1 -1
- package/out/src/tools/ingest-csv.js +0 -11
- package/out/src/tools/ingest-csv.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts +4 -0
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +16 -6
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/postcode-triples.d.ts +175 -0
- package/out/src/tools/postcode-triples.d.ts.map +1 -0
- package/out/src/tools/postcode-triples.js +304 -0
- package/out/src/tools/postcode-triples.js.map +1 -0
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +1 -3
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +4 -27
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/tools/sub-venue/harvest.d.ts +100 -0
- package/out/src/tools/sub-venue/harvest.d.ts.map +1 -0
- package/out/src/tools/sub-venue/harvest.js +168 -0
- package/out/src/tools/sub-venue/harvest.js.map +1 -0
- package/out/src/tools/sub-venue/head-nouns.d.ts +50 -0
- package/out/src/tools/sub-venue/head-nouns.d.ts.map +1 -0
- package/out/src/tools/sub-venue/head-nouns.js +210 -0
- package/out/src/tools/sub-venue/head-nouns.js.map +1 -0
- package/out/src/tools/sub-venue/surfaces.d.ts +45 -0
- package/out/src/tools/sub-venue/surfaces.d.ts.map +1 -0
- package/out/src/tools/sub-venue/surfaces.js +77 -0
- package/out/src/tools/sub-venue/surfaces.js.map +1 -0
- package/out/src/tools/sub-venue/table.d.ts +229 -0
- package/out/src/tools/sub-venue/table.d.ts.map +1 -0
- package/out/src/tools/sub-venue/table.js +108 -0
- package/out/src/tools/sub-venue/table.js.map +1 -0
- package/out/src/tools/sub-venue/wikidata.d.ts +26 -0
- package/out/src/tools/sub-venue/wikidata.d.ts.map +1 -0
- package/out/src/tools/sub-venue/wikidata.js +62 -0
- package/out/src/tools/sub-venue/wikidata.js.map +1 -0
- package/out/src/tools/sub-venue-lexicon.d.ts +27 -378
- package/out/src/tools/sub-venue-lexicon.d.ts.map +1 -1
- package/out/src/tools/sub-venue-lexicon.js +30 -565
- package/out/src/tools/sub-venue-lexicon.js.map +1 -1
- package/out/src/{align.d.ts → utils/align.d.ts} +1 -1
- package/out/src/utils/align.d.ts.map +1 -0
- package/out/src/utils/align.js.map +1 -0
- package/out/src/utils/golden.d.ts.map +1 -0
- package/out/src/{golden.js → utils/golden.js} +2 -2
- package/out/src/utils/golden.js.map +1 -0
- package/out/src/utils/index.d.ts +14 -0
- package/out/src/utils/index.d.ts.map +1 -0
- package/out/src/utils/index.js +14 -0
- package/out/src/utils/index.js.map +1 -0
- package/out/src/utils/license.d.ts.map +1 -0
- package/out/src/{license.js → utils/license.js} +1 -1
- package/out/src/utils/license.js.map +1 -0
- package/out/src/{parquet.d.ts → utils/parquet.d.ts} +24 -5
- package/out/src/utils/parquet.d.ts.map +1 -0
- package/out/src/{parquet.js → utils/parquet.js} +6 -3
- package/out/src/utils/parquet.js.map +1 -0
- package/out/src/{split.d.ts → utils/split.d.ts} +1 -1
- package/out/src/utils/split.d.ts.map +1 -0
- package/out/src/{split.js → utils/split.js} +2 -2
- package/out/src/utils/split.js.map +1 -0
- package/out/src/utils/tokenize.d.ts.map +1 -0
- package/out/src/utils/tokenize.js.map +1 -0
- package/out/src/utils/wof-json.d.ts.map +1 -0
- package/out/src/utils/wof-json.js.map +1 -0
- package/package.json +287 -21
- package/src/adapters/ban/adapter.ts +7 -5
- package/src/adapters/fcc-bdc/adapter.ts +5 -4
- package/src/adapters/geonames/adapter.ts +3 -3
- package/src/adapters/geonames-postal/adapter.ts +4 -4
- package/src/adapters/gnaf/adapter.ts +3 -3
- package/src/adapters/gnaf/assemble.ts +1 -1
- package/src/adapters/index.ts +3 -2
- package/src/adapters/openaddresses/adapter.ts +4 -4
- package/src/adapters/overture/adapter.ts +3 -3
- package/src/adapters/state-hi-schools/adapter.ts +8 -7
- package/src/adapters/state-ia-contractors/adapter.ts +13 -8
- package/src/adapters/state-ny-notaries/adapter.ts +13 -32
- package/src/adapters/state-tx-notaries/adapter.ts +13 -8
- package/src/adapters/synth-po-box/adapter.ts +4 -4
- package/src/adapters/tiger/adapter.ts +5 -3
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +8 -7
- package/src/adapters/usgov-imls-pls/adapter.ts +12 -8
- package/src/adapters/usgov-irs-bmf/adapter.ts +7 -6
- package/src/adapters/usgov-nad/adapter.ts +9 -9
- package/src/adapters/usgov-nppes/adapter.ts +13 -12
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +8 -7
- package/src/{adapter.ts → adapters/utils/index.ts} +15 -3
- package/src/adapters/wof-admin-jp/adapter.ts +9 -12
- package/src/adapters/wof-admin-json/adapter.ts +15 -54
- package/src/adapters/wof-json-rows.ts +79 -0
- package/src/adapters/wof-postalcode-json/adapter.ts +16 -50
- package/src/build.ts +8 -8
- package/src/index.ts +3 -17
- package/src/name-prone-us-suffixes.ts +12 -0
- package/src/parquet-wrapper/reader.ts +9 -0
- package/src/runner.ts +2 -2
- package/src/shard-recipes/anchor-absorption.ts +5 -5
- package/src/shard-recipes/bare-country.ts +95 -0
- package/src/shard-recipes/boundary-stress.ts +3 -2
- package/src/shard-recipes/country-balanced.ts +16 -27
- package/src/shard-recipes/cz-pcfirst-preposition.ts +29 -10
- package/src/shard-recipes/fr-admin-split.ts +6 -4
- package/src/shard-recipes/fr-bare-street.ts +85 -10
- package/src/shard-recipes/fr-lieudit.ts +6 -4
- package/src/shard-recipes/fr-order.ts +13 -26
- package/src/shard-recipes/german.ts +9 -18
- package/src/shard-recipes/house-venue.ts +2 -1
- package/src/shard-recipes/index.ts +8 -1
- package/src/shard-recipes/intersection.ts +9 -17
- package/src/shard-recipes/locale.ts +16 -24
- package/src/shard-recipes/no-street.ts +2 -1
- package/src/shard-recipes/po-box-cedex.ts +76 -58
- package/src/shard-recipes/po-box.ts +2 -1
- package/src/shard-recipes/reviewed-postcode-tail.ts +189 -0
- package/src/shard-recipes/scaffold.ts +99 -24
- package/src/shard-recipes/street-affix.ts +284 -45
- package/src/shard-recipes/street-bare.ts +4 -3
- package/src/shard-recipes/street.ts +3 -2
- package/src/shard-recipes/sub-venue-sources.ts +13 -23
- package/src/shard-recipes/sub-venue.ts +11 -9
- package/src/shard-recipes/trailing-region.ts +183 -0
- package/src/shard-recipes/unit.ts +8 -17
- package/src/{synthesize-boundary-stress.ts → synthesizers/boundary-stress.ts} +2 -2
- package/src/{synthesize-german.ts → synthesizers/german.ts} +3 -2
- package/src/{synthesize-house-venue.ts → synthesizers/house-venue.ts} +20 -6
- package/src/{synthesize-intersection.ts → synthesizers/intersection.ts} +1 -1
- package/src/{synthesize-no-street.ts → synthesizers/no-street.ts} +2 -2
- package/src/{synthesize-po-box.ts → synthesizers/po-box.ts} +1 -1
- package/src/{synthesize-street.ts → synthesizers/street.ts} +5 -3
- package/src/{synthesize.ts → synthesizers/utils.ts} +6 -4
- package/src/tools/align-shard.ts +3 -2
- package/src/tools/fetch/ban.ts +2 -22
- package/src/tools/fetch/download.ts +22 -4
- package/src/tools/fetch/geonames-dump.ts +267 -0
- package/src/tools/fetch/geonames-postal.ts +180 -0
- package/src/tools/fetch/imls-pls.ts +6 -20
- package/src/tools/fetch/index.ts +11 -0
- package/src/tools/fetch/nad.ts +21 -10
- package/src/tools/fetch/nppes.ts +17 -23
- package/src/tools/fetch/openaddresses.ts +18 -22
- package/src/tools/fetch/ourairports.ts +7 -2
- package/src/tools/fetch/ppd.ts +1 -1
- package/src/tools/fetch/state-hi-schools.ts +3 -23
- package/src/tools/fetch/state-sources.ts +1 -1
- package/src/tools/fetch/tiger-full.ts +17 -24
- package/src/tools/golden-expand.ts +44 -33
- package/src/tools/golden-relabel-street.ts +5 -68
- package/src/tools/index.ts +2 -2
- package/src/tools/ingest-csv.ts +0 -13
- package/src/tools/overlay-manifest.ts +20 -8
- package/src/tools/postcode-triples.ts +375 -0
- package/src/tools/shard-kryptonite.ts +4 -5
- package/src/tools/shard-translit.ts +16 -35
- package/src/tools/sub-venue/harvest.ts +235 -0
- package/src/tools/sub-venue/head-nouns.ts +242 -0
- package/src/tools/sub-venue/surfaces.ts +95 -0
- package/src/tools/sub-venue/table.ts +281 -0
- package/src/tools/sub-venue/wikidata.ts +85 -0
- package/src/tools/sub-venue-lexicon.ts +67 -886
- package/src/{align.ts → utils/align.ts} +2 -1
- package/src/{golden.ts → utils/golden.ts} +2 -3
- package/src/utils/index.ts +14 -0
- package/src/{license.ts → utils/license.ts} +1 -1
- package/src/{parquet.ts → utils/parquet.ts} +22 -9
- package/src/{split.ts → utils/split.ts} +3 -3
- package/out/src/adapter.d.ts.map +0 -1
- package/out/src/adapter.js.map +0 -1
- package/out/src/align.d.ts.map +0 -1
- package/out/src/align.js.map +0 -1
- package/out/src/format.d.ts +0 -14
- package/out/src/format.d.ts.map +0 -1
- package/out/src/format.js +0 -14
- package/out/src/format.js.map +0 -1
- package/out/src/golden.d.ts.map +0 -1
- package/out/src/golden.js.map +0 -1
- package/out/src/license.d.ts.map +0 -1
- package/out/src/license.js.map +0 -1
- package/out/src/parquet.d.ts.map +0 -1
- package/out/src/parquet.js.map +0 -1
- package/out/src/split.d.ts.map +0 -1
- package/out/src/split.js.map +0 -1
- package/out/src/synthesize-anchor-absorption.d.ts.map +0 -1
- package/out/src/synthesize-anchor-absorption.js.map +0 -1
- package/out/src/synthesize-boundary-stress.d.ts.map +0 -1
- package/out/src/synthesize-boundary-stress.js.map +0 -1
- package/out/src/synthesize-german.d.ts.map +0 -1
- package/out/src/synthesize-german.js.map +0 -1
- package/out/src/synthesize-house-venue.d.ts.map +0 -1
- package/out/src/synthesize-house-venue.js.map +0 -1
- package/out/src/synthesize-intersection.d.ts.map +0 -1
- package/out/src/synthesize-intersection.js.map +0 -1
- package/out/src/synthesize-no-street.d.ts.map +0 -1
- package/out/src/synthesize-no-street.js.map +0 -1
- package/out/src/synthesize-po-box.d.ts.map +0 -1
- package/out/src/synthesize-po-box.js.map +0 -1
- package/out/src/synthesize-street.d.ts.map +0 -1
- package/out/src/synthesize-street.js.map +0 -1
- package/out/src/synthesize.d.ts.map +0 -1
- package/out/src/synthesize.js.map +0 -1
- package/out/src/tokenize.d.ts.map +0 -1
- package/out/src/tokenize.js.map +0 -1
- package/out/src/wof-json.d.ts.map +0 -1
- package/out/src/wof-json.js.map +0 -1
- package/src/format.ts +0 -14
- /package/out/src/{align.js → utils/align.js} +0 -0
- /package/out/src/{golden.d.ts → utils/golden.d.ts} +0 -0
- /package/out/src/{license.d.ts → utils/license.d.ts} +0 -0
- /package/out/src/{tokenize.d.ts → utils/tokenize.d.ts} +0 -0
- /package/out/src/{tokenize.js → utils/tokenize.js} +0 -0
- /package/out/src/{wof-json.d.ts → utils/wof-json.d.ts} +0 -0
- /package/out/src/{wof-json.js → utils/wof-json.js} +0 -0
- /package/src/{synthesize-anchor-absorption.ts → synthesizers/anchor-absorption.ts} +0 -0
- /package/src/{tokenize.ts → utils/tokenize.ts} +0 -0
- /package/src/{wof-json.ts → utils/wof-json.ts} +0 -0
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
* @file Deterministic canonical-row emission shared by WOF GeoJSON adapters.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type { ComponentTag } from "@mailwoman/core/types"
|
|
9
|
+
import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
|
|
10
|
+
|
|
11
|
+
import type { AdapterOptions, CanonicalRow } from "#types"
|
|
12
|
+
import type { WOFRecord } from "#utils"
|
|
13
|
+
|
|
14
|
+
export interface WOFVariantSpec {
|
|
15
|
+
suffix: string
|
|
16
|
+
components: Partial<Record<ComponentTag, string>>
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
interface EmitWOFJSONRowsOptions {
|
|
20
|
+
records: ReadonlyMap<number, WOFRecord>
|
|
21
|
+
ancestry: ReadonlyMap<number, WOFRecord[]>
|
|
22
|
+
adapterOptions: AdapterOptions
|
|
23
|
+
adapterID: string
|
|
24
|
+
localeByCountry: Readonly<Record<string, string>>
|
|
25
|
+
shouldEmit?: (record: WOFRecord) => boolean
|
|
26
|
+
nameSlotsFor: (record: WOFRecord) => Array<{ key: string; value: string }>
|
|
27
|
+
variantsFor: (record: WOFRecord, ancestry: WOFRecord[], selfName: string) => WOFVariantSpec[]
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Emit aligned rows in WOF-id order, enforcing the adapter limit across all name and hierarchy variants.
|
|
32
|
+
*/
|
|
33
|
+
export function* emitWOFJSONRows(options: EmitWOFJSONRowsOptions): Generator<CanonicalRow> {
|
|
34
|
+
const {
|
|
35
|
+
records,
|
|
36
|
+
ancestry,
|
|
37
|
+
adapterOptions,
|
|
38
|
+
adapterID,
|
|
39
|
+
localeByCountry,
|
|
40
|
+
shouldEmit = () => true,
|
|
41
|
+
nameSlotsFor,
|
|
42
|
+
variantsFor,
|
|
43
|
+
} = options
|
|
44
|
+
|
|
45
|
+
let emitted = 0
|
|
46
|
+
|
|
47
|
+
for (const id of [...records.keys()].toSorted((a, b) => a - b)) {
|
|
48
|
+
if (adapterOptions.signal?.aborted) return
|
|
49
|
+
const record = records.get(id)!
|
|
50
|
+
|
|
51
|
+
if (!shouldEmit(record)) continue
|
|
52
|
+
|
|
53
|
+
for (const slot of nameSlotsFor(record)) {
|
|
54
|
+
for (const variant of variantsFor(record, ancestry.get(id) ?? [], slot.value)) {
|
|
55
|
+
if (adapterOptions.limit !== undefined && emitted >= adapterOptions.limit) return
|
|
56
|
+
|
|
57
|
+
const raw = formatAddress(variant.components, record.country, { separator: ", " })
|
|
58
|
+
|
|
59
|
+
if (!raw) continue
|
|
60
|
+
const components = reconcileComponents(variant.components, raw)
|
|
61
|
+
|
|
62
|
+
if (!Object.keys(components).length) continue
|
|
63
|
+
|
|
64
|
+
yield {
|
|
65
|
+
raw,
|
|
66
|
+
components,
|
|
67
|
+
country: record.country,
|
|
68
|
+
locale: localeByCountry[record.country],
|
|
69
|
+
source: adapterID,
|
|
70
|
+
source_id: `${adapterID}-${record.id}-${slot.key}-${variant.suffix}`,
|
|
71
|
+
corpus_version: "",
|
|
72
|
+
license: "CC0-1.0",
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
emitted++
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
@@ -40,9 +40,10 @@
|
|
|
40
40
|
import type { WhosOnFirstPlacetype } from "@mailwoman/core/resources/whosonfirst"
|
|
41
41
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
42
42
|
|
|
43
|
-
import {
|
|
44
|
-
import
|
|
45
|
-
|
|
43
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
|
|
44
|
+
import { buildAncestryIndex, normalizeNameKey, walkFeatures, type WOFRecord } from "#utils"
|
|
45
|
+
|
|
46
|
+
import { emitWOFJSONRows, type WOFVariantSpec } from "../wof-json-rows.ts"
|
|
46
47
|
|
|
47
48
|
const COUNTRY_DISPLAY_NAME: Record<string, string> = {
|
|
48
49
|
US: "United States of America",
|
|
@@ -71,16 +72,11 @@ function placetypeToTag(placetype: WhosOnFirstPlacetype | string): ComponentTag
|
|
|
71
72
|
}
|
|
72
73
|
}
|
|
73
74
|
|
|
74
|
-
interface VariantSpec {
|
|
75
|
-
suffix: string
|
|
76
|
-
components: Partial<Record<ComponentTag, string>>
|
|
77
|
-
}
|
|
78
|
-
|
|
79
75
|
/**
|
|
80
76
|
* Compute hierarchy variants for a postcode record. `selfName` is the postcode surface form (canonical `wof:name` for
|
|
81
77
|
* the `default` slot, a `name:*` localized variant otherwise).
|
|
82
78
|
*/
|
|
83
|
-
export function postcodeVariantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: string):
|
|
79
|
+
export function postcodeVariantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: string): WOFVariantSpec[] {
|
|
84
80
|
if (placetypeToTag(row.placetype) !== "postcode") return []
|
|
85
81
|
|
|
86
82
|
const locality = ancestry.find((a) => placetypeToTag(a.placetype) === "locality")
|
|
@@ -88,7 +84,7 @@ export function postcodeVariantsFor(row: WOFRecord, ancestry: WOFRecord[], selfN
|
|
|
88
84
|
const country = ancestry.find((a) => placetypeToTag(a.placetype) === "country")
|
|
89
85
|
const countryDisplay = COUNTRY_DISPLAY_NAME[row.country] ?? country?.name ?? row.country
|
|
90
86
|
|
|
91
|
-
const variants:
|
|
87
|
+
const variants: WOFVariantSpec[] = [{ suffix: "self", components: { postcode: selfName } }]
|
|
92
88
|
|
|
93
89
|
if (locality) {
|
|
94
90
|
variants.push({
|
|
@@ -167,46 +163,16 @@ export function createWOFPostalcodeAdapter(): CorpusAdapter {
|
|
|
167
163
|
const ancestry = buildAncestryIndex(byID)
|
|
168
164
|
|
|
169
165
|
// Pass 2: emit postcode rows only, sorted by id for determinism.
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
const slots = nameSlotsFor(rec)
|
|
181
|
-
|
|
182
|
-
for (const slot of slots) {
|
|
183
|
-
const variants = postcodeVariantsFor(rec, chain, slot.value)
|
|
184
|
-
|
|
185
|
-
for (const variant of variants) {
|
|
186
|
-
if (opts.limit !== undefined && emitted >= opts.limit) return
|
|
187
|
-
|
|
188
|
-
const raw = formatAddress(variant.components, rec.country, { separator: ", " })
|
|
189
|
-
|
|
190
|
-
if (!raw) continue
|
|
191
|
-
const aligned = reconcileComponents(variant.components, raw)
|
|
192
|
-
|
|
193
|
-
if (!Object.keys(aligned).length) continue
|
|
194
|
-
|
|
195
|
-
yield {
|
|
196
|
-
raw,
|
|
197
|
-
components: aligned,
|
|
198
|
-
country: rec.country,
|
|
199
|
-
locale: LOCALE_BY_COUNTRY[rec.country],
|
|
200
|
-
source: WOF_POSTALCODE_ADAPTER_ID,
|
|
201
|
-
source_id: `${WOF_POSTALCODE_ADAPTER_ID}-${rec.id}-${slot.key}-${variant.suffix}`,
|
|
202
|
-
corpus_version: "",
|
|
203
|
-
license: "CC0-1.0",
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
emitted++
|
|
207
|
-
}
|
|
208
|
-
}
|
|
209
|
-
}
|
|
166
|
+
yield* emitWOFJSONRows({
|
|
167
|
+
records: byID,
|
|
168
|
+
ancestry,
|
|
169
|
+
adapterOptions: opts,
|
|
170
|
+
adapterID: WOF_POSTALCODE_ADAPTER_ID,
|
|
171
|
+
localeByCountry: LOCALE_BY_COUNTRY,
|
|
172
|
+
shouldEmit: (record) => placetypeToTag(record.placetype) === "postcode",
|
|
173
|
+
nameSlotsFor,
|
|
174
|
+
variantsFor: postcodeVariantsFor,
|
|
175
|
+
})
|
|
210
176
|
},
|
|
211
177
|
}
|
|
212
178
|
}
|
package/src/build.ts
CHANGED
|
@@ -55,20 +55,20 @@ import { $public } from "@mailwoman/core/env"
|
|
|
55
55
|
import { parseJSONStrict } from "@mailwoman/core/objects"
|
|
56
56
|
import { JSONSpliterator } from "spliterator"
|
|
57
57
|
|
|
58
|
-
import { defaultAdapterRegistry } from "
|
|
59
|
-
import {
|
|
60
|
-
import {
|
|
61
|
-
import {
|
|
62
|
-
import {
|
|
58
|
+
import { defaultAdapterRegistry } from "#adapters/utils"
|
|
59
|
+
import { runAdapter, type AdapterRunManifest } from "#runner"
|
|
60
|
+
import { defaultAugmentationsForCountry, synthesizeRow } from "#synthesizers/utils"
|
|
61
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter, LabeledRow } from "#types"
|
|
62
|
+
import { alignRow } from "#utils/align"
|
|
63
|
+
import { licenseExcluded } from "#utils/license"
|
|
64
|
+
import { writeShards, type ShardManifest } from "#utils/parquet"
|
|
63
65
|
import {
|
|
64
66
|
defaultHoldouts,
|
|
65
67
|
splitForRow,
|
|
66
68
|
writeSplitManifestsFromLabeledFiles,
|
|
67
69
|
type SplitManifest,
|
|
68
70
|
type SplitName,
|
|
69
|
-
} from "
|
|
70
|
-
import { defaultAugmentationsForCountry, synthesizeRow } from "./synthesize.ts"
|
|
71
|
-
import type { AdapterOptions, CanonicalRow, CorpusAdapter, LabeledRow } from "./types.ts"
|
|
71
|
+
} from "#utils/split"
|
|
72
72
|
|
|
73
73
|
/**
|
|
74
74
|
* Stage tags surfaced to `onProgress`.
|
package/src/index.ts
CHANGED
|
@@ -4,25 +4,11 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
export * from "./
|
|
7
|
+
export * from "./adapters/utils/index.ts"
|
|
8
8
|
export * from "./adapters/gnaf/assemble.ts"
|
|
9
9
|
export * from "./adapters/index.ts"
|
|
10
|
-
export * from "./align.ts"
|
|
11
10
|
export * from "./build.ts"
|
|
12
|
-
export * from "./format.ts"
|
|
13
|
-
export * from "./golden.ts"
|
|
14
|
-
export * from "./parquet.ts"
|
|
15
|
-
export * from "./parquet-wrapper/index.ts"
|
|
16
11
|
export * from "./runner.ts"
|
|
12
|
+
export * from "./parquet-wrapper/index.ts"
|
|
17
13
|
export * from "./shard-recipes/index.ts"
|
|
18
|
-
export * from "
|
|
19
|
-
export * from "./synthesize-anchor-absorption.ts"
|
|
20
|
-
export * from "./synthesize-german.ts"
|
|
21
|
-
export * from "./synthesize-house-venue.ts"
|
|
22
|
-
export * from "./synthesize-intersection.ts"
|
|
23
|
-
export * from "./synthesize-no-street.ts"
|
|
24
|
-
export * from "./synthesize-po-box.ts"
|
|
25
|
-
export * from "./synthesize-street.ts"
|
|
26
|
-
export * from "./synthesize.ts"
|
|
27
|
-
export * from "./tokenize.ts"
|
|
28
|
-
export * from "./types.ts"
|
|
14
|
+
export * from "#types"
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Re-export shim: the name-prone set moved to `@mailwoman/codex/us` (data in
|
|
7
|
+
* `codex/us/street-suffix.json`, 2026-08-10) so the golden relabel flags, the #1569 shard
|
|
8
|
+
* recipe, AND the Python relabel pass (via the `gazetteer affix-relabel` v2 lexicon) all read
|
|
9
|
+
* one record. Import from `@mailwoman/codex/us` in new code.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
export { NAME_PRONE_US_SUFFIXES } from "@mailwoman/codex/us"
|
|
@@ -67,6 +67,15 @@ export class ParquetReader<T extends ParquetRecordLike> extends BaseParquetReade
|
|
|
67
67
|
* caller wanting the whole record should use.
|
|
68
68
|
*/
|
|
69
69
|
public async *project<K extends keyof T>(...columns: K[]): AsyncGenerator<Pick<T, K>, void, unknown> {
|
|
70
|
+
const missing = columns.filter((column) => !Object.hasOwn(this.schema.schema, column))
|
|
71
|
+
|
|
72
|
+
if (missing.length) {
|
|
73
|
+
throw new Error(
|
|
74
|
+
`Parquet projection requested column${missing.length === 1 ? "" : "s"} absent from the file schema: ` +
|
|
75
|
+
missing.map(String).join(", ")
|
|
76
|
+
)
|
|
77
|
+
}
|
|
78
|
+
|
|
70
79
|
const cursor = this.getCursor(columns.map((column) => [column as string]))
|
|
71
80
|
|
|
72
81
|
for (;;) {
|
package/src/runner.ts
CHANGED
|
@@ -31,8 +31,8 @@ import { createWriteStream, type WriteStream } from "node:fs"
|
|
|
31
31
|
import { mkdir, writeFile } from "node:fs/promises"
|
|
32
32
|
import { dirname, join } from "node:path"
|
|
33
33
|
|
|
34
|
-
import { canonicalDedupKey, streamingSha256, type AdapterRegistry, type StreamingHasher } from "
|
|
35
|
-
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "
|
|
34
|
+
import { canonicalDedupKey, streamingSha256, type AdapterRegistry, type StreamingHasher } from "#adapters/utils"
|
|
35
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
|
|
36
36
|
|
|
37
37
|
/**
|
|
38
38
|
* Snapshot of the runner's state, emitted on every progress tick.
|
|
@@ -17,10 +17,10 @@ import { readFileSync } from "node:fs"
|
|
|
17
17
|
import { parseJSONStrict } from "@mailwoman/core/objects"
|
|
18
18
|
import { dataRootPath } from "@mailwoman/core/utils"
|
|
19
19
|
|
|
20
|
-
import {
|
|
21
|
-
import { alignRow } from "
|
|
22
|
-
|
|
23
|
-
import { makeLcg, type ShardRecipe } from "./scaffold.ts"
|
|
20
|
+
import { synthesizeAnchorAbsorptionRow } from "#synthesizers/anchor-absorption"
|
|
21
|
+
import { alignRow } from "#utils"
|
|
22
|
+
|
|
23
|
+
import { makeLcg, shardSourceID, type ShardRecipe } from "./scaffold.ts"
|
|
24
24
|
|
|
25
25
|
/**
|
|
26
26
|
* The leading-5-digit source: the real US ZIPs in the postcode anchor's pilot lookup. Resolved through the data-root
|
|
@@ -76,7 +76,7 @@ export const anchorAbsorptionRecipe: ShardRecipe = {
|
|
|
76
76
|
country,
|
|
77
77
|
locale: synth.locale,
|
|
78
78
|
source,
|
|
79
|
-
source_id:
|
|
79
|
+
source_id: shardSourceID(source, { v: String(i) }),
|
|
80
80
|
}
|
|
81
81
|
|
|
82
82
|
const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `bare-country` — the country name as the WHOLE query. Addresses almost never consist of one bare
|
|
7
|
+
* country name, so the class is out of the model's training distribution and the tag is a coin
|
|
8
|
+
* flip: measured on the 2026-08-13 panel, France/Germany/United States/New Zealand parsed
|
|
9
|
+
* `country` while Japan/China/Nigeria/Australia/Deutschland parsed `locality`. The retrieval-side
|
|
10
|
+
* bare-country race (#1651) covers the answer either way; this shard closes the PARSE half so the
|
|
11
|
+
* tag itself is right.
|
|
12
|
+
*
|
|
13
|
+
* Source is the codex country table — every ISO canonical English name plus the curated surface
|
|
14
|
+
* forms (endonyms and long-form names). The 2–3 letter CODES are deliberately excluded: a bare
|
|
15
|
+
* `JP` or `GER` is an abbreviation register with its own ambiguity surface (`IN`, `DE`, `TO` are
|
|
16
|
+
* English words), and teaching it here would be an unmeasured claim. Case augmentation is the
|
|
17
|
+
* loader's job, as everywhere.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { COUNTRY_SURFACE_FORMS, CountryNames, matchCountry } from "@mailwoman/codex/country"
|
|
21
|
+
|
|
22
|
+
import { alignAndWrite, makeMulberry32, type ShardRecipe, shardSourceID } from "./scaffold.ts"
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Surfaces shorter than this are the code register (`JP`, `GER`), not a name — excluded (see the module doc).
|
|
26
|
+
*/
|
|
27
|
+
const MIN_NAME_LENGTH = 4
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Every (surface, iso2) pair the shard emits: the ISO canonical names plus each country's curated surface forms,
|
|
31
|
+
* deduplicated on the surface string (a form shared across countries — none known — would keep its first bearer).
|
|
32
|
+
*/
|
|
33
|
+
function* bareCountrySurfaces(): Generator<{ surface: string; iso2: string }> {
|
|
34
|
+
const seen = new Set<string>()
|
|
35
|
+
|
|
36
|
+
for (const [iso2, forms] of Object.entries(COUNTRY_SURFACE_FORMS)) {
|
|
37
|
+
for (const surface of forms) {
|
|
38
|
+
if (surface.length < MIN_NAME_LENGTH || seen.has(surface)) continue
|
|
39
|
+
seen.add(surface)
|
|
40
|
+
yield { surface, iso2 }
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
for (const name of CountryNames) {
|
|
45
|
+
if (name.length < MIN_NAME_LENGTH || seen.has(name)) continue
|
|
46
|
+
const matched = matchCountry(name)
|
|
47
|
+
|
|
48
|
+
if (!matched) continue
|
|
49
|
+
seen.add(name)
|
|
50
|
+
yield { surface: name, iso2: matched.iso2 }
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
56
|
+
* and `description` below for the surface form it generates.
|
|
57
|
+
*/
|
|
58
|
+
export const bareCountryRecipe: ShardRecipe = {
|
|
59
|
+
name: "bare-country",
|
|
60
|
+
description: "The country name as the whole query (#1651 parse half): ISO names + curated endonyms, no codes",
|
|
61
|
+
mode: "generate",
|
|
62
|
+
async run(opts, write) {
|
|
63
|
+
// Seeded for parity with the other recipes; the surfaces themselves drive the content.
|
|
64
|
+
makeMulberry32(opts.seed)
|
|
65
|
+
let read = 0
|
|
66
|
+
let emitted = 0
|
|
67
|
+
let skipped = 0
|
|
68
|
+
|
|
69
|
+
for (const { surface, iso2 } of bareCountrySurfaces()) {
|
|
70
|
+
read++
|
|
71
|
+
|
|
72
|
+
const components: Record<string, string> = { country: surface }
|
|
73
|
+
const source_id = shardSourceID("synth-bare-country", { country: surface, cc: iso2, v: String(read) })
|
|
74
|
+
|
|
75
|
+
const canonical = {
|
|
76
|
+
raw: surface,
|
|
77
|
+
components,
|
|
78
|
+
country: iso2,
|
|
79
|
+
locale: "und",
|
|
80
|
+
source: "synth-bare-country",
|
|
81
|
+
source_id,
|
|
82
|
+
corpus_version: "0.11.0",
|
|
83
|
+
license: "Synthetic — bare-country; surfaces from the codex country table (ISO 3166 + curated forms)",
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
if (alignAndWrite(write, canonical, "bare-country")) {
|
|
87
|
+
emitted++
|
|
88
|
+
} else {
|
|
89
|
+
skipped++
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
return { read, emitted, skipped }
|
|
94
|
+
},
|
|
95
|
+
}
|
|
@@ -12,8 +12,9 @@
|
|
|
12
12
|
* `synthesizeBoundaryStressRow` is NOT re-exported from the corpus index — imported directly here.
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
15
|
+
import { type BoundaryStressTemplate, synthesizeBoundaryStressRow } from "#synthesizers/boundary-stress"
|
|
16
|
+
import { alignRow } from "#utils"
|
|
17
|
+
|
|
17
18
|
import { makeMulberry32, type ShardRecipe, shardSourceID } from "./scaffold.ts"
|
|
18
19
|
|
|
19
20
|
/**
|
|
@@ -24,16 +24,16 @@
|
|
|
24
24
|
* legacy script used.
|
|
25
25
|
*/
|
|
26
26
|
|
|
27
|
-
import { spawnSync } from "node:child_process"
|
|
28
|
-
|
|
29
27
|
import { COUNTRY_SURFACE_FORMS, CountryNames } from "@mailwoman/codex/country"
|
|
28
|
+
import { isPresent } from "@mailwoman/core/objects"
|
|
30
29
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
31
30
|
import { dataRootPath } from "@mailwoman/core/utils"
|
|
32
31
|
|
|
33
|
-
import { stableSourceID } from "
|
|
34
|
-
import {
|
|
35
|
-
import
|
|
36
|
-
|
|
32
|
+
import { stableSourceID } from "#adapters/utils"
|
|
33
|
+
import type { CanonicalRow } from "#types"
|
|
34
|
+
import { alignRow } from "#utils"
|
|
35
|
+
|
|
36
|
+
import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
|
|
37
37
|
|
|
38
38
|
// v2: the country TOKEN is decoupled from the skeleton's locale and drawn from a BROAD pool — every
|
|
39
39
|
// ISO canonical name + every curated surface form (endonyms/abbrevs). Surface forms are over-weighted
|
|
@@ -154,26 +154,15 @@ interface CountryTuple {
|
|
|
154
154
|
order: string
|
|
155
155
|
}
|
|
156
156
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
const r = spawnSync("bash", ["-c", `unzip -p "${source.zip}" "${source.csv}" | head -n ${maxLines}`], {
|
|
163
|
-
maxBuffer: 1024 * 1024 * 1024,
|
|
164
|
-
encoding: "buffer",
|
|
165
|
-
})
|
|
166
|
-
|
|
167
|
-
if (r.status !== 0) {
|
|
168
|
-
console.error(` WARN: unzip failed for ${source.zip} (status ${r.status})`)
|
|
169
|
-
|
|
170
|
-
return []
|
|
171
|
-
}
|
|
172
|
-
|
|
157
|
+
/**
|
|
158
|
+
* The countrywide extracts (FR/IT/NL) are GB-scale, so this reads only as far as `limit` distinct tuples: the `break`
|
|
159
|
+
* closes the reader, which releases the archive without inflating the rest of the member.
|
|
160
|
+
*/
|
|
161
|
+
async function readTuples(source: CountrySource, limit: number): Promise<CountryTuple[]> {
|
|
173
162
|
const tuples: CountryTuple[] = []
|
|
174
163
|
const seen = new Set<string>()
|
|
175
164
|
|
|
176
|
-
for (const row of
|
|
165
|
+
for await (const row of readZippedCSVRecords(source.zip, source.csv)) {
|
|
177
166
|
if (tuples.length >= limit) break
|
|
178
167
|
|
|
179
168
|
const street = row.street ?? "",
|
|
@@ -233,14 +222,14 @@ function renderCountry(
|
|
|
233
222
|
let body: string
|
|
234
223
|
|
|
235
224
|
if (order === "us") {
|
|
236
|
-
const regPc = [reg, pc].filter(
|
|
225
|
+
const regPc = [reg, pc].filter(isPresent).join(" ")
|
|
237
226
|
body = `${hn} ${street}, ${loc}${regPc ? ", " + regPc : ""}`
|
|
238
227
|
} else if (order === "fr") {
|
|
239
228
|
// French is NUMBER-street, postcode-city: "84 Route de la Fontaine, 75008 Paris".
|
|
240
|
-
body = `${hn} ${street}, ${[pc, loc].filter(
|
|
229
|
+
body = `${hn} ${street}, ${[pc, loc].filter(isPresent).join(" ")}`
|
|
241
230
|
} else {
|
|
242
231
|
// DE/IT/NL: street-number, postcode-city: "Pariser Platz 1, 10117 Berlin".
|
|
243
|
-
const pcCity = [pc, loc].filter(
|
|
232
|
+
const pcCity = [pc, loc].filter(isPresent).join(" ")
|
|
244
233
|
body = `${street} ${hn}, ${pcCity}`
|
|
245
234
|
}
|
|
246
235
|
|
|
@@ -449,7 +438,7 @@ export const countryBalancedRecipe: ShardRecipe = {
|
|
|
449
438
|
const pool: CountryTuple[] = []
|
|
450
439
|
|
|
451
440
|
for (const s of sources) {
|
|
452
|
-
const t = readTuples(s, perSource)
|
|
441
|
+
const t = await readTuples(s, perSource)
|
|
453
442
|
|
|
454
443
|
console.error(` ${s.csv} (${s.iso2}): ${t.length} tuples`)
|
|
455
444
|
|
|
@@ -16,6 +16,13 @@
|
|
|
16
16
|
|
|
17
17
|
import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
|
|
18
18
|
|
|
19
|
+
/**
|
|
20
|
+
* The order-cycle slot for the STREET-LESS form (`«city» «pc», Česko`) — the exact surface of the
|
|
21
|
+
* `cz-full-praha-100-00` board row, whose absence from the street-bearing orders was the v4.5.0 no-promote's measured
|
|
22
|
+
* gap.
|
|
23
|
+
*/
|
|
24
|
+
const STREETLESS_ORDER = 3
|
|
25
|
+
|
|
19
26
|
/**
|
|
20
27
|
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
21
28
|
* and `description` below for the surface form it generates.
|
|
@@ -44,27 +51,39 @@ export const czPcFirstPrepositionRecipe: ShardRecipe = {
|
|
|
44
51
|
continue
|
|
45
52
|
}
|
|
46
53
|
|
|
47
|
-
const order = read %
|
|
54
|
+
const order = read % 4
|
|
55
|
+
// The OFFICIAL Czech rendering spaces the PSČ as `NNN NN` ('512 44'); OpenAddresses stores it
|
|
56
|
+
// unspaced ('51244'), and a model trained only on the source form reads the spaced surface as
|
|
57
|
+
// house_number + garbage (the 'Praha 100 00' mangle). Alternate the two renderings so both
|
|
58
|
+
// orthographies are attested — the LABEL is the postcode either way.
|
|
59
|
+
const spaced = read % 2 === 0 && /^\d{5}$/.test(postcode)
|
|
60
|
+
const postcodeSurface = spaced ? `${postcode.slice(0, 3)} ${postcode.slice(3)}` : postcode
|
|
48
61
|
let raw: string
|
|
49
62
|
|
|
50
|
-
const components: Record<string, string> =
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
locality: city,
|
|
55
|
-
}
|
|
63
|
+
const components: Record<string, string> =
|
|
64
|
+
order === STREETLESS_ORDER
|
|
65
|
+
? { locality: city, postcode: postcodeSurface, country: "Česko" }
|
|
66
|
+
: { street, house_number: number, postcode: postcodeSurface, locality: city }
|
|
56
67
|
|
|
57
68
|
if (order === 0) {
|
|
58
|
-
raw = `${
|
|
69
|
+
raw = `${postcodeSurface} ${city}, ${street} ${number}`
|
|
59
70
|
} else if (order === 1) {
|
|
60
|
-
raw = `${street} ${number}, ${
|
|
71
|
+
raw = `${street} ${number}, ${postcodeSurface} ${city}`
|
|
72
|
+
} else if (order === 2) {
|
|
73
|
+
raw = `${city}, ${postcodeSurface}, ${street} ${number}`
|
|
61
74
|
} else {
|
|
62
|
-
|
|
75
|
+
// order === STREETLESS_ORDER: the street-less form — `«city» «pc», Česko` — the exact surface of the
|
|
76
|
+
// cz-full-praha-100-00 board row. The v4.5.0 no-promote receipt measured the gap: every
|
|
77
|
+
// prior order was street-bearing, so the model never saw a spaced PSČ beside a bare
|
|
78
|
+
// locality and mangled 'Praha 100 00, Czechia' into house_number spans. Street/number
|
|
79
|
+
// stay OUT of the components for this form (they are not in the surface).
|
|
80
|
+
raw = `${city} ${postcodeSurface}, Česko`
|
|
63
81
|
}
|
|
64
82
|
|
|
65
83
|
const source_id = shardSourceID("synth-cz-pcfirst-preposition", {
|
|
66
84
|
...components,
|
|
67
85
|
o: String(order),
|
|
86
|
+
s: spaced ? "1" : "0",
|
|
68
87
|
v: String(read),
|
|
69
88
|
})
|
|
70
89
|
|
|
@@ -32,11 +32,13 @@
|
|
|
32
32
|
|
|
33
33
|
import { departementForCodePostal } from "@mailwoman/codex/fr"
|
|
34
34
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
35
|
+
import { tempRootPath } from "@mailwoman/core/utils"
|
|
35
36
|
import { CSVSpliterator, Delimiters } from "spliterator"
|
|
36
37
|
|
|
37
|
-
import { stableSourceID } from "
|
|
38
|
-
import {
|
|
39
|
-
import
|
|
38
|
+
import { stableSourceID } from "#adapters/utils"
|
|
39
|
+
import type { CanonicalRow } from "#types"
|
|
40
|
+
import { alignRow } from "#utils"
|
|
41
|
+
|
|
40
42
|
import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
|
|
41
43
|
|
|
42
44
|
/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
|
|
@@ -45,7 +47,7 @@ import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
|
|
|
45
47
|
would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
|
|
46
48
|
extracted as named constants above. */
|
|
47
49
|
|
|
48
|
-
const DEFAULT_COMMUNES = "
|
|
50
|
+
const DEFAULT_COMMUNES = tempRootPath("reg", "fr-communes.tsv")
|
|
49
51
|
const LICENSE = "BAN (Base Adresse Nationale) commune+postcode tuples, rendered admin-split — see ingest SOURCE"
|
|
50
52
|
|
|
51
53
|
/**
|