@mailwoman/corpus 9.1.0 → 9.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/data/PROVENANCE.md +13 -0
- package/data/reviewed-ve-postcode-tuples.json +74 -0
- package/out/src/adapters/ban/adapter.d.ts +1 -1
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +4 -3
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +2 -2
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +1 -1
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +1 -1
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +2 -2
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +1 -1
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/index.d.ts +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +2 -2
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +1 -1
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +1 -1
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +6 -5
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +11 -6
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +11 -17
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +11 -6
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +2 -2
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +2 -2
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +1 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +1 -1
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +6 -5
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +10 -6
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +5 -4
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +7 -7
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +9 -9
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +6 -5
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/utils/index.d.ts +7 -1
- package/out/src/adapters/utils/index.d.ts.map +1 -1
- package/out/src/adapters/utils/index.js +10 -2
- package/out/src/adapters/utils/index.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +5 -5
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +4 -15
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +11 -35
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-json-rows.d.ts +32 -0
- package/out/src/adapters/wof-json-rows.d.ts.map +1 -0
- package/out/src/adapters/wof-json-rows.js +45 -0
- package/out/src/adapters/wof-json-rows.js.map +1 -0
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +4 -9
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +12 -37
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/build.d.ts +4 -3
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +7 -4
- package/out/src/build.js.map +1 -1
- package/out/src/index.d.ts +1 -1
- package/out/src/index.d.ts.map +1 -1
- package/out/src/index.js +1 -1
- package/out/src/index.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +5 -0
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/runner.d.ts +2 -2
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +1 -1
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +4 -5
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/bare-country.d.ts +25 -0
- package/out/src/shard-recipes/bare-country.d.ts.map +1 -0
- package/out/src/shard-recipes/bare-country.js +87 -0
- package/out/src/shard-recipes/bare-country.js.map +1 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +2 -2
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +6 -5
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +28 -10
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +4 -3
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.js +72 -11
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +4 -3
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +4 -3
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +3 -3
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.js +1 -1
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +6 -0
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +3 -3
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +1 -1
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +5 -4
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/no-street.js +1 -1
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +2 -2
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +1 -1
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/reviewed-postcode-tail.d.ts +47 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.d.ts.map +1 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.js +125 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.js.map +1 -0
- package/out/src/shard-recipes/scaffold.d.ts +33 -4
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +5 -6
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +2 -2
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.js +3 -3
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.js +2 -2
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.d.ts +2 -2
- package/out/src/shard-recipes/sub-venue-sources.d.ts.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.js +1 -1
- package/out/src/shard-recipes/sub-venue-sources.js.map +1 -1
- package/out/src/shard-recipes/sub-venue.d.ts +1 -1
- package/out/src/shard-recipes/sub-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/sub-venue.js +6 -5
- package/out/src/shard-recipes/sub-venue.js.map +1 -1
- package/out/src/shard-recipes/trailing-region.d.ts +74 -0
- package/out/src/shard-recipes/trailing-region.d.ts.map +1 -0
- package/out/src/shard-recipes/trailing-region.js +157 -0
- package/out/src/shard-recipes/trailing-region.js.map +1 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +2 -2
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/synthesizers/boundary-stress.d.ts +1 -1
- package/out/src/synthesizers/boundary-stress.d.ts.map +1 -1
- package/out/src/synthesizers/german.d.ts +1 -1
- package/out/src/synthesizers/german.d.ts.map +1 -1
- package/out/src/synthesizers/german.js.map +1 -1
- package/out/src/synthesizers/house-venue.d.ts +1 -1
- package/out/src/synthesizers/house-venue.d.ts.map +1 -1
- package/out/src/synthesizers/house-venue.js +16 -2
- package/out/src/synthesizers/house-venue.js.map +1 -1
- package/out/src/synthesizers/intersection.d.ts +1 -1
- package/out/src/synthesizers/intersection.d.ts.map +1 -1
- package/out/src/synthesizers/no-street.d.ts +1 -1
- package/out/src/synthesizers/no-street.d.ts.map +1 -1
- package/out/src/synthesizers/po-box.d.ts +1 -1
- package/out/src/synthesizers/po-box.d.ts.map +1 -1
- package/out/src/synthesizers/street.d.ts +1 -1
- package/out/src/synthesizers/street.d.ts.map +1 -1
- package/out/src/synthesizers/street.js +3 -2
- package/out/src/synthesizers/street.js.map +1 -1
- package/out/src/synthesizers/utils.d.ts +1 -1
- package/out/src/synthesizers/utils.d.ts.map +1 -1
- package/out/src/synthesizers/utils.js +3 -2
- package/out/src/synthesizers/utils.js.map +1 -1
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +2 -2
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/fetch/ban.d.ts.map +1 -1
- package/out/src/tools/fetch/ban.js +2 -18
- package/out/src/tools/fetch/ban.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts +12 -3
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +20 -4
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/geonames-dump.d.ts +87 -0
- package/out/src/tools/fetch/geonames-dump.d.ts.map +1 -0
- package/out/src/tools/fetch/geonames-dump.js +178 -0
- package/out/src/tools/fetch/geonames-dump.js.map +1 -0
- package/out/src/tools/fetch/geonames-postal.d.ts +76 -0
- package/out/src/tools/fetch/geonames-postal.d.ts.map +1 -0
- package/out/src/tools/fetch/geonames-postal.js +125 -0
- package/out/src/tools/fetch/geonames-postal.js.map +1 -0
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +2 -2
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +11 -0
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +11 -0
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts.map +1 -1
- package/out/src/tools/fetch/nad.js +20 -11
- package/out/src/tools/fetch/nad.js.map +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +9 -7
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/src/tools/fetch/openaddresses.js +18 -17
- package/out/src/tools/fetch/openaddresses.js.map +1 -1
- package/out/src/tools/fetch/ourairports.d.ts.map +1 -1
- package/out/src/tools/fetch/ourairports.js +7 -2
- package/out/src/tools/fetch/ourairports.js.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +0 -4
- package/out/src/tools/fetch/ppd.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.js +1 -1
- package/out/src/tools/fetch/ppd.js.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.js +3 -19
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
- package/out/src/tools/fetch/state-sources.d.ts +4 -0
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
- package/out/src/tools/fetch/state-sources.js +1 -5
- package/out/src/tools/fetch/state-sources.js.map +1 -1
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
- package/out/src/tools/fetch/tiger-full.js +15 -21
- package/out/src/tools/fetch/tiger-full.js.map +1 -1
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +34 -25
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-relabel-street.js +2 -2
- package/out/src/tools/golden-relabel-street.js.map +1 -1
- package/out/src/tools/index.d.ts +2 -2
- package/out/src/tools/index.d.ts.map +1 -1
- package/out/src/tools/index.js +2 -2
- package/out/src/tools/index.js.map +1 -1
- package/out/src/tools/ingest-csv.d.ts.map +1 -1
- package/out/src/tools/ingest-csv.js +0 -11
- package/out/src/tools/ingest-csv.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts +4 -0
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +16 -6
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/postcode-triples.d.ts +175 -0
- package/out/src/tools/postcode-triples.d.ts.map +1 -0
- package/out/src/tools/postcode-triples.js +304 -0
- package/out/src/tools/postcode-triples.js.map +1 -0
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +1 -2
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +4 -26
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/tools/sub-venue/harvest.d.ts +100 -0
- package/out/src/tools/sub-venue/harvest.d.ts.map +1 -0
- package/out/src/tools/sub-venue/harvest.js +168 -0
- package/out/src/tools/sub-venue/harvest.js.map +1 -0
- package/out/src/tools/sub-venue/head-nouns.d.ts +50 -0
- package/out/src/tools/sub-venue/head-nouns.d.ts.map +1 -0
- package/out/src/tools/sub-venue/head-nouns.js +210 -0
- package/out/src/tools/sub-venue/head-nouns.js.map +1 -0
- package/out/src/tools/sub-venue/surfaces.d.ts +45 -0
- package/out/src/tools/sub-venue/surfaces.d.ts.map +1 -0
- package/out/src/tools/sub-venue/surfaces.js +77 -0
- package/out/src/tools/sub-venue/surfaces.js.map +1 -0
- package/out/src/tools/sub-venue/table.d.ts +229 -0
- package/out/src/tools/sub-venue/table.d.ts.map +1 -0
- package/out/src/tools/sub-venue/table.js +108 -0
- package/out/src/tools/sub-venue/table.js.map +1 -0
- package/out/src/tools/sub-venue/wikidata.d.ts +26 -0
- package/out/src/tools/sub-venue/wikidata.d.ts.map +1 -0
- package/out/src/tools/sub-venue/wikidata.js +62 -0
- package/out/src/tools/sub-venue/wikidata.js.map +1 -0
- package/out/src/tools/sub-venue-lexicon.d.ts +27 -378
- package/out/src/tools/sub-venue-lexicon.d.ts.map +1 -1
- package/out/src/tools/sub-venue-lexicon.js +30 -565
- package/out/src/tools/sub-venue-lexicon.js.map +1 -1
- package/out/src/utils/align.d.ts +1 -1
- package/out/src/utils/align.d.ts.map +1 -1
- package/out/src/utils/align.js.map +1 -1
- package/out/src/utils/golden.js +1 -1
- package/out/src/utils/golden.js.map +1 -1
- package/out/src/utils/license.js +1 -1
- package/out/src/utils/license.js.map +1 -1
- package/out/src/utils/parquet.d.ts +24 -5
- package/out/src/utils/parquet.d.ts.map +1 -1
- package/out/src/utils/parquet.js +6 -2
- package/out/src/utils/parquet.js.map +1 -1
- package/out/src/utils/split.d.ts +1 -1
- package/out/src/utils/split.d.ts.map +1 -1
- package/out/src/utils/split.js +2 -2
- package/out/src/utils/split.js.map +1 -1
- package/package.json +230 -10
- package/src/adapters/ban/adapter.ts +6 -4
- package/src/adapters/fcc-bdc/adapter.ts +4 -3
- package/src/adapters/geonames/adapter.ts +3 -2
- package/src/adapters/geonames-postal/adapter.ts +4 -3
- package/src/adapters/gnaf/adapter.ts +3 -2
- package/src/adapters/index.ts +2 -2
- package/src/adapters/openaddresses/adapter.ts +4 -3
- package/src/adapters/overture/adapter.ts +3 -2
- package/src/adapters/state-hi-schools/adapter.ts +8 -6
- package/src/adapters/state-ia-contractors/adapter.ts +13 -7
- package/src/adapters/state-ny-notaries/adapter.ts +13 -31
- package/src/adapters/state-tx-notaries/adapter.ts +13 -7
- package/src/adapters/synth-po-box/adapter.ts +5 -4
- package/src/adapters/tiger/adapter.ts +3 -2
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +8 -6
- package/src/adapters/usgov-imls-pls/adapter.ts +12 -7
- package/src/adapters/usgov-irs-bmf/adapter.ts +7 -5
- package/src/adapters/usgov-nad/adapter.ts +9 -8
- package/src/adapters/usgov-nppes/adapter.ts +13 -11
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +8 -6
- package/src/adapters/utils/index.ts +16 -3
- package/src/adapters/wof-admin-jp/adapter.ts +9 -12
- package/src/adapters/wof-admin-json/adapter.ts +16 -54
- package/src/adapters/wof-json-rows.ts +79 -0
- package/src/adapters/wof-postalcode-json/adapter.ts +17 -50
- package/src/build.ts +10 -10
- package/src/index.ts +1 -1
- package/src/parquet-wrapper/reader.ts +9 -0
- package/src/runner.ts +2 -7
- package/src/shard-recipes/anchor-absorption.ts +5 -5
- package/src/shard-recipes/bare-country.ts +95 -0
- package/src/shard-recipes/boundary-stress.ts +2 -5
- package/src/shard-recipes/country-balanced.ts +8 -6
- package/src/shard-recipes/cz-pcfirst-preposition.ts +29 -10
- package/src/shard-recipes/fr-admin-split.ts +6 -4
- package/src/shard-recipes/fr-bare-street.ts +85 -10
- package/src/shard-recipes/fr-lieudit.ts +6 -4
- package/src/shard-recipes/fr-order.ts +6 -4
- package/src/shard-recipes/german.ts +4 -3
- package/src/shard-recipes/house-venue.ts +1 -1
- package/src/shard-recipes/index.ts +6 -0
- package/src/shard-recipes/intersection.ts +5 -4
- package/src/shard-recipes/locale.ts +6 -8
- package/src/shard-recipes/no-street.ts +1 -1
- package/src/shard-recipes/po-box-cedex.ts +4 -3
- package/src/shard-recipes/po-box.ts +1 -5
- package/src/shard-recipes/reviewed-postcode-tail.ts +189 -0
- package/src/shard-recipes/scaffold.ts +38 -7
- package/src/shard-recipes/street-affix.ts +4 -3
- package/src/shard-recipes/street-bare.ts +3 -3
- package/src/shard-recipes/street.ts +2 -2
- package/src/shard-recipes/sub-venue-sources.ts +4 -14
- package/src/shard-recipes/sub-venue.ts +10 -8
- package/src/shard-recipes/trailing-region.ts +183 -0
- package/src/shard-recipes/unit.ts +4 -3
- package/src/synthesizers/boundary-stress.ts +1 -1
- package/src/synthesizers/german.ts +2 -1
- package/src/synthesizers/house-venue.ts +17 -3
- package/src/synthesizers/intersection.ts +1 -1
- package/src/synthesizers/no-street.ts +1 -1
- package/src/synthesizers/po-box.ts +1 -1
- package/src/synthesizers/street.ts +5 -3
- package/src/synthesizers/utils.ts +5 -3
- package/src/tools/align-shard.ts +3 -2
- package/src/tools/fetch/ban.ts +2 -22
- package/src/tools/fetch/download.ts +22 -4
- package/src/tools/fetch/geonames-dump.ts +267 -0
- package/src/tools/fetch/geonames-postal.ts +180 -0
- package/src/tools/fetch/imls-pls.ts +2 -2
- package/src/tools/fetch/index.ts +11 -0
- package/src/tools/fetch/nad.ts +21 -10
- package/src/tools/fetch/nppes.ts +8 -6
- package/src/tools/fetch/openaddresses.ts +18 -22
- package/src/tools/fetch/ourairports.ts +7 -2
- package/src/tools/fetch/ppd.ts +1 -1
- package/src/tools/fetch/state-hi-schools.ts +3 -23
- package/src/tools/fetch/state-sources.ts +1 -1
- package/src/tools/fetch/tiger-full.ts +17 -24
- package/src/tools/golden-expand.ts +44 -33
- package/src/tools/golden-relabel-street.ts +2 -2
- package/src/tools/index.ts +2 -2
- package/src/tools/ingest-csv.ts +0 -13
- package/src/tools/overlay-manifest.ts +20 -8
- package/src/tools/postcode-triples.ts +375 -0
- package/src/tools/shard-kryptonite.ts +5 -4
- package/src/tools/shard-translit.ts +10 -35
- package/src/tools/sub-venue/harvest.ts +235 -0
- package/src/tools/sub-venue/head-nouns.ts +242 -0
- package/src/tools/sub-venue/surfaces.ts +95 -0
- package/src/tools/sub-venue/table.ts +281 -0
- package/src/tools/sub-venue/wikidata.ts +85 -0
- package/src/tools/sub-venue-lexicon.ts +67 -886
- package/src/utils/align.ts +2 -1
- package/src/utils/golden.ts +1 -1
- package/src/utils/license.ts +1 -1
- package/src/utils/parquet.ts +22 -8
- package/src/utils/split.ts +4 -3
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
import { anchorAbsorptionRecipe } from "./anchor-absorption.ts"
|
|
12
|
+
import { bareCountryRecipe } from "./bare-country.ts"
|
|
12
13
|
import { boundaryStressRecipe } from "./boundary-stress.ts"
|
|
13
14
|
import { countryBalancedRecipe } from "./country-balanced.ts"
|
|
14
15
|
import { czPcFirstPrepositionRecipe } from "./cz-pcfirst-preposition.ts"
|
|
@@ -27,12 +28,14 @@ import { noStreetLedRecipe } from "./no-street-led.ts"
|
|
|
27
28
|
import { noStreetRecipe } from "./no-street.ts"
|
|
28
29
|
import { poBoxCedexRecipe } from "./po-box-cedex.ts"
|
|
29
30
|
import { poBoxRecipe } from "./po-box.ts"
|
|
31
|
+
import { reviewedPostcodeTailRecipe } from "./reviewed-postcode-tail.ts"
|
|
30
32
|
import type { ShardRecipe } from "./scaffold.ts"
|
|
31
33
|
import { siBareVillageRecipe } from "./si-bare-village.ts"
|
|
32
34
|
import { streetAffixRecipe, suffixBoundaryRecipe } from "./street-affix.ts"
|
|
33
35
|
import { streetBareRecipe } from "./street-bare.ts"
|
|
34
36
|
import { streetRecipe } from "./street.ts"
|
|
35
37
|
import { subVenueRecipe } from "./sub-venue.ts"
|
|
38
|
+
import { trailingRegionRecipe } from "./trailing-region.ts"
|
|
36
39
|
import { unitRecipe } from "./unit.ts"
|
|
37
40
|
|
|
38
41
|
export * from "./scaffold.ts"
|
|
@@ -41,6 +44,9 @@ export * from "./scaffold.ts"
|
|
|
41
44
|
* Every registered recipe, in display order.
|
|
42
45
|
*/
|
|
43
46
|
const RECIPES: readonly ShardRecipe[] = [
|
|
47
|
+
bareCountryRecipe,
|
|
48
|
+
trailingRegionRecipe,
|
|
49
|
+
reviewedPostcodeTailRecipe,
|
|
44
50
|
streetRecipe,
|
|
45
51
|
streetBareRecipe,
|
|
46
52
|
streetAffixRecipe,
|
|
@@ -34,16 +34,17 @@
|
|
|
34
34
|
* - `$MAILWOMAN_DATA_ROOT/oa-cache/us__il__cook.zip` (ZIP→city tails)
|
|
35
35
|
*/
|
|
36
36
|
|
|
37
|
-
import { existsSync,
|
|
37
|
+
import { existsSync, writeFileSync } from "node:fs"
|
|
38
38
|
|
|
39
39
|
import type { DuckDBConnection } from "@duckdb/node-api"
|
|
40
40
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
41
41
|
import { dataRootPath, repoRootPath } from "@mailwoman/core/utils"
|
|
42
|
-
import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
|
|
43
|
-
import type { CanonicalRow, LabeledRow } from "@mailwoman/corpus/types"
|
|
44
|
-
import { alignRow } from "@mailwoman/corpus/utils"
|
|
45
42
|
import { JSONSpliterator } from "spliterator"
|
|
46
43
|
|
|
44
|
+
import { stableSourceID } from "#adapters/utils"
|
|
45
|
+
import type { CanonicalRow, LabeledRow } from "#types"
|
|
46
|
+
import { alignRow } from "#utils"
|
|
47
|
+
|
|
47
48
|
import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
|
|
48
49
|
|
|
49
50
|
interface County {
|
|
@@ -29,16 +29,14 @@ import { createReadStream } from "node:fs"
|
|
|
29
29
|
|
|
30
30
|
import { COUNTRY_SURFACE_FORMS } from "@mailwoman/codex/country"
|
|
31
31
|
import { readZipEntry } from "@mailwoman/core/fs/zip"
|
|
32
|
+
import { isPresent } from "@mailwoman/core/objects"
|
|
32
33
|
import { dataRootPath } from "@mailwoman/core/utils"
|
|
33
|
-
import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
|
|
34
|
-
import {
|
|
35
|
-
type LocaleBaseTuple,
|
|
36
|
-
type SynthesizedLocaleRow,
|
|
37
|
-
synthesizeLocaleRow,
|
|
38
|
-
} from "@mailwoman/corpus/synthesizers/german"
|
|
39
|
-
import { alignRow } from "@mailwoman/corpus/utils"
|
|
40
34
|
import { CSVSpliterator } from "spliterator"
|
|
41
35
|
|
|
36
|
+
import { stableSourceID } from "#adapters/utils"
|
|
37
|
+
import { type LocaleBaseTuple, type SynthesizedLocaleRow, synthesizeLocaleRow } from "#synthesizers/german"
|
|
38
|
+
import { alignRow } from "#utils"
|
|
39
|
+
|
|
42
40
|
import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
|
|
43
41
|
|
|
44
42
|
/**
|
|
@@ -296,7 +294,7 @@ export async function readTuples(part: LocalePart, rng: () => number): Promise<L
|
|
|
296
294
|
// pre-joined STREET column, so `cols.tipoVial === -1` and this is a no-op there.
|
|
297
295
|
const street =
|
|
298
296
|
cols.tipoVial >= 0
|
|
299
|
-
? [get(cells, cols.tipoVial), get(cells, cols.street)].filter(
|
|
297
|
+
? [get(cells, cols.tipoVial), get(cells, cols.street)].filter(isPresent).join(" ")
|
|
300
298
|
: get(cells, cols.street)
|
|
301
299
|
|
|
302
300
|
const rawCity = get(cells, cols.city)
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* scripts/build-no-street-shard.mjs.
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import { synthesizeNoStreetRow, type NoStreetBaseTuple } from "
|
|
13
|
+
import { synthesizeNoStreetRow, type NoStreetBaseTuple } from "#synthesizers/no-street"
|
|
14
14
|
|
|
15
15
|
import { alignAndWrite, makeLcg, readTuples, shardSourceID, type ShardRecipe } from "./scaffold.ts"
|
|
16
16
|
|
|
@@ -39,14 +39,15 @@ import { isNZDeliveryService, isNZPostcode } from "@mailwoman/codex/nz"
|
|
|
39
39
|
import { isPOBox } from "@mailwoman/codex/us"
|
|
40
40
|
import { readZipEntry } from "@mailwoman/core/fs/zip"
|
|
41
41
|
import { dataRootPath } from "@mailwoman/core/utils"
|
|
42
|
+
import { TextSpliterator, TSVSpliterator } from "spliterator"
|
|
43
|
+
|
|
42
44
|
import {
|
|
43
45
|
maybeNoisifyBoxNumber,
|
|
44
46
|
PO_BOX_LOCALE_TEMPLATES,
|
|
45
47
|
synthesizeMilitaryPoBoxRow,
|
|
46
48
|
type LocaleTemplate,
|
|
47
|
-
} from "
|
|
48
|
-
import { alignRow } from "
|
|
49
|
-
import { TextSpliterator, TSVSpliterator } from "spliterator"
|
|
49
|
+
} from "#synthesizers/po-box"
|
|
50
|
+
import { alignRow } from "#utils"
|
|
50
51
|
|
|
51
52
|
import {
|
|
52
53
|
makeMulberry32,
|
|
@@ -9,11 +9,7 @@
|
|
|
9
9
|
* scripts/build-po-box-shard.mjs.
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
|
-
import {
|
|
13
|
-
synthesizeMilitaryPoBoxRow,
|
|
14
|
-
synthesizePoBoxRow,
|
|
15
|
-
type PoBoxBaseTuple,
|
|
16
|
-
} from "@mailwoman/corpus/synthesizers/po-box"
|
|
12
|
+
import { synthesizeMilitaryPoBoxRow, synthesizePoBoxRow, type PoBoxBaseTuple } from "#synthesizers/po-box"
|
|
17
13
|
|
|
18
14
|
import { alignAndWrite, makeLcg, readTuples, shardSourceID, type ShardRecipe } from "./scaffold.ts"
|
|
19
15
|
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* A bounded, reviewed source for Venezuela's `locality postcode, region` convention (#1821).
|
|
7
|
+
* Bulk sources yielded zero Venezuelan tuples, so this recipe reads four reviewed facts and applies
|
|
8
|
+
* only transformations that do not create another postcode-to-place join.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { readFileSync } from "node:fs"
|
|
12
|
+
import { fileURLToPath } from "node:url"
|
|
13
|
+
|
|
14
|
+
import { parseJSONStrict } from "@mailwoman/core/objects"
|
|
15
|
+
|
|
16
|
+
import { alignAndWrite, type ShardRecipe, shardSourceID } from "./scaffold.ts"
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* A separate sampler bucket so a receipt measures these reviewed after-locality rows and no other postcode placement.
|
|
20
|
+
*/
|
|
21
|
+
export const REVIEWED_POSTCODE_TAIL_SOURCE = "synth-reviewed-postcode-tail"
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* The four postcode-to-place facts reviewed for #1821 and committed in the package data file.
|
|
25
|
+
*/
|
|
26
|
+
const REVIEWED_TUPLE_COUNT = 4
|
|
27
|
+
|
|
28
|
+
interface ReviewedTupleProvenance {
|
|
29
|
+
publisher: string
|
|
30
|
+
url: string
|
|
31
|
+
retrievedAt: string
|
|
32
|
+
reviewStatus: "reviewed"
|
|
33
|
+
sourceLicenseNote: string
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface ReviewedPostcodeTuple {
|
|
37
|
+
id: string
|
|
38
|
+
locality: string
|
|
39
|
+
postcode: string
|
|
40
|
+
region: string
|
|
41
|
+
country: "Venezuela"
|
|
42
|
+
cc: "VE"
|
|
43
|
+
locale: "es-VE"
|
|
44
|
+
postcodePlacement: "after_locality"
|
|
45
|
+
provenance: ReviewedTupleProvenance
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
interface ReviewedPostcodeTupleFile {
|
|
49
|
+
version: string
|
|
50
|
+
reviewedAt: string
|
|
51
|
+
tuples: ReviewedPostcodeTuple[]
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
type VariantID =
|
|
55
|
+
| "canonical-country"
|
|
56
|
+
| "canonical-no-country"
|
|
57
|
+
| "comma-free"
|
|
58
|
+
| "uppercase"
|
|
59
|
+
| "left-context"
|
|
60
|
+
| "accent-folded"
|
|
61
|
+
|
|
62
|
+
interface Variant {
|
|
63
|
+
id: VariantID
|
|
64
|
+
raw: string
|
|
65
|
+
components: Record<string, string>
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/
|
|
69
|
+
|
|
70
|
+
export function defaultReviewedPostcodeTuplePath(): string {
|
|
71
|
+
return fileURLToPath(import.meta.resolve("@mailwoman/corpus/data/reviewed-ve-postcode-tuples.json"))
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function readReviewedPostcodeTuples(path = defaultReviewedPostcodeTuplePath()): ReviewedPostcodeTuple[] {
|
|
75
|
+
const document = parseJSONStrict<ReviewedPostcodeTupleFile>(readFileSync(path, "utf8"))
|
|
76
|
+
|
|
77
|
+
if (!document.version || !ISO_DATE.test(document.reviewedAt) || document.tuples.length !== REVIEWED_TUPLE_COUNT) {
|
|
78
|
+
throw new Error("reviewed postcode tuple file must declare a version, review date, and exactly four tuples")
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const ids = new Set<string>()
|
|
82
|
+
const facts = new Set<string>()
|
|
83
|
+
|
|
84
|
+
for (const tuple of document.tuples) {
|
|
85
|
+
const provenance = tuple.provenance
|
|
86
|
+
const fact = `${tuple.locality}\u0000${tuple.postcode}\u0000${tuple.region}`
|
|
87
|
+
|
|
88
|
+
if (
|
|
89
|
+
!tuple.id ||
|
|
90
|
+
!tuple.locality ||
|
|
91
|
+
!tuple.postcode ||
|
|
92
|
+
!tuple.region ||
|
|
93
|
+
tuple.country !== "Venezuela" ||
|
|
94
|
+
tuple.cc !== "VE" ||
|
|
95
|
+
tuple.locale !== "es-VE" ||
|
|
96
|
+
tuple.postcodePlacement !== "after_locality" ||
|
|
97
|
+
!provenance?.publisher ||
|
|
98
|
+
!URL.canParse(provenance.url) ||
|
|
99
|
+
!ISO_DATE.test(provenance.retrievedAt) ||
|
|
100
|
+
provenance.reviewStatus !== "reviewed" ||
|
|
101
|
+
!provenance.sourceLicenseNote
|
|
102
|
+
) {
|
|
103
|
+
throw new Error(`invalid reviewed postcode tuple: ${tuple.id || "<missing id>"}`)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
if (ids.has(tuple.id) || facts.has(fact)) throw new Error(`duplicate reviewed postcode tuple: ${tuple.id}`)
|
|
107
|
+
ids.add(tuple.id)
|
|
108
|
+
facts.add(fact)
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
return document.tuples
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const foldAccents = (value: string): string => value.normalize("NFD").replaceAll(/\p{M}/gu, "").normalize("NFC")
|
|
115
|
+
|
|
116
|
+
export function reviewedPostcodeTailVariants(tuple: ReviewedPostcodeTuple): Variant[] {
|
|
117
|
+
const tail = `${tuple.locality} ${tuple.postcode}, ${tuple.region}`
|
|
118
|
+
const withCountry = `${tail}, ${tuple.country}`
|
|
119
|
+
const base = { locality: tuple.locality, postcode: tuple.postcode, region: tuple.region, country: tuple.country }
|
|
120
|
+
const upper = Object.fromEntries(Object.entries(base).map(([key, value]) => [key, value.toUpperCase()]))
|
|
121
|
+
|
|
122
|
+
const variants: Variant[] = [
|
|
123
|
+
{ id: "canonical-country", raw: withCountry, components: base },
|
|
124
|
+
{
|
|
125
|
+
id: "canonical-no-country",
|
|
126
|
+
raw: tail,
|
|
127
|
+
components: { locality: tuple.locality, postcode: tuple.postcode, region: tuple.region },
|
|
128
|
+
},
|
|
129
|
+
{ id: "comma-free", raw: `${tuple.locality} ${tuple.postcode} ${tuple.region} ${tuple.country}`, components: base },
|
|
130
|
+
{ id: "uppercase", raw: withCountry.toUpperCase(), components: upper },
|
|
131
|
+
{
|
|
132
|
+
id: "left-context",
|
|
133
|
+
raw: `Comercio Ejemplo, Calle Principal, ${withCountry}`,
|
|
134
|
+
components: { venue: "Comercio Ejemplo", street: "Calle Principal", ...base },
|
|
135
|
+
},
|
|
136
|
+
]
|
|
137
|
+
|
|
138
|
+
const foldedLocality = foldAccents(tuple.locality)
|
|
139
|
+
const foldedRegion = foldAccents(tuple.region)
|
|
140
|
+
|
|
141
|
+
if (foldedLocality !== tuple.locality || foldedRegion !== tuple.region) {
|
|
142
|
+
variants.push({
|
|
143
|
+
id: "accent-folded",
|
|
144
|
+
raw: `${foldedLocality} ${tuple.postcode}, ${foldedRegion}, ${tuple.country}`,
|
|
145
|
+
components: { locality: foldedLocality, postcode: tuple.postcode, region: foldedRegion, country: tuple.country },
|
|
146
|
+
})
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
return variants
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Emit bounded surface variants of the committed reviewed facts without constructing another geographic join.
|
|
154
|
+
*/
|
|
155
|
+
export const reviewedPostcodeTailRecipe: ShardRecipe = {
|
|
156
|
+
name: "reviewed-postcode-tail",
|
|
157
|
+
description: "Reviewed Venezuela locality-postcode-region tails with bounded surface variants",
|
|
158
|
+
mode: "generate",
|
|
159
|
+
async run(opts, write) {
|
|
160
|
+
const tuples = readReviewedPostcodeTuples(opts.input)
|
|
161
|
+
let emitted = 0
|
|
162
|
+
let skipped = 0
|
|
163
|
+
|
|
164
|
+
for (const tuple of tuples) {
|
|
165
|
+
for (const variant of reviewedPostcodeTailVariants(tuple)) {
|
|
166
|
+
const sourceID = shardSourceID(REVIEWED_POSTCODE_TAIL_SOURCE, { tuple: tuple.id, variant: variant.id })
|
|
167
|
+
|
|
168
|
+
const canonical = {
|
|
169
|
+
raw: variant.raw,
|
|
170
|
+
components: variant.components,
|
|
171
|
+
country: tuple.cc,
|
|
172
|
+
locale: tuple.locale,
|
|
173
|
+
source: REVIEWED_POSTCODE_TAIL_SOURCE,
|
|
174
|
+
source_id: sourceID,
|
|
175
|
+
corpus_version: "0.11.0",
|
|
176
|
+
license: `Synthetic rendering of reviewed postal facts — ${tuple.provenance.publisher}; source terms recorded in reviewed-ve-postcode-tuples.json`,
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
if (alignAndWrite(write, canonical, "reviewed-postcode-tail", tuple.id)) {
|
|
180
|
+
emitted++
|
|
181
|
+
} else {
|
|
182
|
+
skipped++
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
return { read: tuples.length, emitted, skipped }
|
|
188
|
+
},
|
|
189
|
+
}
|
|
@@ -13,29 +13,60 @@ import { existsSync } from "node:fs"
|
|
|
13
13
|
|
|
14
14
|
import { readZipEntry } from "@mailwoman/core/fs/zip"
|
|
15
15
|
import { tryParsingJSON } from "@mailwoman/core/objects"
|
|
16
|
-
import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
|
|
17
|
-
import { alignRow } from "@mailwoman/corpus/utils"
|
|
18
16
|
import type { PathBuilderLike } from "path-ts"
|
|
19
17
|
import type { AsyncChunkIterator, AsyncDataResource } from "spliterator"
|
|
20
18
|
import { AsyncSequence, CSVSpliterator } from "spliterator"
|
|
21
19
|
|
|
20
|
+
import { stableSourceIDFromParts } from "#adapters/utils"
|
|
21
|
+
import { alignRow } from "#utils"
|
|
22
|
+
|
|
22
23
|
/**
|
|
23
|
-
* {@link
|
|
24
|
-
* `
|
|
25
|
-
* builders kept per-variant ids unique — the strict typing is just too narrow for that. Centralizes the one cast.
|
|
24
|
+
* {@link stableSourceIDFromParts} under the name the recipes use: arbitrary disambiguator keys (e.g. a variant index
|
|
25
|
+
* `v`) that aren't `ComponentTag`s, which is how the legacy builders kept per-variant ids unique.
|
|
26
26
|
*/
|
|
27
27
|
export function shardSourceID(adapterID: string, parts: Record<string, string | undefined>): string {
|
|
28
|
-
return
|
|
28
|
+
return stableSourceIDFromParts(adapterID, parts)
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
+
/**
|
|
32
|
+
* Where a country's convention writes the postcode inside the `«locality», «region»[, «country»]` admin tail.
|
|
33
|
+
*
|
|
34
|
+
* Load-bearing rather than cosmetic: the same digits change TAG with position. Measured on the shipped model,
|
|
35
|
+
* `Heladería Frappé Manía, Avenida Country Club, Barcelona 6001, Anzoátegui, Venezuela` tags `6001` as `house_number`
|
|
36
|
+
* and loses the locality into the street, while the identical row written `… 6001 Barcelona, Anzoátegui, Venezuela`
|
|
37
|
+
* tags it `postcode` and recovers `locality: Barcelona`. So a shard emitting one placement teaches one family of
|
|
38
|
+
* countries.
|
|
39
|
+
*
|
|
40
|
+
* Each is attested by a gauntlet board row, which is the bar for adding another:
|
|
41
|
+
*
|
|
42
|
+
* - `leading` — `«postcode» «locality», «region»`. `Rua da Praia, 15, 8600-315 Lagos, Algarve, Portugal`
|
|
43
|
+
* (`pt_structured`). The default, and what every tuple written before this field existed means.
|
|
44
|
+
* - `after_locality` — `«locality» «postcode», «region»`. `…, Barcelona 6001, Anzoátegui, Venezuela`
|
|
45
|
+
* (`ve_city_postcode_trailing_state`).
|
|
46
|
+
* - `after_region` — `«locality», «region» «postcode»`. `12 MG Road, Indiranagar, Bengaluru, Karnataka 560038, India`
|
|
47
|
+
* (`in_structured`).
|
|
48
|
+
*/
|
|
49
|
+
export type PostcodePlacement = "leading" | "after_locality" | "after_region"
|
|
50
|
+
|
|
31
51
|
/**
|
|
32
52
|
* A (locality, region, postcode, country) source tuple — the input to tuples-mode recipes.
|
|
33
53
|
*/
|
|
34
54
|
export interface ShardTuple {
|
|
35
55
|
locality?: string
|
|
56
|
+
/**
|
|
57
|
+
* The segment before the locality, when the source has one. A shard whose every row BEGINS with the locality teaches
|
|
58
|
+
* that the first named segment is the locality, which flips the model's default — measured on the v4.8.0 candidate,
|
|
59
|
+
* where `Ye Three Lords, 27 Minories, London EC3N 1DE` came back `locality: "Ye Three Lords"`.
|
|
60
|
+
*/
|
|
61
|
+
dependentLocality?: string
|
|
36
62
|
region?: string
|
|
37
63
|
postcode?: string
|
|
38
64
|
country?: string
|
|
65
|
+
/**
|
|
66
|
+
* Defaults to `leading` when absent, which is what every tuple file written before this field existed means — so an
|
|
67
|
+
* old tuples file produces byte-identical rows.
|
|
68
|
+
*/
|
|
69
|
+
postcodePlacement?: PostcodePlacement
|
|
39
70
|
[k: string]: unknown
|
|
40
71
|
}
|
|
41
72
|
|
|
@@ -266,7 +297,7 @@ export interface ShardRecipeOpts {
|
|
|
266
297
|
* `sub-venue`: the `poi.db` spatial layer, read for the en-US and fr-FR venue + confound pools (the two of poi.db's
|
|
267
298
|
* four countries this shard has legs for). Default `$MAILWOMAN_DATA_ROOT/poi/poi.db`.
|
|
268
299
|
*/
|
|
269
|
-
|
|
300
|
+
poiDB?: string
|
|
270
301
|
/**
|
|
271
302
|
* `sub-venue`: GB/US/FR address-context tuples JSONL. Default the house-venue v3 tuples
|
|
272
303
|
* (`$MAILWOMAN_DATA_ROOT/corpus/intermediate/house-venue-tuples-v3.jsonl`); DE and ES read OpenAddresses directly.
|
|
@@ -38,9 +38,10 @@ import {
|
|
|
38
38
|
} from "@mailwoman/codex/us"
|
|
39
39
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
40
40
|
import { dataRootPath } from "@mailwoman/core/utils"
|
|
41
|
-
|
|
42
|
-
import
|
|
43
|
-
import {
|
|
41
|
+
|
|
42
|
+
import { stableSourceID } from "#adapters/utils"
|
|
43
|
+
import type { CanonicalRow } from "#types"
|
|
44
|
+
import { alignRow } from "#utils"
|
|
44
45
|
|
|
45
46
|
import { NAME_PRONE_US_SUFFIXES } from "../name-prone-us-suffixes.ts"
|
|
46
47
|
import { makeMulberry32, readCSVRecords, readZippedCSVRecords, shardSourceID, type ShardRecipe } from "./scaffold.ts"
|
|
@@ -17,9 +17,9 @@
|
|
|
17
17
|
* reproduces the legacy run byte-for-byte.
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
|
-
import { DEFAULT_US_BASES } from "
|
|
21
|
-
import { synthesizeStreetRow, type StreetBaseTuple } from "
|
|
22
|
-
import { alignRow } from "
|
|
20
|
+
import { DEFAULT_US_BASES } from "#synthesizers/intersection"
|
|
21
|
+
import { synthesizeStreetRow, type StreetBaseTuple } from "#synthesizers/street"
|
|
22
|
+
import { alignRow } from "#utils"
|
|
23
23
|
|
|
24
24
|
import { makeMulberry32, shardSourceID, type CanonicalShardRow, type ShardRecipe } from "./scaffold.ts"
|
|
25
25
|
|
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
* {@link synthesizeStreetRow} → aligned LabeledRow. Ported from scripts/build-street-shard.mjs.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
-
import { stableSourceID } from "
|
|
11
|
-
import { synthesizeStreetRow, type StreetBaseTuple } from "
|
|
10
|
+
import { stableSourceID } from "#adapters/utils"
|
|
11
|
+
import { synthesizeStreetRow, type StreetBaseTuple } from "#synthesizers/street"
|
|
12
12
|
|
|
13
13
|
import { alignAndWrite, makeLcg, readTuples, type ShardRecipe } from "./scaffold.ts"
|
|
14
14
|
|
|
@@ -31,14 +31,15 @@ import { fileURLToPath } from "node:url"
|
|
|
31
31
|
|
|
32
32
|
import { parseJSONStrict } from "@mailwoman/core/objects"
|
|
33
33
|
import { dataRootPath } from "@mailwoman/core/utils"
|
|
34
|
-
|
|
34
|
+
|
|
35
|
+
import type { LocaleBaseTuple } from "#synthesizers/german"
|
|
35
36
|
import {
|
|
36
37
|
classifyIdentifier,
|
|
37
38
|
readSubVenueJSONL,
|
|
38
39
|
type SubVenueLexiconTable,
|
|
39
40
|
SUBVENUE_PROMOTIONS,
|
|
40
41
|
type SubVenuePromotion,
|
|
41
|
-
} from "
|
|
42
|
+
} from "#tools"
|
|
42
43
|
|
|
43
44
|
import { readTuples as readLocaleTuples, type LocalePart } from "./locale.ts"
|
|
44
45
|
import { makeMulberry32, readTuples as readShardTuples } from "./scaffold.ts"
|
|
@@ -568,17 +569,6 @@ export const EMPTY_NAME_POOLS: NamePools = {
|
|
|
568
569
|
unpromotedShapes: [],
|
|
569
570
|
}
|
|
570
571
|
|
|
571
|
-
/**
|
|
572
|
-
* An extract row, as `sub-venue-extract` writes it. `SubVenueHarvestRow` plus the `tier` the harvest does not need and
|
|
573
|
-
* this recipe does: `venue` rows (station, airport, campus) fill the venue slot.
|
|
574
|
-
*/
|
|
575
|
-
interface SubVenueExtractRow {
|
|
576
|
-
designatorID: string
|
|
577
|
-
tier?: string
|
|
578
|
-
name?: string | null
|
|
579
|
-
ref?: string | null
|
|
580
|
-
}
|
|
581
|
-
|
|
582
572
|
/**
|
|
583
573
|
* What every pool reader needs to know about the leg it is reading for.
|
|
584
574
|
*/
|
|
@@ -602,7 +592,7 @@ function isLongerProperName(low: string, name: string, designatorPhrases: readon
|
|
|
602
592
|
* Read one OSM extract and split it into name pools.
|
|
603
593
|
*/
|
|
604
594
|
export function readExtractPools(path: string, query: PoolQuery): NamePools {
|
|
605
|
-
const rows = readSubVenueJSONL(path)
|
|
595
|
+
const rows = readSubVenueJSONL(path)
|
|
606
596
|
const venues = new Set<string>()
|
|
607
597
|
const attested = new Set<string>()
|
|
608
598
|
const rejectedVenues = new Set<string>()
|
|
@@ -60,11 +60,13 @@
|
|
|
60
60
|
* `JP` key. The JP extract is harvested and ready; the leg belongs to the JP shard.
|
|
61
61
|
*/
|
|
62
62
|
|
|
63
|
+
import { isPresent } from "@mailwoman/core/objects"
|
|
63
64
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
64
65
|
import { dataRootPath } from "@mailwoman/core/utils"
|
|
65
|
-
|
|
66
|
-
import type {
|
|
67
|
-
import {
|
|
66
|
+
|
|
67
|
+
import type { LocaleBaseTuple } from "#synthesizers/german"
|
|
68
|
+
import type { SubVenueLexiconTable } from "#tools"
|
|
69
|
+
import { alignRow } from "#utils"
|
|
68
70
|
|
|
69
71
|
import { makeMulberry32, shardSourceID, type ShardRecipe } from "./scaffold.ts"
|
|
70
72
|
import {
|
|
@@ -374,7 +376,7 @@ export function renderGroups(
|
|
|
374
376
|
|
|
375
377
|
const raw = groups
|
|
376
378
|
.map((group) => group.map((piece) => fold(piece.text)).join(" "))
|
|
377
|
-
.filter(
|
|
379
|
+
.filter(isPresent)
|
|
378
380
|
.join(separator)
|
|
379
381
|
|
|
380
382
|
const components: Partial<Record<ComponentTag, string>> = {}
|
|
@@ -826,13 +828,13 @@ function buildLegPools(
|
|
|
826
828
|
leg: SubVenueLeg,
|
|
827
829
|
query: PoolQuery,
|
|
828
830
|
contextByCountry: ReadonlyMap<string, LocaleBaseTuple[]>,
|
|
829
|
-
paths: { extractsDir: string;
|
|
831
|
+
paths: { extractsDir: string; poiDB: string }
|
|
830
832
|
): LegPools {
|
|
831
833
|
const extractPools = leg.extract ? readExtractPools(`${paths.extractsDir}/${leg.extract}`, query) : EMPTY_NAME_POOLS
|
|
832
834
|
|
|
833
835
|
// poi.db holds four countries; only these two legs are inside it.
|
|
834
836
|
const poiPools =
|
|
835
|
-
leg.country === "US" || leg.country === "FR" ? readPOIPools(paths.
|
|
837
|
+
leg.country === "US" || leg.country === "FR" ? readPOIPools(paths.poiDB, leg.country, query) : EMPTY_NAME_POOLS
|
|
836
838
|
|
|
837
839
|
const names = mergeNamePools(extractPools, poiPools)
|
|
838
840
|
let context = contextByCountry.get(leg.country) ?? []
|
|
@@ -906,7 +908,7 @@ export const subVenueRecipe: ShardRecipe = {
|
|
|
906
908
|
const count = opts.count
|
|
907
909
|
const negativeFraction = opts.negativeFraction ?? DEFAULT_NEGATIVE_FRACTION
|
|
908
910
|
const extractsDir = opts.extractsDir ?? dataRootPath("sub-venue", "extracts")
|
|
909
|
-
const
|
|
911
|
+
const poiDB = opts.poiDB ?? dataRootPath("poi", "poi.db")
|
|
910
912
|
const tuplesPath = opts.subVenueTuples ?? dataRootPath("corpus", "intermediate", "house-venue-tuples-v3.jsonl")
|
|
911
913
|
const lexicon: SubVenueLexiconTable = readSubVenueLexicon(opts.lexicon ?? defaultLexiconPath())
|
|
912
914
|
|
|
@@ -937,7 +939,7 @@ export const subVenueRecipe: ShardRecipe = {
|
|
|
937
939
|
english: leg.english,
|
|
938
940
|
}
|
|
939
941
|
|
|
940
|
-
const pools = buildLegPools(leg, query, contextByCountry, { extractsDir,
|
|
942
|
+
const pools = buildLegPools(leg, query, contextByCountry, { extractsDir, poiDB })
|
|
941
943
|
|
|
942
944
|
legPromoted.set(leg.locale, promoted)
|
|
943
945
|
legPools.set(leg.locale, pools)
|