@mailwoman/corpus 9.1.0 → 9.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/data/PROVENANCE.md +13 -0
- package/data/reviewed-ve-postcode-tuples.json +74 -0
- package/out/src/adapters/ban/adapter.d.ts +1 -1
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +4 -3
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +2 -2
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +1 -1
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +1 -1
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +2 -2
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +1 -1
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/index.d.ts +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +2 -2
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +1 -1
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +1 -1
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +6 -5
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +11 -6
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +11 -17
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +11 -6
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +2 -2
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +2 -2
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +1 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +1 -1
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +6 -5
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +10 -6
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +5 -4
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +7 -7
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +9 -9
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +6 -5
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/utils/index.d.ts +7 -1
- package/out/src/adapters/utils/index.d.ts.map +1 -1
- package/out/src/adapters/utils/index.js +10 -2
- package/out/src/adapters/utils/index.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +5 -5
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +4 -15
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +11 -35
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-json-rows.d.ts +32 -0
- package/out/src/adapters/wof-json-rows.d.ts.map +1 -0
- package/out/src/adapters/wof-json-rows.js +45 -0
- package/out/src/adapters/wof-json-rows.js.map +1 -0
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +4 -9
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +12 -37
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/build.d.ts +4 -3
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +7 -4
- package/out/src/build.js.map +1 -1
- package/out/src/index.d.ts +1 -1
- package/out/src/index.d.ts.map +1 -1
- package/out/src/index.js +1 -1
- package/out/src/index.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +5 -0
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/runner.d.ts +2 -2
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +1 -1
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +4 -5
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/bare-country.d.ts +25 -0
- package/out/src/shard-recipes/bare-country.d.ts.map +1 -0
- package/out/src/shard-recipes/bare-country.js +87 -0
- package/out/src/shard-recipes/bare-country.js.map +1 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +2 -2
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +6 -5
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +28 -10
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +4 -3
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.js +72 -11
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +4 -3
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +4 -3
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +3 -3
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.js +1 -1
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +6 -0
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +3 -3
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +1 -1
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +5 -4
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/no-street.js +1 -1
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +2 -2
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +1 -1
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/reviewed-postcode-tail.d.ts +47 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.d.ts.map +1 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.js +125 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.js.map +1 -0
- package/out/src/shard-recipes/scaffold.d.ts +33 -4
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +5 -6
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +2 -2
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.js +3 -3
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.js +2 -2
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.d.ts +2 -2
- package/out/src/shard-recipes/sub-venue-sources.d.ts.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.js +1 -1
- package/out/src/shard-recipes/sub-venue-sources.js.map +1 -1
- package/out/src/shard-recipes/sub-venue.d.ts +1 -1
- package/out/src/shard-recipes/sub-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/sub-venue.js +6 -5
- package/out/src/shard-recipes/sub-venue.js.map +1 -1
- package/out/src/shard-recipes/trailing-region.d.ts +74 -0
- package/out/src/shard-recipes/trailing-region.d.ts.map +1 -0
- package/out/src/shard-recipes/trailing-region.js +157 -0
- package/out/src/shard-recipes/trailing-region.js.map +1 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +2 -2
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/synthesizers/boundary-stress.d.ts +1 -1
- package/out/src/synthesizers/boundary-stress.d.ts.map +1 -1
- package/out/src/synthesizers/german.d.ts +1 -1
- package/out/src/synthesizers/german.d.ts.map +1 -1
- package/out/src/synthesizers/german.js.map +1 -1
- package/out/src/synthesizers/house-venue.d.ts +1 -1
- package/out/src/synthesizers/house-venue.d.ts.map +1 -1
- package/out/src/synthesizers/house-venue.js +16 -2
- package/out/src/synthesizers/house-venue.js.map +1 -1
- package/out/src/synthesizers/intersection.d.ts +1 -1
- package/out/src/synthesizers/intersection.d.ts.map +1 -1
- package/out/src/synthesizers/no-street.d.ts +1 -1
- package/out/src/synthesizers/no-street.d.ts.map +1 -1
- package/out/src/synthesizers/po-box.d.ts +1 -1
- package/out/src/synthesizers/po-box.d.ts.map +1 -1
- package/out/src/synthesizers/street.d.ts +1 -1
- package/out/src/synthesizers/street.d.ts.map +1 -1
- package/out/src/synthesizers/street.js +3 -2
- package/out/src/synthesizers/street.js.map +1 -1
- package/out/src/synthesizers/utils.d.ts +1 -1
- package/out/src/synthesizers/utils.d.ts.map +1 -1
- package/out/src/synthesizers/utils.js +3 -2
- package/out/src/synthesizers/utils.js.map +1 -1
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +2 -2
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/fetch/ban.d.ts.map +1 -1
- package/out/src/tools/fetch/ban.js +2 -18
- package/out/src/tools/fetch/ban.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts +12 -3
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +20 -4
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/geonames-dump.d.ts +87 -0
- package/out/src/tools/fetch/geonames-dump.d.ts.map +1 -0
- package/out/src/tools/fetch/geonames-dump.js +178 -0
- package/out/src/tools/fetch/geonames-dump.js.map +1 -0
- package/out/src/tools/fetch/geonames-postal.d.ts +76 -0
- package/out/src/tools/fetch/geonames-postal.d.ts.map +1 -0
- package/out/src/tools/fetch/geonames-postal.js +125 -0
- package/out/src/tools/fetch/geonames-postal.js.map +1 -0
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +2 -2
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +11 -0
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +11 -0
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts.map +1 -1
- package/out/src/tools/fetch/nad.js +20 -11
- package/out/src/tools/fetch/nad.js.map +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +9 -7
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/src/tools/fetch/openaddresses.js +18 -17
- package/out/src/tools/fetch/openaddresses.js.map +1 -1
- package/out/src/tools/fetch/ourairports.d.ts.map +1 -1
- package/out/src/tools/fetch/ourairports.js +7 -2
- package/out/src/tools/fetch/ourairports.js.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +0 -4
- package/out/src/tools/fetch/ppd.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.js +1 -1
- package/out/src/tools/fetch/ppd.js.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.js +3 -19
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
- package/out/src/tools/fetch/state-sources.d.ts +4 -0
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
- package/out/src/tools/fetch/state-sources.js +1 -5
- package/out/src/tools/fetch/state-sources.js.map +1 -1
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
- package/out/src/tools/fetch/tiger-full.js +15 -21
- package/out/src/tools/fetch/tiger-full.js.map +1 -1
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +34 -25
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-relabel-street.js +2 -2
- package/out/src/tools/golden-relabel-street.js.map +1 -1
- package/out/src/tools/index.d.ts +2 -2
- package/out/src/tools/index.d.ts.map +1 -1
- package/out/src/tools/index.js +2 -2
- package/out/src/tools/index.js.map +1 -1
- package/out/src/tools/ingest-csv.d.ts.map +1 -1
- package/out/src/tools/ingest-csv.js +0 -11
- package/out/src/tools/ingest-csv.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts +4 -0
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +16 -6
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/postcode-triples.d.ts +175 -0
- package/out/src/tools/postcode-triples.d.ts.map +1 -0
- package/out/src/tools/postcode-triples.js +304 -0
- package/out/src/tools/postcode-triples.js.map +1 -0
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +1 -2
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +4 -26
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/tools/sub-venue/harvest.d.ts +100 -0
- package/out/src/tools/sub-venue/harvest.d.ts.map +1 -0
- package/out/src/tools/sub-venue/harvest.js +168 -0
- package/out/src/tools/sub-venue/harvest.js.map +1 -0
- package/out/src/tools/sub-venue/head-nouns.d.ts +50 -0
- package/out/src/tools/sub-venue/head-nouns.d.ts.map +1 -0
- package/out/src/tools/sub-venue/head-nouns.js +210 -0
- package/out/src/tools/sub-venue/head-nouns.js.map +1 -0
- package/out/src/tools/sub-venue/surfaces.d.ts +45 -0
- package/out/src/tools/sub-venue/surfaces.d.ts.map +1 -0
- package/out/src/tools/sub-venue/surfaces.js +77 -0
- package/out/src/tools/sub-venue/surfaces.js.map +1 -0
- package/out/src/tools/sub-venue/table.d.ts +229 -0
- package/out/src/tools/sub-venue/table.d.ts.map +1 -0
- package/out/src/tools/sub-venue/table.js +108 -0
- package/out/src/tools/sub-venue/table.js.map +1 -0
- package/out/src/tools/sub-venue/wikidata.d.ts +26 -0
- package/out/src/tools/sub-venue/wikidata.d.ts.map +1 -0
- package/out/src/tools/sub-venue/wikidata.js +62 -0
- package/out/src/tools/sub-venue/wikidata.js.map +1 -0
- package/out/src/tools/sub-venue-lexicon.d.ts +27 -378
- package/out/src/tools/sub-venue-lexicon.d.ts.map +1 -1
- package/out/src/tools/sub-venue-lexicon.js +30 -565
- package/out/src/tools/sub-venue-lexicon.js.map +1 -1
- package/out/src/utils/align.d.ts +1 -1
- package/out/src/utils/align.d.ts.map +1 -1
- package/out/src/utils/align.js.map +1 -1
- package/out/src/utils/golden.js +1 -1
- package/out/src/utils/golden.js.map +1 -1
- package/out/src/utils/license.js +1 -1
- package/out/src/utils/license.js.map +1 -1
- package/out/src/utils/parquet.d.ts +24 -5
- package/out/src/utils/parquet.d.ts.map +1 -1
- package/out/src/utils/parquet.js +6 -2
- package/out/src/utils/parquet.js.map +1 -1
- package/out/src/utils/split.d.ts +1 -1
- package/out/src/utils/split.d.ts.map +1 -1
- package/out/src/utils/split.js +2 -2
- package/out/src/utils/split.js.map +1 -1
- package/package.json +230 -10
- package/src/adapters/ban/adapter.ts +6 -4
- package/src/adapters/fcc-bdc/adapter.ts +4 -3
- package/src/adapters/geonames/adapter.ts +3 -2
- package/src/adapters/geonames-postal/adapter.ts +4 -3
- package/src/adapters/gnaf/adapter.ts +3 -2
- package/src/adapters/index.ts +2 -2
- package/src/adapters/openaddresses/adapter.ts +4 -3
- package/src/adapters/overture/adapter.ts +3 -2
- package/src/adapters/state-hi-schools/adapter.ts +8 -6
- package/src/adapters/state-ia-contractors/adapter.ts +13 -7
- package/src/adapters/state-ny-notaries/adapter.ts +13 -31
- package/src/adapters/state-tx-notaries/adapter.ts +13 -7
- package/src/adapters/synth-po-box/adapter.ts +5 -4
- package/src/adapters/tiger/adapter.ts +3 -2
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +8 -6
- package/src/adapters/usgov-imls-pls/adapter.ts +12 -7
- package/src/adapters/usgov-irs-bmf/adapter.ts +7 -5
- package/src/adapters/usgov-nad/adapter.ts +9 -8
- package/src/adapters/usgov-nppes/adapter.ts +13 -11
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +8 -6
- package/src/adapters/utils/index.ts +16 -3
- package/src/adapters/wof-admin-jp/adapter.ts +9 -12
- package/src/adapters/wof-admin-json/adapter.ts +16 -54
- package/src/adapters/wof-json-rows.ts +79 -0
- package/src/adapters/wof-postalcode-json/adapter.ts +17 -50
- package/src/build.ts +10 -10
- package/src/index.ts +1 -1
- package/src/parquet-wrapper/reader.ts +9 -0
- package/src/runner.ts +2 -7
- package/src/shard-recipes/anchor-absorption.ts +5 -5
- package/src/shard-recipes/bare-country.ts +95 -0
- package/src/shard-recipes/boundary-stress.ts +2 -5
- package/src/shard-recipes/country-balanced.ts +8 -6
- package/src/shard-recipes/cz-pcfirst-preposition.ts +29 -10
- package/src/shard-recipes/fr-admin-split.ts +6 -4
- package/src/shard-recipes/fr-bare-street.ts +85 -10
- package/src/shard-recipes/fr-lieudit.ts +6 -4
- package/src/shard-recipes/fr-order.ts +6 -4
- package/src/shard-recipes/german.ts +4 -3
- package/src/shard-recipes/house-venue.ts +1 -1
- package/src/shard-recipes/index.ts +6 -0
- package/src/shard-recipes/intersection.ts +5 -4
- package/src/shard-recipes/locale.ts +6 -8
- package/src/shard-recipes/no-street.ts +1 -1
- package/src/shard-recipes/po-box-cedex.ts +4 -3
- package/src/shard-recipes/po-box.ts +1 -5
- package/src/shard-recipes/reviewed-postcode-tail.ts +189 -0
- package/src/shard-recipes/scaffold.ts +38 -7
- package/src/shard-recipes/street-affix.ts +4 -3
- package/src/shard-recipes/street-bare.ts +3 -3
- package/src/shard-recipes/street.ts +2 -2
- package/src/shard-recipes/sub-venue-sources.ts +4 -14
- package/src/shard-recipes/sub-venue.ts +10 -8
- package/src/shard-recipes/trailing-region.ts +183 -0
- package/src/shard-recipes/unit.ts +4 -3
- package/src/synthesizers/boundary-stress.ts +1 -1
- package/src/synthesizers/german.ts +2 -1
- package/src/synthesizers/house-venue.ts +17 -3
- package/src/synthesizers/intersection.ts +1 -1
- package/src/synthesizers/no-street.ts +1 -1
- package/src/synthesizers/po-box.ts +1 -1
- package/src/synthesizers/street.ts +5 -3
- package/src/synthesizers/utils.ts +5 -3
- package/src/tools/align-shard.ts +3 -2
- package/src/tools/fetch/ban.ts +2 -22
- package/src/tools/fetch/download.ts +22 -4
- package/src/tools/fetch/geonames-dump.ts +267 -0
- package/src/tools/fetch/geonames-postal.ts +180 -0
- package/src/tools/fetch/imls-pls.ts +2 -2
- package/src/tools/fetch/index.ts +11 -0
- package/src/tools/fetch/nad.ts +21 -10
- package/src/tools/fetch/nppes.ts +8 -6
- package/src/tools/fetch/openaddresses.ts +18 -22
- package/src/tools/fetch/ourairports.ts +7 -2
- package/src/tools/fetch/ppd.ts +1 -1
- package/src/tools/fetch/state-hi-schools.ts +3 -23
- package/src/tools/fetch/state-sources.ts +1 -1
- package/src/tools/fetch/tiger-full.ts +17 -24
- package/src/tools/golden-expand.ts +44 -33
- package/src/tools/golden-relabel-street.ts +2 -2
- package/src/tools/index.ts +2 -2
- package/src/tools/ingest-csv.ts +0 -13
- package/src/tools/overlay-manifest.ts +20 -8
- package/src/tools/postcode-triples.ts +375 -0
- package/src/tools/shard-kryptonite.ts +5 -4
- package/src/tools/shard-translit.ts +10 -35
- package/src/tools/sub-venue/harvest.ts +235 -0
- package/src/tools/sub-venue/head-nouns.ts +242 -0
- package/src/tools/sub-venue/surfaces.ts +95 -0
- package/src/tools/sub-venue/table.ts +281 -0
- package/src/tools/sub-venue/wikidata.ts +85 -0
- package/src/tools/sub-venue-lexicon.ts +67 -886
- package/src/utils/align.ts +2 -1
- package/src/utils/golden.ts +1 -1
- package/src/utils/license.ts +1 -1
- package/src/utils/parquet.ts +22 -8
- package/src/utils/split.ts +4 -3
|
@@ -22,16 +22,17 @@
|
|
|
22
22
|
|
|
23
23
|
import { tryParsingJSON } from "@mailwoman/core/objects"
|
|
24
24
|
import { makeLcg } from "@mailwoman/core/utils"
|
|
25
|
-
import {
|
|
25
|
+
import { TextSpliterator } from "spliterator"
|
|
26
|
+
|
|
27
|
+
import { stableSourceID } from "#adapters/utils"
|
|
26
28
|
import {
|
|
27
29
|
countryToLocale,
|
|
28
30
|
REGION_OPTIONAL_LOCALES,
|
|
29
31
|
synthesizeMilitaryPoBoxRow,
|
|
30
32
|
synthesizePoBoxRow,
|
|
31
33
|
type PoBoxBaseTuple,
|
|
32
|
-
} from "
|
|
33
|
-
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "
|
|
34
|
-
import { TextSpliterator } from "spliterator"
|
|
34
|
+
} from "#synthesizers/po-box"
|
|
35
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
|
|
35
36
|
|
|
36
37
|
/**
|
|
37
38
|
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
@@ -36,10 +36,11 @@
|
|
|
36
36
|
|
|
37
37
|
import { DatabaseSync } from "node:sqlite"
|
|
38
38
|
|
|
39
|
-
import { lookupFipsState } from "@mailwoman/corpus/codex/us-fips-state"
|
|
40
|
-
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
|
|
41
39
|
import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
|
|
42
40
|
|
|
41
|
+
import { lookupFipsState } from "#codex/us-fips-state"
|
|
42
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
|
|
43
|
+
|
|
43
44
|
import { decomposeStreet } from "./street-decompose.ts"
|
|
44
45
|
|
|
45
46
|
/**
|
|
@@ -29,12 +29,14 @@
|
|
|
29
29
|
* terms.
|
|
30
30
|
*/
|
|
31
31
|
|
|
32
|
-
import {
|
|
33
|
-
import { lookupStateAbbreviation } from "@mailwoman/corpus/codex/us-fips-state"
|
|
34
|
-
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
|
|
32
|
+
import { isPresent } from "@mailwoman/core/objects"
|
|
35
33
|
import { reconcileComponents } from "@mailwoman/formatter"
|
|
36
34
|
import { CSVSpliterator } from "spliterator"
|
|
37
35
|
|
|
36
|
+
import { splitStreetLine, stableSourceID } from "#adapters/utils"
|
|
37
|
+
import { lookupStateAbbreviation } from "#codex/us-fips-state"
|
|
38
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
|
|
39
|
+
|
|
38
40
|
/**
|
|
39
41
|
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
40
42
|
* it came from.
|
|
@@ -89,10 +91,10 @@ function composeRaw(
|
|
|
89
91
|
state: string,
|
|
90
92
|
postcode: string
|
|
91
93
|
): string {
|
|
92
|
-
const streetPart = [house, street].filter(
|
|
93
|
-
const cityPart = [city.trim(), [state, postcode].filter(
|
|
94
|
+
const streetPart = [house, street].filter(isPresent).join(" ").trim()
|
|
95
|
+
const cityPart = [city.trim(), [state, postcode].filter(isPresent).join(" ").trim()].filter(isPresent).join(", ")
|
|
94
96
|
|
|
95
|
-
return [venue.trim(), streetPart, cityPart].filter(
|
|
97
|
+
return [venue.trim(), streetPart, cityPart].filter(isPresent).join(", ")
|
|
96
98
|
}
|
|
97
99
|
|
|
98
100
|
export function createUsgovHrsaFqhcAdapter(): CorpusAdapter {
|
|
@@ -18,12 +18,14 @@
|
|
|
18
18
|
* License: stamped `"Public Domain"` per IMLS federal government distribution terms.
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
-
import {
|
|
22
|
-
import { lookupStateAbbreviation } from "@mailwoman/corpus/codex/us-fips-state"
|
|
23
|
-
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
|
|
21
|
+
import { isPresent } from "@mailwoman/core/objects"
|
|
24
22
|
import { reconcileComponents } from "@mailwoman/formatter"
|
|
25
23
|
import { CSVSpliterator } from "spliterator"
|
|
26
24
|
|
|
25
|
+
import { splitStreetLine, stableSourceID } from "#adapters/utils"
|
|
26
|
+
import { lookupStateAbbreviation } from "#codex/us-fips-state"
|
|
27
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
|
|
28
|
+
|
|
27
29
|
/**
|
|
28
30
|
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
29
31
|
* it came from.
|
|
@@ -74,7 +76,6 @@ export function createUsgovImlsPlsAdapter(): CorpusAdapter {
|
|
|
74
76
|
const city = (record.CITY ?? "").trim()
|
|
75
77
|
const zip = (record.ZIP ?? "").trim()
|
|
76
78
|
const stateAbbr = (record.STABR ?? "").trim()
|
|
77
|
-
const county = (record.CNTY ?? "").trim()
|
|
78
79
|
|
|
79
80
|
if (!libName || !city || !zip) continue
|
|
80
81
|
|
|
@@ -99,10 +100,14 @@ export function createUsgovImlsPlsAdapter(): CorpusAdapter {
|
|
|
99
100
|
// a postal-surface component here.
|
|
100
101
|
}
|
|
101
102
|
|
|
102
|
-
const streetPart = [split.house_number, split.street].filter(
|
|
103
|
+
const streetPart = [split.house_number, split.street].filter(isPresent).join(" ").trim()
|
|
103
104
|
|
|
104
|
-
const raw = [
|
|
105
|
-
|
|
105
|
+
const raw = [
|
|
106
|
+
libName,
|
|
107
|
+
streetPart,
|
|
108
|
+
[city, [stateAbbr, zip].filter(isPresent).join(" ")].filter(isPresent).join(", "),
|
|
109
|
+
]
|
|
110
|
+
.filter(isPresent)
|
|
106
111
|
.join(", ")
|
|
107
112
|
|
|
108
113
|
const aligned = reconcileComponents(components, raw)
|
|
@@ -19,11 +19,13 @@
|
|
|
19
19
|
* federal).
|
|
20
20
|
*/
|
|
21
21
|
|
|
22
|
-
import {
|
|
23
|
-
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
|
|
22
|
+
import { isPresent } from "@mailwoman/core/objects"
|
|
24
23
|
import { reconcileComponents } from "@mailwoman/formatter"
|
|
25
24
|
import { CSVSpliterator } from "spliterator"
|
|
26
25
|
|
|
26
|
+
import { splitStreetLine, stableSourceID } from "#adapters/utils"
|
|
27
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
|
|
28
|
+
|
|
27
29
|
/**
|
|
28
30
|
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
29
31
|
* it came from.
|
|
@@ -71,9 +73,9 @@ function composeRaw(
|
|
|
71
73
|
state: string,
|
|
72
74
|
postcode: string
|
|
73
75
|
): string {
|
|
74
|
-
const cityPart = [city.trim(), [state, postcode].filter(
|
|
76
|
+
const cityPart = [city.trim(), [state, postcode].filter(isPresent).join(" ").trim()].filter(isPresent).join(", ")
|
|
75
77
|
|
|
76
|
-
return [venue, streetPart, cityPart].filter(
|
|
78
|
+
return [venue, streetPart, cityPart].filter(isPresent).join(", ")
|
|
77
79
|
}
|
|
78
80
|
|
|
79
81
|
export function createUsgovIrsBmfAdapter(): CorpusAdapter {
|
|
@@ -116,7 +118,7 @@ export function createUsgovIrsBmfAdapter(): CorpusAdapter {
|
|
|
116
118
|
if (!split) continue
|
|
117
119
|
|
|
118
120
|
const streetPart =
|
|
119
|
-
"po_box" in split ? split.po_box : [split.house_number, split.street].filter(
|
|
121
|
+
"po_box" in split ? split.po_box : [split.house_number, split.street].filter(isPresent).join(" ")
|
|
120
122
|
|
|
121
123
|
const components: CanonicalRow["components"] = {
|
|
122
124
|
...(venue ? { venue } : {}),
|
|
@@ -33,11 +33,12 @@
|
|
|
33
33
|
import { readdir } from "node:fs/promises"
|
|
34
34
|
import { join } from "node:path"
|
|
35
35
|
|
|
36
|
-
import { tryParsingJSON } from "@mailwoman/core/objects"
|
|
37
|
-
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
|
|
36
|
+
import { isPresent, tryParsingJSON } from "@mailwoman/core/objects"
|
|
38
37
|
import { reconcileComponents } from "@mailwoman/formatter"
|
|
39
38
|
import { TextSpliterator } from "spliterator"
|
|
40
39
|
|
|
40
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
|
|
41
|
+
|
|
41
42
|
/**
|
|
42
43
|
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
43
44
|
* it came from.
|
|
@@ -175,7 +176,7 @@ function composeHouseNumber(r: NADRecord): string | undefined {
|
|
|
175
176
|
const pre = (r.AddNum_Pre ?? "").toString().trim()
|
|
176
177
|
const suf = (r.AddNum_Suf ?? "").toString().trim()
|
|
177
178
|
|
|
178
|
-
return [pre, num, suf].filter(
|
|
179
|
+
return [pre, num, suf].filter(isPresent).join(" ").trim() || undefined
|
|
179
180
|
}
|
|
180
181
|
|
|
181
182
|
interface DecomposedNADStreet {
|
|
@@ -194,9 +195,9 @@ function decomposeNADStreet(r: NADRecord): DecomposedNADStreet | undefined {
|
|
|
194
195
|
const preSep = (r.St_PreSep ?? "").toString().trim()
|
|
195
196
|
const posTyp = (r.St_PosTyp ?? "").toString().trim()
|
|
196
197
|
const posDir = (r.St_PosDir ?? "").toString().trim()
|
|
197
|
-
const prefix = [preDir, preTyp, preSep].filter(
|
|
198
|
-
const suffix = [posTyp, posDir].filter(
|
|
199
|
-
const full = [prefix, name, suffix].filter(
|
|
198
|
+
const prefix = [preDir, preTyp, preSep].filter(isPresent).join(" ") || undefined
|
|
199
|
+
const suffix = [posTyp, posDir].filter(isPresent).join(" ") || undefined
|
|
200
|
+
const full = [prefix, name, suffix].filter(isPresent).join(" ")
|
|
200
201
|
|
|
201
202
|
return { prefix, street: name, suffix, full }
|
|
202
203
|
}
|
|
@@ -230,10 +231,10 @@ function composeRaw(parts: {
|
|
|
230
231
|
region: string
|
|
231
232
|
postcode: string
|
|
232
233
|
}): string {
|
|
233
|
-
const streetLine = [parts.houseNumber, parts.street, parts.unit].filter(
|
|
234
|
+
const streetLine = [parts.houseNumber, parts.street, parts.unit].filter(isPresent).join(" ").trim()
|
|
234
235
|
const tail = `${parts.locality}, ${parts.region} ${parts.postcode}`
|
|
235
236
|
|
|
236
|
-
return [parts.venue, streetLine || undefined, tail].filter(
|
|
237
|
+
return [parts.venue, streetLine || undefined, tail].filter(isPresent).join(", ")
|
|
237
238
|
}
|
|
238
239
|
|
|
239
240
|
export function createUsgovNADAdapter(): CorpusAdapter {
|
|
@@ -21,12 +21,14 @@
|
|
|
21
21
|
* License: stamped `"Public Domain"` per CMS's federal government distribution terms.
|
|
22
22
|
*/
|
|
23
23
|
|
|
24
|
-
import {
|
|
25
|
-
import { lookupStateAbbreviation } from "@mailwoman/corpus/codex/us-fips-state"
|
|
26
|
-
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
|
|
24
|
+
import { isPresent } from "@mailwoman/core/objects"
|
|
27
25
|
import { reconcileComponents } from "@mailwoman/formatter"
|
|
28
26
|
import { CSVSpliterator } from "spliterator"
|
|
29
27
|
|
|
28
|
+
import { splitStreetLine, stableSourceID } from "#adapters/utils"
|
|
29
|
+
import { lookupStateAbbreviation } from "#codex/us-fips-state"
|
|
30
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
|
|
31
|
+
|
|
30
32
|
/**
|
|
31
33
|
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
32
34
|
* it came from.
|
|
@@ -48,7 +50,7 @@ interface NPPESRow {
|
|
|
48
50
|
"Provider Second Line Business Practice Location Address": string
|
|
49
51
|
"Provider Business Practice Location Address City Name": string
|
|
50
52
|
"Provider Business Practice Location Address State Name": string
|
|
51
|
-
"Provider Business Practice Location Address
|
|
53
|
+
"Provider Business Practice Location Address Postcode": string
|
|
52
54
|
}
|
|
53
55
|
|
|
54
56
|
function composeRaw(
|
|
@@ -59,10 +61,11 @@ function composeRaw(
|
|
|
59
61
|
state: string,
|
|
60
62
|
postcode: string
|
|
61
63
|
): string {
|
|
62
|
-
const streetPart = [house, street].filter(
|
|
63
|
-
|
|
64
|
+
const streetPart = [house, street].filter(isPresent).join(" ").trim()
|
|
65
|
+
|
|
66
|
+
const cityPart = [city.trim(), [state, postcode].filter(isPresent).join(" ").trim()].filter(isPresent).join(", ")
|
|
64
67
|
|
|
65
|
-
return [venue, streetPart, cityPart].filter(
|
|
68
|
+
return [venue, streetPart, cityPart].filter(isPresent).join(", ")
|
|
66
69
|
}
|
|
67
70
|
|
|
68
71
|
export function createUsgovNPPESAdapter(): CorpusAdapter {
|
|
@@ -91,7 +94,6 @@ export function createUsgovNPPESAdapter(): CorpusAdapter {
|
|
|
91
94
|
if (opts.limit !== undefined && emitted >= opts.limit) break
|
|
92
95
|
|
|
93
96
|
const npi = (record.NPI ?? "").trim()
|
|
94
|
-
const entityType = (record["Entity Type Code"] ?? "").trim()
|
|
95
97
|
const orgName = (record["Provider Organization Name (Legal Business Name)"] ?? "").trim()
|
|
96
98
|
const lastName = (record["Provider Last Name (Legal Name)"] ?? "").trim()
|
|
97
99
|
const firstName = (record["Provider First Name"] ?? "").trim()
|
|
@@ -100,7 +102,7 @@ export function createUsgovNPPESAdapter(): CorpusAdapter {
|
|
|
100
102
|
const address2 = (record["Provider Second Line Business Practice Location Address"] ?? "").trim()
|
|
101
103
|
const city = (record["Provider Business Practice Location Address City Name"] ?? "").trim()
|
|
102
104
|
const stateRaw = (record["Provider Business Practice Location Address State Name"] ?? "").trim()
|
|
103
|
-
const postcode = (record["Provider Business Practice Location Address
|
|
105
|
+
const postcode = (record["Provider Business Practice Location Address Postcode"] ?? "").trim()
|
|
104
106
|
|
|
105
107
|
if (!city || !postcode) continue
|
|
106
108
|
|
|
@@ -108,12 +110,12 @@ export function createUsgovNPPESAdapter(): CorpusAdapter {
|
|
|
108
110
|
|
|
109
111
|
if (!state) continue
|
|
110
112
|
|
|
111
|
-
const fullStreet = [address1, address2].filter(
|
|
113
|
+
const fullStreet = [address1, address2].filter(isPresent).join(" ")
|
|
112
114
|
const split = splitStreetLine(fullStreet)
|
|
113
115
|
|
|
114
116
|
if (!split) continue
|
|
115
117
|
|
|
116
|
-
const venue = orgName || [firstName, lastName].filter(
|
|
118
|
+
const venue = orgName || [firstName, lastName].filter(isPresent).join(" ") || undefined
|
|
117
119
|
|
|
118
120
|
const components: CanonicalRow["components"] = {
|
|
119
121
|
...(venue ? { venue } : {}),
|
|
@@ -28,12 +28,14 @@
|
|
|
28
28
|
* distribution terms.
|
|
29
29
|
*/
|
|
30
30
|
|
|
31
|
-
import {
|
|
32
|
-
import { lookupStateAbbreviation } from "@mailwoman/corpus/codex/us-fips-state"
|
|
33
|
-
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
|
|
31
|
+
import { isPresent } from "@mailwoman/core/objects"
|
|
34
32
|
import { reconcileComponents } from "@mailwoman/formatter"
|
|
35
33
|
import { CSVSpliterator } from "spliterator"
|
|
36
34
|
|
|
35
|
+
import { splitStreetLine, stableSourceID } from "#adapters/utils"
|
|
36
|
+
import { lookupStateAbbreviation } from "#codex/us-fips-state"
|
|
37
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
|
|
38
|
+
|
|
37
39
|
/**
|
|
38
40
|
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
39
41
|
* it came from.
|
|
@@ -114,10 +116,10 @@ function composeRaw(
|
|
|
114
116
|
state: string,
|
|
115
117
|
postcode: string
|
|
116
118
|
): string {
|
|
117
|
-
const streetPart = [house, street].filter(
|
|
118
|
-
const cityPart = [city.trim(), [state, postcode].filter(
|
|
119
|
+
const streetPart = [house, street].filter(isPresent).join(" ").trim()
|
|
120
|
+
const cityPart = [city.trim(), [state, postcode].filter(isPresent).join(" ").trim()].filter(isPresent).join(", ")
|
|
119
121
|
|
|
120
|
-
return [venue.trim(), streetPart, cityPart].filter(
|
|
122
|
+
return [venue.trim(), streetPart, cityPart].filter(isPresent).join(", ")
|
|
121
123
|
}
|
|
122
124
|
|
|
123
125
|
export function createUsgovSamhsaTreatmentLocatorAdapter(): CorpusAdapter {
|
|
@@ -26,7 +26,8 @@ import { createHash, type Hash } from "node:crypto"
|
|
|
26
26
|
|
|
27
27
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
28
28
|
import { sha256Hex } from "@mailwoman/core/utils"
|
|
29
|
-
|
|
29
|
+
|
|
30
|
+
import type { CanonicalRow, CorpusAdapter } from "#types"
|
|
30
31
|
|
|
31
32
|
/**
|
|
32
33
|
* Lookup table for corpus adapters.
|
|
@@ -103,8 +104,20 @@ export const defaultAdapterRegistry = new InMemoryAdapterRegistry()
|
|
|
103
104
|
* length.
|
|
104
105
|
*/
|
|
105
106
|
export function stableSourceID(adapterID: string, components: Partial<Record<ComponentTag, string>>): string {
|
|
106
|
-
|
|
107
|
-
|
|
107
|
+
return stableSourceIDFromParts(adapterID, components)
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* {@link stableSourceID} over arbitrary disambiguator keys — a variant index, a slot number, anything that is not a
|
|
112
|
+
* `ComponentTag`. Every key handed in is sorted and hashed either way; only the key vocabulary differs, and the narrow
|
|
113
|
+
* signature above is what stops an adapter hashing a misspelled component name.
|
|
114
|
+
*/
|
|
115
|
+
export function stableSourceIDFromParts(
|
|
116
|
+
adapterID: string,
|
|
117
|
+
parts: Readonly<Record<string, string | undefined>>
|
|
118
|
+
): string {
|
|
119
|
+
const sortedKeys = Object.keys(parts).toSorted()
|
|
120
|
+
const payload = sortedKeys.map((k) => `${k}=${parts[k] ?? ""}`).join("\u001F")
|
|
108
121
|
// One `update` over the joined string, not three chained ones — identical byte stream either way,
|
|
109
122
|
// but the separator has to stay inline or every id in every shard changes.
|
|
110
123
|
const digest = sha256Hex(`${adapterID}\u001E${payload}`)
|
|
@@ -31,7 +31,10 @@
|
|
|
31
31
|
|
|
32
32
|
import { DatabaseSync } from "node:sqlite"
|
|
33
33
|
|
|
34
|
-
import
|
|
34
|
+
import { isPresent } from "@mailwoman/core/objects"
|
|
35
|
+
import { allRows, getRow } from "@mailwoman/core/utils"
|
|
36
|
+
|
|
37
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
|
|
35
38
|
|
|
36
39
|
/**
|
|
37
40
|
* Ancestry hops walked upward from a WOF record before giving up — deeper than any real JP admin chain.
|
|
@@ -52,12 +55,6 @@ interface PlaceRow {
|
|
|
52
55
|
country: string
|
|
53
56
|
}
|
|
54
57
|
|
|
55
|
-
interface NameRow {
|
|
56
|
-
id: number
|
|
57
|
-
name: string
|
|
58
|
-
language: string
|
|
59
|
-
}
|
|
60
|
-
|
|
61
58
|
/**
|
|
62
59
|
* Walk parent chain up to 6 levels, over a place table already resident in memory.
|
|
63
60
|
*
|
|
@@ -125,7 +122,7 @@ export function synthesizeJpAddress(
|
|
|
125
122
|
components.dependent_locality = neighbourhoodName
|
|
126
123
|
}
|
|
127
124
|
|
|
128
|
-
const raw = [regionName, localityName, neighbourhoodName].filter(
|
|
125
|
+
const raw = [regionName, localityName, neighbourhoodName].filter(isPresent).join("")
|
|
129
126
|
|
|
130
127
|
return { raw, components }
|
|
131
128
|
}
|
|
@@ -161,16 +158,16 @@ export function createWOFAdminJpAdapter(): CorpusAdapter {
|
|
|
161
158
|
// seed; there are tens of thousands of neighbourhood seeds.
|
|
162
159
|
const byID = new Map<number, PlaceRow>()
|
|
163
160
|
|
|
164
|
-
for (const row of
|
|
165
|
-
.prepare(`SELECT id, name, placetype, parent_id, country FROM spr WHERE country='JP'`)
|
|
166
|
-
|
|
161
|
+
for (const row of allRows<PlaceRow>(
|
|
162
|
+
db.prepare(`SELECT id, name, placetype, parent_id, country FROM spr WHERE country='JP'`)
|
|
163
|
+
)) {
|
|
167
164
|
byID.set(row.id, row)
|
|
168
165
|
}
|
|
169
166
|
|
|
170
167
|
const outsideStmt = db.prepare(`SELECT id, name, placetype, parent_id, country FROM spr WHERE id = ?`)
|
|
171
168
|
|
|
172
169
|
const resolveOutside = (id: number): PlaceRow | undefined => {
|
|
173
|
-
const row = outsideStmt
|
|
170
|
+
const row = getRow<PlaceRow>(outsideStmt, id)
|
|
174
171
|
|
|
175
172
|
if (row) {
|
|
176
173
|
byID.set(id, row)
|
|
@@ -41,9 +41,11 @@
|
|
|
41
41
|
|
|
42
42
|
import type { WhosOnFirstPlacetype } from "@mailwoman/core/resources/whosonfirst"
|
|
43
43
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
44
|
-
|
|
45
|
-
import {
|
|
46
|
-
import {
|
|
44
|
+
|
|
45
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
|
|
46
|
+
import { buildAncestryIndex, normalizeNameKey, walkFeatures, type WOFRecord } from "#utils"
|
|
47
|
+
|
|
48
|
+
import { emitWOFJSONRows, type WOFVariantSpec } from "../wof-json-rows.ts"
|
|
47
49
|
|
|
48
50
|
/**
|
|
49
51
|
* Display name for the country, keyed by ISO 3166-1 alpha-2.
|
|
@@ -95,18 +97,6 @@ function placetypeToTag(placetype: WhosOnFirstPlacetype | string): ComponentTag
|
|
|
95
97
|
}
|
|
96
98
|
}
|
|
97
99
|
|
|
98
|
-
interface VariantSpec {
|
|
99
|
-
/**
|
|
100
|
-
* Hierarchy-variant id appended to `source_id`.
|
|
101
|
-
*/
|
|
102
|
-
suffix: string
|
|
103
|
-
|
|
104
|
-
/**
|
|
105
|
-
* Component tag → display string the adapter will hand to the runner.
|
|
106
|
-
*/
|
|
107
|
-
components: Partial<Record<ComponentTag, string>>
|
|
108
|
-
}
|
|
109
|
-
|
|
110
100
|
/**
|
|
111
101
|
* Compute the hierarchy variants for a record given its ancestry chain and the chosen `selfName`.
|
|
112
102
|
*
|
|
@@ -117,7 +107,7 @@ interface VariantSpec {
|
|
|
117
107
|
* Country variants substitute `COUNTRY_DISPLAY_NAME` for the default slot so the OpenCage template produces the
|
|
118
108
|
* canonicalized form (`"United States of America"`), matching the legacy SQLite adapter's behavior.
|
|
119
109
|
*/
|
|
120
|
-
export function variantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: string):
|
|
110
|
+
export function variantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: string): WOFVariantSpec[] {
|
|
121
111
|
const selfTag = placetypeToTag(row.placetype)
|
|
122
112
|
|
|
123
113
|
if (!selfTag) return []
|
|
@@ -126,7 +116,7 @@ export function variantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: str
|
|
|
126
116
|
const country = ancestry.find((a) => placetypeToTag(a.placetype) === "country")
|
|
127
117
|
const countryDisplay = COUNTRY_DISPLAY_NAME[row.country] ?? country?.name ?? row.country
|
|
128
118
|
|
|
129
|
-
const variants:
|
|
119
|
+
const variants: WOFVariantSpec[] = []
|
|
130
120
|
|
|
131
121
|
switch (selfTag) {
|
|
132
122
|
case "locality":
|
|
@@ -247,43 +237,15 @@ export function createWOFAdminAdapter(): CorpusAdapter {
|
|
|
247
237
|
const ancestry = buildAncestryIndex(byID)
|
|
248
238
|
|
|
249
239
|
// Pass 2: emit rows in sorted-id order for deterministic JSONL.
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
for (const slot of slots) {
|
|
260
|
-
const variants = variantsFor(rec, chain, slot.value)
|
|
261
|
-
|
|
262
|
-
for (const variant of variants) {
|
|
263
|
-
if (opts.limit !== undefined && emitted >= opts.limit) return
|
|
264
|
-
|
|
265
|
-
const raw = formatAddress(variant.components, rec.country, { separator: ", " })
|
|
266
|
-
|
|
267
|
-
if (!raw) continue
|
|
268
|
-
const aligned = reconcileComponents(variant.components, raw)
|
|
269
|
-
|
|
270
|
-
if (!Object.keys(aligned).length) continue
|
|
271
|
-
|
|
272
|
-
yield {
|
|
273
|
-
raw,
|
|
274
|
-
components: aligned,
|
|
275
|
-
country: rec.country,
|
|
276
|
-
locale: LOCALE_BY_COUNTRY[rec.country],
|
|
277
|
-
source: WOF_ADMIN_ADAPTER_ID,
|
|
278
|
-
source_id: `${WOF_ADMIN_ADAPTER_ID}-${rec.id}-${slot.key}-${variant.suffix}`,
|
|
279
|
-
corpus_version: "",
|
|
280
|
-
license: "CC0-1.0",
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
emitted++
|
|
284
|
-
}
|
|
285
|
-
}
|
|
286
|
-
}
|
|
240
|
+
yield* emitWOFJSONRows({
|
|
241
|
+
records: byID,
|
|
242
|
+
ancestry,
|
|
243
|
+
adapterOptions: opts,
|
|
244
|
+
adapterID: WOF_ADMIN_ADAPTER_ID,
|
|
245
|
+
localeByCountry: LOCALE_BY_COUNTRY,
|
|
246
|
+
nameSlotsFor,
|
|
247
|
+
variantsFor,
|
|
248
|
+
})
|
|
287
249
|
},
|
|
288
250
|
}
|
|
289
251
|
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
* @file Deterministic canonical-row emission shared by WOF GeoJSON adapters.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type { ComponentTag } from "@mailwoman/core/types"
|
|
9
|
+
import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
|
|
10
|
+
|
|
11
|
+
import type { AdapterOptions, CanonicalRow } from "#types"
|
|
12
|
+
import type { WOFRecord } from "#utils"
|
|
13
|
+
|
|
14
|
+
export interface WOFVariantSpec {
|
|
15
|
+
suffix: string
|
|
16
|
+
components: Partial<Record<ComponentTag, string>>
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
interface EmitWOFJSONRowsOptions {
|
|
20
|
+
records: ReadonlyMap<number, WOFRecord>
|
|
21
|
+
ancestry: ReadonlyMap<number, WOFRecord[]>
|
|
22
|
+
adapterOptions: AdapterOptions
|
|
23
|
+
adapterID: string
|
|
24
|
+
localeByCountry: Readonly<Record<string, string>>
|
|
25
|
+
shouldEmit?: (record: WOFRecord) => boolean
|
|
26
|
+
nameSlotsFor: (record: WOFRecord) => Array<{ key: string; value: string }>
|
|
27
|
+
variantsFor: (record: WOFRecord, ancestry: WOFRecord[], selfName: string) => WOFVariantSpec[]
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Emit aligned rows in WOF-id order, enforcing the adapter limit across all name and hierarchy variants.
|
|
32
|
+
*/
|
|
33
|
+
export function* emitWOFJSONRows(options: EmitWOFJSONRowsOptions): Generator<CanonicalRow> {
|
|
34
|
+
const {
|
|
35
|
+
records,
|
|
36
|
+
ancestry,
|
|
37
|
+
adapterOptions,
|
|
38
|
+
adapterID,
|
|
39
|
+
localeByCountry,
|
|
40
|
+
shouldEmit = () => true,
|
|
41
|
+
nameSlotsFor,
|
|
42
|
+
variantsFor,
|
|
43
|
+
} = options
|
|
44
|
+
|
|
45
|
+
let emitted = 0
|
|
46
|
+
|
|
47
|
+
for (const id of [...records.keys()].toSorted((a, b) => a - b)) {
|
|
48
|
+
if (adapterOptions.signal?.aborted) return
|
|
49
|
+
const record = records.get(id)!
|
|
50
|
+
|
|
51
|
+
if (!shouldEmit(record)) continue
|
|
52
|
+
|
|
53
|
+
for (const slot of nameSlotsFor(record)) {
|
|
54
|
+
for (const variant of variantsFor(record, ancestry.get(id) ?? [], slot.value)) {
|
|
55
|
+
if (adapterOptions.limit !== undefined && emitted >= adapterOptions.limit) return
|
|
56
|
+
|
|
57
|
+
const raw = formatAddress(variant.components, record.country, { separator: ", " })
|
|
58
|
+
|
|
59
|
+
if (!raw) continue
|
|
60
|
+
const components = reconcileComponents(variant.components, raw)
|
|
61
|
+
|
|
62
|
+
if (!Object.keys(components).length) continue
|
|
63
|
+
|
|
64
|
+
yield {
|
|
65
|
+
raw,
|
|
66
|
+
components,
|
|
67
|
+
country: record.country,
|
|
68
|
+
locale: localeByCountry[record.country],
|
|
69
|
+
source: adapterID,
|
|
70
|
+
source_id: `${adapterID}-${record.id}-${slot.key}-${variant.suffix}`,
|
|
71
|
+
corpus_version: "",
|
|
72
|
+
license: "CC0-1.0",
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
emitted++
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|