@mailwoman/corpus 9.1.0 → 9.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/data/PROVENANCE.md +13 -0
- package/data/reviewed-ve-postcode-tuples.json +74 -0
- package/out/src/adapters/ban/adapter.d.ts +1 -1
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +4 -3
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +2 -2
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +1 -1
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +1 -1
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +2 -2
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +1 -1
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/index.d.ts +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +2 -2
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +1 -1
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +1 -1
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +6 -5
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +11 -6
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +11 -17
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +11 -6
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +2 -2
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +2 -2
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +1 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +1 -1
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +6 -5
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +10 -6
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +5 -4
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +7 -7
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +9 -9
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +6 -5
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/utils/index.d.ts +7 -1
- package/out/src/adapters/utils/index.d.ts.map +1 -1
- package/out/src/adapters/utils/index.js +10 -2
- package/out/src/adapters/utils/index.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +5 -5
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +4 -15
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +11 -35
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-json-rows.d.ts +32 -0
- package/out/src/adapters/wof-json-rows.d.ts.map +1 -0
- package/out/src/adapters/wof-json-rows.js +45 -0
- package/out/src/adapters/wof-json-rows.js.map +1 -0
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +4 -9
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +12 -37
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/build.d.ts +4 -3
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +7 -4
- package/out/src/build.js.map +1 -1
- package/out/src/index.d.ts +1 -1
- package/out/src/index.d.ts.map +1 -1
- package/out/src/index.js +1 -1
- package/out/src/index.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +5 -0
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/runner.d.ts +2 -2
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +1 -1
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +4 -5
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/bare-country.d.ts +25 -0
- package/out/src/shard-recipes/bare-country.d.ts.map +1 -0
- package/out/src/shard-recipes/bare-country.js +87 -0
- package/out/src/shard-recipes/bare-country.js.map +1 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +2 -2
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +6 -5
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +28 -10
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +4 -3
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.js +72 -11
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +4 -3
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +4 -3
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +3 -3
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.js +1 -1
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +6 -0
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +3 -3
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +1 -1
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +5 -4
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/no-street.js +1 -1
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +2 -2
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +1 -1
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/reviewed-postcode-tail.d.ts +47 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.d.ts.map +1 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.js +125 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.js.map +1 -0
- package/out/src/shard-recipes/scaffold.d.ts +33 -4
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +5 -6
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +2 -2
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.js +3 -3
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.js +2 -2
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.d.ts +2 -2
- package/out/src/shard-recipes/sub-venue-sources.d.ts.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.js +1 -1
- package/out/src/shard-recipes/sub-venue-sources.js.map +1 -1
- package/out/src/shard-recipes/sub-venue.d.ts +1 -1
- package/out/src/shard-recipes/sub-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/sub-venue.js +6 -5
- package/out/src/shard-recipes/sub-venue.js.map +1 -1
- package/out/src/shard-recipes/trailing-region.d.ts +74 -0
- package/out/src/shard-recipes/trailing-region.d.ts.map +1 -0
- package/out/src/shard-recipes/trailing-region.js +157 -0
- package/out/src/shard-recipes/trailing-region.js.map +1 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +2 -2
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/synthesizers/boundary-stress.d.ts +1 -1
- package/out/src/synthesizers/boundary-stress.d.ts.map +1 -1
- package/out/src/synthesizers/german.d.ts +1 -1
- package/out/src/synthesizers/german.d.ts.map +1 -1
- package/out/src/synthesizers/german.js.map +1 -1
- package/out/src/synthesizers/house-venue.d.ts +1 -1
- package/out/src/synthesizers/house-venue.d.ts.map +1 -1
- package/out/src/synthesizers/house-venue.js +16 -2
- package/out/src/synthesizers/house-venue.js.map +1 -1
- package/out/src/synthesizers/intersection.d.ts +1 -1
- package/out/src/synthesizers/intersection.d.ts.map +1 -1
- package/out/src/synthesizers/no-street.d.ts +1 -1
- package/out/src/synthesizers/no-street.d.ts.map +1 -1
- package/out/src/synthesizers/po-box.d.ts +1 -1
- package/out/src/synthesizers/po-box.d.ts.map +1 -1
- package/out/src/synthesizers/street.d.ts +1 -1
- package/out/src/synthesizers/street.d.ts.map +1 -1
- package/out/src/synthesizers/street.js +3 -2
- package/out/src/synthesizers/street.js.map +1 -1
- package/out/src/synthesizers/utils.d.ts +1 -1
- package/out/src/synthesizers/utils.d.ts.map +1 -1
- package/out/src/synthesizers/utils.js +3 -2
- package/out/src/synthesizers/utils.js.map +1 -1
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +2 -2
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/fetch/ban.d.ts.map +1 -1
- package/out/src/tools/fetch/ban.js +2 -18
- package/out/src/tools/fetch/ban.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts +12 -3
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +20 -4
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/geonames-dump.d.ts +87 -0
- package/out/src/tools/fetch/geonames-dump.d.ts.map +1 -0
- package/out/src/tools/fetch/geonames-dump.js +178 -0
- package/out/src/tools/fetch/geonames-dump.js.map +1 -0
- package/out/src/tools/fetch/geonames-postal.d.ts +76 -0
- package/out/src/tools/fetch/geonames-postal.d.ts.map +1 -0
- package/out/src/tools/fetch/geonames-postal.js +125 -0
- package/out/src/tools/fetch/geonames-postal.js.map +1 -0
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +2 -2
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +11 -0
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +11 -0
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts.map +1 -1
- package/out/src/tools/fetch/nad.js +20 -11
- package/out/src/tools/fetch/nad.js.map +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +9 -7
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/src/tools/fetch/openaddresses.js +18 -17
- package/out/src/tools/fetch/openaddresses.js.map +1 -1
- package/out/src/tools/fetch/ourairports.d.ts.map +1 -1
- package/out/src/tools/fetch/ourairports.js +7 -2
- package/out/src/tools/fetch/ourairports.js.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +0 -4
- package/out/src/tools/fetch/ppd.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.js +1 -1
- package/out/src/tools/fetch/ppd.js.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.js +3 -19
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
- package/out/src/tools/fetch/state-sources.d.ts +4 -0
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
- package/out/src/tools/fetch/state-sources.js +1 -5
- package/out/src/tools/fetch/state-sources.js.map +1 -1
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
- package/out/src/tools/fetch/tiger-full.js +15 -21
- package/out/src/tools/fetch/tiger-full.js.map +1 -1
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +34 -25
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-relabel-street.js +2 -2
- package/out/src/tools/golden-relabel-street.js.map +1 -1
- package/out/src/tools/index.d.ts +2 -2
- package/out/src/tools/index.d.ts.map +1 -1
- package/out/src/tools/index.js +2 -2
- package/out/src/tools/index.js.map +1 -1
- package/out/src/tools/ingest-csv.d.ts.map +1 -1
- package/out/src/tools/ingest-csv.js +0 -11
- package/out/src/tools/ingest-csv.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts +4 -0
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +16 -6
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/postcode-triples.d.ts +175 -0
- package/out/src/tools/postcode-triples.d.ts.map +1 -0
- package/out/src/tools/postcode-triples.js +304 -0
- package/out/src/tools/postcode-triples.js.map +1 -0
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +1 -2
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +4 -26
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/tools/sub-venue/harvest.d.ts +100 -0
- package/out/src/tools/sub-venue/harvest.d.ts.map +1 -0
- package/out/src/tools/sub-venue/harvest.js +168 -0
- package/out/src/tools/sub-venue/harvest.js.map +1 -0
- package/out/src/tools/sub-venue/head-nouns.d.ts +50 -0
- package/out/src/tools/sub-venue/head-nouns.d.ts.map +1 -0
- package/out/src/tools/sub-venue/head-nouns.js +210 -0
- package/out/src/tools/sub-venue/head-nouns.js.map +1 -0
- package/out/src/tools/sub-venue/surfaces.d.ts +45 -0
- package/out/src/tools/sub-venue/surfaces.d.ts.map +1 -0
- package/out/src/tools/sub-venue/surfaces.js +77 -0
- package/out/src/tools/sub-venue/surfaces.js.map +1 -0
- package/out/src/tools/sub-venue/table.d.ts +229 -0
- package/out/src/tools/sub-venue/table.d.ts.map +1 -0
- package/out/src/tools/sub-venue/table.js +108 -0
- package/out/src/tools/sub-venue/table.js.map +1 -0
- package/out/src/tools/sub-venue/wikidata.d.ts +26 -0
- package/out/src/tools/sub-venue/wikidata.d.ts.map +1 -0
- package/out/src/tools/sub-venue/wikidata.js +62 -0
- package/out/src/tools/sub-venue/wikidata.js.map +1 -0
- package/out/src/tools/sub-venue-lexicon.d.ts +27 -378
- package/out/src/tools/sub-venue-lexicon.d.ts.map +1 -1
- package/out/src/tools/sub-venue-lexicon.js +30 -565
- package/out/src/tools/sub-venue-lexicon.js.map +1 -1
- package/out/src/utils/align.d.ts +1 -1
- package/out/src/utils/align.d.ts.map +1 -1
- package/out/src/utils/align.js.map +1 -1
- package/out/src/utils/golden.js +1 -1
- package/out/src/utils/golden.js.map +1 -1
- package/out/src/utils/license.js +1 -1
- package/out/src/utils/license.js.map +1 -1
- package/out/src/utils/parquet.d.ts +24 -5
- package/out/src/utils/parquet.d.ts.map +1 -1
- package/out/src/utils/parquet.js +6 -2
- package/out/src/utils/parquet.js.map +1 -1
- package/out/src/utils/split.d.ts +1 -1
- package/out/src/utils/split.d.ts.map +1 -1
- package/out/src/utils/split.js +2 -2
- package/out/src/utils/split.js.map +1 -1
- package/package.json +230 -10
- package/src/adapters/ban/adapter.ts +6 -4
- package/src/adapters/fcc-bdc/adapter.ts +4 -3
- package/src/adapters/geonames/adapter.ts +3 -2
- package/src/adapters/geonames-postal/adapter.ts +4 -3
- package/src/adapters/gnaf/adapter.ts +3 -2
- package/src/adapters/index.ts +2 -2
- package/src/adapters/openaddresses/adapter.ts +4 -3
- package/src/adapters/overture/adapter.ts +3 -2
- package/src/adapters/state-hi-schools/adapter.ts +8 -6
- package/src/adapters/state-ia-contractors/adapter.ts +13 -7
- package/src/adapters/state-ny-notaries/adapter.ts +13 -31
- package/src/adapters/state-tx-notaries/adapter.ts +13 -7
- package/src/adapters/synth-po-box/adapter.ts +5 -4
- package/src/adapters/tiger/adapter.ts +3 -2
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +8 -6
- package/src/adapters/usgov-imls-pls/adapter.ts +12 -7
- package/src/adapters/usgov-irs-bmf/adapter.ts +7 -5
- package/src/adapters/usgov-nad/adapter.ts +9 -8
- package/src/adapters/usgov-nppes/adapter.ts +13 -11
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +8 -6
- package/src/adapters/utils/index.ts +16 -3
- package/src/adapters/wof-admin-jp/adapter.ts +9 -12
- package/src/adapters/wof-admin-json/adapter.ts +16 -54
- package/src/adapters/wof-json-rows.ts +79 -0
- package/src/adapters/wof-postalcode-json/adapter.ts +17 -50
- package/src/build.ts +10 -10
- package/src/index.ts +1 -1
- package/src/parquet-wrapper/reader.ts +9 -0
- package/src/runner.ts +2 -7
- package/src/shard-recipes/anchor-absorption.ts +5 -5
- package/src/shard-recipes/bare-country.ts +95 -0
- package/src/shard-recipes/boundary-stress.ts +2 -5
- package/src/shard-recipes/country-balanced.ts +8 -6
- package/src/shard-recipes/cz-pcfirst-preposition.ts +29 -10
- package/src/shard-recipes/fr-admin-split.ts +6 -4
- package/src/shard-recipes/fr-bare-street.ts +85 -10
- package/src/shard-recipes/fr-lieudit.ts +6 -4
- package/src/shard-recipes/fr-order.ts +6 -4
- package/src/shard-recipes/german.ts +4 -3
- package/src/shard-recipes/house-venue.ts +1 -1
- package/src/shard-recipes/index.ts +6 -0
- package/src/shard-recipes/intersection.ts +5 -4
- package/src/shard-recipes/locale.ts +6 -8
- package/src/shard-recipes/no-street.ts +1 -1
- package/src/shard-recipes/po-box-cedex.ts +4 -3
- package/src/shard-recipes/po-box.ts +1 -5
- package/src/shard-recipes/reviewed-postcode-tail.ts +189 -0
- package/src/shard-recipes/scaffold.ts +38 -7
- package/src/shard-recipes/street-affix.ts +4 -3
- package/src/shard-recipes/street-bare.ts +3 -3
- package/src/shard-recipes/street.ts +2 -2
- package/src/shard-recipes/sub-venue-sources.ts +4 -14
- package/src/shard-recipes/sub-venue.ts +10 -8
- package/src/shard-recipes/trailing-region.ts +183 -0
- package/src/shard-recipes/unit.ts +4 -3
- package/src/synthesizers/boundary-stress.ts +1 -1
- package/src/synthesizers/german.ts +2 -1
- package/src/synthesizers/house-venue.ts +17 -3
- package/src/synthesizers/intersection.ts +1 -1
- package/src/synthesizers/no-street.ts +1 -1
- package/src/synthesizers/po-box.ts +1 -1
- package/src/synthesizers/street.ts +5 -3
- package/src/synthesizers/utils.ts +5 -3
- package/src/tools/align-shard.ts +3 -2
- package/src/tools/fetch/ban.ts +2 -22
- package/src/tools/fetch/download.ts +22 -4
- package/src/tools/fetch/geonames-dump.ts +267 -0
- package/src/tools/fetch/geonames-postal.ts +180 -0
- package/src/tools/fetch/imls-pls.ts +2 -2
- package/src/tools/fetch/index.ts +11 -0
- package/src/tools/fetch/nad.ts +21 -10
- package/src/tools/fetch/nppes.ts +8 -6
- package/src/tools/fetch/openaddresses.ts +18 -22
- package/src/tools/fetch/ourairports.ts +7 -2
- package/src/tools/fetch/ppd.ts +1 -1
- package/src/tools/fetch/state-hi-schools.ts +3 -23
- package/src/tools/fetch/state-sources.ts +1 -1
- package/src/tools/fetch/tiger-full.ts +17 -24
- package/src/tools/golden-expand.ts +44 -33
- package/src/tools/golden-relabel-street.ts +2 -2
- package/src/tools/index.ts +2 -2
- package/src/tools/ingest-csv.ts +0 -13
- package/src/tools/overlay-manifest.ts +20 -8
- package/src/tools/postcode-triples.ts +375 -0
- package/src/tools/shard-kryptonite.ts +5 -4
- package/src/tools/shard-translit.ts +10 -35
- package/src/tools/sub-venue/harvest.ts +235 -0
- package/src/tools/sub-venue/head-nouns.ts +242 -0
- package/src/tools/sub-venue/surfaces.ts +95 -0
- package/src/tools/sub-venue/table.ts +281 -0
- package/src/tools/sub-venue/wikidata.ts +85 -0
- package/src/tools/sub-venue-lexicon.ts +67 -886
- package/src/utils/align.ts +2 -1
- package/src/utils/golden.ts +1 -1
- package/src/utils/license.ts +1 -1
- package/src/utils/parquet.ts +22 -8
- package/src/utils/split.ts +4 -3
|
@@ -39,9 +39,11 @@
|
|
|
39
39
|
|
|
40
40
|
import type { WhosOnFirstPlacetype } from "@mailwoman/core/resources/whosonfirst"
|
|
41
41
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
42
|
-
|
|
43
|
-
import {
|
|
44
|
-
import {
|
|
42
|
+
|
|
43
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
|
|
44
|
+
import { buildAncestryIndex, normalizeNameKey, walkFeatures, type WOFRecord } from "#utils"
|
|
45
|
+
|
|
46
|
+
import { emitWOFJSONRows, type WOFVariantSpec } from "../wof-json-rows.ts"
|
|
45
47
|
|
|
46
48
|
const COUNTRY_DISPLAY_NAME: Record<string, string> = {
|
|
47
49
|
US: "United States of America",
|
|
@@ -70,16 +72,11 @@ function placetypeToTag(placetype: WhosOnFirstPlacetype | string): ComponentTag
|
|
|
70
72
|
}
|
|
71
73
|
}
|
|
72
74
|
|
|
73
|
-
interface VariantSpec {
|
|
74
|
-
suffix: string
|
|
75
|
-
components: Partial<Record<ComponentTag, string>>
|
|
76
|
-
}
|
|
77
|
-
|
|
78
75
|
/**
|
|
79
76
|
* Compute hierarchy variants for a postcode record. `selfName` is the postcode surface form (canonical `wof:name` for
|
|
80
77
|
* the `default` slot, a `name:*` localized variant otherwise).
|
|
81
78
|
*/
|
|
82
|
-
export function postcodeVariantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: string):
|
|
79
|
+
export function postcodeVariantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: string): WOFVariantSpec[] {
|
|
83
80
|
if (placetypeToTag(row.placetype) !== "postcode") return []
|
|
84
81
|
|
|
85
82
|
const locality = ancestry.find((a) => placetypeToTag(a.placetype) === "locality")
|
|
@@ -87,7 +84,7 @@ export function postcodeVariantsFor(row: WOFRecord, ancestry: WOFRecord[], selfN
|
|
|
87
84
|
const country = ancestry.find((a) => placetypeToTag(a.placetype) === "country")
|
|
88
85
|
const countryDisplay = COUNTRY_DISPLAY_NAME[row.country] ?? country?.name ?? row.country
|
|
89
86
|
|
|
90
|
-
const variants:
|
|
87
|
+
const variants: WOFVariantSpec[] = [{ suffix: "self", components: { postcode: selfName } }]
|
|
91
88
|
|
|
92
89
|
if (locality) {
|
|
93
90
|
variants.push({
|
|
@@ -166,46 +163,16 @@ export function createWOFPostalcodeAdapter(): CorpusAdapter {
|
|
|
166
163
|
const ancestry = buildAncestryIndex(byID)
|
|
167
164
|
|
|
168
165
|
// Pass 2: emit postcode rows only, sorted by id for determinism.
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
const slots = nameSlotsFor(rec)
|
|
180
|
-
|
|
181
|
-
for (const slot of slots) {
|
|
182
|
-
const variants = postcodeVariantsFor(rec, chain, slot.value)
|
|
183
|
-
|
|
184
|
-
for (const variant of variants) {
|
|
185
|
-
if (opts.limit !== undefined && emitted >= opts.limit) return
|
|
186
|
-
|
|
187
|
-
const raw = formatAddress(variant.components, rec.country, { separator: ", " })
|
|
188
|
-
|
|
189
|
-
if (!raw) continue
|
|
190
|
-
const aligned = reconcileComponents(variant.components, raw)
|
|
191
|
-
|
|
192
|
-
if (!Object.keys(aligned).length) continue
|
|
193
|
-
|
|
194
|
-
yield {
|
|
195
|
-
raw,
|
|
196
|
-
components: aligned,
|
|
197
|
-
country: rec.country,
|
|
198
|
-
locale: LOCALE_BY_COUNTRY[rec.country],
|
|
199
|
-
source: WOF_POSTALCODE_ADAPTER_ID,
|
|
200
|
-
source_id: `${WOF_POSTALCODE_ADAPTER_ID}-${rec.id}-${slot.key}-${variant.suffix}`,
|
|
201
|
-
corpus_version: "",
|
|
202
|
-
license: "CC0-1.0",
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
emitted++
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
}
|
|
166
|
+
yield* emitWOFJSONRows({
|
|
167
|
+
records: byID,
|
|
168
|
+
ancestry,
|
|
169
|
+
adapterOptions: opts,
|
|
170
|
+
adapterID: WOF_POSTALCODE_ADAPTER_ID,
|
|
171
|
+
localeByCountry: LOCALE_BY_COUNTRY,
|
|
172
|
+
shouldEmit: (record) => placetypeToTag(record.placetype) === "postcode",
|
|
173
|
+
nameSlotsFor,
|
|
174
|
+
variantsFor: postcodeVariantsFor,
|
|
175
|
+
})
|
|
209
176
|
},
|
|
210
177
|
}
|
|
211
178
|
}
|
package/src/build.ts
CHANGED
|
@@ -53,22 +53,22 @@ import { join } from "node:path"
|
|
|
53
53
|
|
|
54
54
|
import { $public } from "@mailwoman/core/env"
|
|
55
55
|
import { parseJSONStrict } from "@mailwoman/core/objects"
|
|
56
|
-
import {
|
|
57
|
-
|
|
58
|
-
import {
|
|
59
|
-
import
|
|
56
|
+
import { JSONSpliterator } from "spliterator"
|
|
57
|
+
|
|
58
|
+
import { defaultAdapterRegistry } from "#adapters/utils"
|
|
59
|
+
import { runAdapter, type AdapterRunManifest } from "#runner"
|
|
60
|
+
import { defaultAugmentationsForCountry, synthesizeRow } from "#synthesizers/utils"
|
|
61
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter, LabeledRow } from "#types"
|
|
62
|
+
import { alignRow } from "#utils/align"
|
|
63
|
+
import { licenseExcluded } from "#utils/license"
|
|
64
|
+
import { writeShards, type ShardManifest } from "#utils/parquet"
|
|
60
65
|
import {
|
|
61
|
-
alignRow,
|
|
62
|
-
licenseExcluded,
|
|
63
|
-
writeShards,
|
|
64
66
|
defaultHoldouts,
|
|
65
67
|
splitForRow,
|
|
66
68
|
writeSplitManifestsFromLabeledFiles,
|
|
67
|
-
type ShardManifest,
|
|
68
69
|
type SplitManifest,
|
|
69
70
|
type SplitName,
|
|
70
|
-
} from "
|
|
71
|
-
import { JSONSpliterator } from "spliterator"
|
|
71
|
+
} from "#utils/split"
|
|
72
72
|
|
|
73
73
|
/**
|
|
74
74
|
* Stage tags surfaced to `onProgress`.
|
package/src/index.ts
CHANGED
|
@@ -67,6 +67,15 @@ export class ParquetReader<T extends ParquetRecordLike> extends BaseParquetReade
|
|
|
67
67
|
* caller wanting the whole record should use.
|
|
68
68
|
*/
|
|
69
69
|
public async *project<K extends keyof T>(...columns: K[]): AsyncGenerator<Pick<T, K>, void, unknown> {
|
|
70
|
+
const missing = columns.filter((column) => !Object.hasOwn(this.schema.schema, column))
|
|
71
|
+
|
|
72
|
+
if (missing.length) {
|
|
73
|
+
throw new Error(
|
|
74
|
+
`Parquet projection requested column${missing.length === 1 ? "" : "s"} absent from the file schema: ` +
|
|
75
|
+
missing.map(String).join(", ")
|
|
76
|
+
)
|
|
77
|
+
}
|
|
78
|
+
|
|
70
79
|
const cursor = this.getCursor(columns.map((column) => [column as string]))
|
|
71
80
|
|
|
72
81
|
for (;;) {
|
package/src/runner.ts
CHANGED
|
@@ -31,13 +31,8 @@ import { createWriteStream, type WriteStream } from "node:fs"
|
|
|
31
31
|
import { mkdir, writeFile } from "node:fs/promises"
|
|
32
32
|
import { dirname, join } from "node:path"
|
|
33
33
|
|
|
34
|
-
import {
|
|
35
|
-
|
|
36
|
-
streamingSha256,
|
|
37
|
-
type AdapterRegistry,
|
|
38
|
-
type StreamingHasher,
|
|
39
|
-
} from "@mailwoman/corpus/adapters/utils"
|
|
40
|
-
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
|
|
34
|
+
import { canonicalDedupKey, streamingSha256, type AdapterRegistry, type StreamingHasher } from "#adapters/utils"
|
|
35
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
|
|
41
36
|
|
|
42
37
|
/**
|
|
43
38
|
* Snapshot of the runner's state, emitted on every progress tick.
|
|
@@ -16,11 +16,11 @@ import { readFileSync } from "node:fs"
|
|
|
16
16
|
|
|
17
17
|
import { parseJSONStrict } from "@mailwoman/core/objects"
|
|
18
18
|
import { dataRootPath } from "@mailwoman/core/utils"
|
|
19
|
-
import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
|
|
20
|
-
import { synthesizeAnchorAbsorptionRow } from "@mailwoman/corpus/synthesizers/anchor-absorption"
|
|
21
|
-
import { alignRow } from "@mailwoman/corpus/utils"
|
|
22
19
|
|
|
23
|
-
import {
|
|
20
|
+
import { synthesizeAnchorAbsorptionRow } from "#synthesizers/anchor-absorption"
|
|
21
|
+
import { alignRow } from "#utils"
|
|
22
|
+
|
|
23
|
+
import { makeLcg, shardSourceID, type ShardRecipe } from "./scaffold.ts"
|
|
24
24
|
|
|
25
25
|
/**
|
|
26
26
|
* The leading-5-digit source: the real US ZIPs in the postcode anchor's pilot lookup. Resolved through the data-root
|
|
@@ -76,7 +76,7 @@ export const anchorAbsorptionRecipe: ShardRecipe = {
|
|
|
76
76
|
country,
|
|
77
77
|
locale: synth.locale,
|
|
78
78
|
source,
|
|
79
|
-
source_id:
|
|
79
|
+
source_id: shardSourceID(source, { v: String(i) }),
|
|
80
80
|
}
|
|
81
81
|
|
|
82
82
|
const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `bare-country` — the country name as the WHOLE query. Addresses almost never consist of one bare
|
|
7
|
+
* country name, so the class is out of the model's training distribution and the tag is a coin
|
|
8
|
+
* flip: measured on the 2026-08-13 panel, France/Germany/United States/New Zealand parsed
|
|
9
|
+
* `country` while Japan/China/Nigeria/Australia/Deutschland parsed `locality`. The retrieval-side
|
|
10
|
+
* bare-country race (#1651) covers the answer either way; this shard closes the PARSE half so the
|
|
11
|
+
* tag itself is right.
|
|
12
|
+
*
|
|
13
|
+
* Source is the codex country table — every ISO canonical English name plus the curated surface
|
|
14
|
+
* forms (endonyms and long-form names). The 2–3 letter CODES are deliberately excluded: a bare
|
|
15
|
+
* `JP` or `GER` is an abbreviation register with its own ambiguity surface (`IN`, `DE`, `TO` are
|
|
16
|
+
* English words), and teaching it here would be an unmeasured claim. Case augmentation is the
|
|
17
|
+
* loader's job, as everywhere.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { COUNTRY_SURFACE_FORMS, CountryNames, matchCountry } from "@mailwoman/codex/country"
|
|
21
|
+
|
|
22
|
+
import { alignAndWrite, makeMulberry32, type ShardRecipe, shardSourceID } from "./scaffold.ts"
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Surfaces shorter than this are the code register (`JP`, `GER`), not a name — excluded (see the module doc).
|
|
26
|
+
*/
|
|
27
|
+
const MIN_NAME_LENGTH = 4
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Every (surface, iso2) pair the shard emits: the ISO canonical names plus each country's curated surface forms,
|
|
31
|
+
* deduplicated on the surface string (a form shared across countries — none known — would keep its first bearer).
|
|
32
|
+
*/
|
|
33
|
+
function* bareCountrySurfaces(): Generator<{ surface: string; iso2: string }> {
|
|
34
|
+
const seen = new Set<string>()
|
|
35
|
+
|
|
36
|
+
for (const [iso2, forms] of Object.entries(COUNTRY_SURFACE_FORMS)) {
|
|
37
|
+
for (const surface of forms) {
|
|
38
|
+
if (surface.length < MIN_NAME_LENGTH || seen.has(surface)) continue
|
|
39
|
+
seen.add(surface)
|
|
40
|
+
yield { surface, iso2 }
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
for (const name of CountryNames) {
|
|
45
|
+
if (name.length < MIN_NAME_LENGTH || seen.has(name)) continue
|
|
46
|
+
const matched = matchCountry(name)
|
|
47
|
+
|
|
48
|
+
if (!matched) continue
|
|
49
|
+
seen.add(name)
|
|
50
|
+
yield { surface: name, iso2: matched.iso2 }
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
56
|
+
* and `description` below for the surface form it generates.
|
|
57
|
+
*/
|
|
58
|
+
export const bareCountryRecipe: ShardRecipe = {
|
|
59
|
+
name: "bare-country",
|
|
60
|
+
description: "The country name as the whole query (#1651 parse half): ISO names + curated endonyms, no codes",
|
|
61
|
+
mode: "generate",
|
|
62
|
+
async run(opts, write) {
|
|
63
|
+
// Seeded for parity with the other recipes; the surfaces themselves drive the content.
|
|
64
|
+
makeMulberry32(opts.seed)
|
|
65
|
+
let read = 0
|
|
66
|
+
let emitted = 0
|
|
67
|
+
let skipped = 0
|
|
68
|
+
|
|
69
|
+
for (const { surface, iso2 } of bareCountrySurfaces()) {
|
|
70
|
+
read++
|
|
71
|
+
|
|
72
|
+
const components: Record<string, string> = { country: surface }
|
|
73
|
+
const source_id = shardSourceID("synth-bare-country", { country: surface, cc: iso2, v: String(read) })
|
|
74
|
+
|
|
75
|
+
const canonical = {
|
|
76
|
+
raw: surface,
|
|
77
|
+
components,
|
|
78
|
+
country: iso2,
|
|
79
|
+
locale: "und",
|
|
80
|
+
source: "synth-bare-country",
|
|
81
|
+
source_id,
|
|
82
|
+
corpus_version: "0.11.0",
|
|
83
|
+
license: "Synthetic — bare-country; surfaces from the codex country table (ISO 3166 + curated forms)",
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
if (alignAndWrite(write, canonical, "bare-country")) {
|
|
87
|
+
emitted++
|
|
88
|
+
} else {
|
|
89
|
+
skipped++
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
return { read, emitted, skipped }
|
|
94
|
+
},
|
|
95
|
+
}
|
|
@@ -12,11 +12,8 @@
|
|
|
12
12
|
* `synthesizeBoundaryStressRow` is NOT re-exported from the corpus index — imported directly here.
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
-
import {
|
|
16
|
-
|
|
17
|
-
synthesizeBoundaryStressRow,
|
|
18
|
-
} from "@mailwoman/corpus/synthesizers/boundary-stress"
|
|
19
|
-
import { alignRow } from "@mailwoman/corpus/utils"
|
|
15
|
+
import { type BoundaryStressTemplate, synthesizeBoundaryStressRow } from "#synthesizers/boundary-stress"
|
|
16
|
+
import { alignRow } from "#utils"
|
|
20
17
|
|
|
21
18
|
import { makeMulberry32, type ShardRecipe, shardSourceID } from "./scaffold.ts"
|
|
22
19
|
|
|
@@ -25,11 +25,13 @@
|
|
|
25
25
|
*/
|
|
26
26
|
|
|
27
27
|
import { COUNTRY_SURFACE_FORMS, CountryNames } from "@mailwoman/codex/country"
|
|
28
|
+
import { isPresent } from "@mailwoman/core/objects"
|
|
28
29
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
29
30
|
import { dataRootPath } from "@mailwoman/core/utils"
|
|
30
|
-
|
|
31
|
-
import
|
|
32
|
-
import {
|
|
31
|
+
|
|
32
|
+
import { stableSourceID } from "#adapters/utils"
|
|
33
|
+
import type { CanonicalRow } from "#types"
|
|
34
|
+
import { alignRow } from "#utils"
|
|
33
35
|
|
|
34
36
|
import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
|
|
35
37
|
|
|
@@ -220,14 +222,14 @@ function renderCountry(
|
|
|
220
222
|
let body: string
|
|
221
223
|
|
|
222
224
|
if (order === "us") {
|
|
223
|
-
const regPc = [reg, pc].filter(
|
|
225
|
+
const regPc = [reg, pc].filter(isPresent).join(" ")
|
|
224
226
|
body = `${hn} ${street}, ${loc}${regPc ? ", " + regPc : ""}`
|
|
225
227
|
} else if (order === "fr") {
|
|
226
228
|
// French is NUMBER-street, postcode-city: "84 Route de la Fontaine, 75008 Paris".
|
|
227
|
-
body = `${hn} ${street}, ${[pc, loc].filter(
|
|
229
|
+
body = `${hn} ${street}, ${[pc, loc].filter(isPresent).join(" ")}`
|
|
228
230
|
} else {
|
|
229
231
|
// DE/IT/NL: street-number, postcode-city: "Pariser Platz 1, 10117 Berlin".
|
|
230
|
-
const pcCity = [pc, loc].filter(
|
|
232
|
+
const pcCity = [pc, loc].filter(isPresent).join(" ")
|
|
231
233
|
body = `${street} ${hn}, ${pcCity}`
|
|
232
234
|
}
|
|
233
235
|
|
|
@@ -16,6 +16,13 @@
|
|
|
16
16
|
|
|
17
17
|
import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
|
|
18
18
|
|
|
19
|
+
/**
|
|
20
|
+
* The order-cycle slot for the STREET-LESS form (`«city» «pc», Česko`) — the exact surface of the
|
|
21
|
+
* `cz-full-praha-100-00` board row, whose absence from the street-bearing orders was the v4.5.0 no-promote's measured
|
|
22
|
+
* gap.
|
|
23
|
+
*/
|
|
24
|
+
const STREETLESS_ORDER = 3
|
|
25
|
+
|
|
19
26
|
/**
|
|
20
27
|
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
21
28
|
* and `description` below for the surface form it generates.
|
|
@@ -44,27 +51,39 @@ export const czPcFirstPrepositionRecipe: ShardRecipe = {
|
|
|
44
51
|
continue
|
|
45
52
|
}
|
|
46
53
|
|
|
47
|
-
const order = read %
|
|
54
|
+
const order = read % 4
|
|
55
|
+
// The OFFICIAL Czech rendering spaces the PSČ as `NNN NN` ('512 44'); OpenAddresses stores it
|
|
56
|
+
// unspaced ('51244'), and a model trained only on the source form reads the spaced surface as
|
|
57
|
+
// house_number + garbage (the 'Praha 100 00' mangle). Alternate the two renderings so both
|
|
58
|
+
// orthographies are attested — the LABEL is the postcode either way.
|
|
59
|
+
const spaced = read % 2 === 0 && /^\d{5}$/.test(postcode)
|
|
60
|
+
const postcodeSurface = spaced ? `${postcode.slice(0, 3)} ${postcode.slice(3)}` : postcode
|
|
48
61
|
let raw: string
|
|
49
62
|
|
|
50
|
-
const components: Record<string, string> =
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
locality: city,
|
|
55
|
-
}
|
|
63
|
+
const components: Record<string, string> =
|
|
64
|
+
order === STREETLESS_ORDER
|
|
65
|
+
? { locality: city, postcode: postcodeSurface, country: "Česko" }
|
|
66
|
+
: { street, house_number: number, postcode: postcodeSurface, locality: city }
|
|
56
67
|
|
|
57
68
|
if (order === 0) {
|
|
58
|
-
raw = `${
|
|
69
|
+
raw = `${postcodeSurface} ${city}, ${street} ${number}`
|
|
59
70
|
} else if (order === 1) {
|
|
60
|
-
raw = `${street} ${number}, ${
|
|
71
|
+
raw = `${street} ${number}, ${postcodeSurface} ${city}`
|
|
72
|
+
} else if (order === 2) {
|
|
73
|
+
raw = `${city}, ${postcodeSurface}, ${street} ${number}`
|
|
61
74
|
} else {
|
|
62
|
-
|
|
75
|
+
// order === STREETLESS_ORDER: the street-less form — `«city» «pc», Česko` — the exact surface of the
|
|
76
|
+
// cz-full-praha-100-00 board row. The v4.5.0 no-promote receipt measured the gap: every
|
|
77
|
+
// prior order was street-bearing, so the model never saw a spaced PSČ beside a bare
|
|
78
|
+
// locality and mangled 'Praha 100 00, Czechia' into house_number spans. Street/number
|
|
79
|
+
// stay OUT of the components for this form (they are not in the surface).
|
|
80
|
+
raw = `${city} ${postcodeSurface}, Česko`
|
|
63
81
|
}
|
|
64
82
|
|
|
65
83
|
const source_id = shardSourceID("synth-cz-pcfirst-preposition", {
|
|
66
84
|
...components,
|
|
67
85
|
o: String(order),
|
|
86
|
+
s: spaced ? "1" : "0",
|
|
68
87
|
v: String(read),
|
|
69
88
|
})
|
|
70
89
|
|
|
@@ -32,11 +32,13 @@
|
|
|
32
32
|
|
|
33
33
|
import { departementForCodePostal } from "@mailwoman/codex/fr"
|
|
34
34
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
35
|
-
import {
|
|
36
|
-
import type { CanonicalRow } from "@mailwoman/corpus/types"
|
|
37
|
-
import { alignRow } from "@mailwoman/corpus/utils"
|
|
35
|
+
import { tempRootPath } from "@mailwoman/core/utils"
|
|
38
36
|
import { CSVSpliterator, Delimiters } from "spliterator"
|
|
39
37
|
|
|
38
|
+
import { stableSourceID } from "#adapters/utils"
|
|
39
|
+
import type { CanonicalRow } from "#types"
|
|
40
|
+
import { alignRow } from "#utils"
|
|
41
|
+
|
|
40
42
|
import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
|
|
41
43
|
|
|
42
44
|
/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
|
|
@@ -45,7 +47,7 @@ import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
|
|
|
45
47
|
would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
|
|
46
48
|
extracted as named constants above. */
|
|
47
49
|
|
|
48
|
-
const DEFAULT_COMMUNES = "
|
|
50
|
+
const DEFAULT_COMMUNES = tempRootPath("reg", "fr-communes.tsv")
|
|
49
51
|
const LICENSE = "BAN (Base Adresse Nationale) commune+postcode tuples, rendered admin-split — see ingest SOURCE"
|
|
50
52
|
|
|
51
53
|
/**
|
|
@@ -20,9 +20,25 @@
|
|
|
20
20
|
* run including this shard.
|
|
21
21
|
*/
|
|
22
22
|
|
|
23
|
+
import { FR_VOIE_TYPES } from "@mailwoman/codex/fr"
|
|
24
|
+
|
|
23
25
|
import { decomposeFrStreet } from "../adapters/ban/street-decompose.ts"
|
|
24
26
|
import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
|
|
25
27
|
|
|
28
|
+
/**
|
|
29
|
+
* Canonical voie type (lowercase, accent-kept) → its most common written abbreviation, from the codex table's first
|
|
30
|
+
* entry. Types with no attested abbreviation stay canonical in the abbreviated form.
|
|
31
|
+
*/
|
|
32
|
+
const FR_VOIE_ABBREV: Record<string, string> = Object.fromEntries(
|
|
33
|
+
Object.entries(FR_VOIE_TYPES).flatMap(([canonical, abbrevs]) => (abbrevs[0] ? [[canonical, abbrevs[0]]] : []))
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* The order-cycle slot for the BARE-STREET-ONLY form (`«voie» «name»`, no number, no locality) — the absence
|
|
38
|
+
* counterweight to the locality-terminated comma-free forms (see the cycle comment).
|
|
39
|
+
*/
|
|
40
|
+
const BARE_STREET_ONLY_FORM = 3
|
|
41
|
+
|
|
26
42
|
/**
|
|
27
43
|
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
28
44
|
* and `description` below for the surface form it generates.
|
|
@@ -30,7 +46,7 @@ import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourc
|
|
|
30
46
|
export const frBareStreetRecipe: ShardRecipe = {
|
|
31
47
|
name: "fr-bare-street",
|
|
32
48
|
description:
|
|
33
|
-
"FR bare
|
|
49
|
+
"FR bare street+city, NO postcode (#251): comma / comma-free / abbreviated-voie surfaces over one label set",
|
|
34
50
|
mode: "tuples",
|
|
35
51
|
async run(opts, write) {
|
|
36
52
|
// Seeded for parity with the other recipes; unused beyond reproducibility (the tuples drive the content).
|
|
@@ -53,23 +69,82 @@ export const frBareStreetRecipe: ShardRecipe = {
|
|
|
53
69
|
|
|
54
70
|
const { prefix, street } = decomposeFrStreet(fullStreet)
|
|
55
71
|
|
|
56
|
-
//
|
|
72
|
+
// A no-prefix nom_voie ("La Ville Mois") is not the prefix-led class the numbered forms
|
|
73
|
+
// exercise — but as a BARE surface it is exactly the non-voie-led counterweight the v4.5.1
|
|
74
|
+
// probe showed missing ('Savile Row'-shaped spans still fell to the trailing-locality
|
|
75
|
+
// prior; the voie-led bare form protected only voie-led spans). Alternate rows emit the
|
|
76
|
+
// whole span as a bare street; the rest skip as before.
|
|
57
77
|
if (!prefix || !street) {
|
|
78
|
+
if (read % 2 === 0 && fullStreet.split(" ").length >= 2) {
|
|
79
|
+
const bare = {
|
|
80
|
+
raw: fullStreet,
|
|
81
|
+
components: { street: fullStreet } as Record<string, string>,
|
|
82
|
+
country: "FR",
|
|
83
|
+
locale: "fr-FR",
|
|
84
|
+
source: "synth-fr-bare-street",
|
|
85
|
+
source_id: shardSourceID("synth-fr-bare-street", {
|
|
86
|
+
street: fullStreet,
|
|
87
|
+
f: "bare-nonvoie",
|
|
88
|
+
v: String(read),
|
|
89
|
+
}),
|
|
90
|
+
corpus_version: "0.9.4",
|
|
91
|
+
license:
|
|
92
|
+
"Synthetic — fr-bare-street; (street, number, city) from BAN (Base Adresse Nationale, Licence Ouverte)",
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
if (alignAndWrite(write, bare, "fr-bare-street")) {
|
|
96
|
+
emitted++
|
|
97
|
+
} else {
|
|
98
|
+
skipped++
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
continue
|
|
102
|
+
}
|
|
103
|
+
|
|
58
104
|
skipped++
|
|
59
105
|
|
|
60
106
|
continue
|
|
61
107
|
}
|
|
62
108
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
109
|
+
// Four surfaces over the same tuple, cycled deterministically. The comma form was the
|
|
110
|
+
// original lever; the COMMA-FREE form is the colloquial register users actually type
|
|
111
|
+
// ('12 rue de Rome Paris' — the street↔locality boundary with NO delimiter, the fr-fr
|
|
112
|
+
// panel's named loss); the ABBREVIATED form is the typeahead register the geocoder-tester
|
|
113
|
+
// FR slice attests at scale; and the BARE-STREET-ONLY form is the absence counterweight —
|
|
114
|
+
// without it, every delimiter-free surface in the mix ENDS in a locality, the model learns
|
|
115
|
+
// "trailing span = locality" as categorical, and bare street names across locales flip to
|
|
116
|
+
// locality wholesale (the v4.5.0 no-promote's measured erosion: 'Calle de Alcalá',
|
|
117
|
+
// 'Madison Square West', and COMER's fork all fell to that prior). Tags are identical
|
|
118
|
+
// where present; each component VALUE is the span as written (BIO alignment binds value
|
|
119
|
+
// to surface); the bare form carries NO number and NO locality because the surface has
|
|
120
|
+
// neither.
|
|
121
|
+
const form = read % 4
|
|
122
|
+
const prefixSurface = form >= 2 ? (FR_VOIE_ABBREV[prefix.toLowerCase()] ?? prefix) : prefix
|
|
123
|
+
|
|
124
|
+
const components: Record<string, string> =
|
|
125
|
+
form === BARE_STREET_ONLY_FORM
|
|
126
|
+
? { street_prefix: prefix, street }
|
|
127
|
+
: { house_number: number, street_prefix: prefixSurface, street, locality }
|
|
128
|
+
|
|
129
|
+
// When the tuple carries a WOF-attested neighbourhood, the comma slot renders the
|
|
130
|
+
// THREE-slot middle surface — the dependent-locality counterweight (the v4.5.1 erosion's
|
|
131
|
+
// untouched half: the two-slot comma-free endings squeezed the middle tag out).
|
|
132
|
+
const hood = String(t.neighbourhood ?? "").trim()
|
|
133
|
+
|
|
134
|
+
if (form === 0 && hood) {
|
|
135
|
+
components.dependent_locality = hood
|
|
68
136
|
}
|
|
69
137
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
138
|
+
const raw =
|
|
139
|
+
form === 0
|
|
140
|
+
? hood
|
|
141
|
+
? `${number} ${prefix} ${street}, ${hood}, ${locality}`
|
|
142
|
+
: `${number} ${prefix} ${street}, ${locality}`
|
|
143
|
+
: form === BARE_STREET_ONLY_FORM
|
|
144
|
+
? `${prefix} ${street}`
|
|
145
|
+
: `${number} ${prefixSurface} ${street} ${locality}`
|
|
146
|
+
|
|
147
|
+
const source_id = shardSourceID("synth-fr-bare-street", { ...components, f: String(form), v: String(read) })
|
|
73
148
|
|
|
74
149
|
const canonical = {
|
|
75
150
|
raw,
|
|
@@ -29,11 +29,13 @@ import { join } from "node:path"
|
|
|
29
29
|
|
|
30
30
|
import { extractBANAddrPoints } from "@mailwoman/ban/sdk"
|
|
31
31
|
import { COUNTRY_SURFACE_FORMS } from "@mailwoman/codex/country"
|
|
32
|
+
import { isPresent } from "@mailwoman/core/objects"
|
|
32
33
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
33
34
|
import { dataRootPath } from "@mailwoman/core/utils"
|
|
34
|
-
|
|
35
|
-
import
|
|
36
|
-
import {
|
|
35
|
+
|
|
36
|
+
import { stableSourceID } from "#adapters/utils"
|
|
37
|
+
import type { CanonicalRow } from "#types"
|
|
38
|
+
import { alignRow } from "#utils"
|
|
37
39
|
|
|
38
40
|
import { decomposeFrStreet } from "../adapters/ban/street-decompose.ts"
|
|
39
41
|
import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
|
|
@@ -156,7 +158,7 @@ function composeRaw(
|
|
|
156
158
|
}
|
|
157
159
|
|
|
158
160
|
lines.push(dependentLocality)
|
|
159
|
-
const cityLine = [postcode, locality].filter(
|
|
161
|
+
const cityLine = [postcode, locality].filter(isPresent).join(" ").trim()
|
|
160
162
|
|
|
161
163
|
if (cityLine) {
|
|
162
164
|
lines.push(cityLine)
|
|
@@ -30,11 +30,13 @@
|
|
|
30
30
|
* inlined the variant logic and never called it).
|
|
31
31
|
*/
|
|
32
32
|
|
|
33
|
+
import { isPresent } from "@mailwoman/core/objects"
|
|
33
34
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
34
35
|
import { dataRootPath } from "@mailwoman/core/utils"
|
|
35
|
-
|
|
36
|
-
import
|
|
37
|
-
import {
|
|
36
|
+
|
|
37
|
+
import { stableSourceID } from "#adapters/utils"
|
|
38
|
+
import type { CanonicalRow } from "#types"
|
|
39
|
+
import { alignRow } from "#utils"
|
|
38
40
|
|
|
39
41
|
import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
|
|
40
42
|
|
|
@@ -196,7 +198,7 @@ export const frOrderRecipe: ShardRecipe = {
|
|
|
196
198
|
const { raw, components } = rendered
|
|
197
199
|
|
|
198
200
|
// Safety check: every component must appear verbatim in raw (alignment precondition).
|
|
199
|
-
const componentValues = Object.values(components).filter(
|
|
201
|
+
const componentValues = Object.values(components).filter(isPresent)
|
|
200
202
|
|
|
201
203
|
if (!componentValues.every((v) => raw.includes(v))) {
|
|
202
204
|
skipped++
|
|
@@ -16,9 +16,10 @@
|
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
18
|
import { dataRootPath } from "@mailwoman/core/utils"
|
|
19
|
-
|
|
20
|
-
import {
|
|
21
|
-
import {
|
|
19
|
+
|
|
20
|
+
import { stableSourceID } from "#adapters/utils"
|
|
21
|
+
import { synthesizeGermanRow, type LocaleBaseTuple } from "#synthesizers/german"
|
|
22
|
+
import { alignRow } from "#utils"
|
|
22
23
|
|
|
23
24
|
import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
|
|
24
25
|
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* scripts/build-house-venue-shard.mjs.
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import { synthesizeHouseVenueRow, type HouseVenueBaseTuple } from "
|
|
13
|
+
import { synthesizeHouseVenueRow, type HouseVenueBaseTuple } from "#synthesizers/house-venue"
|
|
14
14
|
|
|
15
15
|
import { alignAndWrite, makeLcg, readTuples, shardSourceID, type ShardRecipe } from "./scaffold.ts"
|
|
16
16
|
|