@mailwoman/corpus 8.6.0 → 9.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/data/PROVENANCE.md +214 -0
- package/data/sub-venue-lexicon.json +18269 -0
- package/out/src/adapters/ban/adapter.d.ts +5 -4
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +62 -69
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +18 -25
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +1 -7
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +4 -26
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +1 -1
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +72 -78
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +46 -51
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +6 -9
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +7 -12
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +5 -10
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +1 -1
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +5 -9
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +57 -75
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +60 -78
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +69 -87
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +71 -89
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +3 -3
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +10 -18
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +1 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +2 -2
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js +18 -27
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +4 -4
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +64 -82
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +62 -84
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +59 -66
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +6 -9
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +60 -78
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +53 -73
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/{adapter.d.ts → adapters/utils/index.d.ts} +43 -2
- package/out/src/adapters/utils/index.d.ts.map +1 -0
- package/out/src/{adapter.js → adapters/utils/index.js} +47 -2
- package/out/src/adapters/utils/index.js.map +1 -0
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +24 -6
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +2 -2
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +2 -2
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +2 -2
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +2 -2
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/build.d.ts +3 -4
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +6 -8
- package/out/src/build.js.map +1 -1
- package/out/src/index.d.ts +3 -17
- package/out/src/index.d.ts.map +1 -1
- package/out/src/index.js +3 -17
- package/out/src/index.js.map +1 -1
- package/out/src/name-prone-us-suffixes.d.ts +12 -0
- package/out/src/name-prone-us-suffixes.d.ts.map +1 -0
- package/out/src/name-prone-us-suffixes.js +12 -0
- package/out/src/name-prone-us-suffixes.js.map +1 -0
- package/out/src/parquet-wrapper/reader.d.ts +8 -0
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +16 -0
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/schema.d.ts +4 -0
- package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
- package/out/src/parquet-wrapper/schema.js.map +1 -1
- package/out/src/runner.d.ts +2 -2
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +1 -1
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +5 -4
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +2 -2
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +75 -75
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +2 -2
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +8 -5
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +2 -2
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +18 -72
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +17 -70
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/house-venue.js +1 -1
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +4 -1
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts +2 -2
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +28 -72
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +5 -5
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +25 -35
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +8 -5
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street-led.js +8 -5
- package/out/src/shard-recipes/no-street-led.js.map +1 -1
- package/out/src/shard-recipes/no-street.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street.js +1 -1
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +3 -3
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +101 -127
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +1 -1
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +58 -9
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +87 -29
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +54 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +253 -111
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
- package/out/src/shard-recipes/street-bare.js +3 -3
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.d.ts.map +1 -1
- package/out/src/shard-recipes/street.js +2 -2
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.d.ts +228 -0
- package/out/src/shard-recipes/sub-venue-sources.d.ts.map +1 -0
- package/out/src/shard-recipes/sub-venue-sources.js +664 -0
- package/out/src/shard-recipes/sub-venue-sources.js.map +1 -0
- package/out/src/shard-recipes/sub-venue.d.ts +269 -0
- package/out/src/shard-recipes/sub-venue.d.ts.map +1 -0
- package/out/src/shard-recipes/sub-venue.js +709 -0
- package/out/src/shard-recipes/sub-venue.js.map +1 -0
- package/out/src/shard-recipes/unit.d.ts +1 -1
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +26 -74
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/{synthesize-anchor-absorption.d.ts → synthesizers/anchor-absorption.d.ts} +1 -1
- package/out/src/synthesizers/anchor-absorption.d.ts.map +1 -0
- package/out/src/{synthesize-anchor-absorption.js → synthesizers/anchor-absorption.js} +1 -1
- package/out/src/synthesizers/anchor-absorption.js.map +1 -0
- package/out/src/{synthesize-boundary-stress.d.ts → synthesizers/boundary-stress.d.ts} +3 -3
- package/out/src/synthesizers/boundary-stress.d.ts.map +1 -0
- package/out/src/{synthesize-boundary-stress.js → synthesizers/boundary-stress.js} +2 -2
- package/out/src/synthesizers/boundary-stress.js.map +1 -0
- package/out/src/{synthesize-german.d.ts → synthesizers/german.d.ts} +2 -2
- package/out/src/synthesizers/german.d.ts.map +1 -0
- package/out/src/{synthesize-german.js → synthesizers/german.js} +2 -2
- package/out/src/synthesizers/german.js.map +1 -0
- package/out/src/{synthesize-house-venue.d.ts → synthesizers/house-venue.d.ts} +4 -4
- package/out/src/synthesizers/house-venue.d.ts.map +1 -0
- package/out/src/{synthesize-house-venue.js → synthesizers/house-venue.js} +4 -4
- package/out/src/synthesizers/house-venue.js.map +1 -0
- package/out/src/{synthesize-intersection.d.ts → synthesizers/intersection.d.ts} +2 -2
- package/out/src/synthesizers/intersection.d.ts.map +1 -0
- package/out/src/{synthesize-intersection.js → synthesizers/intersection.js} +1 -1
- package/out/src/synthesizers/intersection.js.map +1 -0
- package/out/src/{synthesize-no-street.d.ts → synthesizers/no-street.d.ts} +3 -3
- package/out/src/synthesizers/no-street.d.ts.map +1 -0
- package/out/src/{synthesize-no-street.js → synthesizers/no-street.js} +2 -2
- package/out/src/synthesizers/no-street.js.map +1 -0
- package/out/src/{synthesize-po-box.d.ts → synthesizers/po-box.d.ts} +2 -2
- package/out/src/synthesizers/po-box.d.ts.map +1 -0
- package/out/src/{synthesize-po-box.js → synthesizers/po-box.js} +1 -1
- package/out/src/synthesizers/po-box.js.map +1 -0
- package/out/src/{synthesize-street.d.ts → synthesizers/street.d.ts} +2 -2
- package/out/src/synthesizers/street.d.ts.map +1 -0
- package/out/src/{synthesize-street.js → synthesizers/street.js} +2 -2
- package/out/src/synthesizers/street.js.map +1 -0
- package/out/src/{synthesize.d.ts → synthesizers/utils.d.ts} +3 -3
- package/out/src/synthesizers/utils.d.ts.map +1 -0
- package/out/src/{synthesize.js → synthesizers/utils.js} +4 -15
- package/out/src/synthesizers/utils.js.map +1 -0
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +4 -8
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/audit.d.ts.map +1 -1
- package/out/src/tools/audit.js +4 -4
- package/out/src/tools/audit.js.map +1 -1
- package/out/src/tools/corpus-stats.d.ts +2 -2
- package/out/src/tools/corpus-stats.d.ts.map +1 -1
- package/out/src/tools/corpus-stats.js +18 -31
- package/out/src/tools/corpus-stats.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +5 -6
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +3 -15
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +14 -1
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +14 -1
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts +1 -1
- package/out/src/tools/fetch/nad.js +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +8 -14
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts +1 -1
- package/out/src/tools/fetch/openaddresses.js +1 -1
- package/out/src/tools/fetch/ourairports.d.ts +45 -0
- package/out/src/tools/fetch/ourairports.d.ts.map +1 -0
- package/out/src/tools/fetch/ourairports.js +123 -0
- package/out/src/tools/fetch/ourairports.js.map +1 -0
- package/out/src/tools/fetch/wikidata-subvenue.d.ts +171 -0
- package/out/src/tools/fetch/wikidata-subvenue.d.ts.map +1 -0
- package/out/src/tools/fetch/wikidata-subvenue.js +275 -0
- package/out/src/tools/fetch/wikidata-subvenue.js.map +1 -0
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +18 -16
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-promote.d.ts.map +1 -1
- package/out/src/tools/golden-promote.js +12 -6
- package/out/src/tools/golden-promote.js.map +1 -1
- package/out/src/tools/golden-relabel-street.d.ts +196 -0
- package/out/src/tools/golden-relabel-street.d.ts.map +1 -0
- package/out/src/tools/golden-relabel-street.js +440 -0
- package/out/src/tools/golden-relabel-street.js.map +1 -0
- package/out/src/tools/index.d.ts +4 -0
- package/out/src/tools/index.d.ts.map +1 -1
- package/out/src/tools/index.js +4 -0
- package/out/src/tools/index.js.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
- package/out/src/tools/jsonl-to-parquet.js +3 -2
- package/out/src/tools/jsonl-to-parquet.js.map +1 -1
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
- package/out/src/tools/lint-shard-vocab.js +3 -2
- package/out/src/tools/lint-shard-vocab.js.map +1 -1
- package/out/src/tools/lint-shard.d.ts +3 -3
- package/out/src/tools/lint-shard.d.ts.map +1 -1
- package/out/src/tools/lint-shard.js +23 -32
- package/out/src/tools/lint-shard.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +2 -1
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/overture-subvenue.d.ts +112 -0
- package/out/src/tools/overture-subvenue.d.ts.map +1 -0
- package/out/src/tools/overture-subvenue.js +144 -0
- package/out/src/tools/overture-subvenue.js.map +1 -0
- package/out/src/tools/shard-kryptonite.d.ts +1 -1
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +6 -6
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts +6 -3
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +9 -8
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/tools/sub-venue-lexicon.d.ts +507 -0
- package/out/src/tools/sub-venue-lexicon.d.ts.map +1 -0
- package/out/src/tools/sub-venue-lexicon.js +817 -0
- package/out/src/tools/sub-venue-lexicon.js.map +1 -0
- package/out/src/tools/sub-venue-promotions.d.ts +94 -0
- package/out/src/tools/sub-venue-promotions.d.ts.map +1 -0
- package/out/src/tools/sub-venue-promotions.js +266 -0
- package/out/src/tools/sub-venue-promotions.js.map +1 -0
- package/out/src/{align.d.ts → utils/align.d.ts} +1 -1
- package/out/src/utils/align.d.ts.map +1 -0
- package/out/src/utils/align.js.map +1 -0
- package/out/src/{golden.d.ts → utils/golden.d.ts} +5 -0
- package/out/src/utils/golden.d.ts.map +1 -0
- package/out/src/{golden.js → utils/golden.js} +24 -10
- package/out/src/utils/golden.js.map +1 -0
- package/out/src/utils/index.d.ts +14 -0
- package/out/src/utils/index.d.ts.map +1 -0
- package/out/src/utils/index.js +14 -0
- package/out/src/utils/index.js.map +1 -0
- package/out/src/utils/license.d.ts.map +1 -0
- package/out/src/utils/license.js.map +1 -0
- package/out/src/{parquet.d.ts → utils/parquet.d.ts} +2 -2
- package/out/src/utils/parquet.d.ts.map +1 -0
- package/out/src/{parquet.js → utils/parquet.js} +3 -15
- package/out/src/utils/parquet.js.map +1 -0
- package/out/src/{split.d.ts → utils/split.d.ts} +8 -1
- package/out/src/utils/split.d.ts.map +1 -0
- package/out/src/{split.js → utils/split.js} +7 -0
- package/out/src/utils/split.js.map +1 -0
- package/out/src/utils/tokenize.d.ts.map +1 -0
- package/out/src/utils/tokenize.js.map +1 -0
- package/out/src/{wof-json.d.ts → utils/wof-json.d.ts} +2 -2
- package/out/src/utils/wof-json.d.ts.map +1 -0
- package/out/src/{wof-json.js → utils/wof-json.js} +5 -8
- package/out/src/utils/wof-json.js.map +1 -0
- package/package.json +66 -20
- package/src/adapters/ban/adapter.ts +58 -68
- package/src/adapters/ban/street-decompose.ts +17 -25
- package/src/adapters/fcc-bdc/adapter.ts +5 -36
- package/src/adapters/geonames/adapter.ts +67 -76
- package/src/adapters/geonames-postal/adapter.ts +45 -54
- package/src/adapters/gnaf/adapter.ts +7 -11
- package/src/adapters/gnaf/assemble.ts +7 -11
- package/src/adapters/index.ts +3 -2
- package/src/adapters/openaddresses/adapter.ts +6 -12
- package/src/adapters/overture/adapter.ts +6 -10
- package/src/adapters/state-hi-schools/adapter.ts +53 -77
- package/src/adapters/state-ia-contractors/adapter.ts +55 -80
- package/src/adapters/state-ny-notaries/adapter.ts +64 -89
- package/src/adapters/state-tx-notaries/adapter.ts +63 -88
- package/src/adapters/synth-po-box/adapter.ts +12 -23
- package/src/adapters/tiger/adapter.ts +4 -3
- package/src/adapters/tiger/street-decompose.ts +21 -31
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +64 -88
- package/src/adapters/usgov-imls-pls/adapter.ts +57 -86
- package/src/adapters/usgov-irs-bmf/adapter.ts +55 -66
- package/src/adapters/usgov-nad/adapter.ts +7 -11
- package/src/adapters/usgov-nppes/adapter.ts +54 -79
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +48 -75
- package/src/{adapter.ts → adapters/utils/index.ts} +59 -3
- package/src/adapters/wof-admin-jp/adapter.ts +35 -9
- package/src/adapters/wof-admin-json/adapter.ts +3 -4
- package/src/adapters/wof-postalcode-json/adapter.ts +3 -4
- package/src/build.ts +12 -11
- package/src/index.ts +3 -17
- package/src/name-prone-us-suffixes.ts +12 -0
- package/src/parquet-wrapper/reader.ts +19 -0
- package/src/parquet-wrapper/schema.ts +4 -0
- package/src/runner.ts +7 -2
- package/src/shard-recipes/anchor-absorption.ts +5 -4
- package/src/shard-recipes/boundary-stress.ts +6 -2
- package/src/shard-recipes/country-balanced.ts +77 -91
- package/src/shard-recipes/fr-admin-split.ts +3 -3
- package/src/shard-recipes/fr-fragment.ts +10 -7
- package/src/shard-recipes/fr-lieudit.ts +3 -3
- package/src/shard-recipes/fr-order.ts +18 -83
- package/src/shard-recipes/german.ts +17 -80
- package/src/shard-recipes/house-venue.ts +2 -1
- package/src/shard-recipes/index.ts +4 -1
- package/src/shard-recipes/intersection.ts +37 -74
- package/src/shard-recipes/locale.ts +31 -39
- package/src/shard-recipes/no-fragment.ts +10 -7
- package/src/shard-recipes/no-street-led.ts +10 -7
- package/src/shard-recipes/no-street.ts +2 -1
- package/src/shard-recipes/po-box-cedex.ts +127 -161
- package/src/shard-recipes/po-box.ts +6 -1
- package/src/shard-recipes/scaffold.ts +125 -27
- package/src/shard-recipes/street-affix.ts +335 -134
- package/src/shard-recipes/street-bare.ts +4 -3
- package/src/shard-recipes/street.ts +3 -2
- package/src/shard-recipes/sub-venue-sources.ts +895 -0
- package/src/shard-recipes/sub-venue.ts +982 -0
- package/src/shard-recipes/unit.ts +27 -85
- package/src/{synthesize-boundary-stress.ts → synthesizers/boundary-stress.ts} +2 -2
- package/src/{synthesize-german.ts → synthesizers/german.ts} +2 -2
- package/src/{synthesize-house-venue.ts → synthesizers/house-venue.ts} +4 -4
- package/src/{synthesize-intersection.ts → synthesizers/intersection.ts} +1 -1
- package/src/{synthesize-no-street.ts → synthesizers/no-street.ts} +2 -2
- package/src/{synthesize-po-box.ts → synthesizers/po-box.ts} +1 -1
- package/src/{synthesize-street.ts → synthesizers/street.ts} +2 -2
- package/src/{synthesize.ts → synthesizers/utils.ts} +4 -18
- package/src/tools/align-shard.ts +4 -7
- package/src/tools/audit.ts +6 -5
- package/src/tools/corpus-stats.ts +25 -33
- package/src/tools/fetch/download.ts +7 -5
- package/src/tools/fetch/imls-pls.ts +4 -18
- package/src/tools/fetch/index.ts +14 -1
- package/src/tools/fetch/nad.ts +1 -1
- package/src/tools/fetch/nppes.ts +9 -16
- package/src/tools/fetch/openaddresses.ts +1 -1
- package/src/tools/fetch/ourairports.ts +166 -0
- package/src/tools/fetch/wikidata-subvenue.ts +386 -0
- package/src/tools/golden-expand.ts +18 -13
- package/src/tools/golden-promote.ts +15 -6
- package/src/tools/golden-relabel-street.ts +685 -0
- package/src/tools/index.ts +4 -0
- package/src/tools/jsonl-to-parquet.ts +3 -2
- package/src/tools/lint-shard-vocab.ts +3 -2
- package/src/tools/lint-shard.ts +33 -36
- package/src/tools/overlay-manifest.ts +2 -1
- package/src/tools/overture-subvenue.ts +215 -0
- package/src/tools/shard-kryptonite.ts +8 -9
- package/src/tools/shard-translit.ts +22 -11
- package/src/tools/sub-venue-lexicon.ts +1250 -0
- package/src/tools/sub-venue-promotions.ts +330 -0
- package/src/{align.ts → utils/align.ts} +1 -1
- package/src/{golden.ts → utils/golden.ts} +25 -11
- package/src/utils/index.ts +14 -0
- package/src/{parquet.ts → utils/parquet.ts} +5 -19
- package/src/{split.ts → utils/split.ts} +8 -2
- package/src/{wof-json.ts → utils/wof-json.ts} +5 -8
- package/out/src/adapter.d.ts.map +0 -1
- package/out/src/adapter.js.map +0 -1
- package/out/src/align.d.ts.map +0 -1
- package/out/src/align.js.map +0 -1
- package/out/src/format.d.ts +0 -14
- package/out/src/format.d.ts.map +0 -1
- package/out/src/format.js +0 -14
- package/out/src/format.js.map +0 -1
- package/out/src/golden.d.ts.map +0 -1
- package/out/src/golden.js.map +0 -1
- package/out/src/license.d.ts.map +0 -1
- package/out/src/license.js.map +0 -1
- package/out/src/parquet.d.ts.map +0 -1
- package/out/src/parquet.js.map +0 -1
- package/out/src/split.d.ts.map +0 -1
- package/out/src/split.js.map +0 -1
- package/out/src/synthesize-anchor-absorption.d.ts.map +0 -1
- package/out/src/synthesize-anchor-absorption.js.map +0 -1
- package/out/src/synthesize-boundary-stress.d.ts.map +0 -1
- package/out/src/synthesize-boundary-stress.js.map +0 -1
- package/out/src/synthesize-german.d.ts.map +0 -1
- package/out/src/synthesize-german.js.map +0 -1
- package/out/src/synthesize-house-venue.d.ts.map +0 -1
- package/out/src/synthesize-house-venue.js.map +0 -1
- package/out/src/synthesize-intersection.d.ts.map +0 -1
- package/out/src/synthesize-intersection.js.map +0 -1
- package/out/src/synthesize-no-street.d.ts.map +0 -1
- package/out/src/synthesize-no-street.js.map +0 -1
- package/out/src/synthesize-po-box.d.ts.map +0 -1
- package/out/src/synthesize-po-box.js.map +0 -1
- package/out/src/synthesize-street.d.ts.map +0 -1
- package/out/src/synthesize-street.js.map +0 -1
- package/out/src/synthesize.d.ts.map +0 -1
- package/out/src/synthesize.js.map +0 -1
- package/out/src/tokenize.d.ts.map +0 -1
- package/out/src/tokenize.js.map +0 -1
- package/out/src/wof-json.d.ts.map +0 -1
- package/out/src/wof-json.js.map +0 -1
- package/src/format.ts +0 -14
- /package/out/src/{align.js → utils/align.js} +0 -0
- /package/out/src/{license.d.ts → utils/license.d.ts} +0 -0
- /package/out/src/{license.js → utils/license.js} +0 -0
- /package/out/src/{tokenize.d.ts → utils/tokenize.d.ts} +0 -0
- /package/out/src/{tokenize.js → utils/tokenize.js} +0 -0
- /package/src/{synthesize-anchor-absorption.ts → synthesizers/anchor-absorption.ts} +0 -0
- /package/src/{license.ts → utils/license.ts} +0 -0
- /package/src/{tokenize.ts → utils/tokenize.ts} +0 -0
|
@@ -20,9 +20,9 @@
|
|
|
20
20
|
* locality-hash holdout (hash%10===0).
|
|
21
21
|
*
|
|
22
22
|
* Prerequisites (read once, before the generation loop — these do NOT consume `random`): the cached
|
|
23
|
-
* OA zips in /
|
|
24
|
-
* GeoNames POSTAL-CODE dumps for AU/NZ
|
|
25
|
-
*
|
|
23
|
+
* OA zips in `$MAILWOMAN_DATA_ROOT/oa-cache`, the GeoNames Canada dump at
|
|
24
|
+
* `$MAILWOMAN_DATA_ROOT/geonames/CA.zip`, and the GeoNames POSTAL-CODE dumps for AU/NZ
|
|
25
|
+
* (`$MAILWOMAN_DATA_ROOT/geonames-postal/{AU,NZ}.zip`).
|
|
26
26
|
*
|
|
27
27
|
* Byte-fidelity: the legacy script seeded its own mulberry32 from `--seed`
|
|
28
28
|
* (`mulberry32(opts.seed)`); this recipe re-creates the SAME generator
|
|
@@ -30,22 +30,32 @@
|
|
|
30
30
|
* reproduces the legacy run byte-for-byte.
|
|
31
31
|
*/
|
|
32
32
|
|
|
33
|
-
import {
|
|
33
|
+
import { existsSync } from "node:fs"
|
|
34
34
|
|
|
35
35
|
import { isAuDeliveryService, isAuPostcode, isAuStateAbbreviation } from "@mailwoman/codex/au"
|
|
36
36
|
import { FSA_LETTER_TO_PROVINCE, normalizeCaPostalCode } from "@mailwoman/codex/ca"
|
|
37
37
|
import { isCedex } from "@mailwoman/codex/fr"
|
|
38
|
-
import {
|
|
38
|
+
import { isNZDeliveryService, isNZPostcode } from "@mailwoman/codex/nz"
|
|
39
39
|
import { isPOBox } from "@mailwoman/codex/us"
|
|
40
|
-
|
|
41
|
-
import {
|
|
40
|
+
import { readZipEntry } from "@mailwoman/core/fs/zip"
|
|
41
|
+
import { dataRootPath } from "@mailwoman/core/utils"
|
|
42
42
|
import {
|
|
43
43
|
maybeNoisifyBoxNumber,
|
|
44
44
|
PO_BOX_LOCALE_TEMPLATES,
|
|
45
45
|
synthesizeMilitaryPoBoxRow,
|
|
46
46
|
type LocaleTemplate,
|
|
47
|
-
} from "
|
|
48
|
-
import {
|
|
47
|
+
} from "@mailwoman/corpus/synthesizers/po-box"
|
|
48
|
+
import { alignRow } from "@mailwoman/corpus/utils"
|
|
49
|
+
import { TextSpliterator, TSVSpliterator } from "spliterator"
|
|
50
|
+
|
|
51
|
+
import {
|
|
52
|
+
makeMulberry32,
|
|
53
|
+
readCSVRecords,
|
|
54
|
+
readZippedCSVRecords,
|
|
55
|
+
shardSourceID,
|
|
56
|
+
type CanonicalShardRow,
|
|
57
|
+
type ShardRecipe,
|
|
58
|
+
} from "./scaffold.ts"
|
|
49
59
|
|
|
50
60
|
// ── Base-skeleton sources ────────────────────────────────────────────────────────────────────────
|
|
51
61
|
// Same OA cache as the unit/affix shards. US train = every NON-Vermont state; US eval = Vermont (the
|
|
@@ -58,20 +68,29 @@ import { makeMulberry32, shardSourceID, type CanonicalShardRow, type ShardRecipe
|
|
|
58
68
|
extracted as named constants above. */
|
|
59
69
|
|
|
60
70
|
const US_TRAIN_SOURCES = [
|
|
61
|
-
{ zip: "
|
|
62
|
-
{ zip: "
|
|
63
|
-
{ zip: "
|
|
64
|
-
{ zip: "
|
|
65
|
-
{ zip: "
|
|
66
|
-
{ zip: "
|
|
67
|
-
{ zip: "
|
|
71
|
+
{ zip: dataRootPath("oa-cache", "us__ca__berkeley.zip"), csv: "us/ca/berkeley.csv", region: "CA" },
|
|
72
|
+
{ zip: dataRootPath("oa-cache", "us__ca__marin.zip"), csv: "us/ca/marin.csv", region: "CA" },
|
|
73
|
+
{ zip: dataRootPath("oa-cache", "us__dc__statewide.zip"), csv: "us/dc/statewide.csv", region: "DC" },
|
|
74
|
+
{ zip: dataRootPath("oa-cache", "us__ia__statewide.zip"), csv: "us/ia/statewide.csv", region: "IA" },
|
|
75
|
+
{ zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv", region: "IL" },
|
|
76
|
+
{ zip: dataRootPath("oa-cache", "us__mt__statewide.zip"), csv: "us/mt/statewide.csv", region: "MT" },
|
|
77
|
+
{ zip: dataRootPath("oa-cache", "us__sd__statewide.zip"), csv: "us/sd/statewide.csv", region: "SD" },
|
|
68
78
|
]
|
|
69
79
|
|
|
70
|
-
const US_EVAL_SOURCE = {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
80
|
+
const US_EVAL_SOURCE = {
|
|
81
|
+
zip: dataRootPath("oa-cache", "us__vt__statewide.zip"),
|
|
82
|
+
csv: "us/vt/statewide.csv",
|
|
83
|
+
region: "VT",
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const FR_SOURCE = { zip: dataRootPath("oa-cache", "fr__countrywide.zip"), csv: "fr/countrywide.csv" }
|
|
87
|
+
// GeoNames has two per-country exports with different schemas, and this recipe reads both. The MAIN
|
|
88
|
+
// dump (feature class + population columns) lives under `geonames/`; the POSTAL-CODE dump lives
|
|
89
|
+
// under `geonames-postal/`, where the directory carries the "postal" distinction the filename used
|
|
90
|
+
// to.
|
|
91
|
+
const GEONAMES_CA = dataRootPath("geonames", "CA.zip")
|
|
92
|
+
const GEONAMES_POSTAL_AU = { zip: dataRootPath("geonames-postal", "AU.zip"), txt: "AU.txt" }
|
|
93
|
+
const GEONAMES_POSTAL_NZ = { zip: dataRootPath("geonames-postal", "NZ.zip"), txt: "NZ.txt" }
|
|
75
94
|
|
|
76
95
|
/**
|
|
77
96
|
* ── Surface vocabulary (codex + corpus templates — see the header) ──────────────────────────────.
|
|
@@ -172,20 +191,20 @@ interface USTuple {
|
|
|
172
191
|
postcode: string
|
|
173
192
|
}
|
|
174
193
|
|
|
175
|
-
interface
|
|
194
|
+
interface FRTuple {
|
|
176
195
|
house_number: string
|
|
177
196
|
street: string
|
|
178
197
|
locality: string
|
|
179
198
|
postcode: string
|
|
180
199
|
}
|
|
181
200
|
|
|
182
|
-
interface
|
|
201
|
+
interface AUTuple {
|
|
183
202
|
locality: string
|
|
184
203
|
region: string
|
|
185
204
|
postcode: string
|
|
186
205
|
}
|
|
187
206
|
|
|
188
|
-
interface
|
|
207
|
+
interface NZTuple {
|
|
189
208
|
locality: string
|
|
190
209
|
postcode: string
|
|
191
210
|
}
|
|
@@ -214,44 +233,6 @@ function localityHash(name: string): number {
|
|
|
214
233
|
|
|
215
234
|
const isHoldoutLocality = (name: string): boolean => localityHash(name) % 10 === 0
|
|
216
235
|
|
|
217
|
-
/**
|
|
218
|
-
* Minimal RFC-4180-ish splitter (handles quoted fields).
|
|
219
|
-
*/
|
|
220
|
-
function splitCSV(line: string): string[] {
|
|
221
|
-
const out: string[] = []
|
|
222
|
-
let cur = ""
|
|
223
|
-
let inQ = false
|
|
224
|
-
|
|
225
|
-
for (let i = 0; i < line.length; i++) {
|
|
226
|
-
const c = line[i]!
|
|
227
|
-
|
|
228
|
-
if (inQ) {
|
|
229
|
-
if (c === '"') {
|
|
230
|
-
if (line[i + 1] === '"') {
|
|
231
|
-
cur += '"'
|
|
232
|
-
|
|
233
|
-
i++
|
|
234
|
-
} else {
|
|
235
|
-
inQ = false
|
|
236
|
-
}
|
|
237
|
-
} else {
|
|
238
|
-
cur += c
|
|
239
|
-
}
|
|
240
|
-
} else if (c === '"') {
|
|
241
|
-
inQ = true
|
|
242
|
-
} else if (c === ",") {
|
|
243
|
-
out.push(cur)
|
|
244
|
-
cur = ""
|
|
245
|
-
} else {
|
|
246
|
-
cur += c
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
out.push(cur)
|
|
251
|
-
|
|
252
|
-
return out
|
|
253
|
-
}
|
|
254
|
-
|
|
255
236
|
const MAX_LOCALITY_LENGTH = 40
|
|
256
237
|
|
|
257
238
|
const cleanLocality = (loc: string) =>
|
|
@@ -260,93 +241,75 @@ const cleanLocality = (loc: string) =>
|
|
|
260
241
|
/**
|
|
261
242
|
* Stream real US tuples (number/street/city/postcode) out of a cached OA zip.
|
|
262
243
|
*/
|
|
263
|
-
function readUsTuples(source: { zip: string; csv: string; region: string }): USTuple[] {
|
|
264
|
-
const r = spawnSync("unzip", ["-p", source.zip, source.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
|
|
265
|
-
|
|
266
|
-
if (r.status !== 0) {
|
|
267
|
-
console.error(` WARN: unzip failed for ${source.zip} (status ${r.status})`)
|
|
268
|
-
|
|
269
|
-
return []
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
const lines = r.stdout.toString("utf8").split(/\r?\n/)
|
|
273
|
-
|
|
274
|
-
if (lines.length < 2) return []
|
|
275
|
-
const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
|
|
276
|
-
const idx = (name: string) => header.indexOf(name)
|
|
277
|
-
|
|
278
|
-
const iNum = idx("number"),
|
|
279
|
-
iStreet = idx("street"),
|
|
280
|
-
iCity = idx("city"),
|
|
281
|
-
iPost = idx("postcode")
|
|
282
|
-
|
|
283
|
-
const get = (cells: string[], i: number) => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
|
|
244
|
+
async function readUsTuples(source: { zip: string; csv: string; region: string }): Promise<USTuple[]> {
|
|
284
245
|
const tuples: USTuple[] = []
|
|
285
246
|
const seen = new Set<string>()
|
|
286
247
|
|
|
287
|
-
for (
|
|
288
|
-
|
|
289
|
-
const cells = splitCSV(lines[li]!)
|
|
290
|
-
const locality = get(cells, iCity)
|
|
248
|
+
for await (const row of readZippedCSVRecords(source.zip, source.csv)) {
|
|
249
|
+
const locality = row.city ?? ""
|
|
291
250
|
|
|
292
251
|
if (!cleanLocality(locality)) continue
|
|
293
252
|
const key = locality.toLowerCase()
|
|
294
253
|
|
|
295
|
-
const street =
|
|
296
|
-
house_number =
|
|
254
|
+
const street = row.street ?? "",
|
|
255
|
+
house_number = row.number ?? ""
|
|
297
256
|
|
|
298
257
|
// One tuple per (locality, street) pair keeps the pool varied without ballooning memory.
|
|
299
258
|
const pairKey = `${key}|${street}`.toLowerCase()
|
|
300
259
|
|
|
301
260
|
if (seen.has(pairKey)) continue
|
|
302
261
|
seen.add(pairKey)
|
|
303
|
-
tuples.push({ house_number, street, locality, region: source.region, postcode:
|
|
262
|
+
tuples.push({ house_number, street, locality, region: source.region, postcode: row.postcode ?? "" })
|
|
304
263
|
}
|
|
305
264
|
|
|
306
265
|
return tuples
|
|
307
266
|
}
|
|
308
267
|
|
|
309
268
|
/**
|
|
310
|
-
*
|
|
311
|
-
*
|
|
312
|
-
* parsing stays in splitCSV.
|
|
269
|
+
* The FR stride: keep line 3, then every 211th. Both numbers are the awk pre-filter's, kept exactly — they select which
|
|
270
|
+
* rows land in the shard, so changing either re-rolls the sample.
|
|
313
271
|
*/
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
"bash",
|
|
317
|
-
["-c", `unzip -p "${FR_SOURCE.zip}" "${FR_SOURCE.csv}" | awk 'NR==1 || NR%211==3' | head -n ${limit + 1}`],
|
|
318
|
-
{ maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" }
|
|
319
|
-
)
|
|
272
|
+
const FR_STRIDE = 211
|
|
273
|
+
const FR_STRIDE_OFFSET = 3
|
|
320
274
|
|
|
321
|
-
|
|
322
|
-
|
|
275
|
+
/**
|
|
276
|
+
* GeoNames population floor for a Canadian locality to enter the pool.
|
|
277
|
+
*/
|
|
278
|
+
const CA_LOCALITY_MIN_POPULATION = 1000
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Stride-sampled FR tuples (number/street/city/postcode). The countrywide CSV is 2.5 GB and insee-ordered; `awk NR%K`
|
|
282
|
+
* strides the whole country instead of reading one département.
|
|
283
|
+
*
|
|
284
|
+
* The stride counts PHYSICAL lines, so a record carrying a newline inside a quoted field is two lines to the stride and
|
|
285
|
+
* can be cut in half by it. That is a sampling artefact of striding before the parse, not of the parse: whichever lines
|
|
286
|
+
* survive are re-assembled into records by the CSV reader, and a halved record fails the field checks below. Hence
|
|
287
|
+
* `skipEmpty: false` and no quote handling on the line split — a blank line still advances the count, exactly as it did
|
|
288
|
+
* when this was `awk NR`, and the rows the stride selects are the rows every existing FR shard was built from.
|
|
289
|
+
*/
|
|
290
|
+
async function readFrTuples(limit: number): Promise<FRTuple[]> {
|
|
291
|
+
if (!existsSync(FR_SOURCE.zip)) {
|
|
292
|
+
console.error(` WARN: ${FR_SOURCE.zip} is not cached — skipping ${FR_SOURCE.csv}`)
|
|
323
293
|
|
|
324
294
|
return []
|
|
325
295
|
}
|
|
326
296
|
|
|
327
|
-
const
|
|
297
|
+
const encoder = new TextEncoder()
|
|
328
298
|
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
299
|
+
const strided = TextSpliterator.fromAsync(readZipEntry(FR_SOURCE.zip, FR_SOURCE.csv), { skipEmpty: false })
|
|
300
|
+
// `index` is 0-based where awk's NR is 1-based, hence the +1: keep the header, then every 211th line at offset 3.
|
|
301
|
+
.filter((_line, index) => index === 0 || (index + 1) % FR_STRIDE === FR_STRIDE_OFFSET)
|
|
302
|
+
.take(limit + 1)
|
|
303
|
+
.map((line) => encoder.encode(line + "\n"))
|
|
332
304
|
|
|
333
|
-
const
|
|
334
|
-
iStreet = idx("street"),
|
|
335
|
-
iCity = idx("city"),
|
|
336
|
-
iPost = idx("postcode")
|
|
337
|
-
|
|
338
|
-
const get = (cells: string[], i: number) => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
|
|
339
|
-
const tuples: FrTuple[] = []
|
|
305
|
+
const tuples: FRTuple[] = []
|
|
340
306
|
const seen = new Set<string>()
|
|
341
307
|
|
|
342
|
-
for (
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
postcode = get(cells, iPost),
|
|
348
|
-
street = get(cells, iStreet),
|
|
349
|
-
house_number = get(cells, iNum)
|
|
308
|
+
for await (const row of readCSVRecords(strided)) {
|
|
309
|
+
const locality = row.city ?? "",
|
|
310
|
+
postcode = row.postcode ?? "",
|
|
311
|
+
street = row.street ?? "",
|
|
312
|
+
house_number = row.number ?? ""
|
|
350
313
|
|
|
351
314
|
if (!cleanLocality(locality) || !/^\d{5}$/.test(postcode) || !street || !house_number) continue
|
|
352
315
|
const key = `${locality}|${street}`.toLowerCase()
|
|
@@ -363,23 +326,29 @@ function readFrTuples(limit: number): FrTuple[] {
|
|
|
363
326
|
* Canadian locality pools from the GeoNames dump (CC-BY 4.0): feature class P, admin1 10 (Québec) / 08 (Ontario),
|
|
364
327
|
* population > 1000. GeoNames is the provenance-tracked source — no hand list.
|
|
365
328
|
*/
|
|
366
|
-
function readCaLocalities(admin1: string): string[] {
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
[
|
|
370
|
-
"-c",
|
|
371
|
-
`unzip -p "${GEONAMES_CA}" CA.txt | awk -F'\\t' '$7=="P" && $9=="CA" && $11=="${admin1}" && $15>1000 {print $2}'`,
|
|
372
|
-
],
|
|
373
|
-
{ maxBuffer: 1024 * 1024 * 256, encoding: "utf8" }
|
|
374
|
-
)
|
|
375
|
-
|
|
376
|
-
if (r.status !== 0) {
|
|
377
|
-
console.error(` WARN: GeoNames read failed (admin1=${admin1}) — is ${GEONAMES_CA} present?`)
|
|
329
|
+
async function readCaLocalities(admin1: string): Promise<string[]> {
|
|
330
|
+
if (!existsSync(GEONAMES_CA)) {
|
|
331
|
+
console.error(` WARN: ${GEONAMES_CA} is not cached — skipping CA localities (admin1=${admin1})`)
|
|
378
332
|
|
|
379
333
|
return []
|
|
380
334
|
}
|
|
381
335
|
|
|
382
|
-
|
|
336
|
+
const localities = new Set<string>()
|
|
337
|
+
|
|
338
|
+
// GeoNames main dump columns, 0-based: 1 name, 6 feature class, 8 country, 10 admin1, 14 population.
|
|
339
|
+
for await (const columns of TSVSpliterator.fromAsync(readZipEntry(GEONAMES_CA, "CA.txt"), { header: false })) {
|
|
340
|
+
if (columns[6] !== "P" || columns[8] !== "CA" || columns[10] !== admin1) continue
|
|
341
|
+
|
|
342
|
+
if (Number(columns[14]) <= CA_LOCALITY_MIN_POPULATION) continue
|
|
343
|
+
|
|
344
|
+
const name = columns[1] ?? ""
|
|
345
|
+
|
|
346
|
+
if (cleanLocality(name)) {
|
|
347
|
+
localities.add(name)
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
return [...localities]
|
|
383
352
|
}
|
|
384
353
|
|
|
385
354
|
/**
|
|
@@ -389,25 +358,22 @@ function readCaLocalities(admin1: string): string[] {
|
|
|
389
358
|
* validated against the codex 4-digit shape — a dump row that fails the contract is skipped, not emitted as a junk
|
|
390
359
|
* label.
|
|
391
360
|
*/
|
|
392
|
-
function readPostalTuples(
|
|
361
|
+
async function readPostalTuples(
|
|
393
362
|
source: { zip: string; txt: string },
|
|
394
363
|
opts: { withState: boolean }
|
|
395
|
-
): Array<
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
if (r.status !== 0) {
|
|
399
|
-
console.error(` WARN: unzip failed for ${source.zip} (status ${r.status})`)
|
|
364
|
+
): Promise<Array<AUTuple | NZTuple>> {
|
|
365
|
+
if (!existsSync(source.zip)) {
|
|
366
|
+
console.error(` WARN: ${source.zip} is not cached — skipping ${source.txt}`)
|
|
400
367
|
|
|
401
368
|
return []
|
|
402
369
|
}
|
|
403
370
|
|
|
404
|
-
const tuples: Array<
|
|
371
|
+
const tuples: Array<AUTuple | NZTuple> = []
|
|
405
372
|
const seen = new Set<string>()
|
|
406
|
-
const validPostcode = opts.withState ? isAuPostcode :
|
|
373
|
+
const validPostcode = opts.withState ? isAuPostcode : isNZPostcode
|
|
407
374
|
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
const cols = line.split("\t")
|
|
375
|
+
// A GeoNames dump carries no header row, so every line is data.
|
|
376
|
+
for await (const cols of TSVSpliterator.fromAsync(readZipEntry(source.zip, source.txt), { header: false })) {
|
|
411
377
|
const postcode = (cols[1] ?? "").trim()
|
|
412
378
|
const locality = (cols[2] ?? "").trim()
|
|
413
379
|
const region = (cols[4] ?? "").trim()
|
|
@@ -514,7 +480,7 @@ function makeCedex(random: () => number): string {
|
|
|
514
480
|
/**
|
|
515
481
|
* Compose an AU or NZ delivery-service phrase from the codex-sourced leaders. Same contract as makePoBoxPhrase: a
|
|
516
482
|
* phrase built from a codex-known designator and a clean id must round-trip the codex matcher (isAuDeliveryService /
|
|
517
|
-
*
|
|
483
|
+
* isNZDeliveryService) — a failure is a generation bug, loud. Noisy ids (commas / embedded spaces) are corpus-designed
|
|
518
484
|
* adversarial forms, exempt.
|
|
519
485
|
*/
|
|
520
486
|
function makeAuNzPoBoxPhrase(
|
|
@@ -532,14 +498,14 @@ function makeAuNzPoBoxPhrase(
|
|
|
532
498
|
let num = maybeNoisifyBoxNumber(pickBoxNumber(random), random)
|
|
533
499
|
|
|
534
500
|
// NZ CMB identifiers are alpha-led per the ADV358 example ("CMB B99").
|
|
535
|
-
if (leader === "CMB" && validate ===
|
|
501
|
+
if (leader === "CMB" && validate === isNZDeliveryService) {
|
|
536
502
|
num = `B${num}`
|
|
537
503
|
}
|
|
538
504
|
|
|
539
505
|
const phrase = `${caseDial(random, leader)} ${num}`
|
|
540
506
|
// The "clean id" shape differs per system: ADV358 identifiers carry no separators at all, the AU
|
|
541
507
|
// AMAS id (like the US one) tolerates dashes. Noisy ids outside the clean shape are exempt.
|
|
542
|
-
const cleanID = validate ===
|
|
508
|
+
const cleanID = validate === isNZDeliveryService ? /^[\dA-Za-z]+$/ : /^[\dA-Za-z][\dA-Za-z-]*$/
|
|
543
509
|
|
|
544
510
|
if (cleanID.test(num) && !validate(phrase)) {
|
|
545
511
|
throw new Error(`generated a phrase the codex matcher rejects: "${phrase}"`)
|
|
@@ -614,7 +580,7 @@ function renderPmbUs(random: () => number, t: USTuple): Rendered {
|
|
|
614
580
|
return { fmt: "pmb-bare", raw: `${road} ${phrase}`, components: { house_number: hn, street, po_box: phrase } }
|
|
615
581
|
}
|
|
616
582
|
|
|
617
|
-
function renderBpFr(random: () => number, t:
|
|
583
|
+
function renderBpFr(random: () => number, t: FRTuple): Rendered {
|
|
618
584
|
const phrase = makePoBoxPhrase(random, FR_LEADERS)
|
|
619
585
|
const { locality, postcode: pc } = t
|
|
620
586
|
const upper = random() < 0.5
|
|
@@ -651,7 +617,7 @@ function renderBpFr(random: () => number, t: FrTuple): Rendered {
|
|
|
651
617
|
}
|
|
652
618
|
}
|
|
653
619
|
|
|
654
|
-
function renderCedexFr(random: () => number, t:
|
|
620
|
+
function renderCedexFr(random: () => number, t: FRTuple): Rendered {
|
|
655
621
|
const cedex = makeCedex(random)
|
|
656
622
|
const { house_number: hn, street, locality, postcode: pc } = t
|
|
657
623
|
const loc = random() < 0.6 ? locality.toUpperCase() : locality
|
|
@@ -703,7 +669,7 @@ function renderCaEn(random: () => number, loc: string): Rendered {
|
|
|
703
669
|
return { fmt: "ca-en-bare", raw: phrase, components: { po_box: phrase } }
|
|
704
670
|
}
|
|
705
671
|
|
|
706
|
-
function
|
|
672
|
+
function renderAUPoBox(random: () => number, t: AUTuple): Rendered {
|
|
707
673
|
const phrase = makeAuNzPoBoxPhrase(random, AU_LEADERS_CURRENT, AU_LEADERS_LEGACY, isAuDeliveryService)
|
|
708
674
|
const { locality, region: reg, postcode: pc } = t
|
|
709
675
|
const r = random()
|
|
@@ -737,8 +703,8 @@ function renderAuPoBox(random: () => number, t: AuTuple): Rendered {
|
|
|
737
703
|
return { fmt: "au-venue", raw: `${v}, ${phrase}, ${loc} ${reg} ${pc}`, components: { venue: v, ...base } }
|
|
738
704
|
}
|
|
739
705
|
|
|
740
|
-
function
|
|
741
|
-
const phrase = makeAuNzPoBoxPhrase(random, NZ_LEADERS_COMMON, NZ_LEADERS_RARE,
|
|
706
|
+
function renderNZPoBox(random: () => number, t: NZTuple): Rendered {
|
|
707
|
+
const phrase = makeAuNzPoBoxPhrase(random, NZ_LEADERS_COMMON, NZ_LEADERS_RARE, isNZDeliveryService)
|
|
742
708
|
const { locality, postcode: pc } = t
|
|
743
709
|
// NZ addresses are written mixed-case ("PO Box 4099, Timaru 7942") — no region line (ADV358).
|
|
744
710
|
const base = { po_box: phrase, locality, postcode: pc }
|
|
@@ -795,7 +761,7 @@ export const poBoxCedexRecipe: ShardRecipe = {
|
|
|
795
761
|
const usPool: USTuple[] = []
|
|
796
762
|
|
|
797
763
|
for (const s of opts.golden ? [US_EVAL_SOURCE] : US_TRAIN_SOURCES) {
|
|
798
|
-
const t = readUsTuples(s)
|
|
764
|
+
const t = await readUsTuples(s)
|
|
799
765
|
|
|
800
766
|
console.error(` ${s.csv}: ${t.length} tuples`)
|
|
801
767
|
|
|
@@ -805,21 +771,21 @@ export const poBoxCedexRecipe: ShardRecipe = {
|
|
|
805
771
|
}
|
|
806
772
|
|
|
807
773
|
// FR + CA pools: stable locality-hash holdout (golden gets hash%10==0, train the rest).
|
|
808
|
-
const frAll = readFrTuples(80_000)
|
|
774
|
+
const frAll = await readFrTuples(80_000)
|
|
809
775
|
const frPool = frAll.filter((t) => isHoldoutLocality(t.locality) === opts.golden)
|
|
810
776
|
|
|
811
777
|
console.error(` ${FR_SOURCE.csv}: ${frAll.length} tuples (${frPool.length} after holdout split)`)
|
|
812
778
|
|
|
813
|
-
const qcAll = readCaLocalities("10")
|
|
814
|
-
const onAll = readCaLocalities("08")
|
|
779
|
+
const qcAll = await readCaLocalities("10")
|
|
780
|
+
const onAll = await readCaLocalities("08")
|
|
815
781
|
const qcPool = qcAll.filter((l) => isHoldoutLocality(l) === opts.golden)
|
|
816
782
|
const onPool = onAll.filter((l) => isHoldoutLocality(l) === opts.golden)
|
|
817
783
|
|
|
818
784
|
console.error(` GeoNames CA: QC ${qcAll.length}→${qcPool.length}, ON ${onAll.length}→${onPool.length}`)
|
|
819
785
|
|
|
820
786
|
// AU/NZ pools: same stable locality-hash holdout as FR/CA.
|
|
821
|
-
const auAll = readPostalTuples(GEONAMES_POSTAL_AU, { withState: true }) as
|
|
822
|
-
const nzAll = readPostalTuples(GEONAMES_POSTAL_NZ, { withState: false }) as
|
|
787
|
+
const auAll = (await readPostalTuples(GEONAMES_POSTAL_AU, { withState: true })) as AUTuple[]
|
|
788
|
+
const nzAll = (await readPostalTuples(GEONAMES_POSTAL_NZ, { withState: false })) as NZTuple[]
|
|
823
789
|
const auPool = auAll.filter((t) => isHoldoutLocality(t.locality) === opts.golden)
|
|
824
790
|
const nzPool = nzAll.filter((t) => isHoldoutLocality(t.locality) === opts.golden)
|
|
825
791
|
|
|
@@ -827,7 +793,7 @@ export const poBoxCedexRecipe: ShardRecipe = {
|
|
|
827
793
|
|
|
828
794
|
if (!usPool.length || !frPool.length || !qcPool.length || !onPool.length || !auPool.length || !nzPool.length) {
|
|
829
795
|
throw new Error(
|
|
830
|
-
|
|
796
|
+
`A base pool is empty — check ${dataRootPath("oa-cache")}, ${GEONAMES_CA}, ${GEONAMES_POSTAL_AU.zip}, and ${GEONAMES_POSTAL_NZ.zip}.`
|
|
831
797
|
)
|
|
832
798
|
}
|
|
833
799
|
|
|
@@ -877,11 +843,11 @@ export const poBoxCedexRecipe: ShardRecipe = {
|
|
|
877
843
|
country = "CA"
|
|
878
844
|
locale = "fr-CA"
|
|
879
845
|
} else if (cls === "po-box-au") {
|
|
880
|
-
rendered =
|
|
846
|
+
rendered = renderAUPoBox(random, pick(random, auPool))
|
|
881
847
|
country = "AU"
|
|
882
848
|
locale = "en-AU"
|
|
883
849
|
} else if (cls === "po-box-nz") {
|
|
884
|
-
rendered =
|
|
850
|
+
rendered = renderNZPoBox(random, pick(random, nzPool))
|
|
885
851
|
country = "NZ"
|
|
886
852
|
locale = "en-NZ"
|
|
887
853
|
} else if (cls === "po-box-us-military") {
|
|
@@ -9,7 +9,12 @@
|
|
|
9
9
|
* scripts/build-po-box-shard.mjs.
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
|
-
import {
|
|
12
|
+
import {
|
|
13
|
+
synthesizeMilitaryPoBoxRow,
|
|
14
|
+
synthesizePoBoxRow,
|
|
15
|
+
type PoBoxBaseTuple,
|
|
16
|
+
} from "@mailwoman/corpus/synthesizers/po-box"
|
|
17
|
+
|
|
13
18
|
import { alignAndWrite, makeLcg, readTuples, shardSourceID, type ShardRecipe } from "./scaffold.ts"
|
|
14
19
|
|
|
15
20
|
const LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"
|