@mailwoman/corpus 8.6.0 → 9.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/data/PROVENANCE.md +214 -0
- package/data/sub-venue-lexicon.json +18269 -0
- package/out/src/adapters/ban/adapter.d.ts +5 -4
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +62 -69
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +18 -25
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +1 -7
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +4 -26
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +1 -1
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +72 -78
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +46 -51
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +6 -9
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +7 -12
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +5 -10
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +1 -1
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +5 -9
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +57 -75
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +60 -78
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +69 -87
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +71 -89
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +3 -3
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +10 -18
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +1 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +2 -2
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js +18 -27
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +4 -4
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +64 -82
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +62 -84
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +59 -66
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +6 -9
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +60 -78
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +53 -73
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/{adapter.d.ts → adapters/utils/index.d.ts} +43 -2
- package/out/src/adapters/utils/index.d.ts.map +1 -0
- package/out/src/{adapter.js → adapters/utils/index.js} +47 -2
- package/out/src/adapters/utils/index.js.map +1 -0
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +24 -6
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +2 -2
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +2 -2
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +2 -2
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +2 -2
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/build.d.ts +3 -4
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +6 -8
- package/out/src/build.js.map +1 -1
- package/out/src/index.d.ts +3 -17
- package/out/src/index.d.ts.map +1 -1
- package/out/src/index.js +3 -17
- package/out/src/index.js.map +1 -1
- package/out/src/name-prone-us-suffixes.d.ts +12 -0
- package/out/src/name-prone-us-suffixes.d.ts.map +1 -0
- package/out/src/name-prone-us-suffixes.js +12 -0
- package/out/src/name-prone-us-suffixes.js.map +1 -0
- package/out/src/parquet-wrapper/reader.d.ts +8 -0
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +16 -0
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/schema.d.ts +4 -0
- package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
- package/out/src/parquet-wrapper/schema.js.map +1 -1
- package/out/src/runner.d.ts +2 -2
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +1 -1
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +5 -4
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +2 -2
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +75 -75
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +2 -2
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +8 -5
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +2 -2
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +18 -72
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +17 -70
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/house-venue.js +1 -1
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +4 -1
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts +2 -2
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +28 -72
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +5 -5
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +25 -35
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +8 -5
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street-led.js +8 -5
- package/out/src/shard-recipes/no-street-led.js.map +1 -1
- package/out/src/shard-recipes/no-street.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street.js +1 -1
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +3 -3
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +101 -127
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +1 -1
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +58 -9
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +87 -29
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +54 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +253 -111
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
- package/out/src/shard-recipes/street-bare.js +3 -3
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.d.ts.map +1 -1
- package/out/src/shard-recipes/street.js +2 -2
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.d.ts +228 -0
- package/out/src/shard-recipes/sub-venue-sources.d.ts.map +1 -0
- package/out/src/shard-recipes/sub-venue-sources.js +664 -0
- package/out/src/shard-recipes/sub-venue-sources.js.map +1 -0
- package/out/src/shard-recipes/sub-venue.d.ts +269 -0
- package/out/src/shard-recipes/sub-venue.d.ts.map +1 -0
- package/out/src/shard-recipes/sub-venue.js +709 -0
- package/out/src/shard-recipes/sub-venue.js.map +1 -0
- package/out/src/shard-recipes/unit.d.ts +1 -1
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +26 -74
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/{synthesize-anchor-absorption.d.ts → synthesizers/anchor-absorption.d.ts} +1 -1
- package/out/src/synthesizers/anchor-absorption.d.ts.map +1 -0
- package/out/src/{synthesize-anchor-absorption.js → synthesizers/anchor-absorption.js} +1 -1
- package/out/src/synthesizers/anchor-absorption.js.map +1 -0
- package/out/src/{synthesize-boundary-stress.d.ts → synthesizers/boundary-stress.d.ts} +3 -3
- package/out/src/synthesizers/boundary-stress.d.ts.map +1 -0
- package/out/src/{synthesize-boundary-stress.js → synthesizers/boundary-stress.js} +2 -2
- package/out/src/synthesizers/boundary-stress.js.map +1 -0
- package/out/src/{synthesize-german.d.ts → synthesizers/german.d.ts} +2 -2
- package/out/src/synthesizers/german.d.ts.map +1 -0
- package/out/src/{synthesize-german.js → synthesizers/german.js} +2 -2
- package/out/src/synthesizers/german.js.map +1 -0
- package/out/src/{synthesize-house-venue.d.ts → synthesizers/house-venue.d.ts} +4 -4
- package/out/src/synthesizers/house-venue.d.ts.map +1 -0
- package/out/src/{synthesize-house-venue.js → synthesizers/house-venue.js} +4 -4
- package/out/src/synthesizers/house-venue.js.map +1 -0
- package/out/src/{synthesize-intersection.d.ts → synthesizers/intersection.d.ts} +2 -2
- package/out/src/synthesizers/intersection.d.ts.map +1 -0
- package/out/src/{synthesize-intersection.js → synthesizers/intersection.js} +1 -1
- package/out/src/synthesizers/intersection.js.map +1 -0
- package/out/src/{synthesize-no-street.d.ts → synthesizers/no-street.d.ts} +3 -3
- package/out/src/synthesizers/no-street.d.ts.map +1 -0
- package/out/src/{synthesize-no-street.js → synthesizers/no-street.js} +2 -2
- package/out/src/synthesizers/no-street.js.map +1 -0
- package/out/src/{synthesize-po-box.d.ts → synthesizers/po-box.d.ts} +2 -2
- package/out/src/synthesizers/po-box.d.ts.map +1 -0
- package/out/src/{synthesize-po-box.js → synthesizers/po-box.js} +1 -1
- package/out/src/synthesizers/po-box.js.map +1 -0
- package/out/src/{synthesize-street.d.ts → synthesizers/street.d.ts} +2 -2
- package/out/src/synthesizers/street.d.ts.map +1 -0
- package/out/src/{synthesize-street.js → synthesizers/street.js} +2 -2
- package/out/src/synthesizers/street.js.map +1 -0
- package/out/src/{synthesize.d.ts → synthesizers/utils.d.ts} +3 -3
- package/out/src/synthesizers/utils.d.ts.map +1 -0
- package/out/src/{synthesize.js → synthesizers/utils.js} +4 -15
- package/out/src/synthesizers/utils.js.map +1 -0
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +4 -8
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/audit.d.ts.map +1 -1
- package/out/src/tools/audit.js +4 -4
- package/out/src/tools/audit.js.map +1 -1
- package/out/src/tools/corpus-stats.d.ts +2 -2
- package/out/src/tools/corpus-stats.d.ts.map +1 -1
- package/out/src/tools/corpus-stats.js +18 -31
- package/out/src/tools/corpus-stats.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +5 -6
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +3 -15
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +14 -1
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +14 -1
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts +1 -1
- package/out/src/tools/fetch/nad.js +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +8 -14
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts +1 -1
- package/out/src/tools/fetch/openaddresses.js +1 -1
- package/out/src/tools/fetch/ourairports.d.ts +45 -0
- package/out/src/tools/fetch/ourairports.d.ts.map +1 -0
- package/out/src/tools/fetch/ourairports.js +123 -0
- package/out/src/tools/fetch/ourairports.js.map +1 -0
- package/out/src/tools/fetch/wikidata-subvenue.d.ts +171 -0
- package/out/src/tools/fetch/wikidata-subvenue.d.ts.map +1 -0
- package/out/src/tools/fetch/wikidata-subvenue.js +275 -0
- package/out/src/tools/fetch/wikidata-subvenue.js.map +1 -0
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +18 -16
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-promote.d.ts.map +1 -1
- package/out/src/tools/golden-promote.js +12 -6
- package/out/src/tools/golden-promote.js.map +1 -1
- package/out/src/tools/golden-relabel-street.d.ts +196 -0
- package/out/src/tools/golden-relabel-street.d.ts.map +1 -0
- package/out/src/tools/golden-relabel-street.js +440 -0
- package/out/src/tools/golden-relabel-street.js.map +1 -0
- package/out/src/tools/index.d.ts +4 -0
- package/out/src/tools/index.d.ts.map +1 -1
- package/out/src/tools/index.js +4 -0
- package/out/src/tools/index.js.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
- package/out/src/tools/jsonl-to-parquet.js +3 -2
- package/out/src/tools/jsonl-to-parquet.js.map +1 -1
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
- package/out/src/tools/lint-shard-vocab.js +3 -2
- package/out/src/tools/lint-shard-vocab.js.map +1 -1
- package/out/src/tools/lint-shard.d.ts +3 -3
- package/out/src/tools/lint-shard.d.ts.map +1 -1
- package/out/src/tools/lint-shard.js +23 -32
- package/out/src/tools/lint-shard.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +2 -1
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/overture-subvenue.d.ts +112 -0
- package/out/src/tools/overture-subvenue.d.ts.map +1 -0
- package/out/src/tools/overture-subvenue.js +144 -0
- package/out/src/tools/overture-subvenue.js.map +1 -0
- package/out/src/tools/shard-kryptonite.d.ts +1 -1
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +6 -6
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts +6 -3
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +9 -8
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/tools/sub-venue-lexicon.d.ts +507 -0
- package/out/src/tools/sub-venue-lexicon.d.ts.map +1 -0
- package/out/src/tools/sub-venue-lexicon.js +817 -0
- package/out/src/tools/sub-venue-lexicon.js.map +1 -0
- package/out/src/tools/sub-venue-promotions.d.ts +94 -0
- package/out/src/tools/sub-venue-promotions.d.ts.map +1 -0
- package/out/src/tools/sub-venue-promotions.js +266 -0
- package/out/src/tools/sub-venue-promotions.js.map +1 -0
- package/out/src/{align.d.ts → utils/align.d.ts} +1 -1
- package/out/src/utils/align.d.ts.map +1 -0
- package/out/src/utils/align.js.map +1 -0
- package/out/src/{golden.d.ts → utils/golden.d.ts} +5 -0
- package/out/src/utils/golden.d.ts.map +1 -0
- package/out/src/{golden.js → utils/golden.js} +24 -10
- package/out/src/utils/golden.js.map +1 -0
- package/out/src/utils/index.d.ts +14 -0
- package/out/src/utils/index.d.ts.map +1 -0
- package/out/src/utils/index.js +14 -0
- package/out/src/utils/index.js.map +1 -0
- package/out/src/utils/license.d.ts.map +1 -0
- package/out/src/utils/license.js.map +1 -0
- package/out/src/{parquet.d.ts → utils/parquet.d.ts} +2 -2
- package/out/src/utils/parquet.d.ts.map +1 -0
- package/out/src/{parquet.js → utils/parquet.js} +3 -15
- package/out/src/utils/parquet.js.map +1 -0
- package/out/src/{split.d.ts → utils/split.d.ts} +8 -1
- package/out/src/utils/split.d.ts.map +1 -0
- package/out/src/{split.js → utils/split.js} +7 -0
- package/out/src/utils/split.js.map +1 -0
- package/out/src/utils/tokenize.d.ts.map +1 -0
- package/out/src/utils/tokenize.js.map +1 -0
- package/out/src/{wof-json.d.ts → utils/wof-json.d.ts} +2 -2
- package/out/src/utils/wof-json.d.ts.map +1 -0
- package/out/src/{wof-json.js → utils/wof-json.js} +5 -8
- package/out/src/utils/wof-json.js.map +1 -0
- package/package.json +66 -20
- package/src/adapters/ban/adapter.ts +58 -68
- package/src/adapters/ban/street-decompose.ts +17 -25
- package/src/adapters/fcc-bdc/adapter.ts +5 -36
- package/src/adapters/geonames/adapter.ts +67 -76
- package/src/adapters/geonames-postal/adapter.ts +45 -54
- package/src/adapters/gnaf/adapter.ts +7 -11
- package/src/adapters/gnaf/assemble.ts +7 -11
- package/src/adapters/index.ts +3 -2
- package/src/adapters/openaddresses/adapter.ts +6 -12
- package/src/adapters/overture/adapter.ts +6 -10
- package/src/adapters/state-hi-schools/adapter.ts +53 -77
- package/src/adapters/state-ia-contractors/adapter.ts +55 -80
- package/src/adapters/state-ny-notaries/adapter.ts +64 -89
- package/src/adapters/state-tx-notaries/adapter.ts +63 -88
- package/src/adapters/synth-po-box/adapter.ts +12 -23
- package/src/adapters/tiger/adapter.ts +4 -3
- package/src/adapters/tiger/street-decompose.ts +21 -31
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +64 -88
- package/src/adapters/usgov-imls-pls/adapter.ts +57 -86
- package/src/adapters/usgov-irs-bmf/adapter.ts +55 -66
- package/src/adapters/usgov-nad/adapter.ts +7 -11
- package/src/adapters/usgov-nppes/adapter.ts +54 -79
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +48 -75
- package/src/{adapter.ts → adapters/utils/index.ts} +59 -3
- package/src/adapters/wof-admin-jp/adapter.ts +35 -9
- package/src/adapters/wof-admin-json/adapter.ts +3 -4
- package/src/adapters/wof-postalcode-json/adapter.ts +3 -4
- package/src/build.ts +12 -11
- package/src/index.ts +3 -17
- package/src/name-prone-us-suffixes.ts +12 -0
- package/src/parquet-wrapper/reader.ts +19 -0
- package/src/parquet-wrapper/schema.ts +4 -0
- package/src/runner.ts +7 -2
- package/src/shard-recipes/anchor-absorption.ts +5 -4
- package/src/shard-recipes/boundary-stress.ts +6 -2
- package/src/shard-recipes/country-balanced.ts +77 -91
- package/src/shard-recipes/fr-admin-split.ts +3 -3
- package/src/shard-recipes/fr-fragment.ts +10 -7
- package/src/shard-recipes/fr-lieudit.ts +3 -3
- package/src/shard-recipes/fr-order.ts +18 -83
- package/src/shard-recipes/german.ts +17 -80
- package/src/shard-recipes/house-venue.ts +2 -1
- package/src/shard-recipes/index.ts +4 -1
- package/src/shard-recipes/intersection.ts +37 -74
- package/src/shard-recipes/locale.ts +31 -39
- package/src/shard-recipes/no-fragment.ts +10 -7
- package/src/shard-recipes/no-street-led.ts +10 -7
- package/src/shard-recipes/no-street.ts +2 -1
- package/src/shard-recipes/po-box-cedex.ts +127 -161
- package/src/shard-recipes/po-box.ts +6 -1
- package/src/shard-recipes/scaffold.ts +125 -27
- package/src/shard-recipes/street-affix.ts +335 -134
- package/src/shard-recipes/street-bare.ts +4 -3
- package/src/shard-recipes/street.ts +3 -2
- package/src/shard-recipes/sub-venue-sources.ts +895 -0
- package/src/shard-recipes/sub-venue.ts +982 -0
- package/src/shard-recipes/unit.ts +27 -85
- package/src/{synthesize-boundary-stress.ts → synthesizers/boundary-stress.ts} +2 -2
- package/src/{synthesize-german.ts → synthesizers/german.ts} +2 -2
- package/src/{synthesize-house-venue.ts → synthesizers/house-venue.ts} +4 -4
- package/src/{synthesize-intersection.ts → synthesizers/intersection.ts} +1 -1
- package/src/{synthesize-no-street.ts → synthesizers/no-street.ts} +2 -2
- package/src/{synthesize-po-box.ts → synthesizers/po-box.ts} +1 -1
- package/src/{synthesize-street.ts → synthesizers/street.ts} +2 -2
- package/src/{synthesize.ts → synthesizers/utils.ts} +4 -18
- package/src/tools/align-shard.ts +4 -7
- package/src/tools/audit.ts +6 -5
- package/src/tools/corpus-stats.ts +25 -33
- package/src/tools/fetch/download.ts +7 -5
- package/src/tools/fetch/imls-pls.ts +4 -18
- package/src/tools/fetch/index.ts +14 -1
- package/src/tools/fetch/nad.ts +1 -1
- package/src/tools/fetch/nppes.ts +9 -16
- package/src/tools/fetch/openaddresses.ts +1 -1
- package/src/tools/fetch/ourairports.ts +166 -0
- package/src/tools/fetch/wikidata-subvenue.ts +386 -0
- package/src/tools/golden-expand.ts +18 -13
- package/src/tools/golden-promote.ts +15 -6
- package/src/tools/golden-relabel-street.ts +685 -0
- package/src/tools/index.ts +4 -0
- package/src/tools/jsonl-to-parquet.ts +3 -2
- package/src/tools/lint-shard-vocab.ts +3 -2
- package/src/tools/lint-shard.ts +33 -36
- package/src/tools/overlay-manifest.ts +2 -1
- package/src/tools/overture-subvenue.ts +215 -0
- package/src/tools/shard-kryptonite.ts +8 -9
- package/src/tools/shard-translit.ts +22 -11
- package/src/tools/sub-venue-lexicon.ts +1250 -0
- package/src/tools/sub-venue-promotions.ts +330 -0
- package/src/{align.ts → utils/align.ts} +1 -1
- package/src/{golden.ts → utils/golden.ts} +25 -11
- package/src/utils/index.ts +14 -0
- package/src/{parquet.ts → utils/parquet.ts} +5 -19
- package/src/{split.ts → utils/split.ts} +8 -2
- package/src/{wof-json.ts → utils/wof-json.ts} +5 -8
- package/out/src/adapter.d.ts.map +0 -1
- package/out/src/adapter.js.map +0 -1
- package/out/src/align.d.ts.map +0 -1
- package/out/src/align.js.map +0 -1
- package/out/src/format.d.ts +0 -14
- package/out/src/format.d.ts.map +0 -1
- package/out/src/format.js +0 -14
- package/out/src/format.js.map +0 -1
- package/out/src/golden.d.ts.map +0 -1
- package/out/src/golden.js.map +0 -1
- package/out/src/license.d.ts.map +0 -1
- package/out/src/license.js.map +0 -1
- package/out/src/parquet.d.ts.map +0 -1
- package/out/src/parquet.js.map +0 -1
- package/out/src/split.d.ts.map +0 -1
- package/out/src/split.js.map +0 -1
- package/out/src/synthesize-anchor-absorption.d.ts.map +0 -1
- package/out/src/synthesize-anchor-absorption.js.map +0 -1
- package/out/src/synthesize-boundary-stress.d.ts.map +0 -1
- package/out/src/synthesize-boundary-stress.js.map +0 -1
- package/out/src/synthesize-german.d.ts.map +0 -1
- package/out/src/synthesize-german.js.map +0 -1
- package/out/src/synthesize-house-venue.d.ts.map +0 -1
- package/out/src/synthesize-house-venue.js.map +0 -1
- package/out/src/synthesize-intersection.d.ts.map +0 -1
- package/out/src/synthesize-intersection.js.map +0 -1
- package/out/src/synthesize-no-street.d.ts.map +0 -1
- package/out/src/synthesize-no-street.js.map +0 -1
- package/out/src/synthesize-po-box.d.ts.map +0 -1
- package/out/src/synthesize-po-box.js.map +0 -1
- package/out/src/synthesize-street.d.ts.map +0 -1
- package/out/src/synthesize-street.js.map +0 -1
- package/out/src/synthesize.d.ts.map +0 -1
- package/out/src/synthesize.js.map +0 -1
- package/out/src/tokenize.d.ts.map +0 -1
- package/out/src/tokenize.js.map +0 -1
- package/out/src/wof-json.d.ts.map +0 -1
- package/out/src/wof-json.js.map +0 -1
- package/src/format.ts +0 -14
- /package/out/src/{align.js → utils/align.js} +0 -0
- /package/out/src/{license.d.ts → utils/license.d.ts} +0 -0
- /package/out/src/{license.js → utils/license.js} +0 -0
- /package/out/src/{tokenize.d.ts → utils/tokenize.d.ts} +0 -0
- /package/out/src/{tokenize.js → utils/tokenize.js} +0 -0
- /package/src/{synthesize-anchor-absorption.ts → synthesizers/anchor-absorption.ts} +0 -0
- /package/src/{license.ts → utils/license.ts} +0 -0
- /package/src/{tokenize.ts → utils/tokenize.ts} +0 -0
|
@@ -24,15 +24,14 @@
|
|
|
24
24
|
* legacy script used.
|
|
25
25
|
*/
|
|
26
26
|
|
|
27
|
-
import { spawnSync } from "node:child_process"
|
|
28
|
-
|
|
29
27
|
import { COUNTRY_SURFACE_FORMS, CountryNames } from "@mailwoman/codex/country"
|
|
30
28
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
29
|
+
import { dataRootPath } from "@mailwoman/core/utils"
|
|
30
|
+
import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
|
|
31
|
+
import type { CanonicalRow } from "@mailwoman/corpus/types"
|
|
32
|
+
import { alignRow } from "@mailwoman/corpus/utils"
|
|
31
33
|
|
|
32
|
-
import {
|
|
33
|
-
import { alignRow } from "../align.ts"
|
|
34
|
-
import type { CanonicalRow } from "../types.ts"
|
|
35
|
-
import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
|
|
34
|
+
import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
|
|
36
35
|
|
|
37
36
|
// v2: the country TOKEN is decoupled from the skeleton's locale and drawn from a BROAD pool — every
|
|
38
37
|
// ISO canonical name + every curated surface form (endonyms/abbrevs). Surface forms are over-weighted
|
|
@@ -72,24 +71,72 @@ interface CountrySource {
|
|
|
72
71
|
* countrywide extracts (FR/IT/NL) read region from the CSV when present.
|
|
73
72
|
*/
|
|
74
73
|
const SOURCES: readonly CountrySource[] = [
|
|
75
|
-
{
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
74
|
+
{
|
|
75
|
+
zip: dataRootPath("oa-cache", "us__ia__statewide.zip"),
|
|
76
|
+
csv: "us/ia/statewide.csv",
|
|
77
|
+
iso2: "US",
|
|
78
|
+
region: "IA",
|
|
79
|
+
order: "us",
|
|
80
|
+
},
|
|
81
|
+
{ zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv", iso2: "US", region: "IL", order: "us" },
|
|
82
|
+
{
|
|
83
|
+
zip: dataRootPath("oa-cache", "us__mt__statewide.zip"),
|
|
84
|
+
csv: "us/mt/statewide.csv",
|
|
85
|
+
iso2: "US",
|
|
86
|
+
region: "MT",
|
|
87
|
+
order: "us",
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
zip: dataRootPath("oa-cache", "us__sd__statewide.zip"),
|
|
91
|
+
csv: "us/sd/statewide.csv",
|
|
92
|
+
iso2: "US",
|
|
93
|
+
region: "SD",
|
|
94
|
+
order: "us",
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
zip: dataRootPath("oa-cache", "de__sn__statewide.zip"),
|
|
98
|
+
csv: "de/sn/statewide.csv",
|
|
99
|
+
iso2: "DE",
|
|
100
|
+
region: "",
|
|
101
|
+
order: "eu",
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
zip: dataRootPath("oa-cache", "fr__countrywide.zip"),
|
|
105
|
+
csv: "fr/countrywide.csv",
|
|
106
|
+
iso2: "FR",
|
|
107
|
+
region: "",
|
|
108
|
+
order: "fr",
|
|
109
|
+
},
|
|
81
110
|
// ES uses the Spanish IGN schema, not the OA standard columns — skipped here (codex still recognizes
|
|
82
111
|
// "España"/"Spain"). A dedicated IGN adapter is a follow-up.
|
|
83
|
-
{
|
|
84
|
-
|
|
112
|
+
{
|
|
113
|
+
zip: dataRootPath("oa-cache", "it__countrywide.zip"),
|
|
114
|
+
csv: "it/countrywide.csv",
|
|
115
|
+
iso2: "IT",
|
|
116
|
+
region: "",
|
|
117
|
+
order: "eu",
|
|
118
|
+
},
|
|
119
|
+
{
|
|
120
|
+
zip: dataRootPath("oa-cache", "nl__countrywide.zip"),
|
|
121
|
+
csv: "nl/countrywide.csv",
|
|
122
|
+
iso2: "NL",
|
|
123
|
+
region: "",
|
|
124
|
+
order: "eu",
|
|
125
|
+
},
|
|
85
126
|
]
|
|
86
127
|
|
|
87
128
|
/**
|
|
88
129
|
* Held-out for --golden: Vermont (US holdout) + Berlin (DE holdout) — geographic split, never trained.
|
|
89
130
|
*/
|
|
90
131
|
const EVAL_SOURCES: readonly CountrySource[] = [
|
|
91
|
-
{
|
|
92
|
-
|
|
132
|
+
{
|
|
133
|
+
zip: dataRootPath("oa-cache", "us__vt__statewide.zip"),
|
|
134
|
+
csv: "us/vt/statewide.csv",
|
|
135
|
+
iso2: "US",
|
|
136
|
+
region: "VT",
|
|
137
|
+
order: "us",
|
|
138
|
+
},
|
|
139
|
+
{ zip: dataRootPath("oa-cache", "de__berlin.zip"), csv: "de/berlin.csv", iso2: "DE", region: "", order: "eu" },
|
|
93
140
|
]
|
|
94
141
|
|
|
95
142
|
/**
|
|
@@ -105,81 +152,20 @@ interface CountryTuple {
|
|
|
105
152
|
order: string
|
|
106
153
|
}
|
|
107
154
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
for (let i = 0; i < line.length; i++) {
|
|
115
|
-
const c = line[i]
|
|
116
|
-
|
|
117
|
-
if (inQ) {
|
|
118
|
-
if (c === '"') {
|
|
119
|
-
if (line[i + 1] === '"') {
|
|
120
|
-
cur += '"'
|
|
121
|
-
|
|
122
|
-
i++
|
|
123
|
-
} else {
|
|
124
|
-
inQ = false
|
|
125
|
-
}
|
|
126
|
-
} else {
|
|
127
|
-
cur += c
|
|
128
|
-
}
|
|
129
|
-
} else if (c === '"') {
|
|
130
|
-
inQ = true
|
|
131
|
-
} else if (c === ",") {
|
|
132
|
-
out.push(cur)
|
|
133
|
-
cur = ""
|
|
134
|
-
} else {
|
|
135
|
-
cur += c
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
out.push(cur)
|
|
140
|
-
|
|
141
|
-
return out
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
function readTuples(source: CountrySource, limit: number): CountryTuple[] {
|
|
145
|
-
// countrywide extracts (FR/IT/NL) are GB-scale — cap the bytes with `head` (read ~8 lines per wanted
|
|
146
|
-
// tuple to survive dedup/skips) so the toString stays under V8's string limit.
|
|
147
|
-
const maxLines = Math.max(limit * 8, 20_000) + 1
|
|
148
|
-
|
|
149
|
-
const r = spawnSync("bash", ["-c", `unzip -p "${source.zip}" "${source.csv}" | head -n ${maxLines}`], {
|
|
150
|
-
maxBuffer: 1024 * 1024 * 1024,
|
|
151
|
-
encoding: "buffer",
|
|
152
|
-
})
|
|
153
|
-
|
|
154
|
-
if (r.status !== 0) {
|
|
155
|
-
console.error(` WARN: unzip failed for ${source.zip} (status ${r.status})`)
|
|
156
|
-
|
|
157
|
-
return []
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
const lines = r.stdout.toString("utf8").split(/\r?\n/)
|
|
161
|
-
|
|
162
|
-
if (lines.length < 2) return []
|
|
163
|
-
const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
|
|
164
|
-
const idx = (n: string): number => header.indexOf(n)
|
|
165
|
-
|
|
166
|
-
const iNum = idx("number"),
|
|
167
|
-
iStreet = idx("street"),
|
|
168
|
-
iCity = idx("city"),
|
|
169
|
-
iRegion = idx("region"),
|
|
170
|
-
iPost = idx("postcode")
|
|
171
|
-
|
|
172
|
-
const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
|
|
155
|
+
/**
|
|
156
|
+
* The countrywide extracts (FR/IT/NL) are GB-scale, so this reads only as far as `limit` distinct tuples: the `break`
|
|
157
|
+
* closes the reader, which releases the archive without inflating the rest of the member.
|
|
158
|
+
*/
|
|
159
|
+
async function readTuples(source: CountrySource, limit: number): Promise<CountryTuple[]> {
|
|
173
160
|
const tuples: CountryTuple[] = []
|
|
174
161
|
const seen = new Set<string>()
|
|
175
162
|
|
|
176
|
-
for (
|
|
177
|
-
if (
|
|
178
|
-
const cells = splitCSV(lines[li]!)
|
|
163
|
+
for await (const row of readZippedCSVRecords(source.zip, source.csv)) {
|
|
164
|
+
if (tuples.length >= limit) break
|
|
179
165
|
|
|
180
|
-
const street =
|
|
181
|
-
locality =
|
|
182
|
-
house_number =
|
|
166
|
+
const street = row.street ?? "",
|
|
167
|
+
locality = row.city ?? "",
|
|
168
|
+
house_number = row.number ?? ""
|
|
183
169
|
|
|
184
170
|
if (!street || !locality || !house_number) continue
|
|
185
171
|
const key = `${house_number}|${street}|${locality}`.toLowerCase()
|
|
@@ -191,8 +177,8 @@ function readTuples(source: CountrySource, limit: number): CountryTuple[] {
|
|
|
191
177
|
house_number,
|
|
192
178
|
street,
|
|
193
179
|
locality,
|
|
194
|
-
region:
|
|
195
|
-
postcode:
|
|
180
|
+
region: row.region || source.region,
|
|
181
|
+
postcode: row.postcode ?? "",
|
|
196
182
|
iso2: source.iso2,
|
|
197
183
|
order: source.order,
|
|
198
184
|
})
|
|
@@ -450,7 +436,7 @@ export const countryBalancedRecipe: ShardRecipe = {
|
|
|
450
436
|
const pool: CountryTuple[] = []
|
|
451
437
|
|
|
452
438
|
for (const s of sources) {
|
|
453
|
-
const t = readTuples(s, perSource)
|
|
439
|
+
const t = await readTuples(s, perSource)
|
|
454
440
|
|
|
455
441
|
console.error(` ${s.csv} (${s.iso2}): ${t.length} tuples`)
|
|
456
442
|
|
|
@@ -460,7 +446,7 @@ export const countryBalancedRecipe: ShardRecipe = {
|
|
|
460
446
|
}
|
|
461
447
|
|
|
462
448
|
if (!pool.length) {
|
|
463
|
-
throw new Error(
|
|
449
|
+
throw new Error(`No tuples — are the cached OA zips present in ${dataRootPath("oa-cache")}?`)
|
|
464
450
|
}
|
|
465
451
|
|
|
466
452
|
let emitted = 0
|
|
@@ -32,11 +32,11 @@
|
|
|
32
32
|
|
|
33
33
|
import { departementForCodePostal } from "@mailwoman/codex/fr"
|
|
34
34
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
35
|
+
import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
|
|
36
|
+
import type { CanonicalRow } from "@mailwoman/corpus/types"
|
|
37
|
+
import { alignRow } from "@mailwoman/corpus/utils"
|
|
35
38
|
import { CSVSpliterator, Delimiters } from "spliterator"
|
|
36
39
|
|
|
37
|
-
import { stableSourceID } from "../adapter.ts"
|
|
38
|
-
import { alignRow } from "../align.ts"
|
|
39
|
-
import type { CanonicalRow } from "../types.ts"
|
|
40
40
|
import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
|
|
41
41
|
|
|
42
42
|
/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
* blow-up). Disable that mask for any run including this shard.
|
|
69
69
|
*/
|
|
70
70
|
|
|
71
|
-
import {
|
|
71
|
+
import { TextSpliterator } from "spliterator"
|
|
72
72
|
|
|
73
73
|
import { decomposeFrStreet } from "../adapters/ban/street-decompose.ts"
|
|
74
74
|
import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
|
|
@@ -187,12 +187,15 @@ export const frFragmentRecipe: ShardRecipe = {
|
|
|
187
187
|
)
|
|
188
188
|
}
|
|
189
189
|
|
|
190
|
-
const excluded = new Set(
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
190
|
+
const excluded = new Set<string>()
|
|
191
|
+
|
|
192
|
+
for await (const line of TextSpliterator.fromAsync(excludePath)) {
|
|
193
|
+
const trimmed = line.trim()
|
|
194
|
+
|
|
195
|
+
if (trimmed && !trimmed.startsWith("#")) {
|
|
196
|
+
excluded.add(trimmed)
|
|
197
|
+
}
|
|
198
|
+
}
|
|
196
199
|
|
|
197
200
|
if (!excluded.size) throw new Error(`fr-fragment: --exclude-surfaces "${excludePath}" listed no surfaces`)
|
|
198
201
|
|
|
@@ -31,11 +31,11 @@ import { extractBANAddrPoints } from "@mailwoman/ban/sdk"
|
|
|
31
31
|
import { COUNTRY_SURFACE_FORMS } from "@mailwoman/codex/country"
|
|
32
32
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
33
33
|
import { dataRootPath } from "@mailwoman/core/utils"
|
|
34
|
+
import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
|
|
35
|
+
import type { CanonicalRow } from "@mailwoman/corpus/types"
|
|
36
|
+
import { alignRow } from "@mailwoman/corpus/utils"
|
|
34
37
|
|
|
35
|
-
import { stableSourceID } from "../adapter.ts"
|
|
36
38
|
import { decomposeFrStreet } from "../adapters/ban/street-decompose.ts"
|
|
37
|
-
import { alignRow } from "../align.ts"
|
|
38
|
-
import type { CanonicalRow } from "../types.ts"
|
|
39
39
|
import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
|
|
40
40
|
|
|
41
41
|
/**
|
|
@@ -30,14 +30,13 @@
|
|
|
30
30
|
* inlined the variant logic and never called it).
|
|
31
31
|
*/
|
|
32
32
|
|
|
33
|
-
import { spawnSync } from "node:child_process"
|
|
34
|
-
|
|
35
33
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
34
|
+
import { dataRootPath } from "@mailwoman/core/utils"
|
|
35
|
+
import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
|
|
36
|
+
import type { CanonicalRow } from "@mailwoman/corpus/types"
|
|
37
|
+
import { alignRow } from "@mailwoman/corpus/utils"
|
|
36
38
|
|
|
37
|
-
import {
|
|
38
|
-
import { alignRow } from "../align.ts"
|
|
39
|
-
import type { CanonicalRow } from "../types.ts"
|
|
40
|
-
import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
|
|
39
|
+
import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
|
|
41
40
|
|
|
42
41
|
/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
|
|
43
42
|
cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
|
|
@@ -45,7 +44,7 @@ import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
|
|
|
45
44
|
would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
|
|
46
45
|
extracted as named constants above. */
|
|
47
46
|
|
|
48
|
-
const SOURCE = { zip: "
|
|
47
|
+
const SOURCE = { zip: dataRootPath("oa-cache", "fr__countrywide.zip"), csv: "fr/countrywide.csv" }
|
|
49
48
|
|
|
50
49
|
/**
|
|
51
50
|
* Ordinal suffixes used in French house numbers (BAN corpus), to cover the "8 bis" sub-mode.
|
|
@@ -71,84 +70,20 @@ interface FrTuple {
|
|
|
71
70
|
}
|
|
72
71
|
|
|
73
72
|
/**
|
|
74
|
-
*
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
const out: string[] = []
|
|
78
|
-
let cur = ""
|
|
79
|
-
let inQ = false
|
|
80
|
-
|
|
81
|
-
for (let i = 0; i < line.length; i++) {
|
|
82
|
-
const c = line[i]
|
|
83
|
-
|
|
84
|
-
if (inQ) {
|
|
85
|
-
if (c === '"') {
|
|
86
|
-
if (line[i + 1] === '"') {
|
|
87
|
-
cur += '"'
|
|
88
|
-
|
|
89
|
-
i++
|
|
90
|
-
} else {
|
|
91
|
-
inQ = false
|
|
92
|
-
}
|
|
93
|
-
} else {
|
|
94
|
-
cur += c
|
|
95
|
-
}
|
|
96
|
-
} else if (c === '"') {
|
|
97
|
-
inQ = true
|
|
98
|
-
} else if (c === ",") {
|
|
99
|
-
out.push(cur)
|
|
100
|
-
cur = ""
|
|
101
|
-
} else {
|
|
102
|
-
cur += c
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
out.push(cur)
|
|
107
|
-
|
|
108
|
-
return out
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
/**
|
|
112
|
-
* Stream FR tuples out of the cached OA zip. The countrywide extract is GB-scale; cap with `head` to stay under V8's
|
|
113
|
-
* string limit. Only keeps rows with a house_number (the shard's core signal) and a postcode (required for
|
|
114
|
-
* reversed-order rendering to be meaningful).
|
|
73
|
+
* Stream FR tuples out of the cached OA zip. The countrywide extract is GB-scale, so this reads only as far as `limit`
|
|
74
|
+
* distinct tuples — the `break` closes the reader and releases the archive. Only keeps rows with a house_number (the
|
|
75
|
+
* shard's core signal) and a postcode (required for reversed-order rendering to be meaningful).
|
|
115
76
|
*/
|
|
116
|
-
function readTuples(limit: number): FrTuple[] {
|
|
117
|
-
const maxLines = Math.max(limit * 8, 40_000) + 1
|
|
118
|
-
|
|
119
|
-
const r = spawnSync("bash", ["-c", `unzip -p "${SOURCE.zip}" "${SOURCE.csv}" | head -n ${maxLines}`], {
|
|
120
|
-
maxBuffer: 1024 * 1024 * 1024,
|
|
121
|
-
encoding: "buffer",
|
|
122
|
-
})
|
|
123
|
-
|
|
124
|
-
if (r.status !== 0) {
|
|
125
|
-
console.error(` WARN: unzip failed for ${SOURCE.zip} (status ${r.status})`)
|
|
126
|
-
|
|
127
|
-
return []
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
const lines = r.stdout.toString("utf8").split(/\r?\n/)
|
|
131
|
-
|
|
132
|
-
if (lines.length < 2) return []
|
|
133
|
-
const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
|
|
134
|
-
const idx = (name: string): number => header.indexOf(name)
|
|
135
|
-
|
|
136
|
-
const iNum = idx("number"),
|
|
137
|
-
iStreet = idx("street"),
|
|
138
|
-
iCity = idx("city"),
|
|
139
|
-
iPost = idx("postcode")
|
|
140
|
-
|
|
141
|
-
const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
|
|
77
|
+
async function readTuples(limit: number): Promise<FrTuple[]> {
|
|
142
78
|
const tuples: FrTuple[] = []
|
|
143
79
|
const seen = new Set<string>()
|
|
144
80
|
|
|
145
|
-
for (
|
|
146
|
-
if (
|
|
147
|
-
const
|
|
148
|
-
const
|
|
149
|
-
const
|
|
150
|
-
const
|
|
151
|
-
const postcode = get(cells, iPost)
|
|
81
|
+
for await (const row of readZippedCSVRecords(SOURCE.zip, SOURCE.csv)) {
|
|
82
|
+
if (tuples.length >= limit) break
|
|
83
|
+
const street = row.street ?? ""
|
|
84
|
+
const locality = row.city ?? ""
|
|
85
|
+
const house_number = row.number ?? ""
|
|
86
|
+
const postcode = row.postcode ?? ""
|
|
152
87
|
|
|
153
88
|
// Require all four fields: HN is the signal; postcode drives reversed-order variants.
|
|
154
89
|
if (!street || !locality || !house_number || !postcode) continue
|
|
@@ -211,12 +146,12 @@ export const frOrderRecipe: ShardRecipe = {
|
|
|
211
146
|
|
|
212
147
|
// Over-read from the CSV so the dedup + filter pass can fill `count` rows.
|
|
213
148
|
const poolLimit = Math.max(count * 8, 40_000)
|
|
214
|
-
const pool = readTuples(poolLimit)
|
|
149
|
+
const pool = await readTuples(poolLimit)
|
|
215
150
|
|
|
216
151
|
console.error(` ${SOURCE.csv}: ${pool.length} unique tuples (capped read)`)
|
|
217
152
|
|
|
218
153
|
if (!pool.length) {
|
|
219
|
-
throw new Error(
|
|
154
|
+
throw new Error(`No FR tuples found — is ${SOURCE.zip} present?`)
|
|
220
155
|
}
|
|
221
156
|
|
|
222
157
|
let emitted = 0
|
|
@@ -15,12 +15,12 @@
|
|
|
15
15
|
* docs/articles/evals/resolver-geo/2026-06-06-anchor-pilot.md (the order-artifact correction).
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import {
|
|
18
|
+
import { dataRootPath } from "@mailwoman/core/utils"
|
|
19
|
+
import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
|
|
20
|
+
import { synthesizeGermanRow, type LocaleBaseTuple } from "@mailwoman/corpus/synthesizers/german"
|
|
21
|
+
import { alignRow } from "@mailwoman/corpus/utils"
|
|
19
22
|
|
|
20
|
-
import {
|
|
21
|
-
import { alignRow } from "../align.ts"
|
|
22
|
-
import { synthesizeGermanRow, type LocaleBaseTuple } from "../synthesize-german.ts"
|
|
23
|
-
import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
|
|
23
|
+
import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
|
|
24
24
|
|
|
25
25
|
/**
|
|
26
26
|
* A German OA source (cached zip) + the Bundesland the file covers (OA's REGION column is empty for DE).
|
|
@@ -37,89 +37,26 @@ interface GermanSource {
|
|
|
37
37
|
* Berlin (a city-state, region==locality); sn/statewide → Sachsen.
|
|
38
38
|
*/
|
|
39
39
|
const SOURCES: GermanSource[] = [
|
|
40
|
-
{ zip: "
|
|
41
|
-
{ zip: "
|
|
40
|
+
{ zip: dataRootPath("oa-cache", "de__berlin.zip"), csv: "de/berlin.csv", region: "Berlin" },
|
|
41
|
+
{ zip: dataRootPath("oa-cache", "de__sn__statewide.zip"), csv: "de/sn/statewide.csv", region: "Sachsen" },
|
|
42
42
|
]
|
|
43
43
|
|
|
44
44
|
/**
|
|
45
|
-
*
|
|
45
|
+
* Stream real German tuples out of a cached OA zip.
|
|
46
46
|
*/
|
|
47
|
-
function
|
|
48
|
-
const out: string[] = []
|
|
49
|
-
let cur = ""
|
|
50
|
-
let inQ = false
|
|
51
|
-
|
|
52
|
-
for (let i = 0; i < line.length; i++) {
|
|
53
|
-
const c = line[i]!
|
|
54
|
-
|
|
55
|
-
if (inQ) {
|
|
56
|
-
if (c === '"') {
|
|
57
|
-
if (line[i + 1] === '"') {
|
|
58
|
-
cur += '"'
|
|
59
|
-
|
|
60
|
-
i++
|
|
61
|
-
} else {
|
|
62
|
-
inQ = false
|
|
63
|
-
}
|
|
64
|
-
} else {
|
|
65
|
-
cur += c
|
|
66
|
-
}
|
|
67
|
-
} else if (c === '"') {
|
|
68
|
-
inQ = true
|
|
69
|
-
} else if (c === ",") {
|
|
70
|
-
out.push(cur)
|
|
71
|
-
cur = ""
|
|
72
|
-
} else {
|
|
73
|
-
cur += c
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
out.push(cur)
|
|
78
|
-
|
|
79
|
-
return out
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
/**
|
|
83
|
-
* Stream real German tuples out of a cached OA zip (buffered `unzip -p`).
|
|
84
|
-
*/
|
|
85
|
-
function readGermanTuples(source: GermanSource): LocaleBaseTuple[] {
|
|
86
|
-
const r = spawnSync("unzip", ["-p", source.zip, source.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
|
|
87
|
-
|
|
88
|
-
if (r.status !== 0) {
|
|
89
|
-
console.error(` WARN: unzip failed for ${source.zip} (status ${r.status})`)
|
|
90
|
-
|
|
91
|
-
return []
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
const lines = r.stdout.toString("utf8").split(/\r?\n/)
|
|
95
|
-
|
|
96
|
-
if (lines.length < 2) return []
|
|
97
|
-
const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
|
|
98
|
-
const idx = (name: string): number => header.indexOf(name)
|
|
99
|
-
|
|
100
|
-
const iNum = idx("number"),
|
|
101
|
-
iStreet = idx("street"),
|
|
102
|
-
iCity = idx("city"),
|
|
103
|
-
iRegion = idx("region"),
|
|
104
|
-
iPost = idx("postcode")
|
|
105
|
-
|
|
106
|
-
const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
|
|
47
|
+
async function readGermanTuples(source: GermanSource): Promise<LocaleBaseTuple[]> {
|
|
107
48
|
const tuples: LocaleBaseTuple[] = []
|
|
108
49
|
const seen = new Set<string>()
|
|
109
50
|
|
|
110
|
-
for (
|
|
111
|
-
const
|
|
112
|
-
|
|
113
|
-
if (!lineStr) continue
|
|
114
|
-
const cells = splitCSV(lineStr)
|
|
115
|
-
const street = get(cells, iStreet)
|
|
116
|
-
const locality = get(cells, iCity)
|
|
51
|
+
for await (const row of readZippedCSVRecords(source.zip, source.csv)) {
|
|
52
|
+
const street = row.street ?? ""
|
|
53
|
+
const locality = row.city ?? ""
|
|
117
54
|
|
|
118
55
|
if (!street || !locality) continue
|
|
119
|
-
const house_number =
|
|
120
|
-
const postcode =
|
|
56
|
+
const house_number = row.number ?? ""
|
|
57
|
+
const postcode = row.postcode ?? ""
|
|
121
58
|
// OA's REGION column is empty for DE — fall back to the source's Bundesland (set per file).
|
|
122
|
-
const region =
|
|
59
|
+
const region = row.region || source.region || ""
|
|
123
60
|
const key = `${house_number}|${street}|${locality}|${postcode}`.toLowerCase()
|
|
124
61
|
|
|
125
62
|
if (seen.has(key)) continue
|
|
@@ -155,7 +92,7 @@ export const germanRecipe: ShardRecipe = {
|
|
|
155
92
|
const pool: LocaleBaseTuple[] = []
|
|
156
93
|
|
|
157
94
|
for (const s of SOURCES) {
|
|
158
|
-
const t = readGermanTuples(s)
|
|
95
|
+
const t = await readGermanTuples(s)
|
|
159
96
|
|
|
160
97
|
console.error(` ${s.csv}: ${t.length} unique tuples`)
|
|
161
98
|
|
|
@@ -165,7 +102,7 @@ export const germanRecipe: ShardRecipe = {
|
|
|
165
102
|
}
|
|
166
103
|
|
|
167
104
|
if (!pool.length) {
|
|
168
|
-
throw new Error(
|
|
105
|
+
throw new Error(`No German tuples found — are the cached zips present in ${dataRootPath("oa-cache")}?`)
|
|
169
106
|
}
|
|
170
107
|
|
|
171
108
|
let emitted = 0
|
|
@@ -10,7 +10,8 @@
|
|
|
10
10
|
* scripts/build-house-venue-shard.mjs.
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import { synthesizeHouseVenueRow, type HouseVenueBaseTuple } from "
|
|
13
|
+
import { synthesizeHouseVenueRow, type HouseVenueBaseTuple } from "@mailwoman/corpus/synthesizers/house-venue"
|
|
14
|
+
|
|
14
15
|
import { alignAndWrite, makeLcg, readTuples, shardSourceID, type ShardRecipe } from "./scaffold.ts"
|
|
15
16
|
|
|
16
17
|
const LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"
|
|
@@ -29,9 +29,10 @@ import { poBoxCedexRecipe } from "./po-box-cedex.ts"
|
|
|
29
29
|
import { poBoxRecipe } from "./po-box.ts"
|
|
30
30
|
import type { ShardRecipe } from "./scaffold.ts"
|
|
31
31
|
import { siBareVillageRecipe } from "./si-bare-village.ts"
|
|
32
|
-
import { streetAffixRecipe } from "./street-affix.ts"
|
|
32
|
+
import { streetAffixRecipe, suffixBoundaryRecipe } from "./street-affix.ts"
|
|
33
33
|
import { streetBareRecipe } from "./street-bare.ts"
|
|
34
34
|
import { streetRecipe } from "./street.ts"
|
|
35
|
+
import { subVenueRecipe } from "./sub-venue.ts"
|
|
35
36
|
import { unitRecipe } from "./unit.ts"
|
|
36
37
|
|
|
37
38
|
export * from "./scaffold.ts"
|
|
@@ -43,6 +44,7 @@ const RECIPES: readonly ShardRecipe[] = [
|
|
|
43
44
|
streetRecipe,
|
|
44
45
|
streetBareRecipe,
|
|
45
46
|
streetAffixRecipe,
|
|
47
|
+
suffixBoundaryRecipe,
|
|
46
48
|
noStreetRecipe,
|
|
47
49
|
houseVenueRecipe,
|
|
48
50
|
poBoxRecipe,
|
|
@@ -64,6 +66,7 @@ const RECIPES: readonly ShardRecipe[] = [
|
|
|
64
66
|
countryBalancedRecipe,
|
|
65
67
|
boundaryStressRecipe,
|
|
66
68
|
anchorAbsorptionRecipe,
|
|
69
|
+
subVenueRecipe,
|
|
67
70
|
]
|
|
68
71
|
|
|
69
72
|
/**
|