@mailwoman/corpus 8.6.0 → 9.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/data/PROVENANCE.md +214 -0
- package/data/sub-venue-lexicon.json +18269 -0
- package/out/src/adapters/ban/adapter.d.ts +5 -4
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +62 -69
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +18 -25
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +1 -7
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +4 -26
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +1 -1
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +72 -78
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +46 -51
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +6 -9
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +7 -12
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +5 -10
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +1 -1
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +5 -9
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +57 -75
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +60 -78
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +69 -87
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +71 -89
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +3 -3
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +10 -18
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +1 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +2 -2
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js +18 -27
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +4 -4
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +64 -82
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +62 -84
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +59 -66
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +6 -9
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +60 -78
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +53 -73
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/{adapter.d.ts → adapters/utils/index.d.ts} +43 -2
- package/out/src/adapters/utils/index.d.ts.map +1 -0
- package/out/src/{adapter.js → adapters/utils/index.js} +47 -2
- package/out/src/adapters/utils/index.js.map +1 -0
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +24 -6
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +2 -2
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +2 -2
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +2 -2
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +2 -2
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/build.d.ts +3 -4
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +6 -8
- package/out/src/build.js.map +1 -1
- package/out/src/index.d.ts +3 -17
- package/out/src/index.d.ts.map +1 -1
- package/out/src/index.js +3 -17
- package/out/src/index.js.map +1 -1
- package/out/src/name-prone-us-suffixes.d.ts +12 -0
- package/out/src/name-prone-us-suffixes.d.ts.map +1 -0
- package/out/src/name-prone-us-suffixes.js +12 -0
- package/out/src/name-prone-us-suffixes.js.map +1 -0
- package/out/src/parquet-wrapper/reader.d.ts +8 -0
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +16 -0
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/schema.d.ts +4 -0
- package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
- package/out/src/parquet-wrapper/schema.js.map +1 -1
- package/out/src/runner.d.ts +2 -2
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +1 -1
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +5 -4
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +2 -2
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +75 -75
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +2 -2
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +8 -5
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +2 -2
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +18 -72
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +17 -70
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/house-venue.js +1 -1
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +4 -1
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts +2 -2
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +28 -72
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +5 -5
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +25 -35
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +8 -5
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street-led.js +8 -5
- package/out/src/shard-recipes/no-street-led.js.map +1 -1
- package/out/src/shard-recipes/no-street.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street.js +1 -1
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +3 -3
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +101 -127
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +1 -1
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +58 -9
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +87 -29
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +54 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +253 -111
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
- package/out/src/shard-recipes/street-bare.js +3 -3
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.d.ts.map +1 -1
- package/out/src/shard-recipes/street.js +2 -2
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.d.ts +228 -0
- package/out/src/shard-recipes/sub-venue-sources.d.ts.map +1 -0
- package/out/src/shard-recipes/sub-venue-sources.js +664 -0
- package/out/src/shard-recipes/sub-venue-sources.js.map +1 -0
- package/out/src/shard-recipes/sub-venue.d.ts +269 -0
- package/out/src/shard-recipes/sub-venue.d.ts.map +1 -0
- package/out/src/shard-recipes/sub-venue.js +709 -0
- package/out/src/shard-recipes/sub-venue.js.map +1 -0
- package/out/src/shard-recipes/unit.d.ts +1 -1
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +26 -74
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/{synthesize-anchor-absorption.d.ts → synthesizers/anchor-absorption.d.ts} +1 -1
- package/out/src/synthesizers/anchor-absorption.d.ts.map +1 -0
- package/out/src/{synthesize-anchor-absorption.js → synthesizers/anchor-absorption.js} +1 -1
- package/out/src/synthesizers/anchor-absorption.js.map +1 -0
- package/out/src/{synthesize-boundary-stress.d.ts → synthesizers/boundary-stress.d.ts} +3 -3
- package/out/src/synthesizers/boundary-stress.d.ts.map +1 -0
- package/out/src/{synthesize-boundary-stress.js → synthesizers/boundary-stress.js} +2 -2
- package/out/src/synthesizers/boundary-stress.js.map +1 -0
- package/out/src/{synthesize-german.d.ts → synthesizers/german.d.ts} +2 -2
- package/out/src/synthesizers/german.d.ts.map +1 -0
- package/out/src/{synthesize-german.js → synthesizers/german.js} +2 -2
- package/out/src/synthesizers/german.js.map +1 -0
- package/out/src/{synthesize-house-venue.d.ts → synthesizers/house-venue.d.ts} +4 -4
- package/out/src/synthesizers/house-venue.d.ts.map +1 -0
- package/out/src/{synthesize-house-venue.js → synthesizers/house-venue.js} +4 -4
- package/out/src/synthesizers/house-venue.js.map +1 -0
- package/out/src/{synthesize-intersection.d.ts → synthesizers/intersection.d.ts} +2 -2
- package/out/src/synthesizers/intersection.d.ts.map +1 -0
- package/out/src/{synthesize-intersection.js → synthesizers/intersection.js} +1 -1
- package/out/src/synthesizers/intersection.js.map +1 -0
- package/out/src/{synthesize-no-street.d.ts → synthesizers/no-street.d.ts} +3 -3
- package/out/src/synthesizers/no-street.d.ts.map +1 -0
- package/out/src/{synthesize-no-street.js → synthesizers/no-street.js} +2 -2
- package/out/src/synthesizers/no-street.js.map +1 -0
- package/out/src/{synthesize-po-box.d.ts → synthesizers/po-box.d.ts} +2 -2
- package/out/src/synthesizers/po-box.d.ts.map +1 -0
- package/out/src/{synthesize-po-box.js → synthesizers/po-box.js} +1 -1
- package/out/src/synthesizers/po-box.js.map +1 -0
- package/out/src/{synthesize-street.d.ts → synthesizers/street.d.ts} +2 -2
- package/out/src/synthesizers/street.d.ts.map +1 -0
- package/out/src/{synthesize-street.js → synthesizers/street.js} +2 -2
- package/out/src/synthesizers/street.js.map +1 -0
- package/out/src/{synthesize.d.ts → synthesizers/utils.d.ts} +3 -3
- package/out/src/synthesizers/utils.d.ts.map +1 -0
- package/out/src/{synthesize.js → synthesizers/utils.js} +4 -15
- package/out/src/synthesizers/utils.js.map +1 -0
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +4 -8
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/audit.d.ts.map +1 -1
- package/out/src/tools/audit.js +4 -4
- package/out/src/tools/audit.js.map +1 -1
- package/out/src/tools/corpus-stats.d.ts +2 -2
- package/out/src/tools/corpus-stats.d.ts.map +1 -1
- package/out/src/tools/corpus-stats.js +18 -31
- package/out/src/tools/corpus-stats.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +5 -6
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +3 -15
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +14 -1
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +14 -1
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts +1 -1
- package/out/src/tools/fetch/nad.js +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +8 -14
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts +1 -1
- package/out/src/tools/fetch/openaddresses.js +1 -1
- package/out/src/tools/fetch/ourairports.d.ts +45 -0
- package/out/src/tools/fetch/ourairports.d.ts.map +1 -0
- package/out/src/tools/fetch/ourairports.js +123 -0
- package/out/src/tools/fetch/ourairports.js.map +1 -0
- package/out/src/tools/fetch/wikidata-subvenue.d.ts +171 -0
- package/out/src/tools/fetch/wikidata-subvenue.d.ts.map +1 -0
- package/out/src/tools/fetch/wikidata-subvenue.js +275 -0
- package/out/src/tools/fetch/wikidata-subvenue.js.map +1 -0
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +18 -16
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-promote.d.ts.map +1 -1
- package/out/src/tools/golden-promote.js +12 -6
- package/out/src/tools/golden-promote.js.map +1 -1
- package/out/src/tools/golden-relabel-street.d.ts +196 -0
- package/out/src/tools/golden-relabel-street.d.ts.map +1 -0
- package/out/src/tools/golden-relabel-street.js +440 -0
- package/out/src/tools/golden-relabel-street.js.map +1 -0
- package/out/src/tools/index.d.ts +4 -0
- package/out/src/tools/index.d.ts.map +1 -1
- package/out/src/tools/index.js +4 -0
- package/out/src/tools/index.js.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
- package/out/src/tools/jsonl-to-parquet.js +3 -2
- package/out/src/tools/jsonl-to-parquet.js.map +1 -1
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
- package/out/src/tools/lint-shard-vocab.js +3 -2
- package/out/src/tools/lint-shard-vocab.js.map +1 -1
- package/out/src/tools/lint-shard.d.ts +3 -3
- package/out/src/tools/lint-shard.d.ts.map +1 -1
- package/out/src/tools/lint-shard.js +23 -32
- package/out/src/tools/lint-shard.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +2 -1
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/overture-subvenue.d.ts +112 -0
- package/out/src/tools/overture-subvenue.d.ts.map +1 -0
- package/out/src/tools/overture-subvenue.js +144 -0
- package/out/src/tools/overture-subvenue.js.map +1 -0
- package/out/src/tools/shard-kryptonite.d.ts +1 -1
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +6 -6
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts +6 -3
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +9 -8
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/tools/sub-venue-lexicon.d.ts +507 -0
- package/out/src/tools/sub-venue-lexicon.d.ts.map +1 -0
- package/out/src/tools/sub-venue-lexicon.js +817 -0
- package/out/src/tools/sub-venue-lexicon.js.map +1 -0
- package/out/src/tools/sub-venue-promotions.d.ts +94 -0
- package/out/src/tools/sub-venue-promotions.d.ts.map +1 -0
- package/out/src/tools/sub-venue-promotions.js +266 -0
- package/out/src/tools/sub-venue-promotions.js.map +1 -0
- package/out/src/{align.d.ts → utils/align.d.ts} +1 -1
- package/out/src/utils/align.d.ts.map +1 -0
- package/out/src/utils/align.js.map +1 -0
- package/out/src/{golden.d.ts → utils/golden.d.ts} +5 -0
- package/out/src/utils/golden.d.ts.map +1 -0
- package/out/src/{golden.js → utils/golden.js} +24 -10
- package/out/src/utils/golden.js.map +1 -0
- package/out/src/utils/index.d.ts +14 -0
- package/out/src/utils/index.d.ts.map +1 -0
- package/out/src/utils/index.js +14 -0
- package/out/src/utils/index.js.map +1 -0
- package/out/src/utils/license.d.ts.map +1 -0
- package/out/src/utils/license.js.map +1 -0
- package/out/src/{parquet.d.ts → utils/parquet.d.ts} +2 -2
- package/out/src/utils/parquet.d.ts.map +1 -0
- package/out/src/{parquet.js → utils/parquet.js} +3 -15
- package/out/src/utils/parquet.js.map +1 -0
- package/out/src/{split.d.ts → utils/split.d.ts} +8 -1
- package/out/src/utils/split.d.ts.map +1 -0
- package/out/src/{split.js → utils/split.js} +7 -0
- package/out/src/utils/split.js.map +1 -0
- package/out/src/utils/tokenize.d.ts.map +1 -0
- package/out/src/utils/tokenize.js.map +1 -0
- package/out/src/{wof-json.d.ts → utils/wof-json.d.ts} +2 -2
- package/out/src/utils/wof-json.d.ts.map +1 -0
- package/out/src/{wof-json.js → utils/wof-json.js} +5 -8
- package/out/src/utils/wof-json.js.map +1 -0
- package/package.json +66 -20
- package/src/adapters/ban/adapter.ts +58 -68
- package/src/adapters/ban/street-decompose.ts +17 -25
- package/src/adapters/fcc-bdc/adapter.ts +5 -36
- package/src/adapters/geonames/adapter.ts +67 -76
- package/src/adapters/geonames-postal/adapter.ts +45 -54
- package/src/adapters/gnaf/adapter.ts +7 -11
- package/src/adapters/gnaf/assemble.ts +7 -11
- package/src/adapters/index.ts +3 -2
- package/src/adapters/openaddresses/adapter.ts +6 -12
- package/src/adapters/overture/adapter.ts +6 -10
- package/src/adapters/state-hi-schools/adapter.ts +53 -77
- package/src/adapters/state-ia-contractors/adapter.ts +55 -80
- package/src/adapters/state-ny-notaries/adapter.ts +64 -89
- package/src/adapters/state-tx-notaries/adapter.ts +63 -88
- package/src/adapters/synth-po-box/adapter.ts +12 -23
- package/src/adapters/tiger/adapter.ts +4 -3
- package/src/adapters/tiger/street-decompose.ts +21 -31
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +64 -88
- package/src/adapters/usgov-imls-pls/adapter.ts +57 -86
- package/src/adapters/usgov-irs-bmf/adapter.ts +55 -66
- package/src/adapters/usgov-nad/adapter.ts +7 -11
- package/src/adapters/usgov-nppes/adapter.ts +54 -79
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +48 -75
- package/src/{adapter.ts → adapters/utils/index.ts} +59 -3
- package/src/adapters/wof-admin-jp/adapter.ts +35 -9
- package/src/adapters/wof-admin-json/adapter.ts +3 -4
- package/src/adapters/wof-postalcode-json/adapter.ts +3 -4
- package/src/build.ts +12 -11
- package/src/index.ts +3 -17
- package/src/name-prone-us-suffixes.ts +12 -0
- package/src/parquet-wrapper/reader.ts +19 -0
- package/src/parquet-wrapper/schema.ts +4 -0
- package/src/runner.ts +7 -2
- package/src/shard-recipes/anchor-absorption.ts +5 -4
- package/src/shard-recipes/boundary-stress.ts +6 -2
- package/src/shard-recipes/country-balanced.ts +77 -91
- package/src/shard-recipes/fr-admin-split.ts +3 -3
- package/src/shard-recipes/fr-fragment.ts +10 -7
- package/src/shard-recipes/fr-lieudit.ts +3 -3
- package/src/shard-recipes/fr-order.ts +18 -83
- package/src/shard-recipes/german.ts +17 -80
- package/src/shard-recipes/house-venue.ts +2 -1
- package/src/shard-recipes/index.ts +4 -1
- package/src/shard-recipes/intersection.ts +37 -74
- package/src/shard-recipes/locale.ts +31 -39
- package/src/shard-recipes/no-fragment.ts +10 -7
- package/src/shard-recipes/no-street-led.ts +10 -7
- package/src/shard-recipes/no-street.ts +2 -1
- package/src/shard-recipes/po-box-cedex.ts +127 -161
- package/src/shard-recipes/po-box.ts +6 -1
- package/src/shard-recipes/scaffold.ts +125 -27
- package/src/shard-recipes/street-affix.ts +335 -134
- package/src/shard-recipes/street-bare.ts +4 -3
- package/src/shard-recipes/street.ts +3 -2
- package/src/shard-recipes/sub-venue-sources.ts +895 -0
- package/src/shard-recipes/sub-venue.ts +982 -0
- package/src/shard-recipes/unit.ts +27 -85
- package/src/{synthesize-boundary-stress.ts → synthesizers/boundary-stress.ts} +2 -2
- package/src/{synthesize-german.ts → synthesizers/german.ts} +2 -2
- package/src/{synthesize-house-venue.ts → synthesizers/house-venue.ts} +4 -4
- package/src/{synthesize-intersection.ts → synthesizers/intersection.ts} +1 -1
- package/src/{synthesize-no-street.ts → synthesizers/no-street.ts} +2 -2
- package/src/{synthesize-po-box.ts → synthesizers/po-box.ts} +1 -1
- package/src/{synthesize-street.ts → synthesizers/street.ts} +2 -2
- package/src/{synthesize.ts → synthesizers/utils.ts} +4 -18
- package/src/tools/align-shard.ts +4 -7
- package/src/tools/audit.ts +6 -5
- package/src/tools/corpus-stats.ts +25 -33
- package/src/tools/fetch/download.ts +7 -5
- package/src/tools/fetch/imls-pls.ts +4 -18
- package/src/tools/fetch/index.ts +14 -1
- package/src/tools/fetch/nad.ts +1 -1
- package/src/tools/fetch/nppes.ts +9 -16
- package/src/tools/fetch/openaddresses.ts +1 -1
- package/src/tools/fetch/ourairports.ts +166 -0
- package/src/tools/fetch/wikidata-subvenue.ts +386 -0
- package/src/tools/golden-expand.ts +18 -13
- package/src/tools/golden-promote.ts +15 -6
- package/src/tools/golden-relabel-street.ts +685 -0
- package/src/tools/index.ts +4 -0
- package/src/tools/jsonl-to-parquet.ts +3 -2
- package/src/tools/lint-shard-vocab.ts +3 -2
- package/src/tools/lint-shard.ts +33 -36
- package/src/tools/overlay-manifest.ts +2 -1
- package/src/tools/overture-subvenue.ts +215 -0
- package/src/tools/shard-kryptonite.ts +8 -9
- package/src/tools/shard-translit.ts +22 -11
- package/src/tools/sub-venue-lexicon.ts +1250 -0
- package/src/tools/sub-venue-promotions.ts +330 -0
- package/src/{align.ts → utils/align.ts} +1 -1
- package/src/{golden.ts → utils/golden.ts} +25 -11
- package/src/utils/index.ts +14 -0
- package/src/{parquet.ts → utils/parquet.ts} +5 -19
- package/src/{split.ts → utils/split.ts} +8 -2
- package/src/{wof-json.ts → utils/wof-json.ts} +5 -8
- package/out/src/adapter.d.ts.map +0 -1
- package/out/src/adapter.js.map +0 -1
- package/out/src/align.d.ts.map +0 -1
- package/out/src/align.js.map +0 -1
- package/out/src/format.d.ts +0 -14
- package/out/src/format.d.ts.map +0 -1
- package/out/src/format.js +0 -14
- package/out/src/format.js.map +0 -1
- package/out/src/golden.d.ts.map +0 -1
- package/out/src/golden.js.map +0 -1
- package/out/src/license.d.ts.map +0 -1
- package/out/src/license.js.map +0 -1
- package/out/src/parquet.d.ts.map +0 -1
- package/out/src/parquet.js.map +0 -1
- package/out/src/split.d.ts.map +0 -1
- package/out/src/split.js.map +0 -1
- package/out/src/synthesize-anchor-absorption.d.ts.map +0 -1
- package/out/src/synthesize-anchor-absorption.js.map +0 -1
- package/out/src/synthesize-boundary-stress.d.ts.map +0 -1
- package/out/src/synthesize-boundary-stress.js.map +0 -1
- package/out/src/synthesize-german.d.ts.map +0 -1
- package/out/src/synthesize-german.js.map +0 -1
- package/out/src/synthesize-house-venue.d.ts.map +0 -1
- package/out/src/synthesize-house-venue.js.map +0 -1
- package/out/src/synthesize-intersection.d.ts.map +0 -1
- package/out/src/synthesize-intersection.js.map +0 -1
- package/out/src/synthesize-no-street.d.ts.map +0 -1
- package/out/src/synthesize-no-street.js.map +0 -1
- package/out/src/synthesize-po-box.d.ts.map +0 -1
- package/out/src/synthesize-po-box.js.map +0 -1
- package/out/src/synthesize-street.d.ts.map +0 -1
- package/out/src/synthesize-street.js.map +0 -1
- package/out/src/synthesize.d.ts.map +0 -1
- package/out/src/synthesize.js.map +0 -1
- package/out/src/tokenize.d.ts.map +0 -1
- package/out/src/tokenize.js.map +0 -1
- package/out/src/wof-json.d.ts.map +0 -1
- package/out/src/wof-json.js.map +0 -1
- package/src/format.ts +0 -14
- /package/out/src/{align.js → utils/align.js} +0 -0
- /package/out/src/{license.d.ts → utils/license.d.ts} +0 -0
- /package/out/src/{license.js → utils/license.js} +0 -0
- /package/out/src/{tokenize.d.ts → utils/tokenize.d.ts} +0 -0
- /package/out/src/{tokenize.js → utils/tokenize.js} +0 -0
- /package/src/{synthesize-anchor-absorption.ts → synthesizers/anchor-absorption.ts} +0 -0
- /package/src/{license.ts → utils/license.ts} +0 -0
- /package/src/{tokenize.ts → utils/tokenize.ts} +0 -0
|
@@ -29,16 +29,16 @@
|
|
|
29
29
|
* stream is still open during `run`), and the dead `renderReversed` helper (the legacy `main`
|
|
30
30
|
* inlined the variant logic and never called it).
|
|
31
31
|
*/
|
|
32
|
-
import {
|
|
33
|
-
import { stableSourceID } from "
|
|
34
|
-
import { alignRow } from "
|
|
35
|
-
import { makeMulberry32 } from "./scaffold.js";
|
|
32
|
+
import { dataRootPath } from "@mailwoman/core/utils";
|
|
33
|
+
import { stableSourceID } from "@mailwoman/corpus/adapters/utils";
|
|
34
|
+
import { alignRow } from "@mailwoman/corpus/utils";
|
|
35
|
+
import { makeMulberry32, readZippedCSVRecords } from "./scaffold.js";
|
|
36
36
|
/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
|
|
37
37
|
cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
|
|
38
38
|
output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
|
|
39
39
|
would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
|
|
40
40
|
extracted as named constants above. */
|
|
41
|
-
const SOURCE = { zip: "
|
|
41
|
+
const SOURCE = { zip: dataRootPath("oa-cache", "fr__countrywide.zip"), csv: "fr/countrywide.csv" };
|
|
42
42
|
/**
|
|
43
43
|
* Ordinal suffixes used in French house numbers (BAN corpus), to cover the "8 bis" sub-mode.
|
|
44
44
|
*/
|
|
@@ -52,74 +52,20 @@ const ORDINAL_PROB = 0.12;
|
|
|
52
52
|
*/
|
|
53
53
|
const ALLCAPS_PROB = 0.1;
|
|
54
54
|
/**
|
|
55
|
-
*
|
|
55
|
+
* Stream FR tuples out of the cached OA zip. The countrywide extract is GB-scale, so this reads only as far as `limit`
|
|
56
|
+
* distinct tuples — the `break` closes the reader and releases the archive. Only keeps rows with a house_number (the
|
|
57
|
+
* shard's core signal) and a postcode (required for reversed-order rendering to be meaningful).
|
|
56
58
|
*/
|
|
57
|
-
function
|
|
58
|
-
const out = [];
|
|
59
|
-
let cur = "";
|
|
60
|
-
let inQ = false;
|
|
61
|
-
for (let i = 0; i < line.length; i++) {
|
|
62
|
-
const c = line[i];
|
|
63
|
-
if (inQ) {
|
|
64
|
-
if (c === '"') {
|
|
65
|
-
if (line[i + 1] === '"') {
|
|
66
|
-
cur += '"';
|
|
67
|
-
i++;
|
|
68
|
-
}
|
|
69
|
-
else {
|
|
70
|
-
inQ = false;
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
else {
|
|
74
|
-
cur += c;
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
else if (c === '"') {
|
|
78
|
-
inQ = true;
|
|
79
|
-
}
|
|
80
|
-
else if (c === ",") {
|
|
81
|
-
out.push(cur);
|
|
82
|
-
cur = "";
|
|
83
|
-
}
|
|
84
|
-
else {
|
|
85
|
-
cur += c;
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
out.push(cur);
|
|
89
|
-
return out;
|
|
90
|
-
}
|
|
91
|
-
/**
|
|
92
|
-
* Stream FR tuples out of the cached OA zip. The countrywide extract is GB-scale; cap with `head` to stay under V8's
|
|
93
|
-
* string limit. Only keeps rows with a house_number (the shard's core signal) and a postcode (required for
|
|
94
|
-
* reversed-order rendering to be meaningful).
|
|
95
|
-
*/
|
|
96
|
-
function readTuples(limit) {
|
|
97
|
-
const maxLines = Math.max(limit * 8, 40_000) + 1;
|
|
98
|
-
const r = spawnSync("bash", ["-c", `unzip -p "${SOURCE.zip}" "${SOURCE.csv}" | head -n ${maxLines}`], {
|
|
99
|
-
maxBuffer: 1024 * 1024 * 1024,
|
|
100
|
-
encoding: "buffer",
|
|
101
|
-
});
|
|
102
|
-
if (r.status !== 0) {
|
|
103
|
-
console.error(` WARN: unzip failed for ${SOURCE.zip} (status ${r.status})`);
|
|
104
|
-
return [];
|
|
105
|
-
}
|
|
106
|
-
const lines = r.stdout.toString("utf8").split(/\r?\n/);
|
|
107
|
-
if (lines.length < 2)
|
|
108
|
-
return [];
|
|
109
|
-
const header = splitCSV(lines[0]).map((h) => h.trim().toLowerCase());
|
|
110
|
-
const idx = (name) => header.indexOf(name);
|
|
111
|
-
const iNum = idx("number"), iStreet = idx("street"), iCity = idx("city"), iPost = idx("postcode");
|
|
112
|
-
const get = (cells, i) => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "");
|
|
59
|
+
async function readTuples(limit) {
|
|
113
60
|
const tuples = [];
|
|
114
61
|
const seen = new Set();
|
|
115
|
-
for (
|
|
116
|
-
if (
|
|
117
|
-
|
|
118
|
-
const
|
|
119
|
-
const
|
|
120
|
-
const
|
|
121
|
-
const
|
|
122
|
-
const postcode = get(cells, iPost);
|
|
62
|
+
for await (const row of readZippedCSVRecords(SOURCE.zip, SOURCE.csv)) {
|
|
63
|
+
if (tuples.length >= limit)
|
|
64
|
+
break;
|
|
65
|
+
const street = row.street ?? "";
|
|
66
|
+
const locality = row.city ?? "";
|
|
67
|
+
const house_number = row.number ?? "";
|
|
68
|
+
const postcode = row.postcode ?? "";
|
|
123
69
|
// Require all four fields: HN is the signal; postcode drives reversed-order variants.
|
|
124
70
|
if (!street || !locality || !house_number || !postcode)
|
|
125
71
|
continue;
|
|
@@ -172,10 +118,10 @@ export const frOrderRecipe = {
|
|
|
172
118
|
const reversedFraction = opts.reversedFraction ?? 0.5;
|
|
173
119
|
// Over-read from the CSV so the dedup + filter pass can fill `count` rows.
|
|
174
120
|
const poolLimit = Math.max(count * 8, 40_000);
|
|
175
|
-
const pool = readTuples(poolLimit);
|
|
121
|
+
const pool = await readTuples(poolLimit);
|
|
176
122
|
console.error(` ${SOURCE.csv}: ${pool.length} unique tuples (capped read)`);
|
|
177
123
|
if (!pool.length) {
|
|
178
|
-
throw new Error(
|
|
124
|
+
throw new Error(`No FR tuples found — is ${SOURCE.zip} present?`);
|
|
179
125
|
}
|
|
180
126
|
let emitted = 0;
|
|
181
127
|
let skipped = 0;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fr-order.js","sourceRoot":"","sources":["../../../src/shard-recipes/fr-order.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;
|
|
1
|
+
{"version":3,"file":"fr-order.js","sourceRoot":"","sources":["../../../src/shard-recipes/fr-order.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAA;AAEjE,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAA;AAElD,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAoB,MAAM,eAAe,CAAA;AAEtF;;;;yCAIyC;AAEzC,MAAM,MAAM,GAAG,EAAE,GAAG,EAAE,YAAY,CAAC,UAAU,EAAE,qBAAqB,CAAC,EAAE,GAAG,EAAE,oBAAoB,EAAE,CAAA;AAElG;;GAEG;AACH,MAAM,gBAAgB,GAAsB,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAA;AACpE;;GAEG;AACH,MAAM,YAAY,GAAG,IAAI,CAAA;AACzB;;GAEG;AACH,MAAM,YAAY,GAAG,GAAG,CAAA;AAYxB;;;;GAIG;AACH,KAAK,UAAU,UAAU,CAAC,KAAa;IACtC,MAAM,MAAM,GAAc,EAAE,CAAA;IAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,oBAAoB,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QACtE,IAAI,MAAM,CAAC,MAAM,IAAI,KAAK;YAAE,MAAK;QACjC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAA;QAC/B,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,IAAI,EAAE,CAAA;QAC/B,MAAM,YAAY,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAA;QACrC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAA;QAEnC,sFAAsF;QACtF,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,IAAI,CAAC,QAAQ;YAAE,SAAQ;QAChE,MAAM,GAAG,GAAG,GAAG,YAAY,IAAI,MAAM,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC,WAAW,EAAE,CAAA;QAE7E,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAQ;QAC3B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACb,MAAM,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;IAC1D,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;;GAGG;AACH,SAAS,eAAe,CAAC,MAAoB,EAAE,YAAoB;IAClE,IAAI,MAAM,EAAE,IAAI,YAAY;QAAE,OAAO,YAAY,CAAA;IACjD,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAE,CAAA;IAEhF,uFAAuF;IACvF,OAAO,GAAG,YAAY,IAAI,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,CAAA;AAC3E,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CACvB,EAAU,EACV,MAAc,EACd,QAAgB,EAChB,QAAgB;IAEhB,MAAM,GAAG,GAAG,GAAG,EAAE,IAAI,MAAM,KAAK,QAAQ,IAAI,QAAQ,EAAE,CAAA;IAEtD,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAA;AAC7E,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,aAAa,GAAgB;IACzC,IAAI,EAAE,UAAU;IAChB,WAAW,EAAE,qGAAqG;IAClH,IAAI,EAAE,UAAU;IAChB,OAAO,EAAE;QACR,EAAE,IAAI,EAAE,yBAAyB,EAAE,WAAW,EAAE,+CAA+C,EAAE;QACjG,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,6CAA6C,EAAE;KAChF;IACD,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;QAC/E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAA;QACxB,+GAA+G;QAC/G,4GAA4G;QAC5G,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,IAAI,gBAAgB,CAAA;QAClD,MAAM,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,IAAI,GAAG,CAAA;QAErD,2EAA2E;QAC3E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,MAAM,CAAC,CAAA;QAC7C,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,CAAA;QAExC,OAAO,CAAC,KAAK,CAAC,KAAK,MAAM,CAAC,GAAG,KAAK,IAAI,CAAC,MAAM,8BAA8B,CAAC,CAAA;QAE5E,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,2BAA2B,MAAM,CAAC,GAAG,WAAW,CAAC,CAAA;QAClE,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAA;QAErB,OAAO,OAAO,GAAG,KAAK,IAAI,KAAK,EAAE,GAAG,KAAK,GAAG,EAAE,EAAE,CAAC;YAChD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAE,CAAA;YAC5C,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAA;YACjC,MAAM,QAAQ,GAAG,MAAM,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAA;YACtF,MAAM,YAAY,GAAG,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAA;YAE/D,qDAAqD;YACrD,MAAM,UAAU,GAAG,MAAM,EAAE,GAAG,gBAAgB,CAAA;YAE9C,IAAI,QAA4E,CAAA;YAEhF,IAAI,UAAU,EAAE,CAAC;gBAChB,MAAM,WAAW,GAAG,MAAM,EAAE,CAAA;gBAC5B,IAAI,GAAW,CAAA;gBAEf,IAAI,WAAW,GAAG,IAAI,EAAE,CAAC;oBACxB,qDAAqD;oBACrD,GAAG,GAAG,GAAG,QAAQ,IAAI,QAAQ,KAAK,YAAY,IAAI,MAAM,EAAE,CAAA;gBAC3D,CAAC;qBAAM,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;oBAC9B,sFAAsF;oBACtF,GAAG,GAAG,GAAG,QAAQ,KAAK,QAAQ,KAAK,YAAY,IAAI,MAAM,EAAE,CAAA;gBAC5D,CAAC;qBAAM,IAAI,WAAW,GAAG,IAAI,EAAE,CAAC;oBAC/B,iFAAiF;oBACjF,GAAG,GAAG,GAAG,QAAQ,IAAI,YAAY,IAAI,MAAM,IAAI,QAAQ,EAAE,CAAA;gBAC1D,CAAC;qBAAM,CAAC;oBACP,gEAAgE;oBAChE,GAAG,GAAG,GAAG,QAAQ,KAAK,YAAY,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAA;gBAC5D,CAAC;gBAED,QAAQ,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAA;YAC7E,CAAC;iBAAM,CAAC;gBACP,QAAQ,GAAG,eAAe,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;YACrE,CAAC;YAED,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,QAAQ,CAAA;YAEpC,sFAAsF;YACtF,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,OAAO,CAAa,CAAA;YAE7E,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,4EAA4E;YAC5E,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;gBAEhE,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE;gBACvC,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,YAAY,EAAE,UAAU,CAAC,YAAY;gBACrC,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,QAAQ,EAAE,UAAU,CAAC,QAAQ;aAC7B,CAAC,CAAA;YAEF,MAAM,SAAS,GAAiB;gBAC/B,GAAG;gBACH,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM;gBACN,SAAS,EAAE,QAAQ;gBACnB,cAAc,EAAE,OAAO;gBACvB,OAAO,EAAE,8FAA8F;aACvG,CAAA;YAED,MAAM,OAAO,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAA;YAEnC,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChD,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,KAAK,CACJ,IAAI,CAAC,SAAS,CAAC;gBACd,GAAG,OAAO,CAAC,GAAG;gBACd,YAAY,EAAE,UAAU;gBACxB,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW;gBAClD,aAAa,EAAE,IAAI;aACnB,CAAC,GAAG,IAAI,CACT,CAAA;YAED,OAAO,EAAE,CAAA;QACV,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;CACD,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"german.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/german.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAOH,OAAO,
|
|
1
|
+
{"version":3,"file":"german.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/german.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAOH,OAAO,EAAwC,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AA+CtF;;;GAGG;AACH,eAAO,MAAM,YAAY,EAAE,WA+F1B,CAAA"}
|
|
@@ -14,88 +14,35 @@
|
|
|
14
14
|
* German order so well it reads the US/feed-order eval as a "collapse". See
|
|
15
15
|
* docs/articles/evals/resolver-geo/2026-06-06-anchor-pilot.md (the order-artifact correction).
|
|
16
16
|
*/
|
|
17
|
-
import {
|
|
18
|
-
import { stableSourceID } from "
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
21
|
-
import { makeMulberry32 } from "./scaffold.js";
|
|
17
|
+
import { dataRootPath } from "@mailwoman/core/utils";
|
|
18
|
+
import { stableSourceID } from "@mailwoman/corpus/adapters/utils";
|
|
19
|
+
import { synthesizeGermanRow } from "@mailwoman/corpus/synthesizers/german";
|
|
20
|
+
import { alignRow } from "@mailwoman/corpus/utils";
|
|
21
|
+
import { makeMulberry32, readZippedCSVRecords } from "./scaffold.js";
|
|
22
22
|
/**
|
|
23
23
|
* `region` is the Bundesland the source covers. OA's REGION column is empty for DE, but the region is implied by the
|
|
24
24
|
* per-state file — the international order needs it for the "City, Region Postcode" tail (v0.9.3 / #327). berlin.csv →
|
|
25
25
|
* Berlin (a city-state, region==locality); sn/statewide → Sachsen.
|
|
26
26
|
*/
|
|
27
27
|
const SOURCES = [
|
|
28
|
-
{ zip: "
|
|
29
|
-
{ zip: "
|
|
28
|
+
{ zip: dataRootPath("oa-cache", "de__berlin.zip"), csv: "de/berlin.csv", region: "Berlin" },
|
|
29
|
+
{ zip: dataRootPath("oa-cache", "de__sn__statewide.zip"), csv: "de/sn/statewide.csv", region: "Sachsen" },
|
|
30
30
|
];
|
|
31
31
|
/**
|
|
32
|
-
*
|
|
32
|
+
* Stream real German tuples out of a cached OA zip.
|
|
33
33
|
*/
|
|
34
|
-
function
|
|
35
|
-
const out = [];
|
|
36
|
-
let cur = "";
|
|
37
|
-
let inQ = false;
|
|
38
|
-
for (let i = 0; i < line.length; i++) {
|
|
39
|
-
const c = line[i];
|
|
40
|
-
if (inQ) {
|
|
41
|
-
if (c === '"') {
|
|
42
|
-
if (line[i + 1] === '"') {
|
|
43
|
-
cur += '"';
|
|
44
|
-
i++;
|
|
45
|
-
}
|
|
46
|
-
else {
|
|
47
|
-
inQ = false;
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
else {
|
|
51
|
-
cur += c;
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
else if (c === '"') {
|
|
55
|
-
inQ = true;
|
|
56
|
-
}
|
|
57
|
-
else if (c === ",") {
|
|
58
|
-
out.push(cur);
|
|
59
|
-
cur = "";
|
|
60
|
-
}
|
|
61
|
-
else {
|
|
62
|
-
cur += c;
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
out.push(cur);
|
|
66
|
-
return out;
|
|
67
|
-
}
|
|
68
|
-
/**
|
|
69
|
-
* Stream real German tuples out of a cached OA zip (buffered `unzip -p`).
|
|
70
|
-
*/
|
|
71
|
-
function readGermanTuples(source) {
|
|
72
|
-
const r = spawnSync("unzip", ["-p", source.zip, source.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" });
|
|
73
|
-
if (r.status !== 0) {
|
|
74
|
-
console.error(` WARN: unzip failed for ${source.zip} (status ${r.status})`);
|
|
75
|
-
return [];
|
|
76
|
-
}
|
|
77
|
-
const lines = r.stdout.toString("utf8").split(/\r?\n/);
|
|
78
|
-
if (lines.length < 2)
|
|
79
|
-
return [];
|
|
80
|
-
const header = splitCSV(lines[0]).map((h) => h.trim().toLowerCase());
|
|
81
|
-
const idx = (name) => header.indexOf(name);
|
|
82
|
-
const iNum = idx("number"), iStreet = idx("street"), iCity = idx("city"), iRegion = idx("region"), iPost = idx("postcode");
|
|
83
|
-
const get = (cells, i) => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "");
|
|
34
|
+
async function readGermanTuples(source) {
|
|
84
35
|
const tuples = [];
|
|
85
36
|
const seen = new Set();
|
|
86
|
-
for (
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
continue;
|
|
90
|
-
const cells = splitCSV(lineStr);
|
|
91
|
-
const street = get(cells, iStreet);
|
|
92
|
-
const locality = get(cells, iCity);
|
|
37
|
+
for await (const row of readZippedCSVRecords(source.zip, source.csv)) {
|
|
38
|
+
const street = row.street ?? "";
|
|
39
|
+
const locality = row.city ?? "";
|
|
93
40
|
if (!street || !locality)
|
|
94
41
|
continue;
|
|
95
|
-
const house_number =
|
|
96
|
-
const postcode =
|
|
42
|
+
const house_number = row.number ?? "";
|
|
43
|
+
const postcode = row.postcode ?? "";
|
|
97
44
|
// OA's REGION column is empty for DE — fall back to the source's Bundesland (set per file).
|
|
98
|
-
const region =
|
|
45
|
+
const region = row.region || source.region || "";
|
|
99
46
|
const key = `${house_number}|${street}|${locality}|${postcode}`.toLowerCase();
|
|
100
47
|
if (seen.has(key))
|
|
101
48
|
continue;
|
|
@@ -125,14 +72,14 @@ export const germanRecipe = {
|
|
|
125
72
|
// Pool real tuples from every German source, then sample `count` rows from it.
|
|
126
73
|
const pool = [];
|
|
127
74
|
for (const s of SOURCES) {
|
|
128
|
-
const t = readGermanTuples(s);
|
|
75
|
+
const t = await readGermanTuples(s);
|
|
129
76
|
console.error(` ${s.csv}: ${t.length} unique tuples`);
|
|
130
77
|
for (const x of t) {
|
|
131
78
|
pool.push(x);
|
|
132
79
|
} // NOT pool.push(...t) — spreading ~840K args overflows the stack
|
|
133
80
|
}
|
|
134
81
|
if (!pool.length) {
|
|
135
|
-
throw new Error(
|
|
82
|
+
throw new Error(`No German tuples found — are the cached zips present in ${dataRootPath("oa-cache")}?`);
|
|
136
83
|
}
|
|
137
84
|
let emitted = 0;
|
|
138
85
|
let skipped = 0;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"german.js","sourceRoot":"","sources":["../../../src/shard-recipes/german.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"german.js","sourceRoot":"","sources":["../../../src/shard-recipes/german.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAA;AACjE,OAAO,EAAE,mBAAmB,EAAwB,MAAM,uCAAuC,CAAA;AACjG,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAA;AAElD,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAoB,MAAM,eAAe,CAAA;AAWtF;;;;GAIG;AACH,MAAM,OAAO,GAAmB;IAC/B,EAAE,GAAG,EAAE,YAAY,CAAC,UAAU,EAAE,gBAAgB,CAAC,EAAE,GAAG,EAAE,eAAe,EAAE,MAAM,EAAE,QAAQ,EAAE;IAC3F,EAAE,GAAG,EAAE,YAAY,CAAC,UAAU,EAAE,uBAAuB,CAAC,EAAE,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,SAAS,EAAE;CACzG,CAAA;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAAC,MAAoB;IACnD,MAAM,MAAM,GAAsB,EAAE,CAAA;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,oBAAoB,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QACtE,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAA;QAC/B,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,IAAI,EAAE,CAAA;QAE/B,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ;YAAE,SAAQ;QAClC,MAAM,YAAY,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAA;QACrC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAA;QACnC,4FAA4F;QAC5F,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,IAAI,EAAE,CAAA;QAChD,MAAM,GAAG,GAAG,GAAG,YAAY,IAAI,MAAM,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC,WAAW,EAAE,CAAA;QAE7E,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAQ;QAC3B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACb,MAAM,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAA;IAClE,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAgB;IACxC,IAAI,EAAE,QAAQ;IACd,WAAW,EAAE,6FAA6F;IAC1G,IAAI,EAAE,UAAU;IAChB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,WAAW,EAAE,oDAAoD,EAAE,CAAC;IAC7G,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,6EAA6E;QAC7E,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,IAAI,cAAc,CAAA;QAChD,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,GAAG,CAAA;QAE7C,IAAI,CAAC,CAAC,YAAY,IAAI,CAAC,IAAI,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,0CAA0C,YAAY,EAAE,CAAC,CAAA;QAC1E,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAA;QAEhC,+EAA+E;QAC/E,MAAM,IAAI,GAAsB,EAAE,CAAA;QAElC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,MAAM,gBAAgB,CAAC,CAAC,CAAC,CAAA;YAEnC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,gBAAgB,CAAC,CAAA;YAEtD,KAAK,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACb,CAAC,CAAC,iEAAiE;QACpE,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,2DAA2D,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,CAAA;QACxG,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAA;QAErB,OAAO,OAAO,GAAG,KAAK,IAAI,KAAK,EAAE,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAE,CAAA;YAC5C,iGAAiG;YACjG,2EAA2E;YAC3E,MAAM,KAAK,GAAG,MAAM,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAA;YAClE,MAAM,KAAK,GAAG,mBAAmB,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAA;YAE1D,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,6FAA6F;YAC7F,gEAAgE;YAChE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;gBAEpG,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE;gBACvC,MAAM,EAAE,KAAK,CAAC,UAAU,CAAC,MAAM;gBAC/B,YAAY,EAAE,KAAK,CAAC,UAAU,CAAC,YAAY;gBAC3C,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ;gBACnC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ;aACnC,CAAC,CAAA;YAEF,MAAM,SAAS,GAAG;gBACjB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM;gBACN,SAAS,EAAE,QAAQ;gBACnB,cAAc,EAAE,OAAO;gBACvB,OAAO,EAAE,qDAAqD,KAAK,6BAA6B;aAChG,CAAA;YAED,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,CAAC,CAAA;YAErE,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChD,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;YAEjH,OAAO,EAAE,CAAA;QACV,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;CACD,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"house-venue.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/house-venue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;
|
|
1
|
+
{"version":3,"file":"house-venue.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/house-venue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAIH,OAAO,EAAqD,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AAInG;;;GAGG;AACH,eAAO,MAAM,gBAAgB,EAAE,WAiE9B,CAAA"}
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* signal that no-street's distributional shift cost the model. Ported from
|
|
10
10
|
* scripts/build-house-venue-shard.mjs.
|
|
11
11
|
*/
|
|
12
|
-
import { synthesizeHouseVenueRow } from "
|
|
12
|
+
import { synthesizeHouseVenueRow } from "@mailwoman/corpus/synthesizers/house-venue";
|
|
13
13
|
import { alignAndWrite, makeLcg, readTuples, shardSourceID } from "./scaffold.js";
|
|
14
14
|
const LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples";
|
|
15
15
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"house-venue.js","sourceRoot":"","sources":["../../../src/shard-recipes/house-venue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,uBAAuB,EAA4B,MAAM,
|
|
1
|
+
{"version":3,"file":"house-venue.js","sourceRoot":"","sources":["../../../src/shard-recipes/house-venue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,uBAAuB,EAA4B,MAAM,4CAA4C,CAAA;AAE9G,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,UAAU,EAAE,aAAa,EAAoB,MAAM,eAAe,CAAA;AAEnG,MAAM,OAAO,GAAG,6DAA6D,CAAA;AAE7E;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAgB;IAC5C,IAAI,EAAE,aAAa;IACnB,WAAW,EAAE,8FAA8F;IAC3G,IAAI,EAAE,QAAQ;IACd,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAA;QACtF,0FAA0F;QAC1F,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,IAAI,mBAAmB,CAAA;QACrD,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAClD,IAAI,EAAE,CAAA;YAEN,0FAA0F;YAC1F,qEAAqE;YACrE,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,EAAE,CAAC;gBACvG,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxC,MAAM,KAAK,GAAG,uBAAuB,CAAC,KAA4B,EAAE,EAAE,MAAM,EAAE,CAAC,CAAA;gBAE/E,IAAI,CAAC,KAAK,EAAE,CAAC;oBACZ,OAAO,EAAE,CAAA;oBAET,SAAQ;gBACT,CAAC;gBAED,MAAM,EAAE,GAAG,aAAa,CACvB,KAAK,EACL;oBACC,GAAG,EAAE,KAAK,CAAC,GAAG;oBACd,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM;oBACN,SAAS,EAAE,aAAa,CAAC,MAAM,EAAE;wBAChC,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,OAAO,EAAE,KAAK,CAAC,OAAO;wBACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;qBACZ,CAAC;oBACF,cAAc,EAAE,OAAO;oBACvB,OAAO,EAAE,OAAO;iBAChB,EACD,KAAK,CAAC,QAAQ,CACd,CAAA;gBAED,IAAI,EAAE,EAAE,CAAC;oBACR,OAAO,EAAE,CAAA;gBACV,CAAC;qBAAM,CAAC;oBACP,OAAO,EAAE,CAAA;gBACV,CAAC;YACF,CAAC;QACF,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAClC,CAAC;CACD,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAqBH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAqBH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAQhD,cAAc,eAAe,CAAA;AAkC7B;;GAEG;AACH,eAAO,MAAM,aAAa,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,CAA4C,CAAA;AAEvG;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAEpE;AAED;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,SAAS,WAAW,EAAE,CAEzD"}
|
|
@@ -27,9 +27,10 @@ import { noStreetRecipe } from "./no-street.js";
|
|
|
27
27
|
import { poBoxCedexRecipe } from "./po-box-cedex.js";
|
|
28
28
|
import { poBoxRecipe } from "./po-box.js";
|
|
29
29
|
import { siBareVillageRecipe } from "./si-bare-village.js";
|
|
30
|
-
import { streetAffixRecipe } from "./street-affix.js";
|
|
30
|
+
import { streetAffixRecipe, suffixBoundaryRecipe } from "./street-affix.js";
|
|
31
31
|
import { streetBareRecipe } from "./street-bare.js";
|
|
32
32
|
import { streetRecipe } from "./street.js";
|
|
33
|
+
import { subVenueRecipe } from "./sub-venue.js";
|
|
33
34
|
import { unitRecipe } from "./unit.js";
|
|
34
35
|
export * from "./scaffold.js";
|
|
35
36
|
/**
|
|
@@ -39,6 +40,7 @@ const RECIPES = [
|
|
|
39
40
|
streetRecipe,
|
|
40
41
|
streetBareRecipe,
|
|
41
42
|
streetAffixRecipe,
|
|
43
|
+
suffixBoundaryRecipe,
|
|
42
44
|
noStreetRecipe,
|
|
43
45
|
houseVenueRecipe,
|
|
44
46
|
poBoxRecipe,
|
|
@@ -60,6 +62,7 @@ const RECIPES = [
|
|
|
60
62
|
countryBalancedRecipe,
|
|
61
63
|
boundaryStressRecipe,
|
|
62
64
|
anchorAbsorptionRecipe,
|
|
65
|
+
subVenueRecipe,
|
|
63
66
|
];
|
|
64
67
|
/**
|
|
65
68
|
* Recipe name → recipe.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAA;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAC7D,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAA;AACxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAA;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAC7D,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAA;AACxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAA;AAC3E,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AAEtC,cAAc,eAAe,CAAA;AAE7B;;GAEG;AACH,MAAM,OAAO,GAA2B;IACvC,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,oBAAoB;IACpB,cAAc;IACd,gBAAgB;IAChB,WAAW;IACX,gBAAgB;IAChB,UAAU;IACV,kBAAkB;IAClB,YAAY;IACZ,YAAY;IACZ,aAAa;IACb,kBAAkB;IAClB,kBAAkB;IAClB,gBAAgB;IAChB,eAAe;IACf,0BAA0B;IAC1B,gBAAgB;IAChB,iBAAiB;IACjB,gBAAgB;IAChB,mBAAmB;IACnB,qBAAqB;IACrB,oBAAoB;IACpB,sBAAsB;IACtB,cAAc;CACd,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAqC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AAEvG;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,OAAO,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB;IAC/B,OAAO,OAAO,CAAA;AACf,CAAC"}
|
|
@@ -30,8 +30,8 @@
|
|
|
30
30
|
* External inputs (`--edges-dir`, opts.edgesDir; both already on disk — do not re-download):
|
|
31
31
|
*
|
|
32
32
|
* - <edges-dir>/tl_2023_{17031,34027,50023}_edges.shp (unzipped TIGER 2023 EDGES; default
|
|
33
|
-
* /
|
|
34
|
-
* - /
|
|
33
|
+
* `$MAILWOMAN_DATA_ROOT/census/tiger2023-edges`, where `mailwoman situs interpolation` puts them)
|
|
34
|
+
* - `$MAILWOMAN_DATA_ROOT/oa-cache/us__il__cook.zip` (ZIP→city tails)
|
|
35
35
|
*/
|
|
36
36
|
import { type ShardRecipe } from "./scaffold.ts";
|
|
37
37
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"intersection.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/intersection.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAYH,OAAO,
|
|
1
|
+
{"version":3,"file":"intersection.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/intersection.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAYH,OAAO,EAAwC,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AAwYtF;;;GAGG;AACH,eAAO,MAAM,kBAAkB,EAAE,WA0LhC,CAAA"}
|
|
@@ -30,22 +30,27 @@
|
|
|
30
30
|
* External inputs (`--edges-dir`, opts.edgesDir; both already on disk — do not re-download):
|
|
31
31
|
*
|
|
32
32
|
* - <edges-dir>/tl_2023_{17031,34027,50023}_edges.shp (unzipped TIGER 2023 EDGES; default
|
|
33
|
-
* /
|
|
34
|
-
* - /
|
|
33
|
+
* `$MAILWOMAN_DATA_ROOT/census/tiger2023-edges`, where `mailwoman situs interpolation` puts them)
|
|
34
|
+
* - `$MAILWOMAN_DATA_ROOT/oa-cache/us__il__cook.zip` (ZIP→city tails)
|
|
35
35
|
*/
|
|
36
|
-
import { spawnSync } from "node:child_process";
|
|
37
36
|
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
38
|
-
import { repoRootPath } from "@mailwoman/core/utils";
|
|
39
|
-
import { stableSourceID } from "
|
|
40
|
-
import { alignRow } from "
|
|
41
|
-
import {
|
|
37
|
+
import { dataRootPath, repoRootPath } from "@mailwoman/core/utils";
|
|
38
|
+
import { stableSourceID } from "@mailwoman/corpus/adapters/utils";
|
|
39
|
+
import { alignRow } from "@mailwoman/corpus/utils";
|
|
40
|
+
import { JSONSpliterator } from "spliterator";
|
|
41
|
+
import { makeMulberry32, readZippedCSVRecords } from "./scaffold.js";
|
|
42
42
|
const TRAIN_COUNTIES = [
|
|
43
43
|
{ fips: "17031", state: "IL", regime: "grid-city" },
|
|
44
44
|
{ fips: "34027", state: "NJ", regime: "suburb" },
|
|
45
45
|
];
|
|
46
46
|
const GOLDEN_COUNTIES = [{ fips: "50023", state: "VT", regime: "rural" }];
|
|
47
47
|
const EVAL_GOLD_PATH = repoRootPath("data", "eval", "external", "intersection-real.jsonl");
|
|
48
|
-
|
|
48
|
+
/**
|
|
49
|
+
* Where `mailwoman situs interpolation` unpacks the per-county TIGER EDGES shapefiles — a Census download, so it sits
|
|
50
|
+
* beside the other Census vintages under `census/`. `--edges-dir` overrides it.
|
|
51
|
+
*/
|
|
52
|
+
const DEFAULT_EDGES_DIR = dataRootPath("census", "tiger2023-edges");
|
|
53
|
+
const OA_COOK = { zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv" };
|
|
49
54
|
const FORMS = [
|
|
50
55
|
{ id: "amp", w: 0.2, render: (a, b) => `${a} & ${b}` },
|
|
51
56
|
{ id: "and", w: 0.2, render: (a, b) => `${a} and ${b}` },
|
|
@@ -93,43 +98,6 @@ function weightedPick(items, random) {
|
|
|
93
98
|
}
|
|
94
99
|
return items.at(-1);
|
|
95
100
|
}
|
|
96
|
-
/**
|
|
97
|
-
* Minimal RFC-4180-ish splitter (handles quoted fields) — same as the affix builder.
|
|
98
|
-
*/
|
|
99
|
-
function splitCSV(line) {
|
|
100
|
-
const out = [];
|
|
101
|
-
let cur = "";
|
|
102
|
-
let inQ = false;
|
|
103
|
-
for (let i = 0; i < line.length; i++) {
|
|
104
|
-
const c = line[i];
|
|
105
|
-
if (inQ) {
|
|
106
|
-
if (c === '"') {
|
|
107
|
-
if (line[i + 1] === '"') {
|
|
108
|
-
cur += '"';
|
|
109
|
-
i++;
|
|
110
|
-
}
|
|
111
|
-
else {
|
|
112
|
-
inQ = false;
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
|
-
else {
|
|
116
|
-
cur += c;
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
else if (c === '"') {
|
|
120
|
-
inQ = true;
|
|
121
|
-
}
|
|
122
|
-
else if (c === ",") {
|
|
123
|
-
out.push(cur);
|
|
124
|
-
cur = "";
|
|
125
|
-
}
|
|
126
|
-
else {
|
|
127
|
-
cur += c;
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
out.push(cur);
|
|
131
|
-
return out;
|
|
132
|
-
}
|
|
133
101
|
/**
|
|
134
102
|
* Order-insensitive crossing key, for eval-leakage exclusion + pair dedup.
|
|
135
103
|
*/
|
|
@@ -137,17 +105,14 @@ const pairKey = (a, b) => [a.toLowerCase(), b.toLowerCase()].toSorted().join("\u
|
|
|
137
105
|
/**
|
|
138
106
|
* Load the eval's crossings so neither train nor golden ever sees them.
|
|
139
107
|
*/
|
|
140
|
-
function readEvalExclusions() {
|
|
108
|
+
async function readEvalExclusions() {
|
|
141
109
|
const nodes = new Set();
|
|
142
110
|
const pairs = new Set();
|
|
143
111
|
if (!existsSync(EVAL_GOLD_PATH)) {
|
|
144
112
|
console.error(` WARN: eval gold not found at ${EVAL_GOLD_PATH} — no eval-leakage exclusion applied`);
|
|
145
113
|
return { nodes, pairs };
|
|
146
114
|
}
|
|
147
|
-
for (const
|
|
148
|
-
if (!line)
|
|
149
|
-
continue;
|
|
150
|
-
const row = JSON.parse(line);
|
|
115
|
+
for await (const row of JSONSpliterator.fromAsync(EVAL_GOLD_PATH)) {
|
|
151
116
|
nodes.add(Number(row.node));
|
|
152
117
|
pairs.add(pairKey(row.components.intersection_a, row.components.intersection_b));
|
|
153
118
|
}
|
|
@@ -197,26 +162,12 @@ async function extractCrossings(db, edgesDir, county, seed) {
|
|
|
197
162
|
/**
|
|
198
163
|
* ZIP → majority city from the cached OA Cook-county CSV (real ZIP/city pairings).
|
|
199
164
|
*/
|
|
200
|
-
function buildZipCityMap() {
|
|
201
|
-
const r = spawnSync("unzip", ["-p", OA_COOK.zip, OA_COOK.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" });
|
|
202
|
-
if (r.status !== 0) {
|
|
203
|
-
console.error(` WARN: unzip failed for ${OA_COOK.zip} — city tails disabled`);
|
|
204
|
-
return new Map();
|
|
205
|
-
}
|
|
206
|
-
const lines = r.stdout.toString("utf8").split(/\r?\n/);
|
|
207
|
-
if (lines.length < 2)
|
|
208
|
-
return new Map();
|
|
209
|
-
const header = splitCSV(lines[0]).map((h) => h.trim().toLowerCase());
|
|
210
|
-
const iCity = header.indexOf("city");
|
|
211
|
-
const iPost = header.indexOf("postcode");
|
|
165
|
+
async function buildZipCityMap() {
|
|
212
166
|
const counts = new Map();
|
|
213
167
|
// zip → Map(city → n)
|
|
214
|
-
for (
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
const cells = splitCSV(lines[li]);
|
|
218
|
-
const city = (cells[iCity] ?? "").trim();
|
|
219
|
-
const zip = (cells[iPost] ?? "").trim();
|
|
168
|
+
for await (const row of readZippedCSVRecords(OA_COOK.zip, OA_COOK.csv)) {
|
|
169
|
+
const city = row.city ?? "";
|
|
170
|
+
const zip = row.postcode ?? "";
|
|
220
171
|
if (!city || !/^\d{5}$/.test(zip) || BAD_NAME.test(city))
|
|
221
172
|
continue;
|
|
222
173
|
let byCity = counts.get(zip);
|
|
@@ -364,15 +315,20 @@ export const intersectionRecipe = {
|
|
|
364
315
|
name: "intersection",
|
|
365
316
|
description: "Real-pair intersection rows (US): TIGER 2023 EDGES crossings → audited intersection_a/b labels",
|
|
366
317
|
mode: "generate",
|
|
367
|
-
options: [
|
|
318
|
+
options: [
|
|
319
|
+
{
|
|
320
|
+
flag: "--edges-dir <dir>",
|
|
321
|
+
description: `Unzipped TIGER 2023 EDGES dir. Default ${DEFAULT_EDGES_DIR}`,
|
|
322
|
+
},
|
|
323
|
+
],
|
|
368
324
|
async run(opts, write) {
|
|
369
325
|
// Legacy build-intersection-shard.mjs seeded `mulberry32(opts.seed)`.
|
|
370
326
|
const random = makeMulberry32(opts.seed);
|
|
371
327
|
const count = opts.count ?? 40_000;
|
|
372
328
|
const source = opts.sourceName ?? "synth-intersection";
|
|
373
|
-
const edgesDir = opts.edgesDir ??
|
|
329
|
+
const edgesDir = opts.edgesDir ?? DEFAULT_EDGES_DIR;
|
|
374
330
|
const counties = opts.golden ? GOLDEN_COUNTIES : TRAIN_COUNTIES;
|
|
375
|
-
const exclusions = readEvalExclusions();
|
|
331
|
+
const exclusions = await readEvalExclusions();
|
|
376
332
|
console.error(` eval exclusions: ${exclusions.nodes.size} nodes, ${exclusions.pairs.size} pairs`);
|
|
377
333
|
const { DuckDBInstance } = await import("@duckdb/node-api");
|
|
378
334
|
const instance = await DuckDBInstance.create();
|
|
@@ -408,7 +364,7 @@ export const intersectionRecipe = {
|
|
|
408
364
|
if (!pool.length) {
|
|
409
365
|
throw new Error(`No crossings found — are the TIGER EDGES shapefiles present in ${edgesDir}?`);
|
|
410
366
|
}
|
|
411
|
-
const zipCity = opts.golden ? new Map() : buildZipCityMap();
|
|
367
|
+
const zipCity = opts.golden ? new Map() : await buildZipCityMap();
|
|
412
368
|
if (!opts.golden) {
|
|
413
369
|
console.error(` zip→city map: ${zipCity.size} ZIPs (OA Cook)`);
|
|
414
370
|
}
|