@mailwoman/corpus 9.0.0 → 9.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/data/PROVENANCE.md +227 -0
- package/data/reviewed-ve-postcode-tuples.json +74 -0
- package/data/sub-venue-lexicon.json +18269 -0
- package/out/src/adapters/ban/adapter.d.ts +1 -1
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +5 -4
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +3 -3
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +1 -1
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +2 -2
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +3 -3
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +2 -2
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +1 -1
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +3 -3
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +1 -1
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +2 -2
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +7 -6
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +12 -7
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +12 -18
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +12 -7
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +3 -3
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +3 -3
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +1 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +2 -2
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +7 -6
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +11 -7
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +6 -5
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +8 -8
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +10 -10
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +7 -6
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/{adapter.d.ts → adapters/utils/index.d.ts} +8 -2
- package/out/src/adapters/utils/index.d.ts.map +1 -0
- package/out/src/{adapter.js → adapters/utils/index.js} +11 -3
- package/out/src/adapters/utils/index.js.map +1 -0
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +5 -5
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +4 -15
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +11 -35
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-json-rows.d.ts +32 -0
- package/out/src/adapters/wof-json-rows.d.ts.map +1 -0
- package/out/src/adapters/wof-json-rows.js +45 -0
- package/out/src/adapters/wof-json-rows.js.map +1 -0
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +4 -9
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +12 -37
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/build.d.ts +4 -4
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +7 -7
- package/out/src/build.js.map +1 -1
- package/out/src/index.d.ts +3 -17
- package/out/src/index.d.ts.map +1 -1
- package/out/src/index.js +3 -17
- package/out/src/index.js.map +1 -1
- package/out/src/name-prone-us-suffixes.d.ts +12 -0
- package/out/src/name-prone-us-suffixes.d.ts.map +1 -0
- package/out/src/name-prone-us-suffixes.js +12 -0
- package/out/src/name-prone-us-suffixes.js.map +1 -0
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +5 -0
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/runner.d.ts +2 -2
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +1 -1
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +4 -5
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/bare-country.d.ts +25 -0
- package/out/src/shard-recipes/bare-country.d.ts.map +1 -0
- package/out/src/shard-recipes/bare-country.js +87 -0
- package/out/src/shard-recipes/bare-country.js.map +1 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +2 -2
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +14 -21
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +28 -10
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +4 -3
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.js +72 -11
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +4 -3
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +11 -20
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +8 -14
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/house-venue.js +1 -1
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +8 -1
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +7 -13
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +5 -5
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +13 -20
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/no-street.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street.js +1 -1
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +60 -43
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +1 -1
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/reviewed-postcode-tail.d.ts +47 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.d.ts.map +1 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.js +125 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.js.map +1 -0
- package/out/src/shard-recipes/scaffold.d.ts +51 -23
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +59 -23
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +54 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +203 -35
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
- package/out/src/shard-recipes/street-bare.js +3 -3
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.d.ts.map +1 -1
- package/out/src/shard-recipes/street.js +2 -2
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.d.ts +3 -6
- package/out/src/shard-recipes/sub-venue-sources.d.ts.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.js +4 -9
- package/out/src/shard-recipes/sub-venue-sources.js.map +1 -1
- package/out/src/shard-recipes/sub-venue.d.ts +2 -2
- package/out/src/shard-recipes/sub-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/sub-venue.js +7 -6
- package/out/src/shard-recipes/sub-venue.js.map +1 -1
- package/out/src/shard-recipes/trailing-region.d.ts +74 -0
- package/out/src/shard-recipes/trailing-region.d.ts.map +1 -0
- package/out/src/shard-recipes/trailing-region.js +157 -0
- package/out/src/shard-recipes/trailing-region.js.map +1 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +6 -12
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/{synthesize-anchor-absorption.d.ts → synthesizers/anchor-absorption.d.ts} +1 -1
- package/out/src/synthesizers/anchor-absorption.d.ts.map +1 -0
- package/out/src/{synthesize-anchor-absorption.js → synthesizers/anchor-absorption.js} +1 -1
- package/out/src/synthesizers/anchor-absorption.js.map +1 -0
- package/out/src/{synthesize-boundary-stress.d.ts → synthesizers/boundary-stress.d.ts} +3 -3
- package/out/src/synthesizers/boundary-stress.d.ts.map +1 -0
- package/out/src/{synthesize-boundary-stress.js → synthesizers/boundary-stress.js} +2 -2
- package/out/src/synthesizers/boundary-stress.js.map +1 -0
- package/out/src/{synthesize-german.d.ts → synthesizers/german.d.ts} +2 -2
- package/out/src/synthesizers/german.d.ts.map +1 -0
- package/out/src/{synthesize-german.js → synthesizers/german.js} +2 -2
- package/out/src/synthesizers/german.js.map +1 -0
- package/out/src/{synthesize-house-venue.d.ts → synthesizers/house-venue.d.ts} +4 -4
- package/out/src/synthesizers/house-venue.d.ts.map +1 -0
- package/out/src/{synthesize-house-venue.js → synthesizers/house-venue.js} +20 -6
- package/out/src/synthesizers/house-venue.js.map +1 -0
- package/out/src/{synthesize-intersection.d.ts → synthesizers/intersection.d.ts} +2 -2
- package/out/src/synthesizers/intersection.d.ts.map +1 -0
- package/out/src/{synthesize-intersection.js → synthesizers/intersection.js} +1 -1
- package/out/src/synthesizers/intersection.js.map +1 -0
- package/out/src/{synthesize-no-street.d.ts → synthesizers/no-street.d.ts} +3 -3
- package/out/src/synthesizers/no-street.d.ts.map +1 -0
- package/out/src/{synthesize-no-street.js → synthesizers/no-street.js} +2 -2
- package/out/src/synthesizers/no-street.js.map +1 -0
- package/out/src/{synthesize-po-box.d.ts → synthesizers/po-box.d.ts} +2 -2
- package/out/src/synthesizers/po-box.d.ts.map +1 -0
- package/out/src/{synthesize-po-box.js → synthesizers/po-box.js} +1 -1
- package/out/src/synthesizers/po-box.js.map +1 -0
- package/out/src/{synthesize-street.d.ts → synthesizers/street.d.ts} +2 -2
- package/out/src/synthesizers/street.d.ts.map +1 -0
- package/out/src/{synthesize-street.js → synthesizers/street.js} +4 -3
- package/out/src/synthesizers/street.js.map +1 -0
- package/out/src/{synthesize.d.ts → synthesizers/utils.d.ts} +3 -3
- package/out/src/synthesizers/utils.d.ts.map +1 -0
- package/out/src/{synthesize.js → synthesizers/utils.js} +5 -4
- package/out/src/synthesizers/utils.js.map +1 -0
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +2 -2
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/fetch/ban.d.ts.map +1 -1
- package/out/src/tools/fetch/ban.js +2 -18
- package/out/src/tools/fetch/ban.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts +12 -3
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +20 -4
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/geonames-dump.d.ts +87 -0
- package/out/src/tools/fetch/geonames-dump.d.ts.map +1 -0
- package/out/src/tools/fetch/geonames-dump.js +178 -0
- package/out/src/tools/fetch/geonames-dump.js.map +1 -0
- package/out/src/tools/fetch/geonames-postal.d.ts +76 -0
- package/out/src/tools/fetch/geonames-postal.d.ts.map +1 -0
- package/out/src/tools/fetch/geonames-postal.js +125 -0
- package/out/src/tools/fetch/geonames-postal.js.map +1 -0
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +5 -17
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +11 -0
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +11 -0
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts.map +1 -1
- package/out/src/tools/fetch/nad.js +20 -11
- package/out/src/tools/fetch/nad.js.map +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +17 -22
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/src/tools/fetch/openaddresses.js +18 -17
- package/out/src/tools/fetch/openaddresses.js.map +1 -1
- package/out/src/tools/fetch/ourairports.d.ts.map +1 -1
- package/out/src/tools/fetch/ourairports.js +7 -2
- package/out/src/tools/fetch/ourairports.js.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +0 -4
- package/out/src/tools/fetch/ppd.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.js +1 -1
- package/out/src/tools/fetch/ppd.js.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.js +3 -19
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
- package/out/src/tools/fetch/state-sources.d.ts +4 -0
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
- package/out/src/tools/fetch/state-sources.js +1 -5
- package/out/src/tools/fetch/state-sources.js.map +1 -1
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
- package/out/src/tools/fetch/tiger-full.js +15 -21
- package/out/src/tools/fetch/tiger-full.js.map +1 -1
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +34 -25
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-relabel-street.d.ts.map +1 -1
- package/out/src/tools/golden-relabel-street.js +4 -77
- package/out/src/tools/golden-relabel-street.js.map +1 -1
- package/out/src/tools/index.d.ts +2 -2
- package/out/src/tools/index.d.ts.map +1 -1
- package/out/src/tools/index.js +2 -2
- package/out/src/tools/index.js.map +1 -1
- package/out/src/tools/ingest-csv.d.ts.map +1 -1
- package/out/src/tools/ingest-csv.js +0 -11
- package/out/src/tools/ingest-csv.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts +4 -0
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +16 -6
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/postcode-triples.d.ts +175 -0
- package/out/src/tools/postcode-triples.d.ts.map +1 -0
- package/out/src/tools/postcode-triples.js +304 -0
- package/out/src/tools/postcode-triples.js.map +1 -0
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +1 -3
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +4 -27
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/tools/sub-venue/harvest.d.ts +100 -0
- package/out/src/tools/sub-venue/harvest.d.ts.map +1 -0
- package/out/src/tools/sub-venue/harvest.js +168 -0
- package/out/src/tools/sub-venue/harvest.js.map +1 -0
- package/out/src/tools/sub-venue/head-nouns.d.ts +50 -0
- package/out/src/tools/sub-venue/head-nouns.d.ts.map +1 -0
- package/out/src/tools/sub-venue/head-nouns.js +210 -0
- package/out/src/tools/sub-venue/head-nouns.js.map +1 -0
- package/out/src/tools/sub-venue/surfaces.d.ts +45 -0
- package/out/src/tools/sub-venue/surfaces.d.ts.map +1 -0
- package/out/src/tools/sub-venue/surfaces.js +77 -0
- package/out/src/tools/sub-venue/surfaces.js.map +1 -0
- package/out/src/tools/sub-venue/table.d.ts +229 -0
- package/out/src/tools/sub-venue/table.d.ts.map +1 -0
- package/out/src/tools/sub-venue/table.js +108 -0
- package/out/src/tools/sub-venue/table.js.map +1 -0
- package/out/src/tools/sub-venue/wikidata.d.ts +26 -0
- package/out/src/tools/sub-venue/wikidata.d.ts.map +1 -0
- package/out/src/tools/sub-venue/wikidata.js +62 -0
- package/out/src/tools/sub-venue/wikidata.js.map +1 -0
- package/out/src/tools/sub-venue-lexicon.d.ts +27 -378
- package/out/src/tools/sub-venue-lexicon.d.ts.map +1 -1
- package/out/src/tools/sub-venue-lexicon.js +30 -565
- package/out/src/tools/sub-venue-lexicon.js.map +1 -1
- package/out/src/{align.d.ts → utils/align.d.ts} +1 -1
- package/out/src/utils/align.d.ts.map +1 -0
- package/out/src/utils/align.js.map +1 -0
- package/out/src/utils/golden.d.ts.map +1 -0
- package/out/src/{golden.js → utils/golden.js} +2 -2
- package/out/src/utils/golden.js.map +1 -0
- package/out/src/utils/index.d.ts +14 -0
- package/out/src/utils/index.d.ts.map +1 -0
- package/out/src/utils/index.js +14 -0
- package/out/src/utils/index.js.map +1 -0
- package/out/src/utils/license.d.ts.map +1 -0
- package/out/src/{license.js → utils/license.js} +1 -1
- package/out/src/utils/license.js.map +1 -0
- package/out/src/{parquet.d.ts → utils/parquet.d.ts} +24 -5
- package/out/src/utils/parquet.d.ts.map +1 -0
- package/out/src/{parquet.js → utils/parquet.js} +6 -3
- package/out/src/utils/parquet.js.map +1 -0
- package/out/src/{split.d.ts → utils/split.d.ts} +1 -1
- package/out/src/utils/split.d.ts.map +1 -0
- package/out/src/{split.js → utils/split.js} +2 -2
- package/out/src/utils/split.js.map +1 -0
- package/out/src/utils/tokenize.d.ts.map +1 -0
- package/out/src/utils/tokenize.js.map +1 -0
- package/out/src/utils/wof-json.d.ts.map +1 -0
- package/out/src/utils/wof-json.js.map +1 -0
- package/package.json +287 -21
- package/src/adapters/ban/adapter.ts +7 -5
- package/src/adapters/fcc-bdc/adapter.ts +5 -4
- package/src/adapters/geonames/adapter.ts +3 -3
- package/src/adapters/geonames-postal/adapter.ts +4 -4
- package/src/adapters/gnaf/adapter.ts +3 -3
- package/src/adapters/gnaf/assemble.ts +1 -1
- package/src/adapters/index.ts +3 -2
- package/src/adapters/openaddresses/adapter.ts +4 -4
- package/src/adapters/overture/adapter.ts +3 -3
- package/src/adapters/state-hi-schools/adapter.ts +8 -7
- package/src/adapters/state-ia-contractors/adapter.ts +13 -8
- package/src/adapters/state-ny-notaries/adapter.ts +13 -32
- package/src/adapters/state-tx-notaries/adapter.ts +13 -8
- package/src/adapters/synth-po-box/adapter.ts +4 -4
- package/src/adapters/tiger/adapter.ts +5 -3
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +8 -7
- package/src/adapters/usgov-imls-pls/adapter.ts +12 -8
- package/src/adapters/usgov-irs-bmf/adapter.ts +7 -6
- package/src/adapters/usgov-nad/adapter.ts +9 -9
- package/src/adapters/usgov-nppes/adapter.ts +13 -12
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +8 -7
- package/src/{adapter.ts → adapters/utils/index.ts} +15 -3
- package/src/adapters/wof-admin-jp/adapter.ts +9 -12
- package/src/adapters/wof-admin-json/adapter.ts +15 -54
- package/src/adapters/wof-json-rows.ts +79 -0
- package/src/adapters/wof-postalcode-json/adapter.ts +16 -50
- package/src/build.ts +8 -8
- package/src/index.ts +3 -17
- package/src/name-prone-us-suffixes.ts +12 -0
- package/src/parquet-wrapper/reader.ts +9 -0
- package/src/runner.ts +2 -2
- package/src/shard-recipes/anchor-absorption.ts +5 -5
- package/src/shard-recipes/bare-country.ts +95 -0
- package/src/shard-recipes/boundary-stress.ts +3 -2
- package/src/shard-recipes/country-balanced.ts +16 -27
- package/src/shard-recipes/cz-pcfirst-preposition.ts +29 -10
- package/src/shard-recipes/fr-admin-split.ts +6 -4
- package/src/shard-recipes/fr-bare-street.ts +85 -10
- package/src/shard-recipes/fr-lieudit.ts +6 -4
- package/src/shard-recipes/fr-order.ts +13 -26
- package/src/shard-recipes/german.ts +9 -18
- package/src/shard-recipes/house-venue.ts +2 -1
- package/src/shard-recipes/index.ts +8 -1
- package/src/shard-recipes/intersection.ts +9 -17
- package/src/shard-recipes/locale.ts +16 -24
- package/src/shard-recipes/no-street.ts +2 -1
- package/src/shard-recipes/po-box-cedex.ts +76 -58
- package/src/shard-recipes/po-box.ts +2 -1
- package/src/shard-recipes/reviewed-postcode-tail.ts +189 -0
- package/src/shard-recipes/scaffold.ts +99 -24
- package/src/shard-recipes/street-affix.ts +284 -45
- package/src/shard-recipes/street-bare.ts +4 -3
- package/src/shard-recipes/street.ts +3 -2
- package/src/shard-recipes/sub-venue-sources.ts +13 -23
- package/src/shard-recipes/sub-venue.ts +11 -9
- package/src/shard-recipes/trailing-region.ts +183 -0
- package/src/shard-recipes/unit.ts +8 -17
- package/src/{synthesize-boundary-stress.ts → synthesizers/boundary-stress.ts} +2 -2
- package/src/{synthesize-german.ts → synthesizers/german.ts} +3 -2
- package/src/{synthesize-house-venue.ts → synthesizers/house-venue.ts} +20 -6
- package/src/{synthesize-intersection.ts → synthesizers/intersection.ts} +1 -1
- package/src/{synthesize-no-street.ts → synthesizers/no-street.ts} +2 -2
- package/src/{synthesize-po-box.ts → synthesizers/po-box.ts} +1 -1
- package/src/{synthesize-street.ts → synthesizers/street.ts} +5 -3
- package/src/{synthesize.ts → synthesizers/utils.ts} +6 -4
- package/src/tools/align-shard.ts +3 -2
- package/src/tools/fetch/ban.ts +2 -22
- package/src/tools/fetch/download.ts +22 -4
- package/src/tools/fetch/geonames-dump.ts +267 -0
- package/src/tools/fetch/geonames-postal.ts +180 -0
- package/src/tools/fetch/imls-pls.ts +6 -20
- package/src/tools/fetch/index.ts +11 -0
- package/src/tools/fetch/nad.ts +21 -10
- package/src/tools/fetch/nppes.ts +17 -23
- package/src/tools/fetch/openaddresses.ts +18 -22
- package/src/tools/fetch/ourairports.ts +7 -2
- package/src/tools/fetch/ppd.ts +1 -1
- package/src/tools/fetch/state-hi-schools.ts +3 -23
- package/src/tools/fetch/state-sources.ts +1 -1
- package/src/tools/fetch/tiger-full.ts +17 -24
- package/src/tools/golden-expand.ts +44 -33
- package/src/tools/golden-relabel-street.ts +5 -68
- package/src/tools/index.ts +2 -2
- package/src/tools/ingest-csv.ts +0 -13
- package/src/tools/overlay-manifest.ts +20 -8
- package/src/tools/postcode-triples.ts +375 -0
- package/src/tools/shard-kryptonite.ts +4 -5
- package/src/tools/shard-translit.ts +16 -35
- package/src/tools/sub-venue/harvest.ts +235 -0
- package/src/tools/sub-venue/head-nouns.ts +242 -0
- package/src/tools/sub-venue/surfaces.ts +95 -0
- package/src/tools/sub-venue/table.ts +281 -0
- package/src/tools/sub-venue/wikidata.ts +85 -0
- package/src/tools/sub-venue-lexicon.ts +67 -886
- package/src/{align.ts → utils/align.ts} +2 -1
- package/src/{golden.ts → utils/golden.ts} +2 -3
- package/src/utils/index.ts +14 -0
- package/src/{license.ts → utils/license.ts} +1 -1
- package/src/{parquet.ts → utils/parquet.ts} +22 -9
- package/src/{split.ts → utils/split.ts} +3 -3
- package/out/src/adapter.d.ts.map +0 -1
- package/out/src/adapter.js.map +0 -1
- package/out/src/align.d.ts.map +0 -1
- package/out/src/align.js.map +0 -1
- package/out/src/format.d.ts +0 -14
- package/out/src/format.d.ts.map +0 -1
- package/out/src/format.js +0 -14
- package/out/src/format.js.map +0 -1
- package/out/src/golden.d.ts.map +0 -1
- package/out/src/golden.js.map +0 -1
- package/out/src/license.d.ts.map +0 -1
- package/out/src/license.js.map +0 -1
- package/out/src/parquet.d.ts.map +0 -1
- package/out/src/parquet.js.map +0 -1
- package/out/src/split.d.ts.map +0 -1
- package/out/src/split.js.map +0 -1
- package/out/src/synthesize-anchor-absorption.d.ts.map +0 -1
- package/out/src/synthesize-anchor-absorption.js.map +0 -1
- package/out/src/synthesize-boundary-stress.d.ts.map +0 -1
- package/out/src/synthesize-boundary-stress.js.map +0 -1
- package/out/src/synthesize-german.d.ts.map +0 -1
- package/out/src/synthesize-german.js.map +0 -1
- package/out/src/synthesize-house-venue.d.ts.map +0 -1
- package/out/src/synthesize-house-venue.js.map +0 -1
- package/out/src/synthesize-intersection.d.ts.map +0 -1
- package/out/src/synthesize-intersection.js.map +0 -1
- package/out/src/synthesize-no-street.d.ts.map +0 -1
- package/out/src/synthesize-no-street.js.map +0 -1
- package/out/src/synthesize-po-box.d.ts.map +0 -1
- package/out/src/synthesize-po-box.js.map +0 -1
- package/out/src/synthesize-street.d.ts.map +0 -1
- package/out/src/synthesize-street.js.map +0 -1
- package/out/src/synthesize.d.ts.map +0 -1
- package/out/src/synthesize.js.map +0 -1
- package/out/src/tokenize.d.ts.map +0 -1
- package/out/src/tokenize.js.map +0 -1
- package/out/src/wof-json.d.ts.map +0 -1
- package/out/src/wof-json.js.map +0 -1
- package/src/format.ts +0 -14
- /package/out/src/{align.js → utils/align.js} +0 -0
- /package/out/src/{golden.d.ts → utils/golden.d.ts} +0 -0
- /package/out/src/{license.d.ts → utils/license.d.ts} +0 -0
- /package/out/src/{tokenize.d.ts → utils/tokenize.d.ts} +0 -0
- /package/out/src/{tokenize.js → utils/tokenize.js} +0 -0
- /package/out/src/{wof-json.d.ts → utils/wof-json.d.ts} +0 -0
- /package/out/src/{wof-json.js → utils/wof-json.js} +0 -0
- /package/src/{synthesize-anchor-absorption.ts → synthesizers/anchor-absorption.ts} +0 -0
- /package/src/{tokenize.ts → utils/tokenize.ts} +0 -0
- /package/src/{wof-json.ts → utils/wof-json.ts} +0 -0
|
@@ -27,8 +27,6 @@
|
|
|
27
27
|
* FR/DE postcode (the v0.9.8 blemish).
|
|
28
28
|
*/
|
|
29
29
|
|
|
30
|
-
import { spawnSync } from "node:child_process"
|
|
31
|
-
|
|
32
30
|
import {
|
|
33
31
|
DirectionalAbbreviation,
|
|
34
32
|
lookupDirectional,
|
|
@@ -41,10 +39,12 @@ import {
|
|
|
41
39
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
42
40
|
import { dataRootPath } from "@mailwoman/core/utils"
|
|
43
41
|
|
|
44
|
-
import { stableSourceID } from "
|
|
45
|
-
import {
|
|
46
|
-
import
|
|
47
|
-
|
|
42
|
+
import { stableSourceID } from "#adapters/utils"
|
|
43
|
+
import type { CanonicalRow } from "#types"
|
|
44
|
+
import { alignRow } from "#utils"
|
|
45
|
+
|
|
46
|
+
import { NAME_PRONE_US_SUFFIXES } from "../name-prone-us-suffixes.ts"
|
|
47
|
+
import { makeMulberry32, readCSVRecords, readZippedCSVRecords, shardSourceID, type ShardRecipe } from "./scaffold.ts"
|
|
48
48
|
|
|
49
49
|
// Same OA cache as the unit shard. Train = every NON-Vermont state; eval = Vermont (the holdout).
|
|
50
50
|
/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
|
|
@@ -140,6 +140,7 @@ interface USTuple {
|
|
|
140
140
|
locality: string
|
|
141
141
|
region: string
|
|
142
142
|
postcode: string
|
|
143
|
+
base_source_id: string
|
|
143
144
|
}
|
|
144
145
|
|
|
145
146
|
/**
|
|
@@ -163,19 +164,11 @@ type Prefix = Pick<NonNullable<ReturnType<typeof matchLeadingDirectional>>, "can
|
|
|
163
164
|
/**
|
|
164
165
|
* Stream real US tuples (number/street/city/postcode) out of a cached OA zip.
|
|
165
166
|
*/
|
|
166
|
-
function readTuples(source: USSource): USTuple[] {
|
|
167
|
-
const r = spawnSync("unzip", ["-p", source.zip, source.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
|
|
168
|
-
|
|
169
|
-
if (r.status !== 0) {
|
|
170
|
-
console.error(` WARN: unzip failed for ${source.zip} (status ${r.status})`)
|
|
171
|
-
|
|
172
|
-
return []
|
|
173
|
-
}
|
|
174
|
-
|
|
167
|
+
async function readTuples(source: USSource): Promise<USTuple[]> {
|
|
175
168
|
const tuples: USTuple[] = []
|
|
176
169
|
const seen = new Set<string>()
|
|
177
170
|
|
|
178
|
-
for (const row of
|
|
171
|
+
for await (const row of readZippedCSVRecords(source.zip, source.csv)) {
|
|
179
172
|
const street = row.street ?? ""
|
|
180
173
|
const locality = row.city ?? ""
|
|
181
174
|
const house_number = row.number ?? ""
|
|
@@ -185,7 +178,15 @@ function readTuples(source: USSource): USTuple[] {
|
|
|
185
178
|
|
|
186
179
|
if (seen.has(key)) continue
|
|
187
180
|
seen.add(key)
|
|
188
|
-
|
|
181
|
+
|
|
182
|
+
tuples.push({
|
|
183
|
+
house_number,
|
|
184
|
+
street,
|
|
185
|
+
locality,
|
|
186
|
+
region: source.region,
|
|
187
|
+
postcode: row.postcode ?? "",
|
|
188
|
+
base_source_id: `openaddresses:${source.csv}:${key}`,
|
|
189
|
+
})
|
|
189
190
|
}
|
|
190
191
|
|
|
191
192
|
return tuples
|
|
@@ -205,7 +206,10 @@ const isSuffixOrDirectional = (word: string): boolean =>
|
|
|
205
206
|
* Split an OA street into { prefix?, name, suffix } using the codex. Requires a trailing suffix and a non-empty name
|
|
206
207
|
* that isn't itself an affix token. Returns null when the street has no usable suffix.
|
|
207
208
|
*/
|
|
208
|
-
function parseStreet(
|
|
209
|
+
function parseStreet(
|
|
210
|
+
street: string,
|
|
211
|
+
opts: { allowNameProneTail?: boolean } = {}
|
|
212
|
+
): { prefix: Prefix | null; name: string; suffix: string } | null {
|
|
209
213
|
let words = street.trim().split(/\s+/)
|
|
210
214
|
|
|
211
215
|
if (words.length < 2) return null
|
|
@@ -225,11 +229,42 @@ function parseStreet(street: string): { prefix: Prefix | null; name: string; suf
|
|
|
225
229
|
const suffix = trail.canonical
|
|
226
230
|
const name = words.slice(0, -1).join(" ")
|
|
227
231
|
|
|
228
|
-
if (!name
|
|
232
|
+
if (!name) return null
|
|
233
|
+
|
|
234
|
+
if (isSuffixOrDirectional(name)) {
|
|
235
|
+
const nameTail = matchTrailingSuffix(name)
|
|
236
|
+
|
|
237
|
+
if (!opts.allowNameProneTail || !nameTail || !NAME_PRONE_US_SUFFIXES.has(nameTail.canonical)) return null
|
|
238
|
+
}
|
|
229
239
|
|
|
230
240
|
return { prefix, name, suffix }
|
|
231
241
|
}
|
|
232
242
|
|
|
243
|
+
export type SuffixBoundaryClass = "terminal-only" | "terminal-contrast"
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Classify a real street surface for #1569. `terminal-only` has an ambiguous suffix-eligible name word immediately
|
|
247
|
+
* before a different terminal type (`Blue Hill Rd`); `terminal-contrast` ends at the ambiguous word itself (`Sutton
|
|
248
|
+
* Hollow`). The contrast check intentionally wins when both final words are name-prone: only the LAST token is the
|
|
249
|
+
* suffix under the canonical rule.
|
|
250
|
+
*/
|
|
251
|
+
export function classifySuffixBoundaryStreet(street: string): SuffixBoundaryClass | null {
|
|
252
|
+
const words = street.trim().split(/\s+/)
|
|
253
|
+
|
|
254
|
+
if (words.length < 2) return null
|
|
255
|
+
const terminal = matchTrailingSuffix(words.at(-1)!)
|
|
256
|
+
|
|
257
|
+
if (!terminal) return null
|
|
258
|
+
|
|
259
|
+
if (NAME_PRONE_US_SUFFIXES.has(terminal.canonical)) return "terminal-contrast"
|
|
260
|
+
|
|
261
|
+
if (words.length < 3) return null
|
|
262
|
+
|
|
263
|
+
const penultimate = matchTrailingSuffix(words.at(-2)!)
|
|
264
|
+
|
|
265
|
+
return penultimate && NAME_PRONE_US_SUFFIXES.has(penultimate.canonical) ? "terminal-only" : null
|
|
266
|
+
}
|
|
267
|
+
|
|
233
268
|
/**
|
|
234
269
|
* Render the affix-split street in random surface forms (abbrev vs expanded per affix), Title-cased.
|
|
235
270
|
*/
|
|
@@ -279,18 +314,65 @@ function renderStreet(
|
|
|
279
314
|
*/
|
|
280
315
|
const VENUES = ["John Doe", "Jane Smith", "Acme Inc", "Wayne Enterprises", "Maria Garcia", "Riverside Clinic"]
|
|
281
316
|
|
|
317
|
+
/**
|
|
318
|
+
* Real-venue pool for the suffix-boundary v2 venue shell (corpus 0.19.0). The 2026-08-10 recipe review found the
|
|
319
|
+
* v0.18.x shard's venue-led rows detectable as templates (six fixed venue strings), and the frozen B1 board's failures
|
|
320
|
+
* concentrate in exactly that shell (rich rows 5/43 vs bare 53/65 at v4.3.3 step-40k). HRSA's health-center site file
|
|
321
|
+
* supplies thousands of REAL US facility names ('Alburg Health Center'-register), US-government public domain, already
|
|
322
|
+
* a corpus source (`usgov-hrsa-fqhc`). Kept verbatim (including the ~11% all-caps names — real register diversity);
|
|
323
|
+
* comma-carrying names are dropped because the venue layout uses commas as its field delimiter.
|
|
324
|
+
*/
|
|
325
|
+
const VENUE_POOL_CSV = dataRootPath(
|
|
326
|
+
"corpus",
|
|
327
|
+
"sources",
|
|
328
|
+
"usgov-hrsa-fqhc",
|
|
329
|
+
"Health_Center_Service_Delivery_and_LookAlike_Sites.csv"
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
async function readVenuePool(csvPath: string): Promise<string[]> {
|
|
333
|
+
const seen = new Set<string>()
|
|
334
|
+
const pool: string[] = []
|
|
335
|
+
|
|
336
|
+
for await (const record of readCSVRecords(csvPath)) {
|
|
337
|
+
const name = (record["site name"] ?? record["Site Name"] ?? "").trim().replaceAll(/\s+/gu, " ")
|
|
338
|
+
|
|
339
|
+
if (name.length < 3 || name.length > 60 || name.includes(",") || !/\p{L}/u.test(name)) continue
|
|
340
|
+
const key = name.toLowerCase()
|
|
341
|
+
|
|
342
|
+
if (seen.has(key)) continue
|
|
343
|
+
seen.add(key)
|
|
344
|
+
pool.push(name)
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
return pool
|
|
348
|
+
}
|
|
349
|
+
|
|
282
350
|
const tail = (loc: string, reg: string, pc: string): string => (pc ? `${loc}, ${reg} ${pc}` : `${loc}, ${reg}`)
|
|
283
351
|
|
|
352
|
+
/**
|
|
353
|
+
* Layout-shell options for {@link renderRow}. `cuts` are the cumulative random() cutoffs for [full, bare, street-only] —
|
|
354
|
+
* the remainder is the venue shell. Defaults reproduce the original street-affix distribution (40/25/20/15) with the
|
|
355
|
+
* six template venues.
|
|
356
|
+
*/
|
|
357
|
+
interface RenderRowOpts {
|
|
358
|
+
venues?: readonly string[]
|
|
359
|
+
cuts?: readonly [number, number, number]
|
|
360
|
+
}
|
|
361
|
+
|
|
284
362
|
/**
|
|
285
363
|
* Embed the rendered street in a RANDOM layout so the model recognizes affixes wherever the street sits: full address,
|
|
286
364
|
* bare house+street, street-only (pure affix parse), or venue-prefixed.
|
|
287
365
|
*/
|
|
288
|
-
function renderRow(
|
|
366
|
+
export function renderRow(
|
|
289
367
|
random: () => number,
|
|
290
368
|
base: USTuple,
|
|
291
369
|
street: string,
|
|
292
|
-
streetComponents: Partial<Record<ComponentTag, string
|
|
370
|
+
streetComponents: Partial<Record<ComponentTag, string>>,
|
|
371
|
+
opts: RenderRowOpts = {}
|
|
293
372
|
): { fmt: string; raw: string; components: Partial<Record<ComponentTag, string>> } {
|
|
373
|
+
const venues = opts.venues ?? VENUES
|
|
374
|
+
const [fullCut, bareCut, streetOnlyCut] = opts.cuts ?? [0.4, 0.65, 0.85]
|
|
375
|
+
|
|
294
376
|
const hn = base.house_number,
|
|
295
377
|
loc = base.locality,
|
|
296
378
|
reg = base.region,
|
|
@@ -300,17 +382,17 @@ function renderRow(
|
|
|
300
382
|
const withRoad: Partial<Record<ComponentTag, string>> = { house_number: hn, ...streetComponents }
|
|
301
383
|
const r = random()
|
|
302
384
|
|
|
303
|
-
if (r <
|
|
385
|
+
if (r < fullCut)
|
|
304
386
|
return {
|
|
305
387
|
fmt: "full",
|
|
306
388
|
raw: `${road}, ${tail(loc, reg, pc)}`,
|
|
307
389
|
components: { ...withRoad, locality: loc, region: reg, ...(pc ? { postcode: pc } : {}) },
|
|
308
390
|
}
|
|
309
391
|
|
|
310
|
-
if (r <
|
|
392
|
+
if (r < bareCut) return { fmt: "bare", raw: road, components: withRoad }
|
|
311
393
|
|
|
312
|
-
if (r <
|
|
313
|
-
const v =
|
|
394
|
+
if (r < streetOnlyCut) return { fmt: "street-only", raw: street, components: { ...streetComponents } }
|
|
395
|
+
const v = venues[Math.floor(random() * venues.length)]!
|
|
314
396
|
|
|
315
397
|
return {
|
|
316
398
|
fmt: "venue",
|
|
@@ -320,28 +402,15 @@ function renderRow(
|
|
|
320
402
|
}
|
|
321
403
|
|
|
322
404
|
/**
|
|
323
|
-
* Capped reader for the multi-locale BALANCE sources. The FR/IT/NL countrywide extracts are GB-scale, so
|
|
324
|
-
*
|
|
325
|
-
*
|
|
405
|
+
* Capped reader for the multi-locale BALANCE sources. The FR/IT/NL countrywide extracts are GB-scale, so this reads
|
|
406
|
+
* only as far as `limit` distinct tuples — the `break` closes the reader and releases the archive. Only keeps tuples
|
|
407
|
+
* that carry a POSTCODE.
|
|
326
408
|
*/
|
|
327
|
-
function readBalanceTuples(source: BalanceSource, limit: number): BalanceTuple[] {
|
|
328
|
-
const maxLines = Math.max(limit * 8, 20_000) + 1
|
|
329
|
-
|
|
330
|
-
const r = spawnSync("bash", ["-c", `unzip -p "${source.zip}" "${source.csv}" | head -n ${maxLines}`], {
|
|
331
|
-
maxBuffer: 1024 * 1024 * 1024,
|
|
332
|
-
encoding: "buffer",
|
|
333
|
-
})
|
|
334
|
-
|
|
335
|
-
if (r.status !== 0) {
|
|
336
|
-
console.error(` WARN: unzip failed for ${source.zip} (status ${r.status})`)
|
|
337
|
-
|
|
338
|
-
return []
|
|
339
|
-
}
|
|
340
|
-
|
|
409
|
+
async function readBalanceTuples(source: BalanceSource, limit: number): Promise<BalanceTuple[]> {
|
|
341
410
|
const tuples: BalanceTuple[] = []
|
|
342
411
|
const seen = new Set<string>()
|
|
343
412
|
|
|
344
|
-
for (const row of
|
|
413
|
+
for await (const row of readZippedCSVRecords(source.zip, source.csv)) {
|
|
345
414
|
if (tuples.length >= limit) break
|
|
346
415
|
|
|
347
416
|
const street = row.street ?? "",
|
|
@@ -412,7 +481,7 @@ export const streetAffixRecipe: ShardRecipe = {
|
|
|
412
481
|
const pool: USTuple[] = []
|
|
413
482
|
|
|
414
483
|
for (const s of sources) {
|
|
415
|
-
const t = readTuples(s)
|
|
484
|
+
const t = await readTuples(s)
|
|
416
485
|
|
|
417
486
|
console.error(` ${s.csv}: ${t.length} unique tuples`)
|
|
418
487
|
|
|
@@ -513,7 +582,7 @@ export const streetAffixRecipe: ShardRecipe = {
|
|
|
513
582
|
const mlPool: BalanceTuple[] = []
|
|
514
583
|
|
|
515
584
|
for (const s of mlSources) {
|
|
516
|
-
const t = readBalanceTuples(s, perSource)
|
|
585
|
+
const t = await readBalanceTuples(s, perSource)
|
|
517
586
|
|
|
518
587
|
console.error(` balance ${s.csv} (${s.iso2}): ${t.length} tuples`)
|
|
519
588
|
|
|
@@ -585,3 +654,173 @@ export const streetAffixRecipe: ShardRecipe = {
|
|
|
585
654
|
return { emitted: emitted + balanceEmitted, skipped: skipped + balanceSkipped }
|
|
586
655
|
},
|
|
587
656
|
}
|
|
657
|
+
|
|
658
|
+
/**
|
|
659
|
+
* #1569 root-fix shard. Both classes come from real non-Vermont OA streets and use the affix recipe's existing layout
|
|
660
|
+
* diversity. v4.3.1 makes terminal-only 80% of the mix: the first 40/60 run moved a 100-row TRAIN sample only 4→11
|
|
661
|
+
* while contrast was already 95/100 before training (93/100 after). Post-run audit found that the global affix relabel
|
|
662
|
+
* pass corrupts many already-decomposed target rows into double suffixes; do not retrain this recipe until relabel is
|
|
663
|
+
* idempotent over a decomposed street family. The 20% contrast leg remains explicit, additive to the already-strong
|
|
664
|
+
* base distribution, and B2 still gates it unchanged.
|
|
665
|
+
*
|
|
666
|
+
* V2 (corpus 0.19.0, 2026-08-10 recipe review): the v4.3.3 board split (rich venue-led rows 5/43 vs bare 53/65) showed
|
|
667
|
+
* the model separating template rows from real ones, and the venue shell was the giveaway — six fixed venue strings. v2
|
|
668
|
+
* draws the venue shell from thousands of REAL HRSA facility names and raises its share (venue 30%, full 35%, bare 20%,
|
|
669
|
+
* street-only 15%). Dose policy moved to the recipe's config side: weight ≤4 effective passes per run (Muennighoff
|
|
670
|
+
* 2023's repetition knee) and the source is excluded from the augmentation pool — see the v4.4.0 config.
|
|
671
|
+
*/
|
|
672
|
+
export const suffixBoundaryRecipe: ShardRecipe = {
|
|
673
|
+
name: "suffix-boundary",
|
|
674
|
+
description: "US #1569 suffix boundary: OA terminal-only positives + over-sampled terminal contrasts",
|
|
675
|
+
mode: "generate",
|
|
676
|
+
async run(opts, write) {
|
|
677
|
+
const random = makeMulberry32(opts.seed)
|
|
678
|
+
const count = opts.count ?? 30_000
|
|
679
|
+
const source = opts.sourceName ?? "synth-suffix-boundary"
|
|
680
|
+
const sources = opts.golden ? [EVAL_SOURCE] : TRAIN_SOURCES
|
|
681
|
+
|
|
682
|
+
// v2 venue shell: real facility names, loud when the source file is missing or thin.
|
|
683
|
+
const venuePool = await readVenuePool(VENUE_POOL_CSV)
|
|
684
|
+
|
|
685
|
+
if (venuePool.length < 500) {
|
|
686
|
+
throw new Error(
|
|
687
|
+
`suffix-boundary v2 venue pool too thin: ${venuePool.length} names from ${VENUE_POOL_CSV} — ` +
|
|
688
|
+
"is the usgov-hrsa-fqhc source present?"
|
|
689
|
+
)
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
const shell: RenderRowOpts = { venues: venuePool, cuts: [0.35, 0.55, 0.7] }
|
|
693
|
+
|
|
694
|
+
const pool: Record<SuffixBoundaryClass, USTuple[]> = {
|
|
695
|
+
"terminal-only": [],
|
|
696
|
+
"terminal-contrast": [],
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
for (const s of sources) {
|
|
700
|
+
for (const tuple of await readTuples(s)) {
|
|
701
|
+
const rowClass = classifySuffixBoundaryStreet(tuple.street)
|
|
702
|
+
|
|
703
|
+
if (rowClass) {
|
|
704
|
+
pool[rowClass].push(tuple)
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
if (!pool["terminal-only"].length || !pool["terminal-contrast"].length) {
|
|
710
|
+
throw new Error(
|
|
711
|
+
`Suffix-boundary source class missing: terminal-only=${pool["terminal-only"].length}, ` +
|
|
712
|
+
`terminal-contrast=${pool["terminal-contrast"].length}`
|
|
713
|
+
)
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
let emitted = 0
|
|
717
|
+
let skipped = 0
|
|
718
|
+
const classCounts: Record<SuffixBoundaryClass, number> = { "terminal-only": 0, "terminal-contrast": 0 }
|
|
719
|
+
const terminalOnlyTarget = Math.round(count * 0.8)
|
|
720
|
+
|
|
721
|
+
const classTargets: Record<SuffixBoundaryClass, number> = {
|
|
722
|
+
"terminal-only": terminalOnlyTarget,
|
|
723
|
+
"terminal-contrast": count - terminalOnlyTarget,
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
const formatCounts: Record<string, number> = {}
|
|
727
|
+
let stemPairs = 0
|
|
728
|
+
|
|
729
|
+
const emitOne = (
|
|
730
|
+
rowClass: SuffixBoundaryClass,
|
|
731
|
+
base: USTuple,
|
|
732
|
+
parsed: NonNullable<ReturnType<typeof parseStreet>>,
|
|
733
|
+
method: string
|
|
734
|
+
): boolean => {
|
|
735
|
+
const { street, components: streetComponents } = renderStreet(random, parsed)
|
|
736
|
+
const { fmt, raw, components } = renderRow(random, base, street, streetComponents, shell)
|
|
737
|
+
|
|
738
|
+
const canonical: CanonicalRow = {
|
|
739
|
+
raw,
|
|
740
|
+
components,
|
|
741
|
+
country: "US",
|
|
742
|
+
locale: "en-US",
|
|
743
|
+
source,
|
|
744
|
+
source_id: shardSourceID(source, { ...components, class: rowClass, n: String(emitted) }),
|
|
745
|
+
corpus_version: "0.19.0",
|
|
746
|
+
license:
|
|
747
|
+
"OpenAddresses US (non-VT) skeletons; terminal suffix labels via USPS Pub-28 codex; venue shell from HRSA site names (US public domain)",
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
const aligned = alignRow(canonical)
|
|
751
|
+
|
|
752
|
+
if (aligned.kind !== "labeled" || !aligned.row) {
|
|
753
|
+
skipped++
|
|
754
|
+
|
|
755
|
+
return false
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
write(JSON.stringify({ ...aligned.row, synth_method: method, synth_base_id: base.base_source_id }) + "\n")
|
|
759
|
+
|
|
760
|
+
classCounts[rowClass]++
|
|
761
|
+
formatCounts[fmt] = (formatCounts[fmt] ?? 0) + 1
|
|
762
|
+
|
|
763
|
+
emitted++
|
|
764
|
+
|
|
765
|
+
return true
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
// oxlint-disable-next-line eslint/no-unmodified-loop-condition -- emitted/skipped advance inside emitOne (a closure); the rule cannot see through the call
|
|
769
|
+
while (emitted < count && emitted + skipped < count * 10) {
|
|
770
|
+
const terminalOnlyOpen = classCounts["terminal-only"] < classTargets["terminal-only"]
|
|
771
|
+
const terminalContrastOpen = classCounts["terminal-contrast"] < classTargets["terminal-contrast"]
|
|
772
|
+
|
|
773
|
+
const rowClass: SuffixBoundaryClass =
|
|
774
|
+
terminalOnlyOpen && terminalContrastOpen
|
|
775
|
+
? random() < 0.8
|
|
776
|
+
? "terminal-only"
|
|
777
|
+
: "terminal-contrast"
|
|
778
|
+
: terminalContrastOpen
|
|
779
|
+
? "terminal-contrast"
|
|
780
|
+
: "terminal-only"
|
|
781
|
+
|
|
782
|
+
const classPool = pool[rowClass]
|
|
783
|
+
const base = classPool[Math.floor(random() * classPool.length)]!
|
|
784
|
+
const parsed = parseStreet(base.street, { allowNameProneTail: rowClass === "terminal-only" })
|
|
785
|
+
|
|
786
|
+
if (!parsed) {
|
|
787
|
+
skipped++
|
|
788
|
+
|
|
789
|
+
continue
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
if (!emitOne(rowClass, base, parsed, `suffix-boundary:${rowClass}`)) continue
|
|
793
|
+
|
|
794
|
+
// Stem-pair hard negative (v2, 2026-08-10 recipe review): after a terminal-only row
|
|
795
|
+
// ('Menlo Park' + 'Road'), also emit the SAME stem without its true suffix as a
|
|
796
|
+
// contrast row ('Menlo' + 'Park' under the canonical last-token rule). Sharing the stem
|
|
797
|
+
// forces the model to key on the licensing evidence — the trailing true suffix — rather
|
|
798
|
+
// than on name identity. Dose-neutral: these fill the existing 20% contrast target.
|
|
799
|
+
if (
|
|
800
|
+
rowClass === "terminal-only" &&
|
|
801
|
+
classCounts["terminal-contrast"] < classTargets["terminal-contrast"] &&
|
|
802
|
+
emitted < count &&
|
|
803
|
+
random() < 0.5
|
|
804
|
+
) {
|
|
805
|
+
const stemParsed = parseStreet(parsed.name)
|
|
806
|
+
|
|
807
|
+
if (
|
|
808
|
+
stemParsed &&
|
|
809
|
+
emitOne("terminal-contrast", base, stemParsed, "suffix-boundary:terminal-contrast:stem-pair")
|
|
810
|
+
) {
|
|
811
|
+
stemPairs++
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
console.error(
|
|
817
|
+
`Done: emitted ${emitted}, skipped ${skipped}; source pools ${JSON.stringify({
|
|
818
|
+
terminalOnly: pool["terminal-only"].length,
|
|
819
|
+
terminalContrast: pool["terminal-contrast"].length,
|
|
820
|
+
})}; classes ${JSON.stringify(classCounts)}; stem pairs ${stemPairs}; ` +
|
|
821
|
+
`venue pool ${venuePool.length}; formats ${JSON.stringify(formatCounts)}`
|
|
822
|
+
)
|
|
823
|
+
|
|
824
|
+
return { emitted, skipped }
|
|
825
|
+
},
|
|
826
|
+
}
|
|
@@ -17,9 +17,10 @@
|
|
|
17
17
|
* reproduces the legacy run byte-for-byte.
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
|
-
import {
|
|
21
|
-
import {
|
|
22
|
-
import {
|
|
20
|
+
import { DEFAULT_US_BASES } from "#synthesizers/intersection"
|
|
21
|
+
import { synthesizeStreetRow, type StreetBaseTuple } from "#synthesizers/street"
|
|
22
|
+
import { alignRow } from "#utils"
|
|
23
|
+
|
|
23
24
|
import { makeMulberry32, shardSourceID, type CanonicalShardRow, type ShardRecipe } from "./scaffold.ts"
|
|
24
25
|
|
|
25
26
|
/**
|
|
@@ -7,8 +7,9 @@
|
|
|
7
7
|
* {@link synthesizeStreetRow} → aligned LabeledRow. Ported from scripts/build-street-shard.mjs.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
-
import { stableSourceID } from "
|
|
11
|
-
import { synthesizeStreetRow, type StreetBaseTuple } from "
|
|
10
|
+
import { stableSourceID } from "#adapters/utils"
|
|
11
|
+
import { synthesizeStreetRow, type StreetBaseTuple } from "#synthesizers/street"
|
|
12
|
+
|
|
12
13
|
import { alignAndWrite, makeLcg, readTuples, type ShardRecipe } from "./scaffold.ts"
|
|
13
14
|
|
|
14
15
|
/**
|
|
@@ -26,30 +26,31 @@
|
|
|
26
26
|
*/
|
|
27
27
|
|
|
28
28
|
import { readFileSync } from "node:fs"
|
|
29
|
-
import { createRequire } from "node:module"
|
|
30
|
-
import { dirname, resolve } from "node:path"
|
|
31
29
|
import { DatabaseSync } from "node:sqlite"
|
|
30
|
+
import { fileURLToPath } from "node:url"
|
|
32
31
|
|
|
33
32
|
import { parseJSONStrict } from "@mailwoman/core/objects"
|
|
34
33
|
import { dataRootPath } from "@mailwoman/core/utils"
|
|
35
34
|
|
|
36
|
-
import type { LocaleBaseTuple } from "
|
|
37
|
-
import {
|
|
38
|
-
|
|
35
|
+
import type { LocaleBaseTuple } from "#synthesizers/german"
|
|
36
|
+
import {
|
|
37
|
+
classifyIdentifier,
|
|
38
|
+
readSubVenueJSONL,
|
|
39
|
+
type SubVenueLexiconTable,
|
|
40
|
+
SUBVENUE_PROMOTIONS,
|
|
41
|
+
type SubVenuePromotion,
|
|
42
|
+
} from "#tools"
|
|
43
|
+
|
|
39
44
|
import { readTuples as readLocaleTuples, type LocalePart } from "./locale.ts"
|
|
40
45
|
import { makeMulberry32, readTuples as readShardTuples } from "./scaffold.ts"
|
|
41
46
|
|
|
42
47
|
//#region Lexicon
|
|
43
48
|
|
|
44
|
-
const require_ = createRequire(import.meta.url)
|
|
45
|
-
|
|
46
49
|
/**
|
|
47
|
-
* The committed lexicon, resolved through the package manifest
|
|
48
|
-
* sits at `corpus/data/` from both the source tree and the compiled `out/` tree, and `createRequire().resolve` finds it
|
|
49
|
-
* from either without a `__isCompiledTree` flag of its own.
|
|
50
|
+
* The committed lexicon, resolved through the package manifest.
|
|
50
51
|
*/
|
|
51
52
|
export function defaultLexiconPath(): string {
|
|
52
|
-
return
|
|
53
|
+
return fileURLToPath(import.meta.resolve("@mailwoman/corpus/data/sub-venue-lexicon.json"))
|
|
53
54
|
}
|
|
54
55
|
|
|
55
56
|
/**
|
|
@@ -568,17 +569,6 @@ export const EMPTY_NAME_POOLS: NamePools = {
|
|
|
568
569
|
unpromotedShapes: [],
|
|
569
570
|
}
|
|
570
571
|
|
|
571
|
-
/**
|
|
572
|
-
* An extract row, as `sub-venue-extract` writes it. `SubVenueHarvestRow` plus the `tier` the harvest does not need and
|
|
573
|
-
* this recipe does: `venue` rows (station, airport, campus) fill the venue slot.
|
|
574
|
-
*/
|
|
575
|
-
interface SubVenueExtractRow {
|
|
576
|
-
designatorID: string
|
|
577
|
-
tier?: string
|
|
578
|
-
name?: string | null
|
|
579
|
-
ref?: string | null
|
|
580
|
-
}
|
|
581
|
-
|
|
582
572
|
/**
|
|
583
573
|
* What every pool reader needs to know about the leg it is reading for.
|
|
584
574
|
*/
|
|
@@ -602,7 +592,7 @@ function isLongerProperName(low: string, name: string, designatorPhrases: readon
|
|
|
602
592
|
* Read one OSM extract and split it into name pools.
|
|
603
593
|
*/
|
|
604
594
|
export function readExtractPools(path: string, query: PoolQuery): NamePools {
|
|
605
|
-
const rows = readSubVenueJSONL(path)
|
|
595
|
+
const rows = readSubVenueJSONL(path)
|
|
606
596
|
const venues = new Set<string>()
|
|
607
597
|
const attested = new Set<string>()
|
|
608
598
|
const rejectedVenues = new Set<string>()
|
|
@@ -60,12 +60,14 @@
|
|
|
60
60
|
* `JP` key. The JP extract is harvested and ready; the leg belongs to the JP shard.
|
|
61
61
|
*/
|
|
62
62
|
|
|
63
|
+
import { isPresent } from "@mailwoman/core/objects"
|
|
63
64
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
64
65
|
import { dataRootPath } from "@mailwoman/core/utils"
|
|
65
66
|
|
|
66
|
-
import {
|
|
67
|
-
import type {
|
|
68
|
-
import
|
|
67
|
+
import type { LocaleBaseTuple } from "#synthesizers/german"
|
|
68
|
+
import type { SubVenueLexiconTable } from "#tools"
|
|
69
|
+
import { alignRow } from "#utils"
|
|
70
|
+
|
|
69
71
|
import { makeMulberry32, shardSourceID, type ShardRecipe } from "./scaffold.ts"
|
|
70
72
|
import {
|
|
71
73
|
buildIdentifierModel,
|
|
@@ -374,7 +376,7 @@ export function renderGroups(
|
|
|
374
376
|
|
|
375
377
|
const raw = groups
|
|
376
378
|
.map((group) => group.map((piece) => fold(piece.text)).join(" "))
|
|
377
|
-
.filter(
|
|
379
|
+
.filter(isPresent)
|
|
378
380
|
.join(separator)
|
|
379
381
|
|
|
380
382
|
const components: Partial<Record<ComponentTag, string>> = {}
|
|
@@ -394,7 +396,7 @@ export function renderGroups(
|
|
|
394
396
|
* The street + tail groups for a country, in that country's own order.
|
|
395
397
|
*
|
|
396
398
|
* DE/ES/FR put the postcode before the locality and DE/ES put the house number after the street; GB and US keep the
|
|
397
|
-
* anglophone order and US carries a region. These are the same orders `
|
|
399
|
+
* anglophone order and US carries a region. These are the same orders `synthesizers/german.ts` renders, restated here
|
|
398
400
|
* because this recipe assembles its groups piece-by-piece (it has to, to place a sub-venue group in front of them).
|
|
399
401
|
*/
|
|
400
402
|
export function addressGroups(country: string, tuple: LocaleBaseTuple, withStreet: boolean): Group[] {
|
|
@@ -826,13 +828,13 @@ function buildLegPools(
|
|
|
826
828
|
leg: SubVenueLeg,
|
|
827
829
|
query: PoolQuery,
|
|
828
830
|
contextByCountry: ReadonlyMap<string, LocaleBaseTuple[]>,
|
|
829
|
-
paths: { extractsDir: string;
|
|
831
|
+
paths: { extractsDir: string; poiDB: string }
|
|
830
832
|
): LegPools {
|
|
831
833
|
const extractPools = leg.extract ? readExtractPools(`${paths.extractsDir}/${leg.extract}`, query) : EMPTY_NAME_POOLS
|
|
832
834
|
|
|
833
835
|
// poi.db holds four countries; only these two legs are inside it.
|
|
834
836
|
const poiPools =
|
|
835
|
-
leg.country === "US" || leg.country === "FR" ? readPOIPools(paths.
|
|
837
|
+
leg.country === "US" || leg.country === "FR" ? readPOIPools(paths.poiDB, leg.country, query) : EMPTY_NAME_POOLS
|
|
836
838
|
|
|
837
839
|
const names = mergeNamePools(extractPools, poiPools)
|
|
838
840
|
let context = contextByCountry.get(leg.country) ?? []
|
|
@@ -906,7 +908,7 @@ export const subVenueRecipe: ShardRecipe = {
|
|
|
906
908
|
const count = opts.count
|
|
907
909
|
const negativeFraction = opts.negativeFraction ?? DEFAULT_NEGATIVE_FRACTION
|
|
908
910
|
const extractsDir = opts.extractsDir ?? dataRootPath("sub-venue", "extracts")
|
|
909
|
-
const
|
|
911
|
+
const poiDB = opts.poiDB ?? dataRootPath("poi", "poi.db")
|
|
910
912
|
const tuplesPath = opts.subVenueTuples ?? dataRootPath("corpus", "intermediate", "house-venue-tuples-v3.jsonl")
|
|
911
913
|
const lexicon: SubVenueLexiconTable = readSubVenueLexicon(opts.lexicon ?? defaultLexiconPath())
|
|
912
914
|
|
|
@@ -937,7 +939,7 @@ export const subVenueRecipe: ShardRecipe = {
|
|
|
937
939
|
english: leg.english,
|
|
938
940
|
}
|
|
939
941
|
|
|
940
|
-
const pools = buildLegPools(leg, query, contextByCountry, { extractsDir,
|
|
942
|
+
const pools = buildLegPools(leg, query, contextByCountry, { extractsDir, poiDB })
|
|
941
943
|
|
|
942
944
|
legPromoted.set(leg.locale, promoted)
|
|
943
945
|
legPools.set(leg.locale, pools)
|