@mailwoman/corpus 8.0.0 → 8.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/adapter.d.ts +15 -5
- package/out/src/adapter.d.ts.map +1 -1
- package/out/src/adapter.js +11 -9
- package/out/src/adapter.js.map +1 -1
- package/out/src/adapters/ban/adapter.d.ts +7 -0
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +9 -2
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +5 -1
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +14 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +15 -2
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +11 -0
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +24 -5
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +11 -0
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +14 -1
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +10 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +11 -2
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts +18 -6
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +16 -6
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +11 -0
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +18 -3
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +11 -0
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +12 -1
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +11 -0
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +11 -0
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +15 -2
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +11 -0
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +13 -2
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +14 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +19 -4
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +12 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +17 -2
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +12 -1
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +11 -0
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +15 -2
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +7 -0
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +18 -3
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +13 -3
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +15 -5
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +7 -0
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +9 -2
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/align.d.ts +12 -4
- package/out/src/align.d.ts.map +1 -1
- package/out/src/align.js +9 -5
- package/out/src/align.js.map +1 -1
- package/out/src/build.d.ts +24 -8
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +3 -3
- package/out/src/build.js.map +1 -1
- package/out/src/codex/us-fips-state.d.ts +6 -2
- package/out/src/codex/us-fips-state.d.ts.map +1 -1
- package/out/src/codex/us-fips-state.js +3 -1
- package/out/src/codex/us-fips-state.js.map +1 -1
- package/out/src/golden.d.ts +21 -7
- package/out/src/golden.d.ts.map +1 -1
- package/out/src/golden.js +18 -10
- package/out/src/golden.js.map +1 -1
- package/out/src/license.d.ts +3 -1
- package/out/src/license.d.ts.map +1 -1
- package/out/src/license.js +4 -2
- package/out/src/license.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts +4 -2
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +5 -4
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/schema.d.ts +14 -10
- package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
- package/out/src/parquet-wrapper/schema.js +6 -2
- package/out/src/parquet-wrapper/schema.js.map +1 -1
- package/out/src/parquet-wrapper/writer.d.ts +12 -4
- package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
- package/out/src/parquet-wrapper/writer.js +12 -4
- package/out/src/parquet-wrapper/writer.js.map +1 -1
- package/out/src/parquet.d.ts +21 -7
- package/out/src/parquet.d.ts.map +1 -1
- package/out/src/parquet.js +15 -4
- package/out/src/parquet.js.map +1 -1
- package/out/src/runner.d.ts +39 -13
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +7 -3
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts +4 -0
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +14 -5
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/boundary-stress.d.ts +4 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +13 -7
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts +4 -0
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +39 -11
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts +4 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +14 -3
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts +4 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.js +4 -0
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts +4 -0
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +25 -8
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts +4 -0
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +21 -8
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts +4 -0
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +27 -8
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts +4 -0
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +16 -6
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.d.ts +4 -0
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/house-venue.js +4 -0
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts +9 -3
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +12 -4
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts +4 -0
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +27 -13
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +4 -0
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +9 -4
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/nl-postcode.d.ts +4 -0
- package/out/src/shard-recipes/nl-postcode.d.ts.map +1 -1
- package/out/src/shard-recipes/nl-postcode.js +8 -2
- package/out/src/shard-recipes/nl-postcode.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts +4 -0
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +8 -2
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts +4 -0
- package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street-led.js +5 -1
- package/out/src/shard-recipes/no-street-led.js.map +1 -1
- package/out/src/shard-recipes/no-street.d.ts +4 -0
- package/out/src/shard-recipes/no-street.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street.js +4 -0
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +4 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +99 -43
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts +4 -0
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +4 -0
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +48 -16
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +10 -6
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/si-bare-village.d.ts +4 -0
- package/out/src/shard-recipes/si-bare-village.d.ts.map +1 -1
- package/out/src/shard-recipes/si-bare-village.js +4 -0
- package/out/src/shard-recipes/si-bare-village.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +4 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +29 -10
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.d.ts +4 -0
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
- package/out/src/shard-recipes/street-bare.js +4 -0
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.d.ts +4 -0
- package/out/src/shard-recipes/street.d.ts.map +1 -1
- package/out/src/shard-recipes/street.js +4 -0
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/unit.d.ts +4 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +45 -14
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/split.d.ts +18 -6
- package/out/src/split.d.ts.map +1 -1
- package/out/src/split.js +4 -2
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize-anchor-absorption.d.ts +7 -1
- package/out/src/synthesize-anchor-absorption.d.ts.map +1 -1
- package/out/src/synthesize-anchor-absorption.js +28 -14
- package/out/src/synthesize-anchor-absorption.js.map +1 -1
- package/out/src/synthesize-boundary-stress.d.ts +3 -1
- package/out/src/synthesize-boundary-stress.d.ts.map +1 -1
- package/out/src/synthesize-boundary-stress.js +35 -28
- package/out/src/synthesize-boundary-stress.js.map +1 -1
- package/out/src/synthesize-german.d.ts +15 -5
- package/out/src/synthesize-german.d.ts.map +1 -1
- package/out/src/synthesize-german.js +9 -3
- package/out/src/synthesize-german.js.map +1 -1
- package/out/src/synthesize-house-venue.d.ts.map +1 -1
- package/out/src/synthesize-house-venue.js +15 -15
- package/out/src/synthesize-house-venue.js.map +1 -1
- package/out/src/synthesize-intersection.d.ts +9 -3
- package/out/src/synthesize-intersection.d.ts.map +1 -1
- package/out/src/synthesize-intersection.js +25 -6
- package/out/src/synthesize-intersection.js.map +1 -1
- package/out/src/synthesize-no-street.d.ts +3 -1
- package/out/src/synthesize-no-street.d.ts.map +1 -1
- package/out/src/synthesize-no-street.js +5 -6
- package/out/src/synthesize-no-street.js.map +1 -1
- package/out/src/synthesize-po-box.d.ts +15 -5
- package/out/src/synthesize-po-box.d.ts.map +1 -1
- package/out/src/synthesize-po-box.js +21 -7
- package/out/src/synthesize-po-box.js.map +1 -1
- package/out/src/synthesize-street.d.ts +3 -1
- package/out/src/synthesize-street.d.ts.map +1 -1
- package/out/src/synthesize-street.js +6 -1
- package/out/src/synthesize-street.js.map +1 -1
- package/out/src/synthesize.d.ts +39 -13
- package/out/src/synthesize.d.ts.map +1 -1
- package/out/src/synthesize.js +66 -36
- package/out/src/synthesize.js.map +1 -1
- package/out/src/tokenize.d.ts +15 -5
- package/out/src/tokenize.d.ts.map +1 -1
- package/out/src/tokenize.js.map +1 -1
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +3 -1
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/audit.d.ts.map +1 -1
- package/out/src/tools/audit.js +20 -8
- package/out/src/tools/audit.js.map +1 -1
- package/out/src/tools/corpus-stats.d.ts.map +1 -1
- package/out/src/tools/corpus-stats.js.map +1 -1
- package/out/src/tools/fetch/ban.d.ts.map +1 -1
- package/out/src/tools/fetch/ban.js +17 -10
- package/out/src/tools/fetch/ban.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts +30 -10
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +26 -6
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/hrsa.d.ts.map +1 -1
- package/out/src/tools/fetch/hrsa.js.map +1 -1
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +18 -18
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +3 -1
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +3 -1
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts +21 -7
- package/out/src/tools/fetch/nad.d.ts.map +1 -1
- package/out/src/tools/fetch/nad.js +5 -5
- package/out/src/tools/fetch/nad.js.map +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +9 -15
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts +3 -1
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/src/tools/fetch/openaddresses.js +32 -23
- package/out/src/tools/fetch/openaddresses.js.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +3 -1
- package/out/src/tools/fetch/ppd.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.js +14 -3
- package/out/src/tools/fetch/ppd.js.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.js +13 -6
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
- package/out/src/tools/fetch/state-sources.js +8 -3
- package/out/src/tools/fetch/state-sources.js.map +1 -1
- package/out/src/tools/fetch/tiger-full.d.ts +9 -3
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
- package/out/src/tools/fetch/tiger-full.js +35 -20
- package/out/src/tools/fetch/tiger-full.js.map +1 -1
- package/out/src/tools/gb-title-case.d.ts.map +1 -1
- package/out/src/tools/gb-title-case.js.map +1 -1
- package/out/src/tools/golden-expand.d.ts +24 -8
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +20 -12
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-promote.d.ts +18 -6
- package/out/src/tools/golden-promote.d.ts.map +1 -1
- package/out/src/tools/golden-promote.js +22 -7
- package/out/src/tools/golden-promote.js.map +1 -1
- package/out/src/tools/ingest-csv.d.ts +3 -1
- package/out/src/tools/ingest-csv.d.ts.map +1 -1
- package/out/src/tools/ingest-csv.js +17 -19
- package/out/src/tools/ingest-csv.js.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts +18 -6
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
- package/out/src/tools/jsonl-to-parquet.js +11 -5
- package/out/src/tools/jsonl-to-parquet.js.map +1 -1
- package/out/src/tools/lint-shard-vocab.d.ts +36 -12
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
- package/out/src/tools/lint-shard-vocab.js +25 -11
- package/out/src/tools/lint-shard-vocab.js.map +1 -1
- package/out/src/tools/lint-shard.d.ts +30 -10
- package/out/src/tools/lint-shard.d.ts.map +1 -1
- package/out/src/tools/lint-shard.js +33 -20
- package/out/src/tools/lint-shard.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +6 -4
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/shard-kryptonite.d.ts +6 -2
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +1 -1
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts +9 -3
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +2 -2
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/types.d.ts +51 -17
- package/out/src/types.d.ts.map +1 -1
- package/out/src/wof-json.d.ts +9 -3
- package/out/src/wof-json.d.ts.map +1 -1
- package/out/src/wof-json.js +6 -4
- package/out/src/wof-json.js.map +1 -1
- package/package.json +7 -6
- package/src/adapter.ts +24 -13
- package/src/adapters/ban/adapter.ts +12 -2
- package/src/adapters/ban/street-decompose.ts +7 -1
- package/src/adapters/fcc-bdc/adapter.ts +19 -3
- package/src/adapters/geonames/adapter.ts +28 -5
- package/src/adapters/geonames-postal/adapter.ts +17 -1
- package/src/adapters/gnaf/adapter.ts +15 -2
- package/src/adapters/gnaf/assemble.ts +40 -10
- package/src/adapters/index.ts +13 -0
- package/src/adapters/openaddresses/adapter.ts +22 -3
- package/src/adapters/overture/adapter.ts +18 -2
- package/src/adapters/state-hi-schools/adapter.ts +15 -0
- package/src/adapters/state-ia-contractors/adapter.ts +15 -0
- package/src/adapters/state-ny-notaries/adapter.ts +15 -0
- package/src/adapters/state-tx-notaries/adapter.ts +18 -2
- package/src/adapters/synth-po-box/adapter.ts +19 -2
- package/src/adapters/tiger/adapter.ts +23 -4
- package/src/adapters/tiger/street-decompose.ts +1 -0
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +18 -2
- package/src/adapters/usgov-imls-pls/adapter.ts +15 -0
- package/src/adapters/usgov-irs-bmf/adapter.ts +19 -2
- package/src/adapters/usgov-nad/adapter.ts +14 -1
- package/src/adapters/usgov-nppes/adapter.ts +13 -0
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +21 -3
- package/src/adapters/wof-admin-jp/adapter.ts +20 -3
- package/src/adapters/wof-admin-json/adapter.ts +22 -7
- package/src/adapters/wof-postalcode-json/adapter.ts +10 -2
- package/src/align.ts +21 -8
- package/src/build.ts +48 -11
- package/src/codex/us-fips-state.ts +6 -2
- package/src/golden.ts +28 -13
- package/src/license.ts +4 -2
- package/src/parquet-wrapper/reader.ts +6 -4
- package/src/parquet-wrapper/schema.ts +14 -10
- package/src/parquet-wrapper/writer.ts +12 -4
- package/src/parquet.ts +39 -8
- package/src/runner.ts +49 -15
- package/src/shard-recipes/anchor-absorption.ts +21 -5
- package/src/shard-recipes/boundary-stress.ts +20 -7
- package/src/shard-recipes/country-balanced.ts +78 -13
- package/src/shard-recipes/cz-pcfirst-preposition.ts +9 -0
- package/src/shard-recipes/fr-admin-split.ts +30 -5
- package/src/shard-recipes/fr-bare-street.ts +10 -0
- package/src/shard-recipes/fr-fragment.ts +33 -8
- package/src/shard-recipes/fr-lieudit.ts +35 -9
- package/src/shard-recipes/fr-order.ts +47 -9
- package/src/shard-recipes/german.ts +32 -6
- package/src/shard-recipes/house-venue.ts +7 -0
- package/src/shard-recipes/index.ts +12 -4
- package/src/shard-recipes/intersection.ts +63 -14
- package/src/shard-recipes/locale.ts +37 -4
- package/src/shard-recipes/nl-postcode.ts +12 -2
- package/src/shard-recipes/no-fragment.ts +12 -2
- package/src/shard-recipes/no-street-led.ts +11 -1
- package/src/shard-recipes/no-street.ts +7 -0
- package/src/shard-recipes/po-box-cedex.ts +145 -45
- package/src/shard-recipes/po-box.ts +7 -0
- package/src/shard-recipes/scaffold.ts +52 -20
- package/src/shard-recipes/si-bare-village.ts +9 -0
- package/src/shard-recipes/street-affix.ts +88 -13
- package/src/shard-recipes/street-bare.ts +10 -0
- package/src/shard-recipes/street.ts +7 -0
- package/src/shard-recipes/unit.ts +79 -16
- package/src/split.ts +28 -7
- package/src/synthesize-anchor-absorption.ts +41 -14
- package/src/synthesize-boundary-stress.ts +57 -30
- package/src/synthesize-german.ts +30 -7
- package/src/synthesize-house-venue.ts +22 -16
- package/src/synthesize-intersection.ts +34 -7
- package/src/synthesize-no-street.ts +18 -10
- package/src/synthesize-po-box.ts +42 -10
- package/src/synthesize-street.ts +10 -2
- package/src/synthesize.ts +81 -38
- package/src/tokenize.ts +15 -5
- package/src/tools/align-shard.ts +6 -1
- package/src/tools/audit.ts +57 -11
- package/src/tools/corpus-stats.ts +9 -0
- package/src/tools/fetch/ban.ts +32 -10
- package/src/tools/fetch/download.ts +49 -12
- package/src/tools/fetch/hrsa.ts +3 -0
- package/src/tools/fetch/imls-pls.ts +28 -18
- package/src/tools/fetch/index.ts +3 -1
- package/src/tools/fetch/nad.ts +39 -12
- package/src/tools/fetch/nppes.ts +17 -15
- package/src/tools/fetch/openaddresses.ts +52 -24
- package/src/tools/fetch/ppd.ts +22 -3
- package/src/tools/fetch/state-hi-schools.ts +17 -6
- package/src/tools/fetch/state-sources.ts +16 -3
- package/src/tools/fetch/tiger-full.ts +66 -23
- package/src/tools/gb-title-case.ts +1 -0
- package/src/tools/golden-expand.ts +64 -20
- package/src/tools/golden-promote.ts +53 -13
- package/src/tools/ingest-csv.ts +41 -26
- package/src/tools/jsonl-to-parquet.ts +30 -10
- package/src/tools/lint-shard-vocab.ts +75 -24
- package/src/tools/lint-shard.ts +75 -31
- package/src/tools/overlay-manifest.ts +7 -4
- package/src/tools/shard-kryptonite.ts +11 -3
- package/src/tools/shard-translit.ts +20 -5
- package/src/types.ts +51 -17
- package/src/wof-json.ts +18 -7
|
@@ -51,6 +51,12 @@ import { makeMulberry32, shardSourceID, type CanonicalShardRow, type ShardRecipe
|
|
|
51
51
|
// Same OA cache as the unit/affix shards. US train = every NON-Vermont state; US eval = Vermont (the
|
|
52
52
|
// corpus defaultHoldout). FR comes from the BAN-derived countrywide extract (stride-sampled — the
|
|
53
53
|
// file is 2.5 GB and insee-ordered, so a head-only read would be all département 01).
|
|
54
|
+
/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
|
|
55
|
+
cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
|
|
56
|
+
output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
|
|
57
|
+
would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
|
|
58
|
+
extracted as named constants above. */
|
|
59
|
+
|
|
54
60
|
const US_TRAIN_SOURCES = [
|
|
55
61
|
{ zip: "/tmp/oa-cache/us__ca__berkeley.zip", csv: "us/ca/berkeley.csv", region: "CA" },
|
|
56
62
|
{ zip: "/tmp/oa-cache/us__ca__marin.zip", csv: "us/ca/marin.csv", region: "CA" },
|
|
@@ -60,49 +66,85 @@ const US_TRAIN_SOURCES = [
|
|
|
60
66
|
{ zip: "/tmp/oa-cache/us__mt__statewide.zip", csv: "us/mt/statewide.csv", region: "MT" },
|
|
61
67
|
{ zip: "/tmp/oa-cache/us__sd__statewide.zip", csv: "us/sd/statewide.csv", region: "SD" },
|
|
62
68
|
]
|
|
69
|
+
|
|
63
70
|
const US_EVAL_SOURCE = { zip: "/tmp/oa-cache/us__vt__statewide.zip", csv: "us/vt/statewide.csv", region: "VT" }
|
|
64
71
|
const FR_SOURCE = { zip: "/tmp/oa-cache/fr__countrywide.zip", csv: "fr/countrywide.csv" }
|
|
65
72
|
const GEONAMES_CA = "/tmp/geonames-cache/CA.zip"
|
|
66
73
|
const GEONAMES_POSTAL_AU = { zip: "/tmp/geonames-cache/AU-postal.zip", txt: "AU.txt" }
|
|
67
74
|
const GEONAMES_POSTAL_NZ = { zip: "/tmp/geonames-cache/NZ-postal.zip", txt: "NZ.txt" }
|
|
68
75
|
|
|
69
|
-
|
|
76
|
+
/**
|
|
77
|
+
* ── Surface vocabulary (codex + corpus templates — see the header) ──────────────────────────────.
|
|
78
|
+
*/
|
|
70
79
|
const T: Record<string, LocaleTemplate> = Object.fromEntries(PO_BOX_LOCALE_TEMPLATES.map((t) => [t.locale, t]))
|
|
71
80
|
// US: the corpus en-US leaders carry the common mass; the codex-only USPS Pub-28 designators
|
|
72
81
|
// (Caller/Drawer/Lockbox — firm-holdout and rural forms) ride at low weight. "Box" is in both.
|
|
73
|
-
|
|
74
|
-
|
|
82
|
+
/**
|
|
83
|
+
* PO Box, P.O. Box, P.O.Box, PO BOX, POB, Post Office Box, Box.
|
|
84
|
+
*/
|
|
85
|
+
const US_LEADERS_COMMON = T["en-US"]!.leaders
|
|
86
|
+
/**
|
|
87
|
+
* Codex US_PO_BOX_DESIGNATORS tail.
|
|
88
|
+
*/
|
|
89
|
+
const US_LEADERS_RARE = ["Caller", "Firm Caller", "Drawer", "Lockbox"]
|
|
75
90
|
// "#" EXCLUDED (v4.4.0 probe finding): bare "#N" is a secondary-unit designator per USPS Pub 28 and
|
|
76
91
|
// the shipped unit lever labels it `unit` — the corpus template's po_box reading CONTRADICTS a
|
|
77
92
|
// shipped convention. PMB stays — a genuine commercial-mail-receiving designator, no unit collision.
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
const
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
93
|
+
/**
|
|
94
|
+
* PMB.
|
|
95
|
+
*/
|
|
96
|
+
const US_PMB_LEADERS = T["en-US"]!.pmb!.filter((l) => l !== "#")
|
|
97
|
+
/**
|
|
98
|
+
* BP, B.P., Boîte Postale, BP.
|
|
99
|
+
*/
|
|
100
|
+
const FR_LEADERS = T["fr-FR"]!.leaders
|
|
101
|
+
/**
|
|
102
|
+
* CP, C.P., Case Postale, BP, B.P.
|
|
103
|
+
*/
|
|
104
|
+
const CA_FR_LEADERS = T["fr-CA"]!.leaders
|
|
105
|
+
/**
|
|
106
|
+
* PO Box, P.O. Box, POB, Post Office Box.
|
|
107
|
+
*/
|
|
108
|
+
const CA_EN_LEADERS = T["en-CA"]!.leaders
|
|
109
|
+
/**
|
|
110
|
+
* AU (#517): codex/au is the vocabulary truth. Current designators (live auspost.com.au pages) at full weight; the
|
|
111
|
+
* AMAS-legacy rural/community tail rides at the same 10% rare-dial as the US Caller/Drawer tail. Every emitted phrase
|
|
112
|
+
* must round-trip the codex matcher (makeAuNzPoBoxPhrase).
|
|
113
|
+
*/
|
|
85
114
|
const AU_LEADERS_CURRENT = ["PO Box", "P.O. Box", "Post Office Box", "GPO Box", "Locked Bag", "Private Bag"]
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
115
|
+
/**
|
|
116
|
+
* Codex legacy: true (recognize-only forms)
|
|
117
|
+
*/
|
|
118
|
+
const AU_LEADERS_LEGACY = ["RMB", "RSD", "CMB"]
|
|
119
|
+
/**
|
|
120
|
+
* NZ (#517): the ADV358 box/bag types that carry an identifier. CMB rides rare (its "CMB B99" identifier shape is
|
|
121
|
+
* alpha-led, covered by makeAuNzPoBoxPhrase). Counter Delivery / Poste Restante are identifier-less counter services —
|
|
122
|
+
* no number to learn, excluded from synthesis.
|
|
123
|
+
*/
|
|
90
124
|
const NZ_LEADERS_COMMON = ["PO Box", "Private Bag"]
|
|
91
125
|
const NZ_LEADERS_RARE = ["CMB"]
|
|
92
126
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
127
|
+
/**
|
|
128
|
+
* Canadian postcode synthesis: valid first letters per province from the codex FSA prior, interior letters per the
|
|
129
|
+
* codex pattern (excludes the visually ambiguous D F I O Q U). The LDU digits are random — the SHAPE is the training
|
|
130
|
+
* signal, not the (unknowable) live assignment.
|
|
131
|
+
*/
|
|
96
132
|
const QC_FSA_LETTERS = Object.entries(FSA_LETTER_TO_PROVINCE)
|
|
97
133
|
.filter(([, p]) => p === "QC")
|
|
98
|
-
.map(([l]) => l)
|
|
134
|
+
.map(([l]) => l)
|
|
135
|
+
|
|
136
|
+
// G H J
|
|
99
137
|
const ON_FSA_LETTERS = Object.entries(FSA_LETTER_TO_PROVINCE)
|
|
100
138
|
.filter(([, p]) => p === "ON")
|
|
101
|
-
.map(([l]) => l)
|
|
139
|
+
.map(([l]) => l)
|
|
140
|
+
|
|
141
|
+
// K L M N P
|
|
102
142
|
const CA_INTERIOR_LETTERS = "ABCEGHJKLMNPRSTVWXYZ"
|
|
103
143
|
|
|
104
|
-
|
|
105
|
-
|
|
144
|
+
/**
|
|
145
|
+
* Class mix — po_box mass leans US (the production arena), cedex gets a real block, and the CA-fr class exists because
|
|
146
|
+
* the #511 Montréal rows ("Case Postale 200, H3A 1B9 Montréal, QC") fail today.
|
|
147
|
+
*/
|
|
106
148
|
const CLASS_MIX: ReadonlyArray<[string, number]> = [
|
|
107
149
|
["po-box-us", 0.27],
|
|
108
150
|
["po-box-us-military", 0.05], // #517: CMR/PSC/Unit + Box, APO/FPO/DPO + AA/AE/AP — the arena's 0/3 class
|
|
@@ -115,7 +157,9 @@ const CLASS_MIX: ReadonlyArray<[string, number]> = [
|
|
|
115
157
|
["po-box-nz", 0.06],
|
|
116
158
|
]
|
|
117
159
|
|
|
118
|
-
/**
|
|
160
|
+
/**
|
|
161
|
+
* Synthetic recipient/venue prefixes — the arena's "JOHN DOE, ACME INC, …" pattern.
|
|
162
|
+
*/
|
|
119
163
|
const VENUES_EN = ["John Doe", "Jane Smith", "Acme Inc", "Wayne Enterprises", "Maria Garcia", "Riverside Clinic"]
|
|
120
164
|
const VENUES_FR = ["Société Dupont", "Cabinet Martin", "Hôpital Central", "Mairie Annexe", "Imprimerie Moderne"]
|
|
121
165
|
|
|
@@ -127,21 +171,25 @@ interface USTuple {
|
|
|
127
171
|
region: string
|
|
128
172
|
postcode: string
|
|
129
173
|
}
|
|
174
|
+
|
|
130
175
|
interface FrTuple {
|
|
131
176
|
house_number: string
|
|
132
177
|
street: string
|
|
133
178
|
locality: string
|
|
134
179
|
postcode: string
|
|
135
180
|
}
|
|
181
|
+
|
|
136
182
|
interface AuTuple {
|
|
137
183
|
locality: string
|
|
138
184
|
region: string
|
|
139
185
|
postcode: string
|
|
140
186
|
}
|
|
187
|
+
|
|
141
188
|
interface NzTuple {
|
|
142
189
|
locality: string
|
|
143
190
|
postcode: string
|
|
144
191
|
}
|
|
192
|
+
|
|
145
193
|
interface Rendered {
|
|
146
194
|
fmt: string
|
|
147
195
|
raw: string
|
|
@@ -150,7 +198,9 @@ interface Rendered {
|
|
|
150
198
|
|
|
151
199
|
// ── Holdout + CSV helpers ────────────────────────────────────────────────────────────────────────
|
|
152
200
|
|
|
153
|
-
/**
|
|
201
|
+
/**
|
|
202
|
+
* Stable locality hash for the FR/CA train↔golden split (djb2; hash%10===0 → golden-only).
|
|
203
|
+
*/
|
|
154
204
|
function localityHash(name: string): number {
|
|
155
205
|
let h = 5381
|
|
156
206
|
const s = name.toLowerCase()
|
|
@@ -161,9 +211,12 @@ function localityHash(name: string): number {
|
|
|
161
211
|
|
|
162
212
|
return h
|
|
163
213
|
}
|
|
214
|
+
|
|
164
215
|
const isHoldoutLocality = (name: string): boolean => localityHash(name) % 10 === 0
|
|
165
216
|
|
|
166
|
-
/**
|
|
217
|
+
/**
|
|
218
|
+
* Minimal RFC-4180-ish splitter (handles quoted fields).
|
|
219
|
+
*/
|
|
167
220
|
function splitCSV(line: string): string[] {
|
|
168
221
|
const out: string[] = []
|
|
169
222
|
let cur = ""
|
|
@@ -176,6 +229,7 @@ function splitCSV(line: string): string[] {
|
|
|
176
229
|
if (c === '"') {
|
|
177
230
|
if (line[i + 1] === '"') {
|
|
178
231
|
cur += '"'
|
|
232
|
+
|
|
179
233
|
i++
|
|
180
234
|
} else {
|
|
181
235
|
inQ = false
|
|
@@ -192,14 +246,20 @@ function splitCSV(line: string): string[] {
|
|
|
192
246
|
cur += c
|
|
193
247
|
}
|
|
194
248
|
}
|
|
249
|
+
|
|
195
250
|
out.push(cur)
|
|
196
251
|
|
|
197
252
|
return out
|
|
198
253
|
}
|
|
199
254
|
|
|
200
|
-
const
|
|
255
|
+
const MAX_LOCALITY_LENGTH = 40
|
|
256
|
+
|
|
257
|
+
const cleanLocality = (loc: string) =>
|
|
258
|
+
loc && loc.length <= MAX_LOCALITY_LENGTH && !/\d|,/.test(loc) && !/cedex/i.test(loc)
|
|
201
259
|
|
|
202
|
-
/**
|
|
260
|
+
/**
|
|
261
|
+
* Stream real US tuples (number/street/city/postcode) out of a cached OA zip.
|
|
262
|
+
*/
|
|
203
263
|
function readUsTuples(source: { zip: string; csv: string; region: string }): USTuple[] {
|
|
204
264
|
const r = spawnSync("unzip", ["-p", source.zip, source.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
|
|
205
265
|
|
|
@@ -208,15 +268,18 @@ function readUsTuples(source: { zip: string; csv: string; region: string }): UST
|
|
|
208
268
|
|
|
209
269
|
return []
|
|
210
270
|
}
|
|
271
|
+
|
|
211
272
|
const lines = r.stdout.toString("utf8").split(/\r?\n/)
|
|
212
273
|
|
|
213
274
|
if (lines.length < 2) return []
|
|
214
275
|
const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
|
|
215
276
|
const idx = (name: string) => header.indexOf(name)
|
|
277
|
+
|
|
216
278
|
const iNum = idx("number"),
|
|
217
279
|
iStreet = idx("street"),
|
|
218
280
|
iCity = idx("city"),
|
|
219
281
|
iPost = idx("postcode")
|
|
282
|
+
|
|
220
283
|
const get = (cells: string[], i: number) => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
|
|
221
284
|
const tuples: USTuple[] = []
|
|
222
285
|
const seen = new Set<string>()
|
|
@@ -228,8 +291,10 @@ function readUsTuples(source: { zip: string; csv: string; region: string }): UST
|
|
|
228
291
|
|
|
229
292
|
if (!cleanLocality(locality)) continue
|
|
230
293
|
const key = locality.toLowerCase()
|
|
294
|
+
|
|
231
295
|
const street = get(cells, iStreet),
|
|
232
296
|
house_number = get(cells, iNum)
|
|
297
|
+
|
|
233
298
|
// One tuple per (locality, street) pair keeps the pool varied without ballooning memory.
|
|
234
299
|
const pairKey = `${key}|${street}`.toLowerCase()
|
|
235
300
|
|
|
@@ -253,20 +318,23 @@ function readFrTuples(limit: number): FrTuple[] {
|
|
|
253
318
|
{ maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" }
|
|
254
319
|
)
|
|
255
320
|
|
|
256
|
-
if (r.status !== 0 && r.stdout.length
|
|
321
|
+
if (r.status !== 0 && !r.stdout.length) {
|
|
257
322
|
console.error(` WARN: unzip failed for ${FR_SOURCE.zip}`)
|
|
258
323
|
|
|
259
324
|
return []
|
|
260
325
|
}
|
|
326
|
+
|
|
261
327
|
const lines = r.stdout.toString("utf8").split(/\r?\n/)
|
|
262
328
|
|
|
263
329
|
if (lines.length < 2) return []
|
|
264
330
|
const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
|
|
265
331
|
const idx = (n: string) => header.indexOf(n)
|
|
332
|
+
|
|
266
333
|
const iNum = idx("number"),
|
|
267
334
|
iStreet = idx("street"),
|
|
268
335
|
iCity = idx("city"),
|
|
269
336
|
iPost = idx("postcode")
|
|
337
|
+
|
|
270
338
|
const get = (cells: string[], i: number) => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
|
|
271
339
|
const tuples: FrTuple[] = []
|
|
272
340
|
const seen = new Set<string>()
|
|
@@ -274,6 +342,7 @@ function readFrTuples(limit: number): FrTuple[] {
|
|
|
274
342
|
for (let li = 1; li < lines.length; li++) {
|
|
275
343
|
if (!lines[li]) continue
|
|
276
344
|
const cells = splitCSV(lines[li]!)
|
|
345
|
+
|
|
277
346
|
const locality = get(cells, iCity),
|
|
278
347
|
postcode = get(cells, iPost),
|
|
279
348
|
street = get(cells, iStreet),
|
|
@@ -331,6 +400,7 @@ function readPostalTuples(
|
|
|
331
400
|
|
|
332
401
|
return []
|
|
333
402
|
}
|
|
403
|
+
|
|
334
404
|
const tuples: Array<AuTuple | NzTuple> = []
|
|
335
405
|
const seen = new Set<string>()
|
|
336
406
|
const validPostcode = opts.withState ? isAuPostcode : isNzPostcode
|
|
@@ -357,7 +427,9 @@ function readPostalTuples(
|
|
|
357
427
|
|
|
358
428
|
// ── Rendering helpers ────────────────────────────────────────────────────────────────────────────
|
|
359
429
|
|
|
360
|
-
/**
|
|
430
|
+
/**
|
|
431
|
+
* Box-number distribution (mirrors the corpus defaultPickNumber bands: 70% are 1-3 digits).
|
|
432
|
+
*/
|
|
361
433
|
function pickBoxNumber(random: () => number): string {
|
|
362
434
|
const r = random()
|
|
363
435
|
|
|
@@ -367,10 +439,12 @@ function pickBoxNumber(random: () => number): string {
|
|
|
367
439
|
|
|
368
440
|
if (r < 0.95) return String(1000 + Math.floor(random() * 9000))
|
|
369
441
|
|
|
370
|
-
return String(
|
|
442
|
+
return String(10_000 + Math.floor(random() * 90_000))
|
|
371
443
|
}
|
|
372
444
|
|
|
373
|
-
/**
|
|
445
|
+
/**
|
|
446
|
+
* Case dial for the designator phrase: mostly template casing, sometimes UPPER, rarely lower.
|
|
447
|
+
*/
|
|
374
448
|
function caseDial(random: () => number, s: string): string {
|
|
375
449
|
const r = random()
|
|
376
450
|
|
|
@@ -381,13 +455,17 @@ function caseDial(random: () => number, s: string): string {
|
|
|
381
455
|
return s.toLowerCase()
|
|
382
456
|
}
|
|
383
457
|
|
|
384
|
-
|
|
385
|
-
|
|
458
|
+
/**
|
|
459
|
+
* Leaders the codex PO_BOX_RE genuinely covers (everything en-US except "POB"; PMB/"#" are the corpus's CMRA forms,
|
|
460
|
+
* outside USPS Pub-28 §29). Used to scope the isPOBox round-trip assertion.
|
|
461
|
+
*/
|
|
386
462
|
const CODEX_COVERED_LEADERS = new Set(
|
|
387
463
|
["PO Box", "P.O. Box", "P.O.Box", "PO BOX", "Post Office Box", "Box", ...US_LEADERS_RARE].map((l) => l.toLowerCase())
|
|
388
464
|
)
|
|
389
465
|
|
|
390
|
-
/**
|
|
466
|
+
/**
|
|
467
|
+
* Compose a po_box phrase. "#" joins without a space ("#500", the golden PMB variant).
|
|
468
|
+
*/
|
|
391
469
|
function makePoBoxPhrase(
|
|
392
470
|
random: () => number,
|
|
393
471
|
leaders: ReadonlyArray<string>,
|
|
@@ -398,6 +476,7 @@ function makePoBoxPhrase(
|
|
|
398
476
|
if (rareLeaders && random() < 0.1) {
|
|
399
477
|
leader = rareLeaders[Math.floor(random() * rareLeaders.length)]!
|
|
400
478
|
}
|
|
479
|
+
|
|
401
480
|
const num = maybeNoisifyBoxNumber(pickBoxNumber(random), random)
|
|
402
481
|
const phrase = leader === "#" ? `#${num}` : `${caseDial(random, leader)} ${num}`
|
|
403
482
|
|
|
@@ -418,6 +497,7 @@ function makePoBoxPhrase(
|
|
|
418
497
|
function makeCedex(random: () => number): string {
|
|
419
498
|
const r = random()
|
|
420
499
|
const word = r < 0.6 ? "CEDEX" : r < 0.9 ? "Cedex" : "cedex"
|
|
500
|
+
|
|
421
501
|
const phrase = (() => {
|
|
422
502
|
if (random() < 0.2) return word // "33077 BORDEAUX CEDEX" — un-numbered offices are common
|
|
423
503
|
const n = 1 + Math.floor(random() * 20)
|
|
@@ -448,12 +528,14 @@ function makeAuNzPoBoxPhrase(
|
|
|
448
528
|
if (rareLeaders && random() < 0.1) {
|
|
449
529
|
leader = rareLeaders[Math.floor(random() * rareLeaders.length)]!
|
|
450
530
|
}
|
|
531
|
+
|
|
451
532
|
let num = maybeNoisifyBoxNumber(pickBoxNumber(random), random)
|
|
452
533
|
|
|
453
534
|
// NZ CMB identifiers are alpha-led per the ADV358 example ("CMB B99").
|
|
454
535
|
if (leader === "CMB" && validate === isNzDeliveryService) {
|
|
455
536
|
num = `B${num}`
|
|
456
537
|
}
|
|
538
|
+
|
|
457
539
|
const phrase = `${caseDial(random, leader)} ${num}`
|
|
458
540
|
// The "clean id" shape differs per system: ADV358 identifiers carry no separators at all, the AU
|
|
459
541
|
// AMAS id (like the US one) tolerates dashes. Noisy ids outside the clean shape are exempt.
|
|
@@ -466,7 +548,9 @@ function makeAuNzPoBoxPhrase(
|
|
|
466
548
|
return phrase
|
|
467
549
|
}
|
|
468
550
|
|
|
469
|
-
/**
|
|
551
|
+
/**
|
|
552
|
+
* Synthesize a codex-valid Canadian postcode for a province's FSA letters ("H2X 3V4").
|
|
553
|
+
*/
|
|
470
554
|
function makeCaPostcode(random: () => number, fsaLetters: string[]): string {
|
|
471
555
|
const L = () => CA_INTERIOR_LETTERS[Math.floor(random() * CA_INTERIOR_LETTERS.length)]!
|
|
472
556
|
const D = () => String(Math.floor(random() * 10))
|
|
@@ -505,6 +589,7 @@ function renderPoBoxUs(random: () => number, t: USTuple): Rendered {
|
|
|
505
589
|
components: { venue: v, ...base, ...(pc ? { postcode: pc } : {}) },
|
|
506
590
|
}
|
|
507
591
|
}
|
|
592
|
+
|
|
508
593
|
// USPS label form: comma-less, all-caps ("PO BOX 123 BURLINGTON VT 05401").
|
|
509
594
|
const up = (s: string) => s.toUpperCase()
|
|
510
595
|
|
|
@@ -556,6 +641,7 @@ function renderBpFr(random: () => number, t: FrTuple): Rendered {
|
|
|
556
641
|
components: { po_box: phrase, postcode: pc, locality: locUp, cedex },
|
|
557
642
|
}
|
|
558
643
|
}
|
|
644
|
+
|
|
559
645
|
const v = pick(random, VENUES_FR)
|
|
560
646
|
|
|
561
647
|
return {
|
|
@@ -676,6 +762,7 @@ function renderNzPoBox(random: () => number, t: NzTuple): Rendered {
|
|
|
676
762
|
* `locality` before `region: "ON"` goes looking.
|
|
677
763
|
*/
|
|
678
764
|
const COMPONENT_ORDER = ["house_number", "street", "po_box", "venue", "locality", "postcode", "region", "cedex"]
|
|
765
|
+
|
|
679
766
|
function orderComponents(components: Record<string, string>): Record<string, string> {
|
|
680
767
|
const out: Record<string, string> = {}
|
|
681
768
|
|
|
@@ -690,6 +777,10 @@ function orderComponents(components: Record<string, string>): Record<string, str
|
|
|
690
777
|
return out
|
|
691
778
|
}
|
|
692
779
|
|
|
780
|
+
/**
|
|
781
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
782
|
+
* and `description` below for the surface form it generates.
|
|
783
|
+
*/
|
|
693
784
|
export const poBoxCedexRecipe: ShardRecipe = {
|
|
694
785
|
name: "po-box-cedex",
|
|
695
786
|
description: "PO box / CEDEX coverage rows (US/FR/CA/AU/NZ) — self-generated from cached OA + GeoNames pools",
|
|
@@ -697,7 +788,7 @@ export const poBoxCedexRecipe: ShardRecipe = {
|
|
|
697
788
|
options: [{ flag: "--golden", description: "Emit the leakage-safe holdout variant ({raw, components, country})" }],
|
|
698
789
|
async run(opts, write) {
|
|
699
790
|
const random = makeMulberry32(opts.seed)
|
|
700
|
-
const count = opts.count ??
|
|
791
|
+
const count = opts.count ?? 50_000
|
|
701
792
|
const source = opts.sourceName ?? "synth-po-box-cedex"
|
|
702
793
|
|
|
703
794
|
// US pool: VT only for golden, non-VT for train (the established geographic holdout).
|
|
@@ -705,36 +796,36 @@ export const poBoxCedexRecipe: ShardRecipe = {
|
|
|
705
796
|
|
|
706
797
|
for (const s of opts.golden ? [US_EVAL_SOURCE] : US_TRAIN_SOURCES) {
|
|
707
798
|
const t = readUsTuples(s)
|
|
799
|
+
|
|
708
800
|
console.error(` ${s.csv}: ${t.length} tuples`)
|
|
709
801
|
|
|
710
802
|
for (const x of t) {
|
|
711
803
|
usPool.push(x)
|
|
712
804
|
}
|
|
713
805
|
}
|
|
806
|
+
|
|
714
807
|
// FR + CA pools: stable locality-hash holdout (golden gets hash%10==0, train the rest).
|
|
715
|
-
const frAll = readFrTuples(
|
|
808
|
+
const frAll = readFrTuples(80_000)
|
|
716
809
|
const frPool = frAll.filter((t) => isHoldoutLocality(t.locality) === opts.golden)
|
|
810
|
+
|
|
717
811
|
console.error(` ${FR_SOURCE.csv}: ${frAll.length} tuples (${frPool.length} after holdout split)`)
|
|
812
|
+
|
|
718
813
|
const qcAll = readCaLocalities("10")
|
|
719
814
|
const onAll = readCaLocalities("08")
|
|
720
815
|
const qcPool = qcAll.filter((l) => isHoldoutLocality(l) === opts.golden)
|
|
721
816
|
const onPool = onAll.filter((l) => isHoldoutLocality(l) === opts.golden)
|
|
817
|
+
|
|
722
818
|
console.error(` GeoNames CA: QC ${qcAll.length}→${qcPool.length}, ON ${onAll.length}→${onPool.length}`)
|
|
819
|
+
|
|
723
820
|
// AU/NZ pools: same stable locality-hash holdout as FR/CA.
|
|
724
821
|
const auAll = readPostalTuples(GEONAMES_POSTAL_AU, { withState: true }) as AuTuple[]
|
|
725
822
|
const nzAll = readPostalTuples(GEONAMES_POSTAL_NZ, { withState: false }) as NzTuple[]
|
|
726
823
|
const auPool = auAll.filter((t) => isHoldoutLocality(t.locality) === opts.golden)
|
|
727
824
|
const nzPool = nzAll.filter((t) => isHoldoutLocality(t.locality) === opts.golden)
|
|
825
|
+
|
|
728
826
|
console.error(` GeoNames postal: AU ${auAll.length}→${auPool.length}, NZ ${nzAll.length}→${nzPool.length}`)
|
|
729
827
|
|
|
730
|
-
if (
|
|
731
|
-
usPool.length === 0 ||
|
|
732
|
-
frPool.length === 0 ||
|
|
733
|
-
qcPool.length === 0 ||
|
|
734
|
-
onPool.length === 0 ||
|
|
735
|
-
auPool.length === 0 ||
|
|
736
|
-
nzPool.length === 0
|
|
737
|
-
) {
|
|
828
|
+
if (!usPool.length || !frPool.length || !qcPool.length || !onPool.length || !auPool.length || !nzPool.length) {
|
|
738
829
|
throw new Error(
|
|
739
830
|
"A base pool is empty — check /tmp/oa-cache and /tmp/geonames-cache (CA.zip, AU-postal.zip, NZ-postal.zip)."
|
|
740
831
|
)
|
|
@@ -749,7 +840,7 @@ export const poBoxCedexRecipe: ShardRecipe = {
|
|
|
749
840
|
if (r < acc) return name
|
|
750
841
|
}
|
|
751
842
|
|
|
752
|
-
return CLASS_MIX
|
|
843
|
+
return CLASS_MIX.at(-1)![0]
|
|
753
844
|
}
|
|
754
845
|
|
|
755
846
|
let emitted = 0
|
|
@@ -806,19 +897,24 @@ export const poBoxCedexRecipe: ShardRecipe = {
|
|
|
806
897
|
country = "CA"
|
|
807
898
|
locale = "en-CA"
|
|
808
899
|
}
|
|
900
|
+
|
|
809
901
|
const { raw, components } = rendered
|
|
810
902
|
|
|
811
903
|
// Every component surface must survive verbatim in raw, else alignment can't label it.
|
|
812
904
|
if (!Object.values(components).every((s) => raw.includes(s))) {
|
|
813
905
|
skipped++
|
|
906
|
+
|
|
814
907
|
continue
|
|
815
908
|
}
|
|
816
909
|
|
|
817
910
|
if (opts.golden) {
|
|
818
911
|
write(JSON.stringify({ raw, components: orderComponents(components), country }) + "\n")
|
|
912
|
+
|
|
819
913
|
emitted++
|
|
914
|
+
|
|
820
915
|
continue
|
|
821
916
|
}
|
|
917
|
+
|
|
822
918
|
const canonical: CanonicalShardRow = {
|
|
823
919
|
raw,
|
|
824
920
|
components: orderComponents(components),
|
|
@@ -838,13 +934,17 @@ export const poBoxCedexRecipe: ShardRecipe = {
|
|
|
838
934
|
? "GeoNames NZ postal dump (CC-BY 4.0) locality/postcode tails + NZ Post ADV358 Delivery Service Types (@mailwoman/codex/nz)"
|
|
839
935
|
: "OpenAddresses US (non-VT) skeletons + USPS Pub-28 §29 PO-box designators (codex/corpus templates)",
|
|
840
936
|
}
|
|
937
|
+
|
|
841
938
|
const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
|
|
842
939
|
|
|
843
940
|
if (aligned.kind !== "labeled" || !aligned.row) {
|
|
844
941
|
skipped++
|
|
942
|
+
|
|
845
943
|
continue
|
|
846
944
|
}
|
|
945
|
+
|
|
847
946
|
write(JSON.stringify({ ...aligned.row, synth_method: cls, synth_base_id: null }) + "\n")
|
|
947
|
+
|
|
848
948
|
emitted++
|
|
849
949
|
}
|
|
850
950
|
|
|
@@ -14,6 +14,10 @@ import { alignAndWrite, makeLcg, readTuples, shardSourceID, type ShardRecipe } f
|
|
|
14
14
|
|
|
15
15
|
const LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"
|
|
16
16
|
|
|
17
|
+
/**
|
|
18
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
19
|
+
* and `description` below for the surface form it generates.
|
|
20
|
+
*/
|
|
17
21
|
export const poBoxRecipe: ShardRecipe = {
|
|
18
22
|
name: "po-box",
|
|
19
23
|
description: "PO box rows: tuples → synthesizePoBoxRow (+ optional US military/diplomatic rows)",
|
|
@@ -38,6 +42,7 @@ export const poBoxRecipe: ShardRecipe = {
|
|
|
38
42
|
|
|
39
43
|
if (!tuple.locality || !tuple.postcode || !tuple.country || (!tuple.region && !regionOptional)) {
|
|
40
44
|
skipped++
|
|
45
|
+
|
|
41
46
|
continue
|
|
42
47
|
}
|
|
43
48
|
|
|
@@ -45,6 +50,7 @@ export const poBoxRecipe: ShardRecipe = {
|
|
|
45
50
|
const synth = synthesizePoBoxRow(tuple as PoBoxBaseTuple, { random, pmbRatio })
|
|
46
51
|
|
|
47
52
|
if (!synth) continue
|
|
53
|
+
|
|
48
54
|
const ok = alignAndWrite(
|
|
49
55
|
write,
|
|
50
56
|
{
|
|
@@ -77,6 +83,7 @@ export const poBoxRecipe: ShardRecipe = {
|
|
|
77
83
|
// Default 0 → byte-stable (random() not called when off). US-only.
|
|
78
84
|
if (militaryRatio > 0 && random() < militaryRatio) {
|
|
79
85
|
const mil = synthesizeMilitaryPoBoxRow({ random })
|
|
86
|
+
|
|
80
87
|
const ok = alignAndWrite(
|
|
81
88
|
write,
|
|
82
89
|
{
|