@mailwoman/corpus 8.0.0 → 8.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/adapter.d.ts +15 -5
- package/out/src/adapter.d.ts.map +1 -1
- package/out/src/adapter.js +11 -9
- package/out/src/adapter.js.map +1 -1
- package/out/src/adapters/ban/adapter.d.ts +7 -0
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +9 -2
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +5 -1
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +14 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +15 -2
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +11 -0
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +24 -5
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +11 -0
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +14 -1
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +10 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +11 -2
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts +18 -6
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +16 -6
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +11 -0
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +18 -3
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +11 -0
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +12 -1
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +11 -0
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +11 -0
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +15 -2
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +11 -0
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +13 -2
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +14 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +19 -4
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +12 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +17 -2
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +12 -1
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +11 -0
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +15 -2
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +7 -0
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +18 -3
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +13 -3
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +15 -5
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +7 -0
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +9 -2
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/align.d.ts +12 -4
- package/out/src/align.d.ts.map +1 -1
- package/out/src/align.js +9 -5
- package/out/src/align.js.map +1 -1
- package/out/src/build.d.ts +24 -8
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +3 -3
- package/out/src/build.js.map +1 -1
- package/out/src/codex/us-fips-state.d.ts +6 -2
- package/out/src/codex/us-fips-state.d.ts.map +1 -1
- package/out/src/codex/us-fips-state.js +3 -1
- package/out/src/codex/us-fips-state.js.map +1 -1
- package/out/src/golden.d.ts +21 -7
- package/out/src/golden.d.ts.map +1 -1
- package/out/src/golden.js +18 -10
- package/out/src/golden.js.map +1 -1
- package/out/src/license.d.ts +3 -1
- package/out/src/license.d.ts.map +1 -1
- package/out/src/license.js +4 -2
- package/out/src/license.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts +4 -2
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +5 -4
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/schema.d.ts +14 -10
- package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
- package/out/src/parquet-wrapper/schema.js +6 -2
- package/out/src/parquet-wrapper/schema.js.map +1 -1
- package/out/src/parquet-wrapper/writer.d.ts +12 -4
- package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
- package/out/src/parquet-wrapper/writer.js +12 -4
- package/out/src/parquet-wrapper/writer.js.map +1 -1
- package/out/src/parquet.d.ts +21 -7
- package/out/src/parquet.d.ts.map +1 -1
- package/out/src/parquet.js +15 -4
- package/out/src/parquet.js.map +1 -1
- package/out/src/runner.d.ts +39 -13
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +7 -3
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts +4 -0
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +14 -5
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/boundary-stress.d.ts +4 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +13 -7
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts +4 -0
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +39 -11
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts +4 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +14 -3
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts +4 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.js +4 -0
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts +4 -0
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +25 -8
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts +4 -0
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +21 -8
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts +4 -0
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +27 -8
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts +4 -0
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +16 -6
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.d.ts +4 -0
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/house-venue.js +4 -0
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts +9 -3
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +12 -4
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts +4 -0
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +27 -13
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +4 -0
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +9 -4
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/nl-postcode.d.ts +4 -0
- package/out/src/shard-recipes/nl-postcode.d.ts.map +1 -1
- package/out/src/shard-recipes/nl-postcode.js +8 -2
- package/out/src/shard-recipes/nl-postcode.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts +4 -0
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +8 -2
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts +4 -0
- package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street-led.js +5 -1
- package/out/src/shard-recipes/no-street-led.js.map +1 -1
- package/out/src/shard-recipes/no-street.d.ts +4 -0
- package/out/src/shard-recipes/no-street.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street.js +4 -0
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +4 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +99 -43
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts +4 -0
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +4 -0
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +48 -16
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +10 -6
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/si-bare-village.d.ts +4 -0
- package/out/src/shard-recipes/si-bare-village.d.ts.map +1 -1
- package/out/src/shard-recipes/si-bare-village.js +4 -0
- package/out/src/shard-recipes/si-bare-village.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +4 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +29 -10
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.d.ts +4 -0
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
- package/out/src/shard-recipes/street-bare.js +4 -0
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.d.ts +4 -0
- package/out/src/shard-recipes/street.d.ts.map +1 -1
- package/out/src/shard-recipes/street.js +4 -0
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/unit.d.ts +4 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +45 -14
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/split.d.ts +18 -6
- package/out/src/split.d.ts.map +1 -1
- package/out/src/split.js +4 -2
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize-anchor-absorption.d.ts +7 -1
- package/out/src/synthesize-anchor-absorption.d.ts.map +1 -1
- package/out/src/synthesize-anchor-absorption.js +28 -14
- package/out/src/synthesize-anchor-absorption.js.map +1 -1
- package/out/src/synthesize-boundary-stress.d.ts +3 -1
- package/out/src/synthesize-boundary-stress.d.ts.map +1 -1
- package/out/src/synthesize-boundary-stress.js +35 -28
- package/out/src/synthesize-boundary-stress.js.map +1 -1
- package/out/src/synthesize-german.d.ts +15 -5
- package/out/src/synthesize-german.d.ts.map +1 -1
- package/out/src/synthesize-german.js +9 -3
- package/out/src/synthesize-german.js.map +1 -1
- package/out/src/synthesize-house-venue.d.ts.map +1 -1
- package/out/src/synthesize-house-venue.js +15 -15
- package/out/src/synthesize-house-venue.js.map +1 -1
- package/out/src/synthesize-intersection.d.ts +9 -3
- package/out/src/synthesize-intersection.d.ts.map +1 -1
- package/out/src/synthesize-intersection.js +25 -6
- package/out/src/synthesize-intersection.js.map +1 -1
- package/out/src/synthesize-no-street.d.ts +3 -1
- package/out/src/synthesize-no-street.d.ts.map +1 -1
- package/out/src/synthesize-no-street.js +5 -6
- package/out/src/synthesize-no-street.js.map +1 -1
- package/out/src/synthesize-po-box.d.ts +15 -5
- package/out/src/synthesize-po-box.d.ts.map +1 -1
- package/out/src/synthesize-po-box.js +21 -7
- package/out/src/synthesize-po-box.js.map +1 -1
- package/out/src/synthesize-street.d.ts +3 -1
- package/out/src/synthesize-street.d.ts.map +1 -1
- package/out/src/synthesize-street.js +6 -1
- package/out/src/synthesize-street.js.map +1 -1
- package/out/src/synthesize.d.ts +39 -13
- package/out/src/synthesize.d.ts.map +1 -1
- package/out/src/synthesize.js +66 -36
- package/out/src/synthesize.js.map +1 -1
- package/out/src/tokenize.d.ts +15 -5
- package/out/src/tokenize.d.ts.map +1 -1
- package/out/src/tokenize.js.map +1 -1
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +3 -1
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/audit.d.ts.map +1 -1
- package/out/src/tools/audit.js +20 -8
- package/out/src/tools/audit.js.map +1 -1
- package/out/src/tools/corpus-stats.d.ts.map +1 -1
- package/out/src/tools/corpus-stats.js.map +1 -1
- package/out/src/tools/fetch/ban.d.ts.map +1 -1
- package/out/src/tools/fetch/ban.js +17 -10
- package/out/src/tools/fetch/ban.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts +30 -10
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +26 -6
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/hrsa.d.ts.map +1 -1
- package/out/src/tools/fetch/hrsa.js.map +1 -1
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +18 -18
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +3 -1
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +3 -1
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts +21 -7
- package/out/src/tools/fetch/nad.d.ts.map +1 -1
- package/out/src/tools/fetch/nad.js +5 -5
- package/out/src/tools/fetch/nad.js.map +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +9 -15
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts +3 -1
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/src/tools/fetch/openaddresses.js +32 -23
- package/out/src/tools/fetch/openaddresses.js.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +3 -1
- package/out/src/tools/fetch/ppd.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.js +14 -3
- package/out/src/tools/fetch/ppd.js.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.js +13 -6
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
- package/out/src/tools/fetch/state-sources.js +8 -3
- package/out/src/tools/fetch/state-sources.js.map +1 -1
- package/out/src/tools/fetch/tiger-full.d.ts +9 -3
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
- package/out/src/tools/fetch/tiger-full.js +35 -20
- package/out/src/tools/fetch/tiger-full.js.map +1 -1
- package/out/src/tools/gb-title-case.d.ts.map +1 -1
- package/out/src/tools/gb-title-case.js.map +1 -1
- package/out/src/tools/golden-expand.d.ts +24 -8
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +20 -12
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-promote.d.ts +18 -6
- package/out/src/tools/golden-promote.d.ts.map +1 -1
- package/out/src/tools/golden-promote.js +22 -7
- package/out/src/tools/golden-promote.js.map +1 -1
- package/out/src/tools/ingest-csv.d.ts +3 -1
- package/out/src/tools/ingest-csv.d.ts.map +1 -1
- package/out/src/tools/ingest-csv.js +17 -19
- package/out/src/tools/ingest-csv.js.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts +18 -6
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
- package/out/src/tools/jsonl-to-parquet.js +11 -5
- package/out/src/tools/jsonl-to-parquet.js.map +1 -1
- package/out/src/tools/lint-shard-vocab.d.ts +36 -12
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
- package/out/src/tools/lint-shard-vocab.js +25 -11
- package/out/src/tools/lint-shard-vocab.js.map +1 -1
- package/out/src/tools/lint-shard.d.ts +30 -10
- package/out/src/tools/lint-shard.d.ts.map +1 -1
- package/out/src/tools/lint-shard.js +33 -20
- package/out/src/tools/lint-shard.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +6 -4
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/shard-kryptonite.d.ts +6 -2
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +1 -1
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts +9 -3
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +2 -2
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/types.d.ts +51 -17
- package/out/src/types.d.ts.map +1 -1
- package/out/src/wof-json.d.ts +9 -3
- package/out/src/wof-json.d.ts.map +1 -1
- package/out/src/wof-json.js +6 -4
- package/out/src/wof-json.js.map +1 -1
- package/package.json +7 -6
- package/src/adapter.ts +24 -13
- package/src/adapters/ban/adapter.ts +12 -2
- package/src/adapters/ban/street-decompose.ts +7 -1
- package/src/adapters/fcc-bdc/adapter.ts +19 -3
- package/src/adapters/geonames/adapter.ts +28 -5
- package/src/adapters/geonames-postal/adapter.ts +17 -1
- package/src/adapters/gnaf/adapter.ts +15 -2
- package/src/adapters/gnaf/assemble.ts +40 -10
- package/src/adapters/index.ts +13 -0
- package/src/adapters/openaddresses/adapter.ts +22 -3
- package/src/adapters/overture/adapter.ts +18 -2
- package/src/adapters/state-hi-schools/adapter.ts +15 -0
- package/src/adapters/state-ia-contractors/adapter.ts +15 -0
- package/src/adapters/state-ny-notaries/adapter.ts +15 -0
- package/src/adapters/state-tx-notaries/adapter.ts +18 -2
- package/src/adapters/synth-po-box/adapter.ts +19 -2
- package/src/adapters/tiger/adapter.ts +23 -4
- package/src/adapters/tiger/street-decompose.ts +1 -0
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +18 -2
- package/src/adapters/usgov-imls-pls/adapter.ts +15 -0
- package/src/adapters/usgov-irs-bmf/adapter.ts +19 -2
- package/src/adapters/usgov-nad/adapter.ts +14 -1
- package/src/adapters/usgov-nppes/adapter.ts +13 -0
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +21 -3
- package/src/adapters/wof-admin-jp/adapter.ts +20 -3
- package/src/adapters/wof-admin-json/adapter.ts +22 -7
- package/src/adapters/wof-postalcode-json/adapter.ts +10 -2
- package/src/align.ts +21 -8
- package/src/build.ts +48 -11
- package/src/codex/us-fips-state.ts +6 -2
- package/src/golden.ts +28 -13
- package/src/license.ts +4 -2
- package/src/parquet-wrapper/reader.ts +6 -4
- package/src/parquet-wrapper/schema.ts +14 -10
- package/src/parquet-wrapper/writer.ts +12 -4
- package/src/parquet.ts +39 -8
- package/src/runner.ts +49 -15
- package/src/shard-recipes/anchor-absorption.ts +21 -5
- package/src/shard-recipes/boundary-stress.ts +20 -7
- package/src/shard-recipes/country-balanced.ts +78 -13
- package/src/shard-recipes/cz-pcfirst-preposition.ts +9 -0
- package/src/shard-recipes/fr-admin-split.ts +30 -5
- package/src/shard-recipes/fr-bare-street.ts +10 -0
- package/src/shard-recipes/fr-fragment.ts +33 -8
- package/src/shard-recipes/fr-lieudit.ts +35 -9
- package/src/shard-recipes/fr-order.ts +47 -9
- package/src/shard-recipes/german.ts +32 -6
- package/src/shard-recipes/house-venue.ts +7 -0
- package/src/shard-recipes/index.ts +12 -4
- package/src/shard-recipes/intersection.ts +63 -14
- package/src/shard-recipes/locale.ts +37 -4
- package/src/shard-recipes/nl-postcode.ts +12 -2
- package/src/shard-recipes/no-fragment.ts +12 -2
- package/src/shard-recipes/no-street-led.ts +11 -1
- package/src/shard-recipes/no-street.ts +7 -0
- package/src/shard-recipes/po-box-cedex.ts +145 -45
- package/src/shard-recipes/po-box.ts +7 -0
- package/src/shard-recipes/scaffold.ts +52 -20
- package/src/shard-recipes/si-bare-village.ts +9 -0
- package/src/shard-recipes/street-affix.ts +88 -13
- package/src/shard-recipes/street-bare.ts +10 -0
- package/src/shard-recipes/street.ts +7 -0
- package/src/shard-recipes/unit.ts +79 -16
- package/src/split.ts +28 -7
- package/src/synthesize-anchor-absorption.ts +41 -14
- package/src/synthesize-boundary-stress.ts +57 -30
- package/src/synthesize-german.ts +30 -7
- package/src/synthesize-house-venue.ts +22 -16
- package/src/synthesize-intersection.ts +34 -7
- package/src/synthesize-no-street.ts +18 -10
- package/src/synthesize-po-box.ts +42 -10
- package/src/synthesize-street.ts +10 -2
- package/src/synthesize.ts +81 -38
- package/src/tokenize.ts +15 -5
- package/src/tools/align-shard.ts +6 -1
- package/src/tools/audit.ts +57 -11
- package/src/tools/corpus-stats.ts +9 -0
- package/src/tools/fetch/ban.ts +32 -10
- package/src/tools/fetch/download.ts +49 -12
- package/src/tools/fetch/hrsa.ts +3 -0
- package/src/tools/fetch/imls-pls.ts +28 -18
- package/src/tools/fetch/index.ts +3 -1
- package/src/tools/fetch/nad.ts +39 -12
- package/src/tools/fetch/nppes.ts +17 -15
- package/src/tools/fetch/openaddresses.ts +52 -24
- package/src/tools/fetch/ppd.ts +22 -3
- package/src/tools/fetch/state-hi-schools.ts +17 -6
- package/src/tools/fetch/state-sources.ts +16 -3
- package/src/tools/fetch/tiger-full.ts +66 -23
- package/src/tools/gb-title-case.ts +1 -0
- package/src/tools/golden-expand.ts +64 -20
- package/src/tools/golden-promote.ts +53 -13
- package/src/tools/ingest-csv.ts +41 -26
- package/src/tools/jsonl-to-parquet.ts +30 -10
- package/src/tools/lint-shard-vocab.ts +75 -24
- package/src/tools/lint-shard.ts +75 -31
- package/src/tools/overlay-manifest.ts +7 -4
- package/src/tools/shard-kryptonite.ts +11 -3
- package/src/tools/shard-translit.ts +20 -5
- package/src/types.ts +51 -17
- package/src/wof-json.ts +18 -7
|
@@ -21,7 +21,9 @@ export function shardSourceID(adapterID: string, parts: Record<string, string |
|
|
|
21
21
|
return stableSourceID(adapterID, parts as unknown as Parameters<typeof stableSourceID>[1])
|
|
22
22
|
}
|
|
23
23
|
|
|
24
|
-
/**
|
|
24
|
+
/**
|
|
25
|
+
* A (locality, region, postcode, country) source tuple — the input to tuples-mode recipes.
|
|
26
|
+
*/
|
|
25
27
|
export interface ShardTuple {
|
|
26
28
|
locality?: string
|
|
27
29
|
region?: string
|
|
@@ -38,13 +40,15 @@ export function makeLcg(seed: number): () => number {
|
|
|
38
40
|
let s = seed >>> 0
|
|
39
41
|
|
|
40
42
|
return () => {
|
|
41
|
-
s = (s *
|
|
43
|
+
s = (s * 1_664_525 + 1_013_904_223) % 4_294_967_296
|
|
42
44
|
|
|
43
|
-
return s /
|
|
45
|
+
return s / 4_294_967_296
|
|
44
46
|
}
|
|
45
47
|
}
|
|
46
48
|
|
|
47
|
-
/**
|
|
49
|
+
/**
|
|
50
|
+
* Back-compat alias for {@link makeLcg}.
|
|
51
|
+
*/
|
|
48
52
|
export const makeRandom = makeLcg
|
|
49
53
|
|
|
50
54
|
/**
|
|
@@ -56,15 +60,17 @@ export function makeMulberry32(seed: number): () => number {
|
|
|
56
60
|
let a = seed >>> 0
|
|
57
61
|
|
|
58
62
|
return () => {
|
|
59
|
-
a = (a +
|
|
63
|
+
a = (a + 0x6d_2b_79_f5) | 0
|
|
60
64
|
let t = Math.imul(a ^ (a >>> 15), 1 | a)
|
|
61
65
|
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t
|
|
62
66
|
|
|
63
|
-
return ((t ^ (t >>> 14)) >>> 0) /
|
|
67
|
+
return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296
|
|
64
68
|
}
|
|
65
69
|
}
|
|
66
70
|
|
|
67
|
-
/**
|
|
71
|
+
/**
|
|
72
|
+
* Stream-parse a tuples JSONL file, yielding each parsed object (blank/invalid lines skipped).
|
|
73
|
+
*/
|
|
68
74
|
export async function* readTuples(input: string): AsyncGenerator<ShardTuple> {
|
|
69
75
|
// TextSpliterator (not JSONSpliterator) so a malformed line is SKIPPED, not thrown — the
|
|
70
76
|
// per-line try/catch below is the tolerance this reader has always had.
|
|
@@ -83,7 +89,9 @@ export async function* readTuples(input: string): AsyncGenerator<ShardTuple> {
|
|
|
83
89
|
}
|
|
84
90
|
}
|
|
85
91
|
|
|
86
|
-
/**
|
|
92
|
+
/**
|
|
93
|
+
* A canonical row as the recipes assemble it, before `alignRow` turns it into a `LabeledRow`.
|
|
94
|
+
*/
|
|
87
95
|
export interface CanonicalShardRow {
|
|
88
96
|
raw: string
|
|
89
97
|
components: Record<string, string>
|
|
@@ -113,7 +121,9 @@ export function alignAndWrite(
|
|
|
113
121
|
return true
|
|
114
122
|
}
|
|
115
123
|
|
|
116
|
-
/**
|
|
124
|
+
/**
|
|
125
|
+
* Parsed options a recipe's `run` receives. Common fields + the union of recipe-specific flags.
|
|
126
|
+
*/
|
|
117
127
|
export interface ShardRecipeOpts {
|
|
118
128
|
output: string
|
|
119
129
|
seed: number
|
|
@@ -130,7 +140,9 @@ export interface ShardRecipeOpts {
|
|
|
130
140
|
edgesDir?: string
|
|
131
141
|
country?: string
|
|
132
142
|
intlFraction?: number
|
|
133
|
-
/**
|
|
143
|
+
/**
|
|
144
|
+
* `locale`: fraction of rows that append an explicit country surface form + a `country` component. Default 0.
|
|
145
|
+
*/
|
|
134
146
|
countryFraction?: number
|
|
135
147
|
/**
|
|
136
148
|
* `locale`: tri-state override of the per-part `districtAsLocality` mapping for this invocation. `undefined` (flag
|
|
@@ -143,7 +155,9 @@ export interface ShardRecipeOpts {
|
|
|
143
155
|
bareProb?: number
|
|
144
156
|
hnProb?: number
|
|
145
157
|
communes?: string
|
|
146
|
-
/**
|
|
158
|
+
/**
|
|
159
|
+
* `fr-lieudit`: BAN `adresses-<dept>.csv` directory. Default `$MAILWOMAN_DATA_ROOT/corpus/sources/ban`.
|
|
160
|
+
*/
|
|
147
161
|
banDir?: string
|
|
148
162
|
multilocaleCount?: number
|
|
149
163
|
/**
|
|
@@ -151,18 +165,24 @@ export interface ShardRecipeOpts {
|
|
|
151
165
|
* recipes — a shard that trains on its own eval set measures memorization. See their docstrings.
|
|
152
166
|
*/
|
|
153
167
|
excludeSurfaces?: string
|
|
154
|
-
/**
|
|
168
|
+
/**
|
|
169
|
+
* `no-fragment`: share of rows that are counter-distribution (bare locality OR bare postcode).
|
|
170
|
+
*/
|
|
155
171
|
counterProb?: number
|
|
156
172
|
/**
|
|
157
173
|
* `no-fragment` knob 3: emit N copies of each street+number row whose number has >= longNumberMinDigits digits
|
|
158
174
|
* (oversample the failing long-number class). Default 1 = no boost.
|
|
159
175
|
*/
|
|
160
176
|
longNumberBoost?: number
|
|
161
|
-
/**
|
|
177
|
+
/**
|
|
178
|
+
* `no-fragment` knob 3: minimum digit count for a number to count as "long" and be boosted. Default 3.
|
|
179
|
+
*/
|
|
162
180
|
longNumberMinDigits?: number
|
|
163
181
|
}
|
|
164
182
|
|
|
165
|
-
/**
|
|
183
|
+
/**
|
|
184
|
+
* Tally a recipe returns.
|
|
185
|
+
*/
|
|
166
186
|
export interface ShardStats {
|
|
167
187
|
read?: number
|
|
168
188
|
emitted: number
|
|
@@ -175,21 +195,33 @@ export interface ShardStats {
|
|
|
175
195
|
contaminated?: number
|
|
176
196
|
}
|
|
177
197
|
|
|
178
|
-
/**
|
|
198
|
+
/**
|
|
199
|
+
* A single declared recipe-specific option flag (for the command's --help).
|
|
200
|
+
*/
|
|
179
201
|
export interface ShardRecipeOption {
|
|
180
202
|
flag: string
|
|
181
203
|
description: string
|
|
182
204
|
}
|
|
183
205
|
|
|
184
|
-
/**
|
|
206
|
+
/**
|
|
207
|
+
* A shard recipe: its identity, input mode, and its synthesis `run`.
|
|
208
|
+
*/
|
|
185
209
|
export interface ShardRecipe {
|
|
186
|
-
/**
|
|
210
|
+
/**
|
|
211
|
+
* Recipe id, e.g. "street", "po-box" — the `<recipe>` positional.
|
|
212
|
+
*/
|
|
187
213
|
name: string
|
|
188
|
-
/**
|
|
214
|
+
/**
|
|
215
|
+
* One-line description for `--list` / help.
|
|
216
|
+
*/
|
|
189
217
|
description: string
|
|
190
|
-
/**
|
|
218
|
+
/**
|
|
219
|
+
* `tuples` reads `--input` JSONL; `generate` self-generates `--count` rows.
|
|
220
|
+
*/
|
|
191
221
|
mode: "tuples" | "generate"
|
|
192
|
-
/**
|
|
222
|
+
/**
|
|
223
|
+
* Recipe-specific flags this recipe honors (documentation only).
|
|
224
|
+
*/
|
|
193
225
|
options?: ShardRecipeOption[]
|
|
194
226
|
/**
|
|
195
227
|
* Do the build: create the recipe's PRNG from `opts.seed` (its LEGACY generator — `makeLcg` or `makeMulberry32` — for
|
|
@@ -26,6 +26,10 @@
|
|
|
26
26
|
|
|
27
27
|
import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
|
|
28
28
|
|
|
29
|
+
/**
|
|
30
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
31
|
+
* and `description` below for the surface form it generates.
|
|
32
|
+
*/
|
|
29
33
|
export const siBareVillageRecipe: ShardRecipe = {
|
|
30
34
|
name: "si-bare-village",
|
|
31
35
|
description: "SI no-street village form (#901 run-2): '«V» «n», «pc» «V»' — the fr-bare-street counter-distribution",
|
|
@@ -44,10 +48,13 @@ export const siBareVillageRecipe: ShardRecipe = {
|
|
|
44
48
|
|
|
45
49
|
if (!village || !number || !postcode) {
|
|
46
50
|
skipped++
|
|
51
|
+
|
|
47
52
|
continue
|
|
48
53
|
}
|
|
54
|
+
|
|
49
55
|
const order = read % 3
|
|
50
56
|
let raw: string
|
|
57
|
+
|
|
51
58
|
const components: Record<string, string> = {
|
|
52
59
|
street: village,
|
|
53
60
|
house_number: number,
|
|
@@ -63,7 +70,9 @@ export const siBareVillageRecipe: ShardRecipe = {
|
|
|
63
70
|
components.postcode = postcode
|
|
64
71
|
raw = `${postcode} ${village}, ${village} ${number}`
|
|
65
72
|
}
|
|
73
|
+
|
|
66
74
|
const source_id = shardSourceID("synth-si-bare-village", { ...components, o: String(order), v: String(read) })
|
|
75
|
+
|
|
67
76
|
const canonical = {
|
|
68
77
|
raw,
|
|
69
78
|
components,
|
|
@@ -46,11 +46,18 @@ import type { CanonicalRow } from "../types.ts"
|
|
|
46
46
|
import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
|
|
47
47
|
|
|
48
48
|
// Same OA cache as the unit shard. Train = every NON-Vermont state; eval = Vermont (the holdout).
|
|
49
|
+
/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
|
|
50
|
+
cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
|
|
51
|
+
output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
|
|
52
|
+
would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
|
|
53
|
+
extracted as named constants above. */
|
|
54
|
+
|
|
49
55
|
interface USSource {
|
|
50
56
|
zip: string
|
|
51
57
|
csv: string
|
|
52
58
|
region: string
|
|
53
59
|
}
|
|
60
|
+
|
|
54
61
|
const TRAIN_SOURCES: readonly USSource[] = [
|
|
55
62
|
{ zip: "/tmp/oa-cache/us__ca__berkeley.zip", csv: "us/ca/berkeley.csv", region: "CA" },
|
|
56
63
|
{ zip: "/tmp/oa-cache/us__ca__marin.zip", csv: "us/ca/marin.csv", region: "CA" },
|
|
@@ -60,6 +67,7 @@ const TRAIN_SOURCES: readonly USSource[] = [
|
|
|
60
67
|
{ zip: "/tmp/oa-cache/us__mt__statewide.zip", csv: "us/mt/statewide.csv", region: "MT" },
|
|
61
68
|
{ zip: "/tmp/oa-cache/us__sd__statewide.zip", csv: "us/sd/statewide.csv", region: "SD" },
|
|
62
69
|
]
|
|
70
|
+
|
|
63
71
|
const EVAL_SOURCE: USSource = { zip: "/tmp/oa-cache/us__vt__statewide.zip", csv: "us/vt/statewide.csv", region: "VT" }
|
|
64
72
|
|
|
65
73
|
// Multi-locale BALANCE sources (--multilocale-count > 0). These rows carry NO affix split — they exist
|
|
@@ -73,20 +81,30 @@ interface BalanceSource {
|
|
|
73
81
|
region: string
|
|
74
82
|
order: string
|
|
75
83
|
}
|
|
84
|
+
|
|
76
85
|
const MULTILOCALE_SOURCES: readonly BalanceSource[] = [
|
|
77
86
|
{ zip: "/tmp/oa-cache/de__sn__statewide.zip", csv: "de/sn/statewide.csv", iso2: "DE", region: "", order: "eu" },
|
|
78
87
|
{ zip: "/tmp/oa-cache/fr__countrywide.zip", csv: "fr/countrywide.csv", iso2: "FR", region: "", order: "fr" },
|
|
79
88
|
{ zip: "/tmp/oa-cache/it__countrywide.zip", csv: "it/countrywide.csv", iso2: "IT", region: "", order: "eu" },
|
|
80
89
|
{ zip: "/tmp/oa-cache/nl__countrywide.zip", csv: "nl/countrywide.csv", iso2: "NL", region: "", order: "eu" },
|
|
81
90
|
]
|
|
91
|
+
|
|
82
92
|
const MULTILOCALE_EVAL_SOURCES: readonly BalanceSource[] = [
|
|
83
93
|
{ zip: "/tmp/oa-cache/de__berlin.zip", csv: "de/berlin.csv", iso2: "DE", region: "", order: "eu" },
|
|
84
94
|
]
|
|
85
95
|
|
|
86
|
-
|
|
87
|
-
|
|
96
|
+
/**
|
|
97
|
+
* ["N","E","S","W","NE","NW","SE","SW"].
|
|
98
|
+
*/
|
|
99
|
+
const DIRECTIONAL_ABBRS = Object.values(DirectionalAbbreviation)
|
|
100
|
+
/**
|
|
101
|
+
* Fraction of prefix-less streets that get a synthetic directional.
|
|
102
|
+
*/
|
|
103
|
+
const INJECT_PREFIX_PROB = 0.3
|
|
88
104
|
|
|
89
|
-
/**
|
|
105
|
+
/**
|
|
106
|
+
* A real US skeleton tuple read from a cached OA zip.
|
|
107
|
+
*/
|
|
90
108
|
interface USTuple {
|
|
91
109
|
house_number: string
|
|
92
110
|
street: string
|
|
@@ -95,7 +113,9 @@ interface USTuple {
|
|
|
95
113
|
postcode: string
|
|
96
114
|
}
|
|
97
115
|
|
|
98
|
-
/**
|
|
116
|
+
/**
|
|
117
|
+
* A non-US BALANCE tuple (carries a postcode + native order).
|
|
118
|
+
*/
|
|
99
119
|
interface BalanceTuple {
|
|
100
120
|
house_number: string
|
|
101
121
|
street: string
|
|
@@ -106,10 +126,14 @@ interface BalanceTuple {
|
|
|
106
126
|
order: string
|
|
107
127
|
}
|
|
108
128
|
|
|
109
|
-
/**
|
|
129
|
+
/**
|
|
130
|
+
* Prefix carried through render — the (canonical, abbreviation) pair `renderDirectional` consumes.
|
|
131
|
+
*/
|
|
110
132
|
type Prefix = Pick<NonNullable<ReturnType<typeof matchLeadingDirectional>>, "canonical" | "abbreviation">
|
|
111
133
|
|
|
112
|
-
/**
|
|
134
|
+
/**
|
|
135
|
+
* Minimal RFC-4180-ish splitter (handles quoted fields).
|
|
136
|
+
*/
|
|
113
137
|
function splitCSV(line: string): string[] {
|
|
114
138
|
const out: string[] = []
|
|
115
139
|
let cur = ""
|
|
@@ -122,6 +146,7 @@ function splitCSV(line: string): string[] {
|
|
|
122
146
|
if (c === '"') {
|
|
123
147
|
if (line[i + 1] === '"') {
|
|
124
148
|
cur += '"'
|
|
149
|
+
|
|
125
150
|
i++
|
|
126
151
|
} else {
|
|
127
152
|
inQ = false
|
|
@@ -138,12 +163,15 @@ function splitCSV(line: string): string[] {
|
|
|
138
163
|
cur += c
|
|
139
164
|
}
|
|
140
165
|
}
|
|
166
|
+
|
|
141
167
|
out.push(cur)
|
|
142
168
|
|
|
143
169
|
return out
|
|
144
170
|
}
|
|
145
171
|
|
|
146
|
-
/**
|
|
172
|
+
/**
|
|
173
|
+
* Stream real US tuples (number/street/city/postcode) out of a cached OA zip.
|
|
174
|
+
*/
|
|
147
175
|
function readTuples(source: USSource): USTuple[] {
|
|
148
176
|
const r = spawnSync("unzip", ["-p", source.zip, source.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
|
|
149
177
|
|
|
@@ -152,15 +180,18 @@ function readTuples(source: USSource): USTuple[] {
|
|
|
152
180
|
|
|
153
181
|
return []
|
|
154
182
|
}
|
|
183
|
+
|
|
155
184
|
const lines = r.stdout.toString("utf8").split(/\r?\n/)
|
|
156
185
|
|
|
157
186
|
if (lines.length < 2) return []
|
|
158
187
|
const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
|
|
159
188
|
const idx = (name: string): number => header.indexOf(name)
|
|
189
|
+
|
|
160
190
|
const iNum = idx("number"),
|
|
161
191
|
iStreet = idx("street"),
|
|
162
192
|
iCity = idx("city"),
|
|
163
193
|
iPost = idx("postcode")
|
|
194
|
+
|
|
164
195
|
const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
|
|
165
196
|
const tuples: USTuple[] = []
|
|
166
197
|
const seen = new Set<string>()
|
|
@@ -209,6 +240,7 @@ function parseStreet(street: string): { prefix: Prefix | null; name: string; suf
|
|
|
209
240
|
prefix = { canonical: lead.canonical, abbreviation: lead.abbreviation }
|
|
210
241
|
words = words.slice(1)
|
|
211
242
|
}
|
|
243
|
+
|
|
212
244
|
// Trailing USPS suffix — only if it leaves ≥1 word for the name.
|
|
213
245
|
const trail = matchTrailingSuffix(words.join(" "))
|
|
214
246
|
|
|
@@ -249,11 +281,15 @@ function renderStreet(
|
|
|
249
281
|
parts.push(name)
|
|
250
282
|
|
|
251
283
|
// Suffix: abbreviated ("St") vs expanded ("Street"), Title-cased to match the name.
|
|
252
|
-
const full = title(parsed.suffix)
|
|
284
|
+
const full = title(parsed.suffix)
|
|
285
|
+
|
|
286
|
+
// canonical is uppercase word → "Street"
|
|
253
287
|
const abbr = matchCase(
|
|
254
288
|
US_STREET_SUFFIX_PREFERRED_ABBR[parsed.suffix as keyof typeof US_STREET_SUFFIX_PREFERRED_ABBR],
|
|
255
289
|
"Aa"
|
|
256
|
-
)
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
// "AVE" → "Ave"
|
|
257
293
|
const renderedSuffix = random() < 0.5 ? abbr : full
|
|
258
294
|
components.street_suffix = renderedSuffix
|
|
259
295
|
parts.push(renderedSuffix)
|
|
@@ -261,7 +297,9 @@ function renderStreet(
|
|
|
261
297
|
return { street: parts.join(" "), components }
|
|
262
298
|
}
|
|
263
299
|
|
|
264
|
-
/**
|
|
300
|
+
/**
|
|
301
|
+
* Synthetic recipient/venue prefixes — the arena's "JOHN DOE, ACME INC, …" pattern.
|
|
302
|
+
*/
|
|
265
303
|
const VENUES = ["John Doe", "Jane Smith", "Acme Inc", "Wayne Enterprises", "Maria Garcia", "Riverside Clinic"]
|
|
266
304
|
|
|
267
305
|
const tail = (loc: string, reg: string, pc: string): string => (pc ? `${loc}, ${reg} ${pc}` : `${loc}, ${reg}`)
|
|
@@ -280,6 +318,7 @@ function renderRow(
|
|
|
280
318
|
loc = base.locality,
|
|
281
319
|
reg = base.region,
|
|
282
320
|
pc = base.postcode
|
|
321
|
+
|
|
283
322
|
const road = `${hn} ${street}`
|
|
284
323
|
const withRoad: Partial<Record<ComponentTag, string>> = { house_number: hn, ...streetComponents }
|
|
285
324
|
const r = random()
|
|
@@ -309,7 +348,8 @@ function renderRow(
|
|
|
309
348
|
* tuples that carry a POSTCODE.
|
|
310
349
|
*/
|
|
311
350
|
function readBalanceTuples(source: BalanceSource, limit: number): BalanceTuple[] {
|
|
312
|
-
const maxLines = Math.max(limit * 8,
|
|
351
|
+
const maxLines = Math.max(limit * 8, 20_000) + 1
|
|
352
|
+
|
|
313
353
|
const r = spawnSync("bash", ["-c", `unzip -p "${source.zip}" "${source.csv}" | head -n ${maxLines}`], {
|
|
314
354
|
maxBuffer: 1024 * 1024 * 1024,
|
|
315
355
|
encoding: "buffer",
|
|
@@ -320,16 +360,19 @@ function readBalanceTuples(source: BalanceSource, limit: number): BalanceTuple[]
|
|
|
320
360
|
|
|
321
361
|
return []
|
|
322
362
|
}
|
|
363
|
+
|
|
323
364
|
const lines = r.stdout.toString("utf8").split(/\r?\n/)
|
|
324
365
|
|
|
325
366
|
if (lines.length < 2) return []
|
|
326
367
|
const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
|
|
327
368
|
const idx = (n: string): number => header.indexOf(n)
|
|
369
|
+
|
|
328
370
|
const iNum = idx("number"),
|
|
329
371
|
iStreet = idx("street"),
|
|
330
372
|
iCity = idx("city"),
|
|
331
373
|
iRegion = idx("region"),
|
|
332
374
|
iPost = idx("postcode")
|
|
375
|
+
|
|
333
376
|
const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
|
|
334
377
|
const tuples: BalanceTuple[] = []
|
|
335
378
|
const seen = new Set<string>()
|
|
@@ -337,6 +380,7 @@ function readBalanceTuples(source: BalanceSource, limit: number): BalanceTuple[]
|
|
|
337
380
|
for (let li = 1; li < lines.length && tuples.length < limit; li++) {
|
|
338
381
|
if (!lines[li]) continue
|
|
339
382
|
const cells = splitCSV(lines[li]!)
|
|
383
|
+
|
|
340
384
|
const street = get(cells, iStreet),
|
|
341
385
|
locality = get(cells, iCity),
|
|
342
386
|
house_number = get(cells, iNum),
|
|
@@ -347,6 +391,7 @@ function readBalanceTuples(source: BalanceSource, limit: number): BalanceTuple[]
|
|
|
347
391
|
|
|
348
392
|
if (seen.has(key)) continue
|
|
349
393
|
seen.add(key)
|
|
394
|
+
|
|
350
395
|
tuples.push({
|
|
351
396
|
house_number,
|
|
352
397
|
street,
|
|
@@ -369,6 +414,7 @@ function renderBalanceRow(t: BalanceTuple): { raw: string; components: Partial<R
|
|
|
369
414
|
const { house_number: hn, street, locality: loc, postcode: pc, order } = t
|
|
370
415
|
// region is intentionally omitted — it isn't rendered in `raw`, so labeling it would fail alignment.
|
|
371
416
|
const components: Partial<Record<ComponentTag, string>> = { house_number: hn, street, locality: loc, postcode: pc }
|
|
417
|
+
|
|
372
418
|
const raw =
|
|
373
419
|
order === "fr"
|
|
374
420
|
? `${hn} ${street}, ${pc} ${loc}` // French: number-street, postcode-city
|
|
@@ -378,6 +424,10 @@ function renderBalanceRow(t: BalanceTuple): { raw: string; components: Partial<R
|
|
|
378
424
|
return { raw, components }
|
|
379
425
|
}
|
|
380
426
|
|
|
427
|
+
/**
|
|
428
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
429
|
+
* and `description` below for the surface form it generates.
|
|
430
|
+
*/
|
|
381
431
|
export const streetAffixRecipe: ShardRecipe = {
|
|
382
432
|
name: "street-affix",
|
|
383
433
|
description: "US street-affix rows: OA streets split into street_prefix/street/street_suffix (+ multilocale balance)",
|
|
@@ -391,7 +441,7 @@ export const streetAffixRecipe: ShardRecipe = {
|
|
|
391
441
|
async run(opts, write) {
|
|
392
442
|
// Legacy build-street-affix-shard.mjs seeded `mulberry32(opts.seed)`.
|
|
393
443
|
const random = makeMulberry32(opts.seed)
|
|
394
|
-
const count = opts.count ??
|
|
444
|
+
const count = opts.count ?? 50_000
|
|
395
445
|
const source = opts.sourceName ?? "synth-affix"
|
|
396
446
|
const multilocaleCount = opts.multilocaleCount ?? 0
|
|
397
447
|
const sources = opts.golden ? [EVAL_SOURCE] : TRAIN_SOURCES
|
|
@@ -400,6 +450,7 @@ export const streetAffixRecipe: ShardRecipe = {
|
|
|
400
450
|
|
|
401
451
|
for (const s of sources) {
|
|
402
452
|
const t = readTuples(s)
|
|
453
|
+
|
|
403
454
|
console.error(` ${s.csv}: ${t.length} unique tuples`)
|
|
404
455
|
|
|
405
456
|
for (const x of t) {
|
|
@@ -407,7 +458,7 @@ export const streetAffixRecipe: ShardRecipe = {
|
|
|
407
458
|
}
|
|
408
459
|
}
|
|
409
460
|
|
|
410
|
-
if (pool.length
|
|
461
|
+
if (!pool.length) {
|
|
411
462
|
throw new Error("No US tuples found — are the cached OA zips present in /tmp/oa-cache?")
|
|
412
463
|
}
|
|
413
464
|
|
|
@@ -425,10 +476,13 @@ export const streetAffixRecipe: ShardRecipe = {
|
|
|
425
476
|
|
|
426
477
|
if (!parsed) {
|
|
427
478
|
noAffix++
|
|
479
|
+
|
|
428
480
|
continue
|
|
429
481
|
}
|
|
482
|
+
|
|
430
483
|
const { street, components: streetComponents } = renderStreet(random, parsed)
|
|
431
484
|
const { fmt, raw, components } = renderRow(random, base, street, streetComponents)
|
|
485
|
+
|
|
432
486
|
// Every affix surface form must survive verbatim in raw, else alignment can't label it.
|
|
433
487
|
const surfaces = [streetComponents.street_prefix, streetComponents.street, streetComponents.street_suffix].filter(
|
|
434
488
|
(s): s is string => Boolean(s)
|
|
@@ -436,8 +490,10 @@ export const streetAffixRecipe: ShardRecipe = {
|
|
|
436
490
|
|
|
437
491
|
if (!surfaces.every((s) => raw.includes(s))) {
|
|
438
492
|
skipped++
|
|
493
|
+
|
|
439
494
|
continue
|
|
440
495
|
}
|
|
496
|
+
|
|
441
497
|
formatCounts[fmt] = (formatCounts[fmt] ?? 0) + 1
|
|
442
498
|
const hasP = !!streetComponents.street_prefix
|
|
443
499
|
|
|
@@ -451,9 +507,12 @@ export const streetAffixRecipe: ShardRecipe = {
|
|
|
451
507
|
|
|
452
508
|
if (opts.golden) {
|
|
453
509
|
write(JSON.stringify({ raw, components, country: "US" }) + "\n")
|
|
510
|
+
|
|
454
511
|
emitted++
|
|
512
|
+
|
|
455
513
|
continue
|
|
456
514
|
}
|
|
515
|
+
|
|
457
516
|
const canonical: CanonicalRow = {
|
|
458
517
|
raw,
|
|
459
518
|
components,
|
|
@@ -464,13 +523,17 @@ export const streetAffixRecipe: ShardRecipe = {
|
|
|
464
523
|
corpus_version: "0.4.0",
|
|
465
524
|
license: "OpenAddresses US (non-VT) skeletons, street split via USPS Pub-28 C1/C2 (codex)",
|
|
466
525
|
}
|
|
526
|
+
|
|
467
527
|
const aligned = alignRow(canonical)
|
|
468
528
|
|
|
469
529
|
if (aligned.kind !== "labeled" || !aligned.row) {
|
|
470
530
|
skipped++
|
|
531
|
+
|
|
471
532
|
continue
|
|
472
533
|
}
|
|
534
|
+
|
|
473
535
|
write(JSON.stringify({ ...aligned.row, synth_method: "affix", synth_base_id: null }) + "\n")
|
|
536
|
+
|
|
474
537
|
emitted++
|
|
475
538
|
}
|
|
476
539
|
|
|
@@ -488,15 +551,18 @@ export const streetAffixRecipe: ShardRecipe = {
|
|
|
488
551
|
|
|
489
552
|
for (const s of mlSources) {
|
|
490
553
|
const t = readBalanceTuples(s, perSource)
|
|
554
|
+
|
|
491
555
|
console.error(` balance ${s.csv} (${s.iso2}): ${t.length} tuples`)
|
|
492
556
|
|
|
493
557
|
for (const x of t) {
|
|
494
558
|
mlPool.push(x)
|
|
495
559
|
}
|
|
496
560
|
}
|
|
561
|
+
|
|
497
562
|
const M = mlPool.length
|
|
498
563
|
let mlGuard = 0
|
|
499
564
|
|
|
565
|
+
// oxlint-disable-next-line eslint/no-unmodified-loop-condition -- `M > 0` is an invariant guard against an empty pool, not a progress condition
|
|
500
566
|
while (M > 0 && balanceEmitted < multilocaleCount && mlGuard++ < multilocaleCount * 10) {
|
|
501
567
|
const t = mlPool[Math.floor(random() * M)]!
|
|
502
568
|
const { raw, components } = renderBalanceRow(t)
|
|
@@ -504,16 +570,21 @@ export const streetAffixRecipe: ShardRecipe = {
|
|
|
504
570
|
// Every component surface must survive in raw, else alignment can't label it.
|
|
505
571
|
if (![components.street, components.locality, components.postcode].every((s) => !!s && raw.includes(s))) {
|
|
506
572
|
balanceSkipped++
|
|
573
|
+
|
|
507
574
|
continue
|
|
508
575
|
}
|
|
576
|
+
|
|
509
577
|
balanceISO[t.iso2] = (balanceISO[t.iso2] ?? 0) + 1
|
|
510
578
|
const locale = `${t.iso2.toLowerCase()}-${t.iso2}`
|
|
511
579
|
|
|
512
580
|
if (opts.golden) {
|
|
513
581
|
write(JSON.stringify({ raw, components, country: t.iso2 }) + "\n")
|
|
582
|
+
|
|
514
583
|
balanceEmitted++
|
|
584
|
+
|
|
515
585
|
continue
|
|
516
586
|
}
|
|
587
|
+
|
|
517
588
|
const canonical: CanonicalRow = {
|
|
518
589
|
raw,
|
|
519
590
|
components,
|
|
@@ -524,13 +595,17 @@ export const streetAffixRecipe: ShardRecipe = {
|
|
|
524
595
|
corpus_version: "0.4.0",
|
|
525
596
|
license: "OpenAddresses non-US skeletons (native-order postcode balance for the affix shard)",
|
|
526
597
|
}
|
|
598
|
+
|
|
527
599
|
const aligned = alignRow(canonical)
|
|
528
600
|
|
|
529
601
|
if (aligned.kind !== "labeled" || !aligned.row) {
|
|
530
602
|
balanceSkipped++
|
|
603
|
+
|
|
531
604
|
continue
|
|
532
605
|
}
|
|
606
|
+
|
|
533
607
|
write(JSON.stringify({ ...aligned.row, synth_method: "affix-balance", synth_base_id: null }) + "\n")
|
|
608
|
+
|
|
534
609
|
balanceEmitted++
|
|
535
610
|
}
|
|
536
611
|
}
|
|
@@ -22,6 +22,10 @@ import { DEFAULT_US_BASES } from "../synthesize-intersection.ts"
|
|
|
22
22
|
import { synthesizeStreetRow, type StreetBaseTuple } from "../synthesize-street.ts"
|
|
23
23
|
import { makeMulberry32, shardSourceID, type CanonicalShardRow, type ShardRecipe } from "./scaffold.ts"
|
|
24
24
|
|
|
25
|
+
/**
|
|
26
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
27
|
+
* and `description` below for the surface form it generates.
|
|
28
|
+
*/
|
|
25
29
|
export const streetBareRecipe: ShardRecipe = {
|
|
26
30
|
name: "street-bare",
|
|
27
31
|
description: "Bare-street rows (US): DEFAULT_US_BASES → synthesizeStreetRow (bare) → aligned LabeledRow",
|
|
@@ -43,6 +47,7 @@ export const streetBareRecipe: ShardRecipe = {
|
|
|
43
47
|
|
|
44
48
|
while (emitted < count && guard++ < count * 5) {
|
|
45
49
|
const base = DEFAULT_US_BASES[emitted % DEFAULT_US_BASES.length]!
|
|
50
|
+
|
|
46
51
|
const synth = synthesizeStreetRow(base as StreetBaseTuple, {
|
|
47
52
|
random,
|
|
48
53
|
bareProb,
|
|
@@ -51,8 +56,10 @@ export const streetBareRecipe: ShardRecipe = {
|
|
|
51
56
|
|
|
52
57
|
if (!synth) {
|
|
53
58
|
skipped++
|
|
59
|
+
|
|
54
60
|
continue
|
|
55
61
|
}
|
|
62
|
+
|
|
56
63
|
const isBare = synth.components.region === undefined
|
|
57
64
|
|
|
58
65
|
const canonical: CanonicalShardRow = {
|
|
@@ -79,9 +86,12 @@ export const streetBareRecipe: ShardRecipe = {
|
|
|
79
86
|
|
|
80
87
|
if (aligned.kind !== "labeled" || !aligned.row) {
|
|
81
88
|
skipped++
|
|
89
|
+
|
|
82
90
|
continue
|
|
83
91
|
}
|
|
92
|
+
|
|
84
93
|
write(JSON.stringify({ ...aligned.row, synth_method: "street-bare", synth_base_id: null }) + "\n")
|
|
94
|
+
|
|
85
95
|
emitted++
|
|
86
96
|
}
|
|
87
97
|
|
|
@@ -11,6 +11,10 @@ import { stableSourceID } from "../adapter.ts"
|
|
|
11
11
|
import { synthesizeStreetRow, type StreetBaseTuple } from "../synthesize-street.ts"
|
|
12
12
|
import { alignAndWrite, makeLcg, readTuples, type ShardRecipe } from "./scaffold.ts"
|
|
13
13
|
|
|
14
|
+
/**
|
|
15
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
16
|
+
* and `description` below for the surface form it generates.
|
|
17
|
+
*/
|
|
14
18
|
export const streetRecipe: ShardRecipe = {
|
|
15
19
|
name: "street",
|
|
16
20
|
description: "Street-decomposition rows (US): tuples → synthesizeStreetRow → aligned LabeledRow",
|
|
@@ -29,11 +33,13 @@ export const streetRecipe: ShardRecipe = {
|
|
|
29
33
|
|
|
30
34
|
if (!tuple.locality || !tuple.region || !tuple.postcode || !tuple.country) {
|
|
31
35
|
skipped++
|
|
36
|
+
|
|
32
37
|
continue
|
|
33
38
|
}
|
|
34
39
|
|
|
35
40
|
if (tuple.country !== "US") {
|
|
36
41
|
skipped++
|
|
42
|
+
|
|
37
43
|
continue
|
|
38
44
|
}
|
|
39
45
|
|
|
@@ -41,6 +47,7 @@ export const streetRecipe: ShardRecipe = {
|
|
|
41
47
|
const synth = synthesizeStreetRow(tuple as StreetBaseTuple, { random, includeHouseNumberProb })
|
|
42
48
|
|
|
43
49
|
if (!synth) continue
|
|
50
|
+
|
|
44
51
|
const ok = alignAndWrite(
|
|
45
52
|
write,
|
|
46
53
|
{
|