@mailwoman/corpus 8.0.0 → 8.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/adapter.d.ts +15 -5
- package/out/src/adapter.d.ts.map +1 -1
- package/out/src/adapter.js +11 -9
- package/out/src/adapter.js.map +1 -1
- package/out/src/adapters/ban/adapter.d.ts +7 -0
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +9 -2
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +5 -1
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +14 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +15 -2
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +11 -0
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +24 -5
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +11 -0
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +14 -1
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +10 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +11 -2
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts +18 -6
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +16 -6
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +11 -0
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +18 -3
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +11 -0
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +12 -1
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +11 -0
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +11 -0
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +15 -2
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +11 -0
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +13 -2
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +14 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +19 -4
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +12 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +17 -2
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +12 -1
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +11 -0
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +15 -2
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +7 -0
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +18 -3
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +13 -3
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +15 -5
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +7 -0
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +9 -2
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/align.d.ts +12 -4
- package/out/src/align.d.ts.map +1 -1
- package/out/src/align.js +9 -5
- package/out/src/align.js.map +1 -1
- package/out/src/build.d.ts +24 -8
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +3 -3
- package/out/src/build.js.map +1 -1
- package/out/src/codex/us-fips-state.d.ts +6 -2
- package/out/src/codex/us-fips-state.d.ts.map +1 -1
- package/out/src/codex/us-fips-state.js +3 -1
- package/out/src/codex/us-fips-state.js.map +1 -1
- package/out/src/golden.d.ts +21 -7
- package/out/src/golden.d.ts.map +1 -1
- package/out/src/golden.js +18 -10
- package/out/src/golden.js.map +1 -1
- package/out/src/license.d.ts +3 -1
- package/out/src/license.d.ts.map +1 -1
- package/out/src/license.js +4 -2
- package/out/src/license.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts +4 -2
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +5 -4
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/schema.d.ts +14 -10
- package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
- package/out/src/parquet-wrapper/schema.js +6 -2
- package/out/src/parquet-wrapper/schema.js.map +1 -1
- package/out/src/parquet-wrapper/writer.d.ts +12 -4
- package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
- package/out/src/parquet-wrapper/writer.js +12 -4
- package/out/src/parquet-wrapper/writer.js.map +1 -1
- package/out/src/parquet.d.ts +21 -7
- package/out/src/parquet.d.ts.map +1 -1
- package/out/src/parquet.js +15 -4
- package/out/src/parquet.js.map +1 -1
- package/out/src/runner.d.ts +39 -13
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +7 -3
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts +4 -0
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +14 -5
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/boundary-stress.d.ts +4 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +13 -7
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts +4 -0
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +39 -11
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts +4 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +14 -3
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts +4 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.js +4 -0
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts +4 -0
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +25 -8
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts +4 -0
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +21 -8
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts +4 -0
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +27 -8
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts +4 -0
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +16 -6
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.d.ts +4 -0
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/house-venue.js +4 -0
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts +9 -3
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +12 -4
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts +4 -0
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +27 -13
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +4 -0
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +9 -4
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/nl-postcode.d.ts +4 -0
- package/out/src/shard-recipes/nl-postcode.d.ts.map +1 -1
- package/out/src/shard-recipes/nl-postcode.js +8 -2
- package/out/src/shard-recipes/nl-postcode.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts +4 -0
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +8 -2
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts +4 -0
- package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street-led.js +5 -1
- package/out/src/shard-recipes/no-street-led.js.map +1 -1
- package/out/src/shard-recipes/no-street.d.ts +4 -0
- package/out/src/shard-recipes/no-street.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street.js +4 -0
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +4 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +99 -43
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts +4 -0
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +4 -0
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +48 -16
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +10 -6
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/si-bare-village.d.ts +4 -0
- package/out/src/shard-recipes/si-bare-village.d.ts.map +1 -1
- package/out/src/shard-recipes/si-bare-village.js +4 -0
- package/out/src/shard-recipes/si-bare-village.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +4 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +29 -10
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.d.ts +4 -0
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
- package/out/src/shard-recipes/street-bare.js +4 -0
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.d.ts +4 -0
- package/out/src/shard-recipes/street.d.ts.map +1 -1
- package/out/src/shard-recipes/street.js +4 -0
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/unit.d.ts +4 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +45 -14
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/split.d.ts +18 -6
- package/out/src/split.d.ts.map +1 -1
- package/out/src/split.js +4 -2
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize-anchor-absorption.d.ts +7 -1
- package/out/src/synthesize-anchor-absorption.d.ts.map +1 -1
- package/out/src/synthesize-anchor-absorption.js +28 -14
- package/out/src/synthesize-anchor-absorption.js.map +1 -1
- package/out/src/synthesize-boundary-stress.d.ts +3 -1
- package/out/src/synthesize-boundary-stress.d.ts.map +1 -1
- package/out/src/synthesize-boundary-stress.js +35 -28
- package/out/src/synthesize-boundary-stress.js.map +1 -1
- package/out/src/synthesize-german.d.ts +15 -5
- package/out/src/synthesize-german.d.ts.map +1 -1
- package/out/src/synthesize-german.js +9 -3
- package/out/src/synthesize-german.js.map +1 -1
- package/out/src/synthesize-house-venue.d.ts.map +1 -1
- package/out/src/synthesize-house-venue.js +15 -15
- package/out/src/synthesize-house-venue.js.map +1 -1
- package/out/src/synthesize-intersection.d.ts +9 -3
- package/out/src/synthesize-intersection.d.ts.map +1 -1
- package/out/src/synthesize-intersection.js +25 -6
- package/out/src/synthesize-intersection.js.map +1 -1
- package/out/src/synthesize-no-street.d.ts +3 -1
- package/out/src/synthesize-no-street.d.ts.map +1 -1
- package/out/src/synthesize-no-street.js +5 -6
- package/out/src/synthesize-no-street.js.map +1 -1
- package/out/src/synthesize-po-box.d.ts +15 -5
- package/out/src/synthesize-po-box.d.ts.map +1 -1
- package/out/src/synthesize-po-box.js +21 -7
- package/out/src/synthesize-po-box.js.map +1 -1
- package/out/src/synthesize-street.d.ts +3 -1
- package/out/src/synthesize-street.d.ts.map +1 -1
- package/out/src/synthesize-street.js +6 -1
- package/out/src/synthesize-street.js.map +1 -1
- package/out/src/synthesize.d.ts +39 -13
- package/out/src/synthesize.d.ts.map +1 -1
- package/out/src/synthesize.js +66 -36
- package/out/src/synthesize.js.map +1 -1
- package/out/src/tokenize.d.ts +15 -5
- package/out/src/tokenize.d.ts.map +1 -1
- package/out/src/tokenize.js.map +1 -1
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +3 -1
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/audit.d.ts.map +1 -1
- package/out/src/tools/audit.js +20 -8
- package/out/src/tools/audit.js.map +1 -1
- package/out/src/tools/corpus-stats.d.ts.map +1 -1
- package/out/src/tools/corpus-stats.js.map +1 -1
- package/out/src/tools/fetch/ban.d.ts.map +1 -1
- package/out/src/tools/fetch/ban.js +17 -10
- package/out/src/tools/fetch/ban.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts +30 -10
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +26 -6
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/hrsa.d.ts.map +1 -1
- package/out/src/tools/fetch/hrsa.js.map +1 -1
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +18 -18
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +3 -1
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +3 -1
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts +21 -7
- package/out/src/tools/fetch/nad.d.ts.map +1 -1
- package/out/src/tools/fetch/nad.js +5 -5
- package/out/src/tools/fetch/nad.js.map +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +9 -15
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts +3 -1
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/src/tools/fetch/openaddresses.js +32 -23
- package/out/src/tools/fetch/openaddresses.js.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +3 -1
- package/out/src/tools/fetch/ppd.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.js +14 -3
- package/out/src/tools/fetch/ppd.js.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.js +13 -6
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
- package/out/src/tools/fetch/state-sources.js +8 -3
- package/out/src/tools/fetch/state-sources.js.map +1 -1
- package/out/src/tools/fetch/tiger-full.d.ts +9 -3
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
- package/out/src/tools/fetch/tiger-full.js +35 -20
- package/out/src/tools/fetch/tiger-full.js.map +1 -1
- package/out/src/tools/gb-title-case.d.ts.map +1 -1
- package/out/src/tools/gb-title-case.js.map +1 -1
- package/out/src/tools/golden-expand.d.ts +24 -8
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +20 -12
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-promote.d.ts +18 -6
- package/out/src/tools/golden-promote.d.ts.map +1 -1
- package/out/src/tools/golden-promote.js +22 -7
- package/out/src/tools/golden-promote.js.map +1 -1
- package/out/src/tools/ingest-csv.d.ts +3 -1
- package/out/src/tools/ingest-csv.d.ts.map +1 -1
- package/out/src/tools/ingest-csv.js +17 -19
- package/out/src/tools/ingest-csv.js.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts +18 -6
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
- package/out/src/tools/jsonl-to-parquet.js +11 -5
- package/out/src/tools/jsonl-to-parquet.js.map +1 -1
- package/out/src/tools/lint-shard-vocab.d.ts +36 -12
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
- package/out/src/tools/lint-shard-vocab.js +25 -11
- package/out/src/tools/lint-shard-vocab.js.map +1 -1
- package/out/src/tools/lint-shard.d.ts +30 -10
- package/out/src/tools/lint-shard.d.ts.map +1 -1
- package/out/src/tools/lint-shard.js +33 -20
- package/out/src/tools/lint-shard.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +6 -4
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/shard-kryptonite.d.ts +6 -2
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +1 -1
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts +9 -3
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +2 -2
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/types.d.ts +51 -17
- package/out/src/types.d.ts.map +1 -1
- package/out/src/wof-json.d.ts +9 -3
- package/out/src/wof-json.d.ts.map +1 -1
- package/out/src/wof-json.js +6 -4
- package/out/src/wof-json.js.map +1 -1
- package/package.json +7 -6
- package/src/adapter.ts +24 -13
- package/src/adapters/ban/adapter.ts +12 -2
- package/src/adapters/ban/street-decompose.ts +7 -1
- package/src/adapters/fcc-bdc/adapter.ts +19 -3
- package/src/adapters/geonames/adapter.ts +28 -5
- package/src/adapters/geonames-postal/adapter.ts +17 -1
- package/src/adapters/gnaf/adapter.ts +15 -2
- package/src/adapters/gnaf/assemble.ts +40 -10
- package/src/adapters/index.ts +13 -0
- package/src/adapters/openaddresses/adapter.ts +22 -3
- package/src/adapters/overture/adapter.ts +18 -2
- package/src/adapters/state-hi-schools/adapter.ts +15 -0
- package/src/adapters/state-ia-contractors/adapter.ts +15 -0
- package/src/adapters/state-ny-notaries/adapter.ts +15 -0
- package/src/adapters/state-tx-notaries/adapter.ts +18 -2
- package/src/adapters/synth-po-box/adapter.ts +19 -2
- package/src/adapters/tiger/adapter.ts +23 -4
- package/src/adapters/tiger/street-decompose.ts +1 -0
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +18 -2
- package/src/adapters/usgov-imls-pls/adapter.ts +15 -0
- package/src/adapters/usgov-irs-bmf/adapter.ts +19 -2
- package/src/adapters/usgov-nad/adapter.ts +14 -1
- package/src/adapters/usgov-nppes/adapter.ts +13 -0
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +21 -3
- package/src/adapters/wof-admin-jp/adapter.ts +20 -3
- package/src/adapters/wof-admin-json/adapter.ts +22 -7
- package/src/adapters/wof-postalcode-json/adapter.ts +10 -2
- package/src/align.ts +21 -8
- package/src/build.ts +48 -11
- package/src/codex/us-fips-state.ts +6 -2
- package/src/golden.ts +28 -13
- package/src/license.ts +4 -2
- package/src/parquet-wrapper/reader.ts +6 -4
- package/src/parquet-wrapper/schema.ts +14 -10
- package/src/parquet-wrapper/writer.ts +12 -4
- package/src/parquet.ts +39 -8
- package/src/runner.ts +49 -15
- package/src/shard-recipes/anchor-absorption.ts +21 -5
- package/src/shard-recipes/boundary-stress.ts +20 -7
- package/src/shard-recipes/country-balanced.ts +78 -13
- package/src/shard-recipes/cz-pcfirst-preposition.ts +9 -0
- package/src/shard-recipes/fr-admin-split.ts +30 -5
- package/src/shard-recipes/fr-bare-street.ts +10 -0
- package/src/shard-recipes/fr-fragment.ts +33 -8
- package/src/shard-recipes/fr-lieudit.ts +35 -9
- package/src/shard-recipes/fr-order.ts +47 -9
- package/src/shard-recipes/german.ts +32 -6
- package/src/shard-recipes/house-venue.ts +7 -0
- package/src/shard-recipes/index.ts +12 -4
- package/src/shard-recipes/intersection.ts +63 -14
- package/src/shard-recipes/locale.ts +37 -4
- package/src/shard-recipes/nl-postcode.ts +12 -2
- package/src/shard-recipes/no-fragment.ts +12 -2
- package/src/shard-recipes/no-street-led.ts +11 -1
- package/src/shard-recipes/no-street.ts +7 -0
- package/src/shard-recipes/po-box-cedex.ts +145 -45
- package/src/shard-recipes/po-box.ts +7 -0
- package/src/shard-recipes/scaffold.ts +52 -20
- package/src/shard-recipes/si-bare-village.ts +9 -0
- package/src/shard-recipes/street-affix.ts +88 -13
- package/src/shard-recipes/street-bare.ts +10 -0
- package/src/shard-recipes/street.ts +7 -0
- package/src/shard-recipes/unit.ts +79 -16
- package/src/split.ts +28 -7
- package/src/synthesize-anchor-absorption.ts +41 -14
- package/src/synthesize-boundary-stress.ts +57 -30
- package/src/synthesize-german.ts +30 -7
- package/src/synthesize-house-venue.ts +22 -16
- package/src/synthesize-intersection.ts +34 -7
- package/src/synthesize-no-street.ts +18 -10
- package/src/synthesize-po-box.ts +42 -10
- package/src/synthesize-street.ts +10 -2
- package/src/synthesize.ts +81 -38
- package/src/tokenize.ts +15 -5
- package/src/tools/align-shard.ts +6 -1
- package/src/tools/audit.ts +57 -11
- package/src/tools/corpus-stats.ts +9 -0
- package/src/tools/fetch/ban.ts +32 -10
- package/src/tools/fetch/download.ts +49 -12
- package/src/tools/fetch/hrsa.ts +3 -0
- package/src/tools/fetch/imls-pls.ts +28 -18
- package/src/tools/fetch/index.ts +3 -1
- package/src/tools/fetch/nad.ts +39 -12
- package/src/tools/fetch/nppes.ts +17 -15
- package/src/tools/fetch/openaddresses.ts +52 -24
- package/src/tools/fetch/ppd.ts +22 -3
- package/src/tools/fetch/state-hi-schools.ts +17 -6
- package/src/tools/fetch/state-sources.ts +16 -3
- package/src/tools/fetch/tiger-full.ts +66 -23
- package/src/tools/gb-title-case.ts +1 -0
- package/src/tools/golden-expand.ts +64 -20
- package/src/tools/golden-promote.ts +53 -13
- package/src/tools/ingest-csv.ts +41 -26
- package/src/tools/jsonl-to-parquet.ts +30 -10
- package/src/tools/lint-shard-vocab.ts +75 -24
- package/src/tools/lint-shard.ts +75 -31
- package/src/tools/overlay-manifest.ts +7 -4
- package/src/tools/shard-kryptonite.ts +11 -3
- package/src/tools/shard-translit.ts +20 -5
- package/src/types.ts +51 -17
- package/src/wof-json.ts +18 -7
|
@@ -37,6 +37,12 @@ import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
|
|
|
37
37
|
// v2: the country TOKEN is decoupled from the skeleton's locale and drawn from a BROAD pool — every
|
|
38
38
|
// ISO canonical name + every curated surface form (endonyms/abbrevs). Surface forms are over-weighted
|
|
39
39
|
// so endonyms/abbrevs ("Deutschland","USA","NL") get strong signal.
|
|
40
|
+
/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
|
|
41
|
+
cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
|
|
42
|
+
output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
|
|
43
|
+
would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
|
|
44
|
+
extracted as named constants above. */
|
|
45
|
+
|
|
40
46
|
const COUNTRY_FORM_POOL = (() => {
|
|
41
47
|
const surface = Object.values(COUNTRY_SURFACE_FORMS).flat() // endonyms + abbrevs + canonical (curated)
|
|
42
48
|
const names = [...CountryNames]
|
|
@@ -44,9 +50,15 @@ const COUNTRY_FORM_POOL = (() => {
|
|
|
44
50
|
// all ~249 ISO canonical English names (breadth)
|
|
45
51
|
return { surface, names }
|
|
46
52
|
})()
|
|
47
|
-
const COUNTRY_ABSENT_PROB = 0.3 // negatives: rows with NO country token → teach golden precision
|
|
48
53
|
|
|
49
|
-
/**
|
|
54
|
+
/**
|
|
55
|
+
* Negatives: rows with NO country token → teach golden precision.
|
|
56
|
+
*/
|
|
57
|
+
const COUNTRY_ABSENT_PROB = 0.3
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* A cached OpenAddresses extract + the implied iso2/region/render-order.
|
|
61
|
+
*/
|
|
50
62
|
interface CountrySource {
|
|
51
63
|
zip: string
|
|
52
64
|
csv: string
|
|
@@ -55,8 +67,10 @@ interface CountrySource {
|
|
|
55
67
|
order: string
|
|
56
68
|
}
|
|
57
69
|
|
|
58
|
-
|
|
59
|
-
|
|
70
|
+
/**
|
|
71
|
+
* Multi-locale OA sources. region = implied admin where the extract is single-region (US states, DE Saxony);
|
|
72
|
+
* countrywide extracts (FR/IT/NL) read region from the CSV when present.
|
|
73
|
+
*/
|
|
60
74
|
const SOURCES: readonly CountrySource[] = [
|
|
61
75
|
{ zip: "/tmp/oa-cache/us__ia__statewide.zip", csv: "us/ia/statewide.csv", iso2: "US", region: "IA", order: "us" },
|
|
62
76
|
{ zip: "/tmp/oa-cache/us__il__cook.zip", csv: "us/il/cook.csv", iso2: "US", region: "IL", order: "us" },
|
|
@@ -69,13 +83,18 @@ const SOURCES: readonly CountrySource[] = [
|
|
|
69
83
|
{ zip: "/tmp/oa-cache/it__countrywide.zip", csv: "it/countrywide.csv", iso2: "IT", region: "", order: "eu" },
|
|
70
84
|
{ zip: "/tmp/oa-cache/nl__countrywide.zip", csv: "nl/countrywide.csv", iso2: "NL", region: "", order: "eu" },
|
|
71
85
|
]
|
|
72
|
-
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Held-out for --golden: Vermont (US holdout) + Berlin (DE holdout) — geographic split, never trained.
|
|
89
|
+
*/
|
|
73
90
|
const EVAL_SOURCES: readonly CountrySource[] = [
|
|
74
91
|
{ zip: "/tmp/oa-cache/us__vt__statewide.zip", csv: "us/vt/statewide.csv", iso2: "US", region: "VT", order: "us" },
|
|
75
92
|
{ zip: "/tmp/oa-cache/de__berlin.zip", csv: "de/berlin.csv", iso2: "DE", region: "", order: "eu" },
|
|
76
93
|
]
|
|
77
94
|
|
|
78
|
-
/**
|
|
95
|
+
/**
|
|
96
|
+
* A real tuple read out of a cached OA zip (+ the source's iso2/render-order).
|
|
97
|
+
*/
|
|
79
98
|
interface CountryTuple {
|
|
80
99
|
house_number: string
|
|
81
100
|
street: string
|
|
@@ -88,6 +107,7 @@ interface CountryTuple {
|
|
|
88
107
|
|
|
89
108
|
function splitCSV(line: string): string[] {
|
|
90
109
|
const out: string[] = []
|
|
110
|
+
|
|
91
111
|
let cur = "",
|
|
92
112
|
inQ = false
|
|
93
113
|
|
|
@@ -98,6 +118,7 @@ function splitCSV(line: string): string[] {
|
|
|
98
118
|
if (c === '"') {
|
|
99
119
|
if (line[i + 1] === '"') {
|
|
100
120
|
cur += '"'
|
|
121
|
+
|
|
101
122
|
i++
|
|
102
123
|
} else {
|
|
103
124
|
inQ = false
|
|
@@ -114,6 +135,7 @@ function splitCSV(line: string): string[] {
|
|
|
114
135
|
cur += c
|
|
115
136
|
}
|
|
116
137
|
}
|
|
138
|
+
|
|
117
139
|
out.push(cur)
|
|
118
140
|
|
|
119
141
|
return out
|
|
@@ -122,7 +144,8 @@ function splitCSV(line: string): string[] {
|
|
|
122
144
|
function readTuples(source: CountrySource, limit: number): CountryTuple[] {
|
|
123
145
|
// countrywide extracts (FR/IT/NL) are GB-scale — cap the bytes with `head` (read ~8 lines per wanted
|
|
124
146
|
// tuple to survive dedup/skips) so the toString stays under V8's string limit.
|
|
125
|
-
const maxLines = Math.max(limit * 8,
|
|
147
|
+
const maxLines = Math.max(limit * 8, 20_000) + 1
|
|
148
|
+
|
|
126
149
|
const r = spawnSync("bash", ["-c", `unzip -p "${source.zip}" "${source.csv}" | head -n ${maxLines}`], {
|
|
127
150
|
maxBuffer: 1024 * 1024 * 1024,
|
|
128
151
|
encoding: "buffer",
|
|
@@ -133,16 +156,19 @@ function readTuples(source: CountrySource, limit: number): CountryTuple[] {
|
|
|
133
156
|
|
|
134
157
|
return []
|
|
135
158
|
}
|
|
159
|
+
|
|
136
160
|
const lines = r.stdout.toString("utf8").split(/\r?\n/)
|
|
137
161
|
|
|
138
162
|
if (lines.length < 2) return []
|
|
139
163
|
const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
|
|
140
164
|
const idx = (n: string): number => header.indexOf(n)
|
|
165
|
+
|
|
141
166
|
const iNum = idx("number"),
|
|
142
167
|
iStreet = idx("street"),
|
|
143
168
|
iCity = idx("city"),
|
|
144
169
|
iRegion = idx("region"),
|
|
145
170
|
iPost = idx("postcode")
|
|
171
|
+
|
|
146
172
|
const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
|
|
147
173
|
const tuples: CountryTuple[] = []
|
|
148
174
|
const seen = new Set<string>()
|
|
@@ -150,6 +176,7 @@ function readTuples(source: CountrySource, limit: number): CountryTuple[] {
|
|
|
150
176
|
for (let li = 1; li < lines.length && tuples.length < limit; li++) {
|
|
151
177
|
if (!lines[li]) continue
|
|
152
178
|
const cells = splitCSV(lines[li]!)
|
|
179
|
+
|
|
153
180
|
const street = get(cells, iStreet),
|
|
154
181
|
locality = get(cells, iCity),
|
|
155
182
|
house_number = get(cells, iNum)
|
|
@@ -159,6 +186,7 @@ function readTuples(source: CountrySource, limit: number): CountryTuple[] {
|
|
|
159
186
|
|
|
160
187
|
if (seen.has(key)) continue
|
|
161
188
|
seen.add(key)
|
|
189
|
+
|
|
162
190
|
tuples.push({
|
|
163
191
|
house_number,
|
|
164
192
|
street,
|
|
@@ -173,7 +201,9 @@ function readTuples(source: CountrySource, limit: number): CountryTuple[] {
|
|
|
173
201
|
return tuples
|
|
174
202
|
}
|
|
175
203
|
|
|
176
|
-
/**
|
|
204
|
+
/**
|
|
205
|
+
* Pick a country token from the BROAD pool, or null (a country-absent negative). v2.
|
|
206
|
+
*/
|
|
177
207
|
function pickCountry(random: () => number): string | null {
|
|
178
208
|
if (random() < COUNTRY_ABSENT_PROB) return null // negative — teaches "trailing token != always country"
|
|
179
209
|
// 60% curated surface forms (endonym/abbrev variety), 40% broad ISO canonical names (coverage).
|
|
@@ -182,7 +212,9 @@ function pickCountry(random: () => number): string | null {
|
|
|
182
212
|
return pool[Math.floor(random() * pool.length)]!
|
|
183
213
|
}
|
|
184
214
|
|
|
185
|
-
/**
|
|
215
|
+
/**
|
|
216
|
+
* Render the address body in native-ish order. `country` null → a country-ABSENT negative row.
|
|
217
|
+
*/
|
|
186
218
|
function renderCountry(
|
|
187
219
|
random: () => number,
|
|
188
220
|
t: CountryTuple,
|
|
@@ -198,6 +230,7 @@ function renderCountry(
|
|
|
198
230
|
if (pc) {
|
|
199
231
|
components.postcode = pc
|
|
200
232
|
}
|
|
233
|
+
|
|
201
234
|
let body: string
|
|
202
235
|
|
|
203
236
|
if (order === "us") {
|
|
@@ -217,6 +250,7 @@ function renderCountry(
|
|
|
217
250
|
// postcode is NOT a country (counters the v1 golden over-firing).
|
|
218
251
|
return { fmt: "negative", raw: body, components }
|
|
219
252
|
}
|
|
253
|
+
|
|
220
254
|
const withC: Partial<Record<ComponentTag, string>> = { ...components, country }
|
|
221
255
|
const r = random()
|
|
222
256
|
|
|
@@ -242,6 +276,7 @@ interface Homograph {
|
|
|
242
276
|
cities: readonly string[]
|
|
243
277
|
us: { role: "region" | "locality"; locality: string; region: string; postcodes: readonly string[] }
|
|
244
278
|
}
|
|
279
|
+
|
|
245
280
|
const HOMOGRAPHS: readonly Homograph[] = [
|
|
246
281
|
{
|
|
247
282
|
surface: "Georgia",
|
|
@@ -280,12 +315,14 @@ const HOMOGRAPHS: readonly Homograph[] = [
|
|
|
280
315
|
us: { role: "locality", locality: "Turkey", region: "TX", postcodes: ["79261", "28393"] },
|
|
281
316
|
},
|
|
282
317
|
]
|
|
318
|
+
|
|
283
319
|
// 2-letter codes that are BOTH a US state abbrev AND an ISO country code → must read as region in US ctx.
|
|
284
320
|
interface AbbrevRegion {
|
|
285
321
|
code: string
|
|
286
322
|
localities: readonly string[]
|
|
287
323
|
postcodes: readonly string[]
|
|
288
324
|
}
|
|
325
|
+
|
|
289
326
|
const ABBREV_REGIONS: readonly AbbrevRegion[] = [
|
|
290
327
|
{ code: "CA", localities: ["Los Angeles", "Sacramento", "San Diego"], postcodes: ["90012", "95814", "92101"] }, // California / Canada
|
|
291
328
|
{ code: "GA", localities: ["Atlanta", "Savannah", "Macon"], postcodes: ["30309", "31401", "31201"] }, // Georgia(US) / Georgia
|
|
@@ -294,6 +331,7 @@ const ABBREV_REGIONS: readonly AbbrevRegion[] = [
|
|
|
294
331
|
{ code: "PA", localities: ["Philadelphia", "Pittsburgh"], postcodes: ["19103", "15222"] }, // Pennsylvania / Panama
|
|
295
332
|
{ code: "AL", localities: ["Birmingham", "Montgomery"], postcodes: ["35203", "36104"] }, // Alabama / Albania
|
|
296
333
|
]
|
|
334
|
+
|
|
297
335
|
const STREET_POOL: readonly string[] = [
|
|
298
336
|
"Main Street",
|
|
299
337
|
"Oak Avenue",
|
|
@@ -306,6 +344,7 @@ const STREET_POOL: readonly string[] = [
|
|
|
306
344
|
"2nd Avenue",
|
|
307
345
|
"Maple Drive",
|
|
308
346
|
]
|
|
347
|
+
|
|
309
348
|
const pick = <T>(random: () => number, arr: readonly T[]): T => arr[Math.floor(random() * arr.length)]!
|
|
310
349
|
const houseNo = (random: () => number): string => String(1 + Math.floor(random() * 998))
|
|
311
350
|
|
|
@@ -320,6 +359,7 @@ function renderHomograph(random: () => number): {
|
|
|
320
359
|
iso2: string
|
|
321
360
|
} {
|
|
322
361
|
const h = pick(random, HOMOGRAPHS)
|
|
362
|
+
|
|
323
363
|
const hn = houseNo(random),
|
|
324
364
|
street = pick(random, STREET_POOL)
|
|
325
365
|
|
|
@@ -327,12 +367,14 @@ function renderHomograph(random: () => number): {
|
|
|
327
367
|
const city = pick(random, h.cities)
|
|
328
368
|
const withStreet = random() < 0.6
|
|
329
369
|
const raw = withStreet ? `${hn} ${street}, ${city}, ${h.surface}` : `${city}, ${h.surface}`
|
|
370
|
+
|
|
330
371
|
const components: Partial<Record<ComponentTag, string>> = withStreet
|
|
331
372
|
? { house_number: hn, street, locality: city, country: h.surface }
|
|
332
373
|
: { locality: city, country: h.surface }
|
|
333
374
|
|
|
334
375
|
return { fmt: "homograph-country", raw, components, iso2: h.iso2 }
|
|
335
376
|
}
|
|
377
|
+
|
|
336
378
|
const pc = pick(random, h.us.postcodes)
|
|
337
379
|
|
|
338
380
|
if (h.us.role === "region") {
|
|
@@ -354,7 +396,9 @@ function renderHomograph(random: () => number): {
|
|
|
354
396
|
}
|
|
355
397
|
}
|
|
356
398
|
|
|
357
|
-
/**
|
|
399
|
+
/**
|
|
400
|
+
* An abbrev-as-region negative: "123 Main St, Los Angeles, CA 90012" → region CA, NO country.
|
|
401
|
+
*/
|
|
358
402
|
function renderAbbrevRegion(random: () => number): {
|
|
359
403
|
fmt: string
|
|
360
404
|
raw: string
|
|
@@ -362,6 +406,7 @@ function renderAbbrevRegion(random: () => number): {
|
|
|
362
406
|
iso2: string
|
|
363
407
|
} {
|
|
364
408
|
const a = pick(random, ABBREV_REGIONS)
|
|
409
|
+
|
|
365
410
|
const hn = houseNo(random),
|
|
366
411
|
street = pick(random, STREET_POOL),
|
|
367
412
|
locality = pick(random, a.localities),
|
|
@@ -375,9 +420,19 @@ function renderAbbrevRegion(random: () => number): {
|
|
|
375
420
|
}
|
|
376
421
|
}
|
|
377
422
|
|
|
378
|
-
|
|
379
|
-
|
|
423
|
+
/**
|
|
424
|
+
* Share of rows that are homograph contrast pairs.
|
|
425
|
+
*/
|
|
426
|
+
const HOMOGRAPH_FRAC = 0.22
|
|
427
|
+
/**
|
|
428
|
+
* Share that are code-as-region negatives (cumulative with HOMOGRAPH_FRAC)
|
|
429
|
+
*/
|
|
430
|
+
const ABBREV_FRAC = 0.08
|
|
380
431
|
|
|
432
|
+
/**
|
|
433
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
434
|
+
* and `description` below for the surface form it generates.
|
|
435
|
+
*/
|
|
381
436
|
export const countryBalancedRecipe: ShardRecipe = {
|
|
382
437
|
name: "country-balanced",
|
|
383
438
|
description: "Balanced model-first country rows (#464): OA skeletons + ISO surface forms + homograph contrast pairs",
|
|
@@ -396,6 +451,7 @@ export const countryBalancedRecipe: ShardRecipe = {
|
|
|
396
451
|
|
|
397
452
|
for (const s of sources) {
|
|
398
453
|
const t = readTuples(s, perSource)
|
|
454
|
+
|
|
399
455
|
console.error(` ${s.csv} (${s.iso2}): ${t.length} tuples`)
|
|
400
456
|
|
|
401
457
|
for (const x of t) {
|
|
@@ -403,7 +459,7 @@ export const countryBalancedRecipe: ShardRecipe = {
|
|
|
403
459
|
}
|
|
404
460
|
}
|
|
405
461
|
|
|
406
|
-
if (pool.length
|
|
462
|
+
if (!pool.length) {
|
|
407
463
|
throw new Error("No tuples — are the cached OA zips present in /tmp/oa-cache?")
|
|
408
464
|
}
|
|
409
465
|
|
|
@@ -435,17 +491,22 @@ export const countryBalancedRecipe: ShardRecipe = {
|
|
|
435
491
|
|
|
436
492
|
if (country && !rendered.raw.includes(country)) {
|
|
437
493
|
skipped++
|
|
494
|
+
|
|
438
495
|
continue
|
|
439
496
|
}
|
|
440
497
|
}
|
|
498
|
+
|
|
441
499
|
const { raw, components } = rendered
|
|
442
500
|
const localeTag = rowISO2 === "US" ? "en-US" : `${rowISO2.toLowerCase()}-${rowISO2}`
|
|
443
501
|
|
|
444
502
|
if (opts.golden) {
|
|
445
503
|
write(JSON.stringify({ raw, components, country: rowISO2 }) + "\n")
|
|
504
|
+
|
|
446
505
|
emitted++
|
|
506
|
+
|
|
447
507
|
continue
|
|
448
508
|
}
|
|
509
|
+
|
|
449
510
|
const canonical: CanonicalRow = {
|
|
450
511
|
raw,
|
|
451
512
|
components,
|
|
@@ -456,13 +517,17 @@ export const countryBalancedRecipe: ShardRecipe = {
|
|
|
456
517
|
corpus_version: "0.4.0",
|
|
457
518
|
license: "OpenAddresses multi-locale skeletons + injected ISO-3166 country surface forms (codex)",
|
|
458
519
|
}
|
|
520
|
+
|
|
459
521
|
const aligned = alignRow(canonical)
|
|
460
522
|
|
|
461
523
|
if (aligned.kind !== "labeled" || !aligned.row) {
|
|
462
524
|
skipped++
|
|
525
|
+
|
|
463
526
|
continue
|
|
464
527
|
}
|
|
528
|
+
|
|
465
529
|
write(JSON.stringify({ ...aligned.row, synth_method: "country", synth_base_id: null }) + "\n")
|
|
530
|
+
|
|
466
531
|
emitted++
|
|
467
532
|
}
|
|
468
533
|
|
|
@@ -16,6 +16,10 @@
|
|
|
16
16
|
|
|
17
17
|
import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
|
|
18
18
|
|
|
19
|
+
/**
|
|
20
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
21
|
+
* and `description` below for the surface form it generates.
|
|
22
|
+
*/
|
|
19
23
|
export const czPcFirstPrepositionRecipe: ShardRecipe = {
|
|
20
24
|
name: "cz-pcfirst-preposition",
|
|
21
25
|
description:
|
|
@@ -36,10 +40,13 @@ export const czPcFirstPrepositionRecipe: ShardRecipe = {
|
|
|
36
40
|
|
|
37
41
|
if (!street || !city || !number || !postcode) {
|
|
38
42
|
skipped++
|
|
43
|
+
|
|
39
44
|
continue
|
|
40
45
|
}
|
|
46
|
+
|
|
41
47
|
const order = read % 3
|
|
42
48
|
let raw: string
|
|
49
|
+
|
|
43
50
|
const components: Record<string, string> = {
|
|
44
51
|
street,
|
|
45
52
|
house_number: number,
|
|
@@ -54,11 +61,13 @@ export const czPcFirstPrepositionRecipe: ShardRecipe = {
|
|
|
54
61
|
} else {
|
|
55
62
|
raw = `${city}, ${postcode}, ${street} ${number}`
|
|
56
63
|
}
|
|
64
|
+
|
|
57
65
|
const source_id = shardSourceID("synth-cz-pcfirst-preposition", {
|
|
58
66
|
...components,
|
|
59
67
|
o: String(order),
|
|
60
68
|
v: String(read),
|
|
61
69
|
})
|
|
70
|
+
|
|
62
71
|
const canonical = {
|
|
63
72
|
raw,
|
|
64
73
|
components,
|
|
@@ -39,10 +39,18 @@ import { alignRow } from "../align.ts"
|
|
|
39
39
|
import type { CanonicalRow } from "../types.ts"
|
|
40
40
|
import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
|
|
41
41
|
|
|
42
|
+
/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
|
|
43
|
+
cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
|
|
44
|
+
output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
|
|
45
|
+
would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
|
|
46
|
+
extracted as named constants above. */
|
|
47
|
+
|
|
42
48
|
const DEFAULT_COMMUNES = "/tmp/reg/fr-communes.tsv"
|
|
43
49
|
const LICENSE = "BAN (Base Adresse Nationale) commune+postcode tuples, rendered admin-split — see ingest SOURCE"
|
|
44
50
|
|
|
45
|
-
/**
|
|
51
|
+
/**
|
|
52
|
+
* One distinct commune row from the TSV, with the département derived from its postcode.
|
|
53
|
+
*/
|
|
46
54
|
interface CommuneRow {
|
|
47
55
|
commune: string
|
|
48
56
|
postcode: string
|
|
@@ -51,14 +59,18 @@ interface CommuneRow {
|
|
|
51
59
|
lat: string | undefined
|
|
52
60
|
}
|
|
53
61
|
|
|
54
|
-
/**
|
|
62
|
+
/**
|
|
63
|
+
* One rendered admin-split variant.
|
|
64
|
+
*/
|
|
55
65
|
interface AdminSplitVariant {
|
|
56
66
|
raw: string
|
|
57
67
|
components: Partial<Record<ComponentTag, string>>
|
|
58
68
|
order: string
|
|
59
69
|
}
|
|
60
70
|
|
|
61
|
-
/**
|
|
71
|
+
/**
|
|
72
|
+
* Read the distinct commune TSV (commune, postcode, lon, lat); derive the département name.
|
|
73
|
+
*/
|
|
62
74
|
async function readCommunes(path: string): Promise<CommuneRow[]> {
|
|
63
75
|
const rows: CommuneRow[] = []
|
|
64
76
|
|
|
@@ -129,6 +141,10 @@ function render(random: () => number, c: CommuneRow): AdminSplitVariant {
|
|
|
129
141
|
return out
|
|
130
142
|
}
|
|
131
143
|
|
|
144
|
+
/**
|
|
145
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
146
|
+
* and `description` below for the surface form it generates.
|
|
147
|
+
*/
|
|
132
148
|
export const frAdminSplitRecipe: ShardRecipe = {
|
|
133
149
|
name: "fr-admin-split",
|
|
134
150
|
description: "FR admin-split rows: BAN communes → split département into `region` (+ canonical-FR preservation)",
|
|
@@ -139,14 +155,15 @@ export const frAdminSplitRecipe: ShardRecipe = {
|
|
|
139
155
|
async run(opts, write) {
|
|
140
156
|
// Legacy build-fr-admin-split-shard.mjs seeded `mulberry32(opts.seed)`.
|
|
141
157
|
const random = makeMulberry32(opts.seed)
|
|
142
|
-
const count = opts.count ??
|
|
158
|
+
const count = opts.count ?? 60_000
|
|
143
159
|
const source = opts.sourceName ?? "synth-fr-admin-split"
|
|
144
160
|
const communesPath = opts.communes ?? DEFAULT_COMMUNES
|
|
145
161
|
|
|
146
162
|
const pool = await readCommunes(communesPath)
|
|
163
|
+
|
|
147
164
|
console.error(` ${communesPath}: ${pool.length} communes with derived département`)
|
|
148
165
|
|
|
149
|
-
if (pool.length
|
|
166
|
+
if (!pool.length) {
|
|
150
167
|
throw new Error("No communes — build the TSV from BAN first (see the recipe header).")
|
|
151
168
|
}
|
|
152
169
|
|
|
@@ -165,14 +182,17 @@ export const frAdminSplitRecipe: ShardRecipe = {
|
|
|
165
182
|
|
|
166
183
|
if (!values.every((v) => raw.includes(v))) {
|
|
167
184
|
skipped++
|
|
185
|
+
|
|
168
186
|
continue
|
|
169
187
|
}
|
|
170
188
|
|
|
171
189
|
if (opts.golden) {
|
|
172
190
|
// Held-out eval slice for the centroid gate — carries the truth coordinate.
|
|
173
191
|
write(JSON.stringify({ raw, components, country: "FR", lat: Number(base.lat), lon: Number(base.lon) }) + "\n")
|
|
192
|
+
|
|
174
193
|
emitted++
|
|
175
194
|
orderCounts[order] = (orderCounts[order] ?? 0) + 1
|
|
195
|
+
|
|
176
196
|
continue
|
|
177
197
|
}
|
|
178
198
|
|
|
@@ -181,6 +201,7 @@ export const frAdminSplitRecipe: ShardRecipe = {
|
|
|
181
201
|
region: components.region,
|
|
182
202
|
postcode: components.postcode,
|
|
183
203
|
})
|
|
204
|
+
|
|
184
205
|
const canonical: CanonicalRow = {
|
|
185
206
|
raw,
|
|
186
207
|
components,
|
|
@@ -191,16 +212,20 @@ export const frAdminSplitRecipe: ShardRecipe = {
|
|
|
191
212
|
corpus_version: "0.5.0",
|
|
192
213
|
license: LICENSE,
|
|
193
214
|
}
|
|
215
|
+
|
|
194
216
|
const aligned = alignRow(canonical)
|
|
195
217
|
|
|
196
218
|
if (aligned.kind !== "labeled" || !aligned.row) {
|
|
197
219
|
skipped++
|
|
220
|
+
|
|
198
221
|
continue
|
|
199
222
|
}
|
|
223
|
+
|
|
200
224
|
write(
|
|
201
225
|
JSON.stringify({ ...aligned.row, synth_method: "fr-admin-split", synth_order: order, synth_base_id: null }) +
|
|
202
226
|
"\n"
|
|
203
227
|
)
|
|
228
|
+
|
|
204
229
|
emitted++
|
|
205
230
|
orderCounts[order] = (orderCounts[order] ?? 0) + 1
|
|
206
231
|
}
|
|
@@ -23,6 +23,10 @@
|
|
|
23
23
|
import { decomposeFrStreet } from "../adapters/ban/street-decompose.ts"
|
|
24
24
|
import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
|
|
25
25
|
|
|
26
|
+
/**
|
|
27
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
28
|
+
* and `description` below for the surface form it generates.
|
|
29
|
+
*/
|
|
26
30
|
export const frBareStreetRecipe: ShardRecipe = {
|
|
27
31
|
name: "fr-bare-street",
|
|
28
32
|
description:
|
|
@@ -43,24 +47,30 @@ export const frBareStreetRecipe: ShardRecipe = {
|
|
|
43
47
|
|
|
44
48
|
if (!fullStreet || !number || !locality) {
|
|
45
49
|
skipped++
|
|
50
|
+
|
|
46
51
|
continue
|
|
47
52
|
}
|
|
53
|
+
|
|
48
54
|
const { prefix, street } = decomposeFrStreet(fullStreet)
|
|
49
55
|
|
|
50
56
|
// The failing class is the prefix-led FR street; a no-prefix nom_voie ("La Ville Mois") isn't it.
|
|
51
57
|
if (!prefix || !street) {
|
|
52
58
|
skipped++
|
|
59
|
+
|
|
53
60
|
continue
|
|
54
61
|
}
|
|
62
|
+
|
|
55
63
|
const components: Record<string, string> = {
|
|
56
64
|
house_number: number,
|
|
57
65
|
street_prefix: prefix,
|
|
58
66
|
street,
|
|
59
67
|
locality,
|
|
60
68
|
}
|
|
69
|
+
|
|
61
70
|
// BARE comma form, number-before (FR's dominant order), NO postcode — the whole point.
|
|
62
71
|
const raw = `${number} ${prefix} ${street}, ${locality}`
|
|
63
72
|
const source_id = shardSourceID("synth-fr-bare-street", { ...components, v: String(read) })
|
|
73
|
+
|
|
64
74
|
const canonical = {
|
|
65
75
|
raw,
|
|
66
76
|
components,
|
|
@@ -73,19 +73,30 @@ import { readFileSync } from "node:fs"
|
|
|
73
73
|
import { decomposeFrStreet } from "../adapters/ban/street-decompose.ts"
|
|
74
74
|
import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
|
|
75
75
|
|
|
76
|
-
/**
|
|
76
|
+
/**
|
|
77
|
+
* House numbers, weighted toward the small values that dominate real BAN rows.
|
|
78
|
+
*/
|
|
79
|
+
/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
|
|
80
|
+
cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
|
|
81
|
+
output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
|
|
82
|
+
would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
|
|
83
|
+
extracted as named constants above. */
|
|
84
|
+
|
|
77
85
|
const HOUSE_NUMBERS = [
|
|
78
86
|
1, 1, 2, 2, 3, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 20, 21, 24, 27, 30, 33, 42, 57, 68, 84, 102, 115, 140,
|
|
79
87
|
]
|
|
80
|
-
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* FR alphanumeric house-number forms. `bis`/`ter` are separated; a bare letter is suffixed.
|
|
91
|
+
*/
|
|
81
92
|
const ALNUM_SUFFIXES = ["bis", "ter", "A", "B"]
|
|
82
93
|
|
|
83
94
|
const norm = (value: string): string =>
|
|
84
95
|
value
|
|
85
96
|
.normalize("NFD")
|
|
86
|
-
.
|
|
97
|
+
.replaceAll(/[̀-ͯ]/g, "")
|
|
87
98
|
.toLowerCase()
|
|
88
|
-
.
|
|
99
|
+
.replaceAll(/\s+/g, " ")
|
|
89
100
|
.trim()
|
|
90
101
|
|
|
91
102
|
/**
|
|
@@ -130,12 +141,21 @@ export function frTitleCase(value: string): string {
|
|
|
130
141
|
.join(" ")
|
|
131
142
|
}
|
|
132
143
|
|
|
133
|
-
/**
|
|
144
|
+
/**
|
|
145
|
+
* Does the street name carry a particle? Decides the particle vs bare classification.
|
|
146
|
+
*/
|
|
134
147
|
const PARTICLE = /\b(de la|de l'|du|des|de|d'|le|la|les)\b/i
|
|
135
|
-
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Does the street name carry date material (a year, or a day + French month)?
|
|
151
|
+
*/
|
|
136
152
|
const DATEISH =
|
|
137
153
|
/\b(1[0-9]|20)\d{2}\b|\b\d{1,2}\s+(janvier|f[ée]vrier|mars|avril|mai|juin|juillet|ao[ûu]t|septembre|octobre|novembre|d[ée]cembre)\b/i
|
|
138
154
|
|
|
155
|
+
/**
|
|
156
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
157
|
+
* and `description` below for the surface form it generates.
|
|
158
|
+
*/
|
|
139
159
|
export const frFragmentRecipe: ShardRecipe = {
|
|
140
160
|
name: "fr-fragment",
|
|
141
161
|
description:
|
|
@@ -199,12 +219,14 @@ export const frFragmentRecipe: ShardRecipe = {
|
|
|
199
219
|
|
|
200
220
|
if (!fullStreet) {
|
|
201
221
|
skipped++
|
|
222
|
+
|
|
202
223
|
continue
|
|
203
224
|
}
|
|
204
225
|
|
|
205
226
|
// THE SPLIT. A surface on the fragment board never enters training.
|
|
206
227
|
if (excluded.has(norm(fullStreet))) {
|
|
207
228
|
contaminated++
|
|
229
|
+
|
|
208
230
|
continue
|
|
209
231
|
}
|
|
210
232
|
|
|
@@ -214,6 +236,7 @@ export const frFragmentRecipe: ShardRecipe = {
|
|
|
214
236
|
// is a different problem and would muddy the signal.
|
|
215
237
|
if (!prefix || !street) {
|
|
216
238
|
skipped++
|
|
239
|
+
|
|
217
240
|
continue
|
|
218
241
|
}
|
|
219
242
|
|
|
@@ -226,11 +249,13 @@ export const frFragmentRecipe: ShardRecipe = {
|
|
|
226
249
|
const number = HOUSE_NUMBERS[Math.floor(random() * HOUSE_NUMBERS.length)]!
|
|
227
250
|
const alnum = random() < 0.25
|
|
228
251
|
const suffix = ALNUM_SUFFIXES[Math.floor(random() * ALNUM_SUFFIXES.length)]!
|
|
252
|
+
|
|
229
253
|
const houseNumber = alnum
|
|
230
254
|
? suffix === "bis" || suffix === "ter"
|
|
231
255
|
? `${number} ${suffix}`
|
|
232
256
|
: `${number}${suffix}`
|
|
233
257
|
: String(number)
|
|
258
|
+
|
|
234
259
|
components.house_number = houseNumber
|
|
235
260
|
raw = `${houseNumber} ${prefix} ${street}`
|
|
236
261
|
klass = alnum ? "alnum-housenumber" : "street-housenumber"
|
|
@@ -260,11 +285,11 @@ export const frFragmentRecipe: ShardRecipe = {
|
|
|
260
285
|
}
|
|
261
286
|
}
|
|
262
287
|
|
|
263
|
-
//
|
|
288
|
+
// MARK: counter-distribution — bare localities
|
|
264
289
|
// Minted last so the locality pool is complete. Without these the model can satisfy every row
|
|
265
290
|
// above by flipping its default from "bare => locality" to "bare => street", which trades one
|
|
266
291
|
// broken prior for another and would show up as bare-locality collapsing on the board.
|
|
267
|
-
const pool = [...localities].
|
|
292
|
+
const pool = [...localities].toSorted()
|
|
268
293
|
const wanted = Math.round((emitted / Math.max(1, 1 - bareLocalityProb)) * bareLocalityProb)
|
|
269
294
|
|
|
270
295
|
for (let i = 0; i < wanted && pool.length; i++) {
|