@mailwoman/corpus 8.1.0 → 8.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/adapter.d.ts +15 -5
- package/out/src/adapter.d.ts.map +1 -1
- package/out/src/adapter.js +11 -9
- package/out/src/adapter.js.map +1 -1
- package/out/src/adapters/ban/adapter.d.ts +7 -0
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +9 -2
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +5 -1
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +14 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +15 -2
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +11 -0
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +24 -5
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +11 -0
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +14 -1
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +10 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +11 -2
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts +18 -6
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +16 -6
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +11 -0
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +18 -3
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +11 -0
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +12 -1
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +11 -0
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +11 -0
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +15 -2
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +11 -0
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +13 -2
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +14 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +19 -4
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +12 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +17 -2
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +12 -1
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +11 -0
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +15 -2
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +7 -0
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +18 -3
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +13 -3
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +15 -5
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +7 -0
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +9 -2
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/align.d.ts +12 -4
- package/out/src/align.d.ts.map +1 -1
- package/out/src/align.js +9 -5
- package/out/src/align.js.map +1 -1
- package/out/src/build.d.ts +24 -8
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +3 -3
- package/out/src/build.js.map +1 -1
- package/out/src/codex/us-fips-state.d.ts +6 -2
- package/out/src/codex/us-fips-state.d.ts.map +1 -1
- package/out/src/codex/us-fips-state.js +3 -1
- package/out/src/codex/us-fips-state.js.map +1 -1
- package/out/src/golden.d.ts +21 -7
- package/out/src/golden.d.ts.map +1 -1
- package/out/src/golden.js +18 -10
- package/out/src/golden.js.map +1 -1
- package/out/src/license.d.ts +3 -1
- package/out/src/license.d.ts.map +1 -1
- package/out/src/license.js +4 -2
- package/out/src/license.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts +4 -2
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +5 -4
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/schema.d.ts +14 -10
- package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
- package/out/src/parquet-wrapper/schema.js +6 -2
- package/out/src/parquet-wrapper/schema.js.map +1 -1
- package/out/src/parquet-wrapper/writer.d.ts +12 -4
- package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
- package/out/src/parquet-wrapper/writer.js +12 -4
- package/out/src/parquet-wrapper/writer.js.map +1 -1
- package/out/src/parquet.d.ts +21 -7
- package/out/src/parquet.d.ts.map +1 -1
- package/out/src/parquet.js +15 -4
- package/out/src/parquet.js.map +1 -1
- package/out/src/runner.d.ts +39 -13
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +7 -3
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts +4 -0
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +14 -5
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/boundary-stress.d.ts +4 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +13 -7
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts +4 -0
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +39 -11
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts +4 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +14 -3
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts +4 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.js +4 -0
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts +4 -0
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +25 -8
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts +4 -0
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +21 -8
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts +4 -0
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +27 -8
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts +4 -0
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +16 -6
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.d.ts +4 -0
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/house-venue.js +4 -0
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts +9 -3
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +12 -4
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts +4 -0
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +27 -13
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +4 -0
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +9 -4
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/nl-postcode.d.ts +4 -0
- package/out/src/shard-recipes/nl-postcode.d.ts.map +1 -1
- package/out/src/shard-recipes/nl-postcode.js +8 -2
- package/out/src/shard-recipes/nl-postcode.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts +4 -0
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +8 -2
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts +4 -0
- package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street-led.js +5 -1
- package/out/src/shard-recipes/no-street-led.js.map +1 -1
- package/out/src/shard-recipes/no-street.d.ts +4 -0
- package/out/src/shard-recipes/no-street.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street.js +4 -0
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +4 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +99 -43
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts +4 -0
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +4 -0
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +48 -16
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +10 -6
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/si-bare-village.d.ts +4 -0
- package/out/src/shard-recipes/si-bare-village.d.ts.map +1 -1
- package/out/src/shard-recipes/si-bare-village.js +4 -0
- package/out/src/shard-recipes/si-bare-village.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +4 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +29 -10
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.d.ts +4 -0
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
- package/out/src/shard-recipes/street-bare.js +4 -0
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.d.ts +4 -0
- package/out/src/shard-recipes/street.d.ts.map +1 -1
- package/out/src/shard-recipes/street.js +4 -0
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/unit.d.ts +4 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +45 -14
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/split.d.ts +18 -6
- package/out/src/split.d.ts.map +1 -1
- package/out/src/split.js +4 -2
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize-anchor-absorption.d.ts +7 -1
- package/out/src/synthesize-anchor-absorption.d.ts.map +1 -1
- package/out/src/synthesize-anchor-absorption.js +28 -14
- package/out/src/synthesize-anchor-absorption.js.map +1 -1
- package/out/src/synthesize-boundary-stress.d.ts +3 -1
- package/out/src/synthesize-boundary-stress.d.ts.map +1 -1
- package/out/src/synthesize-boundary-stress.js +35 -28
- package/out/src/synthesize-boundary-stress.js.map +1 -1
- package/out/src/synthesize-german.d.ts +15 -5
- package/out/src/synthesize-german.d.ts.map +1 -1
- package/out/src/synthesize-german.js +9 -3
- package/out/src/synthesize-german.js.map +1 -1
- package/out/src/synthesize-house-venue.d.ts.map +1 -1
- package/out/src/synthesize-house-venue.js +15 -15
- package/out/src/synthesize-house-venue.js.map +1 -1
- package/out/src/synthesize-intersection.d.ts +9 -3
- package/out/src/synthesize-intersection.d.ts.map +1 -1
- package/out/src/synthesize-intersection.js +25 -6
- package/out/src/synthesize-intersection.js.map +1 -1
- package/out/src/synthesize-no-street.d.ts +3 -1
- package/out/src/synthesize-no-street.d.ts.map +1 -1
- package/out/src/synthesize-no-street.js +5 -6
- package/out/src/synthesize-no-street.js.map +1 -1
- package/out/src/synthesize-po-box.d.ts +15 -5
- package/out/src/synthesize-po-box.d.ts.map +1 -1
- package/out/src/synthesize-po-box.js +21 -7
- package/out/src/synthesize-po-box.js.map +1 -1
- package/out/src/synthesize-street.d.ts +3 -1
- package/out/src/synthesize-street.d.ts.map +1 -1
- package/out/src/synthesize-street.js +6 -1
- package/out/src/synthesize-street.js.map +1 -1
- package/out/src/synthesize.d.ts +39 -13
- package/out/src/synthesize.d.ts.map +1 -1
- package/out/src/synthesize.js +66 -36
- package/out/src/synthesize.js.map +1 -1
- package/out/src/tokenize.d.ts +15 -5
- package/out/src/tokenize.d.ts.map +1 -1
- package/out/src/tokenize.js.map +1 -1
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +3 -1
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/audit.d.ts.map +1 -1
- package/out/src/tools/audit.js +20 -8
- package/out/src/tools/audit.js.map +1 -1
- package/out/src/tools/corpus-stats.d.ts.map +1 -1
- package/out/src/tools/corpus-stats.js.map +1 -1
- package/out/src/tools/fetch/ban.d.ts.map +1 -1
- package/out/src/tools/fetch/ban.js +17 -10
- package/out/src/tools/fetch/ban.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts +30 -10
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +26 -6
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/hrsa.d.ts.map +1 -1
- package/out/src/tools/fetch/hrsa.js.map +1 -1
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +18 -18
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +3 -1
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +3 -1
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts +21 -7
- package/out/src/tools/fetch/nad.d.ts.map +1 -1
- package/out/src/tools/fetch/nad.js +5 -5
- package/out/src/tools/fetch/nad.js.map +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +9 -15
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts +3 -1
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/src/tools/fetch/openaddresses.js +32 -23
- package/out/src/tools/fetch/openaddresses.js.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +3 -1
- package/out/src/tools/fetch/ppd.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.js +14 -3
- package/out/src/tools/fetch/ppd.js.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.js +13 -6
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
- package/out/src/tools/fetch/state-sources.js +8 -3
- package/out/src/tools/fetch/state-sources.js.map +1 -1
- package/out/src/tools/fetch/tiger-full.d.ts +9 -3
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
- package/out/src/tools/fetch/tiger-full.js +35 -20
- package/out/src/tools/fetch/tiger-full.js.map +1 -1
- package/out/src/tools/gb-title-case.d.ts.map +1 -1
- package/out/src/tools/gb-title-case.js.map +1 -1
- package/out/src/tools/golden-expand.d.ts +24 -8
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +20 -12
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-promote.d.ts +18 -6
- package/out/src/tools/golden-promote.d.ts.map +1 -1
- package/out/src/tools/golden-promote.js +22 -7
- package/out/src/tools/golden-promote.js.map +1 -1
- package/out/src/tools/ingest-csv.d.ts +3 -1
- package/out/src/tools/ingest-csv.d.ts.map +1 -1
- package/out/src/tools/ingest-csv.js +17 -19
- package/out/src/tools/ingest-csv.js.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts +18 -6
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
- package/out/src/tools/jsonl-to-parquet.js +11 -5
- package/out/src/tools/jsonl-to-parquet.js.map +1 -1
- package/out/src/tools/lint-shard-vocab.d.ts +36 -12
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
- package/out/src/tools/lint-shard-vocab.js +25 -11
- package/out/src/tools/lint-shard-vocab.js.map +1 -1
- package/out/src/tools/lint-shard.d.ts +30 -10
- package/out/src/tools/lint-shard.d.ts.map +1 -1
- package/out/src/tools/lint-shard.js +33 -20
- package/out/src/tools/lint-shard.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +6 -4
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/shard-kryptonite.d.ts +6 -2
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +1 -1
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts +9 -3
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +2 -2
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/types.d.ts +51 -17
- package/out/src/types.d.ts.map +1 -1
- package/out/src/wof-json.d.ts +9 -3
- package/out/src/wof-json.d.ts.map +1 -1
- package/out/src/wof-json.js +6 -4
- package/out/src/wof-json.js.map +1 -1
- package/package.json +7 -6
- package/src/adapter.ts +24 -13
- package/src/adapters/ban/adapter.ts +12 -2
- package/src/adapters/ban/street-decompose.ts +7 -1
- package/src/adapters/fcc-bdc/adapter.ts +19 -3
- package/src/adapters/geonames/adapter.ts +28 -5
- package/src/adapters/geonames-postal/adapter.ts +17 -1
- package/src/adapters/gnaf/adapter.ts +15 -2
- package/src/adapters/gnaf/assemble.ts +40 -10
- package/src/adapters/index.ts +13 -0
- package/src/adapters/openaddresses/adapter.ts +22 -3
- package/src/adapters/overture/adapter.ts +18 -2
- package/src/adapters/state-hi-schools/adapter.ts +15 -0
- package/src/adapters/state-ia-contractors/adapter.ts +15 -0
- package/src/adapters/state-ny-notaries/adapter.ts +15 -0
- package/src/adapters/state-tx-notaries/adapter.ts +18 -2
- package/src/adapters/synth-po-box/adapter.ts +19 -2
- package/src/adapters/tiger/adapter.ts +23 -4
- package/src/adapters/tiger/street-decompose.ts +1 -0
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +18 -2
- package/src/adapters/usgov-imls-pls/adapter.ts +15 -0
- package/src/adapters/usgov-irs-bmf/adapter.ts +19 -2
- package/src/adapters/usgov-nad/adapter.ts +14 -1
- package/src/adapters/usgov-nppes/adapter.ts +13 -0
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +21 -3
- package/src/adapters/wof-admin-jp/adapter.ts +20 -3
- package/src/adapters/wof-admin-json/adapter.ts +22 -7
- package/src/adapters/wof-postalcode-json/adapter.ts +10 -2
- package/src/align.ts +21 -8
- package/src/build.ts +48 -11
- package/src/codex/us-fips-state.ts +6 -2
- package/src/golden.ts +28 -13
- package/src/license.ts +4 -2
- package/src/parquet-wrapper/reader.ts +6 -4
- package/src/parquet-wrapper/schema.ts +14 -10
- package/src/parquet-wrapper/writer.ts +12 -4
- package/src/parquet.ts +39 -8
- package/src/runner.ts +49 -15
- package/src/shard-recipes/anchor-absorption.ts +21 -5
- package/src/shard-recipes/boundary-stress.ts +20 -7
- package/src/shard-recipes/country-balanced.ts +78 -13
- package/src/shard-recipes/cz-pcfirst-preposition.ts +9 -0
- package/src/shard-recipes/fr-admin-split.ts +30 -5
- package/src/shard-recipes/fr-bare-street.ts +10 -0
- package/src/shard-recipes/fr-fragment.ts +33 -8
- package/src/shard-recipes/fr-lieudit.ts +35 -9
- package/src/shard-recipes/fr-order.ts +47 -9
- package/src/shard-recipes/german.ts +32 -6
- package/src/shard-recipes/house-venue.ts +7 -0
- package/src/shard-recipes/index.ts +12 -4
- package/src/shard-recipes/intersection.ts +63 -14
- package/src/shard-recipes/locale.ts +37 -4
- package/src/shard-recipes/nl-postcode.ts +12 -2
- package/src/shard-recipes/no-fragment.ts +12 -2
- package/src/shard-recipes/no-street-led.ts +11 -1
- package/src/shard-recipes/no-street.ts +7 -0
- package/src/shard-recipes/po-box-cedex.ts +145 -45
- package/src/shard-recipes/po-box.ts +7 -0
- package/src/shard-recipes/scaffold.ts +52 -20
- package/src/shard-recipes/si-bare-village.ts +9 -0
- package/src/shard-recipes/street-affix.ts +88 -13
- package/src/shard-recipes/street-bare.ts +10 -0
- package/src/shard-recipes/street.ts +7 -0
- package/src/shard-recipes/unit.ts +79 -16
- package/src/split.ts +28 -7
- package/src/synthesize-anchor-absorption.ts +41 -14
- package/src/synthesize-boundary-stress.ts +57 -30
- package/src/synthesize-german.ts +30 -7
- package/src/synthesize-house-venue.ts +22 -16
- package/src/synthesize-intersection.ts +34 -7
- package/src/synthesize-no-street.ts +18 -10
- package/src/synthesize-po-box.ts +42 -10
- package/src/synthesize-street.ts +10 -2
- package/src/synthesize.ts +81 -38
- package/src/tokenize.ts +15 -5
- package/src/tools/align-shard.ts +6 -1
- package/src/tools/audit.ts +57 -11
- package/src/tools/corpus-stats.ts +9 -0
- package/src/tools/fetch/ban.ts +32 -10
- package/src/tools/fetch/download.ts +49 -12
- package/src/tools/fetch/hrsa.ts +3 -0
- package/src/tools/fetch/imls-pls.ts +28 -18
- package/src/tools/fetch/index.ts +3 -1
- package/src/tools/fetch/nad.ts +39 -12
- package/src/tools/fetch/nppes.ts +17 -15
- package/src/tools/fetch/openaddresses.ts +52 -24
- package/src/tools/fetch/ppd.ts +22 -3
- package/src/tools/fetch/state-hi-schools.ts +17 -6
- package/src/tools/fetch/state-sources.ts +16 -3
- package/src/tools/fetch/tiger-full.ts +66 -23
- package/src/tools/gb-title-case.ts +1 -0
- package/src/tools/golden-expand.ts +64 -20
- package/src/tools/golden-promote.ts +53 -13
- package/src/tools/ingest-csv.ts +41 -26
- package/src/tools/jsonl-to-parquet.ts +30 -10
- package/src/tools/lint-shard-vocab.ts +75 -24
- package/src/tools/lint-shard.ts +75 -31
- package/src/tools/overlay-manifest.ts +7 -4
- package/src/tools/shard-kryptonite.ts +11 -3
- package/src/tools/shard-translit.ts +20 -5
- package/src/types.ts +51 -17
- package/src/wof-json.ts +18 -7
|
@@ -51,16 +51,20 @@ interface County {
|
|
|
51
51
|
state: string
|
|
52
52
|
regime: string
|
|
53
53
|
}
|
|
54
|
+
|
|
54
55
|
const TRAIN_COUNTIES: readonly County[] = [
|
|
55
56
|
{ fips: "17031", state: "IL", regime: "grid-city" },
|
|
56
57
|
{ fips: "34027", state: "NJ", regime: "suburb" },
|
|
57
58
|
]
|
|
59
|
+
|
|
58
60
|
const GOLDEN_COUNTIES: readonly County[] = [{ fips: "50023", state: "VT", regime: "rural" }]
|
|
59
61
|
|
|
60
62
|
const EVAL_GOLD_PATH = repoRootPath("data", "eval", "external", "intersection-real.jsonl")
|
|
61
63
|
const OA_COOK = { zip: "/tmp/oa-cache/us__il__cook.zip", csv: "us/il/cook.csv" }
|
|
62
64
|
|
|
63
|
-
/**
|
|
65
|
+
/**
|
|
66
|
+
* One real crossing extracted from a county's TIGER EDGES shapefile.
|
|
67
|
+
*/
|
|
64
68
|
interface Crossing {
|
|
65
69
|
a: string
|
|
66
70
|
b: string
|
|
@@ -79,6 +83,7 @@ interface Form {
|
|
|
79
83
|
w: number
|
|
80
84
|
render: (a: string, b: string) => string
|
|
81
85
|
}
|
|
86
|
+
|
|
82
87
|
const FORMS: readonly Form[] = [
|
|
83
88
|
{ id: "amp", w: 0.2, render: (a, b) => `${a} & ${b}` },
|
|
84
89
|
{ id: "and", w: 0.2, render: (a, b) => `${a} and ${b}` },
|
|
@@ -100,6 +105,7 @@ interface Tail {
|
|
|
100
105
|
id: string
|
|
101
106
|
w: number
|
|
102
107
|
}
|
|
108
|
+
|
|
103
109
|
const TAILS: readonly Tail[] = [
|
|
104
110
|
{ id: "bare", w: 0.55 },
|
|
105
111
|
{ id: "region", w: 0.16 },
|
|
@@ -113,6 +119,7 @@ interface Casing {
|
|
|
113
119
|
w: number
|
|
114
120
|
apply: (s: string) => string
|
|
115
121
|
}
|
|
122
|
+
|
|
116
123
|
const CASES: readonly Casing[] = [
|
|
117
124
|
{ id: "as-is", w: 0.82, apply: (s) => s },
|
|
118
125
|
{ id: "upper", w: 0.12, apply: (s) => s.toUpperCase() },
|
|
@@ -131,7 +138,9 @@ const CONNECTOR_O_TOKENS = new Set(["and", "at", "of", "corner", "intersection"]
|
|
|
131
138
|
*/
|
|
132
139
|
const BAD_NAME = /[,&@/]|\b(and|at)\b/i
|
|
133
140
|
|
|
134
|
-
/**
|
|
141
|
+
/**
|
|
142
|
+
* Punctuation a connector form may leave between spans (besides whitespace): `, & @ /`.
|
|
143
|
+
*/
|
|
135
144
|
const CONNECTOR_PUNCT_RE = /^[\s,&@/]*$/
|
|
136
145
|
|
|
137
146
|
function weightedPick<T extends { w: number }>(items: readonly T[], random: () => number): T {
|
|
@@ -144,10 +153,12 @@ function weightedPick<T extends { w: number }>(items: readonly T[], random: () =
|
|
|
144
153
|
if (r <= 0) return item
|
|
145
154
|
}
|
|
146
155
|
|
|
147
|
-
return items
|
|
156
|
+
return items.at(-1)!
|
|
148
157
|
}
|
|
149
158
|
|
|
150
|
-
/**
|
|
159
|
+
/**
|
|
160
|
+
* Minimal RFC-4180-ish splitter (handles quoted fields) — same as the affix builder.
|
|
161
|
+
*/
|
|
151
162
|
function splitCSV(line: string): string[] {
|
|
152
163
|
const out: string[] = []
|
|
153
164
|
let cur = ""
|
|
@@ -160,6 +171,7 @@ function splitCSV(line: string): string[] {
|
|
|
160
171
|
if (c === '"') {
|
|
161
172
|
if (line[i + 1] === '"') {
|
|
162
173
|
cur += '"'
|
|
174
|
+
|
|
163
175
|
i++
|
|
164
176
|
} else {
|
|
165
177
|
inQ = false
|
|
@@ -176,15 +188,20 @@ function splitCSV(line: string): string[] {
|
|
|
176
188
|
cur += c
|
|
177
189
|
}
|
|
178
190
|
}
|
|
191
|
+
|
|
179
192
|
out.push(cur)
|
|
180
193
|
|
|
181
194
|
return out
|
|
182
195
|
}
|
|
183
196
|
|
|
184
|
-
/**
|
|
185
|
-
|
|
197
|
+
/**
|
|
198
|
+
* Order-insensitive crossing key, for eval-leakage exclusion + pair dedup.
|
|
199
|
+
*/
|
|
200
|
+
const pairKey = (a: string, b: string): string => [a.toLowerCase(), b.toLowerCase()].toSorted().join("\u001F")
|
|
186
201
|
|
|
187
|
-
/**
|
|
202
|
+
/**
|
|
203
|
+
* Load the eval's crossings so neither train nor golden ever sees them.
|
|
204
|
+
*/
|
|
188
205
|
function readEvalExclusions(): { nodes: Set<number>; pairs: Set<string> } {
|
|
189
206
|
const nodes = new Set<number>()
|
|
190
207
|
const pairs = new Set<string>()
|
|
@@ -217,6 +234,7 @@ async function extractCrossings(
|
|
|
217
234
|
seed: number
|
|
218
235
|
): Promise<Crossing[]> {
|
|
219
236
|
const shp = `${edgesDir}/tl_2023_${county.fips}_edges.shp`
|
|
237
|
+
|
|
220
238
|
const result = await db.runAndReadAll(`
|
|
221
239
|
WITH incidence AS (
|
|
222
240
|
SELECT TNIDF AS node, FULLNAME AS name, ZIPL AS zip
|
|
@@ -238,6 +256,7 @@ async function extractCrossings(
|
|
|
238
256
|
WHERE len(names[1]) >= 6 AND len(names[2]) >= 6
|
|
239
257
|
ORDER BY h
|
|
240
258
|
`)
|
|
259
|
+
|
|
241
260
|
const out: Crossing[] = []
|
|
242
261
|
|
|
243
262
|
for (const r of result.getRowObjects()) {
|
|
@@ -254,7 +273,9 @@ async function extractCrossings(
|
|
|
254
273
|
return out
|
|
255
274
|
}
|
|
256
275
|
|
|
257
|
-
/**
|
|
276
|
+
/**
|
|
277
|
+
* ZIP → majority city from the cached OA Cook-county CSV (real ZIP/city pairings).
|
|
278
|
+
*/
|
|
258
279
|
function buildZipCityMap(): Map<string, string> {
|
|
259
280
|
const r = spawnSync("unzip", ["-p", OA_COOK.zip, OA_COOK.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
|
|
260
281
|
|
|
@@ -263,6 +284,7 @@ function buildZipCityMap(): Map<string, string> {
|
|
|
263
284
|
|
|
264
285
|
return new Map()
|
|
265
286
|
}
|
|
287
|
+
|
|
266
288
|
const lines = r.stdout.toString("utf8").split(/\r?\n/)
|
|
267
289
|
|
|
268
290
|
if (lines.length < 2) return new Map()
|
|
@@ -284,8 +306,10 @@ function buildZipCityMap(): Map<string, string> {
|
|
|
284
306
|
if (!byCity) {
|
|
285
307
|
counts.set(zip, (byCity = new Map()))
|
|
286
308
|
}
|
|
309
|
+
|
|
287
310
|
byCity.set(city, (byCity.get(city) ?? 0) + 1)
|
|
288
311
|
}
|
|
312
|
+
|
|
289
313
|
const map = new Map<string, string>()
|
|
290
314
|
|
|
291
315
|
for (const [zip, byCity] of counts) {
|
|
@@ -392,7 +416,7 @@ function auditRow(row: LabeledRow, components: Partial<Record<ComponentTag, stri
|
|
|
392
416
|
}
|
|
393
417
|
}
|
|
394
418
|
|
|
395
|
-
if (errors.length
|
|
419
|
+
if (errors.length) return errors
|
|
396
420
|
|
|
397
421
|
const compCount = Object.keys(components).length
|
|
398
422
|
|
|
@@ -407,8 +431,10 @@ function auditRow(row: LabeledRow, components: Partial<Record<ComponentTag, stri
|
|
|
407
431
|
|
|
408
432
|
if (indices.length !== 1) {
|
|
409
433
|
errors.push(`${tag}: expected 1 span, got ${indices.length}`)
|
|
434
|
+
|
|
410
435
|
continue
|
|
411
436
|
}
|
|
437
|
+
|
|
412
438
|
const idx = indices[0]!
|
|
413
439
|
const got = raw.slice(span_starts[idx]!, span_ends[idx]!)
|
|
414
440
|
|
|
@@ -425,6 +451,7 @@ function auditRow(row: LabeledRow, components: Partial<Record<ComponentTag, stri
|
|
|
425
451
|
if (span_starts[i]! > cursor) {
|
|
426
452
|
uncovered.push(raw.slice(cursor, span_starts[i]!))
|
|
427
453
|
}
|
|
454
|
+
|
|
428
455
|
cursor = span_ends[i]!
|
|
429
456
|
}
|
|
430
457
|
|
|
@@ -440,7 +467,8 @@ function auditRow(row: LabeledRow, components: Partial<Record<ComponentTag, stri
|
|
|
440
467
|
errors.push(`illegal uncovered word "${word}"`)
|
|
441
468
|
}
|
|
442
469
|
}
|
|
443
|
-
|
|
470
|
+
|
|
471
|
+
const punctOnly = segment.replaceAll(/[\p{L}\p{N}]+/gu, "")
|
|
444
472
|
|
|
445
473
|
if (!CONNECTOR_PUNCT_RE.test(punctOnly)) {
|
|
446
474
|
errors.push(`illegal uncovered punctuation in "${segment}"`)
|
|
@@ -450,6 +478,10 @@ function auditRow(row: LabeledRow, components: Partial<Record<ComponentTag, stri
|
|
|
450
478
|
return errors
|
|
451
479
|
}
|
|
452
480
|
|
|
481
|
+
/**
|
|
482
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
483
|
+
* and `description` below for the surface form it generates.
|
|
484
|
+
*/
|
|
453
485
|
export const intersectionRecipe: ShardRecipe = {
|
|
454
486
|
name: "intersection",
|
|
455
487
|
description: "Real-pair intersection rows (US): TIGER 2023 EDGES crossings → audited intersection_a/b labels",
|
|
@@ -458,11 +490,12 @@ export const intersectionRecipe: ShardRecipe = {
|
|
|
458
490
|
async run(opts, write) {
|
|
459
491
|
// Legacy build-intersection-shard.mjs seeded `mulberry32(opts.seed)`.
|
|
460
492
|
const random = makeMulberry32(opts.seed)
|
|
461
|
-
const count = opts.count ??
|
|
493
|
+
const count = opts.count ?? 40_000
|
|
462
494
|
const source = opts.sourceName ?? "synth-intersection"
|
|
463
495
|
const edgesDir = opts.edgesDir ?? "/tmp/tiger-edges"
|
|
464
496
|
const counties = opts.golden ? GOLDEN_COUNTIES : TRAIN_COUNTIES
|
|
465
497
|
const exclusions = readEvalExclusions()
|
|
498
|
+
|
|
466
499
|
console.error(` eval exclusions: ${exclusions.nodes.size} nodes, ${exclusions.pairs.size} pairs`)
|
|
467
500
|
|
|
468
501
|
const { DuckDBInstance } = await import("@duckdb/node-api")
|
|
@@ -484,26 +517,32 @@ export const intersectionRecipe: ShardRecipe = {
|
|
|
484
517
|
|
|
485
518
|
if (exclusions.nodes.has(c.node) || exclusions.pairs.has(key)) {
|
|
486
519
|
stats.evalExcluded++
|
|
520
|
+
|
|
487
521
|
continue
|
|
488
522
|
}
|
|
489
523
|
|
|
490
524
|
if (BAD_NAME.test(c.a) || BAD_NAME.test(c.b) || c.a.includes(c.b) || c.b.includes(c.a)) {
|
|
491
525
|
stats.badName++
|
|
526
|
+
|
|
492
527
|
continue
|
|
493
528
|
}
|
|
494
529
|
|
|
495
530
|
if (seenPairs.has(key)) {
|
|
496
531
|
stats.dupPair++
|
|
532
|
+
|
|
497
533
|
continue
|
|
498
534
|
}
|
|
535
|
+
|
|
499
536
|
seenPairs.add(key)
|
|
500
537
|
pool.push(c)
|
|
538
|
+
|
|
501
539
|
kept++
|
|
502
540
|
}
|
|
541
|
+
|
|
503
542
|
console.error(` ${county.fips} (${county.state}, ${county.regime}): ${crossings.length} crossings, ${kept} kept`)
|
|
504
543
|
}
|
|
505
544
|
|
|
506
|
-
if (pool.length
|
|
545
|
+
if (!pool.length) {
|
|
507
546
|
throw new Error(`No crossings found — are the TIGER EDGES shapefiles present in ${edgesDir}?`)
|
|
508
547
|
}
|
|
509
548
|
|
|
@@ -531,6 +570,7 @@ export const intersectionRecipe: ShardRecipe = {
|
|
|
531
570
|
|
|
532
571
|
if (seenRaw.has(raw)) {
|
|
533
572
|
skipped++
|
|
573
|
+
|
|
534
574
|
continue
|
|
535
575
|
}
|
|
536
576
|
|
|
@@ -542,7 +582,9 @@ export const intersectionRecipe: ShardRecipe = {
|
|
|
542
582
|
caseCounts[caseID] = (caseCounts[caseID] ?? 0) + 1
|
|
543
583
|
countyCounts[crossing.fips] = (countyCounts[crossing.fips] ?? 0) + 1
|
|
544
584
|
usedCrossings.add(crossing.node)
|
|
585
|
+
|
|
545
586
|
emitted++
|
|
587
|
+
|
|
546
588
|
continue
|
|
547
589
|
}
|
|
548
590
|
|
|
@@ -556,18 +598,22 @@ export const intersectionRecipe: ShardRecipe = {
|
|
|
556
598
|
corpus_version: "0.4.0",
|
|
557
599
|
license: "TIGER/Line 2023 EDGES (US Census, public domain) real street pairs; OA Cook IL zip-to-city tails",
|
|
558
600
|
}
|
|
601
|
+
|
|
559
602
|
// Verbatim-only alignment: raw is built from the component values, so a fuzzy fallback could
|
|
560
603
|
// only ever mislabel (e.g. claim a lookalike window for a near-duplicate street).
|
|
561
604
|
const aligned = alignRow(canonical, { maxEditDistance: 0 })
|
|
562
605
|
|
|
563
606
|
if (aligned.kind !== "labeled" || !aligned.row) {
|
|
564
607
|
skipped++
|
|
608
|
+
|
|
565
609
|
continue
|
|
566
610
|
}
|
|
611
|
+
|
|
567
612
|
const violations = auditRow(aligned.row, components)
|
|
568
613
|
|
|
569
|
-
if (violations.length
|
|
614
|
+
if (violations.length) {
|
|
570
615
|
auditErrors.push({ raw, violations })
|
|
616
|
+
|
|
571
617
|
continue
|
|
572
618
|
}
|
|
573
619
|
|
|
@@ -582,6 +628,7 @@ export const intersectionRecipe: ShardRecipe = {
|
|
|
582
628
|
if (samples.length < FORMS.length && !samples.some((s) => s.form === formID)) {
|
|
583
629
|
samples.push({ form: formID, raw, tokens: aligned.row.tokens, labels: aligned.row.labels })
|
|
584
630
|
}
|
|
631
|
+
|
|
585
632
|
emitted++
|
|
586
633
|
}
|
|
587
634
|
|
|
@@ -599,7 +646,9 @@ export const intersectionRecipe: ShardRecipe = {
|
|
|
599
646
|
source: "TIGER2023 EDGES via DuckDB ST_Read; node = 2 distinct S1* FULLNAMEs; eval crossings excluded",
|
|
600
647
|
samples,
|
|
601
648
|
}
|
|
649
|
+
|
|
602
650
|
writeFileSync(opts.output.replace(/\.jsonl$/, ".report.json"), JSON.stringify(report, null, "\t"))
|
|
651
|
+
|
|
603
652
|
console.error(
|
|
604
653
|
`Done: emitted ${emitted} rows (skipped ${skipped}) from ${usedCrossings.size}/${pool.length} real crossings. → ${opts.output}\n` +
|
|
605
654
|
` forms: ${JSON.stringify(formCounts)}\n` +
|
|
@@ -608,7 +657,7 @@ export const intersectionRecipe: ShardRecipe = {
|
|
|
608
657
|
` audit: ${auditErrors.length} violation(s)`
|
|
609
658
|
)
|
|
610
659
|
|
|
611
|
-
if (auditErrors.length
|
|
660
|
+
if (auditErrors.length) {
|
|
612
661
|
throw new Error(`AUDIT FAILED — first violation: ${JSON.stringify(auditErrors[0])}`)
|
|
613
662
|
}
|
|
614
663
|
|
|
@@ -239,11 +239,14 @@ export async function readTuples(part: LocalePart, rng: () => number): Promise<L
|
|
|
239
239
|
input = createReadStream(part.path)
|
|
240
240
|
} else {
|
|
241
241
|
const child = spawn("unzip", ["-p", part.zip!, part.csv!])
|
|
242
|
+
|
|
242
243
|
child.on("error", (err) => {
|
|
243
244
|
console.error(` WARN: unzip failed for ${part.zip}: ${err.message}`)
|
|
244
245
|
})
|
|
246
|
+
|
|
245
247
|
input = child.stdout!
|
|
246
248
|
}
|
|
249
|
+
|
|
247
250
|
const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
|
|
248
251
|
const reservoir: LocaleBaseTuple[] = []
|
|
249
252
|
let cols: ColumnIndex | null = null
|
|
@@ -263,7 +266,9 @@ export async function readTuples(part: LocalePart, rng: () => number): Promise<L
|
|
|
263
266
|
})) {
|
|
264
267
|
if (header === null) {
|
|
265
268
|
header = cells.map((h) => h.trim().toLowerCase())
|
|
269
|
+
// oxlint-disable-next-line no-loop-func -- the binding is per-iteration (for-of/for-await) and the batch is awaited before the next
|
|
266
270
|
const ix = (name: string): number => header!.indexOf(name)
|
|
271
|
+
|
|
267
272
|
// `cnigRaw` (ES pedanía only): the RAW CNIG header has no NUMBER/STREET/CITY/DISTRICT/REGION/POSTCODE
|
|
268
273
|
// at all — `numero`/`nombre_via`/`poblacion`/`municipio`/`comunidad_autonoma`/`cod_postal` instead.
|
|
269
274
|
// See {@link LocalePart.cnigRaw}.
|
|
@@ -291,6 +296,7 @@ export async function readTuples(part: LocalePart, rng: () => number): Promise<L
|
|
|
291
296
|
}
|
|
292
297
|
|
|
293
298
|
if (cols === null) continue
|
|
299
|
+
|
|
294
300
|
// cnigRaw: STREET is split into a road-type column (`tipo_vial`, e.g. "CARRETERA") and the name
|
|
295
301
|
// (`nombre_via`) — rejoin them exactly as OA's own conform step does (verified byte-identical to the
|
|
296
302
|
// conformed STREET column for the same source row). Every other part's header already carries the
|
|
@@ -299,6 +305,7 @@ export async function readTuples(part: LocalePart, rng: () => number): Promise<L
|
|
|
299
305
|
cols.tipoVial >= 0
|
|
300
306
|
? [get(cells, cols.tipoVial), get(cells, cols.street)].filter(Boolean).join(" ")
|
|
301
307
|
: get(cells, cols.street)
|
|
308
|
+
|
|
302
309
|
const rawCity = get(cells, cols.city)
|
|
303
310
|
|
|
304
311
|
if (!street) continue
|
|
@@ -324,6 +331,7 @@ export async function readTuples(part: LocalePart, rng: () => number): Promise<L
|
|
|
324
331
|
if (cleanedDistrict) {
|
|
325
332
|
locality = cleanedDistrict
|
|
326
333
|
const cleanedCity = cleanCityNoise(rawCity)
|
|
334
|
+
|
|
327
335
|
// ES pedanía lesson (2026-07-22): the CNIG `poblacion` column is filled on ~93% of rows but
|
|
328
336
|
// EQUALS `municipio` on the majority of those (the address point sits in the municipio's own
|
|
329
337
|
// main town, not a below-municipio pedanía) — only ~32.6% of ES rows carry a genuinely
|
|
@@ -348,6 +356,7 @@ export async function readTuples(part: LocalePart, rng: () => number): Promise<L
|
|
|
348
356
|
|
|
349
357
|
continue
|
|
350
358
|
}
|
|
359
|
+
|
|
351
360
|
const tuple: LocaleBaseTuple = {
|
|
352
361
|
house_number: get(cells, cols.num),
|
|
353
362
|
street,
|
|
@@ -356,6 +365,7 @@ export async function readTuples(part: LocalePart, rng: () => number): Promise<L
|
|
|
356
365
|
postcode: get(cells, cols.post),
|
|
357
366
|
...(dependent_locality ? { dependent_locality } : {}),
|
|
358
367
|
}
|
|
368
|
+
|
|
359
369
|
seen++
|
|
360
370
|
|
|
361
371
|
if (reservoir.length < RESERVOIR_CAP) {
|
|
@@ -369,8 +379,8 @@ export async function readTuples(part: LocalePart, rng: () => number): Promise<L
|
|
|
369
379
|
}
|
|
370
380
|
}
|
|
371
381
|
}
|
|
372
|
-
} catch (
|
|
373
|
-
console.error(` WARN: read failed for ${part.path ?? part.zip}: ${(
|
|
382
|
+
} catch (error) {
|
|
383
|
+
console.error(` WARN: read failed for ${part.path ?? part.zip}: ${(error as Error).message}`)
|
|
374
384
|
|
|
375
385
|
return []
|
|
376
386
|
}
|
|
@@ -429,6 +439,10 @@ export function resolveLocaleParts(countrySource: LocaleCountrySource, override:
|
|
|
429
439
|
return override === true && countrySource.pedaniaParts ? countrySource.pedaniaParts : countrySource.parts
|
|
430
440
|
}
|
|
431
441
|
|
|
442
|
+
/**
|
|
443
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
444
|
+
* and `description` below for the surface form it generates.
|
|
445
|
+
*/
|
|
432
446
|
export const localeRecipe: ShardRecipe = {
|
|
433
447
|
name: "locale",
|
|
434
448
|
description: "Per-locale coverage rows (DE/FR/NL/IT/ES/NZ/GB) from real OA tuples, both orders → synthesizeLocaleRow",
|
|
@@ -459,11 +473,13 @@ export const localeRecipe: ShardRecipe = {
|
|
|
459
473
|
`No OA sources registered for --country ${country}. Known: ${Object.keys(COUNTRY_SOURCES).join(", ")}.`
|
|
460
474
|
)
|
|
461
475
|
}
|
|
476
|
+
|
|
462
477
|
const intlFraction = opts.intlFraction ?? 0.4
|
|
463
478
|
|
|
464
479
|
if (!(intlFraction >= 0 && intlFraction <= 1)) {
|
|
465
480
|
throw new Error(`--intl-fraction must be in [0, 1], got ${intlFraction}`)
|
|
466
481
|
}
|
|
482
|
+
|
|
467
483
|
// Default 0 → the `random() < countryFraction` draw below is short-circuited away entirely (never
|
|
468
484
|
// consumed), so every existing locale's emit stream is byte-identical to before this option existed.
|
|
469
485
|
const countryFraction = opts.countryFraction ?? 0
|
|
@@ -471,6 +487,7 @@ export const localeRecipe: ShardRecipe = {
|
|
|
471
487
|
if (!(countryFraction >= 0 && countryFraction <= 1)) {
|
|
472
488
|
throw new Error(`--country-fraction must be in [0, 1], got ${countryFraction}`)
|
|
473
489
|
}
|
|
490
|
+
|
|
474
491
|
const source = opts.sourceName ?? countrySource.source
|
|
475
492
|
const count = opts.count ?? 4000
|
|
476
493
|
// Tri-state: `undefined` (flag absent) touches nothing below — `parts` stays the default list and each
|
|
@@ -485,7 +502,7 @@ export const localeRecipe: ShardRecipe = {
|
|
|
485
502
|
for (let pi = 0; pi < parts.length; pi++) {
|
|
486
503
|
// A reservoir PRNG per part, seeded but independent of the emit loop's `random`, so the sample is
|
|
487
504
|
// reproducible without perturbing the synth/order draws.
|
|
488
|
-
const reservoirRng = makeMulberry32((opts.seed ^ (
|
|
505
|
+
const reservoirRng = makeMulberry32((opts.seed ^ (0x9e_37_79_b9 * (pi + 1))) >>> 0)
|
|
489
506
|
const effectivePart = applyDistrictAsLocalityOverride(parts[pi]!, districtAsLocalityOverride)
|
|
490
507
|
const t = await readTuples(effectivePart, reservoirRng)
|
|
491
508
|
|
|
@@ -494,7 +511,7 @@ export const localeRecipe: ShardRecipe = {
|
|
|
494
511
|
} // NOT pool.push(...t) — spreading huge arrays overflows the stack
|
|
495
512
|
}
|
|
496
513
|
|
|
497
|
-
if (pool.length
|
|
514
|
+
if (!pool.length) {
|
|
498
515
|
throw new Error(`No ${country} tuples found — are the source CSVs/zips present? (see COUNTRY_SOURCES)`)
|
|
499
516
|
}
|
|
500
517
|
|
|
@@ -506,22 +523,27 @@ export const localeRecipe: ShardRecipe = {
|
|
|
506
523
|
while (emitted < count && guard++ < count * 6) {
|
|
507
524
|
const base = pool[Math.floor(random() * N)]!
|
|
508
525
|
const order = random() < intlFraction ? "international" : "native"
|
|
526
|
+
|
|
509
527
|
// Per-country surface-shape draws (#241) — consumed ONLY for that country, so the DE/FR emit
|
|
510
528
|
// streams for a given seed are unchanged by their existence.
|
|
511
529
|
const nativeHouseJoin =
|
|
512
530
|
country === "ES" ? (random() < ES_SPACE_JOIN_FRACTION ? ("space" as const) : ("template" as const)) : undefined
|
|
531
|
+
|
|
513
532
|
const postcodeShape =
|
|
514
533
|
country === "NL"
|
|
515
534
|
? random() < NL_GLUED_POSTCODE_FRACTION
|
|
516
535
|
? ("as-source" as const)
|
|
517
536
|
: ("conventional" as const)
|
|
518
537
|
: undefined
|
|
538
|
+
|
|
519
539
|
const synth = synthesizeLocaleRow(base, country, { random, order, nativeHouseJoin, postcodeShape })
|
|
520
540
|
|
|
521
541
|
if (!synth) {
|
|
522
542
|
skipped++
|
|
543
|
+
|
|
523
544
|
continue
|
|
524
545
|
}
|
|
546
|
+
|
|
525
547
|
applyCountryAppend(synth, country, countryFraction, random)
|
|
526
548
|
|
|
527
549
|
if (opts.golden) {
|
|
@@ -535,22 +557,29 @@ export const localeRecipe: ShardRecipe = {
|
|
|
535
557
|
source,
|
|
536
558
|
source_id: "golden:align-check",
|
|
537
559
|
}
|
|
560
|
+
|
|
538
561
|
const goldenAligned = alignRow(goldenCanonical as Parameters<typeof alignRow>[0])
|
|
539
562
|
|
|
540
563
|
if (goldenAligned.kind !== "labeled" || !goldenAligned.row) {
|
|
541
564
|
skipped++
|
|
565
|
+
|
|
542
566
|
continue
|
|
543
567
|
}
|
|
568
|
+
|
|
544
569
|
write(JSON.stringify({ raw: synth.raw, components: synth.components, country, order }) + "\n")
|
|
570
|
+
|
|
545
571
|
emitted++
|
|
572
|
+
|
|
546
573
|
continue
|
|
547
574
|
}
|
|
575
|
+
|
|
548
576
|
const sourceID = stableSourceID(source, {
|
|
549
577
|
street: synth.components.street,
|
|
550
578
|
house_number: synth.components.house_number,
|
|
551
579
|
locality: synth.components.locality,
|
|
552
580
|
postcode: synth.components.postcode,
|
|
553
581
|
})
|
|
582
|
+
|
|
554
583
|
const canonical = {
|
|
555
584
|
raw: synth.raw,
|
|
556
585
|
components: synth.components,
|
|
@@ -561,13 +590,17 @@ export const localeRecipe: ShardRecipe = {
|
|
|
561
590
|
corpus_version: countrySource.corpusVersion,
|
|
562
591
|
license: `OpenAddresses ${country} tuples, rendered ${order}-order — see ingest SOURCES`,
|
|
563
592
|
}
|
|
593
|
+
|
|
564
594
|
const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
|
|
565
595
|
|
|
566
596
|
if (aligned.kind !== "labeled" || !aligned.row) {
|
|
567
597
|
skipped++
|
|
598
|
+
|
|
568
599
|
continue
|
|
569
600
|
}
|
|
601
|
+
|
|
570
602
|
write(JSON.stringify({ ...aligned.row, synth_method: source, synth_order: order, synth_base_id: null }) + "\n")
|
|
603
|
+
|
|
571
604
|
emitted++
|
|
572
605
|
}
|
|
573
606
|
|
|
@@ -20,11 +20,17 @@
|
|
|
20
20
|
|
|
21
21
|
import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
|
|
22
22
|
|
|
23
|
-
/**
|
|
23
|
+
/**
|
|
24
|
+
* "1012LG" → "1012 LG". The tuples carry the unspaced OA form; the spaced form is the failing case.
|
|
25
|
+
*/
|
|
24
26
|
function spacePostcode(pc: string): string {
|
|
25
27
|
return pc.replace(/^(\d{4})([A-Z]{2})$/, "$1 $2")
|
|
26
28
|
}
|
|
27
29
|
|
|
30
|
+
/**
|
|
31
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
32
|
+
* and `description` below for the surface form it generates.
|
|
33
|
+
*/
|
|
28
34
|
export const nlPostcodeRecipe: ShardRecipe = {
|
|
29
35
|
name: "nl-postcode",
|
|
30
36
|
description:
|
|
@@ -41,13 +47,15 @@ export const nlPostcodeRecipe: ShardRecipe = {
|
|
|
41
47
|
const street = String(t.street ?? "").trim()
|
|
42
48
|
const city = String(t.locality ?? "").trim()
|
|
43
49
|
const number = String(t.number ?? "").trim()
|
|
50
|
+
|
|
44
51
|
const rawPostcode = String(t.postcode ?? "")
|
|
45
52
|
.trim()
|
|
46
53
|
.toUpperCase()
|
|
47
|
-
.
|
|
54
|
+
.replaceAll(/\s+/g, "")
|
|
48
55
|
|
|
49
56
|
if (!street || !city || !number || !/^\d{4}[A-Z]{2}$/.test(rawPostcode)) {
|
|
50
57
|
skipped++
|
|
58
|
+
|
|
51
59
|
continue
|
|
52
60
|
}
|
|
53
61
|
|
|
@@ -75,12 +83,14 @@ export const nlPostcodeRecipe: ShardRecipe = {
|
|
|
75
83
|
postcode,
|
|
76
84
|
locality: city,
|
|
77
85
|
}
|
|
86
|
+
|
|
78
87
|
const source_id = shardSourceID("synth-nl-postcode", {
|
|
79
88
|
...components,
|
|
80
89
|
o: String(order),
|
|
81
90
|
s: spaced ? "1" : "0",
|
|
82
91
|
v: String(read),
|
|
83
92
|
})
|
|
93
|
+
|
|
84
94
|
const canonical = {
|
|
85
95
|
raw,
|
|
86
96
|
components,
|
|
@@ -44,15 +44,21 @@ import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourc
|
|
|
44
44
|
* collapse whitespace, KEEP diacritics. Stripping them (fr-fragment's norm) would fold `Tømmerlien` -> `tommerlien`,
|
|
45
45
|
* miss the board's reserved `tømmerlien`, and leak the surface.
|
|
46
46
|
*/
|
|
47
|
-
const norm = (value: string): string => value.normalize("NFC").toLowerCase().
|
|
47
|
+
const norm = (value: string): string => value.normalize("NFC").toLowerCase().replaceAll(/\s+/g, " ").trim()
|
|
48
48
|
|
|
49
|
-
/**
|
|
49
|
+
/**
|
|
50
|
+
* Title-case a Kartverket ALL-CAPS locality (HELLVIK -> Hellvik); #690, all-caps is OOD.
|
|
51
|
+
*/
|
|
50
52
|
const titleNO = (value: string): string =>
|
|
51
53
|
value
|
|
52
54
|
.split(/\s+/)
|
|
53
55
|
.map((w) => (w ? w[0]!.toUpperCase() + w.slice(1).toLowerCase() : w))
|
|
54
56
|
.join(" ")
|
|
55
57
|
|
|
58
|
+
/**
|
|
59
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
60
|
+
* and `description` below for the surface form it generates.
|
|
61
|
+
*/
|
|
56
62
|
export const noFragmentRecipe: ShardRecipe = {
|
|
57
63
|
name: "no-fragment",
|
|
58
64
|
description:
|
|
@@ -123,6 +129,7 @@ export const noFragmentRecipe: ShardRecipe = {
|
|
|
123
129
|
// emitSeq keeps every emit distinct — knob 3 emits N copies of one long-number row, and
|
|
124
130
|
// (components, read) alone would collide their source_id and let downstream dedup drop the boost.
|
|
125
131
|
const source_id = shardSourceID("synth-no-fragment", { ...components, k: klass, v: `${read}:${emitSeq++}` })
|
|
132
|
+
|
|
126
133
|
const canonical = {
|
|
127
134
|
raw,
|
|
128
135
|
components,
|
|
@@ -158,12 +165,14 @@ export const noFragmentRecipe: ShardRecipe = {
|
|
|
158
165
|
|
|
159
166
|
if (!street) {
|
|
160
167
|
skipped++
|
|
168
|
+
|
|
161
169
|
continue
|
|
162
170
|
}
|
|
163
171
|
|
|
164
172
|
// THE SPLIT. A surface on the digit board never enters training.
|
|
165
173
|
if (excluded.has(norm(street))) {
|
|
166
174
|
contaminated++
|
|
175
|
+
|
|
167
176
|
continue
|
|
168
177
|
}
|
|
169
178
|
|
|
@@ -180,6 +189,7 @@ export const noFragmentRecipe: ShardRecipe = {
|
|
|
180
189
|
|
|
181
190
|
emit(pc, { postcode: pc }, "counter-bare-postcode")
|
|
182
191
|
}
|
|
192
|
+
|
|
183
193
|
continue
|
|
184
194
|
}
|
|
185
195
|
|
|
@@ -27,8 +27,12 @@ import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourc
|
|
|
27
27
|
* check would never match the board's reserved `tømmerlien` and the split would leak silently. Diacritic street heads
|
|
28
28
|
* (…vegen/…veien with ø/å/æ) are the whole point of this shard's boundary; folding them away is not an option.
|
|
29
29
|
*/
|
|
30
|
-
const norm = (value: string): string => value.normalize("NFC").toLowerCase().
|
|
30
|
+
const norm = (value: string): string => value.normalize("NFC").toLowerCase().replaceAll(/\s+/g, " ").trim()
|
|
31
31
|
|
|
32
|
+
/**
|
|
33
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
34
|
+
* and `description` below for the surface form it generates.
|
|
35
|
+
*/
|
|
32
36
|
export const noStreetLedRecipe: ShardRecipe = {
|
|
33
37
|
name: "no-street-led",
|
|
34
38
|
description: "NO street-led boundary form (#901 family): '«st» «n», «pc» «city»' — diacritic street heads",
|
|
@@ -81,16 +85,20 @@ export const noStreetLedRecipe: ShardRecipe = {
|
|
|
81
85
|
|
|
82
86
|
if (!street || !city || !number || !postcode) {
|
|
83
87
|
skipped++
|
|
88
|
+
|
|
84
89
|
continue
|
|
85
90
|
}
|
|
86
91
|
|
|
87
92
|
// A surface on the digit board never enters training.
|
|
88
93
|
if (excluded.has(norm(street))) {
|
|
89
94
|
contaminated++
|
|
95
|
+
|
|
90
96
|
continue
|
|
91
97
|
}
|
|
98
|
+
|
|
92
99
|
const order = read % 3
|
|
93
100
|
let raw: string
|
|
101
|
+
|
|
94
102
|
const components: Record<string, string> = {
|
|
95
103
|
street,
|
|
96
104
|
house_number: number,
|
|
@@ -105,7 +113,9 @@ export const noStreetLedRecipe: ShardRecipe = {
|
|
|
105
113
|
} else {
|
|
106
114
|
raw = `${postcode} ${city}, ${street} ${number}`
|
|
107
115
|
}
|
|
116
|
+
|
|
108
117
|
const source_id = shardSourceID("synth-no-street-led", { ...components, o: String(order), v: String(read) })
|
|
118
|
+
|
|
109
119
|
const canonical = {
|
|
110
120
|
raw,
|
|
111
121
|
components,
|
|
@@ -15,6 +15,10 @@ import { alignAndWrite, makeLcg, readTuples, shardSourceID, type ShardRecipe } f
|
|
|
15
15
|
|
|
16
16
|
const LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"
|
|
17
17
|
|
|
18
|
+
/**
|
|
19
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
20
|
+
* and `description` below for the surface form it generates.
|
|
21
|
+
*/
|
|
18
22
|
export const noStreetRecipe: ShardRecipe = {
|
|
19
23
|
name: "no-street",
|
|
20
24
|
description: "No-street counter-example rows: tuples → synthesizeNoStreetRow → aligned LabeledRow",
|
|
@@ -33,6 +37,7 @@ export const noStreetRecipe: ShardRecipe = {
|
|
|
33
37
|
|
|
34
38
|
if (!tuple.locality || !tuple.region || !tuple.postcode || !tuple.country) {
|
|
35
39
|
skipped++
|
|
40
|
+
|
|
36
41
|
continue
|
|
37
42
|
}
|
|
38
43
|
|
|
@@ -41,8 +46,10 @@ export const noStreetRecipe: ShardRecipe = {
|
|
|
41
46
|
|
|
42
47
|
if (!synth) {
|
|
43
48
|
skipped++
|
|
49
|
+
|
|
44
50
|
continue
|
|
45
51
|
}
|
|
52
|
+
|
|
46
53
|
const ok = alignAndWrite(
|
|
47
54
|
write,
|
|
48
55
|
{
|