@mailwoman/corpus 8.1.0 → 8.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/adapter.d.ts +15 -5
- package/out/src/adapter.d.ts.map +1 -1
- package/out/src/adapter.js +11 -9
- package/out/src/adapter.js.map +1 -1
- package/out/src/adapters/ban/adapter.d.ts +7 -0
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +9 -2
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +5 -1
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +14 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +15 -2
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +11 -0
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +24 -5
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +11 -0
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +14 -1
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +10 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +11 -2
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts +18 -6
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +16 -6
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +11 -0
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +18 -3
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +11 -0
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +12 -1
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +11 -0
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +11 -0
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +15 -2
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +11 -0
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +13 -2
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +14 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +19 -4
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +12 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +17 -2
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +12 -1
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +11 -0
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +15 -2
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +7 -0
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +18 -3
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +13 -3
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +15 -5
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +7 -0
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +9 -2
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/align.d.ts +12 -4
- package/out/src/align.d.ts.map +1 -1
- package/out/src/align.js +9 -5
- package/out/src/align.js.map +1 -1
- package/out/src/build.d.ts +24 -8
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +3 -3
- package/out/src/build.js.map +1 -1
- package/out/src/codex/us-fips-state.d.ts +6 -2
- package/out/src/codex/us-fips-state.d.ts.map +1 -1
- package/out/src/codex/us-fips-state.js +3 -1
- package/out/src/codex/us-fips-state.js.map +1 -1
- package/out/src/golden.d.ts +21 -7
- package/out/src/golden.d.ts.map +1 -1
- package/out/src/golden.js +18 -10
- package/out/src/golden.js.map +1 -1
- package/out/src/license.d.ts +3 -1
- package/out/src/license.d.ts.map +1 -1
- package/out/src/license.js +4 -2
- package/out/src/license.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts +4 -2
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +5 -4
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/schema.d.ts +14 -10
- package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
- package/out/src/parquet-wrapper/schema.js +6 -2
- package/out/src/parquet-wrapper/schema.js.map +1 -1
- package/out/src/parquet-wrapper/writer.d.ts +12 -4
- package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
- package/out/src/parquet-wrapper/writer.js +12 -4
- package/out/src/parquet-wrapper/writer.js.map +1 -1
- package/out/src/parquet.d.ts +21 -7
- package/out/src/parquet.d.ts.map +1 -1
- package/out/src/parquet.js +15 -4
- package/out/src/parquet.js.map +1 -1
- package/out/src/runner.d.ts +39 -13
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +7 -3
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts +4 -0
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +14 -5
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/boundary-stress.d.ts +4 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +13 -7
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts +4 -0
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +39 -11
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts +4 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +14 -3
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts +4 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.js +4 -0
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts +4 -0
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +25 -8
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts +4 -0
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +21 -8
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts +4 -0
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +27 -8
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts +4 -0
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +16 -6
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.d.ts +4 -0
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/house-venue.js +7 -1
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts +9 -3
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +12 -4
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts +4 -0
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +27 -13
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +4 -0
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +9 -4
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/nl-postcode.d.ts +4 -0
- package/out/src/shard-recipes/nl-postcode.d.ts.map +1 -1
- package/out/src/shard-recipes/nl-postcode.js +8 -2
- package/out/src/shard-recipes/nl-postcode.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts +4 -0
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +8 -2
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts +4 -0
- package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street-led.js +5 -1
- package/out/src/shard-recipes/no-street-led.js.map +1 -1
- package/out/src/shard-recipes/no-street.d.ts +4 -0
- package/out/src/shard-recipes/no-street.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street.js +4 -0
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +4 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +99 -43
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts +4 -0
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +4 -0
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +48 -16
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +10 -6
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/si-bare-village.d.ts +4 -0
- package/out/src/shard-recipes/si-bare-village.d.ts.map +1 -1
- package/out/src/shard-recipes/si-bare-village.js +4 -0
- package/out/src/shard-recipes/si-bare-village.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +4 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +29 -10
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.d.ts +4 -0
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
- package/out/src/shard-recipes/street-bare.js +4 -0
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.d.ts +4 -0
- package/out/src/shard-recipes/street.d.ts.map +1 -1
- package/out/src/shard-recipes/street.js +4 -0
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/unit.d.ts +4 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +45 -14
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/split.d.ts +18 -6
- package/out/src/split.d.ts.map +1 -1
- package/out/src/split.js +4 -2
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize-anchor-absorption.d.ts +7 -1
- package/out/src/synthesize-anchor-absorption.d.ts.map +1 -1
- package/out/src/synthesize-anchor-absorption.js +28 -14
- package/out/src/synthesize-anchor-absorption.js.map +1 -1
- package/out/src/synthesize-boundary-stress.d.ts +3 -1
- package/out/src/synthesize-boundary-stress.d.ts.map +1 -1
- package/out/src/synthesize-boundary-stress.js +35 -28
- package/out/src/synthesize-boundary-stress.js.map +1 -1
- package/out/src/synthesize-german.d.ts +15 -5
- package/out/src/synthesize-german.d.ts.map +1 -1
- package/out/src/synthesize-german.js +9 -3
- package/out/src/synthesize-german.js.map +1 -1
- package/out/src/synthesize-house-venue.d.ts.map +1 -1
- package/out/src/synthesize-house-venue.js +34 -18
- package/out/src/synthesize-house-venue.js.map +1 -1
- package/out/src/synthesize-intersection.d.ts +9 -3
- package/out/src/synthesize-intersection.d.ts.map +1 -1
- package/out/src/synthesize-intersection.js +25 -6
- package/out/src/synthesize-intersection.js.map +1 -1
- package/out/src/synthesize-no-street.d.ts +3 -1
- package/out/src/synthesize-no-street.d.ts.map +1 -1
- package/out/src/synthesize-no-street.js +5 -6
- package/out/src/synthesize-no-street.js.map +1 -1
- package/out/src/synthesize-po-box.d.ts +15 -5
- package/out/src/synthesize-po-box.d.ts.map +1 -1
- package/out/src/synthesize-po-box.js +21 -7
- package/out/src/synthesize-po-box.js.map +1 -1
- package/out/src/synthesize-street.d.ts +3 -1
- package/out/src/synthesize-street.d.ts.map +1 -1
- package/out/src/synthesize-street.js +6 -1
- package/out/src/synthesize-street.js.map +1 -1
- package/out/src/synthesize.d.ts +39 -13
- package/out/src/synthesize.d.ts.map +1 -1
- package/out/src/synthesize.js +66 -36
- package/out/src/synthesize.js.map +1 -1
- package/out/src/tokenize.d.ts +15 -5
- package/out/src/tokenize.d.ts.map +1 -1
- package/out/src/tokenize.js.map +1 -1
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +3 -1
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/audit.d.ts.map +1 -1
- package/out/src/tools/audit.js +20 -8
- package/out/src/tools/audit.js.map +1 -1
- package/out/src/tools/corpus-stats.d.ts.map +1 -1
- package/out/src/tools/corpus-stats.js.map +1 -1
- package/out/src/tools/fetch/ban.d.ts.map +1 -1
- package/out/src/tools/fetch/ban.js +17 -10
- package/out/src/tools/fetch/ban.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts +30 -10
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +26 -6
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/hrsa.d.ts.map +1 -1
- package/out/src/tools/fetch/hrsa.js.map +1 -1
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +18 -18
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +3 -1
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +3 -1
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts +21 -7
- package/out/src/tools/fetch/nad.d.ts.map +1 -1
- package/out/src/tools/fetch/nad.js +5 -5
- package/out/src/tools/fetch/nad.js.map +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +9 -15
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts +3 -1
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/src/tools/fetch/openaddresses.js +32 -23
- package/out/src/tools/fetch/openaddresses.js.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +3 -1
- package/out/src/tools/fetch/ppd.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.js +14 -3
- package/out/src/tools/fetch/ppd.js.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.js +13 -6
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
- package/out/src/tools/fetch/state-sources.js +8 -3
- package/out/src/tools/fetch/state-sources.js.map +1 -1
- package/out/src/tools/fetch/tiger-full.d.ts +9 -3
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
- package/out/src/tools/fetch/tiger-full.js +35 -20
- package/out/src/tools/fetch/tiger-full.js.map +1 -1
- package/out/src/tools/gb-title-case.d.ts.map +1 -1
- package/out/src/tools/gb-title-case.js.map +1 -1
- package/out/src/tools/golden-expand.d.ts +24 -8
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +20 -12
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-promote.d.ts +18 -6
- package/out/src/tools/golden-promote.d.ts.map +1 -1
- package/out/src/tools/golden-promote.js +22 -7
- package/out/src/tools/golden-promote.js.map +1 -1
- package/out/src/tools/ingest-csv.d.ts +3 -1
- package/out/src/tools/ingest-csv.d.ts.map +1 -1
- package/out/src/tools/ingest-csv.js +17 -19
- package/out/src/tools/ingest-csv.js.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts +18 -6
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
- package/out/src/tools/jsonl-to-parquet.js +11 -5
- package/out/src/tools/jsonl-to-parquet.js.map +1 -1
- package/out/src/tools/lint-shard-vocab.d.ts +36 -12
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
- package/out/src/tools/lint-shard-vocab.js +25 -11
- package/out/src/tools/lint-shard-vocab.js.map +1 -1
- package/out/src/tools/lint-shard.d.ts +30 -10
- package/out/src/tools/lint-shard.d.ts.map +1 -1
- package/out/src/tools/lint-shard.js +33 -20
- package/out/src/tools/lint-shard.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +6 -4
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/shard-kryptonite.d.ts +6 -2
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +1 -1
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts +9 -3
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +2 -2
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/types.d.ts +51 -17
- package/out/src/types.d.ts.map +1 -1
- package/out/src/wof-json.d.ts +9 -3
- package/out/src/wof-json.d.ts.map +1 -1
- package/out/src/wof-json.js +6 -4
- package/out/src/wof-json.js.map +1 -1
- package/package.json +7 -6
- package/src/adapter.ts +24 -13
- package/src/adapters/ban/adapter.ts +12 -2
- package/src/adapters/ban/street-decompose.ts +7 -1
- package/src/adapters/fcc-bdc/adapter.ts +19 -3
- package/src/adapters/geonames/adapter.ts +28 -5
- package/src/adapters/geonames-postal/adapter.ts +17 -1
- package/src/adapters/gnaf/adapter.ts +15 -2
- package/src/adapters/gnaf/assemble.ts +40 -10
- package/src/adapters/index.ts +13 -0
- package/src/adapters/openaddresses/adapter.ts +22 -3
- package/src/adapters/overture/adapter.ts +18 -2
- package/src/adapters/state-hi-schools/adapter.ts +15 -0
- package/src/adapters/state-ia-contractors/adapter.ts +15 -0
- package/src/adapters/state-ny-notaries/adapter.ts +15 -0
- package/src/adapters/state-tx-notaries/adapter.ts +18 -2
- package/src/adapters/synth-po-box/adapter.ts +19 -2
- package/src/adapters/tiger/adapter.ts +23 -4
- package/src/adapters/tiger/street-decompose.ts +1 -0
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +18 -2
- package/src/adapters/usgov-imls-pls/adapter.ts +15 -0
- package/src/adapters/usgov-irs-bmf/adapter.ts +19 -2
- package/src/adapters/usgov-nad/adapter.ts +14 -1
- package/src/adapters/usgov-nppes/adapter.ts +13 -0
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +21 -3
- package/src/adapters/wof-admin-jp/adapter.ts +20 -3
- package/src/adapters/wof-admin-json/adapter.ts +22 -7
- package/src/adapters/wof-postalcode-json/adapter.ts +10 -2
- package/src/align.ts +21 -8
- package/src/build.ts +48 -11
- package/src/codex/us-fips-state.ts +6 -2
- package/src/golden.ts +28 -13
- package/src/license.ts +4 -2
- package/src/parquet-wrapper/reader.ts +6 -4
- package/src/parquet-wrapper/schema.ts +14 -10
- package/src/parquet-wrapper/writer.ts +12 -4
- package/src/parquet.ts +39 -8
- package/src/runner.ts +49 -15
- package/src/shard-recipes/anchor-absorption.ts +21 -5
- package/src/shard-recipes/boundary-stress.ts +20 -7
- package/src/shard-recipes/country-balanced.ts +78 -13
- package/src/shard-recipes/cz-pcfirst-preposition.ts +9 -0
- package/src/shard-recipes/fr-admin-split.ts +30 -5
- package/src/shard-recipes/fr-bare-street.ts +10 -0
- package/src/shard-recipes/fr-fragment.ts +33 -8
- package/src/shard-recipes/fr-lieudit.ts +35 -9
- package/src/shard-recipes/fr-order.ts +47 -9
- package/src/shard-recipes/german.ts +32 -6
- package/src/shard-recipes/house-venue.ts +10 -1
- package/src/shard-recipes/index.ts +12 -4
- package/src/shard-recipes/intersection.ts +63 -14
- package/src/shard-recipes/locale.ts +37 -4
- package/src/shard-recipes/nl-postcode.ts +12 -2
- package/src/shard-recipes/no-fragment.ts +12 -2
- package/src/shard-recipes/no-street-led.ts +11 -1
- package/src/shard-recipes/no-street.ts +7 -0
- package/src/shard-recipes/po-box-cedex.ts +145 -45
- package/src/shard-recipes/po-box.ts +7 -0
- package/src/shard-recipes/scaffold.ts +52 -20
- package/src/shard-recipes/si-bare-village.ts +9 -0
- package/src/shard-recipes/street-affix.ts +88 -13
- package/src/shard-recipes/street-bare.ts +10 -0
- package/src/shard-recipes/street.ts +7 -0
- package/src/shard-recipes/unit.ts +79 -16
- package/src/split.ts +28 -7
- package/src/synthesize-anchor-absorption.ts +41 -14
- package/src/synthesize-boundary-stress.ts +57 -30
- package/src/synthesize-german.ts +30 -7
- package/src/synthesize-house-venue.ts +43 -19
- package/src/synthesize-intersection.ts +34 -7
- package/src/synthesize-no-street.ts +18 -10
- package/src/synthesize-po-box.ts +42 -10
- package/src/synthesize-street.ts +10 -2
- package/src/synthesize.ts +81 -38
- package/src/tokenize.ts +15 -5
- package/src/tools/align-shard.ts +6 -1
- package/src/tools/audit.ts +57 -11
- package/src/tools/corpus-stats.ts +9 -0
- package/src/tools/fetch/ban.ts +32 -10
- package/src/tools/fetch/download.ts +49 -12
- package/src/tools/fetch/hrsa.ts +3 -0
- package/src/tools/fetch/imls-pls.ts +28 -18
- package/src/tools/fetch/index.ts +3 -1
- package/src/tools/fetch/nad.ts +39 -12
- package/src/tools/fetch/nppes.ts +17 -15
- package/src/tools/fetch/openaddresses.ts +52 -24
- package/src/tools/fetch/ppd.ts +22 -3
- package/src/tools/fetch/state-hi-schools.ts +17 -6
- package/src/tools/fetch/state-sources.ts +16 -3
- package/src/tools/fetch/tiger-full.ts +66 -23
- package/src/tools/gb-title-case.ts +1 -0
- package/src/tools/golden-expand.ts +64 -20
- package/src/tools/golden-promote.ts +53 -13
- package/src/tools/ingest-csv.ts +41 -26
- package/src/tools/jsonl-to-parquet.ts +30 -10
- package/src/tools/lint-shard-vocab.ts +75 -24
- package/src/tools/lint-shard.ts +75 -31
- package/src/tools/overlay-manifest.ts +7 -4
- package/src/tools/shard-kryptonite.ts +11 -3
- package/src/tools/shard-translit.ts +20 -5
- package/src/types.ts +51 -17
- package/src/wof-json.ts +18 -7
package/src/adapter.ts
CHANGED
|
@@ -34,16 +34,24 @@ import type { CanonicalRow, CorpusAdapter } from "./types.ts"
|
|
|
34
34
|
* the dependency graph stays traceable.
|
|
35
35
|
*/
|
|
36
36
|
export interface AdapterRegistry {
|
|
37
|
-
/**
|
|
37
|
+
/**
|
|
38
|
+
* Add an adapter. Throws if `adapter.id` is already registered.
|
|
39
|
+
*/
|
|
38
40
|
register(adapter: CorpusAdapter): void
|
|
39
41
|
|
|
40
|
-
/**
|
|
42
|
+
/**
|
|
43
|
+
* Return the adapter for `id`, or `undefined`.
|
|
44
|
+
*/
|
|
41
45
|
get(id: string): CorpusAdapter | undefined
|
|
42
46
|
|
|
43
|
-
/**
|
|
47
|
+
/**
|
|
48
|
+
* All registered adapters, in insertion order.
|
|
49
|
+
*/
|
|
44
50
|
list(): readonly CorpusAdapter[]
|
|
45
51
|
|
|
46
|
-
/**
|
|
52
|
+
/**
|
|
53
|
+
* Convenience: ids only, in insertion order.
|
|
54
|
+
*/
|
|
47
55
|
ids(): readonly string[]
|
|
48
56
|
}
|
|
49
57
|
|
|
@@ -58,6 +66,7 @@ export class InMemoryAdapterRegistry implements AdapterRegistry {
|
|
|
58
66
|
if (this.#byID.has(adapter.id)) {
|
|
59
67
|
throw new Error(`AdapterRegistry: id ${JSON.stringify(adapter.id)} already registered`)
|
|
60
68
|
}
|
|
69
|
+
|
|
61
70
|
this.#byID.set(adapter.id, adapter)
|
|
62
71
|
}
|
|
63
72
|
|
|
@@ -92,9 +101,9 @@ export const defaultAdapterRegistry = new InMemoryAdapterRegistry()
|
|
|
92
101
|
* length.
|
|
93
102
|
*/
|
|
94
103
|
export function stableSourceID(adapterID: string, components: Partial<Record<ComponentTag, string>>): string {
|
|
95
|
-
const sortedKeys = Object.keys(components).
|
|
96
|
-
const payload = sortedKeys.map((k) => `${k}=${components[k] ?? ""}`).join("\
|
|
97
|
-
const digest = createHash("sha256").update(adapterID).update("\
|
|
104
|
+
const sortedKeys = Object.keys(components).toSorted() as ComponentTag[]
|
|
105
|
+
const payload = sortedKeys.map((k) => `${k}=${components[k] ?? ""}`).join("\u001F")
|
|
106
|
+
const digest = createHash("sha256").update(adapterID).update("\u001E").update(payload).digest("hex")
|
|
98
107
|
|
|
99
108
|
return `${adapterID}-${digest.slice(0, 12)}`
|
|
100
109
|
}
|
|
@@ -110,12 +119,12 @@ export function stableSourceID(adapterID: string, components: Partial<Record<Com
|
|
|
110
119
|
* ensuring each augmentation variant survives.
|
|
111
120
|
*/
|
|
112
121
|
export function canonicalDedupKey(row: CanonicalRow): string {
|
|
113
|
-
const sortedKeys = Object.keys(row.components).
|
|
114
|
-
const compPart = sortedKeys.map((k) => `${k}=${row.components[k] ?? ""}`).join("\
|
|
115
|
-
const rawNorm = row.raw.toLowerCase().
|
|
116
|
-
const synthPart = row.synth ? `\
|
|
122
|
+
const sortedKeys = Object.keys(row.components).toSorted() as ComponentTag[]
|
|
123
|
+
const compPart = sortedKeys.map((k) => `${k}=${row.components[k] ?? ""}`).join("\u001F")
|
|
124
|
+
const rawNorm = row.raw.toLowerCase().replaceAll(/\s+/g, " ").trim()
|
|
125
|
+
const synthPart = row.synth ? `\u001E${row.synth.method}` : ""
|
|
117
126
|
|
|
118
|
-
return `${row.country}\
|
|
127
|
+
return `${row.country}\u001E${rawNorm}\u001E${compPart}${synthPart}`
|
|
119
128
|
}
|
|
120
129
|
|
|
121
130
|
/**
|
|
@@ -130,7 +139,9 @@ export interface StreamingHasher {
|
|
|
130
139
|
digest(): string
|
|
131
140
|
}
|
|
132
141
|
|
|
133
|
-
/**
|
|
142
|
+
/**
|
|
143
|
+
* Default `StreamingHasher` (SHA-256, hex).
|
|
144
|
+
*/
|
|
134
145
|
export function streamingSha256(): StreamingHasher {
|
|
135
146
|
const h: Hash = createHash("sha256")
|
|
136
147
|
let finalized = false
|
|
@@ -39,6 +39,10 @@ import { reconcileComponents } from "../../format.ts"
|
|
|
39
39
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
40
40
|
import { decomposeFrStreet } from "./street-decompose.ts"
|
|
41
41
|
|
|
42
|
+
/**
|
|
43
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
44
|
+
* it came from.
|
|
45
|
+
*/
|
|
42
46
|
export const BAN_ADAPTER_ID = "ban"
|
|
43
47
|
|
|
44
48
|
/**
|
|
@@ -83,13 +87,14 @@ function composeRaw(house: string, street: string, postcode: string, locality: s
|
|
|
83
87
|
if (streetPart) {
|
|
84
88
|
parts.push(streetPart)
|
|
85
89
|
}
|
|
90
|
+
|
|
86
91
|
const cityPart = [postcode, locality].filter(Boolean).join(" ").trim()
|
|
87
92
|
|
|
88
93
|
if (cityPart) {
|
|
89
94
|
parts.push(cityPart)
|
|
90
95
|
}
|
|
91
96
|
|
|
92
|
-
return parts.join(", ").
|
|
97
|
+
return parts.join(", ").replaceAll(/\s+/g, " ").trim()
|
|
93
98
|
}
|
|
94
99
|
|
|
95
100
|
export function createBanAdapter(): CorpusAdapter {
|
|
@@ -104,6 +109,7 @@ export function createBanAdapter(): CorpusAdapter {
|
|
|
104
109
|
}
|
|
105
110
|
|
|
106
111
|
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
112
|
+
|
|
107
113
|
const parser = stream.pipe(
|
|
108
114
|
csvParse({
|
|
109
115
|
delimiter: ";",
|
|
@@ -161,7 +167,7 @@ export function createBanAdapter(): CorpusAdapter {
|
|
|
161
167
|
|
|
162
168
|
const aligned = reconcileComponents(components, raw)
|
|
163
169
|
|
|
164
|
-
if (Object.keys(aligned).length
|
|
170
|
+
if (!Object.keys(aligned).length) continue
|
|
165
171
|
|
|
166
172
|
const sourceID = record.id?.trim()
|
|
167
173
|
? `${BAN_ADAPTER_ID}-${record.id.trim()}`
|
|
@@ -177,6 +183,7 @@ export function createBanAdapter(): CorpusAdapter {
|
|
|
177
183
|
corpus_version: "",
|
|
178
184
|
license: "Licence Ouverte 2.0",
|
|
179
185
|
}
|
|
186
|
+
|
|
180
187
|
emitted++
|
|
181
188
|
}
|
|
182
189
|
} finally {
|
|
@@ -186,4 +193,7 @@ export function createBanAdapter(): CorpusAdapter {
|
|
|
186
193
|
}
|
|
187
194
|
}
|
|
188
195
|
|
|
196
|
+
/**
|
|
197
|
+
* The configured adapter instance registered with the corpus builder.
|
|
198
|
+
*/
|
|
189
199
|
export const banAdapter = createBanAdapter()
|
|
@@ -20,6 +20,11 @@ import { resolve } from "node:path"
|
|
|
20
20
|
|
|
21
21
|
import { resourceDictionaryPath } from "@mailwoman/core/utils"
|
|
22
22
|
|
|
23
|
+
/**
|
|
24
|
+
* Tokens a BAN street needs before a type/article/name decomposition is attempted.
|
|
25
|
+
*/
|
|
26
|
+
const MIN_TOKENS_FOR_DECOMPOSE = 3
|
|
27
|
+
|
|
23
28
|
function loadDictionary(filename: string): Set<string> {
|
|
24
29
|
const candidates = [
|
|
25
30
|
resourceDictionaryPath("libpostal", "fr", filename),
|
|
@@ -51,6 +56,7 @@ function loadDictionary(filename: string): Set<string> {
|
|
|
51
56
|
// try next
|
|
52
57
|
}
|
|
53
58
|
}
|
|
59
|
+
|
|
54
60
|
throw new Error(`Could not load FR libpostal dictionary: ${filename}`)
|
|
55
61
|
}
|
|
56
62
|
|
|
@@ -79,7 +85,7 @@ export function decomposeFrStreet(fullname: string): DecomposedFrStreet {
|
|
|
79
85
|
const norm = (s: string) => s.toLowerCase().replace(/[.,;]$/, "")
|
|
80
86
|
|
|
81
87
|
// Try 2-word prefix first (e.g. "ancien chemin")
|
|
82
|
-
if (tokens.length >=
|
|
88
|
+
if (tokens.length >= MIN_TOKENS_FOR_DECOMPOSE) {
|
|
83
89
|
const twoWord = norm(tokens[0]!) + " " + norm(tokens[1]!)
|
|
84
90
|
|
|
85
91
|
if (STREET_TYPES_FR.has(twoWord)) {
|
|
@@ -40,10 +40,20 @@ import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
|
|
|
40
40
|
import { formatAddress, reconcileComponents } from "../../format.ts"
|
|
41
41
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
42
42
|
|
|
43
|
+
/**
|
|
44
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
45
|
+
* it came from.
|
|
46
|
+
*/
|
|
43
47
|
export const FCC_BDC_ADAPTER_ID = "fcc-bdc"
|
|
48
|
+
/**
|
|
49
|
+
* License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
|
|
50
|
+
* than having to look them up.
|
|
51
|
+
*/
|
|
44
52
|
export const FCC_BDC_DEFAULT_LICENSE = "Public Domain"
|
|
45
53
|
|
|
46
|
-
/**
|
|
54
|
+
/**
|
|
55
|
+
* SQLite row shape — one row per BSL `location_id`. Columns mirror NTIARecord.
|
|
56
|
+
*/
|
|
47
57
|
interface BdcLocationRow {
|
|
48
58
|
location_id: number
|
|
49
59
|
address_primary: string
|
|
@@ -110,7 +120,9 @@ export function buildPostcode(zip: string, suffix: string | null): string {
|
|
|
110
120
|
return `${z}-${s}`
|
|
111
121
|
}
|
|
112
122
|
|
|
113
|
-
/**
|
|
123
|
+
/**
|
|
124
|
+
* Build a BDC adapter. Pure factory so multiple instances can be created in tests.
|
|
125
|
+
*/
|
|
114
126
|
export function createFccBdcAdapter(): CorpusAdapter {
|
|
115
127
|
return {
|
|
116
128
|
id: FCC_BDC_ADAPTER_ID,
|
|
@@ -164,7 +176,7 @@ export function createFccBdcAdapter(): CorpusAdapter {
|
|
|
164
176
|
if (!raw) continue
|
|
165
177
|
const aligned = reconcileComponents(components, raw)
|
|
166
178
|
|
|
167
|
-
if (Object.keys(aligned).length
|
|
179
|
+
if (!Object.keys(aligned).length) continue
|
|
168
180
|
|
|
169
181
|
yield {
|
|
170
182
|
raw,
|
|
@@ -176,6 +188,7 @@ export function createFccBdcAdapter(): CorpusAdapter {
|
|
|
176
188
|
corpus_version: "",
|
|
177
189
|
license: FCC_BDC_DEFAULT_LICENSE,
|
|
178
190
|
}
|
|
191
|
+
|
|
179
192
|
emitted++
|
|
180
193
|
}
|
|
181
194
|
} finally {
|
|
@@ -185,4 +198,7 @@ export function createFccBdcAdapter(): CorpusAdapter {
|
|
|
185
198
|
}
|
|
186
199
|
}
|
|
187
200
|
|
|
201
|
+
/**
|
|
202
|
+
* The configured adapter instance registered with the corpus builder.
|
|
203
|
+
*/
|
|
188
204
|
export const fccBdcAdapter = createFccBdcAdapter()
|
|
@@ -37,10 +37,20 @@ import { stableSourceID } from "../../adapter.ts"
|
|
|
37
37
|
import { reconcileComponents } from "../../format.ts"
|
|
38
38
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
39
39
|
|
|
40
|
+
/**
|
|
41
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
42
|
+
* it came from.
|
|
43
|
+
*/
|
|
40
44
|
export const GEONAMES_ADAPTER_ID = "geonames"
|
|
45
|
+
/**
|
|
46
|
+
* License carried by this source (CC-BY-4.0), attached to each row so downstream consumers inherit the terms rather
|
|
47
|
+
* than having to look them up.
|
|
48
|
+
*/
|
|
41
49
|
export const GEONAMES_DEFAULT_LICENSE = "CC-BY-4.0"
|
|
42
50
|
|
|
43
|
-
|
|
51
|
+
/**
|
|
52
|
+
* GeoNames main-table column indices (0-based; see the export README).
|
|
53
|
+
*/
|
|
44
54
|
const COL = {
|
|
45
55
|
geonameid: 0,
|
|
46
56
|
name: 1,
|
|
@@ -51,10 +61,14 @@ const COL = {
|
|
|
51
61
|
admin1: 10,
|
|
52
62
|
} as const
|
|
53
63
|
|
|
54
|
-
|
|
64
|
+
/**
|
|
65
|
+
* Populated-place feature codes that are NOT current real places — skip them.
|
|
66
|
+
*/
|
|
55
67
|
const NON_CURRENT_PPL = new Set(["PPLH", "PPLQ", "PPLW", "PPLCH"])
|
|
56
68
|
|
|
57
|
-
/**
|
|
69
|
+
/**
|
|
70
|
+
* Load `admin1CodesASCII.txt` → Map("<CC>.<admin1>" → region name). Empty map if absent.
|
|
71
|
+
*/
|
|
58
72
|
function loadAdmin1(dir: string): Map<string, string> {
|
|
59
73
|
const map = new Map<string, string>()
|
|
60
74
|
const fp = join(dir, "admin1CodesASCII.txt")
|
|
@@ -73,7 +87,9 @@ function loadAdmin1(dir: string): Map<string, string> {
|
|
|
73
87
|
return map
|
|
74
88
|
}
|
|
75
89
|
|
|
76
|
-
/**
|
|
90
|
+
/**
|
|
91
|
+
* Load `countryInfo.txt` → Map(ISO → country name). Empty map if absent. The file is `#`-commented.
|
|
92
|
+
*/
|
|
77
93
|
function loadCountries(dir: string): Map<string, string> {
|
|
78
94
|
const map = new Map<string, string>()
|
|
79
95
|
const fp = join(dir, "countryInfo.txt")
|
|
@@ -106,6 +122,7 @@ export function createGeonamesAdapter(): CorpusAdapter {
|
|
|
106
122
|
const countries = loadCountries(dir)
|
|
107
123
|
|
|
108
124
|
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
125
|
+
|
|
109
126
|
const parser = stream.pipe(
|
|
110
127
|
csvParse({ delimiter: "\t", quote: false, relax_column_count: true, skip_empty_lines: true })
|
|
111
128
|
)
|
|
@@ -159,10 +176,12 @@ export function createGeonamesAdapter(): CorpusAdapter {
|
|
|
159
176
|
if (opts.limit !== undefined && emitted >= opts.limit) break
|
|
160
177
|
const aligned = reconcileComponents(v.comp, v.raw)
|
|
161
178
|
|
|
162
|
-
if (Object.keys(aligned).length
|
|
179
|
+
if (!Object.keys(aligned).length) continue
|
|
180
|
+
|
|
163
181
|
const sourceID = geonameid
|
|
164
182
|
? `${GEONAMES_ADAPTER_ID}-${geonameid}-${v.slot}`
|
|
165
183
|
: stableSourceID(GEONAMES_ADAPTER_ID, aligned)
|
|
184
|
+
|
|
166
185
|
yield {
|
|
167
186
|
raw: v.raw,
|
|
168
187
|
components: aligned,
|
|
@@ -172,6 +191,7 @@ export function createGeonamesAdapter(): CorpusAdapter {
|
|
|
172
191
|
corpus_version: "",
|
|
173
192
|
license: GEONAMES_DEFAULT_LICENSE,
|
|
174
193
|
}
|
|
194
|
+
|
|
175
195
|
emitted++
|
|
176
196
|
}
|
|
177
197
|
}
|
|
@@ -182,4 +202,7 @@ export function createGeonamesAdapter(): CorpusAdapter {
|
|
|
182
202
|
}
|
|
183
203
|
}
|
|
184
204
|
|
|
205
|
+
/**
|
|
206
|
+
* The configured adapter instance registered with the corpus builder.
|
|
207
|
+
*/
|
|
185
208
|
export const geonamesAdapter = createGeonamesAdapter()
|
|
@@ -31,10 +31,20 @@ import { stableSourceID } from "../../adapter.ts"
|
|
|
31
31
|
import { reconcileComponents } from "../../format.ts"
|
|
32
32
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
33
33
|
|
|
34
|
+
/**
|
|
35
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
36
|
+
* it came from.
|
|
37
|
+
*/
|
|
34
38
|
export const GEONAMES_POSTAL_ADAPTER_ID = "geonames-postal"
|
|
39
|
+
/**
|
|
40
|
+
* License carried by this source (CC-BY-4.0), attached to each row so downstream consumers inherit the terms rather
|
|
41
|
+
* than having to look them up.
|
|
42
|
+
*/
|
|
35
43
|
export const GEONAMES_POSTAL_DEFAULT_LICENSE = "CC-BY-4.0"
|
|
36
44
|
|
|
37
|
-
|
|
45
|
+
/**
|
|
46
|
+
* GeoNames postal-dump columns (0-based).
|
|
47
|
+
*/
|
|
38
48
|
const COL = { country: 0, postcode: 1, place: 2, admin1Name: 3 } as const
|
|
39
49
|
|
|
40
50
|
export function createGeonamesPostalAdapter(): CorpusAdapter {
|
|
@@ -46,6 +56,7 @@ export function createGeonamesPostalAdapter(): CorpusAdapter {
|
|
|
46
56
|
|
|
47
57
|
async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
|
|
48
58
|
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
59
|
+
|
|
49
60
|
const parser = stream.pipe(
|
|
50
61
|
csvParse({ delimiter: "\t", quote: false, relax_column_count: true, skip_empty_lines: true })
|
|
51
62
|
)
|
|
@@ -89,6 +100,7 @@ export function createGeonamesPostalAdapter(): CorpusAdapter {
|
|
|
89
100
|
const aligned = reconcileComponents(v.comp, v.raw)
|
|
90
101
|
|
|
91
102
|
if (Object.keys(aligned).length < 2) continue
|
|
103
|
+
|
|
92
104
|
yield {
|
|
93
105
|
raw: v.raw,
|
|
94
106
|
components: aligned,
|
|
@@ -98,6 +110,7 @@ export function createGeonamesPostalAdapter(): CorpusAdapter {
|
|
|
98
110
|
corpus_version: "",
|
|
99
111
|
license: GEONAMES_POSTAL_DEFAULT_LICENSE,
|
|
100
112
|
}
|
|
113
|
+
|
|
101
114
|
emitted++
|
|
102
115
|
}
|
|
103
116
|
}
|
|
@@ -108,4 +121,7 @@ export function createGeonamesPostalAdapter(): CorpusAdapter {
|
|
|
108
121
|
}
|
|
109
122
|
}
|
|
110
123
|
|
|
124
|
+
/**
|
|
125
|
+
* The configured adapter instance registered with the corpus builder.
|
|
126
|
+
*/
|
|
111
127
|
export const geonamesPostalAdapter = createGeonamesPostalAdapter()
|
|
@@ -34,8 +34,14 @@ import { stableSourceID } from "../../adapter.ts"
|
|
|
34
34
|
import { reconcileComponents } from "../../format.ts"
|
|
35
35
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
36
36
|
|
|
37
|
+
/**
|
|
38
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
39
|
+
* it came from.
|
|
40
|
+
*/
|
|
37
41
|
export const GNAF_ADAPTER_ID = "gnaf"
|
|
38
|
-
/**
|
|
42
|
+
/**
|
|
43
|
+
* Open G-NAF is freely redistributable with attribution to Geoscape Australia (CC-BY-style).
|
|
44
|
+
*/
|
|
39
45
|
export const GNAF_DEFAULT_LICENSE = "CC-BY-4.0"
|
|
40
46
|
|
|
41
47
|
interface GNAFTuple {
|
|
@@ -102,8 +108,10 @@ export function createGNAFAdapter(): CorpusAdapter {
|
|
|
102
108
|
|
|
103
109
|
const orders = renderOrders(t)
|
|
104
110
|
const order = idx % orders.length
|
|
111
|
+
|
|
105
112
|
idx++
|
|
106
113
|
const raw = orders[order]!
|
|
114
|
+
|
|
107
115
|
const components: CanonicalRow["components"] = {
|
|
108
116
|
house_number: t.house_number,
|
|
109
117
|
street: t.street,
|
|
@@ -119,7 +127,8 @@ export function createGNAFAdapter(): CorpusAdapter {
|
|
|
119
127
|
|
|
120
128
|
const aligned = reconcileComponents(components, raw)
|
|
121
129
|
|
|
122
|
-
if (Object.keys(aligned).length
|
|
130
|
+
if (!Object.keys(aligned).length) continue
|
|
131
|
+
|
|
123
132
|
yield {
|
|
124
133
|
raw,
|
|
125
134
|
components: aligned,
|
|
@@ -130,10 +139,14 @@ export function createGNAFAdapter(): CorpusAdapter {
|
|
|
130
139
|
corpus_version: "",
|
|
131
140
|
license: GNAF_DEFAULT_LICENSE,
|
|
132
141
|
}
|
|
142
|
+
|
|
133
143
|
emitted++
|
|
134
144
|
}
|
|
135
145
|
},
|
|
136
146
|
}
|
|
137
147
|
}
|
|
138
148
|
|
|
149
|
+
/**
|
|
150
|
+
* The configured adapter instance registered with the corpus builder.
|
|
151
|
+
*/
|
|
139
152
|
export const gnafAdapter = createGNAFAdapter()
|
|
@@ -33,17 +33,25 @@ import { join } from "node:path"
|
|
|
33
33
|
import { PSVSpliterator } from "spliterator"
|
|
34
34
|
|
|
35
35
|
export interface GNAFAssembleOptions {
|
|
36
|
-
/**
|
|
36
|
+
/**
|
|
37
|
+
* G-NAF `Standard` directory (holds the per-state `*_psv.psv` tables).
|
|
38
|
+
*/
|
|
37
39
|
standardDir: string
|
|
38
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* Target sample size (uniform reservoir → population-proportional across states).
|
|
42
|
+
*/
|
|
39
43
|
sampleSize: number
|
|
40
|
-
/**
|
|
44
|
+
/**
|
|
45
|
+
* Output JSONL path.
|
|
46
|
+
*/
|
|
41
47
|
out: string
|
|
42
48
|
/**
|
|
43
49
|
* Optional held-out eval JSONL (rows with a `components` field) — its (street,locality,postcode) are excluded.
|
|
44
50
|
*/
|
|
45
51
|
holdoutPath?: string
|
|
46
|
-
/**
|
|
52
|
+
/**
|
|
53
|
+
* Progress sink (the CLI passes a setter).
|
|
54
|
+
*/
|
|
47
55
|
onProgress?: (message: string) => void
|
|
48
56
|
}
|
|
49
57
|
|
|
@@ -54,25 +62,32 @@ export interface GNAFAssembleResult {
|
|
|
54
62
|
byState: Record<string, number>
|
|
55
63
|
}
|
|
56
64
|
|
|
57
|
-
/**
|
|
65
|
+
/**
|
|
66
|
+
* UPPERCASE → Title Case, preserving intra-word apostrophes/hyphens (O'Brien, Coff's Harbour).
|
|
67
|
+
*/
|
|
58
68
|
export function titlecase(s: string): string {
|
|
59
69
|
return s
|
|
60
70
|
.toLowerCase()
|
|
61
|
-
.
|
|
71
|
+
.replaceAll(/(^|[\s'\-/])([a-z])/g, (_m, p: string, c: string) => p + c.toUpperCase())
|
|
62
72
|
.trim()
|
|
63
73
|
}
|
|
64
74
|
|
|
65
|
-
/**
|
|
75
|
+
/**
|
|
76
|
+
* Holdout/dedup key: a street within a locality+postcode (house-number-agnostic — conservative).
|
|
77
|
+
*/
|
|
66
78
|
export function gnafHoldoutKey(street: string, locality: string, postcode: string): string {
|
|
67
79
|
return `${street}|${locality}|${postcode}`.toLowerCase()
|
|
68
80
|
}
|
|
69
81
|
|
|
70
82
|
type Row = Record<string, string | number | undefined>
|
|
83
|
+
|
|
71
84
|
async function* psvObjects(path: string): AsyncIterable<Row> {
|
|
72
85
|
yield* PSVSpliterator.fromAsync(path, { mode: "object", header: true }) as AsyncIterable<Row>
|
|
73
86
|
}
|
|
74
87
|
|
|
75
|
-
/**
|
|
88
|
+
/**
|
|
89
|
+
* Load a small lookup table fully into a Map keyed by `keyCol`.
|
|
90
|
+
*/
|
|
76
91
|
async function loadMap<V>(paths: string[], keyCol: string, pick: (r: Row) => V): Promise<Map<string, V>> {
|
|
77
92
|
const m = new Map<string, V>()
|
|
78
93
|
|
|
@@ -89,7 +104,9 @@ async function loadMap<V>(paths: string[], keyCol: string, pick: (r: Row) => V):
|
|
|
89
104
|
return m
|
|
90
105
|
}
|
|
91
106
|
|
|
92
|
-
/**
|
|
107
|
+
/**
|
|
108
|
+
* Build the held-out key set from an eval JSONL whose rows carry a `components` object.
|
|
109
|
+
*/
|
|
93
110
|
async function loadHoldout(path: string): Promise<Set<string>> {
|
|
94
111
|
const keys = new Set<string>()
|
|
95
112
|
const text = await readFile(path, "utf8")
|
|
@@ -114,6 +131,7 @@ async function loadHoldout(path: string): Promise<Set<string>> {
|
|
|
114
131
|
export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssembleResult> {
|
|
115
132
|
const progress = opts.onProgress ?? (() => {})
|
|
116
133
|
const files = await readdir(opts.standardDir)
|
|
134
|
+
|
|
117
135
|
const pick = (re: RegExp, exclude?: RegExp) =>
|
|
118
136
|
files.filter((f) => re.test(f) && !(exclude && exclude.test(f))).map((f) => join(opts.standardDir, f))
|
|
119
137
|
|
|
@@ -129,16 +147,19 @@ export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssem
|
|
|
129
147
|
}
|
|
130
148
|
|
|
131
149
|
progress(`loading STREET_LOCALITY (${streetPaths.length} files) + LOCALITY (${localityPaths.length})…`)
|
|
150
|
+
|
|
132
151
|
const streetMap = await loadMap(streetPaths, "STREET_LOCALITY_PID", (r) => ({
|
|
133
152
|
name: String(r.STREET_NAME ?? ""),
|
|
134
153
|
type: String(r.STREET_TYPE_CODE ?? ""),
|
|
135
154
|
suffix: String(r.STREET_SUFFIX_CODE ?? ""),
|
|
136
155
|
}))
|
|
156
|
+
|
|
137
157
|
const localityMap = await loadMap(localityPaths, "LOCALITY_PID", (r) => String(r.LOCALITY_NAME ?? ""))
|
|
138
158
|
progress(`streets=${streetMap.size.toLocaleString()} localities=${localityMap.size.toLocaleString()}`)
|
|
139
159
|
|
|
140
160
|
const reservoir: Array<{ house_number: string; street: string; locality: string; region: string; postcode: string }> =
|
|
141
161
|
[]
|
|
162
|
+
|
|
142
163
|
let seen = 0
|
|
143
164
|
let heldOut = 0
|
|
144
165
|
|
|
@@ -159,8 +180,10 @@ export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssem
|
|
|
159
180
|
|
|
160
181
|
if (holdout.has(gnafHoldoutKey(street, locality, postcode))) {
|
|
161
182
|
heldOut++
|
|
183
|
+
|
|
162
184
|
continue
|
|
163
185
|
}
|
|
186
|
+
|
|
164
187
|
let house = numberFirst + (r.NUMBER_FIRST_SUFFIX ? String(r.NUMBER_FIRST_SUFFIX) : "")
|
|
165
188
|
|
|
166
189
|
if (r.NUMBER_LAST) {
|
|
@@ -170,7 +193,9 @@ export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssem
|
|
|
170
193
|
if (r.FLAT_NUMBER) {
|
|
171
194
|
house = `${String(r.FLAT_NUMBER)}/${house}`
|
|
172
195
|
}
|
|
196
|
+
|
|
173
197
|
const tuple = { house_number: house, street, locality, region: state, postcode }
|
|
198
|
+
|
|
174
199
|
seen++
|
|
175
200
|
|
|
176
201
|
if (reservoir.length < opts.sampleSize) {
|
|
@@ -183,6 +208,7 @@ export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssem
|
|
|
183
208
|
}
|
|
184
209
|
}
|
|
185
210
|
}
|
|
211
|
+
|
|
186
212
|
progress(`${state}: ${seen.toLocaleString()} valid joinable seen`)
|
|
187
213
|
}
|
|
188
214
|
|
|
@@ -193,7 +219,11 @@ export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssem
|
|
|
193
219
|
out.write(JSON.stringify(t) + "\n")
|
|
194
220
|
byState[t.region] = (byState[t.region] ?? 0) + 1
|
|
195
221
|
}
|
|
196
|
-
|
|
222
|
+
|
|
223
|
+
await new Promise<void>((res) => {
|
|
224
|
+
out.end(res)
|
|
225
|
+
})
|
|
226
|
+
|
|
197
227
|
progress(`wrote ${reservoir.length.toLocaleString()} tuples → ${opts.out}`)
|
|
198
228
|
|
|
199
229
|
return { written: reservoir.length, seen, heldOut, byState }
|
package/src/adapters/index.ts
CHANGED
|
@@ -81,55 +81,68 @@ for (const adapter of BUILTIN_ADAPTERS) {
|
|
|
81
81
|
|
|
82
82
|
export { BAN_ADAPTER_ID, banAdapter } from "./ban/adapter.ts"
|
|
83
83
|
export { FCC_BDC_ADAPTER_ID, FCC_BDC_DEFAULT_LICENSE, fccBdcAdapter } from "./fcc-bdc/adapter.ts"
|
|
84
|
+
|
|
84
85
|
export {
|
|
85
86
|
GEONAMES_POSTAL_ADAPTER_ID,
|
|
86
87
|
GEONAMES_POSTAL_DEFAULT_LICENSE,
|
|
87
88
|
geonamesPostalAdapter,
|
|
88
89
|
} from "./geonames-postal/adapter.ts"
|
|
90
|
+
|
|
89
91
|
export { GEONAMES_ADAPTER_ID, GEONAMES_DEFAULT_LICENSE, geonamesAdapter } from "./geonames/adapter.ts"
|
|
92
|
+
|
|
90
93
|
export {
|
|
91
94
|
OPENADDRESSES_ADAPTER_ID,
|
|
92
95
|
OPENADDRESSES_DEFAULT_LICENSE,
|
|
93
96
|
openaddressesAdapter,
|
|
94
97
|
} from "./openaddresses/adapter.ts"
|
|
98
|
+
|
|
95
99
|
export {
|
|
96
100
|
STATE_HI_SCHOOLS_ADAPTER_ID,
|
|
97
101
|
STATE_HI_SCHOOLS_DEFAULT_LICENSE,
|
|
98
102
|
stateHiSchoolsAdapter,
|
|
99
103
|
} from "./state-hi-schools/adapter.ts"
|
|
104
|
+
|
|
100
105
|
export {
|
|
101
106
|
STATE_IA_CONTRACTORS_ADAPTER_ID,
|
|
102
107
|
STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
|
|
103
108
|
stateIaContractorsAdapter,
|
|
104
109
|
} from "./state-ia-contractors/adapter.ts"
|
|
110
|
+
|
|
105
111
|
export {
|
|
106
112
|
STATE_NY_NOTARIES_ADAPTER_ID,
|
|
107
113
|
STATE_NY_NOTARIES_DEFAULT_LICENSE,
|
|
108
114
|
stateNyNotariesAdapter,
|
|
109
115
|
} from "./state-ny-notaries/adapter.ts"
|
|
116
|
+
|
|
110
117
|
export {
|
|
111
118
|
STATE_TX_NOTARIES_ADAPTER_ID,
|
|
112
119
|
STATE_TX_NOTARIES_DEFAULT_LICENSE,
|
|
113
120
|
stateTxNotariesAdapter,
|
|
114
121
|
} from "./state-tx-notaries/adapter.ts"
|
|
122
|
+
|
|
115
123
|
export { TIGER_ADAPTER_ID, TIGER_DEFAULT_LICENSE, tigerAdapter } from "./tiger/adapter.ts"
|
|
124
|
+
|
|
116
125
|
export {
|
|
117
126
|
USGOV_HRSA_FQHC_ADAPTER_ID,
|
|
118
127
|
USGOV_HRSA_FQHC_DEFAULT_LICENSE,
|
|
119
128
|
usgovHrsaFqhcAdapter,
|
|
120
129
|
} from "./usgov-hrsa-fqhc/adapter.ts"
|
|
130
|
+
|
|
121
131
|
export {
|
|
122
132
|
USGOV_IMLS_PLS_ADAPTER_ID,
|
|
123
133
|
USGOV_IMLS_PLS_DEFAULT_LICENSE,
|
|
124
134
|
usgovImlsPlsAdapter,
|
|
125
135
|
} from "./usgov-imls-pls/adapter.ts"
|
|
136
|
+
|
|
126
137
|
export { USGOV_IRS_BMF_ADAPTER_ID, USGOV_IRS_BMF_DEFAULT_LICENSE, usgovIrsBmfAdapter } from "./usgov-irs-bmf/adapter.ts"
|
|
127
138
|
export { USGOV_NAD_ADAPTER_ID, USGOV_NAD_DEFAULT_LICENSE, usgovNADAdapter } from "./usgov-nad/adapter.ts"
|
|
128
139
|
export { USGOV_NPPES_ADAPTER_ID, USGOV_NPPES_DEFAULT_LICENSE, usgovNPPESAdapter } from "./usgov-nppes/adapter.ts"
|
|
140
|
+
|
|
129
141
|
export {
|
|
130
142
|
USGOV_SAMHSA_ADAPTER_ID,
|
|
131
143
|
USGOV_SAMHSA_DEFAULT_LICENSE,
|
|
132
144
|
usgovSamhsaTreatmentLocatorAdapter,
|
|
133
145
|
} from "./usgov-samhsa-treatment-locator/adapter.ts"
|
|
146
|
+
|
|
134
147
|
export { WOF_ADMIN_ADAPTER_ID, wofAdminAdapter } from "./wof-admin-json/adapter.ts"
|
|
135
148
|
export { WOF_POSTALCODE_ADAPTER_ID, wofPostalcodeAdapter } from "./wof-postalcode-json/adapter.ts"
|