@mailwoman/corpus 8.0.0 → 8.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/adapter.d.ts +15 -5
- package/out/src/adapter.d.ts.map +1 -1
- package/out/src/adapter.js +11 -9
- package/out/src/adapter.js.map +1 -1
- package/out/src/adapters/ban/adapter.d.ts +7 -0
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +9 -2
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +5 -1
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +14 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +15 -2
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +11 -0
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +24 -5
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +11 -0
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +14 -1
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +10 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +11 -2
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts +18 -6
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +16 -6
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +11 -0
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +18 -3
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +11 -0
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +12 -1
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +11 -0
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +11 -0
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +15 -2
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +11 -0
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +13 -2
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +14 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +19 -4
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +12 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +17 -2
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +12 -1
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +11 -0
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +15 -2
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +7 -0
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +18 -3
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +13 -3
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +15 -5
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +7 -0
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +9 -2
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/align.d.ts +12 -4
- package/out/src/align.d.ts.map +1 -1
- package/out/src/align.js +9 -5
- package/out/src/align.js.map +1 -1
- package/out/src/build.d.ts +24 -8
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +3 -3
- package/out/src/build.js.map +1 -1
- package/out/src/codex/us-fips-state.d.ts +6 -2
- package/out/src/codex/us-fips-state.d.ts.map +1 -1
- package/out/src/codex/us-fips-state.js +3 -1
- package/out/src/codex/us-fips-state.js.map +1 -1
- package/out/src/golden.d.ts +21 -7
- package/out/src/golden.d.ts.map +1 -1
- package/out/src/golden.js +18 -10
- package/out/src/golden.js.map +1 -1
- package/out/src/license.d.ts +3 -1
- package/out/src/license.d.ts.map +1 -1
- package/out/src/license.js +4 -2
- package/out/src/license.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts +4 -2
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +5 -4
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/schema.d.ts +14 -10
- package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
- package/out/src/parquet-wrapper/schema.js +6 -2
- package/out/src/parquet-wrapper/schema.js.map +1 -1
- package/out/src/parquet-wrapper/writer.d.ts +12 -4
- package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
- package/out/src/parquet-wrapper/writer.js +12 -4
- package/out/src/parquet-wrapper/writer.js.map +1 -1
- package/out/src/parquet.d.ts +21 -7
- package/out/src/parquet.d.ts.map +1 -1
- package/out/src/parquet.js +15 -4
- package/out/src/parquet.js.map +1 -1
- package/out/src/runner.d.ts +39 -13
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +7 -3
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts +4 -0
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +14 -5
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/boundary-stress.d.ts +4 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +13 -7
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts +4 -0
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +39 -11
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts +4 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +14 -3
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts +4 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.js +4 -0
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts +4 -0
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +25 -8
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts +4 -0
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +21 -8
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts +4 -0
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +27 -8
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts +4 -0
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +16 -6
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.d.ts +4 -0
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/house-venue.js +4 -0
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts +9 -3
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +12 -4
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts +4 -0
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +27 -13
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +4 -0
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +9 -4
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/nl-postcode.d.ts +4 -0
- package/out/src/shard-recipes/nl-postcode.d.ts.map +1 -1
- package/out/src/shard-recipes/nl-postcode.js +8 -2
- package/out/src/shard-recipes/nl-postcode.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts +4 -0
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +8 -2
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts +4 -0
- package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street-led.js +5 -1
- package/out/src/shard-recipes/no-street-led.js.map +1 -1
- package/out/src/shard-recipes/no-street.d.ts +4 -0
- package/out/src/shard-recipes/no-street.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street.js +4 -0
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +4 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +99 -43
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts +4 -0
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +4 -0
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +48 -16
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +10 -6
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/si-bare-village.d.ts +4 -0
- package/out/src/shard-recipes/si-bare-village.d.ts.map +1 -1
- package/out/src/shard-recipes/si-bare-village.js +4 -0
- package/out/src/shard-recipes/si-bare-village.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +4 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +29 -10
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.d.ts +4 -0
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
- package/out/src/shard-recipes/street-bare.js +4 -0
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.d.ts +4 -0
- package/out/src/shard-recipes/street.d.ts.map +1 -1
- package/out/src/shard-recipes/street.js +4 -0
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/unit.d.ts +4 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +45 -14
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/split.d.ts +18 -6
- package/out/src/split.d.ts.map +1 -1
- package/out/src/split.js +4 -2
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize-anchor-absorption.d.ts +7 -1
- package/out/src/synthesize-anchor-absorption.d.ts.map +1 -1
- package/out/src/synthesize-anchor-absorption.js +28 -14
- package/out/src/synthesize-anchor-absorption.js.map +1 -1
- package/out/src/synthesize-boundary-stress.d.ts +3 -1
- package/out/src/synthesize-boundary-stress.d.ts.map +1 -1
- package/out/src/synthesize-boundary-stress.js +35 -28
- package/out/src/synthesize-boundary-stress.js.map +1 -1
- package/out/src/synthesize-german.d.ts +15 -5
- package/out/src/synthesize-german.d.ts.map +1 -1
- package/out/src/synthesize-german.js +9 -3
- package/out/src/synthesize-german.js.map +1 -1
- package/out/src/synthesize-house-venue.d.ts.map +1 -1
- package/out/src/synthesize-house-venue.js +15 -15
- package/out/src/synthesize-house-venue.js.map +1 -1
- package/out/src/synthesize-intersection.d.ts +9 -3
- package/out/src/synthesize-intersection.d.ts.map +1 -1
- package/out/src/synthesize-intersection.js +25 -6
- package/out/src/synthesize-intersection.js.map +1 -1
- package/out/src/synthesize-no-street.d.ts +3 -1
- package/out/src/synthesize-no-street.d.ts.map +1 -1
- package/out/src/synthesize-no-street.js +5 -6
- package/out/src/synthesize-no-street.js.map +1 -1
- package/out/src/synthesize-po-box.d.ts +15 -5
- package/out/src/synthesize-po-box.d.ts.map +1 -1
- package/out/src/synthesize-po-box.js +21 -7
- package/out/src/synthesize-po-box.js.map +1 -1
- package/out/src/synthesize-street.d.ts +3 -1
- package/out/src/synthesize-street.d.ts.map +1 -1
- package/out/src/synthesize-street.js +6 -1
- package/out/src/synthesize-street.js.map +1 -1
- package/out/src/synthesize.d.ts +39 -13
- package/out/src/synthesize.d.ts.map +1 -1
- package/out/src/synthesize.js +66 -36
- package/out/src/synthesize.js.map +1 -1
- package/out/src/tokenize.d.ts +15 -5
- package/out/src/tokenize.d.ts.map +1 -1
- package/out/src/tokenize.js.map +1 -1
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +3 -1
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/audit.d.ts.map +1 -1
- package/out/src/tools/audit.js +20 -8
- package/out/src/tools/audit.js.map +1 -1
- package/out/src/tools/corpus-stats.d.ts.map +1 -1
- package/out/src/tools/corpus-stats.js.map +1 -1
- package/out/src/tools/fetch/ban.d.ts.map +1 -1
- package/out/src/tools/fetch/ban.js +17 -10
- package/out/src/tools/fetch/ban.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts +30 -10
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +26 -6
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/hrsa.d.ts.map +1 -1
- package/out/src/tools/fetch/hrsa.js.map +1 -1
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +18 -18
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +3 -1
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +3 -1
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts +21 -7
- package/out/src/tools/fetch/nad.d.ts.map +1 -1
- package/out/src/tools/fetch/nad.js +5 -5
- package/out/src/tools/fetch/nad.js.map +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +9 -15
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts +3 -1
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/src/tools/fetch/openaddresses.js +32 -23
- package/out/src/tools/fetch/openaddresses.js.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +3 -1
- package/out/src/tools/fetch/ppd.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.js +14 -3
- package/out/src/tools/fetch/ppd.js.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.js +13 -6
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
- package/out/src/tools/fetch/state-sources.js +8 -3
- package/out/src/tools/fetch/state-sources.js.map +1 -1
- package/out/src/tools/fetch/tiger-full.d.ts +9 -3
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
- package/out/src/tools/fetch/tiger-full.js +35 -20
- package/out/src/tools/fetch/tiger-full.js.map +1 -1
- package/out/src/tools/gb-title-case.d.ts.map +1 -1
- package/out/src/tools/gb-title-case.js.map +1 -1
- package/out/src/tools/golden-expand.d.ts +24 -8
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +20 -12
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-promote.d.ts +18 -6
- package/out/src/tools/golden-promote.d.ts.map +1 -1
- package/out/src/tools/golden-promote.js +22 -7
- package/out/src/tools/golden-promote.js.map +1 -1
- package/out/src/tools/ingest-csv.d.ts +3 -1
- package/out/src/tools/ingest-csv.d.ts.map +1 -1
- package/out/src/tools/ingest-csv.js +17 -19
- package/out/src/tools/ingest-csv.js.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts +18 -6
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
- package/out/src/tools/jsonl-to-parquet.js +11 -5
- package/out/src/tools/jsonl-to-parquet.js.map +1 -1
- package/out/src/tools/lint-shard-vocab.d.ts +36 -12
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
- package/out/src/tools/lint-shard-vocab.js +25 -11
- package/out/src/tools/lint-shard-vocab.js.map +1 -1
- package/out/src/tools/lint-shard.d.ts +30 -10
- package/out/src/tools/lint-shard.d.ts.map +1 -1
- package/out/src/tools/lint-shard.js +33 -20
- package/out/src/tools/lint-shard.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +6 -4
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/shard-kryptonite.d.ts +6 -2
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +1 -1
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts +9 -3
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +2 -2
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/types.d.ts +51 -17
- package/out/src/types.d.ts.map +1 -1
- package/out/src/wof-json.d.ts +9 -3
- package/out/src/wof-json.d.ts.map +1 -1
- package/out/src/wof-json.js +6 -4
- package/out/src/wof-json.js.map +1 -1
- package/package.json +7 -6
- package/src/adapter.ts +24 -13
- package/src/adapters/ban/adapter.ts +12 -2
- package/src/adapters/ban/street-decompose.ts +7 -1
- package/src/adapters/fcc-bdc/adapter.ts +19 -3
- package/src/adapters/geonames/adapter.ts +28 -5
- package/src/adapters/geonames-postal/adapter.ts +17 -1
- package/src/adapters/gnaf/adapter.ts +15 -2
- package/src/adapters/gnaf/assemble.ts +40 -10
- package/src/adapters/index.ts +13 -0
- package/src/adapters/openaddresses/adapter.ts +22 -3
- package/src/adapters/overture/adapter.ts +18 -2
- package/src/adapters/state-hi-schools/adapter.ts +15 -0
- package/src/adapters/state-ia-contractors/adapter.ts +15 -0
- package/src/adapters/state-ny-notaries/adapter.ts +15 -0
- package/src/adapters/state-tx-notaries/adapter.ts +18 -2
- package/src/adapters/synth-po-box/adapter.ts +19 -2
- package/src/adapters/tiger/adapter.ts +23 -4
- package/src/adapters/tiger/street-decompose.ts +1 -0
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +18 -2
- package/src/adapters/usgov-imls-pls/adapter.ts +15 -0
- package/src/adapters/usgov-irs-bmf/adapter.ts +19 -2
- package/src/adapters/usgov-nad/adapter.ts +14 -1
- package/src/adapters/usgov-nppes/adapter.ts +13 -0
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +21 -3
- package/src/adapters/wof-admin-jp/adapter.ts +20 -3
- package/src/adapters/wof-admin-json/adapter.ts +22 -7
- package/src/adapters/wof-postalcode-json/adapter.ts +10 -2
- package/src/align.ts +21 -8
- package/src/build.ts +48 -11
- package/src/codex/us-fips-state.ts +6 -2
- package/src/golden.ts +28 -13
- package/src/license.ts +4 -2
- package/src/parquet-wrapper/reader.ts +6 -4
- package/src/parquet-wrapper/schema.ts +14 -10
- package/src/parquet-wrapper/writer.ts +12 -4
- package/src/parquet.ts +39 -8
- package/src/runner.ts +49 -15
- package/src/shard-recipes/anchor-absorption.ts +21 -5
- package/src/shard-recipes/boundary-stress.ts +20 -7
- package/src/shard-recipes/country-balanced.ts +78 -13
- package/src/shard-recipes/cz-pcfirst-preposition.ts +9 -0
- package/src/shard-recipes/fr-admin-split.ts +30 -5
- package/src/shard-recipes/fr-bare-street.ts +10 -0
- package/src/shard-recipes/fr-fragment.ts +33 -8
- package/src/shard-recipes/fr-lieudit.ts +35 -9
- package/src/shard-recipes/fr-order.ts +47 -9
- package/src/shard-recipes/german.ts +32 -6
- package/src/shard-recipes/house-venue.ts +7 -0
- package/src/shard-recipes/index.ts +12 -4
- package/src/shard-recipes/intersection.ts +63 -14
- package/src/shard-recipes/locale.ts +37 -4
- package/src/shard-recipes/nl-postcode.ts +12 -2
- package/src/shard-recipes/no-fragment.ts +12 -2
- package/src/shard-recipes/no-street-led.ts +11 -1
- package/src/shard-recipes/no-street.ts +7 -0
- package/src/shard-recipes/po-box-cedex.ts +145 -45
- package/src/shard-recipes/po-box.ts +7 -0
- package/src/shard-recipes/scaffold.ts +52 -20
- package/src/shard-recipes/si-bare-village.ts +9 -0
- package/src/shard-recipes/street-affix.ts +88 -13
- package/src/shard-recipes/street-bare.ts +10 -0
- package/src/shard-recipes/street.ts +7 -0
- package/src/shard-recipes/unit.ts +79 -16
- package/src/split.ts +28 -7
- package/src/synthesize-anchor-absorption.ts +41 -14
- package/src/synthesize-boundary-stress.ts +57 -30
- package/src/synthesize-german.ts +30 -7
- package/src/synthesize-house-venue.ts +22 -16
- package/src/synthesize-intersection.ts +34 -7
- package/src/synthesize-no-street.ts +18 -10
- package/src/synthesize-po-box.ts +42 -10
- package/src/synthesize-street.ts +10 -2
- package/src/synthesize.ts +81 -38
- package/src/tokenize.ts +15 -5
- package/src/tools/align-shard.ts +6 -1
- package/src/tools/audit.ts +57 -11
- package/src/tools/corpus-stats.ts +9 -0
- package/src/tools/fetch/ban.ts +32 -10
- package/src/tools/fetch/download.ts +49 -12
- package/src/tools/fetch/hrsa.ts +3 -0
- package/src/tools/fetch/imls-pls.ts +28 -18
- package/src/tools/fetch/index.ts +3 -1
- package/src/tools/fetch/nad.ts +39 -12
- package/src/tools/fetch/nppes.ts +17 -15
- package/src/tools/fetch/openaddresses.ts +52 -24
- package/src/tools/fetch/ppd.ts +22 -3
- package/src/tools/fetch/state-hi-schools.ts +17 -6
- package/src/tools/fetch/state-sources.ts +16 -3
- package/src/tools/fetch/tiger-full.ts +66 -23
- package/src/tools/gb-title-case.ts +1 -0
- package/src/tools/golden-expand.ts +64 -20
- package/src/tools/golden-promote.ts +53 -13
- package/src/tools/ingest-csv.ts +41 -26
- package/src/tools/jsonl-to-parquet.ts +30 -10
- package/src/tools/lint-shard-vocab.ts +75 -24
- package/src/tools/lint-shard.ts +75 -31
- package/src/tools/overlay-manifest.ts +7 -4
- package/src/tools/shard-kryptonite.ts +11 -3
- package/src/tools/shard-translit.ts +20 -5
- package/src/types.ts +51 -17
- package/src/wof-json.ts +18 -7
|
@@ -32,7 +32,15 @@ import {
|
|
|
32
32
|
} from "../../synthesize-po-box.ts"
|
|
33
33
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
34
34
|
|
|
35
|
+
/**
|
|
36
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
37
|
+
* it came from.
|
|
38
|
+
*/
|
|
35
39
|
export const SYNTH_PO_BOX_ADAPTER_ID = "synth-po-box"
|
|
40
|
+
/**
|
|
41
|
+
* License for the synthetic PO-box rows. The output is generated, but it inherits the terms of the real tuples it is
|
|
42
|
+
* derived from, so the attribution travels with it.
|
|
43
|
+
*/
|
|
36
44
|
export const SYNTH_PO_BOX_LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"
|
|
37
45
|
|
|
38
46
|
export interface PoBoxInputRow extends PoBoxBaseTuple {
|
|
@@ -69,9 +77,9 @@ function makeRandom(seed: number): () => number {
|
|
|
69
77
|
let s = seed
|
|
70
78
|
|
|
71
79
|
return () => {
|
|
72
|
-
s = (s *
|
|
80
|
+
s = (s * 1_664_525 + 1_013_904_223) % 4_294_967_296
|
|
73
81
|
|
|
74
|
-
return s /
|
|
82
|
+
return s / 4_294_967_296
|
|
75
83
|
}
|
|
76
84
|
}
|
|
77
85
|
|
|
@@ -113,6 +121,7 @@ export function createSynthPoBoxAdapter(opts: SynthPoBoxAdapterOptions = {}): Co
|
|
|
113
121
|
input = JSON.parse(trimmed) as PoBoxInputRow
|
|
114
122
|
} catch {
|
|
115
123
|
skipped++
|
|
124
|
+
|
|
116
125
|
continue
|
|
117
126
|
}
|
|
118
127
|
|
|
@@ -123,6 +132,7 @@ export function createSynthPoBoxAdapter(opts: SynthPoBoxAdapterOptions = {}): Co
|
|
|
123
132
|
|
|
124
133
|
if (!input.locality || !input.postcode || !input.country || (!input.region && !regionOptional)) {
|
|
125
134
|
skipped++
|
|
135
|
+
|
|
126
136
|
continue
|
|
127
137
|
}
|
|
128
138
|
|
|
@@ -152,6 +162,7 @@ export function createSynthPoBoxAdapter(opts: SynthPoBoxAdapterOptions = {}): Co
|
|
|
152
162
|
corpus_version: "",
|
|
153
163
|
license: SYNTH_PO_BOX_LICENSE,
|
|
154
164
|
}
|
|
165
|
+
|
|
155
166
|
emitted++
|
|
156
167
|
|
|
157
168
|
if (options.limit !== undefined && emitted >= options.limit) break
|
|
@@ -171,12 +182,14 @@ export function createSynthPoBoxAdapter(opts: SynthPoBoxAdapterOptions = {}): Co
|
|
|
171
182
|
random() < militaryRatio
|
|
172
183
|
) {
|
|
173
184
|
const mil = synthesizeMilitaryPoBoxRow({ random })
|
|
185
|
+
|
|
174
186
|
const sourceID = stableSourceID(SYNTH_PO_BOX_ADAPTER_ID, {
|
|
175
187
|
po_box: `${mil.components.po_box}#mil${militarySeq++}`,
|
|
176
188
|
locality: mil.components.locality!,
|
|
177
189
|
region: mil.components.region!,
|
|
178
190
|
postcode: mil.components.postcode!,
|
|
179
191
|
})
|
|
192
|
+
|
|
180
193
|
yield {
|
|
181
194
|
raw: mil.raw,
|
|
182
195
|
components: mil.components,
|
|
@@ -187,6 +200,7 @@ export function createSynthPoBoxAdapter(opts: SynthPoBoxAdapterOptions = {}): Co
|
|
|
187
200
|
corpus_version: "",
|
|
188
201
|
license: SYNTH_PO_BOX_LICENSE,
|
|
189
202
|
}
|
|
203
|
+
|
|
190
204
|
emitted++
|
|
191
205
|
}
|
|
192
206
|
}
|
|
@@ -194,4 +208,7 @@ export function createSynthPoBoxAdapter(opts: SynthPoBoxAdapterOptions = {}): Co
|
|
|
194
208
|
}
|
|
195
209
|
}
|
|
196
210
|
|
|
211
|
+
/**
|
|
212
|
+
* The configured adapter instance registered with the corpus builder.
|
|
213
|
+
*/
|
|
197
214
|
export const synthPoBoxAdapter = createSynthPoBoxAdapter()
|
|
@@ -41,7 +41,15 @@ import { formatAddress, reconcileComponents } from "../../format.ts"
|
|
|
41
41
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
42
42
|
import { decomposeStreet } from "./street-decompose.ts"
|
|
43
43
|
|
|
44
|
+
/**
|
|
45
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
46
|
+
* it came from.
|
|
47
|
+
*/
|
|
44
48
|
export const TIGER_ADAPTER_ID = "tiger"
|
|
49
|
+
/**
|
|
50
|
+
* License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
|
|
51
|
+
* than having to look them up.
|
|
52
|
+
*/
|
|
45
53
|
export const TIGER_DEFAULT_LICENSE = "Public Domain"
|
|
46
54
|
|
|
47
55
|
/**
|
|
@@ -125,7 +133,9 @@ function* streetVariants(row: TigerStreetRow): Iterable<{
|
|
|
125
133
|
}
|
|
126
134
|
}
|
|
127
135
|
|
|
128
|
-
/**
|
|
136
|
+
/**
|
|
137
|
+
* Three locality-level variants, mirroring `wof-admin`'s fan-out.
|
|
138
|
+
*/
|
|
129
139
|
function* placeVariants(row: TigerPlaceRow): Iterable<{
|
|
130
140
|
components: CanonicalRow["components"]
|
|
131
141
|
variantKey: string
|
|
@@ -141,17 +151,21 @@ function* placeVariants(row: TigerPlaceRow): Iterable<{
|
|
|
141
151
|
components: { locality: name },
|
|
142
152
|
variantKey: "locality-only",
|
|
143
153
|
}
|
|
154
|
+
|
|
144
155
|
yield {
|
|
145
156
|
components: { locality: name, region: state.abbreviation },
|
|
146
157
|
variantKey: "with-region",
|
|
147
158
|
}
|
|
159
|
+
|
|
148
160
|
yield {
|
|
149
161
|
components: { locality: name, region: state.abbreviation, country: US_COUNTRY_DISPLAY },
|
|
150
162
|
variantKey: "with-region-country",
|
|
151
163
|
}
|
|
152
164
|
}
|
|
153
165
|
|
|
154
|
-
/**
|
|
166
|
+
/**
|
|
167
|
+
* Build a TIGER adapter. Pure factory so multiple instances can be created in tests.
|
|
168
|
+
*/
|
|
155
169
|
export function createTigerAdapter(): CorpusAdapter {
|
|
156
170
|
return {
|
|
157
171
|
id: TIGER_ADAPTER_ID,
|
|
@@ -181,7 +195,7 @@ export function createTigerAdapter(): CorpusAdapter {
|
|
|
181
195
|
if (!raw) continue
|
|
182
196
|
const aligned = reconcileComponents(variant.components, raw)
|
|
183
197
|
|
|
184
|
-
if (Object.keys(aligned).length
|
|
198
|
+
if (!Object.keys(aligned).length) continue
|
|
185
199
|
|
|
186
200
|
yield {
|
|
187
201
|
raw,
|
|
@@ -193,6 +207,7 @@ export function createTigerAdapter(): CorpusAdapter {
|
|
|
193
207
|
corpus_version: "",
|
|
194
208
|
license: TIGER_DEFAULT_LICENSE,
|
|
195
209
|
}
|
|
210
|
+
|
|
196
211
|
emitted++
|
|
197
212
|
}
|
|
198
213
|
}
|
|
@@ -207,7 +222,7 @@ export function createTigerAdapter(): CorpusAdapter {
|
|
|
207
222
|
if (!raw) continue
|
|
208
223
|
const aligned = reconcileComponents(variant.components, raw)
|
|
209
224
|
|
|
210
|
-
if (Object.keys(aligned).length
|
|
225
|
+
if (!Object.keys(aligned).length) continue
|
|
211
226
|
|
|
212
227
|
yield {
|
|
213
228
|
raw,
|
|
@@ -219,6 +234,7 @@ export function createTigerAdapter(): CorpusAdapter {
|
|
|
219
234
|
corpus_version: "",
|
|
220
235
|
license: TIGER_DEFAULT_LICENSE,
|
|
221
236
|
}
|
|
237
|
+
|
|
222
238
|
emitted++
|
|
223
239
|
}
|
|
224
240
|
}
|
|
@@ -229,4 +245,7 @@ export function createTigerAdapter(): CorpusAdapter {
|
|
|
229
245
|
}
|
|
230
246
|
}
|
|
231
247
|
|
|
248
|
+
/**
|
|
249
|
+
* The configured adapter instance registered with the corpus builder.
|
|
250
|
+
*/
|
|
232
251
|
export const tigerAdapter = createTigerAdapter()
|
|
@@ -38,7 +38,15 @@ import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
|
|
|
38
38
|
import { reconcileComponents } from "../../format.ts"
|
|
39
39
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
40
40
|
|
|
41
|
+
/**
|
|
42
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
43
|
+
* it came from.
|
|
44
|
+
*/
|
|
41
45
|
export const USGOV_HRSA_FQHC_ADAPTER_ID = "usgov-hrsa-fqhc"
|
|
46
|
+
/**
|
|
47
|
+
* License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
|
|
48
|
+
* than having to look them up.
|
|
49
|
+
*/
|
|
42
50
|
export const USGOV_HRSA_FQHC_DEFAULT_LICENSE = "Public Domain"
|
|
43
51
|
|
|
44
52
|
/**
|
|
@@ -52,7 +60,9 @@ interface HrsaSiteRow {
|
|
|
52
60
|
"Site City": string
|
|
53
61
|
"Site State Abbreviation": string
|
|
54
62
|
"Site Postal Code": string
|
|
55
|
-
/**
|
|
63
|
+
/**
|
|
64
|
+
* Optional. Falls back to `stableSourceID` derived from components when missing.
|
|
65
|
+
*/
|
|
56
66
|
"Site ID"?: string
|
|
57
67
|
}
|
|
58
68
|
|
|
@@ -113,6 +123,7 @@ export function createUsgovHrsaFqhcAdapter(): CorpusAdapter {
|
|
|
113
123
|
}
|
|
114
124
|
|
|
115
125
|
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
126
|
+
|
|
116
127
|
const parser = stream.pipe(
|
|
117
128
|
csvParse({
|
|
118
129
|
columns: true,
|
|
@@ -160,9 +171,10 @@ export function createUsgovHrsaFqhcAdapter(): CorpusAdapter {
|
|
|
160
171
|
|
|
161
172
|
const aligned = reconcileComponents(components, raw)
|
|
162
173
|
|
|
163
|
-
if (Object.keys(aligned).length
|
|
174
|
+
if (!Object.keys(aligned).length) continue
|
|
164
175
|
|
|
165
176
|
const siteID = (record["Site ID"] ?? "").trim()
|
|
177
|
+
|
|
166
178
|
const sourceID = siteID
|
|
167
179
|
? `${USGOV_HRSA_FQHC_ADAPTER_ID}-${siteID}`
|
|
168
180
|
: stableSourceID(USGOV_HRSA_FQHC_ADAPTER_ID, aligned)
|
|
@@ -177,6 +189,7 @@ export function createUsgovHrsaFqhcAdapter(): CorpusAdapter {
|
|
|
177
189
|
corpus_version: "",
|
|
178
190
|
license: USGOV_HRSA_FQHC_DEFAULT_LICENSE,
|
|
179
191
|
}
|
|
192
|
+
|
|
180
193
|
emitted++
|
|
181
194
|
}
|
|
182
195
|
} finally {
|
|
@@ -186,4 +199,7 @@ export function createUsgovHrsaFqhcAdapter(): CorpusAdapter {
|
|
|
186
199
|
}
|
|
187
200
|
}
|
|
188
201
|
|
|
202
|
+
/**
|
|
203
|
+
* The configured adapter instance registered with the corpus builder.
|
|
204
|
+
*/
|
|
189
205
|
export const usgovHrsaFqhcAdapter = createUsgovHrsaFqhcAdapter()
|
|
@@ -27,7 +27,15 @@ import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
|
|
|
27
27
|
import { reconcileComponents } from "../../format.ts"
|
|
28
28
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
29
29
|
|
|
30
|
+
/**
|
|
31
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
32
|
+
* it came from.
|
|
33
|
+
*/
|
|
30
34
|
export const USGOV_IMLS_PLS_ADAPTER_ID = "usgov-imls-pls"
|
|
35
|
+
/**
|
|
36
|
+
* License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
|
|
37
|
+
* than having to look them up.
|
|
38
|
+
*/
|
|
31
39
|
export const USGOV_IMLS_PLS_DEFAULT_LICENSE = "Public Domain"
|
|
32
40
|
|
|
33
41
|
const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
|
|
@@ -65,6 +73,7 @@ export function createUsgovImlsPlsAdapter(): CorpusAdapter {
|
|
|
65
73
|
}
|
|
66
74
|
|
|
67
75
|
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
76
|
+
|
|
68
77
|
const parser = stream.pipe(
|
|
69
78
|
csvParse({
|
|
70
79
|
columns: true,
|
|
@@ -113,6 +122,7 @@ export function createUsgovImlsPlsAdapter(): CorpusAdapter {
|
|
|
113
122
|
}
|
|
114
123
|
|
|
115
124
|
const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
|
|
125
|
+
|
|
116
126
|
const raw = [
|
|
117
127
|
libName,
|
|
118
128
|
streetPart,
|
|
@@ -126,6 +136,7 @@ export function createUsgovImlsPlsAdapter(): CorpusAdapter {
|
|
|
126
136
|
if (Object.keys(aligned).length <= 2) continue
|
|
127
137
|
|
|
128
138
|
const fscsKey = (record.FSCSKEY ?? "").trim()
|
|
139
|
+
|
|
129
140
|
const sourceID = fscsKey
|
|
130
141
|
? `${USGOV_IMLS_PLS_ADAPTER_ID}-${fscsKey}`
|
|
131
142
|
: stableSourceID(USGOV_IMLS_PLS_ADAPTER_ID, aligned)
|
|
@@ -140,6 +151,7 @@ export function createUsgovImlsPlsAdapter(): CorpusAdapter {
|
|
|
140
151
|
corpus_version: "",
|
|
141
152
|
license: USGOV_IMLS_PLS_DEFAULT_LICENSE,
|
|
142
153
|
}
|
|
154
|
+
|
|
143
155
|
emitted++
|
|
144
156
|
}
|
|
145
157
|
} finally {
|
|
@@ -149,4 +161,7 @@ export function createUsgovImlsPlsAdapter(): CorpusAdapter {
|
|
|
149
161
|
}
|
|
150
162
|
}
|
|
151
163
|
|
|
164
|
+
/**
|
|
165
|
+
* The configured adapter instance registered with the corpus builder.
|
|
166
|
+
*/
|
|
152
167
|
export const usgovImlsPlsAdapter = createUsgovImlsPlsAdapter()
|
|
@@ -27,11 +27,21 @@ import { stableSourceID } from "../../adapter.ts"
|
|
|
27
27
|
import { reconcileComponents } from "../../format.ts"
|
|
28
28
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
29
29
|
|
|
30
|
+
/**
|
|
31
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
32
|
+
* it came from.
|
|
33
|
+
*/
|
|
30
34
|
export const USGOV_IRS_BMF_ADAPTER_ID = "usgov-irs-bmf"
|
|
35
|
+
/**
|
|
36
|
+
* License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
|
|
37
|
+
* than having to look them up.
|
|
38
|
+
*/
|
|
31
39
|
export const USGOV_IRS_BMF_DEFAULT_LICENSE = "Public Domain"
|
|
32
40
|
|
|
33
41
|
const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
|
|
34
|
-
|
|
42
|
+
/**
|
|
43
|
+
* PO box in its many written forms: "PO BOX 12", "P.O. BOX 12", "P O BOX 12", "POB 12", "BOX 12".
|
|
44
|
+
*/
|
|
35
45
|
const PO_BOX = /^\s*(?:P\.?\s?O\.?\s*BOX|POB|BOX)\s+\w/i
|
|
36
46
|
|
|
37
47
|
interface IrsBmfRow {
|
|
@@ -43,7 +53,9 @@ interface IrsBmfRow {
|
|
|
43
53
|
ZIP: string
|
|
44
54
|
}
|
|
45
55
|
|
|
46
|
-
/**
|
|
56
|
+
/**
|
|
57
|
+
* Classify the street line into a `po_box` or a `{house_number?, street}` split.
|
|
58
|
+
*/
|
|
47
59
|
function splitStreetLine(street: string): { po_box: string } | { house_number?: string; street: string } | null {
|
|
48
60
|
const trimmed = street.trim()
|
|
49
61
|
|
|
@@ -82,6 +94,7 @@ export function createUsgovIrsBmfAdapter(): CorpusAdapter {
|
|
|
82
94
|
}
|
|
83
95
|
|
|
84
96
|
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
97
|
+
|
|
85
98
|
const parser = stream.pipe(
|
|
86
99
|
csvParse({ columns: true, skip_empty_lines: true, relax_quotes: true, relax_column_count: true, trim: true })
|
|
87
100
|
)
|
|
@@ -143,6 +156,7 @@ export function createUsgovIrsBmfAdapter(): CorpusAdapter {
|
|
|
143
156
|
corpus_version: "",
|
|
144
157
|
license: USGOV_IRS_BMF_DEFAULT_LICENSE,
|
|
145
158
|
}
|
|
159
|
+
|
|
146
160
|
emitted++
|
|
147
161
|
}
|
|
148
162
|
} finally {
|
|
@@ -152,4 +166,7 @@ export function createUsgovIrsBmfAdapter(): CorpusAdapter {
|
|
|
152
166
|
}
|
|
153
167
|
}
|
|
154
168
|
|
|
169
|
+
/**
|
|
170
|
+
* The configured adapter instance registered with the corpus builder.
|
|
171
|
+
*/
|
|
155
172
|
export const usgovIrsBmfAdapter = createUsgovIrsBmfAdapter()
|
|
@@ -38,7 +38,15 @@ import { TextSpliterator } from "spliterator"
|
|
|
38
38
|
import { reconcileComponents } from "../../format.ts"
|
|
39
39
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
40
40
|
|
|
41
|
+
/**
|
|
42
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
43
|
+
* it came from.
|
|
44
|
+
*/
|
|
41
45
|
export const USGOV_NAD_ADAPTER_ID = "usgov-nad"
|
|
46
|
+
/**
|
|
47
|
+
* License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
|
|
48
|
+
* than having to look them up.
|
|
49
|
+
*/
|
|
42
50
|
export const USGOV_NAD_DEFAULT_LICENSE = "Public Domain"
|
|
43
51
|
|
|
44
52
|
interface NADRecord {
|
|
@@ -192,6 +200,7 @@ function decomposeNADStreet(r: NADRecord): DecomposedNADStreet | undefined {
|
|
|
192
200
|
|
|
193
201
|
return { prefix, street: name, suffix, full }
|
|
194
202
|
}
|
|
203
|
+
|
|
195
204
|
const full = (r.StNam_Full ?? "").toString().trim()
|
|
196
205
|
|
|
197
206
|
if (full) return { full, street: full }
|
|
@@ -243,7 +252,7 @@ export function createUsgovNADAdapter(): CorpusAdapter {
|
|
|
243
252
|
// Single-file inputs (e.g. a bulk-extracted CSV) are not currently supported — the
|
|
244
253
|
// featureserver shard pattern is the primary distribution.
|
|
245
254
|
const entries = await readdir(opts.inputPath)
|
|
246
|
-
const shards = entries.filter((n) => n.endsWith(".ndjson")).
|
|
255
|
+
const shards = entries.filter((n) => n.endsWith(".ndjson")).toSorted()
|
|
247
256
|
|
|
248
257
|
let emitted = 0
|
|
249
258
|
outer: for (const shard of shards) {
|
|
@@ -328,6 +337,7 @@ export function createUsgovNADAdapter(): CorpusAdapter {
|
|
|
328
337
|
corpus_version: "",
|
|
329
338
|
license: USGOV_NAD_DEFAULT_LICENSE,
|
|
330
339
|
}
|
|
340
|
+
|
|
331
341
|
emitted++
|
|
332
342
|
}
|
|
333
343
|
}
|
|
@@ -335,4 +345,7 @@ export function createUsgovNADAdapter(): CorpusAdapter {
|
|
|
335
345
|
}
|
|
336
346
|
}
|
|
337
347
|
|
|
348
|
+
/**
|
|
349
|
+
* The configured adapter instance registered with the corpus builder.
|
|
350
|
+
*/
|
|
338
351
|
export const usgovNADAdapter = createUsgovNADAdapter()
|
|
@@ -30,7 +30,15 @@ import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
|
|
|
30
30
|
import { reconcileComponents } from "../../format.ts"
|
|
31
31
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
32
32
|
|
|
33
|
+
/**
|
|
34
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
35
|
+
* it came from.
|
|
36
|
+
*/
|
|
33
37
|
export const USGOV_NPPES_ADAPTER_ID = "usgov-nppes"
|
|
38
|
+
/**
|
|
39
|
+
* License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
|
|
40
|
+
* than having to look them up.
|
|
41
|
+
*/
|
|
34
42
|
export const USGOV_NPPES_DEFAULT_LICENSE = "Public Domain"
|
|
35
43
|
|
|
36
44
|
const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
|
|
@@ -86,6 +94,7 @@ export function createUsgovNPPESAdapter(): CorpusAdapter {
|
|
|
86
94
|
}
|
|
87
95
|
|
|
88
96
|
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
97
|
+
|
|
89
98
|
const parser = stream.pipe(
|
|
90
99
|
csvParse({
|
|
91
100
|
columns: true,
|
|
@@ -157,6 +166,7 @@ export function createUsgovNPPESAdapter(): CorpusAdapter {
|
|
|
157
166
|
corpus_version: "",
|
|
158
167
|
license: USGOV_NPPES_DEFAULT_LICENSE,
|
|
159
168
|
}
|
|
169
|
+
|
|
160
170
|
emitted++
|
|
161
171
|
}
|
|
162
172
|
} finally {
|
|
@@ -166,4 +176,7 @@ export function createUsgovNPPESAdapter(): CorpusAdapter {
|
|
|
166
176
|
}
|
|
167
177
|
}
|
|
168
178
|
|
|
179
|
+
/**
|
|
180
|
+
* The configured adapter instance registered with the corpus builder.
|
|
181
|
+
*/
|
|
169
182
|
export const usgovNPPESAdapter = createUsgovNPPESAdapter()
|
|
@@ -37,7 +37,15 @@ import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
|
|
|
37
37
|
import { reconcileComponents } from "../../format.ts"
|
|
38
38
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
39
39
|
|
|
40
|
+
/**
|
|
41
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
42
|
+
* it came from.
|
|
43
|
+
*/
|
|
40
44
|
export const USGOV_SAMHSA_ADAPTER_ID = "usgov-samhsa-treatment-locator"
|
|
45
|
+
/**
|
|
46
|
+
* License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
|
|
47
|
+
* than having to look them up.
|
|
48
|
+
*/
|
|
41
49
|
export const USGOV_SAMHSA_DEFAULT_LICENSE = "Public Domain"
|
|
42
50
|
|
|
43
51
|
/**
|
|
@@ -52,7 +60,9 @@ interface SamhsaSiteRow {
|
|
|
52
60
|
city: string
|
|
53
61
|
state: string
|
|
54
62
|
zip: string
|
|
55
|
-
/**
|
|
63
|
+
/**
|
|
64
|
+
* Optional. Falls back to `stableSourceID` derived from components when missing.
|
|
65
|
+
*/
|
|
56
66
|
frid?: string
|
|
57
67
|
}
|
|
58
68
|
|
|
@@ -109,7 +119,9 @@ function composeVenue(name1: string, name2: string | undefined): string {
|
|
|
109
119
|
return `${n1} - ${n2}`
|
|
110
120
|
}
|
|
111
121
|
|
|
112
|
-
/**
|
|
122
|
+
/**
|
|
123
|
+
* Same envelope-style format as HRSA: venue prefix, street body, city/state/zip suffix.
|
|
124
|
+
*/
|
|
113
125
|
function composeRaw(
|
|
114
126
|
venue: string,
|
|
115
127
|
house: string | undefined,
|
|
@@ -137,6 +149,7 @@ export function createUsgovSamhsaTreatmentLocatorAdapter(): CorpusAdapter {
|
|
|
137
149
|
}
|
|
138
150
|
|
|
139
151
|
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
152
|
+
|
|
140
153
|
const parser = stream.pipe(
|
|
141
154
|
csvParse({
|
|
142
155
|
columns: true,
|
|
@@ -182,9 +195,10 @@ export function createUsgovSamhsaTreatmentLocatorAdapter(): CorpusAdapter {
|
|
|
182
195
|
|
|
183
196
|
const aligned = reconcileComponents(components, raw)
|
|
184
197
|
|
|
185
|
-
if (Object.keys(aligned).length
|
|
198
|
+
if (!Object.keys(aligned).length) continue
|
|
186
199
|
|
|
187
200
|
const frID = (record.frid ?? "").trim()
|
|
201
|
+
|
|
188
202
|
const sourceID = frID
|
|
189
203
|
? `${USGOV_SAMHSA_ADAPTER_ID}-${frID}`
|
|
190
204
|
: stableSourceID(USGOV_SAMHSA_ADAPTER_ID, aligned)
|
|
@@ -199,6 +213,7 @@ export function createUsgovSamhsaTreatmentLocatorAdapter(): CorpusAdapter {
|
|
|
199
213
|
corpus_version: "",
|
|
200
214
|
license: USGOV_SAMHSA_DEFAULT_LICENSE,
|
|
201
215
|
}
|
|
216
|
+
|
|
202
217
|
emitted++
|
|
203
218
|
}
|
|
204
219
|
} finally {
|
|
@@ -208,4 +223,7 @@ export function createUsgovSamhsaTreatmentLocatorAdapter(): CorpusAdapter {
|
|
|
208
223
|
}
|
|
209
224
|
}
|
|
210
225
|
|
|
226
|
+
/**
|
|
227
|
+
* The configured adapter instance registered with the corpus builder.
|
|
228
|
+
*/
|
|
211
229
|
export const usgovSamhsaTreatmentLocatorAdapter = createUsgovSamhsaTreatmentLocatorAdapter()
|
|
@@ -33,6 +33,15 @@ import { DatabaseSync } from "node:sqlite"
|
|
|
33
33
|
|
|
34
34
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
35
35
|
|
|
36
|
+
/**
|
|
37
|
+
* Ancestry hops walked upward from a WOF record before giving up — deeper than any real JP admin chain.
|
|
38
|
+
*/
|
|
39
|
+
const MAX_ANCESTRY_DEPTH = 6
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
43
|
+
* it came from.
|
|
44
|
+
*/
|
|
36
45
|
export const WOF_ADMIN_JP_ADAPTER_ID = "wof-admin-jp"
|
|
37
46
|
|
|
38
47
|
interface PlaceRow {
|
|
@@ -49,13 +58,15 @@ interface NameRow {
|
|
|
49
58
|
language: string
|
|
50
59
|
}
|
|
51
60
|
|
|
52
|
-
/**
|
|
61
|
+
/**
|
|
62
|
+
* Walk parent chain up to 6 levels.
|
|
63
|
+
*/
|
|
53
64
|
function chainOf(db: DatabaseSync, startID: number, _jpnNames: Map<number, string>): PlaceRow[] {
|
|
54
65
|
const stmt = db.prepare(`SELECT id, name, placetype, parent_id, country FROM spr WHERE id = ?`)
|
|
55
66
|
const out: PlaceRow[] = []
|
|
56
67
|
let id = startID
|
|
57
68
|
|
|
58
|
-
for (let i = 0; i <
|
|
69
|
+
for (let i = 0; i < MAX_ANCESTRY_DEPTH && id > 0; i++) {
|
|
59
70
|
const row = stmt.get(id) as PlaceRow | undefined
|
|
60
71
|
|
|
61
72
|
if (!row) break
|
|
@@ -66,7 +77,9 @@ function chainOf(db: DatabaseSync, startID: number, _jpnNames: Map<number, strin
|
|
|
66
77
|
return out
|
|
67
78
|
}
|
|
68
79
|
|
|
69
|
-
/**
|
|
80
|
+
/**
|
|
81
|
+
* Pick the best display name for a place: prefer Japanese variant, fall back to English.
|
|
82
|
+
*/
|
|
70
83
|
function pickName(row: PlaceRow, jpnNames: Map<number, string>): string {
|
|
71
84
|
return jpnNames.get(row.id) ?? row.name
|
|
72
85
|
}
|
|
@@ -164,6 +177,7 @@ export function createWOFAdminJpAdapter(): CorpusAdapter {
|
|
|
164
177
|
corpus_version: "",
|
|
165
178
|
license: "CC-BY-4.0",
|
|
166
179
|
}
|
|
180
|
+
|
|
167
181
|
emitted++
|
|
168
182
|
}
|
|
169
183
|
} finally {
|
|
@@ -173,4 +187,7 @@ export function createWOFAdminJpAdapter(): CorpusAdapter {
|
|
|
173
187
|
}
|
|
174
188
|
}
|
|
175
189
|
|
|
190
|
+
/**
|
|
191
|
+
* The configured adapter instance registered with the corpus builder.
|
|
192
|
+
*/
|
|
176
193
|
export const wofAdminJpAdapter = createWOFAdminJpAdapter()
|
|
@@ -61,13 +61,17 @@ const COUNTRY_DISPLAY_NAME: Record<string, string> = {
|
|
|
61
61
|
FR: "France",
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
-
/**
|
|
64
|
+
/**
|
|
65
|
+
* BCP-47 locale defaulting for the corpus row's `locale` field. Defaulted by country.
|
|
66
|
+
*/
|
|
65
67
|
const LOCALE_BY_COUNTRY: Record<string, string> = {
|
|
66
68
|
US: "en-US",
|
|
67
69
|
FR: "fr-FR",
|
|
68
70
|
}
|
|
69
71
|
|
|
70
|
-
/**
|
|
72
|
+
/**
|
|
73
|
+
* Map a WOF placetype to a Mailwoman `ComponentTag`, or `undefined` to skip.
|
|
74
|
+
*/
|
|
71
75
|
function placetypeToTag(placetype: WhosOnFirstPlacetype | string): ComponentTag | undefined {
|
|
72
76
|
switch (placetype) {
|
|
73
77
|
case "country":
|
|
@@ -93,10 +97,14 @@ function placetypeToTag(placetype: WhosOnFirstPlacetype | string): ComponentTag
|
|
|
93
97
|
}
|
|
94
98
|
|
|
95
99
|
interface VariantSpec {
|
|
96
|
-
/**
|
|
100
|
+
/**
|
|
101
|
+
* Hierarchy-variant id appended to `source_id`.
|
|
102
|
+
*/
|
|
97
103
|
suffix: string
|
|
98
104
|
|
|
99
|
-
/**
|
|
105
|
+
/**
|
|
106
|
+
* Component tag → display string the adapter will hand to the runner.
|
|
107
|
+
*/
|
|
100
108
|
components: Partial<Record<ComponentTag, string>>
|
|
101
109
|
}
|
|
102
110
|
|
|
@@ -202,6 +210,10 @@ export function nameSlotsFor(rec: WOFRecord): Array<{ key: string; value: string
|
|
|
202
210
|
return slots
|
|
203
211
|
}
|
|
204
212
|
|
|
213
|
+
/**
|
|
214
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
215
|
+
* it came from.
|
|
216
|
+
*/
|
|
205
217
|
export const WOF_ADMIN_ADAPTER_ID = "wof-admin"
|
|
206
218
|
|
|
207
219
|
/**
|
|
@@ -236,7 +248,7 @@ export function createWOFAdminAdapter(): CorpusAdapter {
|
|
|
236
248
|
const ancestry = buildAncestryIndex(byID)
|
|
237
249
|
|
|
238
250
|
// Pass 2: emit rows in sorted-id order for deterministic JSONL.
|
|
239
|
-
const ids = [...byID.keys()].
|
|
251
|
+
const ids = [...byID.keys()].toSorted((a, b) => a - b)
|
|
240
252
|
let emitted = 0
|
|
241
253
|
|
|
242
254
|
for (const id of ids) {
|
|
@@ -256,7 +268,7 @@ export function createWOFAdminAdapter(): CorpusAdapter {
|
|
|
256
268
|
if (!raw) continue
|
|
257
269
|
const aligned = reconcileComponents(variant.components, raw)
|
|
258
270
|
|
|
259
|
-
if (Object.keys(aligned).length
|
|
271
|
+
if (!Object.keys(aligned).length) continue
|
|
260
272
|
|
|
261
273
|
yield {
|
|
262
274
|
raw,
|
|
@@ -268,6 +280,7 @@ export function createWOFAdminAdapter(): CorpusAdapter {
|
|
|
268
280
|
corpus_version: "",
|
|
269
281
|
license: "CC0-1.0",
|
|
270
282
|
}
|
|
283
|
+
|
|
271
284
|
emitted++
|
|
272
285
|
}
|
|
273
286
|
}
|
|
@@ -276,5 +289,7 @@ export function createWOFAdminAdapter(): CorpusAdapter {
|
|
|
276
289
|
}
|
|
277
290
|
}
|
|
278
291
|
|
|
279
|
-
/**
|
|
292
|
+
/**
|
|
293
|
+
* Single shared instance, suitable for `defaultAdapterRegistry`.
|
|
294
|
+
*/
|
|
280
295
|
export const wofAdminAdapter = createWOFAdminAdapter()
|