@mailwoman/corpus 8.0.0 → 8.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/adapter.d.ts +15 -5
- package/out/src/adapter.d.ts.map +1 -1
- package/out/src/adapter.js +11 -9
- package/out/src/adapter.js.map +1 -1
- package/out/src/adapters/ban/adapter.d.ts +7 -0
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +9 -2
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +5 -1
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +14 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +15 -2
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +11 -0
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +24 -5
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +11 -0
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +14 -1
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +10 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +11 -2
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts +18 -6
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +16 -6
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +11 -0
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +18 -3
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +11 -0
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +12 -1
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +11 -0
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +11 -0
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +15 -2
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +11 -0
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +13 -2
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +14 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +19 -4
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +12 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +17 -2
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +12 -1
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +11 -0
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +15 -2
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +7 -0
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +18 -3
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +13 -3
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +15 -5
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +7 -0
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +9 -2
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/align.d.ts +12 -4
- package/out/src/align.d.ts.map +1 -1
- package/out/src/align.js +9 -5
- package/out/src/align.js.map +1 -1
- package/out/src/build.d.ts +24 -8
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +3 -3
- package/out/src/build.js.map +1 -1
- package/out/src/codex/us-fips-state.d.ts +6 -2
- package/out/src/codex/us-fips-state.d.ts.map +1 -1
- package/out/src/codex/us-fips-state.js +3 -1
- package/out/src/codex/us-fips-state.js.map +1 -1
- package/out/src/golden.d.ts +21 -7
- package/out/src/golden.d.ts.map +1 -1
- package/out/src/golden.js +18 -10
- package/out/src/golden.js.map +1 -1
- package/out/src/license.d.ts +3 -1
- package/out/src/license.d.ts.map +1 -1
- package/out/src/license.js +4 -2
- package/out/src/license.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts +4 -2
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +5 -4
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/schema.d.ts +14 -10
- package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
- package/out/src/parquet-wrapper/schema.js +6 -2
- package/out/src/parquet-wrapper/schema.js.map +1 -1
- package/out/src/parquet-wrapper/writer.d.ts +12 -4
- package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
- package/out/src/parquet-wrapper/writer.js +12 -4
- package/out/src/parquet-wrapper/writer.js.map +1 -1
- package/out/src/parquet.d.ts +21 -7
- package/out/src/parquet.d.ts.map +1 -1
- package/out/src/parquet.js +15 -4
- package/out/src/parquet.js.map +1 -1
- package/out/src/runner.d.ts +39 -13
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +7 -3
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts +4 -0
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +14 -5
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/boundary-stress.d.ts +4 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +13 -7
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts +4 -0
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +39 -11
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts +4 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +14 -3
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts +4 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.js +4 -0
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts +4 -0
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +25 -8
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts +4 -0
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +21 -8
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts +4 -0
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +27 -8
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts +4 -0
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +16 -6
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.d.ts +4 -0
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/house-venue.js +4 -0
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts +9 -3
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +12 -4
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts +4 -0
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +27 -13
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +4 -0
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +9 -4
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/nl-postcode.d.ts +4 -0
- package/out/src/shard-recipes/nl-postcode.d.ts.map +1 -1
- package/out/src/shard-recipes/nl-postcode.js +8 -2
- package/out/src/shard-recipes/nl-postcode.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts +4 -0
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +8 -2
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts +4 -0
- package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street-led.js +5 -1
- package/out/src/shard-recipes/no-street-led.js.map +1 -1
- package/out/src/shard-recipes/no-street.d.ts +4 -0
- package/out/src/shard-recipes/no-street.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street.js +4 -0
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +4 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +99 -43
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts +4 -0
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +4 -0
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +48 -16
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +10 -6
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/si-bare-village.d.ts +4 -0
- package/out/src/shard-recipes/si-bare-village.d.ts.map +1 -1
- package/out/src/shard-recipes/si-bare-village.js +4 -0
- package/out/src/shard-recipes/si-bare-village.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +4 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +29 -10
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.d.ts +4 -0
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
- package/out/src/shard-recipes/street-bare.js +4 -0
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.d.ts +4 -0
- package/out/src/shard-recipes/street.d.ts.map +1 -1
- package/out/src/shard-recipes/street.js +4 -0
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/unit.d.ts +4 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +45 -14
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/split.d.ts +18 -6
- package/out/src/split.d.ts.map +1 -1
- package/out/src/split.js +4 -2
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize-anchor-absorption.d.ts +7 -1
- package/out/src/synthesize-anchor-absorption.d.ts.map +1 -1
- package/out/src/synthesize-anchor-absorption.js +28 -14
- package/out/src/synthesize-anchor-absorption.js.map +1 -1
- package/out/src/synthesize-boundary-stress.d.ts +3 -1
- package/out/src/synthesize-boundary-stress.d.ts.map +1 -1
- package/out/src/synthesize-boundary-stress.js +35 -28
- package/out/src/synthesize-boundary-stress.js.map +1 -1
- package/out/src/synthesize-german.d.ts +15 -5
- package/out/src/synthesize-german.d.ts.map +1 -1
- package/out/src/synthesize-german.js +9 -3
- package/out/src/synthesize-german.js.map +1 -1
- package/out/src/synthesize-house-venue.d.ts.map +1 -1
- package/out/src/synthesize-house-venue.js +15 -15
- package/out/src/synthesize-house-venue.js.map +1 -1
- package/out/src/synthesize-intersection.d.ts +9 -3
- package/out/src/synthesize-intersection.d.ts.map +1 -1
- package/out/src/synthesize-intersection.js +25 -6
- package/out/src/synthesize-intersection.js.map +1 -1
- package/out/src/synthesize-no-street.d.ts +3 -1
- package/out/src/synthesize-no-street.d.ts.map +1 -1
- package/out/src/synthesize-no-street.js +5 -6
- package/out/src/synthesize-no-street.js.map +1 -1
- package/out/src/synthesize-po-box.d.ts +15 -5
- package/out/src/synthesize-po-box.d.ts.map +1 -1
- package/out/src/synthesize-po-box.js +21 -7
- package/out/src/synthesize-po-box.js.map +1 -1
- package/out/src/synthesize-street.d.ts +3 -1
- package/out/src/synthesize-street.d.ts.map +1 -1
- package/out/src/synthesize-street.js +6 -1
- package/out/src/synthesize-street.js.map +1 -1
- package/out/src/synthesize.d.ts +39 -13
- package/out/src/synthesize.d.ts.map +1 -1
- package/out/src/synthesize.js +66 -36
- package/out/src/synthesize.js.map +1 -1
- package/out/src/tokenize.d.ts +15 -5
- package/out/src/tokenize.d.ts.map +1 -1
- package/out/src/tokenize.js.map +1 -1
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +3 -1
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/audit.d.ts.map +1 -1
- package/out/src/tools/audit.js +20 -8
- package/out/src/tools/audit.js.map +1 -1
- package/out/src/tools/corpus-stats.d.ts.map +1 -1
- package/out/src/tools/corpus-stats.js.map +1 -1
- package/out/src/tools/fetch/ban.d.ts.map +1 -1
- package/out/src/tools/fetch/ban.js +17 -10
- package/out/src/tools/fetch/ban.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts +30 -10
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +26 -6
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/hrsa.d.ts.map +1 -1
- package/out/src/tools/fetch/hrsa.js.map +1 -1
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +18 -18
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +3 -1
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +3 -1
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts +21 -7
- package/out/src/tools/fetch/nad.d.ts.map +1 -1
- package/out/src/tools/fetch/nad.js +5 -5
- package/out/src/tools/fetch/nad.js.map +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +9 -15
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts +3 -1
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/src/tools/fetch/openaddresses.js +32 -23
- package/out/src/tools/fetch/openaddresses.js.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +3 -1
- package/out/src/tools/fetch/ppd.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.js +14 -3
- package/out/src/tools/fetch/ppd.js.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.js +13 -6
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
- package/out/src/tools/fetch/state-sources.js +8 -3
- package/out/src/tools/fetch/state-sources.js.map +1 -1
- package/out/src/tools/fetch/tiger-full.d.ts +9 -3
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
- package/out/src/tools/fetch/tiger-full.js +35 -20
- package/out/src/tools/fetch/tiger-full.js.map +1 -1
- package/out/src/tools/gb-title-case.d.ts.map +1 -1
- package/out/src/tools/gb-title-case.js.map +1 -1
- package/out/src/tools/golden-expand.d.ts +24 -8
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +20 -12
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-promote.d.ts +18 -6
- package/out/src/tools/golden-promote.d.ts.map +1 -1
- package/out/src/tools/golden-promote.js +22 -7
- package/out/src/tools/golden-promote.js.map +1 -1
- package/out/src/tools/ingest-csv.d.ts +3 -1
- package/out/src/tools/ingest-csv.d.ts.map +1 -1
- package/out/src/tools/ingest-csv.js +17 -19
- package/out/src/tools/ingest-csv.js.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts +18 -6
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
- package/out/src/tools/jsonl-to-parquet.js +11 -5
- package/out/src/tools/jsonl-to-parquet.js.map +1 -1
- package/out/src/tools/lint-shard-vocab.d.ts +36 -12
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
- package/out/src/tools/lint-shard-vocab.js +25 -11
- package/out/src/tools/lint-shard-vocab.js.map +1 -1
- package/out/src/tools/lint-shard.d.ts +30 -10
- package/out/src/tools/lint-shard.d.ts.map +1 -1
- package/out/src/tools/lint-shard.js +33 -20
- package/out/src/tools/lint-shard.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +6 -4
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/shard-kryptonite.d.ts +6 -2
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +1 -1
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts +9 -3
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +2 -2
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/types.d.ts +51 -17
- package/out/src/types.d.ts.map +1 -1
- package/out/src/wof-json.d.ts +9 -3
- package/out/src/wof-json.d.ts.map +1 -1
- package/out/src/wof-json.js +6 -4
- package/out/src/wof-json.js.map +1 -1
- package/package.json +7 -6
- package/src/adapter.ts +24 -13
- package/src/adapters/ban/adapter.ts +12 -2
- package/src/adapters/ban/street-decompose.ts +7 -1
- package/src/adapters/fcc-bdc/adapter.ts +19 -3
- package/src/adapters/geonames/adapter.ts +28 -5
- package/src/adapters/geonames-postal/adapter.ts +17 -1
- package/src/adapters/gnaf/adapter.ts +15 -2
- package/src/adapters/gnaf/assemble.ts +40 -10
- package/src/adapters/index.ts +13 -0
- package/src/adapters/openaddresses/adapter.ts +22 -3
- package/src/adapters/overture/adapter.ts +18 -2
- package/src/adapters/state-hi-schools/adapter.ts +15 -0
- package/src/adapters/state-ia-contractors/adapter.ts +15 -0
- package/src/adapters/state-ny-notaries/adapter.ts +15 -0
- package/src/adapters/state-tx-notaries/adapter.ts +18 -2
- package/src/adapters/synth-po-box/adapter.ts +19 -2
- package/src/adapters/tiger/adapter.ts +23 -4
- package/src/adapters/tiger/street-decompose.ts +1 -0
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +18 -2
- package/src/adapters/usgov-imls-pls/adapter.ts +15 -0
- package/src/adapters/usgov-irs-bmf/adapter.ts +19 -2
- package/src/adapters/usgov-nad/adapter.ts +14 -1
- package/src/adapters/usgov-nppes/adapter.ts +13 -0
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +21 -3
- package/src/adapters/wof-admin-jp/adapter.ts +20 -3
- package/src/adapters/wof-admin-json/adapter.ts +22 -7
- package/src/adapters/wof-postalcode-json/adapter.ts +10 -2
- package/src/align.ts +21 -8
- package/src/build.ts +48 -11
- package/src/codex/us-fips-state.ts +6 -2
- package/src/golden.ts +28 -13
- package/src/license.ts +4 -2
- package/src/parquet-wrapper/reader.ts +6 -4
- package/src/parquet-wrapper/schema.ts +14 -10
- package/src/parquet-wrapper/writer.ts +12 -4
- package/src/parquet.ts +39 -8
- package/src/runner.ts +49 -15
- package/src/shard-recipes/anchor-absorption.ts +21 -5
- package/src/shard-recipes/boundary-stress.ts +20 -7
- package/src/shard-recipes/country-balanced.ts +78 -13
- package/src/shard-recipes/cz-pcfirst-preposition.ts +9 -0
- package/src/shard-recipes/fr-admin-split.ts +30 -5
- package/src/shard-recipes/fr-bare-street.ts +10 -0
- package/src/shard-recipes/fr-fragment.ts +33 -8
- package/src/shard-recipes/fr-lieudit.ts +35 -9
- package/src/shard-recipes/fr-order.ts +47 -9
- package/src/shard-recipes/german.ts +32 -6
- package/src/shard-recipes/house-venue.ts +7 -0
- package/src/shard-recipes/index.ts +12 -4
- package/src/shard-recipes/intersection.ts +63 -14
- package/src/shard-recipes/locale.ts +37 -4
- package/src/shard-recipes/nl-postcode.ts +12 -2
- package/src/shard-recipes/no-fragment.ts +12 -2
- package/src/shard-recipes/no-street-led.ts +11 -1
- package/src/shard-recipes/no-street.ts +7 -0
- package/src/shard-recipes/po-box-cedex.ts +145 -45
- package/src/shard-recipes/po-box.ts +7 -0
- package/src/shard-recipes/scaffold.ts +52 -20
- package/src/shard-recipes/si-bare-village.ts +9 -0
- package/src/shard-recipes/street-affix.ts +88 -13
- package/src/shard-recipes/street-bare.ts +10 -0
- package/src/shard-recipes/street.ts +7 -0
- package/src/shard-recipes/unit.ts +79 -16
- package/src/split.ts +28 -7
- package/src/synthesize-anchor-absorption.ts +41 -14
- package/src/synthesize-boundary-stress.ts +57 -30
- package/src/synthesize-german.ts +30 -7
- package/src/synthesize-house-venue.ts +22 -16
- package/src/synthesize-intersection.ts +34 -7
- package/src/synthesize-no-street.ts +18 -10
- package/src/synthesize-po-box.ts +42 -10
- package/src/synthesize-street.ts +10 -2
- package/src/synthesize.ts +81 -38
- package/src/tokenize.ts +15 -5
- package/src/tools/align-shard.ts +6 -1
- package/src/tools/audit.ts +57 -11
- package/src/tools/corpus-stats.ts +9 -0
- package/src/tools/fetch/ban.ts +32 -10
- package/src/tools/fetch/download.ts +49 -12
- package/src/tools/fetch/hrsa.ts +3 -0
- package/src/tools/fetch/imls-pls.ts +28 -18
- package/src/tools/fetch/index.ts +3 -1
- package/src/tools/fetch/nad.ts +39 -12
- package/src/tools/fetch/nppes.ts +17 -15
- package/src/tools/fetch/openaddresses.ts +52 -24
- package/src/tools/fetch/ppd.ts +22 -3
- package/src/tools/fetch/state-hi-schools.ts +17 -6
- package/src/tools/fetch/state-sources.ts +16 -3
- package/src/tools/fetch/tiger-full.ts +66 -23
- package/src/tools/gb-title-case.ts +1 -0
- package/src/tools/golden-expand.ts +64 -20
- package/src/tools/golden-promote.ts +53 -13
- package/src/tools/ingest-csv.ts +41 -26
- package/src/tools/jsonl-to-parquet.ts +30 -10
- package/src/tools/lint-shard-vocab.ts +75 -24
- package/src/tools/lint-shard.ts +75 -31
- package/src/tools/overlay-manifest.ts +7 -4
- package/src/tools/shard-kryptonite.ts +11 -3
- package/src/tools/shard-translit.ts +20 -5
- package/src/types.ts +51 -17
- package/src/wof-json.ts +18 -7
|
@@ -34,8 +34,14 @@ import { stableSourceID } from "../../adapter.ts"
|
|
|
34
34
|
import { reconcileComponents } from "../../format.ts"
|
|
35
35
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
36
36
|
|
|
37
|
+
/**
|
|
38
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
39
|
+
* it came from.
|
|
40
|
+
*/
|
|
37
41
|
export const GNAF_ADAPTER_ID = "gnaf"
|
|
38
|
-
/**
|
|
42
|
+
/**
|
|
43
|
+
* Open G-NAF is freely redistributable with attribution to Geoscape Australia (CC-BY-style).
|
|
44
|
+
*/
|
|
39
45
|
export const GNAF_DEFAULT_LICENSE = "CC-BY-4.0"
|
|
40
46
|
|
|
41
47
|
interface GNAFTuple {
|
|
@@ -102,8 +108,10 @@ export function createGNAFAdapter(): CorpusAdapter {
|
|
|
102
108
|
|
|
103
109
|
const orders = renderOrders(t)
|
|
104
110
|
const order = idx % orders.length
|
|
111
|
+
|
|
105
112
|
idx++
|
|
106
113
|
const raw = orders[order]!
|
|
114
|
+
|
|
107
115
|
const components: CanonicalRow["components"] = {
|
|
108
116
|
house_number: t.house_number,
|
|
109
117
|
street: t.street,
|
|
@@ -119,7 +127,8 @@ export function createGNAFAdapter(): CorpusAdapter {
|
|
|
119
127
|
|
|
120
128
|
const aligned = reconcileComponents(components, raw)
|
|
121
129
|
|
|
122
|
-
if (Object.keys(aligned).length
|
|
130
|
+
if (!Object.keys(aligned).length) continue
|
|
131
|
+
|
|
123
132
|
yield {
|
|
124
133
|
raw,
|
|
125
134
|
components: aligned,
|
|
@@ -130,10 +139,14 @@ export function createGNAFAdapter(): CorpusAdapter {
|
|
|
130
139
|
corpus_version: "",
|
|
131
140
|
license: GNAF_DEFAULT_LICENSE,
|
|
132
141
|
}
|
|
142
|
+
|
|
133
143
|
emitted++
|
|
134
144
|
}
|
|
135
145
|
},
|
|
136
146
|
}
|
|
137
147
|
}
|
|
138
148
|
|
|
149
|
+
/**
|
|
150
|
+
* The configured adapter instance registered with the corpus builder.
|
|
151
|
+
*/
|
|
139
152
|
export const gnafAdapter = createGNAFAdapter()
|
|
@@ -33,17 +33,25 @@ import { join } from "node:path"
|
|
|
33
33
|
import { PSVSpliterator } from "spliterator"
|
|
34
34
|
|
|
35
35
|
export interface GNAFAssembleOptions {
|
|
36
|
-
/**
|
|
36
|
+
/**
|
|
37
|
+
* G-NAF `Standard` directory (holds the per-state `*_psv.psv` tables).
|
|
38
|
+
*/
|
|
37
39
|
standardDir: string
|
|
38
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* Target sample size (uniform reservoir → population-proportional across states).
|
|
42
|
+
*/
|
|
39
43
|
sampleSize: number
|
|
40
|
-
/**
|
|
44
|
+
/**
|
|
45
|
+
* Output JSONL path.
|
|
46
|
+
*/
|
|
41
47
|
out: string
|
|
42
48
|
/**
|
|
43
49
|
* Optional held-out eval JSONL (rows with a `components` field) — its (street,locality,postcode) are excluded.
|
|
44
50
|
*/
|
|
45
51
|
holdoutPath?: string
|
|
46
|
-
/**
|
|
52
|
+
/**
|
|
53
|
+
* Progress sink (the CLI passes a setter).
|
|
54
|
+
*/
|
|
47
55
|
onProgress?: (message: string) => void
|
|
48
56
|
}
|
|
49
57
|
|
|
@@ -54,25 +62,32 @@ export interface GNAFAssembleResult {
|
|
|
54
62
|
byState: Record<string, number>
|
|
55
63
|
}
|
|
56
64
|
|
|
57
|
-
/**
|
|
65
|
+
/**
|
|
66
|
+
* UPPERCASE → Title Case, preserving intra-word apostrophes/hyphens (O'Brien, Coff's Harbour).
|
|
67
|
+
*/
|
|
58
68
|
export function titlecase(s: string): string {
|
|
59
69
|
return s
|
|
60
70
|
.toLowerCase()
|
|
61
|
-
.
|
|
71
|
+
.replaceAll(/(^|[\s'\-/])([a-z])/g, (_m, p: string, c: string) => p + c.toUpperCase())
|
|
62
72
|
.trim()
|
|
63
73
|
}
|
|
64
74
|
|
|
65
|
-
/**
|
|
75
|
+
/**
|
|
76
|
+
* Holdout/dedup key: a street within a locality+postcode (house-number-agnostic — conservative).
|
|
77
|
+
*/
|
|
66
78
|
export function gnafHoldoutKey(street: string, locality: string, postcode: string): string {
|
|
67
79
|
return `${street}|${locality}|${postcode}`.toLowerCase()
|
|
68
80
|
}
|
|
69
81
|
|
|
70
82
|
type Row = Record<string, string | number | undefined>
|
|
83
|
+
|
|
71
84
|
async function* psvObjects(path: string): AsyncIterable<Row> {
|
|
72
85
|
yield* PSVSpliterator.fromAsync(path, { mode: "object", header: true }) as AsyncIterable<Row>
|
|
73
86
|
}
|
|
74
87
|
|
|
75
|
-
/**
|
|
88
|
+
/**
|
|
89
|
+
* Load a small lookup table fully into a Map keyed by `keyCol`.
|
|
90
|
+
*/
|
|
76
91
|
async function loadMap<V>(paths: string[], keyCol: string, pick: (r: Row) => V): Promise<Map<string, V>> {
|
|
77
92
|
const m = new Map<string, V>()
|
|
78
93
|
|
|
@@ -89,7 +104,9 @@ async function loadMap<V>(paths: string[], keyCol: string, pick: (r: Row) => V):
|
|
|
89
104
|
return m
|
|
90
105
|
}
|
|
91
106
|
|
|
92
|
-
/**
|
|
107
|
+
/**
|
|
108
|
+
* Build the held-out key set from an eval JSONL whose rows carry a `components` object.
|
|
109
|
+
*/
|
|
93
110
|
async function loadHoldout(path: string): Promise<Set<string>> {
|
|
94
111
|
const keys = new Set<string>()
|
|
95
112
|
const text = await readFile(path, "utf8")
|
|
@@ -114,6 +131,7 @@ async function loadHoldout(path: string): Promise<Set<string>> {
|
|
|
114
131
|
export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssembleResult> {
|
|
115
132
|
const progress = opts.onProgress ?? (() => {})
|
|
116
133
|
const files = await readdir(opts.standardDir)
|
|
134
|
+
|
|
117
135
|
const pick = (re: RegExp, exclude?: RegExp) =>
|
|
118
136
|
files.filter((f) => re.test(f) && !(exclude && exclude.test(f))).map((f) => join(opts.standardDir, f))
|
|
119
137
|
|
|
@@ -129,16 +147,19 @@ export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssem
|
|
|
129
147
|
}
|
|
130
148
|
|
|
131
149
|
progress(`loading STREET_LOCALITY (${streetPaths.length} files) + LOCALITY (${localityPaths.length})…`)
|
|
150
|
+
|
|
132
151
|
const streetMap = await loadMap(streetPaths, "STREET_LOCALITY_PID", (r) => ({
|
|
133
152
|
name: String(r.STREET_NAME ?? ""),
|
|
134
153
|
type: String(r.STREET_TYPE_CODE ?? ""),
|
|
135
154
|
suffix: String(r.STREET_SUFFIX_CODE ?? ""),
|
|
136
155
|
}))
|
|
156
|
+
|
|
137
157
|
const localityMap = await loadMap(localityPaths, "LOCALITY_PID", (r) => String(r.LOCALITY_NAME ?? ""))
|
|
138
158
|
progress(`streets=${streetMap.size.toLocaleString()} localities=${localityMap.size.toLocaleString()}`)
|
|
139
159
|
|
|
140
160
|
const reservoir: Array<{ house_number: string; street: string; locality: string; region: string; postcode: string }> =
|
|
141
161
|
[]
|
|
162
|
+
|
|
142
163
|
let seen = 0
|
|
143
164
|
let heldOut = 0
|
|
144
165
|
|
|
@@ -159,8 +180,10 @@ export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssem
|
|
|
159
180
|
|
|
160
181
|
if (holdout.has(gnafHoldoutKey(street, locality, postcode))) {
|
|
161
182
|
heldOut++
|
|
183
|
+
|
|
162
184
|
continue
|
|
163
185
|
}
|
|
186
|
+
|
|
164
187
|
let house = numberFirst + (r.NUMBER_FIRST_SUFFIX ? String(r.NUMBER_FIRST_SUFFIX) : "")
|
|
165
188
|
|
|
166
189
|
if (r.NUMBER_LAST) {
|
|
@@ -170,7 +193,9 @@ export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssem
|
|
|
170
193
|
if (r.FLAT_NUMBER) {
|
|
171
194
|
house = `${String(r.FLAT_NUMBER)}/${house}`
|
|
172
195
|
}
|
|
196
|
+
|
|
173
197
|
const tuple = { house_number: house, street, locality, region: state, postcode }
|
|
198
|
+
|
|
174
199
|
seen++
|
|
175
200
|
|
|
176
201
|
if (reservoir.length < opts.sampleSize) {
|
|
@@ -183,6 +208,7 @@ export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssem
|
|
|
183
208
|
}
|
|
184
209
|
}
|
|
185
210
|
}
|
|
211
|
+
|
|
186
212
|
progress(`${state}: ${seen.toLocaleString()} valid joinable seen`)
|
|
187
213
|
}
|
|
188
214
|
|
|
@@ -193,7 +219,11 @@ export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssem
|
|
|
193
219
|
out.write(JSON.stringify(t) + "\n")
|
|
194
220
|
byState[t.region] = (byState[t.region] ?? 0) + 1
|
|
195
221
|
}
|
|
196
|
-
|
|
222
|
+
|
|
223
|
+
await new Promise<void>((res) => {
|
|
224
|
+
out.end(res)
|
|
225
|
+
})
|
|
226
|
+
|
|
197
227
|
progress(`wrote ${reservoir.length.toLocaleString()} tuples → ${opts.out}`)
|
|
198
228
|
|
|
199
229
|
return { written: reservoir.length, seen, heldOut, byState }
|
package/src/adapters/index.ts
CHANGED
|
@@ -81,55 +81,68 @@ for (const adapter of BUILTIN_ADAPTERS) {
|
|
|
81
81
|
|
|
82
82
|
export { BAN_ADAPTER_ID, banAdapter } from "./ban/adapter.ts"
|
|
83
83
|
export { FCC_BDC_ADAPTER_ID, FCC_BDC_DEFAULT_LICENSE, fccBdcAdapter } from "./fcc-bdc/adapter.ts"
|
|
84
|
+
|
|
84
85
|
export {
|
|
85
86
|
GEONAMES_POSTAL_ADAPTER_ID,
|
|
86
87
|
GEONAMES_POSTAL_DEFAULT_LICENSE,
|
|
87
88
|
geonamesPostalAdapter,
|
|
88
89
|
} from "./geonames-postal/adapter.ts"
|
|
90
|
+
|
|
89
91
|
export { GEONAMES_ADAPTER_ID, GEONAMES_DEFAULT_LICENSE, geonamesAdapter } from "./geonames/adapter.ts"
|
|
92
|
+
|
|
90
93
|
export {
|
|
91
94
|
OPENADDRESSES_ADAPTER_ID,
|
|
92
95
|
OPENADDRESSES_DEFAULT_LICENSE,
|
|
93
96
|
openaddressesAdapter,
|
|
94
97
|
} from "./openaddresses/adapter.ts"
|
|
98
|
+
|
|
95
99
|
export {
|
|
96
100
|
STATE_HI_SCHOOLS_ADAPTER_ID,
|
|
97
101
|
STATE_HI_SCHOOLS_DEFAULT_LICENSE,
|
|
98
102
|
stateHiSchoolsAdapter,
|
|
99
103
|
} from "./state-hi-schools/adapter.ts"
|
|
104
|
+
|
|
100
105
|
export {
|
|
101
106
|
STATE_IA_CONTRACTORS_ADAPTER_ID,
|
|
102
107
|
STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
|
|
103
108
|
stateIaContractorsAdapter,
|
|
104
109
|
} from "./state-ia-contractors/adapter.ts"
|
|
110
|
+
|
|
105
111
|
export {
|
|
106
112
|
STATE_NY_NOTARIES_ADAPTER_ID,
|
|
107
113
|
STATE_NY_NOTARIES_DEFAULT_LICENSE,
|
|
108
114
|
stateNyNotariesAdapter,
|
|
109
115
|
} from "./state-ny-notaries/adapter.ts"
|
|
116
|
+
|
|
110
117
|
export {
|
|
111
118
|
STATE_TX_NOTARIES_ADAPTER_ID,
|
|
112
119
|
STATE_TX_NOTARIES_DEFAULT_LICENSE,
|
|
113
120
|
stateTxNotariesAdapter,
|
|
114
121
|
} from "./state-tx-notaries/adapter.ts"
|
|
122
|
+
|
|
115
123
|
export { TIGER_ADAPTER_ID, TIGER_DEFAULT_LICENSE, tigerAdapter } from "./tiger/adapter.ts"
|
|
124
|
+
|
|
116
125
|
export {
|
|
117
126
|
USGOV_HRSA_FQHC_ADAPTER_ID,
|
|
118
127
|
USGOV_HRSA_FQHC_DEFAULT_LICENSE,
|
|
119
128
|
usgovHrsaFqhcAdapter,
|
|
120
129
|
} from "./usgov-hrsa-fqhc/adapter.ts"
|
|
130
|
+
|
|
121
131
|
export {
|
|
122
132
|
USGOV_IMLS_PLS_ADAPTER_ID,
|
|
123
133
|
USGOV_IMLS_PLS_DEFAULT_LICENSE,
|
|
124
134
|
usgovImlsPlsAdapter,
|
|
125
135
|
} from "./usgov-imls-pls/adapter.ts"
|
|
136
|
+
|
|
126
137
|
export { USGOV_IRS_BMF_ADAPTER_ID, USGOV_IRS_BMF_DEFAULT_LICENSE, usgovIrsBmfAdapter } from "./usgov-irs-bmf/adapter.ts"
|
|
127
138
|
export { USGOV_NAD_ADAPTER_ID, USGOV_NAD_DEFAULT_LICENSE, usgovNADAdapter } from "./usgov-nad/adapter.ts"
|
|
128
139
|
export { USGOV_NPPES_ADAPTER_ID, USGOV_NPPES_DEFAULT_LICENSE, usgovNPPESAdapter } from "./usgov-nppes/adapter.ts"
|
|
140
|
+
|
|
129
141
|
export {
|
|
130
142
|
USGOV_SAMHSA_ADAPTER_ID,
|
|
131
143
|
USGOV_SAMHSA_DEFAULT_LICENSE,
|
|
132
144
|
usgovSamhsaTreatmentLocatorAdapter,
|
|
133
145
|
} from "./usgov-samhsa-treatment-locator/adapter.ts"
|
|
146
|
+
|
|
134
147
|
export { WOF_ADMIN_ADAPTER_ID, wofAdminAdapter } from "./wof-admin-json/adapter.ts"
|
|
135
148
|
export { WOF_POSTALCODE_ADAPTER_ID, wofPostalcodeAdapter } from "./wof-postalcode-json/adapter.ts"
|
|
@@ -43,7 +43,15 @@ import { formatAddress, reconcileComponents } from "../../format.ts"
|
|
|
43
43
|
import { SHARE_ALIKE_PATTERN } from "../../license.ts"
|
|
44
44
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
45
45
|
|
|
46
|
+
/**
|
|
47
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
48
|
+
* it came from.
|
|
49
|
+
*/
|
|
46
50
|
export const OPENADDRESSES_ADAPTER_ID = "openaddresses"
|
|
51
|
+
/**
|
|
52
|
+
* License carried by this source (CC-BY-4.0), attached to each row so downstream consumers inherit the terms rather
|
|
53
|
+
* than having to look them up.
|
|
54
|
+
*/
|
|
47
55
|
export const OPENADDRESSES_DEFAULT_LICENSE = "CC-BY-4.0"
|
|
48
56
|
|
|
49
57
|
/**
|
|
@@ -63,7 +71,9 @@ interface OaProperties {
|
|
|
63
71
|
license?: string
|
|
64
72
|
}
|
|
65
73
|
|
|
66
|
-
/**
|
|
74
|
+
/**
|
|
75
|
+
* Return a lowercase-keyed view of a Feature's properties so case variants both work.
|
|
76
|
+
*/
|
|
67
77
|
function normalizeProperties(raw: unknown): OaProperties {
|
|
68
78
|
if (!raw || typeof raw !== "object") return {}
|
|
69
79
|
const out: Record<string, string> = {}
|
|
@@ -79,7 +89,9 @@ function normalizeProperties(raw: unknown): OaProperties {
|
|
|
79
89
|
return out as OaProperties
|
|
80
90
|
}
|
|
81
91
|
|
|
82
|
-
/**
|
|
92
|
+
/**
|
|
93
|
+
* Parse a single ND-GeoJSON line; return null for blanks, comments, or non-Feature shapes.
|
|
94
|
+
*/
|
|
83
95
|
function parseFeatureLine(line: string): OaProperties | null {
|
|
84
96
|
const trimmed = line.trim()
|
|
85
97
|
|
|
@@ -135,6 +147,7 @@ export function createOpenaddressesAdapter(opts: OpenaddressesAdapterOptions = {
|
|
|
135
147
|
"openaddresses adapter: --country is required (OpenAddresses files are country-partitioned but rows lack a country field)"
|
|
136
148
|
)
|
|
137
149
|
}
|
|
150
|
+
|
|
138
151
|
const country = adapterOpts.country
|
|
139
152
|
|
|
140
153
|
// TextSpliterator streams string lines (parseFeatureLine keeps tolerating blank/`#`/
|
|
@@ -172,6 +185,7 @@ export function createOpenaddressesAdapter(opts: OpenaddressesAdapterOptions = {
|
|
|
172
185
|
|
|
173
186
|
if (!allowShareAlike && SHARE_ALIKE_PATTERN.test(license)) {
|
|
174
187
|
shareAlikeBlocked++
|
|
188
|
+
|
|
175
189
|
continue
|
|
176
190
|
}
|
|
177
191
|
|
|
@@ -207,9 +221,10 @@ export function createOpenaddressesAdapter(opts: OpenaddressesAdapterOptions = {
|
|
|
207
221
|
|
|
208
222
|
const aligned = reconcileComponents(components, raw)
|
|
209
223
|
|
|
210
|
-
if (Object.keys(aligned).length
|
|
224
|
+
if (!Object.keys(aligned).length) continue
|
|
211
225
|
|
|
212
226
|
const sourceIDSeed = props.hash?.trim() || props.id?.trim()
|
|
227
|
+
|
|
213
228
|
const sourceID = sourceIDSeed
|
|
214
229
|
? `${OPENADDRESSES_ADAPTER_ID}-${sourceIDSeed}`
|
|
215
230
|
: stableSourceID(OPENADDRESSES_ADAPTER_ID, aligned)
|
|
@@ -223,6 +238,7 @@ export function createOpenaddressesAdapter(opts: OpenaddressesAdapterOptions = {
|
|
|
223
238
|
corpus_version: "",
|
|
224
239
|
license,
|
|
225
240
|
}
|
|
241
|
+
|
|
226
242
|
emitted++
|
|
227
243
|
}
|
|
228
244
|
} finally {
|
|
@@ -234,4 +250,7 @@ export function createOpenaddressesAdapter(opts: OpenaddressesAdapterOptions = {
|
|
|
234
250
|
}
|
|
235
251
|
}
|
|
236
252
|
|
|
253
|
+
/**
|
|
254
|
+
* The configured adapter instance registered with the corpus builder.
|
|
255
|
+
*/
|
|
237
256
|
export const openaddressesAdapter = createOpenaddressesAdapter()
|
|
@@ -36,10 +36,20 @@ import { stableSourceID } from "../../adapter.ts"
|
|
|
36
36
|
import { formatAddress, reconcileComponents } from "../../format.ts"
|
|
37
37
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
38
38
|
|
|
39
|
+
/**
|
|
40
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
41
|
+
* it came from.
|
|
42
|
+
*/
|
|
39
43
|
export const OVERTURE_ADAPTER_ID = "overture"
|
|
44
|
+
/**
|
|
45
|
+
* License carried by this source (CDLA-Permissive-2.0), attached to each row so downstream consumers inherit the terms
|
|
46
|
+
* rather than having to look them up.
|
|
47
|
+
*/
|
|
40
48
|
export const OVERTURE_DEFAULT_LICENSE = "CDLA-Permissive-2.0"
|
|
41
49
|
|
|
42
|
-
/**
|
|
50
|
+
/**
|
|
51
|
+
* The flattened per-row shape emitted by `ingest-overture-addresses.ts --corpus-jsonl`.
|
|
52
|
+
*/
|
|
43
53
|
interface OvertureCorpusRow {
|
|
44
54
|
street?: string
|
|
45
55
|
number?: string
|
|
@@ -74,6 +84,7 @@ export function createOvertureAdapter(): CorpusAdapter {
|
|
|
74
84
|
"overture adapter: --country is required (the Overture JSONL is per-country and rows omit a country field)"
|
|
75
85
|
)
|
|
76
86
|
}
|
|
87
|
+
|
|
77
88
|
const country = opts.country
|
|
78
89
|
|
|
79
90
|
// TextSpliterator streams string lines (parseLine keeps tolerating blank/`#`/malformed
|
|
@@ -109,6 +120,7 @@ export function createOvertureAdapter(): CorpusAdapter {
|
|
|
109
120
|
if (/^\d/.test(number)) {
|
|
110
121
|
components.house_number = number
|
|
111
122
|
}
|
|
123
|
+
|
|
112
124
|
components.street = street
|
|
113
125
|
|
|
114
126
|
if (unit) {
|
|
@@ -129,7 +141,7 @@ export function createOvertureAdapter(): CorpusAdapter {
|
|
|
129
141
|
|
|
130
142
|
const aligned = reconcileComponents(components, raw)
|
|
131
143
|
|
|
132
|
-
if (Object.keys(aligned).length
|
|
144
|
+
if (!Object.keys(aligned).length) continue
|
|
133
145
|
|
|
134
146
|
yield {
|
|
135
147
|
raw,
|
|
@@ -140,10 +152,14 @@ export function createOvertureAdapter(): CorpusAdapter {
|
|
|
140
152
|
corpus_version: "",
|
|
141
153
|
license: OVERTURE_DEFAULT_LICENSE,
|
|
142
154
|
}
|
|
155
|
+
|
|
143
156
|
emitted++
|
|
144
157
|
}
|
|
145
158
|
},
|
|
146
159
|
}
|
|
147
160
|
}
|
|
148
161
|
|
|
162
|
+
/**
|
|
163
|
+
* The configured adapter instance registered with the corpus builder.
|
|
164
|
+
*/
|
|
149
165
|
export const overtureAdapter = createOvertureAdapter()
|
|
@@ -38,7 +38,15 @@ import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
|
|
|
38
38
|
import { reconcileComponents } from "../../format.ts"
|
|
39
39
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
40
40
|
|
|
41
|
+
/**
|
|
42
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
43
|
+
* it came from.
|
|
44
|
+
*/
|
|
41
45
|
export const STATE_HI_SCHOOLS_ADAPTER_ID = "state-hi-schools"
|
|
46
|
+
/**
|
|
47
|
+
* License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
|
|
48
|
+
* than having to look them up.
|
|
49
|
+
*/
|
|
42
50
|
export const STATE_HI_SCHOOLS_DEFAULT_LICENSE = "Public Domain"
|
|
43
51
|
|
|
44
52
|
const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
|
|
@@ -88,6 +96,7 @@ export function createStateHiSchoolsAdapter(): CorpusAdapter {
|
|
|
88
96
|
}
|
|
89
97
|
|
|
90
98
|
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
99
|
+
|
|
91
100
|
const parser = stream.pipe(
|
|
92
101
|
csvParse({
|
|
93
102
|
columns: true,
|
|
@@ -132,6 +141,7 @@ export function createStateHiSchoolsAdapter(): CorpusAdapter {
|
|
|
132
141
|
}
|
|
133
142
|
|
|
134
143
|
const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
|
|
144
|
+
|
|
135
145
|
const raw = [
|
|
136
146
|
name,
|
|
137
147
|
streetPart,
|
|
@@ -145,6 +155,7 @@ export function createStateHiSchoolsAdapter(): CorpusAdapter {
|
|
|
145
155
|
if (Object.keys(aligned).length <= 2) continue
|
|
146
156
|
|
|
147
157
|
const code = (record.code ?? "").toString().trim()
|
|
158
|
+
|
|
148
159
|
const sourceID = code
|
|
149
160
|
? `${STATE_HI_SCHOOLS_ADAPTER_ID}-${code}`
|
|
150
161
|
: stableSourceID(STATE_HI_SCHOOLS_ADAPTER_ID, aligned)
|
|
@@ -159,6 +170,7 @@ export function createStateHiSchoolsAdapter(): CorpusAdapter {
|
|
|
159
170
|
corpus_version: "",
|
|
160
171
|
license: STATE_HI_SCHOOLS_DEFAULT_LICENSE,
|
|
161
172
|
}
|
|
173
|
+
|
|
162
174
|
emitted++
|
|
163
175
|
}
|
|
164
176
|
} finally {
|
|
@@ -168,4 +180,7 @@ export function createStateHiSchoolsAdapter(): CorpusAdapter {
|
|
|
168
180
|
}
|
|
169
181
|
}
|
|
170
182
|
|
|
183
|
+
/**
|
|
184
|
+
* The configured adapter instance registered with the corpus builder.
|
|
185
|
+
*/
|
|
171
186
|
export const stateHiSchoolsAdapter = createStateHiSchoolsAdapter()
|
|
@@ -25,7 +25,15 @@ import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
|
|
|
25
25
|
import { reconcileComponents } from "../../format.ts"
|
|
26
26
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
27
27
|
|
|
28
|
+
/**
|
|
29
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
30
|
+
* it came from.
|
|
31
|
+
*/
|
|
28
32
|
export const STATE_IA_CONTRACTORS_ADAPTER_ID = "state-ia-contractors"
|
|
33
|
+
/**
|
|
34
|
+
* License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
|
|
35
|
+
* than having to look them up.
|
|
36
|
+
*/
|
|
29
37
|
export const STATE_IA_CONTRACTORS_DEFAULT_LICENSE = "Public Domain"
|
|
30
38
|
|
|
31
39
|
const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
|
|
@@ -66,6 +74,7 @@ export function createStateIaContractorsAdapter(): CorpusAdapter {
|
|
|
66
74
|
}
|
|
67
75
|
|
|
68
76
|
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
77
|
+
|
|
69
78
|
const parser = stream.pipe(
|
|
70
79
|
csvParse({
|
|
71
80
|
columns: true,
|
|
@@ -113,6 +122,7 @@ export function createStateIaContractorsAdapter(): CorpusAdapter {
|
|
|
113
122
|
}
|
|
114
123
|
|
|
115
124
|
const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
|
|
125
|
+
|
|
116
126
|
const raw = [venue, streetPart, [city, [stateAbbr, zip].filter(Boolean).join(" ")].filter(Boolean).join(", ")]
|
|
117
127
|
.filter(Boolean)
|
|
118
128
|
.join(", ")
|
|
@@ -122,6 +132,7 @@ export function createStateIaContractorsAdapter(): CorpusAdapter {
|
|
|
122
132
|
if (Object.keys(aligned).length <= 2) continue
|
|
123
133
|
|
|
124
134
|
const regNum = (record["Registration #"] ?? "").trim()
|
|
135
|
+
|
|
125
136
|
const sourceID = regNum
|
|
126
137
|
? `${STATE_IA_CONTRACTORS_ADAPTER_ID}-${regNum}`
|
|
127
138
|
: stableSourceID(STATE_IA_CONTRACTORS_ADAPTER_ID, aligned)
|
|
@@ -136,6 +147,7 @@ export function createStateIaContractorsAdapter(): CorpusAdapter {
|
|
|
136
147
|
corpus_version: "",
|
|
137
148
|
license: STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
|
|
138
149
|
}
|
|
150
|
+
|
|
139
151
|
emitted++
|
|
140
152
|
}
|
|
141
153
|
} finally {
|
|
@@ -145,4 +157,7 @@ export function createStateIaContractorsAdapter(): CorpusAdapter {
|
|
|
145
157
|
}
|
|
146
158
|
}
|
|
147
159
|
|
|
160
|
+
/**
|
|
161
|
+
* The configured adapter instance registered with the corpus builder.
|
|
162
|
+
*/
|
|
148
163
|
export const stateIaContractorsAdapter = createStateIaContractorsAdapter()
|
|
@@ -23,7 +23,15 @@ import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
|
|
|
23
23
|
import { reconcileComponents } from "../../format.ts"
|
|
24
24
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
25
25
|
|
|
26
|
+
/**
|
|
27
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
28
|
+
* it came from.
|
|
29
|
+
*/
|
|
26
30
|
export const STATE_NY_NOTARIES_ADAPTER_ID = "state-ny-notaries"
|
|
31
|
+
/**
|
|
32
|
+
* License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
|
|
33
|
+
* than having to look them up.
|
|
34
|
+
*/
|
|
27
35
|
export const STATE_NY_NOTARIES_DEFAULT_LICENSE = "Public Domain"
|
|
28
36
|
|
|
29
37
|
const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
|
|
@@ -75,6 +83,7 @@ export function createStateNyNotariesAdapter(): CorpusAdapter {
|
|
|
75
83
|
}
|
|
76
84
|
|
|
77
85
|
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
86
|
+
|
|
78
87
|
const parser = stream.pipe(
|
|
79
88
|
csvParse({
|
|
80
89
|
columns: true,
|
|
@@ -134,6 +143,7 @@ export function createStateNyNotariesAdapter(): CorpusAdapter {
|
|
|
134
143
|
}
|
|
135
144
|
|
|
136
145
|
const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
|
|
146
|
+
|
|
137
147
|
const raw = [venue, streetPart, [city, [stateAbbr, zip].filter(Boolean).join(" ")].filter(Boolean).join(", ")]
|
|
138
148
|
.filter(Boolean)
|
|
139
149
|
.join(", ")
|
|
@@ -143,6 +153,7 @@ export function createStateNyNotariesAdapter(): CorpusAdapter {
|
|
|
143
153
|
if (Object.keys(aligned).length <= 2) continue
|
|
144
154
|
|
|
145
155
|
const commNum = (record["Commission Number (UID)"] ?? "").trim()
|
|
156
|
+
|
|
146
157
|
const sourceID = commNum
|
|
147
158
|
? `${STATE_NY_NOTARIES_ADAPTER_ID}-${commNum}`
|
|
148
159
|
: stableSourceID(STATE_NY_NOTARIES_ADAPTER_ID, aligned)
|
|
@@ -157,6 +168,7 @@ export function createStateNyNotariesAdapter(): CorpusAdapter {
|
|
|
157
168
|
corpus_version: "",
|
|
158
169
|
license: STATE_NY_NOTARIES_DEFAULT_LICENSE,
|
|
159
170
|
}
|
|
171
|
+
|
|
160
172
|
emitted++
|
|
161
173
|
}
|
|
162
174
|
} finally {
|
|
@@ -166,4 +178,7 @@ export function createStateNyNotariesAdapter(): CorpusAdapter {
|
|
|
166
178
|
}
|
|
167
179
|
}
|
|
168
180
|
|
|
181
|
+
/**
|
|
182
|
+
* The configured adapter instance registered with the corpus builder.
|
|
183
|
+
*/
|
|
169
184
|
export const stateNyNotariesAdapter = createStateNyNotariesAdapter()
|
|
@@ -24,12 +24,22 @@ import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
|
|
|
24
24
|
import { reconcileComponents } from "../../format.ts"
|
|
25
25
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
26
26
|
|
|
27
|
+
/**
|
|
28
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
29
|
+
* it came from.
|
|
30
|
+
*/
|
|
27
31
|
export const STATE_TX_NOTARIES_ADAPTER_ID = "state-tx-notaries"
|
|
32
|
+
/**
|
|
33
|
+
* License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
|
|
34
|
+
* than having to look them up.
|
|
35
|
+
*/
|
|
28
36
|
export const STATE_TX_NOTARIES_DEFAULT_LICENSE = "Public Domain"
|
|
29
37
|
|
|
30
38
|
const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
|
|
31
39
|
|
|
32
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* Match trailing "CITY, ST ZIP" or "CITY, ST" at the end of an address line.
|
|
42
|
+
*/
|
|
33
43
|
const CITY_STATE_ZIP_SUFFIX = /[,]?\s*([^,]+),\s*([A-Z]{2})\s*(\d{5}(?:-\d{4})?)?\s*$/i
|
|
34
44
|
|
|
35
45
|
interface TxNotaryRow {
|
|
@@ -63,6 +73,7 @@ export function createStateTxNotariesAdapter(): CorpusAdapter {
|
|
|
63
73
|
}
|
|
64
74
|
|
|
65
75
|
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
76
|
+
|
|
66
77
|
const parser = stream.pipe(
|
|
67
78
|
csvParse({
|
|
68
79
|
columns: true,
|
|
@@ -90,7 +101,7 @@ export function createStateTxNotariesAdapter(): CorpusAdapter {
|
|
|
90
101
|
|
|
91
102
|
// Parse embedded city/state/zip from the trailing portion of the address.
|
|
92
103
|
// Addresses look like: "1215 MCMILLAN DR\nCEDAR HILL, TX 75104"
|
|
93
|
-
const addrSingleLine = rawAddress.
|
|
104
|
+
const addrSingleLine = rawAddress.replaceAll("\n", ", ")
|
|
94
105
|
const cszMatch = CITY_STATE_ZIP_SUFFIX.exec(addrSingleLine)
|
|
95
106
|
|
|
96
107
|
if (!cszMatch) continue
|
|
@@ -126,6 +137,7 @@ export function createStateTxNotariesAdapter(): CorpusAdapter {
|
|
|
126
137
|
}
|
|
127
138
|
|
|
128
139
|
const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
|
|
140
|
+
|
|
129
141
|
const raw = [venue, streetPart, [city, [stateAbbr, zip].filter(Boolean).join(" ")].filter(Boolean).join(", ")]
|
|
130
142
|
.filter(Boolean)
|
|
131
143
|
.join(", ")
|
|
@@ -148,6 +160,7 @@ export function createStateTxNotariesAdapter(): CorpusAdapter {
|
|
|
148
160
|
corpus_version: "",
|
|
149
161
|
license: STATE_TX_NOTARIES_DEFAULT_LICENSE,
|
|
150
162
|
}
|
|
163
|
+
|
|
151
164
|
emitted++
|
|
152
165
|
}
|
|
153
166
|
} finally {
|
|
@@ -157,4 +170,7 @@ export function createStateTxNotariesAdapter(): CorpusAdapter {
|
|
|
157
170
|
}
|
|
158
171
|
}
|
|
159
172
|
|
|
173
|
+
/**
|
|
174
|
+
* The configured adapter instance registered with the corpus builder.
|
|
175
|
+
*/
|
|
160
176
|
export const stateTxNotariesAdapter = createStateTxNotariesAdapter()
|