@mailwoman/corpus 8.0.0 → 8.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/adapter.d.ts +15 -5
- package/out/src/adapter.d.ts.map +1 -1
- package/out/src/adapter.js +11 -9
- package/out/src/adapter.js.map +1 -1
- package/out/src/adapters/ban/adapter.d.ts +7 -0
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +9 -2
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +5 -1
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +14 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +15 -2
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +11 -0
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +24 -5
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +11 -0
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +14 -1
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +10 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +11 -2
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts +18 -6
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +16 -6
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +11 -0
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +18 -3
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +11 -0
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +12 -1
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +11 -0
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +11 -0
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +15 -2
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +11 -0
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +13 -2
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +14 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +19 -4
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +12 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +17 -2
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +12 -1
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +11 -0
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +15 -2
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +7 -0
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +18 -3
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +13 -3
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +15 -5
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +7 -0
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +9 -2
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/align.d.ts +12 -4
- package/out/src/align.d.ts.map +1 -1
- package/out/src/align.js +9 -5
- package/out/src/align.js.map +1 -1
- package/out/src/build.d.ts +24 -8
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +3 -3
- package/out/src/build.js.map +1 -1
- package/out/src/codex/us-fips-state.d.ts +6 -2
- package/out/src/codex/us-fips-state.d.ts.map +1 -1
- package/out/src/codex/us-fips-state.js +3 -1
- package/out/src/codex/us-fips-state.js.map +1 -1
- package/out/src/golden.d.ts +21 -7
- package/out/src/golden.d.ts.map +1 -1
- package/out/src/golden.js +18 -10
- package/out/src/golden.js.map +1 -1
- package/out/src/license.d.ts +3 -1
- package/out/src/license.d.ts.map +1 -1
- package/out/src/license.js +4 -2
- package/out/src/license.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts +4 -2
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +5 -4
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/schema.d.ts +14 -10
- package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
- package/out/src/parquet-wrapper/schema.js +6 -2
- package/out/src/parquet-wrapper/schema.js.map +1 -1
- package/out/src/parquet-wrapper/writer.d.ts +12 -4
- package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
- package/out/src/parquet-wrapper/writer.js +12 -4
- package/out/src/parquet-wrapper/writer.js.map +1 -1
- package/out/src/parquet.d.ts +21 -7
- package/out/src/parquet.d.ts.map +1 -1
- package/out/src/parquet.js +15 -4
- package/out/src/parquet.js.map +1 -1
- package/out/src/runner.d.ts +39 -13
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +7 -3
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts +4 -0
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +14 -5
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/boundary-stress.d.ts +4 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +13 -7
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts +4 -0
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +39 -11
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts +4 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +14 -3
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts +4 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.js +4 -0
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts +4 -0
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +25 -8
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts +4 -0
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +21 -8
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts +4 -0
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +27 -8
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts +4 -0
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +16 -6
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.d.ts +4 -0
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/house-venue.js +4 -0
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts +9 -3
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +12 -4
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts +4 -0
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +27 -13
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +4 -0
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +9 -4
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/nl-postcode.d.ts +4 -0
- package/out/src/shard-recipes/nl-postcode.d.ts.map +1 -1
- package/out/src/shard-recipes/nl-postcode.js +8 -2
- package/out/src/shard-recipes/nl-postcode.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts +4 -0
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +8 -2
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts +4 -0
- package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street-led.js +5 -1
- package/out/src/shard-recipes/no-street-led.js.map +1 -1
- package/out/src/shard-recipes/no-street.d.ts +4 -0
- package/out/src/shard-recipes/no-street.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street.js +4 -0
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +4 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +99 -43
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts +4 -0
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +4 -0
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +48 -16
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +10 -6
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/si-bare-village.d.ts +4 -0
- package/out/src/shard-recipes/si-bare-village.d.ts.map +1 -1
- package/out/src/shard-recipes/si-bare-village.js +4 -0
- package/out/src/shard-recipes/si-bare-village.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +4 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +29 -10
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.d.ts +4 -0
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
- package/out/src/shard-recipes/street-bare.js +4 -0
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.d.ts +4 -0
- package/out/src/shard-recipes/street.d.ts.map +1 -1
- package/out/src/shard-recipes/street.js +4 -0
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/unit.d.ts +4 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +45 -14
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/split.d.ts +18 -6
- package/out/src/split.d.ts.map +1 -1
- package/out/src/split.js +4 -2
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize-anchor-absorption.d.ts +7 -1
- package/out/src/synthesize-anchor-absorption.d.ts.map +1 -1
- package/out/src/synthesize-anchor-absorption.js +28 -14
- package/out/src/synthesize-anchor-absorption.js.map +1 -1
- package/out/src/synthesize-boundary-stress.d.ts +3 -1
- package/out/src/synthesize-boundary-stress.d.ts.map +1 -1
- package/out/src/synthesize-boundary-stress.js +35 -28
- package/out/src/synthesize-boundary-stress.js.map +1 -1
- package/out/src/synthesize-german.d.ts +15 -5
- package/out/src/synthesize-german.d.ts.map +1 -1
- package/out/src/synthesize-german.js +9 -3
- package/out/src/synthesize-german.js.map +1 -1
- package/out/src/synthesize-house-venue.d.ts.map +1 -1
- package/out/src/synthesize-house-venue.js +15 -15
- package/out/src/synthesize-house-venue.js.map +1 -1
- package/out/src/synthesize-intersection.d.ts +9 -3
- package/out/src/synthesize-intersection.d.ts.map +1 -1
- package/out/src/synthesize-intersection.js +25 -6
- package/out/src/synthesize-intersection.js.map +1 -1
- package/out/src/synthesize-no-street.d.ts +3 -1
- package/out/src/synthesize-no-street.d.ts.map +1 -1
- package/out/src/synthesize-no-street.js +5 -6
- package/out/src/synthesize-no-street.js.map +1 -1
- package/out/src/synthesize-po-box.d.ts +15 -5
- package/out/src/synthesize-po-box.d.ts.map +1 -1
- package/out/src/synthesize-po-box.js +21 -7
- package/out/src/synthesize-po-box.js.map +1 -1
- package/out/src/synthesize-street.d.ts +3 -1
- package/out/src/synthesize-street.d.ts.map +1 -1
- package/out/src/synthesize-street.js +6 -1
- package/out/src/synthesize-street.js.map +1 -1
- package/out/src/synthesize.d.ts +39 -13
- package/out/src/synthesize.d.ts.map +1 -1
- package/out/src/synthesize.js +66 -36
- package/out/src/synthesize.js.map +1 -1
- package/out/src/tokenize.d.ts +15 -5
- package/out/src/tokenize.d.ts.map +1 -1
- package/out/src/tokenize.js.map +1 -1
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +3 -1
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/audit.d.ts.map +1 -1
- package/out/src/tools/audit.js +20 -8
- package/out/src/tools/audit.js.map +1 -1
- package/out/src/tools/corpus-stats.d.ts.map +1 -1
- package/out/src/tools/corpus-stats.js.map +1 -1
- package/out/src/tools/fetch/ban.d.ts.map +1 -1
- package/out/src/tools/fetch/ban.js +17 -10
- package/out/src/tools/fetch/ban.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts +30 -10
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +26 -6
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/hrsa.d.ts.map +1 -1
- package/out/src/tools/fetch/hrsa.js.map +1 -1
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +18 -18
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +3 -1
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +3 -1
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts +21 -7
- package/out/src/tools/fetch/nad.d.ts.map +1 -1
- package/out/src/tools/fetch/nad.js +5 -5
- package/out/src/tools/fetch/nad.js.map +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +9 -15
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts +3 -1
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/src/tools/fetch/openaddresses.js +32 -23
- package/out/src/tools/fetch/openaddresses.js.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +3 -1
- package/out/src/tools/fetch/ppd.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.js +14 -3
- package/out/src/tools/fetch/ppd.js.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.js +13 -6
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
- package/out/src/tools/fetch/state-sources.js +8 -3
- package/out/src/tools/fetch/state-sources.js.map +1 -1
- package/out/src/tools/fetch/tiger-full.d.ts +9 -3
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
- package/out/src/tools/fetch/tiger-full.js +35 -20
- package/out/src/tools/fetch/tiger-full.js.map +1 -1
- package/out/src/tools/gb-title-case.d.ts.map +1 -1
- package/out/src/tools/gb-title-case.js.map +1 -1
- package/out/src/tools/golden-expand.d.ts +24 -8
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +20 -12
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-promote.d.ts +18 -6
- package/out/src/tools/golden-promote.d.ts.map +1 -1
- package/out/src/tools/golden-promote.js +22 -7
- package/out/src/tools/golden-promote.js.map +1 -1
- package/out/src/tools/ingest-csv.d.ts +3 -1
- package/out/src/tools/ingest-csv.d.ts.map +1 -1
- package/out/src/tools/ingest-csv.js +17 -19
- package/out/src/tools/ingest-csv.js.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts +18 -6
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
- package/out/src/tools/jsonl-to-parquet.js +11 -5
- package/out/src/tools/jsonl-to-parquet.js.map +1 -1
- package/out/src/tools/lint-shard-vocab.d.ts +36 -12
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
- package/out/src/tools/lint-shard-vocab.js +25 -11
- package/out/src/tools/lint-shard-vocab.js.map +1 -1
- package/out/src/tools/lint-shard.d.ts +30 -10
- package/out/src/tools/lint-shard.d.ts.map +1 -1
- package/out/src/tools/lint-shard.js +33 -20
- package/out/src/tools/lint-shard.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +6 -4
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/shard-kryptonite.d.ts +6 -2
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +1 -1
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts +9 -3
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +2 -2
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/types.d.ts +51 -17
- package/out/src/types.d.ts.map +1 -1
- package/out/src/wof-json.d.ts +9 -3
- package/out/src/wof-json.d.ts.map +1 -1
- package/out/src/wof-json.js +6 -4
- package/out/src/wof-json.js.map +1 -1
- package/package.json +7 -6
- package/src/adapter.ts +24 -13
- package/src/adapters/ban/adapter.ts +12 -2
- package/src/adapters/ban/street-decompose.ts +7 -1
- package/src/adapters/fcc-bdc/adapter.ts +19 -3
- package/src/adapters/geonames/adapter.ts +28 -5
- package/src/adapters/geonames-postal/adapter.ts +17 -1
- package/src/adapters/gnaf/adapter.ts +15 -2
- package/src/adapters/gnaf/assemble.ts +40 -10
- package/src/adapters/index.ts +13 -0
- package/src/adapters/openaddresses/adapter.ts +22 -3
- package/src/adapters/overture/adapter.ts +18 -2
- package/src/adapters/state-hi-schools/adapter.ts +15 -0
- package/src/adapters/state-ia-contractors/adapter.ts +15 -0
- package/src/adapters/state-ny-notaries/adapter.ts +15 -0
- package/src/adapters/state-tx-notaries/adapter.ts +18 -2
- package/src/adapters/synth-po-box/adapter.ts +19 -2
- package/src/adapters/tiger/adapter.ts +23 -4
- package/src/adapters/tiger/street-decompose.ts +1 -0
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +18 -2
- package/src/adapters/usgov-imls-pls/adapter.ts +15 -0
- package/src/adapters/usgov-irs-bmf/adapter.ts +19 -2
- package/src/adapters/usgov-nad/adapter.ts +14 -1
- package/src/adapters/usgov-nppes/adapter.ts +13 -0
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +21 -3
- package/src/adapters/wof-admin-jp/adapter.ts +20 -3
- package/src/adapters/wof-admin-json/adapter.ts +22 -7
- package/src/adapters/wof-postalcode-json/adapter.ts +10 -2
- package/src/align.ts +21 -8
- package/src/build.ts +48 -11
- package/src/codex/us-fips-state.ts +6 -2
- package/src/golden.ts +28 -13
- package/src/license.ts +4 -2
- package/src/parquet-wrapper/reader.ts +6 -4
- package/src/parquet-wrapper/schema.ts +14 -10
- package/src/parquet-wrapper/writer.ts +12 -4
- package/src/parquet.ts +39 -8
- package/src/runner.ts +49 -15
- package/src/shard-recipes/anchor-absorption.ts +21 -5
- package/src/shard-recipes/boundary-stress.ts +20 -7
- package/src/shard-recipes/country-balanced.ts +78 -13
- package/src/shard-recipes/cz-pcfirst-preposition.ts +9 -0
- package/src/shard-recipes/fr-admin-split.ts +30 -5
- package/src/shard-recipes/fr-bare-street.ts +10 -0
- package/src/shard-recipes/fr-fragment.ts +33 -8
- package/src/shard-recipes/fr-lieudit.ts +35 -9
- package/src/shard-recipes/fr-order.ts +47 -9
- package/src/shard-recipes/german.ts +32 -6
- package/src/shard-recipes/house-venue.ts +7 -0
- package/src/shard-recipes/index.ts +12 -4
- package/src/shard-recipes/intersection.ts +63 -14
- package/src/shard-recipes/locale.ts +37 -4
- package/src/shard-recipes/nl-postcode.ts +12 -2
- package/src/shard-recipes/no-fragment.ts +12 -2
- package/src/shard-recipes/no-street-led.ts +11 -1
- package/src/shard-recipes/no-street.ts +7 -0
- package/src/shard-recipes/po-box-cedex.ts +145 -45
- package/src/shard-recipes/po-box.ts +7 -0
- package/src/shard-recipes/scaffold.ts +52 -20
- package/src/shard-recipes/si-bare-village.ts +9 -0
- package/src/shard-recipes/street-affix.ts +88 -13
- package/src/shard-recipes/street-bare.ts +10 -0
- package/src/shard-recipes/street.ts +7 -0
- package/src/shard-recipes/unit.ts +79 -16
- package/src/split.ts +28 -7
- package/src/synthesize-anchor-absorption.ts +41 -14
- package/src/synthesize-boundary-stress.ts +57 -30
- package/src/synthesize-german.ts +30 -7
- package/src/synthesize-house-venue.ts +22 -16
- package/src/synthesize-intersection.ts +34 -7
- package/src/synthesize-no-street.ts +18 -10
- package/src/synthesize-po-box.ts +42 -10
- package/src/synthesize-street.ts +10 -2
- package/src/synthesize.ts +81 -38
- package/src/tokenize.ts +15 -5
- package/src/tools/align-shard.ts +6 -1
- package/src/tools/audit.ts +57 -11
- package/src/tools/corpus-stats.ts +9 -0
- package/src/tools/fetch/ban.ts +32 -10
- package/src/tools/fetch/download.ts +49 -12
- package/src/tools/fetch/hrsa.ts +3 -0
- package/src/tools/fetch/imls-pls.ts +28 -18
- package/src/tools/fetch/index.ts +3 -1
- package/src/tools/fetch/nad.ts +39 -12
- package/src/tools/fetch/nppes.ts +17 -15
- package/src/tools/fetch/openaddresses.ts +52 -24
- package/src/tools/fetch/ppd.ts +22 -3
- package/src/tools/fetch/state-hi-schools.ts +17 -6
- package/src/tools/fetch/state-sources.ts +16 -3
- package/src/tools/fetch/tiger-full.ts +66 -23
- package/src/tools/gb-title-case.ts +1 -0
- package/src/tools/golden-expand.ts +64 -20
- package/src/tools/golden-promote.ts +53 -13
- package/src/tools/ingest-csv.ts +41 -26
- package/src/tools/jsonl-to-parquet.ts +30 -10
- package/src/tools/lint-shard-vocab.ts +75 -24
- package/src/tools/lint-shard.ts +75 -31
- package/src/tools/overlay-manifest.ts +7 -4
- package/src/tools/shard-kryptonite.ts +11 -3
- package/src/tools/shard-translit.ts +20 -5
- package/src/types.ts +51 -17
- package/src/wof-json.ts +18 -7
package/out/src/types.d.ts
CHANGED
|
@@ -23,7 +23,9 @@ import type { BIOLabel, ComponentTag } from "@mailwoman/core/types";
|
|
|
23
23
|
* pipeline (see `synthesize.ts`).
|
|
24
24
|
*/
|
|
25
25
|
export interface SourceProvenance {
|
|
26
|
-
/**
|
|
26
|
+
/**
|
|
27
|
+
* Adapter id that emitted this row, e.g. `"wof-admin"`, `"ban"`, `"openaddresses"`.
|
|
28
|
+
*/
|
|
27
29
|
source: string;
|
|
28
30
|
/**
|
|
29
31
|
* Stable id within the adapter's source. For SQLite-backed adapters this is the row's primary key; for CSV/GeoJSON, a
|
|
@@ -69,18 +71,26 @@ export interface SynthMarker {
|
|
|
69
71
|
* that can't be sure leave it empty and let the runner default by country.
|
|
70
72
|
*/
|
|
71
73
|
export interface CanonicalRow extends SourceProvenance {
|
|
72
|
-
/**
|
|
74
|
+
/**
|
|
75
|
+
* Address string as it might appear in source data.
|
|
76
|
+
*/
|
|
73
77
|
raw: string;
|
|
74
78
|
/**
|
|
75
79
|
* Component-by-tag ground truth. Surface forms must occur in `raw` (within the alignment edit distance threshold) or
|
|
76
80
|
* the row will land in the quarantine pile.
|
|
77
81
|
*/
|
|
78
82
|
components: Partial<Record<ComponentTag, string>>;
|
|
79
|
-
/**
|
|
83
|
+
/**
|
|
84
|
+
* ISO 3166-1 alpha-2 country code.
|
|
85
|
+
*/
|
|
80
86
|
country: string;
|
|
81
|
-
/**
|
|
87
|
+
/**
|
|
88
|
+
* Optional BCP-47 locale. Defaulted by country if absent.
|
|
89
|
+
*/
|
|
82
90
|
locale?: string;
|
|
83
|
-
/**
|
|
91
|
+
/**
|
|
92
|
+
* Present only on synthetic rows.
|
|
93
|
+
*/
|
|
84
94
|
synth?: SynthMarker;
|
|
85
95
|
}
|
|
86
96
|
/**
|
|
@@ -93,9 +103,13 @@ export interface CanonicalRow extends SourceProvenance {
|
|
|
93
103
|
* direction (punctuation-mute).
|
|
94
104
|
*/
|
|
95
105
|
export interface LabeledRow extends CanonicalRow {
|
|
96
|
-
/**
|
|
106
|
+
/**
|
|
107
|
+
* SentencePiece subword tokens for `raw`.
|
|
108
|
+
*/
|
|
97
109
|
tokens: readonly string[];
|
|
98
|
-
/**
|
|
110
|
+
/**
|
|
111
|
+
* BIO labels, one per token. Same length as `tokens`.
|
|
112
|
+
*/
|
|
99
113
|
labels: readonly BIOLabel[];
|
|
100
114
|
/**
|
|
101
115
|
* Char-offset label spans over `raw` (parallel arrays, per the #519 ruling): `span_starts[i]` is the inclusive start
|
|
@@ -107,9 +121,13 @@ export interface LabeledRow extends CanonicalRow {
|
|
|
107
121
|
* and not-yet-migrated synthesis paths may lack it. Required once v0.5.0 lands and the token path is deleted.
|
|
108
122
|
*/
|
|
109
123
|
span_starts?: readonly number[];
|
|
110
|
-
/**
|
|
124
|
+
/**
|
|
125
|
+
* Exclusive end offsets, parallel to `span_starts`.
|
|
126
|
+
*/
|
|
111
127
|
span_ends?: readonly number[];
|
|
112
|
-
/**
|
|
128
|
+
/**
|
|
129
|
+
* Component tags, parallel to `span_starts`.
|
|
130
|
+
*/
|
|
113
131
|
span_tags?: readonly ComponentTag[];
|
|
114
132
|
}
|
|
115
133
|
/**
|
|
@@ -138,15 +156,25 @@ export interface QuarantinedRow {
|
|
|
138
156
|
* `signal` allows the runner to cancel a long-running scan cleanly.
|
|
139
157
|
*/
|
|
140
158
|
export interface AdapterOptions {
|
|
141
|
-
/**
|
|
159
|
+
/**
|
|
160
|
+
* Path to the adapter's input data (file, directory, or URL — adapter-specific).
|
|
161
|
+
*/
|
|
142
162
|
inputPath: string;
|
|
143
|
-
/**
|
|
163
|
+
/**
|
|
164
|
+
* Optional output directory, available to adapters that maintain side state (rare).
|
|
165
|
+
*/
|
|
144
166
|
outputDir?: string;
|
|
145
|
-
/**
|
|
167
|
+
/**
|
|
168
|
+
* ISO 3166-1 alpha-2 country filter.
|
|
169
|
+
*/
|
|
146
170
|
country?: string;
|
|
147
|
-
/**
|
|
171
|
+
/**
|
|
172
|
+
* Soft row cap. Adapters should stop iterating once this is reached.
|
|
173
|
+
*/
|
|
148
174
|
limit?: number;
|
|
149
|
-
/**
|
|
175
|
+
/**
|
|
176
|
+
* Cancellation hook. Adapters should respect this on every iteration boundary.
|
|
177
|
+
*/
|
|
150
178
|
signal?: AbortSignal;
|
|
151
179
|
}
|
|
152
180
|
/**
|
|
@@ -160,11 +188,17 @@ export interface AdapterOptions {
|
|
|
160
188
|
* (e.g. OpenAddresses, which carries per-source licenses).
|
|
161
189
|
*/
|
|
162
190
|
export interface CorpusAdapter {
|
|
163
|
-
/**
|
|
191
|
+
/**
|
|
192
|
+
* Stable, machine-friendly id used in paths and CLI args. E.g. `"wof-admin"`.
|
|
193
|
+
*/
|
|
164
194
|
readonly id: string;
|
|
165
|
-
/**
|
|
195
|
+
/**
|
|
196
|
+
* Default SPDX-ish license label for rows from this adapter. Per-row overrides allowed.
|
|
197
|
+
*/
|
|
166
198
|
readonly defaultLicense: string;
|
|
167
|
-
/**
|
|
199
|
+
/**
|
|
200
|
+
* One-sentence description shown by `npx mailwoman corpus list`.
|
|
201
|
+
*/
|
|
168
202
|
readonly description: string;
|
|
169
203
|
/**
|
|
170
204
|
* Async iterable of canonical rows.
|
package/out/src/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEnE;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAChC
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEnE;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;;;OAIG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;;OAGG;IACH,cAAc,EAAE,MAAM,CAAA;IAEtB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAA;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC3B;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;;OAGG;IACH,cAAc,EAAE,MAAM,CAAA;CACtB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,YAAa,SAAQ,gBAAgB;IACrD;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;IAEX;;;OAGG;IACH,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;IAEjD;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,KAAK,CAAC,EAAE,WAAW,CAAA;CACnB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC/C;;OAEG;IACH,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IAEzB;;OAEG;IACH,MAAM,EAAE,SAAS,QAAQ,EAAE,CAAA;IAE3B;;;;;;;;OAQG;IACH,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAE/B;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAE7B;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAA;CACnC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC9B,GAAG,EAAE,YAAY,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;CACd;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,cAAc;IAC9B;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAEhB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IAEd;;OAEG;IACH,MAAM,CAAC,EAAE,WAAW,CAAA;CACpB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,aAAa;IAC7B;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IAEnB;;OAEG;IACH,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAE/B;;OAEG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAE5B;;;;;;;;;;;;;;;OAeG;IACH,IAAI,CAAC,IAAI,EAAE,cAAc,GAAG,aAAa,CAAC,YAAY,CAAC,CAAA;CACvD"}
|
package/out/src/wof-json.d.ts
CHANGED
|
@@ -28,7 +28,9 @@
|
|
|
28
28
|
* treated as active"), which is why the previous SpatiaLite adapter's `is_current = 1` filter
|
|
29
29
|
* silently emitted zero rows from the real corpus.
|
|
30
30
|
*/
|
|
31
|
-
/**
|
|
31
|
+
/**
|
|
32
|
+
* A WOF GeoJSON feature, as published by the per-record bundles.
|
|
33
|
+
*/
|
|
32
34
|
export interface WOFFeature {
|
|
33
35
|
type?: string;
|
|
34
36
|
id?: number | string;
|
|
@@ -41,10 +43,14 @@ export interface WOFFeature {
|
|
|
41
43
|
export interface WOFRecord {
|
|
42
44
|
id: number;
|
|
43
45
|
parent_id: number | null;
|
|
44
|
-
/**
|
|
46
|
+
/**
|
|
47
|
+
* Canonical `wof:name` of the record.
|
|
48
|
+
*/
|
|
45
49
|
name: string;
|
|
46
50
|
placetype: string;
|
|
47
|
-
/**
|
|
51
|
+
/**
|
|
52
|
+
* ISO 3166-1 alpha-2 from `wof:country`.
|
|
53
|
+
*/
|
|
48
54
|
country: string;
|
|
49
55
|
/**
|
|
50
56
|
* Localized name variants from `name:*` properties.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wof-json.d.ts","sourceRoot":"","sources":["../../src/wof-json.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAMH
|
|
1
|
+
{"version":3,"file":"wof-json.d.ts","sourceRoot":"","sources":["../../src/wof-json.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAMH;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAA;CAC3C;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACzB,EAAE,EAAE,MAAM,CAAA;IACV,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;;;;;OAMG;IACH,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CACjC;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAKxE;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAkBvF;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAEvD;AA+CD;;;;;;;;;;GAUG;AACH,wBAAuB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,WAAW,CAAA;CAAO,GAAG,aAAa,CAAC,SAAS,CAAC,CAoClH;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,GAAG,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAA;AAEpD,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,aAAa,CAuB9E"}
|
package/out/src/wof-json.js
CHANGED
|
@@ -54,7 +54,7 @@ export function extractNameVariants(props) {
|
|
|
54
54
|
continue;
|
|
55
55
|
const candidate = Array.isArray(value)
|
|
56
56
|
? value.find((v) => typeof v === "string" && v.trim().length > 0)
|
|
57
|
-
: typeof value === "string" && value.trim().length
|
|
57
|
+
: typeof value === "string" && value.trim().length
|
|
58
58
|
? value
|
|
59
59
|
: undefined;
|
|
60
60
|
if (candidate) {
|
|
@@ -70,15 +70,17 @@ export function extractNameVariants(props) {
|
|
|
70
70
|
* existing source_id separator vocabulary and downstream consumers split on `-`.
|
|
71
71
|
*/
|
|
72
72
|
export function normalizeNameKey(rawKey) {
|
|
73
|
-
return rawKey.
|
|
73
|
+
return rawKey.replaceAll(/[:_]/g, "-");
|
|
74
74
|
}
|
|
75
|
-
/**
|
|
75
|
+
/**
|
|
76
|
+
* Result of parsing a single GeoJSON file. `null` means "skip this row" (any reason).
|
|
77
|
+
*/
|
|
76
78
|
function recordFromFeature(feature) {
|
|
77
79
|
if (!feature || feature.type !== "Feature" || !feature.properties)
|
|
78
80
|
return null;
|
|
79
81
|
const props = feature.properties;
|
|
80
82
|
const rawID = typeof feature.id === "number" ? feature.id : props["wof:id"];
|
|
81
|
-
const id = typeof rawID === "number" ? rawID : typeof rawID === "string" ? Number(rawID) : NaN;
|
|
83
|
+
const id = typeof rawID === "number" ? rawID : typeof rawID === "string" ? Number(rawID) : Number.NaN;
|
|
82
84
|
if (!Number.isFinite(id))
|
|
83
85
|
return null;
|
|
84
86
|
const name = props["wof:name"];
|
package/out/src/wof-json.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wof-json.js","sourceRoot":"","sources":["../../src/wof-json.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAE3C,OAAO,QAAQ,MAAM,WAAW,CAAA;
|
|
1
|
+
{"version":3,"file":"wof-json.js","sourceRoot":"","sources":["../../src/wof-json.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAE3C,OAAO,QAAQ,MAAM,WAAW,CAAA;AAqChC;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAA8B;IAC9D,MAAM,GAAG,GAAG,KAAK,CAAC,eAAe,CAAC,CAAA;IAClC,MAAM,CAAC,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAEnF,OAAO,CAAC,KAAK,CAAC,CAAA;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAA8B;IACjE,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAA;IAErC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAClD,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;YAAE,SAAQ;QAEtC,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YACrC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;YAC9E,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM;gBACjD,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,SAAS,CAAA;QAEb,IAAI,SAAS,EAAE,CAAC;YACf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAA;QAC/B,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAc;IAC9C,OAAO,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;AACvC,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,OAAmB;IAC7C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,UAAU;QAAE,OAAO,IAAI,CAAA;IAC9E,MAAM,KAAK,GAAG,OAAO,CAAC,UAAU,CAAA;IAEhC,MAAM,KAAK,GAAG,OAAO,OAAO,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;IAC3E,MAAM,EAAE,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAA;IAErG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAAE,OAAO,IAAI,CAAA;IAErC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAA;IAE9B,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO,IAAI,CAAA;IAEzD,MAAM,SAAS,GAAG,KAAK,CAAC,eAAe,CAAC,CAAA;IAExC,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAA;IAE5D,MAAM,OAAO,GAAG,KAAK,CAAC,aAAa,CAAC,CAAA;IAEpC,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAExD,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IAEzC,MAAM,SAAS,GAAG,KAAK,CAAC,eAAe,CAAC,CAAA;IAExC,MAAM,SAAS,GACd,OAAO,SAAS,KAAK,QAAQ;QAC5B,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,CAAC,IAAI,EAAE;YAClD,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC;YACnB,CAAC,CAAC,IAAI,CAAA;IAET,OAAO;QACN,EAAE;QACF,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAmB,CAAC,CAAC,CAAC,CAAE,SAAoB,CAAC,CAAC,CAAC,IAAI;QAC9E,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;QACjB,SAAS;QACT,OAAO;QACP,YAAY,EAAE,mBAAmB,CAAC,KAAK,CAAC;KACxC,CAAA;AACF,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,YAAY,CAAC,OAAe,EAAE,OAAiC,EAAE;IACvF,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,EAAE;QAChD,GAAG,EAAE,OAAO;QACZ,QAAQ,EAAE,IAAI;QACd,SAAS,EAAE,IAAI;QACf,cAAc,EAAE,IAAI;KACpB,CAAC,CAAA;IAEF,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;YAAE,OAAM;QAChC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;QAE9B,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,SAAQ;QAExC,IAAI,IAAY,CAAA;QAEhB,IAAI,CAAC;YACJ,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;QACxC,CAAC;QAAC,MAAM,CAAC;YACR,SAAQ;QACT,CAAC;QAED,IAAI,MAAkB,CAAA;QAEtB,IAAI,CAAC;YACJ,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAe,CAAA;QACxC,CAAC;QAAC,MAAM,CAAC;YACR,SAAQ;QACT,CAAC;QAED,MAAM,GAAG,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAA;QAErC,IAAI,GAAG,EAAE,CAAC;YACT,MAAM,GAAG,CAAA;QACV,CAAC;IACF,CAAC;AACF,CAAC;AAYD,MAAM,UAAU,kBAAkB,CAAC,IAA4B;IAC9D,MAAM,KAAK,GAAkB,IAAI,GAAG,EAAE,CAAA;IAEtC,KAAK,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAgB,EAAE,CAAA;QAC7B,MAAM,KAAK,GAAG,IAAI,GAAG,CAAS,CAAC,EAAE,CAAC,CAAC,CAAA;QACnC,IAAI,GAAG,GAAkB,GAAG,CAAC,SAAS,CAAA;QAEtC,OAAO,GAAG,KAAK,IAAI,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAE5B,IAAI,CAAC,MAAM;gBAAE,MAAK;YAElB,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAAE,MAAK;YAC/B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAClB,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;YACpB,GAAG,GAAG,MAAM,CAAC,SAAS,CAAA;QACvB,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAA;IACrB,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mailwoman/corpus",
|
|
3
|
-
"version": "8.
|
|
3
|
+
"version": "8.2.0",
|
|
4
4
|
"description": "Mailwoman corpus pipeline: BIO-labeled dataset builder for the neural classifier.",
|
|
5
5
|
"license": "AGPL-3.0-only OR LicenseRef-Commercial",
|
|
6
6
|
"repository": {
|
|
@@ -20,7 +20,8 @@
|
|
|
20
20
|
"!*.test.ts",
|
|
21
21
|
"!*.test.tsx",
|
|
22
22
|
"!**/*.test.ts",
|
|
23
|
-
"!**/*.test.tsx"
|
|
23
|
+
"!**/*.test.tsx",
|
|
24
|
+
"!test-kit/**/*"
|
|
24
25
|
],
|
|
25
26
|
"type": "module",
|
|
26
27
|
"exports": {
|
|
@@ -90,10 +91,10 @@
|
|
|
90
91
|
},
|
|
91
92
|
"dependencies": {
|
|
92
93
|
"@dsnp/parquetjs": "1.8.9",
|
|
93
|
-
"@mailwoman/ban": "8.
|
|
94
|
-
"@mailwoman/codex": "8.
|
|
95
|
-
"@mailwoman/core": "8.
|
|
96
|
-
"@mailwoman/formatter": "8.
|
|
94
|
+
"@mailwoman/ban": "8.2.0",
|
|
95
|
+
"@mailwoman/codex": "8.2.0",
|
|
96
|
+
"@mailwoman/core": "8.2.0",
|
|
97
|
+
"@mailwoman/formatter": "8.2.0",
|
|
97
98
|
"csv-parse": "^7.0.1",
|
|
98
99
|
"fast-glob": "^3.3.3",
|
|
99
100
|
"fastest-levenshtein": "^1.0.16",
|
package/src/adapter.ts
CHANGED
|
@@ -34,16 +34,24 @@ import type { CanonicalRow, CorpusAdapter } from "./types.ts"
|
|
|
34
34
|
* the dependency graph stays traceable.
|
|
35
35
|
*/
|
|
36
36
|
export interface AdapterRegistry {
|
|
37
|
-
/**
|
|
37
|
+
/**
|
|
38
|
+
* Add an adapter. Throws if `adapter.id` is already registered.
|
|
39
|
+
*/
|
|
38
40
|
register(adapter: CorpusAdapter): void
|
|
39
41
|
|
|
40
|
-
/**
|
|
42
|
+
/**
|
|
43
|
+
* Return the adapter for `id`, or `undefined`.
|
|
44
|
+
*/
|
|
41
45
|
get(id: string): CorpusAdapter | undefined
|
|
42
46
|
|
|
43
|
-
/**
|
|
47
|
+
/**
|
|
48
|
+
* All registered adapters, in insertion order.
|
|
49
|
+
*/
|
|
44
50
|
list(): readonly CorpusAdapter[]
|
|
45
51
|
|
|
46
|
-
/**
|
|
52
|
+
/**
|
|
53
|
+
* Convenience: ids only, in insertion order.
|
|
54
|
+
*/
|
|
47
55
|
ids(): readonly string[]
|
|
48
56
|
}
|
|
49
57
|
|
|
@@ -58,6 +66,7 @@ export class InMemoryAdapterRegistry implements AdapterRegistry {
|
|
|
58
66
|
if (this.#byID.has(adapter.id)) {
|
|
59
67
|
throw new Error(`AdapterRegistry: id ${JSON.stringify(adapter.id)} already registered`)
|
|
60
68
|
}
|
|
69
|
+
|
|
61
70
|
this.#byID.set(adapter.id, adapter)
|
|
62
71
|
}
|
|
63
72
|
|
|
@@ -92,9 +101,9 @@ export const defaultAdapterRegistry = new InMemoryAdapterRegistry()
|
|
|
92
101
|
* length.
|
|
93
102
|
*/
|
|
94
103
|
export function stableSourceID(adapterID: string, components: Partial<Record<ComponentTag, string>>): string {
|
|
95
|
-
const sortedKeys = Object.keys(components).
|
|
96
|
-
const payload = sortedKeys.map((k) => `${k}=${components[k] ?? ""}`).join("\
|
|
97
|
-
const digest = createHash("sha256").update(adapterID).update("\
|
|
104
|
+
const sortedKeys = Object.keys(components).toSorted() as ComponentTag[]
|
|
105
|
+
const payload = sortedKeys.map((k) => `${k}=${components[k] ?? ""}`).join("\u001F")
|
|
106
|
+
const digest = createHash("sha256").update(adapterID).update("\u001E").update(payload).digest("hex")
|
|
98
107
|
|
|
99
108
|
return `${adapterID}-${digest.slice(0, 12)}`
|
|
100
109
|
}
|
|
@@ -110,12 +119,12 @@ export function stableSourceID(adapterID: string, components: Partial<Record<Com
|
|
|
110
119
|
* ensuring each augmentation variant survives.
|
|
111
120
|
*/
|
|
112
121
|
export function canonicalDedupKey(row: CanonicalRow): string {
|
|
113
|
-
const sortedKeys = Object.keys(row.components).
|
|
114
|
-
const compPart = sortedKeys.map((k) => `${k}=${row.components[k] ?? ""}`).join("\
|
|
115
|
-
const rawNorm = row.raw.toLowerCase().
|
|
116
|
-
const synthPart = row.synth ? `\
|
|
122
|
+
const sortedKeys = Object.keys(row.components).toSorted() as ComponentTag[]
|
|
123
|
+
const compPart = sortedKeys.map((k) => `${k}=${row.components[k] ?? ""}`).join("\u001F")
|
|
124
|
+
const rawNorm = row.raw.toLowerCase().replaceAll(/\s+/g, " ").trim()
|
|
125
|
+
const synthPart = row.synth ? `\u001E${row.synth.method}` : ""
|
|
117
126
|
|
|
118
|
-
return `${row.country}\
|
|
127
|
+
return `${row.country}\u001E${rawNorm}\u001E${compPart}${synthPart}`
|
|
119
128
|
}
|
|
120
129
|
|
|
121
130
|
/**
|
|
@@ -130,7 +139,9 @@ export interface StreamingHasher {
|
|
|
130
139
|
digest(): string
|
|
131
140
|
}
|
|
132
141
|
|
|
133
|
-
/**
|
|
142
|
+
/**
|
|
143
|
+
* Default `StreamingHasher` (SHA-256, hex).
|
|
144
|
+
*/
|
|
134
145
|
export function streamingSha256(): StreamingHasher {
|
|
135
146
|
const h: Hash = createHash("sha256")
|
|
136
147
|
let finalized = false
|
|
@@ -39,6 +39,10 @@ import { reconcileComponents } from "../../format.ts"
|
|
|
39
39
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
40
40
|
import { decomposeFrStreet } from "./street-decompose.ts"
|
|
41
41
|
|
|
42
|
+
/**
|
|
43
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
44
|
+
* it came from.
|
|
45
|
+
*/
|
|
42
46
|
export const BAN_ADAPTER_ID = "ban"
|
|
43
47
|
|
|
44
48
|
/**
|
|
@@ -83,13 +87,14 @@ function composeRaw(house: string, street: string, postcode: string, locality: s
|
|
|
83
87
|
if (streetPart) {
|
|
84
88
|
parts.push(streetPart)
|
|
85
89
|
}
|
|
90
|
+
|
|
86
91
|
const cityPart = [postcode, locality].filter(Boolean).join(" ").trim()
|
|
87
92
|
|
|
88
93
|
if (cityPart) {
|
|
89
94
|
parts.push(cityPart)
|
|
90
95
|
}
|
|
91
96
|
|
|
92
|
-
return parts.join(", ").
|
|
97
|
+
return parts.join(", ").replaceAll(/\s+/g, " ").trim()
|
|
93
98
|
}
|
|
94
99
|
|
|
95
100
|
export function createBanAdapter(): CorpusAdapter {
|
|
@@ -104,6 +109,7 @@ export function createBanAdapter(): CorpusAdapter {
|
|
|
104
109
|
}
|
|
105
110
|
|
|
106
111
|
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
112
|
+
|
|
107
113
|
const parser = stream.pipe(
|
|
108
114
|
csvParse({
|
|
109
115
|
delimiter: ";",
|
|
@@ -161,7 +167,7 @@ export function createBanAdapter(): CorpusAdapter {
|
|
|
161
167
|
|
|
162
168
|
const aligned = reconcileComponents(components, raw)
|
|
163
169
|
|
|
164
|
-
if (Object.keys(aligned).length
|
|
170
|
+
if (!Object.keys(aligned).length) continue
|
|
165
171
|
|
|
166
172
|
const sourceID = record.id?.trim()
|
|
167
173
|
? `${BAN_ADAPTER_ID}-${record.id.trim()}`
|
|
@@ -177,6 +183,7 @@ export function createBanAdapter(): CorpusAdapter {
|
|
|
177
183
|
corpus_version: "",
|
|
178
184
|
license: "Licence Ouverte 2.0",
|
|
179
185
|
}
|
|
186
|
+
|
|
180
187
|
emitted++
|
|
181
188
|
}
|
|
182
189
|
} finally {
|
|
@@ -186,4 +193,7 @@ export function createBanAdapter(): CorpusAdapter {
|
|
|
186
193
|
}
|
|
187
194
|
}
|
|
188
195
|
|
|
196
|
+
/**
|
|
197
|
+
* The configured adapter instance registered with the corpus builder.
|
|
198
|
+
*/
|
|
189
199
|
export const banAdapter = createBanAdapter()
|
|
@@ -20,6 +20,11 @@ import { resolve } from "node:path"
|
|
|
20
20
|
|
|
21
21
|
import { resourceDictionaryPath } from "@mailwoman/core/utils"
|
|
22
22
|
|
|
23
|
+
/**
|
|
24
|
+
* Tokens a BAN street needs before a type/article/name decomposition is attempted.
|
|
25
|
+
*/
|
|
26
|
+
const MIN_TOKENS_FOR_DECOMPOSE = 3
|
|
27
|
+
|
|
23
28
|
function loadDictionary(filename: string): Set<string> {
|
|
24
29
|
const candidates = [
|
|
25
30
|
resourceDictionaryPath("libpostal", "fr", filename),
|
|
@@ -51,6 +56,7 @@ function loadDictionary(filename: string): Set<string> {
|
|
|
51
56
|
// try next
|
|
52
57
|
}
|
|
53
58
|
}
|
|
59
|
+
|
|
54
60
|
throw new Error(`Could not load FR libpostal dictionary: ${filename}`)
|
|
55
61
|
}
|
|
56
62
|
|
|
@@ -79,7 +85,7 @@ export function decomposeFrStreet(fullname: string): DecomposedFrStreet {
|
|
|
79
85
|
const norm = (s: string) => s.toLowerCase().replace(/[.,;]$/, "")
|
|
80
86
|
|
|
81
87
|
// Try 2-word prefix first (e.g. "ancien chemin")
|
|
82
|
-
if (tokens.length >=
|
|
88
|
+
if (tokens.length >= MIN_TOKENS_FOR_DECOMPOSE) {
|
|
83
89
|
const twoWord = norm(tokens[0]!) + " " + norm(tokens[1]!)
|
|
84
90
|
|
|
85
91
|
if (STREET_TYPES_FR.has(twoWord)) {
|
|
@@ -40,10 +40,20 @@ import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
|
|
|
40
40
|
import { formatAddress, reconcileComponents } from "../../format.ts"
|
|
41
41
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
42
42
|
|
|
43
|
+
/**
|
|
44
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
45
|
+
* it came from.
|
|
46
|
+
*/
|
|
43
47
|
export const FCC_BDC_ADAPTER_ID = "fcc-bdc"
|
|
48
|
+
/**
|
|
49
|
+
* License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
|
|
50
|
+
* than having to look them up.
|
|
51
|
+
*/
|
|
44
52
|
export const FCC_BDC_DEFAULT_LICENSE = "Public Domain"
|
|
45
53
|
|
|
46
|
-
/**
|
|
54
|
+
/**
|
|
55
|
+
* SQLite row shape — one row per BSL `location_id`. Columns mirror NTIARecord.
|
|
56
|
+
*/
|
|
47
57
|
interface BdcLocationRow {
|
|
48
58
|
location_id: number
|
|
49
59
|
address_primary: string
|
|
@@ -110,7 +120,9 @@ export function buildPostcode(zip: string, suffix: string | null): string {
|
|
|
110
120
|
return `${z}-${s}`
|
|
111
121
|
}
|
|
112
122
|
|
|
113
|
-
/**
|
|
123
|
+
/**
|
|
124
|
+
* Build a BDC adapter. Pure factory so multiple instances can be created in tests.
|
|
125
|
+
*/
|
|
114
126
|
export function createFccBdcAdapter(): CorpusAdapter {
|
|
115
127
|
return {
|
|
116
128
|
id: FCC_BDC_ADAPTER_ID,
|
|
@@ -164,7 +176,7 @@ export function createFccBdcAdapter(): CorpusAdapter {
|
|
|
164
176
|
if (!raw) continue
|
|
165
177
|
const aligned = reconcileComponents(components, raw)
|
|
166
178
|
|
|
167
|
-
if (Object.keys(aligned).length
|
|
179
|
+
if (!Object.keys(aligned).length) continue
|
|
168
180
|
|
|
169
181
|
yield {
|
|
170
182
|
raw,
|
|
@@ -176,6 +188,7 @@ export function createFccBdcAdapter(): CorpusAdapter {
|
|
|
176
188
|
corpus_version: "",
|
|
177
189
|
license: FCC_BDC_DEFAULT_LICENSE,
|
|
178
190
|
}
|
|
191
|
+
|
|
179
192
|
emitted++
|
|
180
193
|
}
|
|
181
194
|
} finally {
|
|
@@ -185,4 +198,7 @@ export function createFccBdcAdapter(): CorpusAdapter {
|
|
|
185
198
|
}
|
|
186
199
|
}
|
|
187
200
|
|
|
201
|
+
/**
|
|
202
|
+
* The configured adapter instance registered with the corpus builder.
|
|
203
|
+
*/
|
|
188
204
|
export const fccBdcAdapter = createFccBdcAdapter()
|
|
@@ -37,10 +37,20 @@ import { stableSourceID } from "../../adapter.ts"
|
|
|
37
37
|
import { reconcileComponents } from "../../format.ts"
|
|
38
38
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
39
39
|
|
|
40
|
+
/**
|
|
41
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
42
|
+
* it came from.
|
|
43
|
+
*/
|
|
40
44
|
export const GEONAMES_ADAPTER_ID = "geonames"
|
|
45
|
+
/**
|
|
46
|
+
* License carried by this source (CC-BY-4.0), attached to each row so downstream consumers inherit the terms rather
|
|
47
|
+
* than having to look them up.
|
|
48
|
+
*/
|
|
41
49
|
export const GEONAMES_DEFAULT_LICENSE = "CC-BY-4.0"
|
|
42
50
|
|
|
43
|
-
|
|
51
|
+
/**
|
|
52
|
+
* GeoNames main-table column indices (0-based; see the export README).
|
|
53
|
+
*/
|
|
44
54
|
const COL = {
|
|
45
55
|
geonameid: 0,
|
|
46
56
|
name: 1,
|
|
@@ -51,10 +61,14 @@ const COL = {
|
|
|
51
61
|
admin1: 10,
|
|
52
62
|
} as const
|
|
53
63
|
|
|
54
|
-
|
|
64
|
+
/**
|
|
65
|
+
* Populated-place feature codes that are NOT current real places — skip them.
|
|
66
|
+
*/
|
|
55
67
|
const NON_CURRENT_PPL = new Set(["PPLH", "PPLQ", "PPLW", "PPLCH"])
|
|
56
68
|
|
|
57
|
-
/**
|
|
69
|
+
/**
|
|
70
|
+
* Load `admin1CodesASCII.txt` → Map("<CC>.<admin1>" → region name). Empty map if absent.
|
|
71
|
+
*/
|
|
58
72
|
function loadAdmin1(dir: string): Map<string, string> {
|
|
59
73
|
const map = new Map<string, string>()
|
|
60
74
|
const fp = join(dir, "admin1CodesASCII.txt")
|
|
@@ -73,7 +87,9 @@ function loadAdmin1(dir: string): Map<string, string> {
|
|
|
73
87
|
return map
|
|
74
88
|
}
|
|
75
89
|
|
|
76
|
-
/**
|
|
90
|
+
/**
|
|
91
|
+
* Load `countryInfo.txt` → Map(ISO → country name). Empty map if absent. The file is `#`-commented.
|
|
92
|
+
*/
|
|
77
93
|
function loadCountries(dir: string): Map<string, string> {
|
|
78
94
|
const map = new Map<string, string>()
|
|
79
95
|
const fp = join(dir, "countryInfo.txt")
|
|
@@ -106,6 +122,7 @@ export function createGeonamesAdapter(): CorpusAdapter {
|
|
|
106
122
|
const countries = loadCountries(dir)
|
|
107
123
|
|
|
108
124
|
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
125
|
+
|
|
109
126
|
const parser = stream.pipe(
|
|
110
127
|
csvParse({ delimiter: "\t", quote: false, relax_column_count: true, skip_empty_lines: true })
|
|
111
128
|
)
|
|
@@ -159,10 +176,12 @@ export function createGeonamesAdapter(): CorpusAdapter {
|
|
|
159
176
|
if (opts.limit !== undefined && emitted >= opts.limit) break
|
|
160
177
|
const aligned = reconcileComponents(v.comp, v.raw)
|
|
161
178
|
|
|
162
|
-
if (Object.keys(aligned).length
|
|
179
|
+
if (!Object.keys(aligned).length) continue
|
|
180
|
+
|
|
163
181
|
const sourceID = geonameid
|
|
164
182
|
? `${GEONAMES_ADAPTER_ID}-${geonameid}-${v.slot}`
|
|
165
183
|
: stableSourceID(GEONAMES_ADAPTER_ID, aligned)
|
|
184
|
+
|
|
166
185
|
yield {
|
|
167
186
|
raw: v.raw,
|
|
168
187
|
components: aligned,
|
|
@@ -172,6 +191,7 @@ export function createGeonamesAdapter(): CorpusAdapter {
|
|
|
172
191
|
corpus_version: "",
|
|
173
192
|
license: GEONAMES_DEFAULT_LICENSE,
|
|
174
193
|
}
|
|
194
|
+
|
|
175
195
|
emitted++
|
|
176
196
|
}
|
|
177
197
|
}
|
|
@@ -182,4 +202,7 @@ export function createGeonamesAdapter(): CorpusAdapter {
|
|
|
182
202
|
}
|
|
183
203
|
}
|
|
184
204
|
|
|
205
|
+
/**
|
|
206
|
+
* The configured adapter instance registered with the corpus builder.
|
|
207
|
+
*/
|
|
185
208
|
export const geonamesAdapter = createGeonamesAdapter()
|
|
@@ -31,10 +31,20 @@ import { stableSourceID } from "../../adapter.ts"
|
|
|
31
31
|
import { reconcileComponents } from "../../format.ts"
|
|
32
32
|
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
33
33
|
|
|
34
|
+
/**
|
|
35
|
+
* Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
|
|
36
|
+
* it came from.
|
|
37
|
+
*/
|
|
34
38
|
export const GEONAMES_POSTAL_ADAPTER_ID = "geonames-postal"
|
|
39
|
+
/**
|
|
40
|
+
* License carried by this source (CC-BY-4.0), attached to each row so downstream consumers inherit the terms rather
|
|
41
|
+
* than having to look them up.
|
|
42
|
+
*/
|
|
35
43
|
export const GEONAMES_POSTAL_DEFAULT_LICENSE = "CC-BY-4.0"
|
|
36
44
|
|
|
37
|
-
|
|
45
|
+
/**
|
|
46
|
+
* GeoNames postal-dump columns (0-based).
|
|
47
|
+
*/
|
|
38
48
|
const COL = { country: 0, postcode: 1, place: 2, admin1Name: 3 } as const
|
|
39
49
|
|
|
40
50
|
export function createGeonamesPostalAdapter(): CorpusAdapter {
|
|
@@ -46,6 +56,7 @@ export function createGeonamesPostalAdapter(): CorpusAdapter {
|
|
|
46
56
|
|
|
47
57
|
async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
|
|
48
58
|
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
59
|
+
|
|
49
60
|
const parser = stream.pipe(
|
|
50
61
|
csvParse({ delimiter: "\t", quote: false, relax_column_count: true, skip_empty_lines: true })
|
|
51
62
|
)
|
|
@@ -89,6 +100,7 @@ export function createGeonamesPostalAdapter(): CorpusAdapter {
|
|
|
89
100
|
const aligned = reconcileComponents(v.comp, v.raw)
|
|
90
101
|
|
|
91
102
|
if (Object.keys(aligned).length < 2) continue
|
|
103
|
+
|
|
92
104
|
yield {
|
|
93
105
|
raw: v.raw,
|
|
94
106
|
components: aligned,
|
|
@@ -98,6 +110,7 @@ export function createGeonamesPostalAdapter(): CorpusAdapter {
|
|
|
98
110
|
corpus_version: "",
|
|
99
111
|
license: GEONAMES_POSTAL_DEFAULT_LICENSE,
|
|
100
112
|
}
|
|
113
|
+
|
|
101
114
|
emitted++
|
|
102
115
|
}
|
|
103
116
|
}
|
|
@@ -108,4 +121,7 @@ export function createGeonamesPostalAdapter(): CorpusAdapter {
|
|
|
108
121
|
}
|
|
109
122
|
}
|
|
110
123
|
|
|
124
|
+
/**
|
|
125
|
+
* The configured adapter instance registered with the corpus builder.
|
|
126
|
+
*/
|
|
111
127
|
export const geonamesPostalAdapter = createGeonamesPostalAdapter()
|