@mailwoman/corpus 8.6.0 → 9.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/data/PROVENANCE.md +214 -0
- package/data/sub-venue-lexicon.json +18269 -0
- package/out/src/adapters/ban/adapter.d.ts +5 -4
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +62 -69
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +18 -25
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +1 -7
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +4 -26
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +1 -1
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +72 -78
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +46 -51
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +6 -9
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +7 -12
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +5 -10
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +1 -1
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +5 -9
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +57 -75
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +60 -78
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +69 -87
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +71 -89
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +3 -3
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +10 -18
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +1 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +2 -2
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js +18 -27
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +4 -4
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +64 -82
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +62 -84
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +59 -66
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +6 -9
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +60 -78
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +53 -73
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/{adapter.d.ts → adapters/utils/index.d.ts} +43 -2
- package/out/src/adapters/utils/index.d.ts.map +1 -0
- package/out/src/{adapter.js → adapters/utils/index.js} +47 -2
- package/out/src/adapters/utils/index.js.map +1 -0
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +24 -6
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +2 -2
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +2 -2
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +2 -2
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +2 -2
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/build.d.ts +3 -4
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +6 -8
- package/out/src/build.js.map +1 -1
- package/out/src/index.d.ts +3 -17
- package/out/src/index.d.ts.map +1 -1
- package/out/src/index.js +3 -17
- package/out/src/index.js.map +1 -1
- package/out/src/name-prone-us-suffixes.d.ts +12 -0
- package/out/src/name-prone-us-suffixes.d.ts.map +1 -0
- package/out/src/name-prone-us-suffixes.js +12 -0
- package/out/src/name-prone-us-suffixes.js.map +1 -0
- package/out/src/parquet-wrapper/reader.d.ts +8 -0
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +16 -0
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/schema.d.ts +4 -0
- package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
- package/out/src/parquet-wrapper/schema.js.map +1 -1
- package/out/src/runner.d.ts +2 -2
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +1 -1
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +5 -4
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +2 -2
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +75 -75
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +2 -2
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +8 -5
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +2 -2
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +18 -72
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +17 -70
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/house-venue.js +1 -1
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +4 -1
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts +2 -2
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +28 -72
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +5 -5
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +25 -35
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +8 -5
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street-led.js +8 -5
- package/out/src/shard-recipes/no-street-led.js.map +1 -1
- package/out/src/shard-recipes/no-street.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street.js +1 -1
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +3 -3
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +101 -127
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +1 -1
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +58 -9
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +87 -29
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +54 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +253 -111
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
- package/out/src/shard-recipes/street-bare.js +3 -3
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.d.ts.map +1 -1
- package/out/src/shard-recipes/street.js +2 -2
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.d.ts +228 -0
- package/out/src/shard-recipes/sub-venue-sources.d.ts.map +1 -0
- package/out/src/shard-recipes/sub-venue-sources.js +664 -0
- package/out/src/shard-recipes/sub-venue-sources.js.map +1 -0
- package/out/src/shard-recipes/sub-venue.d.ts +269 -0
- package/out/src/shard-recipes/sub-venue.d.ts.map +1 -0
- package/out/src/shard-recipes/sub-venue.js +709 -0
- package/out/src/shard-recipes/sub-venue.js.map +1 -0
- package/out/src/shard-recipes/unit.d.ts +1 -1
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +26 -74
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/{synthesize-anchor-absorption.d.ts → synthesizers/anchor-absorption.d.ts} +1 -1
- package/out/src/synthesizers/anchor-absorption.d.ts.map +1 -0
- package/out/src/{synthesize-anchor-absorption.js → synthesizers/anchor-absorption.js} +1 -1
- package/out/src/synthesizers/anchor-absorption.js.map +1 -0
- package/out/src/{synthesize-boundary-stress.d.ts → synthesizers/boundary-stress.d.ts} +3 -3
- package/out/src/synthesizers/boundary-stress.d.ts.map +1 -0
- package/out/src/{synthesize-boundary-stress.js → synthesizers/boundary-stress.js} +2 -2
- package/out/src/synthesizers/boundary-stress.js.map +1 -0
- package/out/src/{synthesize-german.d.ts → synthesizers/german.d.ts} +2 -2
- package/out/src/synthesizers/german.d.ts.map +1 -0
- package/out/src/{synthesize-german.js → synthesizers/german.js} +2 -2
- package/out/src/synthesizers/german.js.map +1 -0
- package/out/src/{synthesize-house-venue.d.ts → synthesizers/house-venue.d.ts} +4 -4
- package/out/src/synthesizers/house-venue.d.ts.map +1 -0
- package/out/src/{synthesize-house-venue.js → synthesizers/house-venue.js} +4 -4
- package/out/src/synthesizers/house-venue.js.map +1 -0
- package/out/src/{synthesize-intersection.d.ts → synthesizers/intersection.d.ts} +2 -2
- package/out/src/synthesizers/intersection.d.ts.map +1 -0
- package/out/src/{synthesize-intersection.js → synthesizers/intersection.js} +1 -1
- package/out/src/synthesizers/intersection.js.map +1 -0
- package/out/src/{synthesize-no-street.d.ts → synthesizers/no-street.d.ts} +3 -3
- package/out/src/synthesizers/no-street.d.ts.map +1 -0
- package/out/src/{synthesize-no-street.js → synthesizers/no-street.js} +2 -2
- package/out/src/synthesizers/no-street.js.map +1 -0
- package/out/src/{synthesize-po-box.d.ts → synthesizers/po-box.d.ts} +2 -2
- package/out/src/synthesizers/po-box.d.ts.map +1 -0
- package/out/src/{synthesize-po-box.js → synthesizers/po-box.js} +1 -1
- package/out/src/synthesizers/po-box.js.map +1 -0
- package/out/src/{synthesize-street.d.ts → synthesizers/street.d.ts} +2 -2
- package/out/src/synthesizers/street.d.ts.map +1 -0
- package/out/src/{synthesize-street.js → synthesizers/street.js} +2 -2
- package/out/src/synthesizers/street.js.map +1 -0
- package/out/src/{synthesize.d.ts → synthesizers/utils.d.ts} +3 -3
- package/out/src/synthesizers/utils.d.ts.map +1 -0
- package/out/src/{synthesize.js → synthesizers/utils.js} +4 -15
- package/out/src/synthesizers/utils.js.map +1 -0
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +4 -8
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/audit.d.ts.map +1 -1
- package/out/src/tools/audit.js +4 -4
- package/out/src/tools/audit.js.map +1 -1
- package/out/src/tools/corpus-stats.d.ts +2 -2
- package/out/src/tools/corpus-stats.d.ts.map +1 -1
- package/out/src/tools/corpus-stats.js +18 -31
- package/out/src/tools/corpus-stats.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +5 -6
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +3 -15
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +14 -1
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +14 -1
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts +1 -1
- package/out/src/tools/fetch/nad.js +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +8 -14
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts +1 -1
- package/out/src/tools/fetch/openaddresses.js +1 -1
- package/out/src/tools/fetch/ourairports.d.ts +45 -0
- package/out/src/tools/fetch/ourairports.d.ts.map +1 -0
- package/out/src/tools/fetch/ourairports.js +123 -0
- package/out/src/tools/fetch/ourairports.js.map +1 -0
- package/out/src/tools/fetch/wikidata-subvenue.d.ts +171 -0
- package/out/src/tools/fetch/wikidata-subvenue.d.ts.map +1 -0
- package/out/src/tools/fetch/wikidata-subvenue.js +275 -0
- package/out/src/tools/fetch/wikidata-subvenue.js.map +1 -0
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +18 -16
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-promote.d.ts.map +1 -1
- package/out/src/tools/golden-promote.js +12 -6
- package/out/src/tools/golden-promote.js.map +1 -1
- package/out/src/tools/golden-relabel-street.d.ts +196 -0
- package/out/src/tools/golden-relabel-street.d.ts.map +1 -0
- package/out/src/tools/golden-relabel-street.js +440 -0
- package/out/src/tools/golden-relabel-street.js.map +1 -0
- package/out/src/tools/index.d.ts +4 -0
- package/out/src/tools/index.d.ts.map +1 -1
- package/out/src/tools/index.js +4 -0
- package/out/src/tools/index.js.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
- package/out/src/tools/jsonl-to-parquet.js +3 -2
- package/out/src/tools/jsonl-to-parquet.js.map +1 -1
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
- package/out/src/tools/lint-shard-vocab.js +3 -2
- package/out/src/tools/lint-shard-vocab.js.map +1 -1
- package/out/src/tools/lint-shard.d.ts +3 -3
- package/out/src/tools/lint-shard.d.ts.map +1 -1
- package/out/src/tools/lint-shard.js +23 -32
- package/out/src/tools/lint-shard.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +2 -1
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/overture-subvenue.d.ts +112 -0
- package/out/src/tools/overture-subvenue.d.ts.map +1 -0
- package/out/src/tools/overture-subvenue.js +144 -0
- package/out/src/tools/overture-subvenue.js.map +1 -0
- package/out/src/tools/shard-kryptonite.d.ts +1 -1
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +6 -6
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts +6 -3
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +9 -8
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/tools/sub-venue-lexicon.d.ts +507 -0
- package/out/src/tools/sub-venue-lexicon.d.ts.map +1 -0
- package/out/src/tools/sub-venue-lexicon.js +817 -0
- package/out/src/tools/sub-venue-lexicon.js.map +1 -0
- package/out/src/tools/sub-venue-promotions.d.ts +94 -0
- package/out/src/tools/sub-venue-promotions.d.ts.map +1 -0
- package/out/src/tools/sub-venue-promotions.js +266 -0
- package/out/src/tools/sub-venue-promotions.js.map +1 -0
- package/out/src/{align.d.ts → utils/align.d.ts} +1 -1
- package/out/src/utils/align.d.ts.map +1 -0
- package/out/src/utils/align.js.map +1 -0
- package/out/src/{golden.d.ts → utils/golden.d.ts} +5 -0
- package/out/src/utils/golden.d.ts.map +1 -0
- package/out/src/{golden.js → utils/golden.js} +24 -10
- package/out/src/utils/golden.js.map +1 -0
- package/out/src/utils/index.d.ts +14 -0
- package/out/src/utils/index.d.ts.map +1 -0
- package/out/src/utils/index.js +14 -0
- package/out/src/utils/index.js.map +1 -0
- package/out/src/utils/license.d.ts.map +1 -0
- package/out/src/utils/license.js.map +1 -0
- package/out/src/{parquet.d.ts → utils/parquet.d.ts} +2 -2
- package/out/src/utils/parquet.d.ts.map +1 -0
- package/out/src/{parquet.js → utils/parquet.js} +3 -15
- package/out/src/utils/parquet.js.map +1 -0
- package/out/src/{split.d.ts → utils/split.d.ts} +8 -1
- package/out/src/utils/split.d.ts.map +1 -0
- package/out/src/{split.js → utils/split.js} +7 -0
- package/out/src/utils/split.js.map +1 -0
- package/out/src/utils/tokenize.d.ts.map +1 -0
- package/out/src/utils/tokenize.js.map +1 -0
- package/out/src/{wof-json.d.ts → utils/wof-json.d.ts} +2 -2
- package/out/src/utils/wof-json.d.ts.map +1 -0
- package/out/src/{wof-json.js → utils/wof-json.js} +5 -8
- package/out/src/utils/wof-json.js.map +1 -0
- package/package.json +66 -20
- package/src/adapters/ban/adapter.ts +58 -68
- package/src/adapters/ban/street-decompose.ts +17 -25
- package/src/adapters/fcc-bdc/adapter.ts +5 -36
- package/src/adapters/geonames/adapter.ts +67 -76
- package/src/adapters/geonames-postal/adapter.ts +45 -54
- package/src/adapters/gnaf/adapter.ts +7 -11
- package/src/adapters/gnaf/assemble.ts +7 -11
- package/src/adapters/index.ts +3 -2
- package/src/adapters/openaddresses/adapter.ts +6 -12
- package/src/adapters/overture/adapter.ts +6 -10
- package/src/adapters/state-hi-schools/adapter.ts +53 -77
- package/src/adapters/state-ia-contractors/adapter.ts +55 -80
- package/src/adapters/state-ny-notaries/adapter.ts +64 -89
- package/src/adapters/state-tx-notaries/adapter.ts +63 -88
- package/src/adapters/synth-po-box/adapter.ts +12 -23
- package/src/adapters/tiger/adapter.ts +4 -3
- package/src/adapters/tiger/street-decompose.ts +21 -31
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +64 -88
- package/src/adapters/usgov-imls-pls/adapter.ts +57 -86
- package/src/adapters/usgov-irs-bmf/adapter.ts +55 -66
- package/src/adapters/usgov-nad/adapter.ts +7 -11
- package/src/adapters/usgov-nppes/adapter.ts +54 -79
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +48 -75
- package/src/{adapter.ts → adapters/utils/index.ts} +59 -3
- package/src/adapters/wof-admin-jp/adapter.ts +35 -9
- package/src/adapters/wof-admin-json/adapter.ts +3 -4
- package/src/adapters/wof-postalcode-json/adapter.ts +3 -4
- package/src/build.ts +12 -11
- package/src/index.ts +3 -17
- package/src/name-prone-us-suffixes.ts +12 -0
- package/src/parquet-wrapper/reader.ts +19 -0
- package/src/parquet-wrapper/schema.ts +4 -0
- package/src/runner.ts +7 -2
- package/src/shard-recipes/anchor-absorption.ts +5 -4
- package/src/shard-recipes/boundary-stress.ts +6 -2
- package/src/shard-recipes/country-balanced.ts +77 -91
- package/src/shard-recipes/fr-admin-split.ts +3 -3
- package/src/shard-recipes/fr-fragment.ts +10 -7
- package/src/shard-recipes/fr-lieudit.ts +3 -3
- package/src/shard-recipes/fr-order.ts +18 -83
- package/src/shard-recipes/german.ts +17 -80
- package/src/shard-recipes/house-venue.ts +2 -1
- package/src/shard-recipes/index.ts +4 -1
- package/src/shard-recipes/intersection.ts +37 -74
- package/src/shard-recipes/locale.ts +31 -39
- package/src/shard-recipes/no-fragment.ts +10 -7
- package/src/shard-recipes/no-street-led.ts +10 -7
- package/src/shard-recipes/no-street.ts +2 -1
- package/src/shard-recipes/po-box-cedex.ts +127 -161
- package/src/shard-recipes/po-box.ts +6 -1
- package/src/shard-recipes/scaffold.ts +125 -27
- package/src/shard-recipes/street-affix.ts +335 -134
- package/src/shard-recipes/street-bare.ts +4 -3
- package/src/shard-recipes/street.ts +3 -2
- package/src/shard-recipes/sub-venue-sources.ts +895 -0
- package/src/shard-recipes/sub-venue.ts +982 -0
- package/src/shard-recipes/unit.ts +27 -85
- package/src/{synthesize-boundary-stress.ts → synthesizers/boundary-stress.ts} +2 -2
- package/src/{synthesize-german.ts → synthesizers/german.ts} +2 -2
- package/src/{synthesize-house-venue.ts → synthesizers/house-venue.ts} +4 -4
- package/src/{synthesize-intersection.ts → synthesizers/intersection.ts} +1 -1
- package/src/{synthesize-no-street.ts → synthesizers/no-street.ts} +2 -2
- package/src/{synthesize-po-box.ts → synthesizers/po-box.ts} +1 -1
- package/src/{synthesize-street.ts → synthesizers/street.ts} +2 -2
- package/src/{synthesize.ts → synthesizers/utils.ts} +4 -18
- package/src/tools/align-shard.ts +4 -7
- package/src/tools/audit.ts +6 -5
- package/src/tools/corpus-stats.ts +25 -33
- package/src/tools/fetch/download.ts +7 -5
- package/src/tools/fetch/imls-pls.ts +4 -18
- package/src/tools/fetch/index.ts +14 -1
- package/src/tools/fetch/nad.ts +1 -1
- package/src/tools/fetch/nppes.ts +9 -16
- package/src/tools/fetch/openaddresses.ts +1 -1
- package/src/tools/fetch/ourairports.ts +166 -0
- package/src/tools/fetch/wikidata-subvenue.ts +386 -0
- package/src/tools/golden-expand.ts +18 -13
- package/src/tools/golden-promote.ts +15 -6
- package/src/tools/golden-relabel-street.ts +685 -0
- package/src/tools/index.ts +4 -0
- package/src/tools/jsonl-to-parquet.ts +3 -2
- package/src/tools/lint-shard-vocab.ts +3 -2
- package/src/tools/lint-shard.ts +33 -36
- package/src/tools/overlay-manifest.ts +2 -1
- package/src/tools/overture-subvenue.ts +215 -0
- package/src/tools/shard-kryptonite.ts +8 -9
- package/src/tools/shard-translit.ts +22 -11
- package/src/tools/sub-venue-lexicon.ts +1250 -0
- package/src/tools/sub-venue-promotions.ts +330 -0
- package/src/{align.ts → utils/align.ts} +1 -1
- package/src/{golden.ts → utils/golden.ts} +25 -11
- package/src/utils/index.ts +14 -0
- package/src/{parquet.ts → utils/parquet.ts} +5 -19
- package/src/{split.ts → utils/split.ts} +8 -2
- package/src/{wof-json.ts → utils/wof-json.ts} +5 -8
- package/out/src/adapter.d.ts.map +0 -1
- package/out/src/adapter.js.map +0 -1
- package/out/src/align.d.ts.map +0 -1
- package/out/src/align.js.map +0 -1
- package/out/src/format.d.ts +0 -14
- package/out/src/format.d.ts.map +0 -1
- package/out/src/format.js +0 -14
- package/out/src/format.js.map +0 -1
- package/out/src/golden.d.ts.map +0 -1
- package/out/src/golden.js.map +0 -1
- package/out/src/license.d.ts.map +0 -1
- package/out/src/license.js.map +0 -1
- package/out/src/parquet.d.ts.map +0 -1
- package/out/src/parquet.js.map +0 -1
- package/out/src/split.d.ts.map +0 -1
- package/out/src/split.js.map +0 -1
- package/out/src/synthesize-anchor-absorption.d.ts.map +0 -1
- package/out/src/synthesize-anchor-absorption.js.map +0 -1
- package/out/src/synthesize-boundary-stress.d.ts.map +0 -1
- package/out/src/synthesize-boundary-stress.js.map +0 -1
- package/out/src/synthesize-german.d.ts.map +0 -1
- package/out/src/synthesize-german.js.map +0 -1
- package/out/src/synthesize-house-venue.d.ts.map +0 -1
- package/out/src/synthesize-house-venue.js.map +0 -1
- package/out/src/synthesize-intersection.d.ts.map +0 -1
- package/out/src/synthesize-intersection.js.map +0 -1
- package/out/src/synthesize-no-street.d.ts.map +0 -1
- package/out/src/synthesize-no-street.js.map +0 -1
- package/out/src/synthesize-po-box.d.ts.map +0 -1
- package/out/src/synthesize-po-box.js.map +0 -1
- package/out/src/synthesize-street.d.ts.map +0 -1
- package/out/src/synthesize-street.js.map +0 -1
- package/out/src/synthesize.d.ts.map +0 -1
- package/out/src/synthesize.js.map +0 -1
- package/out/src/tokenize.d.ts.map +0 -1
- package/out/src/tokenize.js.map +0 -1
- package/out/src/wof-json.d.ts.map +0 -1
- package/out/src/wof-json.js.map +0 -1
- package/src/format.ts +0 -14
- /package/out/src/{align.js → utils/align.js} +0 -0
- /package/out/src/{license.d.ts → utils/license.d.ts} +0 -0
- /package/out/src/{license.js → utils/license.js} +0 -0
- /package/out/src/{tokenize.d.ts → utils/tokenize.d.ts} +0 -0
- /package/out/src/{tokenize.js → utils/tokenize.js} +0 -0
- /package/src/{synthesize-anchor-absorption.ts → synthesizers/anchor-absorption.ts} +0 -0
- /package/src/{license.ts → utils/license.ts} +0 -0
- /package/src/{tokenize.ts → utils/tokenize.ts} +0 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"imls-pls.js","sourceRoot":"","sources":["../../../../src/tools/fetch/imls-pls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;kGACkG;AAElG,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"imls-pls.js","sourceRoot":"","sources":["../../../../src/tools/fetch/imls-pls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;kGACkG;AAElG,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACzD,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAA;AACrC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAE1C,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAA;AACxE,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE3E;;;GAGG;AACH,MAAM,aAAa,GAAG,IAAI,CAAA;AAE1B;;GAEG;AACH,MAAM,OAAO,GAAG,qEAAqE,CAAA;AACrF,MAAM,IAAI,GAAG,gBAAgB,CAAA;AAY7B,MAAM,CAAC,KAAK,UAAU,YAAY,CACjC,OAA4B,EAC5B,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IAChD,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAEnD,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAEvB,0BAA0B;IAE1B,MAAM,QAAQ,GAAG,MAAM,YAAY,CAA0B,YAAY,CAAC,CAAA;IAE1E,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAErD,IAAI,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;YACtF,MAAM,EAAE,CAAC,oEAAoE,CAAC,CAAA;YAE9E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;QAC9D,CAAC;IACF,CAAC;IAED,qBAAqB;IAErB,MAAM,EAAE,CAAC,iBAAiB,OAAO,MAAM,CAAC,CAAA;IAExC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,cAAc,CAAC;QAC/C,GAAG,EAAE,OAAO;QACZ,IAAI,EAAE,OAAO;QACb,SAAS,EAAE,OAAO;QAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,iBAAiB,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IAElE,IAAI,OAAO,GAAG,aAAa,EAAE,CAAC;QAC7B,MAAM,EAAE,CAAC,2BAA2B,OAAO,+BAA+B,CAAC,CAAA;QAE3E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,gDAAgD;IAChD,4DAA4D;IAC5D,qEAAqE;IACrE,MAAM,EAAE,CAAC,+BAA+B,CAAC,CAAA;IACzC,MAAM,OAAO,GAAG,CAAC,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAE1E,IAAI,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;IAEzE,4EAA4E;IAC5E,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;IAClG,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,EAAE,CAAC,2BAA2B,CAAC,CAAA;QAErC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;QACxB,CAAC;QAED,MAAM,EAAE,CAAC,kFAAkF,CAAC,CAAA;QAE5F,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,EAAE,CAAC,4BAA4B,OAAO,EAAE,CAAC,CAAA;IAE/C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IAEhD,MAAM,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;IAChD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAA;IACtC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IAExC,mDAAmD;IAEnD,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAClC,MAAM,EAAE,CAAC,0BAA0B,CAAC,CAAA;IAEpC,uBAAuB;IAEvB,MAAM,QAAQ,GAAmB;QAChC,UAAU,EAAE,OAAO;QACnB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC;QAC3B,MAAM,EAAE,MAAM;QACd,KAAK,EAAE,OAAO;KACd,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,MAAM,EAAE,CAAC,CAAA;IAC1E,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IAEjD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}
|
|
@@ -40,6 +40,13 @@
|
|
|
40
40
|
* Tier A (US PD).
|
|
41
41
|
* - `openaddresses` — OpenAddresses country collections (default: Canada / `ca`). Tier B/C mixed
|
|
42
42
|
* — per-row filter.
|
|
43
|
+
* - `ourairports` — OurAirports global airport CSVs (~83K airports + the country/region/runway
|
|
44
|
+
* joins). Tier A (public domain). The VENUE half of the sub-venue arc (#35); it carries no
|
|
45
|
+
* interior structure, which comes from `@mailwoman/osm/sdk`'s `extractOSMSubVenues`.
|
|
46
|
+
* - `wikidata-subvenue` — the multilingual sub-venue DESIGNATOR vocabulary, pulled as the labels
|
|
47
|
+
* and aliases of eight Wikidata concepts (terminal, gate, concourse, campus, …) plus the
|
|
48
|
+
* airport-terminal instance labels as attested usage. Tier A (CC0). The only module in this
|
|
49
|
+
* family built on `APIClient` — see its docstring for why.
|
|
43
50
|
* - `state-sources` — NY/TX/DE/OR notaries, IA contractors, WA health providers, HI lobbyists.
|
|
44
51
|
* Tier A (state PD-equivalent).
|
|
45
52
|
* - `state-hi-schools` — Hawaii DOE school directory (XLSX → CSV via openpyxl). Tier A (state
|
|
@@ -65,7 +72,7 @@
|
|
|
65
72
|
*
|
|
66
73
|
* # Download Canada (~2 GiB compressed, ~7 GiB uncompressed)
|
|
67
74
|
* mailwoman corpus fetch openaddresses --country ca \
|
|
68
|
-
* --out-root /
|
|
75
|
+
* --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
|
|
69
76
|
*
|
|
70
77
|
* # Or any other OA country code
|
|
71
78
|
* mailwoman corpus fetch openaddresses --country fr
|
|
@@ -90,18 +97,22 @@ import { fetchIMLSPLS } from "./imls-pls.ts";
|
|
|
90
97
|
import { fetchNAD } from "./nad.ts";
|
|
91
98
|
import { fetchNPPES } from "./nppes.ts";
|
|
92
99
|
import { fetchOpenAddresses } from "./openaddresses.ts";
|
|
100
|
+
import { fetchOurAirports } from "./ourairports.ts";
|
|
93
101
|
import { fetchStateHISchools } from "./state-hi-schools.ts";
|
|
94
102
|
import { fetchStateSources } from "./state-sources.ts";
|
|
95
103
|
import { fetchTigerFull } from "./tiger-full.ts";
|
|
104
|
+
import { fetchWikidataSubVenue } from "./wikidata-subvenue.ts";
|
|
96
105
|
export * from "./ban.ts";
|
|
97
106
|
export * from "./hrsa.ts";
|
|
98
107
|
export * from "./imls-pls.ts";
|
|
99
108
|
export * from "./nad.ts";
|
|
100
109
|
export * from "./nppes.ts";
|
|
101
110
|
export * from "./openaddresses.ts";
|
|
111
|
+
export * from "./ourairports.ts";
|
|
102
112
|
export * from "./state-hi-schools.ts";
|
|
103
113
|
export * from "./state-sources.ts";
|
|
104
114
|
export * from "./tiger-full.ts";
|
|
115
|
+
export * from "./wikidata-subvenue.ts";
|
|
105
116
|
/**
|
|
106
117
|
* The fetch-source registry: id → module entry point. Each entry point takes its own options interface.
|
|
107
118
|
*/
|
|
@@ -112,9 +123,11 @@ export declare const FETCH_SOURCES: {
|
|
|
112
123
|
readonly "imls-pls": typeof fetchIMLSPLS;
|
|
113
124
|
readonly nppes: typeof fetchNPPES;
|
|
114
125
|
readonly openaddresses: typeof fetchOpenAddresses;
|
|
126
|
+
readonly ourairports: typeof fetchOurAirports;
|
|
115
127
|
readonly "state-sources": typeof fetchStateSources;
|
|
116
128
|
readonly "state-hi-schools": typeof fetchStateHISchools;
|
|
117
129
|
readonly "tiger-full": typeof fetchTigerFull;
|
|
130
|
+
readonly "wikidata-subvenue": typeof fetchWikidataSubVenue;
|
|
118
131
|
};
|
|
119
132
|
export type FetchSourceID = keyof typeof FETCH_SOURCES;
|
|
120
133
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/index.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4FG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AACvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAA;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAChD,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAA;AAE9D,cAAc,UAAU,CAAA;AACxB,cAAc,WAAW,CAAA;AACzB,cAAc,eAAe,CAAA;AAC7B,cAAc,UAAU,CAAA;AACxB,cAAc,YAAY,CAAA;AAC1B,cAAc,oBAAoB,CAAA;AAClC,cAAc,kBAAkB,CAAA;AAChC,cAAc,uBAAuB,CAAA;AACrC,cAAc,oBAAoB,CAAA;AAClC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,wBAAwB,CAAA;AAEtC;;GAEG;AACH,eAAO,MAAM,aAAa;;;;;;;;;;;;CAYhB,CAAA;AAEV,MAAM,MAAM,aAAa,GAAG,MAAM,OAAO,aAAa,CAAA"}
|
|
@@ -40,6 +40,13 @@
|
|
|
40
40
|
* Tier A (US PD).
|
|
41
41
|
* - `openaddresses` — OpenAddresses country collections (default: Canada / `ca`). Tier B/C mixed
|
|
42
42
|
* — per-row filter.
|
|
43
|
+
* - `ourairports` — OurAirports global airport CSVs (~83K airports + the country/region/runway
|
|
44
|
+
* joins). Tier A (public domain). The VENUE half of the sub-venue arc (#35); it carries no
|
|
45
|
+
* interior structure, which comes from `@mailwoman/osm/sdk`'s `extractOSMSubVenues`.
|
|
46
|
+
* - `wikidata-subvenue` — the multilingual sub-venue DESIGNATOR vocabulary, pulled as the labels
|
|
47
|
+
* and aliases of eight Wikidata concepts (terminal, gate, concourse, campus, …) plus the
|
|
48
|
+
* airport-terminal instance labels as attested usage. Tier A (CC0). The only module in this
|
|
49
|
+
* family built on `APIClient` — see its docstring for why.
|
|
43
50
|
* - `state-sources` — NY/TX/DE/OR notaries, IA contractors, WA health providers, HI lobbyists.
|
|
44
51
|
* Tier A (state PD-equivalent).
|
|
45
52
|
* - `state-hi-schools` — Hawaii DOE school directory (XLSX → CSV via openpyxl). Tier A (state
|
|
@@ -65,7 +72,7 @@
|
|
|
65
72
|
*
|
|
66
73
|
* # Download Canada (~2 GiB compressed, ~7 GiB uncompressed)
|
|
67
74
|
* mailwoman corpus fetch openaddresses --country ca \
|
|
68
|
-
* --out-root /
|
|
75
|
+
* --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
|
|
69
76
|
*
|
|
70
77
|
* # Or any other OA country code
|
|
71
78
|
* mailwoman corpus fetch openaddresses --country fr
|
|
@@ -90,18 +97,22 @@ import { fetchIMLSPLS } from "./imls-pls.js";
|
|
|
90
97
|
import { fetchNAD } from "./nad.js";
|
|
91
98
|
import { fetchNPPES } from "./nppes.js";
|
|
92
99
|
import { fetchOpenAddresses } from "./openaddresses.js";
|
|
100
|
+
import { fetchOurAirports } from "./ourairports.js";
|
|
93
101
|
import { fetchStateHISchools } from "./state-hi-schools.js";
|
|
94
102
|
import { fetchStateSources } from "./state-sources.js";
|
|
95
103
|
import { fetchTigerFull } from "./tiger-full.js";
|
|
104
|
+
import { fetchWikidataSubVenue } from "./wikidata-subvenue.js";
|
|
96
105
|
export * from "./ban.js";
|
|
97
106
|
export * from "./hrsa.js";
|
|
98
107
|
export * from "./imls-pls.js";
|
|
99
108
|
export * from "./nad.js";
|
|
100
109
|
export * from "./nppes.js";
|
|
101
110
|
export * from "./openaddresses.js";
|
|
111
|
+
export * from "./ourairports.js";
|
|
102
112
|
export * from "./state-hi-schools.js";
|
|
103
113
|
export * from "./state-sources.js";
|
|
104
114
|
export * from "./tiger-full.js";
|
|
115
|
+
export * from "./wikidata-subvenue.js";
|
|
105
116
|
/**
|
|
106
117
|
* The fetch-source registry: id → module entry point. Each entry point takes its own options interface.
|
|
107
118
|
*/
|
|
@@ -112,8 +123,10 @@ export const FETCH_SOURCES = {
|
|
|
112
123
|
"imls-pls": fetchIMLSPLS,
|
|
113
124
|
nppes: fetchNPPES,
|
|
114
125
|
openaddresses: fetchOpenAddresses,
|
|
126
|
+
ourairports: fetchOurAirports,
|
|
115
127
|
"state-sources": fetchStateSources,
|
|
116
128
|
"state-hi-schools": fetchStateHISchools,
|
|
117
129
|
"tiger-full": fetchTigerFull,
|
|
130
|
+
"wikidata-subvenue": fetchWikidataSubVenue,
|
|
118
131
|
};
|
|
119
132
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/tools/fetch/index.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/tools/fetch/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4FG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AACvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAA;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAChD,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAA;AAE9D,cAAc,UAAU,CAAA;AACxB,cAAc,WAAW,CAAA;AACzB,cAAc,eAAe,CAAA;AAC7B,cAAc,UAAU,CAAA;AACxB,cAAc,YAAY,CAAA;AAC1B,cAAc,oBAAoB,CAAA;AAClC,cAAc,kBAAkB,CAAA;AAChC,cAAc,uBAAuB,CAAA;AACrC,cAAc,oBAAoB,CAAA;AAClC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,wBAAwB,CAAA;AAEtC;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC5B,GAAG,EAAE,QAAQ;IACb,GAAG,EAAE,QAAQ;IACb,IAAI,EAAE,SAAS;IACf,UAAU,EAAE,YAAY;IACxB,KAAK,EAAE,UAAU;IACjB,aAAa,EAAE,kBAAkB;IACjC,WAAW,EAAE,gBAAgB;IAC7B,eAAe,EAAE,iBAAiB;IAClC,kBAAkB,EAAE,mBAAmB;IACvC,YAAY,EAAE,cAAc;IAC5B,mBAAmB,EAAE,qBAAqB;CACjC,CAAA"}
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
* ## Usage
|
|
27
27
|
*
|
|
28
28
|
* ```sh
|
|
29
|
-
* mailwoman corpus fetch nad --out-root /
|
|
29
|
+
* mailwoman corpus fetch nad --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
|
|
30
30
|
*
|
|
31
31
|
* # Resume from an OID
|
|
32
32
|
* mailwoman corpus fetch nad --start-oid 34400001
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
* ## Usage
|
|
27
27
|
*
|
|
28
28
|
* ```sh
|
|
29
|
-
* mailwoman corpus fetch nad --out-root /
|
|
29
|
+
* mailwoman corpus fetch nad --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
|
|
30
30
|
*
|
|
31
31
|
* # Resume from an OID
|
|
32
32
|
* mailwoman corpus fetch nad --start-oid 34400001
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"nppes.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/nppes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;
|
|
1
|
+
{"version":3,"file":"nppes.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/nppes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAYH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAOnE,MAAM,MAAM,iBAAiB,GAAG,gBAAgB,CAAA;AA0ChD,wBAAsB,UAAU,CAAC,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAyFnH"}
|
|
@@ -22,14 +22,12 @@
|
|
|
22
22
|
*/
|
|
23
23
|
/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
|
|
24
24
|
procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
|
|
25
|
-
import { execFile } from "node:child_process";
|
|
26
25
|
import { existsSync, mkdirSync, statSync } from "node:fs";
|
|
27
26
|
import { rm } from "node:fs/promises";
|
|
28
|
-
import { join } from "node:path";
|
|
29
|
-
import {
|
|
27
|
+
import { basename, join } from "node:path";
|
|
28
|
+
import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip";
|
|
30
29
|
import { sha256File } from "@mailwoman/core/utils";
|
|
31
30
|
import { downloadToFile, readManifest, writeManifest } from "./download.js";
|
|
32
|
-
const execFileAsync = promisify(execFile);
|
|
33
31
|
const INDEX_URL = "https://download.cms.gov/nppes/NPI_Files.html";
|
|
34
32
|
const BASE_URL = "https://download.cms.gov/nppes";
|
|
35
33
|
const SLUG = "usgov-nppes";
|
|
@@ -53,16 +51,12 @@ async function discoverLatestZip() {
|
|
|
53
51
|
return undefined;
|
|
54
52
|
}
|
|
55
53
|
/**
|
|
56
|
-
*
|
|
54
|
+
* The main registry CSV (npidata_pfile_*.csv), which the archive also carries alongside a header file and a per-month
|
|
55
|
+
* change file.
|
|
57
56
|
*/
|
|
58
57
|
async function findNpidataCSV(zipPath) {
|
|
59
|
-
const
|
|
60
|
-
|
|
61
|
-
const match = /npidata_pfile\S+\.csv/i.exec(line);
|
|
62
|
-
if (match?.[0])
|
|
63
|
-
return match[0];
|
|
64
|
-
}
|
|
65
|
-
return undefined;
|
|
58
|
+
const entries = await listZipEntries(zipPath);
|
|
59
|
+
return entries.find((entry) => /npidata_pfile\S+\.csv/i.test(entry.name))?.name;
|
|
66
60
|
}
|
|
67
61
|
export async function fetchNPPES(options, report) {
|
|
68
62
|
const destDir = join(options.outRoot, SLUG);
|
|
@@ -106,8 +100,8 @@ export async function fetchNPPES(options, report) {
|
|
|
106
100
|
return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
|
|
107
101
|
}
|
|
108
102
|
report?.(` Extracting: ${csvName}`);
|
|
109
|
-
|
|
110
|
-
|
|
103
|
+
const csvDest = join(destDir, basename(csvName));
|
|
104
|
+
await extractZipEntry(zipDest, csvName, csvDest);
|
|
111
105
|
const csvSize = statSync(csvDest).size;
|
|
112
106
|
const csvSha = await sha256File(csvDest);
|
|
113
107
|
report?.(` CSV size: ${(csvSize / 1024 / 1024).toFixed(1)} MB`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"nppes.js","sourceRoot":"","sources":["../../../../src/tools/fetch/nppes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH;kGACkG;AAElG,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"nppes.js","sourceRoot":"","sources":["../../../../src/tools/fetch/nppes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH;kGACkG;AAElG,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACzD,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAA;AACrC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAE1C,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAA;AACxE,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE3E,MAAM,SAAS,GAAG,+CAA+C,CAAA;AACjE,MAAM,QAAQ,GAAG,gCAAgC,CAAA;AACjD,MAAM,IAAI,GAAG,aAAa,CAAA;AAY1B;;;GAGG;AACH,KAAK,UAAU,iBAAiB;IAC/B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;QAClC,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;KACnC,CAAC,CAAA;IAEF,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,OAAO,SAAS,EAAE,CAAC,CAAA;IACpF,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAA;IAE7B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,qDAAqD,CAAC,EAAE,CAAC;QAC1F,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QAErB,IAAI,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;IACnD,CAAC;IAED,OAAO,SAAS,CAAA;AACjB,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,cAAc,CAAC,OAAe;IAC5C,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAA;IAE7C,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,wBAAwB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAA;AAChF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAA0B,EAAE,MAA+B;IAC3F,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACvC,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAEnD,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IACvB,MAAM,EAAE,CAAC,kDAAkD,SAAS,MAAM,CAAC,CAAA;IAE3E,MAAM,WAAW,GAAG,MAAM,iBAAiB,EAAE,CAAA;IAE7C,IAAI,CAAC,WAAW,EAAE,CAAC;QAClB,MAAM,EAAE,CAAC,4CAA4C,SAAS,EAAE,CAAC,CAAA;QAEjE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,MAAM,GAAG,GAAG,QAAQ,IAAI,WAAW,EAAE,CAAA;IAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;IAC1C,MAAM,EAAE,CAAC,uBAAuB,WAAW,EAAE,CAAC,CAAA;IAE9C,qEAAqE;IACrE,oBAAoB;IACpB,MAAM,QAAQ,GAAG,MAAM,YAAY,CAA0B,YAAY,CAAC,CAAA;IAE1E,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAErD,IAAI,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;YACtF,MAAM,EAAE,CAAC,oEAAoE,CAAC,CAAA;YAE9E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;QAC9D,CAAC;IACF,CAAC;IAED,gDAAgD;IAEhD,MAAM,EAAE,CAAC,iBAAiB,MAAM,MAAM,CAAC,CAAA;IAEvC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,cAAc,CAAC;QAC/C,GAAG,EAAE,MAAM;QACX,IAAI,EAAE,OAAO;QACb,SAAS,EAAE,SAAS;QACpB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,iBAAiB,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IAElE,iEAAiE;IAEjE,MAAM,EAAE,CAAC,6CAA6C,CAAC,CAAA;IACvD,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAA;IAE7C,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,EAAE,CAAC,iDAAiD,CAAC,CAAA;QAE3D,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,EAAE,CAAC,iBAAiB,OAAO,EAAE,CAAC,CAAA;IAEpC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IAEhD,MAAM,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;IAChD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAA;IACtC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IACxC,MAAM,EAAE,CAAC,eAAe,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IAEhE,wCAAwC;IAExC,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAClC,MAAM,EAAE,CAAC,0BAA0B,CAAC,CAAA;IAEpC,gEAAgE;IAEhE,MAAM,QAAQ,GAAmB;QAChC,UAAU,EAAE,MAAM;QAClB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,OAAO;QACjB,MAAM,EAAE,MAAM;QACd,KAAK,EAAE,OAAO;KACd,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,MAAM,EAAE,CAAC,CAAA;IAC1E,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IAEjD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
* ```sh
|
|
45
45
|
* # With token (preferred). Default country: ca. Supports any OA country code (us-west, fr, …)
|
|
46
46
|
* OA_BATCH_TOKEN=<token> mailwoman corpus fetch openaddresses --country ca \
|
|
47
|
-
* --out-root /
|
|
47
|
+
* --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
|
|
48
48
|
*
|
|
49
49
|
* # Without token (will detect + print instructions, then report the failure):
|
|
50
50
|
* mailwoman corpus fetch openaddresses --country ca
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
* ```sh
|
|
45
45
|
* # With token (preferred). Default country: ca. Supports any OA country code (us-west, fr, …)
|
|
46
46
|
* OA_BATCH_TOKEN=<token> mailwoman corpus fetch openaddresses --country ca \
|
|
47
|
-
* --out-root /
|
|
47
|
+
* --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
|
|
48
48
|
*
|
|
49
49
|
* # Without token (will detect + print instructions, then report the failure):
|
|
50
50
|
* mailwoman corpus fetch openaddresses --country ca
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Fetch the OurAirports CSV dumps — the VENUE side of the sub-venue corpus arc (#35).
|
|
7
|
+
*
|
|
8
|
+
* Source : https://davidmegginson.github.io/ourairports-data/ (the project's own GitHub Pages
|
|
9
|
+
* mirror of the nightly export; `ourairports.com/data/` redirects here).
|
|
10
|
+
* License: PUBLIC DOMAIN. OurAirports places its data in the public domain and asks only for a
|
|
11
|
+
* courtesy credit — no attribution obligation rides on a derived shard, which makes this
|
|
12
|
+
* the one transport source in the arc with no licensing question at all. Tier A.
|
|
13
|
+
*
|
|
14
|
+
* ## What it is good for, and what it is not
|
|
15
|
+
*
|
|
16
|
+
* `airports.csv` is every airport on earth with ICAO/IATA codes, coordinates, `municipality`, and
|
|
17
|
+
* `iso_country` — 12.7 MB, ~83,000 rows as of 2026-08-04. That is the CONTAINING VENUE for a
|
|
18
|
+
* `<sub-venue>, <venue>, <street>, <locality>, <postcode>` corpus line, and it is better at that job
|
|
19
|
+
* than OSM: every row is named, the name is canonical, and `municipality` gives the locality without
|
|
20
|
+
* a spatial join.
|
|
21
|
+
*
|
|
22
|
+
* It carries NO interior structure. There is no terminal, concourse, gate or pier table — the
|
|
23
|
+
* corpus task says as much ("Good for the venue side of each pair, weaker on interior structure")
|
|
24
|
+
* and a row-level read confirms it. Pair it with the OSM `aeroway` extractor
|
|
25
|
+
* (`@mailwoman/osm/sdk`'s `extractOSMSubVenues`), which is where the sub-venue half comes from.
|
|
26
|
+
*
|
|
27
|
+
* ## Why `downloadToFile` and not `APIClient`
|
|
28
|
+
*
|
|
29
|
+
* `AGENTS.md` routes HTTP through `APIClient`, and that rule is about API REQUESTS — small bodies,
|
|
30
|
+
* repeated calls, rate-limited hosts. This is four static file transfers against a GitHub Pages CDN
|
|
31
|
+
* with no rate limit and nothing to pace, run once per refresh. It uses the same `downloadToFile`
|
|
32
|
+
* every other module in this `fetch/` family uses, which is where the retry and timeout live.
|
|
33
|
+
* The Wikidata sibling (`wikidata-subvenue.ts`) IS an API client and is built on `APIClient`
|
|
34
|
+
* accordingly; the split between the two is the one `AGENTS.md` draws.
|
|
35
|
+
*
|
|
36
|
+
* Invoke via `mailwoman corpus fetch ourairports --out-root <path>`.
|
|
37
|
+
*/
|
|
38
|
+
import type { BaseFetchOptions, FetchSummary } from "./download.ts";
|
|
39
|
+
export type FetchOurAirportsOptions = BaseFetchOptions;
|
|
40
|
+
/**
|
|
41
|
+
* Download the OurAirports CSVs into `<outRoot>/ourairports/`, with a sibling `MANIFEST.json` carrying each file's
|
|
42
|
+
* origin URL, sha256, byte count and upstream `Last-Modified`.
|
|
43
|
+
*/
|
|
44
|
+
export declare function fetchOurAirports(options: FetchOurAirportsOptions, report?: (line: string) => void): Promise<FetchSummary>;
|
|
45
|
+
//# sourceMappingURL=ourairports.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ourairports.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/ourairports.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAOH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAsBnE,MAAM,MAAM,uBAAuB,GAAG,gBAAgB,CAAA;AAqCtD;;;GAGG;AACH,wBAAsB,gBAAgB,CACrC,OAAO,EAAE,uBAAuB,EAChC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAwDvB"}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Fetch the OurAirports CSV dumps — the VENUE side of the sub-venue corpus arc (#35).
|
|
7
|
+
*
|
|
8
|
+
* Source : https://davidmegginson.github.io/ourairports-data/ (the project's own GitHub Pages
|
|
9
|
+
* mirror of the nightly export; `ourairports.com/data/` redirects here).
|
|
10
|
+
* License: PUBLIC DOMAIN. OurAirports places its data in the public domain and asks only for a
|
|
11
|
+
* courtesy credit — no attribution obligation rides on a derived shard, which makes this
|
|
12
|
+
* the one transport source in the arc with no licensing question at all. Tier A.
|
|
13
|
+
*
|
|
14
|
+
* ## What it is good for, and what it is not
|
|
15
|
+
*
|
|
16
|
+
* `airports.csv` is every airport on earth with ICAO/IATA codes, coordinates, `municipality`, and
|
|
17
|
+
* `iso_country` — 12.7 MB, ~83,000 rows as of 2026-08-04. That is the CONTAINING VENUE for a
|
|
18
|
+
* `<sub-venue>, <venue>, <street>, <locality>, <postcode>` corpus line, and it is better at that job
|
|
19
|
+
* than OSM: every row is named, the name is canonical, and `municipality` gives the locality without
|
|
20
|
+
* a spatial join.
|
|
21
|
+
*
|
|
22
|
+
* It carries NO interior structure. There is no terminal, concourse, gate or pier table — the
|
|
23
|
+
* corpus task says as much ("Good for the venue side of each pair, weaker on interior structure")
|
|
24
|
+
* and a row-level read confirms it. Pair it with the OSM `aeroway` extractor
|
|
25
|
+
* (`@mailwoman/osm/sdk`'s `extractOSMSubVenues`), which is where the sub-venue half comes from.
|
|
26
|
+
*
|
|
27
|
+
* ## Why `downloadToFile` and not `APIClient`
|
|
28
|
+
*
|
|
29
|
+
* `AGENTS.md` routes HTTP through `APIClient`, and that rule is about API REQUESTS — small bodies,
|
|
30
|
+
* repeated calls, rate-limited hosts. This is four static file transfers against a GitHub Pages CDN
|
|
31
|
+
* with no rate limit and nothing to pace, run once per refresh. It uses the same `downloadToFile`
|
|
32
|
+
* every other module in this `fetch/` family uses, which is where the retry and timeout live.
|
|
33
|
+
* The Wikidata sibling (`wikidata-subvenue.ts`) IS an API client and is built on `APIClient`
|
|
34
|
+
* accordingly; the split between the two is the one `AGENTS.md` draws.
|
|
35
|
+
*
|
|
36
|
+
* Invoke via `mailwoman corpus fetch ourairports --out-root <path>`.
|
|
37
|
+
*/
|
|
38
|
+
import { mkdirSync } from "node:fs";
|
|
39
|
+
import { join } from "node:path";
|
|
40
|
+
import { sha256File } from "@mailwoman/core/utils";
|
|
41
|
+
import { downloadToFile, writeManifest } from "./download.js";
|
|
42
|
+
const SLUG = "ourairports";
|
|
43
|
+
/**
|
|
44
|
+
* The GitHub Pages mirror the project itself publishes. `ourairports.com/data/*.csv` 302s here, so pointing at the
|
|
45
|
+
* mirror directly saves a redirect and is the URL the project's own README gives.
|
|
46
|
+
*/
|
|
47
|
+
const BASE_URL = "https://davidmegginson.github.io/ourairports-data";
|
|
48
|
+
/**
|
|
49
|
+
* The files worth having, and why each one.
|
|
50
|
+
*
|
|
51
|
+
* `airports.csv` is the payload. The other three are small joins that turn its codes into text: `countries.csv` and
|
|
52
|
+
* `regions.csv` expand `iso_country`/`iso_region` into names (a corpus line needs "Germany", not "DE"), and
|
|
53
|
+
* `runways.csv` is the only file carrying per-airport sub-structure of any kind — runway designators, which are NOT
|
|
54
|
+
* sub-venue designators (nobody addresses mail to a runway) but are worth having on disk as the negative class if the
|
|
55
|
+
* shard ever needs one.
|
|
56
|
+
*/
|
|
57
|
+
const FILES = ["airports.csv", "countries.csv", "regions.csv", "runways.csv"];
|
|
58
|
+
/**
|
|
59
|
+
* Read the upstream `Last-Modified` with a HEAD. Returns `null` on any failure — provenance metadata is nice to have
|
|
60
|
+
* and must never fail a download that otherwise succeeded.
|
|
61
|
+
*/
|
|
62
|
+
async function readLastModified(url) {
|
|
63
|
+
try {
|
|
64
|
+
const res = await fetch(url, { method: "HEAD", signal: AbortSignal.timeout(30_000) });
|
|
65
|
+
return res.ok ? res.headers.get("last-modified") : null;
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Download the OurAirports CSVs into `<outRoot>/ourairports/`, with a sibling `MANIFEST.json` carrying each file's
|
|
73
|
+
* origin URL, sha256, byte count and upstream `Last-Modified`.
|
|
74
|
+
*/
|
|
75
|
+
export async function fetchOurAirports(options, report) {
|
|
76
|
+
const destDir = join(options.outRoot, SLUG);
|
|
77
|
+
mkdirSync(destDir, { recursive: true });
|
|
78
|
+
const entries = [];
|
|
79
|
+
const failedCodes = [];
|
|
80
|
+
let fetched = 0;
|
|
81
|
+
let failed = 0;
|
|
82
|
+
for (const filename of FILES) {
|
|
83
|
+
const url = `${BASE_URL}/${filename}`;
|
|
84
|
+
const dest = join(destDir, filename);
|
|
85
|
+
report?.(`=== ${SLUG} / ${filename}`);
|
|
86
|
+
try {
|
|
87
|
+
const [{ bytes }, lastModified] = await Promise.all([
|
|
88
|
+
downloadToFile({
|
|
89
|
+
url,
|
|
90
|
+
dest,
|
|
91
|
+
timeoutMs: 600_000,
|
|
92
|
+
retries: 2,
|
|
93
|
+
headers: { "Accept-Encoding": "gzip, br" },
|
|
94
|
+
report,
|
|
95
|
+
}),
|
|
96
|
+
readLastModified(url),
|
|
97
|
+
]);
|
|
98
|
+
entries.push({
|
|
99
|
+
filename,
|
|
100
|
+
source_url: url,
|
|
101
|
+
sha256: await sha256File(dest),
|
|
102
|
+
bytes,
|
|
103
|
+
last_modified: lastModified,
|
|
104
|
+
});
|
|
105
|
+
fetched++;
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
report?.(`✗ ${filename}: ${error instanceof Error ? error.message : String(error)}`);
|
|
109
|
+
failedCodes.push(filename);
|
|
110
|
+
failed++;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
const manifest = {
|
|
114
|
+
source: "OurAirports",
|
|
115
|
+
base_url: BASE_URL,
|
|
116
|
+
license: "public domain (courtesy credit requested)",
|
|
117
|
+
downloaded_at: new Date().toISOString(),
|
|
118
|
+
files: entries,
|
|
119
|
+
};
|
|
120
|
+
await writeManifest(join(destDir, "MANIFEST.json"), manifest);
|
|
121
|
+
return { fetched, skipped: 0, failed, failedCodes };
|
|
122
|
+
}
|
|
123
|
+
//# sourceMappingURL=ourairports.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ourairports.js","sourceRoot":"","sources":["../../../../src/tools/fetch/ourairports.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AACnC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE7D,MAAM,IAAI,GAAG,aAAa,CAAA;AAE1B;;;GAGG;AACH,MAAM,QAAQ,GAAG,mDAAmD,CAAA;AAEpE;;;;;;;;GAQG;AACH,MAAM,KAAK,GAAG,CAAC,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,aAAa,CAAU,CAAA;AAyBtF;;;GAGG;AACH,KAAK,UAAU,gBAAgB,CAAC,GAAW;IAC1C,IAAI,CAAC;QACJ,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAErF,OAAO,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IACxD,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACrC,OAAgC,EAChC,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvC,MAAM,OAAO,GAA2B,EAAE,CAAA;IAC1C,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,GAAG,QAAQ,IAAI,QAAQ,EAAE,CAAA;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAEpC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,QAAQ,EAAE,CAAC,CAAA;QAErC,IAAI,CAAC;YACJ,MAAM,CAAC,EAAE,KAAK,EAAE,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBACnD,cAAc,CAAC;oBACd,GAAG;oBACH,IAAI;oBACJ,SAAS,EAAE,OAAO;oBAClB,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;oBAC1C,MAAM;iBACN,CAAC;gBACF,gBAAgB,CAAC,GAAG,CAAC;aACrB,CAAC,CAAA;YAEF,OAAO,CAAC,IAAI,CAAC;gBACZ,QAAQ;gBACR,UAAU,EAAE,GAAG;gBACf,MAAM,EAAE,MAAM,UAAU,CAAC,IAAI,CAAC;gBAC9B,KAAK;gBACL,aAAa,EAAE,YAAY;aAC3B,CAAC,CAAA;YAEF,OAAO,EAAE,CAAA;QACV,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,KAAK,QAAQ,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YACpF,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YAE1B,MAAM,EAAE,CAAA;QACT,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAwB;QACrC,MAAM,EAAE,aAAa;QACrB,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,2CAA2C;QACpD,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,KAAK,EAAE,OAAO;KACd,CAAA;IAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAE7D,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACpD,CAAC"}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Fetch the multilingual sub-venue designator vocabulary from Wikidata (#35 wave 1).
|
|
7
|
+
*
|
|
8
|
+
* Source : https://query.wikidata.org/sparql (the Wikidata Query Service).
|
|
9
|
+
* License: CC0. Wikidata's data is public-domain dedicated, so nothing rides on a derived shard.
|
|
10
|
+
* Tier A.
|
|
11
|
+
*
|
|
12
|
+
* ## The pull is CLASS labels, not instance names — and that inversion is the whole design
|
|
13
|
+
*
|
|
14
|
+
* The obvious read of "Wikidata for localized designators" is: fetch every airport terminal entity
|
|
15
|
+
* and read its name in each language. That was tried first and it is the WRONG query. Wikidata
|
|
16
|
+
* knows 246 items that are `instance of / subclass of*` airport terminal, carrying 775 labels
|
|
17
|
+
* between them (measured 2026-08-04), and most of those labels are proper names that translate
|
|
18
|
+
* verbatim — `TWA Flight Center` is spelled `TWA Flight Center` in fifteen languages. The
|
|
19
|
+
* instance layer is thin and its localization is mostly a no-op.
|
|
20
|
+
*
|
|
21
|
+
* The DESIGNATOR is the label of the CLASS. `wd:Q849706` ("airport terminal") is labelled `Terminal`
|
|
22
|
+
* in German, `terminal aéroportuaire` in French, `ターミナルビル` in Japanese, `航站楼` in Chinese —
|
|
23
|
+
* and `skos:altLabel` adds the aliases (`Abfertigungsgebäude`, `Flughafenterminal`, `aerostazione`).
|
|
24
|
+
* Eight concept ids yield 877 label+alias rows across 174 languages, which is the vocabulary the
|
|
25
|
+
* corpus task asked for and the instance query does not contain. {@link SUBVENUE_CONCEPTS} is that
|
|
26
|
+
* list of ids; {@link buildDesignatorLabelQuery} is that query.
|
|
27
|
+
*
|
|
28
|
+
* Instance labels are fetched too ({@link buildTerminalInstanceQuery}), for a different job: they
|
|
29
|
+
* are ATTESTED USAGE — evidence of how a designator combines with a modifier or an identifier in
|
|
30
|
+
* running text. 775 rows is small, and it is a validation set, not a vocabulary.
|
|
31
|
+
*
|
|
32
|
+
* ## What a caller must NOT do with the output
|
|
33
|
+
*
|
|
34
|
+
* A class label is a CONCEPT NAME, not a designator as written in an address. Q849706's Spanish
|
|
35
|
+
* label is `terminal aeroportuaria` and its French is `terminal d'aéroport`; nobody writes either on
|
|
36
|
+
* an envelope, they write `Terminal`. Q247739's Spanish is `puerta de embarque` where the addressed
|
|
37
|
+
* form is `Puerta`. So this fetch produces CANDIDATE SURFACES that need a head-noun/curation pass
|
|
38
|
+
* before any of them reaches `neural/venue-structure.ts`'s designator vocabulary — the lexicon
|
|
39
|
+
* builder marks every one `curated: false` and the burden of promotion is on a human. Wiring the raw
|
|
40
|
+
* pull straight into the span proposer would admit multi-word phrases that match nothing and, worse,
|
|
41
|
+
* admit `hall` in a language where it names an ordinary room.
|
|
42
|
+
*
|
|
43
|
+
* ## Why `APIClient` here when the OurAirports sibling uses `downloadToFile`
|
|
44
|
+
*
|
|
45
|
+
* This is the API-request side of `AGENTS.md`'s split: small JSON bodies, several calls per run, and
|
|
46
|
+
* a host that publishes a rate policy and enforces it with 429s. Pacing, bounded `Retry-After`-aware
|
|
47
|
+
* retry, response caching and `ResourceError` mapping all earn their keep, so it extends
|
|
48
|
+
* {@link APIClient}. `ourairports.ts` is four static file transfers off a CDN and correctly does not.
|
|
49
|
+
*
|
|
50
|
+
* WDQS also REQUIRES a descriptive `User-Agent` naming the tool and a contact — an anonymous or
|
|
51
|
+
* library-default agent is blocked outright by the Wikimedia user-agent policy. See
|
|
52
|
+
* {@link WIKIDATA_USER_AGENT}.
|
|
53
|
+
*
|
|
54
|
+
* Invoke via `mailwoman corpus fetch wikidata-subvenue --out-root <path>`.
|
|
55
|
+
*/
|
|
56
|
+
import { APIClient, type ClockLike } from "@mailwoman/core/api";
|
|
57
|
+
import type { BaseFetchOptions, FetchSummary } from "./download.ts";
|
|
58
|
+
/**
|
|
59
|
+
* The SPARQL endpoint. Public, no credential.
|
|
60
|
+
*/
|
|
61
|
+
export declare const WDQS_ENDPOINT = "https://query.wikidata.org/sparql";
|
|
62
|
+
/**
|
|
63
|
+
* The `User-Agent` every request carries.
|
|
64
|
+
*
|
|
65
|
+
* NOT decoration. The Wikimedia user-agent policy blocks requests whose agent is absent, generic, or a library default,
|
|
66
|
+
* and WDQS enforces it — an unidentified client gets a 403 that no amount of retrying fixes. The policy asks for a tool
|
|
67
|
+
* name, a URL, and a contact address, all three of which are here.
|
|
68
|
+
*/
|
|
69
|
+
export declare const WIKIDATA_USER_AGENT = "mailwoman/1.0 (https://github.com/sister-software/mailwoman; teffen@sister.software) corpus-subvenue-fetch";
|
|
70
|
+
/**
|
|
71
|
+
* One Wikidata concept whose labels are a designator's multilingual surface set.
|
|
72
|
+
*
|
|
73
|
+
* `designatorID` is the mailwoman-side vocabulary term, matching `neural/venue-structure.ts`'s
|
|
74
|
+
* `VENUE_STRUCTURE_DESIGNATORS` wherever the two overlap. `qid` was resolved by `wbsearchentities` and hand-checked
|
|
75
|
+
* against the entity's English description (recorded below) on 2026-08-04 — a QID picked by search alone is how you end
|
|
76
|
+
* up pulling the labels of a Bronx neighbourhood called Concourse.
|
|
77
|
+
*
|
|
78
|
+
* `wing` is ABSENT and that is a finding, not an oversight: Wikidata has no clean concept for "wing of a building".
|
|
79
|
+
* `wbsearchentities` for "wing" returns a surname, two English villages, a rugby position and a drone company. Since
|
|
80
|
+
* `wing` is the single most valuable designator in the arc — `West Wing` is the one modifier case that already parses,
|
|
81
|
+
* and `East Wing` is the one that does not — its localized surfaces have to come from somewhere else. See the wave-1
|
|
82
|
+
* report.
|
|
83
|
+
*/
|
|
84
|
+
export interface SubVenueConcept {
|
|
85
|
+
designatorID: string;
|
|
86
|
+
qid: string;
|
|
87
|
+
/**
|
|
88
|
+
* The entity's English description, recorded so a future reader can tell at a glance whether the QID still names what
|
|
89
|
+
* we think it names.
|
|
90
|
+
*/
|
|
91
|
+
gloss: string;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* The concept table. Eight ids, each verified against its English description on 2026-08-04.
|
|
95
|
+
*/
|
|
96
|
+
export declare const SUBVENUE_CONCEPTS: readonly SubVenueConcept[];
|
|
97
|
+
/**
|
|
98
|
+
* Build the class-label query: `rdfs:label` and `skos:altLabel` for every concept, in every language, tagged with which
|
|
99
|
+
* of the two it came from so the lexicon can rank a label above an alias.
|
|
100
|
+
*
|
|
101
|
+
* `VALUES` rather than a property path over the whole class tree — the concept list is closed and hand-verified, and a
|
|
102
|
+
* `wdt:P279*` walk from `building` would drag in every structure type on earth.
|
|
103
|
+
*/
|
|
104
|
+
export declare function buildDesignatorLabelQuery(concepts?: readonly SubVenueConcept[]): string;
|
|
105
|
+
/**
|
|
106
|
+
* Build the instance-label query — every item that is an `instance of` (through any `subclass of` chain) an airport
|
|
107
|
+
* terminal, with all of its labels. Measured at 246 items / 775 labels on 2026-08-04, well inside WDQS's 60-second
|
|
108
|
+
* budget.
|
|
109
|
+
*
|
|
110
|
+
* A caveat worth knowing before trusting a row: Wikidata's P31 on these is not clean. `Q1322696` (Kigali International
|
|
111
|
+
* Airport) is typed as an airport terminal, so the result set mixes AIRPORTS in with terminals. The consumer filters;
|
|
112
|
+
* this module fetches what the query returns.
|
|
113
|
+
*/
|
|
114
|
+
export declare function buildTerminalInstanceQuery(classQID?: string): string;
|
|
115
|
+
/**
|
|
116
|
+
* The SPARQL JSON results shape, narrowed to the two column types these queries produce.
|
|
117
|
+
*/
|
|
118
|
+
export interface SPARQLResults {
|
|
119
|
+
results: {
|
|
120
|
+
bindings: Array<Record<string, {
|
|
121
|
+
type: string;
|
|
122
|
+
value: string;
|
|
123
|
+
"xml:lang"?: string;
|
|
124
|
+
}>>;
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Whether a decoded body is a SPARQL results envelope. Used as the cache's write validator so an HTML error page served
|
|
129
|
+
* under a 200 is never persisted for the next run to destructure into `undefined`.
|
|
130
|
+
*/
|
|
131
|
+
export declare function isSPARQLResults(value: unknown): value is SPARQLResults;
|
|
132
|
+
export interface CreateWikidataClientOptions {
|
|
133
|
+
/**
|
|
134
|
+
* On-disk response-cache root. Defaults to a `http-cache` directory beside the fetch output.
|
|
135
|
+
*/
|
|
136
|
+
cacheDir: string;
|
|
137
|
+
/**
|
|
138
|
+
* Time source powering the pacer and the retry backoff. Defaults to the system clock; tests inject a fake so no suite
|
|
139
|
+
* ever sleeps a real second.
|
|
140
|
+
*/
|
|
141
|
+
clock?: ClockLike;
|
|
142
|
+
/**
|
|
143
|
+
* Axios overrides, merged over this client's defaults. THE TEST SEAM — pass an `adapter` and no live call is made.
|
|
144
|
+
* Overriding `headers` wholesale would drop the required `User-Agent`, so don't.
|
|
145
|
+
*/
|
|
146
|
+
axios?: ConstructorParameters<typeof APIClient>[0]["axios"];
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* A Wikidata Query Service client: paced, retrying, disk-cached, and correctly identified.
|
|
150
|
+
*/
|
|
151
|
+
export declare class WikidataClient extends APIClient {
|
|
152
|
+
/**
|
|
153
|
+
* Run one SPARQL query and return its results envelope.
|
|
154
|
+
*/
|
|
155
|
+
query(sparql: string): Promise<SPARQLResults>;
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Construct a {@link WikidataClient} with every default resolved.
|
|
159
|
+
*/
|
|
160
|
+
export declare function createWikidataClient(options: CreateWikidataClientOptions): WikidataClient;
|
|
161
|
+
export type FetchWikidataSubVenueOptions = BaseFetchOptions;
|
|
162
|
+
/**
|
|
163
|
+
* Run both queries and write their raw SPARQL JSON into `<outRoot>/wikidata-subvenue/`, with a `MANIFEST.json` carrying
|
|
164
|
+
* the endpoint, the exact queries, the concept table, row counts and sha256s.
|
|
165
|
+
*
|
|
166
|
+
* The RAW envelope is written rather than a reshaped one on purpose: the lexicon build is a separate, pure step
|
|
167
|
+
* (`sub-venue-lexicon.ts`) and keeping the fetch output byte-faithful to what WDQS served means a lexicon regeneration
|
|
168
|
+
* never needs the network.
|
|
169
|
+
*/
|
|
170
|
+
export declare function fetchWikidataSubVenue(options: FetchWikidataSubVenueOptions, report?: (line: string) => void): Promise<FetchSummary>;
|
|
171
|
+
//# sourceMappingURL=wikidata-subvenue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wikidata-subvenue.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/wikidata-subvenue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AAMH,OAAO,EAAE,SAAS,EAAE,KAAK,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAI/D,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAKnE;;GAEG;AACH,eAAO,MAAM,aAAa,sCAAsC,CAAA;AAEhE;;;;;;GAMG;AACH,eAAO,MAAM,mBAAmB,+GAC6E,CAAA;AAkC7G;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,eAAe;IAC/B,YAAY,EAAE,MAAM,CAAA;IACpB,GAAG,EAAE,MAAM,CAAA;IACX;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAA;CACb;AAED;;GAEG;AACH,eAAO,MAAM,iBAAiB,EAAE,SAAS,eAAe,EASvD,CAAA;AAOD;;;;;;GAMG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,GAAE,SAAS,eAAe,EAAsB,GAAG,MAAM,CAU1G;AAED;;;;;;;;GAQG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,GAAE,MAA2B,GAAG,MAAM,CAMxF;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,OAAO,EAAE;QACR,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,UAAU,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC,CAAA;KACrF,CAAA;CACD;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAEtE;AAED,MAAM,WAAW,2BAA2B;IAC3C;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB;;;OAGG;IACH,KAAK,CAAC,EAAE,SAAS,CAAA;IACjB;;;OAGG;IACH,KAAK,CAAC,EAAE,qBAAqB,CAAC,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAA;CAC3D;AAED;;GAEG;AACH,qBAAa,cAAe,SAAQ,SAAS;IAC5C;;OAEG;IACU,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;CAQ1D;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,2BAA2B,GAAG,cAAc,CAkCzF;AAED,MAAM,MAAM,4BAA4B,GAAG,gBAAgB,CAAA;AA0C3D;;;;;;;GAOG;AACH,wBAAsB,qBAAqB,CAC1C,OAAO,EAAE,4BAA4B,EACrC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA+CvB"}
|