@mailwoman/corpus 9.0.0 → 9.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/data/PROVENANCE.md +227 -0
- package/data/reviewed-ve-postcode-tuples.json +74 -0
- package/data/sub-venue-lexicon.json +18269 -0
- package/out/src/adapters/ban/adapter.d.ts +1 -1
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +5 -4
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +3 -3
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +1 -1
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +2 -2
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +3 -3
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +2 -2
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +1 -1
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +3 -3
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +1 -1
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +2 -2
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +7 -6
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +12 -7
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +12 -18
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +12 -7
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +3 -3
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +3 -3
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +1 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +2 -2
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +7 -6
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +11 -7
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +6 -5
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +8 -8
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +10 -10
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +7 -6
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/{adapter.d.ts → adapters/utils/index.d.ts} +8 -2
- package/out/src/adapters/utils/index.d.ts.map +1 -0
- package/out/src/{adapter.js → adapters/utils/index.js} +11 -3
- package/out/src/adapters/utils/index.js.map +1 -0
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +5 -5
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +4 -15
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +11 -35
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-json-rows.d.ts +32 -0
- package/out/src/adapters/wof-json-rows.d.ts.map +1 -0
- package/out/src/adapters/wof-json-rows.js +45 -0
- package/out/src/adapters/wof-json-rows.js.map +1 -0
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +4 -9
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +12 -37
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/build.d.ts +4 -4
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +7 -7
- package/out/src/build.js.map +1 -1
- package/out/src/index.d.ts +3 -17
- package/out/src/index.d.ts.map +1 -1
- package/out/src/index.js +3 -17
- package/out/src/index.js.map +1 -1
- package/out/src/name-prone-us-suffixes.d.ts +12 -0
- package/out/src/name-prone-us-suffixes.d.ts.map +1 -0
- package/out/src/name-prone-us-suffixes.js +12 -0
- package/out/src/name-prone-us-suffixes.js.map +1 -0
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +5 -0
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/runner.d.ts +2 -2
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +1 -1
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +4 -5
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/bare-country.d.ts +25 -0
- package/out/src/shard-recipes/bare-country.d.ts.map +1 -0
- package/out/src/shard-recipes/bare-country.js +87 -0
- package/out/src/shard-recipes/bare-country.js.map +1 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +2 -2
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +14 -21
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +28 -10
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +4 -3
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.js +72 -11
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +4 -3
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +11 -20
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +8 -14
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/house-venue.js +1 -1
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +8 -1
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +7 -13
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +5 -5
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +13 -20
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/no-street.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street.js +1 -1
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +60 -43
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +1 -1
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/reviewed-postcode-tail.d.ts +47 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.d.ts.map +1 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.js +125 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.js.map +1 -0
- package/out/src/shard-recipes/scaffold.d.ts +51 -23
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +59 -23
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +54 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +203 -35
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
- package/out/src/shard-recipes/street-bare.js +3 -3
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.d.ts.map +1 -1
- package/out/src/shard-recipes/street.js +2 -2
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.d.ts +3 -6
- package/out/src/shard-recipes/sub-venue-sources.d.ts.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.js +4 -9
- package/out/src/shard-recipes/sub-venue-sources.js.map +1 -1
- package/out/src/shard-recipes/sub-venue.d.ts +2 -2
- package/out/src/shard-recipes/sub-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/sub-venue.js +7 -6
- package/out/src/shard-recipes/sub-venue.js.map +1 -1
- package/out/src/shard-recipes/trailing-region.d.ts +74 -0
- package/out/src/shard-recipes/trailing-region.d.ts.map +1 -0
- package/out/src/shard-recipes/trailing-region.js +157 -0
- package/out/src/shard-recipes/trailing-region.js.map +1 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +6 -12
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/{synthesize-anchor-absorption.d.ts → synthesizers/anchor-absorption.d.ts} +1 -1
- package/out/src/synthesizers/anchor-absorption.d.ts.map +1 -0
- package/out/src/{synthesize-anchor-absorption.js → synthesizers/anchor-absorption.js} +1 -1
- package/out/src/synthesizers/anchor-absorption.js.map +1 -0
- package/out/src/{synthesize-boundary-stress.d.ts → synthesizers/boundary-stress.d.ts} +3 -3
- package/out/src/synthesizers/boundary-stress.d.ts.map +1 -0
- package/out/src/{synthesize-boundary-stress.js → synthesizers/boundary-stress.js} +2 -2
- package/out/src/synthesizers/boundary-stress.js.map +1 -0
- package/out/src/{synthesize-german.d.ts → synthesizers/german.d.ts} +2 -2
- package/out/src/synthesizers/german.d.ts.map +1 -0
- package/out/src/{synthesize-german.js → synthesizers/german.js} +2 -2
- package/out/src/synthesizers/german.js.map +1 -0
- package/out/src/{synthesize-house-venue.d.ts → synthesizers/house-venue.d.ts} +4 -4
- package/out/src/synthesizers/house-venue.d.ts.map +1 -0
- package/out/src/{synthesize-house-venue.js → synthesizers/house-venue.js} +20 -6
- package/out/src/synthesizers/house-venue.js.map +1 -0
- package/out/src/{synthesize-intersection.d.ts → synthesizers/intersection.d.ts} +2 -2
- package/out/src/synthesizers/intersection.d.ts.map +1 -0
- package/out/src/{synthesize-intersection.js → synthesizers/intersection.js} +1 -1
- package/out/src/synthesizers/intersection.js.map +1 -0
- package/out/src/{synthesize-no-street.d.ts → synthesizers/no-street.d.ts} +3 -3
- package/out/src/synthesizers/no-street.d.ts.map +1 -0
- package/out/src/{synthesize-no-street.js → synthesizers/no-street.js} +2 -2
- package/out/src/synthesizers/no-street.js.map +1 -0
- package/out/src/{synthesize-po-box.d.ts → synthesizers/po-box.d.ts} +2 -2
- package/out/src/synthesizers/po-box.d.ts.map +1 -0
- package/out/src/{synthesize-po-box.js → synthesizers/po-box.js} +1 -1
- package/out/src/synthesizers/po-box.js.map +1 -0
- package/out/src/{synthesize-street.d.ts → synthesizers/street.d.ts} +2 -2
- package/out/src/synthesizers/street.d.ts.map +1 -0
- package/out/src/{synthesize-street.js → synthesizers/street.js} +4 -3
- package/out/src/synthesizers/street.js.map +1 -0
- package/out/src/{synthesize.d.ts → synthesizers/utils.d.ts} +3 -3
- package/out/src/synthesizers/utils.d.ts.map +1 -0
- package/out/src/{synthesize.js → synthesizers/utils.js} +5 -4
- package/out/src/synthesizers/utils.js.map +1 -0
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +2 -2
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/fetch/ban.d.ts.map +1 -1
- package/out/src/tools/fetch/ban.js +2 -18
- package/out/src/tools/fetch/ban.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts +12 -3
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +20 -4
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/geonames-dump.d.ts +87 -0
- package/out/src/tools/fetch/geonames-dump.d.ts.map +1 -0
- package/out/src/tools/fetch/geonames-dump.js +178 -0
- package/out/src/tools/fetch/geonames-dump.js.map +1 -0
- package/out/src/tools/fetch/geonames-postal.d.ts +76 -0
- package/out/src/tools/fetch/geonames-postal.d.ts.map +1 -0
- package/out/src/tools/fetch/geonames-postal.js +125 -0
- package/out/src/tools/fetch/geonames-postal.js.map +1 -0
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +5 -17
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +11 -0
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +11 -0
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts.map +1 -1
- package/out/src/tools/fetch/nad.js +20 -11
- package/out/src/tools/fetch/nad.js.map +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +17 -22
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/src/tools/fetch/openaddresses.js +18 -17
- package/out/src/tools/fetch/openaddresses.js.map +1 -1
- package/out/src/tools/fetch/ourairports.d.ts.map +1 -1
- package/out/src/tools/fetch/ourairports.js +7 -2
- package/out/src/tools/fetch/ourairports.js.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +0 -4
- package/out/src/tools/fetch/ppd.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.js +1 -1
- package/out/src/tools/fetch/ppd.js.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.js +3 -19
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
- package/out/src/tools/fetch/state-sources.d.ts +4 -0
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
- package/out/src/tools/fetch/state-sources.js +1 -5
- package/out/src/tools/fetch/state-sources.js.map +1 -1
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
- package/out/src/tools/fetch/tiger-full.js +15 -21
- package/out/src/tools/fetch/tiger-full.js.map +1 -1
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +34 -25
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-relabel-street.d.ts.map +1 -1
- package/out/src/tools/golden-relabel-street.js +4 -77
- package/out/src/tools/golden-relabel-street.js.map +1 -1
- package/out/src/tools/index.d.ts +2 -2
- package/out/src/tools/index.d.ts.map +1 -1
- package/out/src/tools/index.js +2 -2
- package/out/src/tools/index.js.map +1 -1
- package/out/src/tools/ingest-csv.d.ts.map +1 -1
- package/out/src/tools/ingest-csv.js +0 -11
- package/out/src/tools/ingest-csv.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts +4 -0
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +16 -6
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/postcode-triples.d.ts +175 -0
- package/out/src/tools/postcode-triples.d.ts.map +1 -0
- package/out/src/tools/postcode-triples.js +304 -0
- package/out/src/tools/postcode-triples.js.map +1 -0
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +1 -3
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +4 -27
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/tools/sub-venue/harvest.d.ts +100 -0
- package/out/src/tools/sub-venue/harvest.d.ts.map +1 -0
- package/out/src/tools/sub-venue/harvest.js +168 -0
- package/out/src/tools/sub-venue/harvest.js.map +1 -0
- package/out/src/tools/sub-venue/head-nouns.d.ts +50 -0
- package/out/src/tools/sub-venue/head-nouns.d.ts.map +1 -0
- package/out/src/tools/sub-venue/head-nouns.js +210 -0
- package/out/src/tools/sub-venue/head-nouns.js.map +1 -0
- package/out/src/tools/sub-venue/surfaces.d.ts +45 -0
- package/out/src/tools/sub-venue/surfaces.d.ts.map +1 -0
- package/out/src/tools/sub-venue/surfaces.js +77 -0
- package/out/src/tools/sub-venue/surfaces.js.map +1 -0
- package/out/src/tools/sub-venue/table.d.ts +229 -0
- package/out/src/tools/sub-venue/table.d.ts.map +1 -0
- package/out/src/tools/sub-venue/table.js +108 -0
- package/out/src/tools/sub-venue/table.js.map +1 -0
- package/out/src/tools/sub-venue/wikidata.d.ts +26 -0
- package/out/src/tools/sub-venue/wikidata.d.ts.map +1 -0
- package/out/src/tools/sub-venue/wikidata.js +62 -0
- package/out/src/tools/sub-venue/wikidata.js.map +1 -0
- package/out/src/tools/sub-venue-lexicon.d.ts +27 -378
- package/out/src/tools/sub-venue-lexicon.d.ts.map +1 -1
- package/out/src/tools/sub-venue-lexicon.js +30 -565
- package/out/src/tools/sub-venue-lexicon.js.map +1 -1
- package/out/src/{align.d.ts → utils/align.d.ts} +1 -1
- package/out/src/utils/align.d.ts.map +1 -0
- package/out/src/utils/align.js.map +1 -0
- package/out/src/utils/golden.d.ts.map +1 -0
- package/out/src/{golden.js → utils/golden.js} +2 -2
- package/out/src/utils/golden.js.map +1 -0
- package/out/src/utils/index.d.ts +14 -0
- package/out/src/utils/index.d.ts.map +1 -0
- package/out/src/utils/index.js +14 -0
- package/out/src/utils/index.js.map +1 -0
- package/out/src/utils/license.d.ts.map +1 -0
- package/out/src/{license.js → utils/license.js} +1 -1
- package/out/src/utils/license.js.map +1 -0
- package/out/src/{parquet.d.ts → utils/parquet.d.ts} +24 -5
- package/out/src/utils/parquet.d.ts.map +1 -0
- package/out/src/{parquet.js → utils/parquet.js} +6 -3
- package/out/src/utils/parquet.js.map +1 -0
- package/out/src/{split.d.ts → utils/split.d.ts} +1 -1
- package/out/src/utils/split.d.ts.map +1 -0
- package/out/src/{split.js → utils/split.js} +2 -2
- package/out/src/utils/split.js.map +1 -0
- package/out/src/utils/tokenize.d.ts.map +1 -0
- package/out/src/utils/tokenize.js.map +1 -0
- package/out/src/utils/wof-json.d.ts.map +1 -0
- package/out/src/utils/wof-json.js.map +1 -0
- package/package.json +287 -21
- package/src/adapters/ban/adapter.ts +7 -5
- package/src/adapters/fcc-bdc/adapter.ts +5 -4
- package/src/adapters/geonames/adapter.ts +3 -3
- package/src/adapters/geonames-postal/adapter.ts +4 -4
- package/src/adapters/gnaf/adapter.ts +3 -3
- package/src/adapters/gnaf/assemble.ts +1 -1
- package/src/adapters/index.ts +3 -2
- package/src/adapters/openaddresses/adapter.ts +4 -4
- package/src/adapters/overture/adapter.ts +3 -3
- package/src/adapters/state-hi-schools/adapter.ts +8 -7
- package/src/adapters/state-ia-contractors/adapter.ts +13 -8
- package/src/adapters/state-ny-notaries/adapter.ts +13 -32
- package/src/adapters/state-tx-notaries/adapter.ts +13 -8
- package/src/adapters/synth-po-box/adapter.ts +4 -4
- package/src/adapters/tiger/adapter.ts +5 -3
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +8 -7
- package/src/adapters/usgov-imls-pls/adapter.ts +12 -8
- package/src/adapters/usgov-irs-bmf/adapter.ts +7 -6
- package/src/adapters/usgov-nad/adapter.ts +9 -9
- package/src/adapters/usgov-nppes/adapter.ts +13 -12
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +8 -7
- package/src/{adapter.ts → adapters/utils/index.ts} +15 -3
- package/src/adapters/wof-admin-jp/adapter.ts +9 -12
- package/src/adapters/wof-admin-json/adapter.ts +15 -54
- package/src/adapters/wof-json-rows.ts +79 -0
- package/src/adapters/wof-postalcode-json/adapter.ts +16 -50
- package/src/build.ts +8 -8
- package/src/index.ts +3 -17
- package/src/name-prone-us-suffixes.ts +12 -0
- package/src/parquet-wrapper/reader.ts +9 -0
- package/src/runner.ts +2 -2
- package/src/shard-recipes/anchor-absorption.ts +5 -5
- package/src/shard-recipes/bare-country.ts +95 -0
- package/src/shard-recipes/boundary-stress.ts +3 -2
- package/src/shard-recipes/country-balanced.ts +16 -27
- package/src/shard-recipes/cz-pcfirst-preposition.ts +29 -10
- package/src/shard-recipes/fr-admin-split.ts +6 -4
- package/src/shard-recipes/fr-bare-street.ts +85 -10
- package/src/shard-recipes/fr-lieudit.ts +6 -4
- package/src/shard-recipes/fr-order.ts +13 -26
- package/src/shard-recipes/german.ts +9 -18
- package/src/shard-recipes/house-venue.ts +2 -1
- package/src/shard-recipes/index.ts +8 -1
- package/src/shard-recipes/intersection.ts +9 -17
- package/src/shard-recipes/locale.ts +16 -24
- package/src/shard-recipes/no-street.ts +2 -1
- package/src/shard-recipes/po-box-cedex.ts +76 -58
- package/src/shard-recipes/po-box.ts +2 -1
- package/src/shard-recipes/reviewed-postcode-tail.ts +189 -0
- package/src/shard-recipes/scaffold.ts +99 -24
- package/src/shard-recipes/street-affix.ts +284 -45
- package/src/shard-recipes/street-bare.ts +4 -3
- package/src/shard-recipes/street.ts +3 -2
- package/src/shard-recipes/sub-venue-sources.ts +13 -23
- package/src/shard-recipes/sub-venue.ts +11 -9
- package/src/shard-recipes/trailing-region.ts +183 -0
- package/src/shard-recipes/unit.ts +8 -17
- package/src/{synthesize-boundary-stress.ts → synthesizers/boundary-stress.ts} +2 -2
- package/src/{synthesize-german.ts → synthesizers/german.ts} +3 -2
- package/src/{synthesize-house-venue.ts → synthesizers/house-venue.ts} +20 -6
- package/src/{synthesize-intersection.ts → synthesizers/intersection.ts} +1 -1
- package/src/{synthesize-no-street.ts → synthesizers/no-street.ts} +2 -2
- package/src/{synthesize-po-box.ts → synthesizers/po-box.ts} +1 -1
- package/src/{synthesize-street.ts → synthesizers/street.ts} +5 -3
- package/src/{synthesize.ts → synthesizers/utils.ts} +6 -4
- package/src/tools/align-shard.ts +3 -2
- package/src/tools/fetch/ban.ts +2 -22
- package/src/tools/fetch/download.ts +22 -4
- package/src/tools/fetch/geonames-dump.ts +267 -0
- package/src/tools/fetch/geonames-postal.ts +180 -0
- package/src/tools/fetch/imls-pls.ts +6 -20
- package/src/tools/fetch/index.ts +11 -0
- package/src/tools/fetch/nad.ts +21 -10
- package/src/tools/fetch/nppes.ts +17 -23
- package/src/tools/fetch/openaddresses.ts +18 -22
- package/src/tools/fetch/ourairports.ts +7 -2
- package/src/tools/fetch/ppd.ts +1 -1
- package/src/tools/fetch/state-hi-schools.ts +3 -23
- package/src/tools/fetch/state-sources.ts +1 -1
- package/src/tools/fetch/tiger-full.ts +17 -24
- package/src/tools/golden-expand.ts +44 -33
- package/src/tools/golden-relabel-street.ts +5 -68
- package/src/tools/index.ts +2 -2
- package/src/tools/ingest-csv.ts +0 -13
- package/src/tools/overlay-manifest.ts +20 -8
- package/src/tools/postcode-triples.ts +375 -0
- package/src/tools/shard-kryptonite.ts +4 -5
- package/src/tools/shard-translit.ts +16 -35
- package/src/tools/sub-venue/harvest.ts +235 -0
- package/src/tools/sub-venue/head-nouns.ts +242 -0
- package/src/tools/sub-venue/surfaces.ts +95 -0
- package/src/tools/sub-venue/table.ts +281 -0
- package/src/tools/sub-venue/wikidata.ts +85 -0
- package/src/tools/sub-venue-lexicon.ts +67 -886
- package/src/{align.ts → utils/align.ts} +2 -1
- package/src/{golden.ts → utils/golden.ts} +2 -3
- package/src/utils/index.ts +14 -0
- package/src/{license.ts → utils/license.ts} +1 -1
- package/src/{parquet.ts → utils/parquet.ts} +22 -9
- package/src/{split.ts → utils/split.ts} +3 -3
- package/out/src/adapter.d.ts.map +0 -1
- package/out/src/adapter.js.map +0 -1
- package/out/src/align.d.ts.map +0 -1
- package/out/src/align.js.map +0 -1
- package/out/src/format.d.ts +0 -14
- package/out/src/format.d.ts.map +0 -1
- package/out/src/format.js +0 -14
- package/out/src/format.js.map +0 -1
- package/out/src/golden.d.ts.map +0 -1
- package/out/src/golden.js.map +0 -1
- package/out/src/license.d.ts.map +0 -1
- package/out/src/license.js.map +0 -1
- package/out/src/parquet.d.ts.map +0 -1
- package/out/src/parquet.js.map +0 -1
- package/out/src/split.d.ts.map +0 -1
- package/out/src/split.js.map +0 -1
- package/out/src/synthesize-anchor-absorption.d.ts.map +0 -1
- package/out/src/synthesize-anchor-absorption.js.map +0 -1
- package/out/src/synthesize-boundary-stress.d.ts.map +0 -1
- package/out/src/synthesize-boundary-stress.js.map +0 -1
- package/out/src/synthesize-german.d.ts.map +0 -1
- package/out/src/synthesize-german.js.map +0 -1
- package/out/src/synthesize-house-venue.d.ts.map +0 -1
- package/out/src/synthesize-house-venue.js.map +0 -1
- package/out/src/synthesize-intersection.d.ts.map +0 -1
- package/out/src/synthesize-intersection.js.map +0 -1
- package/out/src/synthesize-no-street.d.ts.map +0 -1
- package/out/src/synthesize-no-street.js.map +0 -1
- package/out/src/synthesize-po-box.d.ts.map +0 -1
- package/out/src/synthesize-po-box.js.map +0 -1
- package/out/src/synthesize-street.d.ts.map +0 -1
- package/out/src/synthesize-street.js.map +0 -1
- package/out/src/synthesize.d.ts.map +0 -1
- package/out/src/synthesize.js.map +0 -1
- package/out/src/tokenize.d.ts.map +0 -1
- package/out/src/tokenize.js.map +0 -1
- package/out/src/wof-json.d.ts.map +0 -1
- package/out/src/wof-json.js.map +0 -1
- package/src/format.ts +0 -14
- /package/out/src/{align.js → utils/align.js} +0 -0
- /package/out/src/{golden.d.ts → utils/golden.d.ts} +0 -0
- /package/out/src/{license.d.ts → utils/license.d.ts} +0 -0
- /package/out/src/{tokenize.d.ts → utils/tokenize.d.ts} +0 -0
- /package/out/src/{tokenize.js → utils/tokenize.js} +0 -0
- /package/out/src/{wof-json.d.ts → utils/wof-json.d.ts} +0 -0
- /package/out/src/{wof-json.js → utils/wof-json.js} +0 -0
- /package/src/{synthesize-anchor-absorption.ts → synthesizers/anchor-absorption.ts} +0 -0
- /package/src/{tokenize.ts → utils/tokenize.ts} +0 -0
- /package/src/{wof-json.ts → utils/wof-json.ts} +0 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"align-shard.js","sourceRoot":"","sources":["../../../src/tools/align-shard.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAE3C
|
|
1
|
+
{"version":3,"file":"align-shard.js","sourceRoot":"","sources":["../../../src/tools/align-shard.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAE3C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AAQjC,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IAAuB;IAChE,+DAA+D;IAC/D,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IACtE,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,iBAAiB,GAA2B,EAAE,CAAA;IAEpD,IAAI,KAAK,EAAE,MAAM,SAAS,IAAI,eAAe,CAAC,SAAS,CAAiC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrG,4FAA4F;QAC5F,SAAS,CAAC,cAAc,GAAG,IAAI,CAAC,aAAa,CAAA;QAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAA;QAElC,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAA;YAElD,OAAO,EAAE,CAAA;QACV,CAAC;aAAM,CAAC;YACP,WAAW,EAAE,CAAA;YACb,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,SAAS,CAAA;YACxC,iBAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACvD,CAAC;IACF,CAAC;IAED,MAAM,IAAI,OAAO,CAAO,CAAC,GAAG,EAAE,EAAE;QAC/B,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACnB,CAAC,CAAC,CAAA;IAEF,OAAO,CAAC,KAAK,CACZ,0BAA0B,OAAO,aAAa,WAAW,kBAAkB,IAAI,CAAC,MAAM,IAAI;QACzF,yBAAyB,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAAE,CAC7D,CAAA;AACF,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ban.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/ban.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;
|
|
1
|
+
{"version":3,"file":"ban.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/ban.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAWH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAsHnE,MAAM,WAAW,gBAAgB;IAChC,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;CACb;AAED,MAAM,MAAM,eAAe,GAAG,gBAAgB,CAAA;AAE9C,wBAAsB,QAAQ,CAAC,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAmI/G"}
|
|
@@ -22,13 +22,13 @@ import { readFile, unlink, writeFile } from "node:fs/promises";
|
|
|
22
22
|
import { join } from "node:path";
|
|
23
23
|
import { setTimeout as sleep } from "node:timers/promises";
|
|
24
24
|
import { gunzipSync } from "node:zlib";
|
|
25
|
+
import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/utils";
|
|
25
26
|
import { sha256File } from "@mailwoman/core/utils";
|
|
26
27
|
import { downloadToFile, loadManifestEntries, writeManifest } from "./download.js";
|
|
27
28
|
/**
|
|
28
29
|
* Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
|
|
29
30
|
* than data.
|
|
30
31
|
*/
|
|
31
|
-
const BYTES_PER_KIB = 1024;
|
|
32
32
|
const BASE_URL = "https://adresse.data.gouv.fr/data/ban/adresses/latest/csv";
|
|
33
33
|
/**
|
|
34
34
|
* All département codes — metropolitan 01-95 (with 2A/2B for Corsica instead of 20) plus overseas DOM/TOM. Codes do not
|
|
@@ -137,22 +137,6 @@ const DEPT_CODES = [
|
|
|
137
137
|
"974",
|
|
138
138
|
"976",
|
|
139
139
|
];
|
|
140
|
-
/**
|
|
141
|
-
* Mimic `numfmt --to=iec` for a friendly byte-size log line.
|
|
142
|
-
*/
|
|
143
|
-
function iec(bytes) {
|
|
144
|
-
if (bytes < BYTES_PER_KIB)
|
|
145
|
-
return String(bytes);
|
|
146
|
-
const units = ["K", "M", "G", "T", "P"];
|
|
147
|
-
let value = bytes / 1024;
|
|
148
|
-
let i = 0;
|
|
149
|
-
while (value >= BYTES_PER_KIB && i < units.length - 1) {
|
|
150
|
-
value /= 1024;
|
|
151
|
-
i++;
|
|
152
|
-
}
|
|
153
|
-
const rounded = value < 10 ? value.toFixed(1) : Math.round(value).toString();
|
|
154
|
-
return `${rounded}${units[i] ?? ""}`;
|
|
155
|
-
}
|
|
156
140
|
export async function fetchBan(options, report) {
|
|
157
141
|
const banDir = join(options.outRoot, "ban");
|
|
158
142
|
const manifestPath = join(banDir, "MANIFEST.json");
|
|
@@ -234,7 +218,7 @@ export async function fetchBan(options, report) {
|
|
|
234
218
|
sha256: sha,
|
|
235
219
|
bytes,
|
|
236
220
|
});
|
|
237
|
-
report?.(` ✓ ${
|
|
221
|
+
report?.(` ✓ ${ByteFormatter.formatIEC(bytes)} sha256=${sha}`);
|
|
238
222
|
fetched++;
|
|
239
223
|
// Be a polite citizen — short pause between requests.
|
|
240
224
|
await sleep(200);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ban.js","sourceRoot":"","sources":["../../../../src/tools/fetch/ban.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC9D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AAEtC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAElF;;;GAGG;
|
|
1
|
+
{"version":3,"file":"ban.js","sourceRoot":"","sources":["../../../../src/tools/fetch/ban.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC9D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AAEtC,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAA;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAElF;;;GAGG;AAEH,MAAM,QAAQ,GAAG,2DAA2D,CAAA;AAE5E;;;GAGG;AACH,MAAM,UAAU,GAAG;IAClB,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;CACL,CAAA;AAaD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAwB,EAAE,MAA+B;IACvF,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;IAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAA;IAClD,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEtC,2FAA2F;IAC3F,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAmB,YAAY,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAErG,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,YAAY,IAAI,MAAM,CAAA;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,KAAK,CAAC,CAAA;QAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;QACtC,MAAM,GAAG,GAAG,GAAG,QAAQ,aAAa,IAAI,SAAS,CAAA;QAEjD,MAAM,EAAE,CAAC,YAAY,IAAI,EAAE,CAAC,CAAA;QAE5B,sEAAsE;QACtE,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACzB,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;YAC7C,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;YAE7C,IAAI,WAAW,IAAI,WAAW,KAAK,WAAW,EAAE,CAAC;gBAChD,MAAM,EAAE,CAAC,8CAA8C,CAAC,CAAA;gBAExD,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,MAAM,EAAE,CAAC,iEAAiE,CAAC,CAAA;YAC3E,MAAM,MAAM,CAAC,OAAO,CAAC,CAAA;QACtB,CAAC;QAED,4BAA4B;QAC5B,IAAI,CAAC;YACJ,MAAM,cAAc,CAAC;gBACpB,GAAG;gBACH,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,OAAO;gBAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;gBAC1C,MAAM;aACN,CAAC,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,wBAAwB,GAAG,KAAM,KAAe,CAAC,OAAO,GAAG,CAAC,CAAA;YAErE,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,2CAA2C;QAC3C,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAA;QAEpC,IAAI,MAAM,GAAG,aAAa,EAAE,CAAC;YAC5B,MAAM,EAAE,CAAC,2BAA2B,MAAM,qCAAqC,CAAC,CAAA;YAChF,MAAM,MAAM,CAAC,MAAM,CAAC,CAAA;YAEpB,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,uCAAuC;QACvC,IAAI,CAAC;YACJ,MAAM,SAAS,CAAC,OAAO,EAAE,UAAU,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;QAC7D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,0BAA2B,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YAC9D,MAAM,MAAM,CAAC,MAAM,CAAC,CAAA;YAEpB,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,CAAC,MAAM,CAAC,CAAA;QAEpB,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1B,MAAM,EAAE,CAAC,sCAAsC,OAAO,EAAE,CAAC,CAAA;YAEzD,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAA;QACpC,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;QAErC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE;YACjB,SAAS,EAAE,IAAI;YACf,QAAQ;YACR,UAAU,EAAE,GAAG;YACf,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,MAAM,EAAE,GAAG;YACX,KAAK;SACL,CAAC,CAAA;QAEF,MAAM,EAAE,CAAC,OAAO,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,YAAY,GAAG,EAAE,CAAC,CAAA;QAEhE,OAAO,EAAE,CAAA;QAET,sDAAsD;QACtD,MAAM,KAAK,CAAC,GAAG,CAAC,CAAA;IACjB,CAAC;IAED,uFAAuF;IACvF,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACtD,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAClE,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IACzC,MAAM,EAAE,CAAC,SAAS,YAAY,SAAS,MAAM,CAAC,MAAM,WAAW,CAAC,CAAA;IAEhE,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,aAAa,OAAO,EAAE,CAAC,CAAA;IAChC,MAAM,EAAE,CAAC,aAAa,OAAO,kCAAkC,CAAC,CAAA;IAChE,MAAM,EAAE,CAAC,aAAa,MAAM,EAAE,CAAC,CAAA;IAE/B,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,EAAE,CAAC,iBAAiB,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IACnD,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACjD,CAAC"}
|
|
@@ -3,9 +3,7 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
* Download
|
|
7
|
-
* one MANIFEST.json idiom instead of the per-script clones the 2026-07-09 dedupe survey counted
|
|
8
|
-
* (6× `downloadToFile`, 2× `isTransientStatus`, 9× manifest writes).
|
|
6
|
+
* Download and manifest utilities for `mailwoman corpus fetch <source>`.
|
|
9
7
|
*/
|
|
10
8
|
/**
|
|
11
9
|
* The option base every `mailwoman corpus fetch <source>` module extends.
|
|
@@ -28,6 +26,17 @@ export interface FetchSummary {
|
|
|
28
26
|
/**
|
|
29
27
|
* A status worth retrying: rate limiting or a server-side failure.
|
|
30
28
|
*/
|
|
29
|
+
/**
|
|
30
|
+
* An HTTP failure that CARRIES its status, so callers branch on `error.status` rather than on message prose. The prose
|
|
31
|
+
* route shipped a real flake: a caller classified "not published upstream" with `message.includes("404")`, and the
|
|
32
|
+
* message contains the URL — an ephemeral test-server port such as `:40453` satisfies it while the actual status is
|
|
33
|
+
* 500. Roughly 1–2% of ephemeral ports contain the substring, which is exactly the kind of sometimes-failure that burns
|
|
34
|
+
* a CI run and vanishes locally.
|
|
35
|
+
*/
|
|
36
|
+
export declare class HTTPStatusError extends Error {
|
|
37
|
+
readonly status: number;
|
|
38
|
+
constructor(status: number, message: string);
|
|
39
|
+
}
|
|
31
40
|
export declare function isTransientStatus(status: number): boolean;
|
|
32
41
|
export interface DownloadOptions {
|
|
33
42
|
url: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/download.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/download.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAuBH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,EAAE,CAAA;CACrB;AAED;;GAEG;AACH;;;;;;GAMG;AACH,qBAAa,eAAgB,SAAQ,KAAK;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;gBAEX,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;CAK3C;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEzD;AAED,MAAM,WAAW,eAAe;IAC/B,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA6CzF;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAQrE;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAS7G;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAElF"}
|
|
@@ -3,9 +3,7 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
* Download
|
|
7
|
-
* one MANIFEST.json idiom instead of the per-script clones the 2026-07-09 dedupe survey counted
|
|
8
|
-
* (6× `downloadToFile`, 2× `isTransientStatus`, 9× manifest writes).
|
|
6
|
+
* Download and manifest utilities for `mailwoman corpus fetch <source>`.
|
|
9
7
|
*/
|
|
10
8
|
import { existsSync } from "node:fs";
|
|
11
9
|
import { readFile, writeFile } from "node:fs/promises";
|
|
@@ -26,6 +24,21 @@ const HTTP_SERVER_ERROR_MAX = 599;
|
|
|
26
24
|
/**
|
|
27
25
|
* A status worth retrying: rate limiting or a server-side failure.
|
|
28
26
|
*/
|
|
27
|
+
/**
|
|
28
|
+
* An HTTP failure that CARRIES its status, so callers branch on `error.status` rather than on message prose. The prose
|
|
29
|
+
* route shipped a real flake: a caller classified "not published upstream" with `message.includes("404")`, and the
|
|
30
|
+
* message contains the URL — an ephemeral test-server port such as `:40453` satisfies it while the actual status is
|
|
31
|
+
* 500. Roughly 1–2% of ephemeral ports contain the substring, which is exactly the kind of sometimes-failure that burns
|
|
32
|
+
* a CI run and vanishes locally.
|
|
33
|
+
*/
|
|
34
|
+
export class HTTPStatusError extends Error {
|
|
35
|
+
status;
|
|
36
|
+
constructor(status, message) {
|
|
37
|
+
super(message);
|
|
38
|
+
this.name = "HTTPStatusError";
|
|
39
|
+
this.status = status;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
29
42
|
export function isTransientStatus(status) {
|
|
30
43
|
return status === HTTP_TOO_MANY_REQUESTS || (status >= HTTP_SERVER_ERROR_MIN && status <= HTTP_SERVER_ERROR_MAX);
|
|
31
44
|
}
|
|
@@ -43,6 +56,9 @@ export async function downloadToFile(options) {
|
|
|
43
56
|
}
|
|
44
57
|
let res;
|
|
45
58
|
try {
|
|
59
|
+
// Raw `fetch`, deliberately: this is the shared FILE downloader and the body is piped to disk below.
|
|
60
|
+
// `APIClient` is the repo default for API requests — small bodies, repeated calls — and buffers a
|
|
61
|
+
// non-stream response in memory, which is the one thing a multi-gigabyte transfer must not do.
|
|
46
62
|
res = await fetch(url, { headers, signal: AbortSignal.timeout(timeoutMs) });
|
|
47
63
|
}
|
|
48
64
|
catch (error) {
|
|
@@ -51,7 +67,7 @@ export async function downloadToFile(options) {
|
|
|
51
67
|
continue;
|
|
52
68
|
}
|
|
53
69
|
if (!res.ok) {
|
|
54
|
-
const error = new
|
|
70
|
+
const error = new HTTPStatusError(res.status, `HTTP ${res.status} ${res.statusText} — ${url}`);
|
|
55
71
|
if (!isTransientStatus(res.status))
|
|
56
72
|
throw error;
|
|
57
73
|
lastError = error;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../../src/tools/fetch/download.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../../src/tools/fetch/download.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE1D,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AAExD;;GAEG;AACH,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAElC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAsBjC;;GAEG;AACH;;;;;;GAMG;AACH,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAChC,MAAM,CAAQ;IAEvB,YAAY,MAAc,EAAE,OAAe;QAC1C,KAAK,CAAC,OAAO,CAAC,CAAA;QACd,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAA;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACrB,CAAC;CACD;AAED,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC/C,OAAO,MAAM,KAAK,sBAAsB,IAAI,CAAC,MAAM,IAAI,qBAAqB,IAAI,MAAM,IAAI,qBAAqB,CAAC,CAAA;AACjH,CAAC;AAqBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAwB;IAC5D,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,GAAG,OAAO,EAAE,OAAO,GAAG,CAAC,EAAE,YAAY,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IACrG,IAAI,SAAkB,CAAA;IAEtB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACrD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YACjB,MAAM,EAAE,CAAC,SAAS,OAAO,IAAI,OAAO,UAAU,YAAY,QAAQ,GAAG,EAAE,CAAC,CAAA;YACxE,MAAM,KAAK,CAAC,YAAY,CAAC,CAAA;QAC1B,CAAC;QAED,IAAI,GAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,qGAAqG;YACrG,kGAAkG;YAClG,+FAA+F;YAC/F,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;QAC5E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,iEAAiE;YACjE,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,MAAM,GAAG,EAAE,CAAC,CAAA;YAE9F,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,MAAM,KAAK,CAAA;YAC/C,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,CAAA;YACnD,MAAM,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;YAE7B,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,CAAA;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,8DAA8D;YAC9D,SAAS,GAAG,KAAK,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAA;AAC5E,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAI,IAAY;IACjD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IAElC,+FAA+F;IAC/F,sEAAsE;IACtE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;IAE3D,OAAO,cAAc,CAAI,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAI,IAAY,EAAE,GAAyB;IACnF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAa,CAAA;IACpC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAM,IAAI,CAAC,CAAA;IAE5C,KAAK,MAAM,KAAK,IAAI,MAAM,IAAI,EAAE,EAAE,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAA;IAC/B,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,QAAiB;IAClE,MAAM,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;AAChE,CAAC"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Fetch GeoNames per-country GAZETTEER dumps — the 19-column `<CC>.txt` files under
|
|
7
|
+
* `https://download.geonames.org/export/dump/` (NOT the postal exports; those are `export/zip/` and
|
|
8
|
+
* `geonames-postal.ts`'s job). The dumps carry feature classes and codes (column 8: `PPLC` national capital,
|
|
9
|
+
* `PPLA` first-order administrative seat), which is what the capitals reference build consumes (#1880).
|
|
10
|
+
*
|
|
11
|
+
* The catalog question is answered by the SOURCE, not by an ISO list: `countryInfo.txt` in the same directory
|
|
12
|
+
* enumerates every country GeoNames publishes, one row per ISO alpha-2 code, and also names each country's
|
|
13
|
+
* capital — the cross-check the capitals build grades its `PPLC` extraction against. Fetch that first; derive
|
|
14
|
+
* the country set from it; then a dump absent from disk is a measured gap against the source's own catalog
|
|
15
|
+
* rather than a silent hole. The dump directory may hold files this tool did not fetch: present files are never
|
|
16
|
+
* overwritten, and a present `<CC>.txt` that is NOT a 19-column gazetteer dump (GeoNames' postal exports share
|
|
17
|
+
* the basename) is reported as `wrong_format_present`, never counted as coverage.
|
|
18
|
+
*/
|
|
19
|
+
import type { BaseFetchOptions, FetchSummary } from "./download.ts";
|
|
20
|
+
export interface FetchGeonamesDumpOptions extends BaseFetchOptions {
|
|
21
|
+
/**
|
|
22
|
+
* ISO alpha-2 codes, any casing. Absent → every country `countryInfo.txt` enumerates.
|
|
23
|
+
*/
|
|
24
|
+
countries?: readonly string[];
|
|
25
|
+
/**
|
|
26
|
+
* Dump directory to read from. Defaults to GeoNames' own; exists so the 404 and coverage behaviour can be exercised
|
|
27
|
+
* against a local server.
|
|
28
|
+
*/
|
|
29
|
+
baseURL?: string;
|
|
30
|
+
/**
|
|
31
|
+
* Refetch a dump whose `<CC>.txt` already exists. Default false — the tool fills gaps.
|
|
32
|
+
*/
|
|
33
|
+
force?: boolean;
|
|
34
|
+
}
|
|
35
|
+
interface GeonamesDumpFileEntry {
|
|
36
|
+
country: string;
|
|
37
|
+
filename: string;
|
|
38
|
+
source_url: string;
|
|
39
|
+
sha256: string;
|
|
40
|
+
bytes: number;
|
|
41
|
+
}
|
|
42
|
+
export interface GeonamesDumpManifest {
|
|
43
|
+
source: string;
|
|
44
|
+
base_url: string;
|
|
45
|
+
license: string;
|
|
46
|
+
attribution: string;
|
|
47
|
+
downloaded_at: string;
|
|
48
|
+
files: GeonamesDumpFileEntry[];
|
|
49
|
+
/**
|
|
50
|
+
* `<CC>.txt` files already on disk and left alone — the hand-fetched population this tool extends.
|
|
51
|
+
*/
|
|
52
|
+
skipped_present: string[];
|
|
53
|
+
/**
|
|
54
|
+
* Countries in the source catalog that the source's dump directory nonetheless 404s — a fact about the source,
|
|
55
|
+
* recorded so a later reader does not spend the fetch to rediscover it.
|
|
56
|
+
*/
|
|
57
|
+
unavailable: string[];
|
|
58
|
+
/**
|
|
59
|
+
* Present `<CC>.txt` files that are NOT 19-column gazetteer dumps — GeoNames' postal exports share the same basename,
|
|
60
|
+
* and seven tier-1 postal files sat at these paths reading as "present" until the capitals build found them
|
|
61
|
+
* capital-less. Left in place (this tool never overwrites data it did not fetch); the fix is to move the file to its
|
|
62
|
+
* own home and rerun.
|
|
63
|
+
*/
|
|
64
|
+
wrong_format_present: string[];
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* True when the first non-empty line carries the gazetteer dump's 19 tab-separated columns. Accepts a partial head read
|
|
68
|
+
* — the first line is the whole question, so callers need not hand it a resident 350 MB dump.
|
|
69
|
+
*/
|
|
70
|
+
export declare function looksLikeGazetteerDump(text: string): boolean;
|
|
71
|
+
/**
|
|
72
|
+
* Parse the ISO codes (column 1) and capital names (column 6) out of `countryInfo.txt` — `#`-prefixed lines are the
|
|
73
|
+
* file's own commentary.
|
|
74
|
+
*/
|
|
75
|
+
export declare function parseCountryInfo(text: string): Array<{
|
|
76
|
+
country: string;
|
|
77
|
+
capital: string;
|
|
78
|
+
}>;
|
|
79
|
+
/**
|
|
80
|
+
* Download `countryInfo.txt` plus every missing `<CC>.zip`, extracting each to `<outRoot>/<CC>.txt` beside the
|
|
81
|
+
* hand-fetched dumps, with a `MANIFEST.json` naming fetched, skipped-present, and source-unavailable countries.
|
|
82
|
+
*/
|
|
83
|
+
export declare function fetchGeonamesDumps(options: FetchGeonamesDumpOptions, report?: (line: string) => void): Promise<FetchSummary & {
|
|
84
|
+
skippedPresent: string[];
|
|
85
|
+
}>;
|
|
86
|
+
export {};
|
|
87
|
+
//# sourceMappingURL=geonames-dump.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"geonames-dump.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/geonames-dump.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AASH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAgBnE,MAAM,WAAW,wBAAyB,SAAQ,gBAAgB;IACjE;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC7B;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;CACf;AAED,UAAU,qBAAqB;IAC9B,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,oBAAoB;IACpC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,CAAA;IACrB,KAAK,EAAE,qBAAqB,EAAE,CAAA;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,EAAE,CAAA;IACzB;;;OAGG;IACH,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB;;;;;OAKG;IACH,oBAAoB,EAAE,MAAM,EAAE,CAAA;CAC9B;AAQD;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAuB5D;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAiB1F;AAqBD;;;GAGG;AACH,wBAAsB,kBAAkB,CACvC,OAAO,EAAE,wBAAwB,EACjC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,GAAG;IAAE,cAAc,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CA2FtD"}
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Fetch GeoNames per-country GAZETTEER dumps — the 19-column `<CC>.txt` files under
|
|
7
|
+
* `https://download.geonames.org/export/dump/` (NOT the postal exports; those are `export/zip/` and
|
|
8
|
+
* `geonames-postal.ts`'s job). The dumps carry feature classes and codes (column 8: `PPLC` national capital,
|
|
9
|
+
* `PPLA` first-order administrative seat), which is what the capitals reference build consumes (#1880).
|
|
10
|
+
*
|
|
11
|
+
* The catalog question is answered by the SOURCE, not by an ISO list: `countryInfo.txt` in the same directory
|
|
12
|
+
* enumerates every country GeoNames publishes, one row per ISO alpha-2 code, and also names each country's
|
|
13
|
+
* capital — the cross-check the capitals build grades its `PPLC` extraction against. Fetch that first; derive
|
|
14
|
+
* the country set from it; then a dump absent from disk is a measured gap against the source's own catalog
|
|
15
|
+
* rather than a silent hole. The dump directory may hold files this tool did not fetch: present files are never
|
|
16
|
+
* overwritten, and a present `<CC>.txt` that is NOT a 19-column gazetteer dump (GeoNames' postal exports share
|
|
17
|
+
* the basename) is reported as `wrong_format_present`, never counted as coverage.
|
|
18
|
+
*/
|
|
19
|
+
import { existsSync, mkdirSync } from "node:fs";
|
|
20
|
+
import { open, readFile, rm } from "node:fs/promises";
|
|
21
|
+
import { join } from "node:path";
|
|
22
|
+
import { extractZipEntry } from "@mailwoman/core/fs/zip";
|
|
23
|
+
import { sha256File } from "@mailwoman/core/utils";
|
|
24
|
+
import { downloadToFile, HTTPStatusError, writeManifest } from "./download.js";
|
|
25
|
+
/**
|
|
26
|
+
* The one status that means "the source does not publish this country" rather than "the transfer failed".
|
|
27
|
+
*/
|
|
28
|
+
const HTTP_NOT_FOUND = 404;
|
|
29
|
+
const SLUG = "geonames-dump";
|
|
30
|
+
/**
|
|
31
|
+
* GeoNames' gazetteer-dump directory — one zip per ISO alpha-2 code holding `<CC>.txt`, plus `countryInfo.txt` as a
|
|
32
|
+
* bare text file.
|
|
33
|
+
*/
|
|
34
|
+
const BASE_URL = "https://download.geonames.org/export/dump";
|
|
35
|
+
/**
|
|
36
|
+
* Column count of a gazetteer dump row — the discriminator against GeoNames' 12-column postal exports, which share the
|
|
37
|
+
* `<CC>.txt` basename.
|
|
38
|
+
*/
|
|
39
|
+
const GAZETTEER_DUMP_COLUMNS = 19;
|
|
40
|
+
/**
|
|
41
|
+
* True when the first non-empty line carries the gazetteer dump's 19 tab-separated columns. Accepts a partial head read
|
|
42
|
+
* — the first line is the whole question, so callers need not hand it a resident 350 MB dump.
|
|
43
|
+
*/
|
|
44
|
+
export function looksLikeGazetteerDump(text) {
|
|
45
|
+
let start = 0;
|
|
46
|
+
while (start < text.length) {
|
|
47
|
+
const end = text.indexOf("\n", start);
|
|
48
|
+
const line = end === -1 ? text.slice(start) : text.slice(start, end);
|
|
49
|
+
if (line.trim()) {
|
|
50
|
+
let tabs = 0;
|
|
51
|
+
for (let i = line.indexOf("\t"); i !== -1; i = line.indexOf("\t", i + 1)) {
|
|
52
|
+
tabs++;
|
|
53
|
+
}
|
|
54
|
+
return tabs === GAZETTEER_DUMP_COLUMNS - 1;
|
|
55
|
+
}
|
|
56
|
+
if (end === -1)
|
|
57
|
+
break;
|
|
58
|
+
start = end + 1;
|
|
59
|
+
}
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Parse the ISO codes (column 1) and capital names (column 6) out of `countryInfo.txt` — `#`-prefixed lines are the
|
|
64
|
+
* file's own commentary.
|
|
65
|
+
*/
|
|
66
|
+
export function parseCountryInfo(text) {
|
|
67
|
+
const rows = [];
|
|
68
|
+
// oxlint-disable-next-line mailwoman/prefer-spliterator -- countryInfo.txt is ~40 KB, one row per country on Earth; it cannot grow past that
|
|
69
|
+
for (const line of text.split("\n")) {
|
|
70
|
+
if (line.startsWith("#") || !line.trim())
|
|
71
|
+
continue;
|
|
72
|
+
// oxlint-disable-next-line mailwoman/prefer-spliterator -- one bounded 19-column catalog row
|
|
73
|
+
const cols = line.split("\t");
|
|
74
|
+
const country = cols[0]?.trim().toUpperCase();
|
|
75
|
+
if (country?.length === 2) {
|
|
76
|
+
rows.push({ country, capital: cols[5]?.trim() ?? "" });
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return rows;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Bytes read to classify a present file: enough to cover a first dump row whose alternate-names column runs long (they
|
|
83
|
+
* reach several KB), a fraction of the largest dumps (US.txt is ~350 MB).
|
|
84
|
+
*/
|
|
85
|
+
const FORMAT_SNIFF_BYTES = 65_536;
|
|
86
|
+
async function readFileHead(path) {
|
|
87
|
+
const handle = await open(path, "r");
|
|
88
|
+
try {
|
|
89
|
+
const buffer = Buffer.alloc(FORMAT_SNIFF_BYTES);
|
|
90
|
+
const { bytesRead } = await handle.read(buffer, 0, FORMAT_SNIFF_BYTES, 0);
|
|
91
|
+
return buffer.subarray(0, bytesRead).toString("utf8");
|
|
92
|
+
}
|
|
93
|
+
finally {
|
|
94
|
+
await handle.close();
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Download `countryInfo.txt` plus every missing `<CC>.zip`, extracting each to `<outRoot>/<CC>.txt` beside the
|
|
99
|
+
* hand-fetched dumps, with a `MANIFEST.json` naming fetched, skipped-present, and source-unavailable countries.
|
|
100
|
+
*/
|
|
101
|
+
export async function fetchGeonamesDumps(options, report) {
|
|
102
|
+
mkdirSync(options.outRoot, { recursive: true });
|
|
103
|
+
const baseURL = options.baseURL ?? BASE_URL;
|
|
104
|
+
const countryInfoDest = join(options.outRoot, "countryInfo.txt");
|
|
105
|
+
await downloadToFile({
|
|
106
|
+
url: `${baseURL}/countryInfo.txt`,
|
|
107
|
+
dest: countryInfoDest,
|
|
108
|
+
timeoutMs: 120_000,
|
|
109
|
+
retries: 2,
|
|
110
|
+
report,
|
|
111
|
+
});
|
|
112
|
+
const catalog = parseCountryInfo(await readFile(countryInfoDest, "utf8"));
|
|
113
|
+
const countries = options.countries?.map((code) => code.trim().toUpperCase()) ?? catalog.map((row) => row.country);
|
|
114
|
+
const entries = [];
|
|
115
|
+
const failedCodes = [];
|
|
116
|
+
const unavailable = [];
|
|
117
|
+
const skippedPresent = [];
|
|
118
|
+
const wrongFormatPresent = [];
|
|
119
|
+
let fetched = 0;
|
|
120
|
+
for (const country of countries) {
|
|
121
|
+
const txtDest = join(options.outRoot, `${country}.txt`);
|
|
122
|
+
if (!options.force && existsSync(txtDest)) {
|
|
123
|
+
if (looksLikeGazetteerDump(await readFileHead(txtDest))) {
|
|
124
|
+
skippedPresent.push(country);
|
|
125
|
+
}
|
|
126
|
+
else {
|
|
127
|
+
report?.(`✗ ${country}.txt is present but is not a 19-column gazetteer dump — move it aside and rerun`);
|
|
128
|
+
wrongFormatPresent.push(country);
|
|
129
|
+
}
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
const filename = `${country}.zip`;
|
|
133
|
+
const url = `${baseURL}/${filename}`;
|
|
134
|
+
const zipDest = join(options.outRoot, filename);
|
|
135
|
+
report?.(`=== ${SLUG} / ${country}`);
|
|
136
|
+
try {
|
|
137
|
+
await downloadToFile({ url, dest: zipDest, timeoutMs: 300_000, retries: 2, report });
|
|
138
|
+
await extractZipEntry(zipDest, `${country}.txt`, txtDest);
|
|
139
|
+
await rm(zipDest, { force: true });
|
|
140
|
+
entries.push({
|
|
141
|
+
country,
|
|
142
|
+
filename: `${country}.txt`,
|
|
143
|
+
source_url: url,
|
|
144
|
+
sha256: await sha256File(txtDest),
|
|
145
|
+
bytes: (await readFile(txtDest)).byteLength,
|
|
146
|
+
});
|
|
147
|
+
fetched++;
|
|
148
|
+
}
|
|
149
|
+
catch (error) {
|
|
150
|
+
await rm(zipDest, { force: true });
|
|
151
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
152
|
+
// Branch on the TYPED status (the geonames-postal lesson): message prose contains the URL, and a URL
|
|
153
|
+
// can contain any substring.
|
|
154
|
+
if (error instanceof HTTPStatusError && error.status === HTTP_NOT_FOUND) {
|
|
155
|
+
report?.(`✗ ${country}: GeoNames publishes no gazetteer dump for this country`);
|
|
156
|
+
unavailable.push(country);
|
|
157
|
+
}
|
|
158
|
+
else {
|
|
159
|
+
report?.(`✗ ${country}: ${message}`);
|
|
160
|
+
}
|
|
161
|
+
failedCodes.push(country);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
const manifest = {
|
|
165
|
+
source: SLUG,
|
|
166
|
+
base_url: baseURL,
|
|
167
|
+
license: "CC-BY-4.0",
|
|
168
|
+
attribution: "GeoNames",
|
|
169
|
+
downloaded_at: new Date().toISOString(),
|
|
170
|
+
files: entries,
|
|
171
|
+
skipped_present: skippedPresent.toSorted(),
|
|
172
|
+
unavailable,
|
|
173
|
+
wrong_format_present: wrongFormatPresent.toSorted(),
|
|
174
|
+
};
|
|
175
|
+
await writeManifest(join(options.outRoot, "MANIFEST.json"), manifest);
|
|
176
|
+
return { fetched, skipped: skippedPresent.length, failed: failedCodes.length, failedCodes, skippedPresent };
|
|
177
|
+
}
|
|
178
|
+
//# sourceMappingURL=geonames-dump.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"geonames-dump.js","sourceRoot":"","sources":["../../../../src/tools/fetch/geonames-dump.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AAC/C,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAA;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE9E;;GAEG;AACH,MAAM,cAAc,GAAG,GAAG,CAAA;AAE1B,MAAM,IAAI,GAAG,eAAe,CAAA;AAE5B;;;GAGG;AACH,MAAM,QAAQ,GAAG,2CAA2C,CAAA;AAmD5D;;;GAGG;AACH,MAAM,sBAAsB,GAAG,EAAE,CAAA;AAEjC;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IAClD,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACrC,MAAM,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;QAEpE,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACjB,IAAI,IAAI,GAAG,CAAC,CAAA;YAEZ,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC1E,IAAI,EAAE,CAAA;YACP,CAAC;YAED,OAAO,IAAI,KAAK,sBAAsB,GAAG,CAAC,CAAA;QAC3C,CAAC;QAED,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,MAAK;QAErB,KAAK,GAAG,GAAG,GAAG,CAAC,CAAA;IAChB,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC5C,MAAM,IAAI,GAAgD,EAAE,CAAA;IAE5D,6IAA6I;IAC7I,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,SAAQ;QAElD,6FAA6F;QAC7F,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;QAE7C,IAAI,OAAO,EAAE,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;QACvD,CAAC;IACF,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,kBAAkB,GAAG,MAAM,CAAA;AAEjC,KAAK,UAAU,YAAY,CAAC,IAAY;IACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IAEpC,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAA;QAC/C,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAA;QAEzE,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;IACtD,CAAC;YAAS,CAAC;QACV,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IACrB,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,OAAiC,EACjC,MAA+B;IAE/B,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE/C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAA;IAC3C,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAA;IAEhE,MAAM,cAAc,CAAC;QACpB,GAAG,EAAE,GAAG,OAAO,kBAAkB;QACjC,IAAI,EAAE,eAAe;QACrB,SAAS,EAAE,OAAO;QAClB,OAAO,EAAE,CAAC;QACV,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,OAAO,GAAG,gBAAgB,CAAC,MAAM,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC,CAAA;IACzE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAElH,MAAM,OAAO,GAA4B,EAAE,CAAA;IAC3C,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,MAAM,cAAc,GAAa,EAAE,CAAA;IACnC,MAAM,kBAAkB,GAAa,EAAE,CAAA;IACvC,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,KAAK,MAAM,OAAO,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,OAAO,MAAM,CAAC,CAAA;QAEvD,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3C,IAAI,sBAAsB,CAAC,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;gBACzD,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAC7B,CAAC;iBAAM,CAAC;gBACP,MAAM,EAAE,CAAC,KAAK,OAAO,iFAAiF,CAAC,CAAA;gBACvG,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACjC,CAAC;YAED,SAAQ;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,GAAG,OAAO,MAAM,CAAA;QACjC,MAAM,GAAG,GAAG,GAAG,OAAO,IAAI,QAAQ,EAAE,CAAA;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAE/C,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,OAAO,EAAE,CAAC,CAAA;QAEpC,IAAI,CAAC;YACJ,MAAM,cAAc,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAA;YACpF,MAAM,eAAe,CAAC,OAAO,EAAE,GAAG,OAAO,MAAM,EAAE,OAAO,CAAC,CAAA;YACzD,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;YAElC,OAAO,CAAC,IAAI,CAAC;gBACZ,OAAO;gBACP,QAAQ,EAAE,GAAG,OAAO,MAAM;gBAC1B,UAAU,EAAE,GAAG;gBACf,MAAM,EAAE,MAAM,UAAU,CAAC,OAAO,CAAC;gBACjC,KAAK,EAAE,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU;aAC3C,CAAC,CAAA;YAEF,OAAO,EAAE,CAAA;QACV,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;YAElC,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAEtE,qGAAqG;YACrG,6BAA6B;YAC7B,IAAI,KAAK,YAAY,eAAe,IAAI,KAAK,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;gBACzE,MAAM,EAAE,CAAC,KAAK,OAAO,yDAAyD,CAAC,CAAA;gBAC/E,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAC1B,CAAC;iBAAM,CAAC;gBACP,MAAM,EAAE,CAAC,KAAK,OAAO,KAAK,OAAO,EAAE,CAAC,CAAA;YACrC,CAAC;YAED,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC1B,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAyB;QACtC,MAAM,EAAE,IAAI;QACZ,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,WAAW;QACpB,WAAW,EAAE,UAAU;QACvB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,KAAK,EAAE,OAAO;QACd,eAAe,EAAE,cAAc,CAAC,QAAQ,EAAE;QAC1C,WAAW;QACX,oBAAoB,EAAE,kBAAkB,CAAC,QAAQ,EAAE;KACnD,CAAA;IAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAErE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,CAAA;AAC5G,CAAC"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Fetch the GeoNames per-country postal-code exports — the only source in this family that carries a
|
|
7
|
+
* `(postcode, locality, region)` triple with the NAMES inline.
|
|
8
|
+
*
|
|
9
|
+
* Source : https://download.geonames.org/export/zip/<CC>.zip
|
|
10
|
+
* License: CC-BY-4.0, attribute "GeoNames". Tier B.
|
|
11
|
+
*
|
|
12
|
+
* ## Why this source and not a join
|
|
13
|
+
*
|
|
14
|
+
* A postcode shard is only useful to the corpus if a postcode reaches a locality and a region. Two routes exist and
|
|
15
|
+
* only one of them works everywhere:
|
|
16
|
+
*
|
|
17
|
+
* - **`parent_id`** — `postalcode-intl.db` carries a real parent that resolves in the admin gazetteer (measured: NL
|
|
18
|
+
* 97.5% of rows linked, FR 90.7%, DE 66.1%, ES 34.9%, IT 27.4%, and 93.8–100% of those resolve to a
|
|
19
|
+
* `locality`/`localadmin`). It is also the ONLY shard that does: `postalcode-geonames-intl.db` and every
|
|
20
|
+
* `postalcode-<cc>-overture.db` carry `parent_id = 0` on every row, so five countries have this route and the rest
|
|
21
|
+
* have none.
|
|
22
|
+
* - **Nearest locality centroid** — the obvious fallback, and it does not work. Scored against the `parent_id` truth
|
|
23
|
+
* on 1,000–1,500 postcodes per country: NL 81.1%, DE 44.7%, ES 35.1%, FR 29.0%, IT 13.5%. A locality's centroid
|
|
24
|
+
* sits at its middle, so a postcode near the edge is routinely closer to a neighbouring town's centroid than to its
|
|
25
|
+
* own. Under 50% for four of the five, which is not a join.
|
|
26
|
+
*
|
|
27
|
+
* The GeoNames export sidesteps both: columns 3 and 4 ARE the place and admin1 names, so there is nothing to join and
|
|
28
|
+
* nothing to approximate. `@mailwoman/corpus`'s `geonames-postal` adapter consumes it directly.
|
|
29
|
+
*
|
|
30
|
+
* ## Coverage is not universal, and the gap is the point
|
|
31
|
+
*
|
|
32
|
+
* GeoNames publishes ~80 countries, NOT all of them. Venezuela returns 404 — so a VE postcode shard cannot be built
|
|
33
|
+
* from this source at any effort, and that is an acquisition question rather than a build one. Ask for a country
|
|
34
|
+
* before assuming it is there; an absent country fails as one entry, never as the whole run.
|
|
35
|
+
*
|
|
36
|
+
* ## Row counts from this source overstate, for some countries by exactly 2×
|
|
37
|
+
*
|
|
38
|
+
* Countries whose postcode format contains a hyphen are published TWICE — once `3750-000`, once `3750000`. Measured
|
|
39
|
+
* on the shard built from this source: PT 395,544 rows over 197,772 distinct codes and PL 40,598 over 20,299, both
|
|
40
|
+
* exactly 2.00×, while AU, CZ and AT (no hyphen in the format) are 1.00×. A consumer sizing a shard from the row
|
|
41
|
+
* count doubles its estimate for those countries.
|
|
42
|
+
*
|
|
43
|
+
* ## Why `downloadToFile` and not `APIClient`
|
|
44
|
+
*
|
|
45
|
+
* Same split `AGENTS.md` draws and the `ourairports` sibling explains: these are static file transfers from a plain
|
|
46
|
+
* file host, run once per refresh. The pacing, retry and caching `APIClient` exists for have nothing to act on here.
|
|
47
|
+
*
|
|
48
|
+
* Invoke via `mailwoman corpus fetch geonames-postal --countries pt,au,nz`.
|
|
49
|
+
*/
|
|
50
|
+
import type { BaseFetchOptions, FetchSummary } from "./download.ts";
|
|
51
|
+
/**
|
|
52
|
+
* Countries fetched when the caller names none.
|
|
53
|
+
*
|
|
54
|
+
* These are the ones the corpus wants and cannot get from `postalcode-intl.db`'s `parent_id` route — see the header for
|
|
55
|
+
* why that route covers exactly five countries. Venezuela is deliberately absent because GeoNames does not publish it.
|
|
56
|
+
*/
|
|
57
|
+
export declare const GEONAMES_POSTAL_DEFAULT_COUNTRIES: readonly ["PT", "AU", "NZ", "IE", "BR", "ZA", "MX"];
|
|
58
|
+
export interface FetchGeonamesPostalOptions extends BaseFetchOptions {
|
|
59
|
+
/**
|
|
60
|
+
* ISO alpha-2 codes, in any casing. Defaults to {@linkcode GEONAMES_POSTAL_DEFAULT_COUNTRIES}.
|
|
61
|
+
*/
|
|
62
|
+
countries?: readonly string[];
|
|
63
|
+
/**
|
|
64
|
+
* Export directory to read from. Defaults to GeoNames' own. Exists so the 404-is-a-coverage-finding behaviour can be
|
|
65
|
+
* exercised against a local server rather than by asking GeoNames for a country it does not have.
|
|
66
|
+
*/
|
|
67
|
+
baseURL?: string;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Download the requested GeoNames postal zips into `<outRoot>/geonames-postal/`, with a sibling `MANIFEST.json`
|
|
71
|
+
* carrying each file's origin URL, sha256 and byte count, plus the countries the source does not publish.
|
|
72
|
+
*
|
|
73
|
+
* A country the source does not carry is counted as failed and named in `failedCodes` — it does not stop the rest.
|
|
74
|
+
*/
|
|
75
|
+
export declare function fetchGeonamesPostal(options: FetchGeonamesPostalOptions, report?: (line: string) => void): Promise<FetchSummary>;
|
|
76
|
+
//# sourceMappingURL=geonames-postal.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"geonames-postal.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/geonames-postal.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AAOH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAenE;;;;;GAKG;AACH,eAAO,MAAM,iCAAiC,qDAAsD,CAAA;AAEpG,MAAM,WAAW,0BAA2B,SAAQ,gBAAgB;IACnE;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC7B;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB;AAwBD;;;;;GAKG;AACH,wBAAsB,mBAAmB,CACxC,OAAO,EAAE,0BAA0B,EACnC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA0DvB"}
|