@mailwoman/corpus 8.0.0 → 8.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/adapter.d.ts +15 -5
- package/out/src/adapter.d.ts.map +1 -1
- package/out/src/adapter.js +11 -9
- package/out/src/adapter.js.map +1 -1
- package/out/src/adapters/ban/adapter.d.ts +7 -0
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +9 -2
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +5 -1
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +14 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +15 -2
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +11 -0
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +24 -5
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +11 -0
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +14 -1
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +10 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +11 -2
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts +18 -6
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +16 -6
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +11 -0
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +18 -3
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +11 -0
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +12 -1
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +11 -0
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +11 -0
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +15 -2
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +11 -0
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +13 -2
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +14 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +19 -4
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +12 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +17 -2
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +12 -1
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +11 -0
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +15 -2
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +7 -0
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +18 -3
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +13 -3
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +15 -5
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +7 -0
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +9 -2
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/align.d.ts +12 -4
- package/out/src/align.d.ts.map +1 -1
- package/out/src/align.js +9 -5
- package/out/src/align.js.map +1 -1
- package/out/src/build.d.ts +24 -8
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +3 -3
- package/out/src/build.js.map +1 -1
- package/out/src/codex/us-fips-state.d.ts +6 -2
- package/out/src/codex/us-fips-state.d.ts.map +1 -1
- package/out/src/codex/us-fips-state.js +3 -1
- package/out/src/codex/us-fips-state.js.map +1 -1
- package/out/src/golden.d.ts +21 -7
- package/out/src/golden.d.ts.map +1 -1
- package/out/src/golden.js +18 -10
- package/out/src/golden.js.map +1 -1
- package/out/src/license.d.ts +3 -1
- package/out/src/license.d.ts.map +1 -1
- package/out/src/license.js +4 -2
- package/out/src/license.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts +4 -2
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +5 -4
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/schema.d.ts +14 -10
- package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
- package/out/src/parquet-wrapper/schema.js +6 -2
- package/out/src/parquet-wrapper/schema.js.map +1 -1
- package/out/src/parquet-wrapper/writer.d.ts +12 -4
- package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
- package/out/src/parquet-wrapper/writer.js +12 -4
- package/out/src/parquet-wrapper/writer.js.map +1 -1
- package/out/src/parquet.d.ts +21 -7
- package/out/src/parquet.d.ts.map +1 -1
- package/out/src/parquet.js +15 -4
- package/out/src/parquet.js.map +1 -1
- package/out/src/runner.d.ts +39 -13
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +7 -3
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts +4 -0
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +14 -5
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/boundary-stress.d.ts +4 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +13 -7
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts +4 -0
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +39 -11
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts +4 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +14 -3
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts +4 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.js +4 -0
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts +4 -0
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +25 -8
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts +4 -0
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +21 -8
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts +4 -0
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +27 -8
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts +4 -0
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +16 -6
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.d.ts +4 -0
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/house-venue.js +4 -0
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts +9 -3
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +12 -4
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts +4 -0
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +27 -13
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +4 -0
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +9 -4
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/nl-postcode.d.ts +4 -0
- package/out/src/shard-recipes/nl-postcode.d.ts.map +1 -1
- package/out/src/shard-recipes/nl-postcode.js +8 -2
- package/out/src/shard-recipes/nl-postcode.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts +4 -0
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +8 -2
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts +4 -0
- package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street-led.js +5 -1
- package/out/src/shard-recipes/no-street-led.js.map +1 -1
- package/out/src/shard-recipes/no-street.d.ts +4 -0
- package/out/src/shard-recipes/no-street.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street.js +4 -0
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +4 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +99 -43
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts +4 -0
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +4 -0
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +48 -16
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +10 -6
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/si-bare-village.d.ts +4 -0
- package/out/src/shard-recipes/si-bare-village.d.ts.map +1 -1
- package/out/src/shard-recipes/si-bare-village.js +4 -0
- package/out/src/shard-recipes/si-bare-village.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +4 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +29 -10
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.d.ts +4 -0
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
- package/out/src/shard-recipes/street-bare.js +4 -0
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.d.ts +4 -0
- package/out/src/shard-recipes/street.d.ts.map +1 -1
- package/out/src/shard-recipes/street.js +4 -0
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/unit.d.ts +4 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +45 -14
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/split.d.ts +18 -6
- package/out/src/split.d.ts.map +1 -1
- package/out/src/split.js +4 -2
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize-anchor-absorption.d.ts +7 -1
- package/out/src/synthesize-anchor-absorption.d.ts.map +1 -1
- package/out/src/synthesize-anchor-absorption.js +28 -14
- package/out/src/synthesize-anchor-absorption.js.map +1 -1
- package/out/src/synthesize-boundary-stress.d.ts +3 -1
- package/out/src/synthesize-boundary-stress.d.ts.map +1 -1
- package/out/src/synthesize-boundary-stress.js +35 -28
- package/out/src/synthesize-boundary-stress.js.map +1 -1
- package/out/src/synthesize-german.d.ts +15 -5
- package/out/src/synthesize-german.d.ts.map +1 -1
- package/out/src/synthesize-german.js +9 -3
- package/out/src/synthesize-german.js.map +1 -1
- package/out/src/synthesize-house-venue.d.ts.map +1 -1
- package/out/src/synthesize-house-venue.js +15 -15
- package/out/src/synthesize-house-venue.js.map +1 -1
- package/out/src/synthesize-intersection.d.ts +9 -3
- package/out/src/synthesize-intersection.d.ts.map +1 -1
- package/out/src/synthesize-intersection.js +25 -6
- package/out/src/synthesize-intersection.js.map +1 -1
- package/out/src/synthesize-no-street.d.ts +3 -1
- package/out/src/synthesize-no-street.d.ts.map +1 -1
- package/out/src/synthesize-no-street.js +5 -6
- package/out/src/synthesize-no-street.js.map +1 -1
- package/out/src/synthesize-po-box.d.ts +15 -5
- package/out/src/synthesize-po-box.d.ts.map +1 -1
- package/out/src/synthesize-po-box.js +21 -7
- package/out/src/synthesize-po-box.js.map +1 -1
- package/out/src/synthesize-street.d.ts +3 -1
- package/out/src/synthesize-street.d.ts.map +1 -1
- package/out/src/synthesize-street.js +6 -1
- package/out/src/synthesize-street.js.map +1 -1
- package/out/src/synthesize.d.ts +39 -13
- package/out/src/synthesize.d.ts.map +1 -1
- package/out/src/synthesize.js +66 -36
- package/out/src/synthesize.js.map +1 -1
- package/out/src/tokenize.d.ts +15 -5
- package/out/src/tokenize.d.ts.map +1 -1
- package/out/src/tokenize.js.map +1 -1
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +3 -1
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/audit.d.ts.map +1 -1
- package/out/src/tools/audit.js +20 -8
- package/out/src/tools/audit.js.map +1 -1
- package/out/src/tools/corpus-stats.d.ts.map +1 -1
- package/out/src/tools/corpus-stats.js.map +1 -1
- package/out/src/tools/fetch/ban.d.ts.map +1 -1
- package/out/src/tools/fetch/ban.js +17 -10
- package/out/src/tools/fetch/ban.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts +30 -10
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +26 -6
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/hrsa.d.ts.map +1 -1
- package/out/src/tools/fetch/hrsa.js.map +1 -1
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +18 -18
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +3 -1
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +3 -1
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts +21 -7
- package/out/src/tools/fetch/nad.d.ts.map +1 -1
- package/out/src/tools/fetch/nad.js +5 -5
- package/out/src/tools/fetch/nad.js.map +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +9 -15
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts +3 -1
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/src/tools/fetch/openaddresses.js +32 -23
- package/out/src/tools/fetch/openaddresses.js.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +3 -1
- package/out/src/tools/fetch/ppd.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.js +14 -3
- package/out/src/tools/fetch/ppd.js.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.js +13 -6
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
- package/out/src/tools/fetch/state-sources.js +8 -3
- package/out/src/tools/fetch/state-sources.js.map +1 -1
- package/out/src/tools/fetch/tiger-full.d.ts +9 -3
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
- package/out/src/tools/fetch/tiger-full.js +35 -20
- package/out/src/tools/fetch/tiger-full.js.map +1 -1
- package/out/src/tools/gb-title-case.d.ts.map +1 -1
- package/out/src/tools/gb-title-case.js.map +1 -1
- package/out/src/tools/golden-expand.d.ts +24 -8
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +20 -12
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-promote.d.ts +18 -6
- package/out/src/tools/golden-promote.d.ts.map +1 -1
- package/out/src/tools/golden-promote.js +22 -7
- package/out/src/tools/golden-promote.js.map +1 -1
- package/out/src/tools/ingest-csv.d.ts +3 -1
- package/out/src/tools/ingest-csv.d.ts.map +1 -1
- package/out/src/tools/ingest-csv.js +17 -19
- package/out/src/tools/ingest-csv.js.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts +18 -6
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
- package/out/src/tools/jsonl-to-parquet.js +11 -5
- package/out/src/tools/jsonl-to-parquet.js.map +1 -1
- package/out/src/tools/lint-shard-vocab.d.ts +36 -12
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
- package/out/src/tools/lint-shard-vocab.js +25 -11
- package/out/src/tools/lint-shard-vocab.js.map +1 -1
- package/out/src/tools/lint-shard.d.ts +30 -10
- package/out/src/tools/lint-shard.d.ts.map +1 -1
- package/out/src/tools/lint-shard.js +33 -20
- package/out/src/tools/lint-shard.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +6 -4
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/shard-kryptonite.d.ts +6 -2
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +1 -1
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts +9 -3
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +2 -2
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/types.d.ts +51 -17
- package/out/src/types.d.ts.map +1 -1
- package/out/src/wof-json.d.ts +9 -3
- package/out/src/wof-json.d.ts.map +1 -1
- package/out/src/wof-json.js +6 -4
- package/out/src/wof-json.js.map +1 -1
- package/package.json +7 -6
- package/src/adapter.ts +24 -13
- package/src/adapters/ban/adapter.ts +12 -2
- package/src/adapters/ban/street-decompose.ts +7 -1
- package/src/adapters/fcc-bdc/adapter.ts +19 -3
- package/src/adapters/geonames/adapter.ts +28 -5
- package/src/adapters/geonames-postal/adapter.ts +17 -1
- package/src/adapters/gnaf/adapter.ts +15 -2
- package/src/adapters/gnaf/assemble.ts +40 -10
- package/src/adapters/index.ts +13 -0
- package/src/adapters/openaddresses/adapter.ts +22 -3
- package/src/adapters/overture/adapter.ts +18 -2
- package/src/adapters/state-hi-schools/adapter.ts +15 -0
- package/src/adapters/state-ia-contractors/adapter.ts +15 -0
- package/src/adapters/state-ny-notaries/adapter.ts +15 -0
- package/src/adapters/state-tx-notaries/adapter.ts +18 -2
- package/src/adapters/synth-po-box/adapter.ts +19 -2
- package/src/adapters/tiger/adapter.ts +23 -4
- package/src/adapters/tiger/street-decompose.ts +1 -0
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +18 -2
- package/src/adapters/usgov-imls-pls/adapter.ts +15 -0
- package/src/adapters/usgov-irs-bmf/adapter.ts +19 -2
- package/src/adapters/usgov-nad/adapter.ts +14 -1
- package/src/adapters/usgov-nppes/adapter.ts +13 -0
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +21 -3
- package/src/adapters/wof-admin-jp/adapter.ts +20 -3
- package/src/adapters/wof-admin-json/adapter.ts +22 -7
- package/src/adapters/wof-postalcode-json/adapter.ts +10 -2
- package/src/align.ts +21 -8
- package/src/build.ts +48 -11
- package/src/codex/us-fips-state.ts +6 -2
- package/src/golden.ts +28 -13
- package/src/license.ts +4 -2
- package/src/parquet-wrapper/reader.ts +6 -4
- package/src/parquet-wrapper/schema.ts +14 -10
- package/src/parquet-wrapper/writer.ts +12 -4
- package/src/parquet.ts +39 -8
- package/src/runner.ts +49 -15
- package/src/shard-recipes/anchor-absorption.ts +21 -5
- package/src/shard-recipes/boundary-stress.ts +20 -7
- package/src/shard-recipes/country-balanced.ts +78 -13
- package/src/shard-recipes/cz-pcfirst-preposition.ts +9 -0
- package/src/shard-recipes/fr-admin-split.ts +30 -5
- package/src/shard-recipes/fr-bare-street.ts +10 -0
- package/src/shard-recipes/fr-fragment.ts +33 -8
- package/src/shard-recipes/fr-lieudit.ts +35 -9
- package/src/shard-recipes/fr-order.ts +47 -9
- package/src/shard-recipes/german.ts +32 -6
- package/src/shard-recipes/house-venue.ts +7 -0
- package/src/shard-recipes/index.ts +12 -4
- package/src/shard-recipes/intersection.ts +63 -14
- package/src/shard-recipes/locale.ts +37 -4
- package/src/shard-recipes/nl-postcode.ts +12 -2
- package/src/shard-recipes/no-fragment.ts +12 -2
- package/src/shard-recipes/no-street-led.ts +11 -1
- package/src/shard-recipes/no-street.ts +7 -0
- package/src/shard-recipes/po-box-cedex.ts +145 -45
- package/src/shard-recipes/po-box.ts +7 -0
- package/src/shard-recipes/scaffold.ts +52 -20
- package/src/shard-recipes/si-bare-village.ts +9 -0
- package/src/shard-recipes/street-affix.ts +88 -13
- package/src/shard-recipes/street-bare.ts +10 -0
- package/src/shard-recipes/street.ts +7 -0
- package/src/shard-recipes/unit.ts +79 -16
- package/src/split.ts +28 -7
- package/src/synthesize-anchor-absorption.ts +41 -14
- package/src/synthesize-boundary-stress.ts +57 -30
- package/src/synthesize-german.ts +30 -7
- package/src/synthesize-house-venue.ts +22 -16
- package/src/synthesize-intersection.ts +34 -7
- package/src/synthesize-no-street.ts +18 -10
- package/src/synthesize-po-box.ts +42 -10
- package/src/synthesize-street.ts +10 -2
- package/src/synthesize.ts +81 -38
- package/src/tokenize.ts +15 -5
- package/src/tools/align-shard.ts +6 -1
- package/src/tools/audit.ts +57 -11
- package/src/tools/corpus-stats.ts +9 -0
- package/src/tools/fetch/ban.ts +32 -10
- package/src/tools/fetch/download.ts +49 -12
- package/src/tools/fetch/hrsa.ts +3 -0
- package/src/tools/fetch/imls-pls.ts +28 -18
- package/src/tools/fetch/index.ts +3 -1
- package/src/tools/fetch/nad.ts +39 -12
- package/src/tools/fetch/nppes.ts +17 -15
- package/src/tools/fetch/openaddresses.ts +52 -24
- package/src/tools/fetch/ppd.ts +22 -3
- package/src/tools/fetch/state-hi-schools.ts +17 -6
- package/src/tools/fetch/state-sources.ts +16 -3
- package/src/tools/fetch/tiger-full.ts +66 -23
- package/src/tools/gb-title-case.ts +1 -0
- package/src/tools/golden-expand.ts +64 -20
- package/src/tools/golden-promote.ts +53 -13
- package/src/tools/ingest-csv.ts +41 -26
- package/src/tools/jsonl-to-parquet.ts +30 -10
- package/src/tools/lint-shard-vocab.ts +75 -24
- package/src/tools/lint-shard.ts +75 -31
- package/src/tools/overlay-manifest.ts +7 -4
- package/src/tools/shard-kryptonite.ts +11 -3
- package/src/tools/shard-translit.ts +20 -5
- package/src/types.ts +51 -17
- package/src/wof-json.ts +18 -7
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../../src/tools/fetch/download.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;
|
|
1
|
+
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../../src/tools/fetch/download.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE1D;;GAEG;AACH,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAElC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAsBjC;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC/C,OAAO,MAAM,KAAK,sBAAsB,IAAI,CAAC,MAAM,IAAI,qBAAqB,IAAI,MAAM,IAAI,qBAAqB,CAAC,CAAA;AACjH,CAAC;AAqBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAwB;IAC5D,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,GAAG,OAAO,EAAE,OAAO,GAAG,CAAC,EAAE,YAAY,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IACrG,IAAI,SAAkB,CAAA;IAEtB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACrD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YACjB,MAAM,EAAE,CAAC,SAAS,OAAO,IAAI,OAAO,UAAU,YAAY,QAAQ,GAAG,EAAE,CAAC,CAAA;YACxE,MAAM,KAAK,CAAC,YAAY,CAAC,CAAA;QAC1B,CAAC;QAED,IAAI,GAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;QAC5E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,iEAAiE;YACjE,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,MAAM,GAAG,EAAE,CAAC,CAAA;YAExE,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,MAAM,KAAK,CAAA;YAC/C,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,CAAA;YACnD,MAAM,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;YAE7B,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,CAAA;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,8DAA8D;YAC9D,SAAS,GAAG,KAAK,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAA;AAC5E,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAI,IAAY;IACjD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IAElC,IAAI,CAAC;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAM,CAAA;IACrD,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAI,IAAY,EAAE,GAAyB;IACnF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAa,CAAA;IACpC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAM,IAAI,CAAC,CAAA;IAE5C,KAAK,MAAM,KAAK,IAAI,MAAM,IAAI,EAAE,EAAE,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAA;IAC/B,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,QAAiB;IAClE,MAAM,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;AAChE,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hrsa.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/hrsa.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAOH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAOnE,MAAM,MAAM,gBAAgB,GAAG,gBAAgB,CAAA;AAU/C,wBAAsB,SAAS,CAAC,OAAO,EAAE,gBAAgB,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,
|
|
1
|
+
{"version":3,"file":"hrsa.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/hrsa.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAOH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAOnE,MAAM,MAAM,gBAAgB,GAAG,gBAAgB,CAAA;AAU/C,wBAAsB,SAAS,CAAC,OAAO,EAAE,gBAAgB,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CA8BjH"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hrsa.js","sourceRoot":"","sources":["../../../../src/tools/fetch/hrsa.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AACnC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE7D,MAAM,IAAI,GAAG,iBAAiB,CAAA;AAC9B,MAAM,QAAQ,GAAG,wDAAwD,CAAA;AACzE,MAAM,UAAU,GAAG,+CAA+C,QAAQ,EAAE,CAAA;AAY5E,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB,EAAE,MAA+B;IACzF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACvC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;IAEpC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,QAAQ,EAAE,CAAC,CAAA;
|
|
1
|
+
{"version":3,"file":"hrsa.js","sourceRoot":"","sources":["../../../../src/tools/fetch/hrsa.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AACnC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE7D,MAAM,IAAI,GAAG,iBAAiB,CAAA;AAC9B,MAAM,QAAQ,GAAG,wDAAwD,CAAA;AACzE,MAAM,UAAU,GAAG,+CAA+C,QAAQ,EAAE,CAAA;AAY5E,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB,EAAE,MAA+B;IACzF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACvC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;IAEpC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,QAAQ,EAAE,CAAC,CAAA;IAErC,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC;QACtC,GAAG,EAAE,UAAU;QACf,IAAI;QACJ,SAAS,EAAE,OAAO;QAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;IAElC,MAAM,QAAQ,GAAmB;QAChC,UAAU,EAAE,UAAU;QACtB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,QAAQ;QAClB,MAAM,EAAE,GAAG;QACX,KAAK;KACL,CAAA;IAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAE7D,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;IAErE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"imls-pls.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/imls-pls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;
|
|
1
|
+
{"version":3,"file":"imls-pls.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/imls-pls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAaH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAiBnE,MAAM,MAAM,mBAAmB,GAAG,gBAAgB,CAAA;AAsBlD,wBAAsB,YAAY,CACjC,OAAO,EAAE,mBAAmB,EAC5B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAgGvB"}
|
|
@@ -19,6 +19,8 @@
|
|
|
19
19
|
* Invoke via `mailwoman corpus fetch imls-pls --out-root <path>`. Idempotent: if dest CSV exists
|
|
20
20
|
* and sha matches MANIFEST, skips download.
|
|
21
21
|
*/
|
|
22
|
+
/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
|
|
23
|
+
procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
|
|
22
24
|
import { execFile } from "node:child_process";
|
|
23
25
|
import { existsSync, mkdirSync, statSync } from "node:fs";
|
|
24
26
|
import { rm } from "node:fs/promises";
|
|
@@ -26,12 +28,20 @@ import { basename, join } from "node:path";
|
|
|
26
28
|
import { promisify } from "node:util";
|
|
27
29
|
import { sha256File } from "@mailwoman/core/utils";
|
|
28
30
|
import { downloadToFile, readManifest, writeManifest } from "./download.js";
|
|
31
|
+
/**
|
|
32
|
+
* Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
|
|
33
|
+
* than data.
|
|
34
|
+
*/
|
|
35
|
+
const BYTES_PER_KIB = 1024;
|
|
29
36
|
const execFileAsync = promisify(execFile);
|
|
30
|
-
|
|
31
|
-
|
|
37
|
+
/**
|
|
38
|
+
* The PLS FY 2023 bulk CSV ZIP (most recent as of 2026-05). If IMLS publishes a newer year, update this URL.
|
|
39
|
+
*/
|
|
32
40
|
const ZIP_URL = "https://www.imls.gov/sites/default/files/2025-08/pls_fy2023_csv.zip";
|
|
33
41
|
const SLUG = "usgov-imls-pls";
|
|
34
|
-
/**
|
|
42
|
+
/**
|
|
43
|
+
* Return the filenames listed inside a ZIP (the trailing column of each `unzip -l` row).
|
|
44
|
+
*/
|
|
35
45
|
async function listZipEntries(zipPath) {
|
|
36
46
|
const listing = await execFileAsync("unzip", ["-l", zipPath]);
|
|
37
47
|
return listing.stdout
|
|
@@ -45,9 +55,7 @@ export async function fetchIMLSPLS(options, report) {
|
|
|
45
55
|
const zipDest = join(destDir, basename(ZIP_URL));
|
|
46
56
|
const manifestPath = join(destDir, "MANIFEST.json");
|
|
47
57
|
report?.(`=== ${SLUG}`);
|
|
48
|
-
//
|
|
49
|
-
// Idempotency check: if outlet CSV already exists and sha matches, skip.
|
|
50
|
-
// ------------------------------------------------------------------
|
|
58
|
+
// MARK: Idempotency check
|
|
51
59
|
const recorded = await readManifest(manifestPath);
|
|
52
60
|
if (recorded?.sha256 && recorded.filename) {
|
|
53
61
|
const recordedPath = join(destDir, recorded.filename);
|
|
@@ -56,9 +64,7 @@ export async function fetchIMLSPLS(options, report) {
|
|
|
56
64
|
return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] };
|
|
57
65
|
}
|
|
58
66
|
}
|
|
59
|
-
//
|
|
60
|
-
// Download ZIP
|
|
61
|
-
// ------------------------------------------------------------------
|
|
67
|
+
// MARK: Download ZIP
|
|
62
68
|
report?.(` Downloading ${ZIP_URL} ...`);
|
|
63
69
|
const { bytes: zipSize } = await downloadToFile({
|
|
64
70
|
url: ZIP_URL,
|
|
@@ -68,15 +74,13 @@ export async function fetchIMLSPLS(options, report) {
|
|
|
68
74
|
report,
|
|
69
75
|
});
|
|
70
76
|
report?.(` Downloaded: ${(zipSize / 1024 / 1024).toFixed(1)} MB`);
|
|
71
|
-
if (zipSize <
|
|
77
|
+
if (zipSize < BYTES_PER_KIB) {
|
|
72
78
|
report?.(` ✗ Response too small (${zipSize} bytes) — probable error page`);
|
|
73
79
|
return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
|
|
74
80
|
}
|
|
75
|
-
// ------------------------------------------------------------------
|
|
76
81
|
// Discover the outlet-level CSV inside the ZIP.
|
|
77
82
|
// Outlet files match: pls_fy*outlet*.csv (case-insensitive)
|
|
78
83
|
// Administrative-entity files match: pls_fy*ae*.csv — we skip those.
|
|
79
|
-
// ------------------------------------------------------------------
|
|
80
84
|
report?.(" Inspecting ZIP contents ...");
|
|
81
85
|
const entries = await listZipEntries(zipDest);
|
|
82
86
|
let csvName = entries.find((name) => /pls_fy.*outlet.*\.csv/i.test(name));
|
|
@@ -97,14 +101,10 @@ export async function fetchIMLSPLS(options, report) {
|
|
|
97
101
|
const csvDest = join(destDir, basename(csvName));
|
|
98
102
|
const csvSize = statSync(csvDest).size;
|
|
99
103
|
const csvSha = await sha256File(csvDest);
|
|
100
|
-
//
|
|
101
|
-
// Remove ZIP (small, but keep destDir clean)
|
|
102
|
-
// ------------------------------------------------------------------
|
|
104
|
+
// MARK: Remove ZIP (small, but keep destDir clean)
|
|
103
105
|
await rm(zipDest, { force: true });
|
|
104
106
|
report?.(" Removed ZIP (CSV kept)");
|
|
105
|
-
//
|
|
106
|
-
// Write MANIFEST
|
|
107
|
-
// ------------------------------------------------------------------
|
|
107
|
+
// MARK: Write MANIFEST
|
|
108
108
|
const manifest = {
|
|
109
109
|
source_url: ZIP_URL,
|
|
110
110
|
downloaded_at: new Date().toISOString(),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"imls-pls.js","sourceRoot":"","sources":["../../../../src/tools/fetch/imls-pls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACzD,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAA;AACrC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAErC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE3E,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAA;AAEzC
|
|
1
|
+
{"version":3,"file":"imls-pls.js","sourceRoot":"","sources":["../../../../src/tools/fetch/imls-pls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;kGACkG;AAElG,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACzD,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAA;AACrC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAErC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE3E;;;GAGG;AACH,MAAM,aAAa,GAAG,IAAI,CAAA;AAE1B,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAA;AAEzC;;GAEG;AACH,MAAM,OAAO,GAAG,qEAAqE,CAAA;AACrF,MAAM,IAAI,GAAG,gBAAgB,CAAA;AAY7B;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,OAAe;IAC5C,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAA;IAE7D,OAAO,OAAO,CAAC,MAAM;SACnB,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;SACnD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;AACpC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CACjC,OAA4B,EAC5B,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IAChD,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAEnD,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAEvB,0BAA0B;IAE1B,MAAM,QAAQ,GAAG,MAAM,YAAY,CAA0B,YAAY,CAAC,CAAA;IAE1E,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAErD,IAAI,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;YACtF,MAAM,EAAE,CAAC,oEAAoE,CAAC,CAAA;YAE9E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;QAC9D,CAAC;IACF,CAAC;IAED,qBAAqB;IAErB,MAAM,EAAE,CAAC,iBAAiB,OAAO,MAAM,CAAC,CAAA;IAExC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,cAAc,CAAC;QAC/C,GAAG,EAAE,OAAO;QACZ,IAAI,EAAE,OAAO;QACb,SAAS,EAAE,OAAO;QAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,iBAAiB,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IAElE,IAAI,OAAO,GAAG,aAAa,EAAE,CAAC;QAC7B,MAAM,EAAE,CAAC,2BAA2B,OAAO,+BAA+B,CAAC,CAAA;QAE3E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,gDAAgD;IAChD,4DAA4D;IAC5D,qEAAqE;IACrE,MAAM,EAAE,CAAC,+BAA+B,CAAC,CAAA;IACzC,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAA;IAE7C,IAAI,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;IAEzE,4EAA4E;IAC5E,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;IAClG,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,EAAE,CAAC,2BAA2B,CAAC,CAAA;QAErC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;QACxB,CAAC;QAED,MAAM,EAAE,CAAC,kFAAkF,CAAC,CAAA;QAE5F,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,EAAE,CAAC,4BAA4B,OAAO,EAAE,CAAC,CAAA;IAC/C,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,CAAA;IAE3E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IAChD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAA;IACtC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IAExC,mDAAmD;IAEnD,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAClC,MAAM,EAAE,CAAC,0BAA0B,CAAC,CAAA;IAEpC,uBAAuB;IAEvB,MAAM,QAAQ,GAAmB;QAChC,UAAU,EAAE,OAAO;QACnB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC;QAC3B,MAAM,EAAE,MAAM;QACd,KAAK,EAAE,OAAO;KACd,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,MAAM,EAAE,CAAC,CAAA;IAC1E,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IAEjD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}
|
|
@@ -102,7 +102,9 @@ export * from "./openaddresses.ts";
|
|
|
102
102
|
export * from "./state-hi-schools.ts";
|
|
103
103
|
export * from "./state-sources.ts";
|
|
104
104
|
export * from "./tiger-full.ts";
|
|
105
|
-
/**
|
|
105
|
+
/**
|
|
106
|
+
* The fetch-source registry: id → module entry point. Each entry point takes its own options interface.
|
|
107
|
+
*/
|
|
106
108
|
export declare const FETCH_SOURCES: {
|
|
107
109
|
readonly ban: typeof fetchBan;
|
|
108
110
|
readonly nad: typeof fetchNAD;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqFG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AACvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAA;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAEhD,cAAc,UAAU,CAAA;AACxB,cAAc,WAAW,CAAA;AACzB,cAAc,eAAe,CAAA;AAC7B,cAAc,UAAU,CAAA;AACxB,cAAc,YAAY,CAAA;AAC1B,cAAc,oBAAoB,CAAA;AAClC,cAAc,uBAAuB,CAAA;AACrC,cAAc,oBAAoB,CAAA;AAClC,cAAc,iBAAiB,CAAA;AAE/B
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqFG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AACvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAA;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAEhD,cAAc,UAAU,CAAA;AACxB,cAAc,WAAW,CAAA;AACzB,cAAc,eAAe,CAAA;AAC7B,cAAc,UAAU,CAAA;AACxB,cAAc,YAAY,CAAA;AAC1B,cAAc,oBAAoB,CAAA;AAClC,cAAc,uBAAuB,CAAA;AACrC,cAAc,oBAAoB,CAAA;AAClC,cAAc,iBAAiB,CAAA;AAE/B;;GAEG;AACH,eAAO,MAAM,aAAa;;;;;;;;;;CAUhB,CAAA;AAEV,MAAM,MAAM,aAAa,GAAG,MAAM,OAAO,aAAa,CAAA"}
|
|
@@ -102,7 +102,9 @@ export * from "./openaddresses.js";
|
|
|
102
102
|
export * from "./state-hi-schools.js";
|
|
103
103
|
export * from "./state-sources.js";
|
|
104
104
|
export * from "./tiger-full.js";
|
|
105
|
-
/**
|
|
105
|
+
/**
|
|
106
|
+
* The fetch-source registry: id → module entry point. Each entry point takes its own options interface.
|
|
107
|
+
*/
|
|
106
108
|
export const FETCH_SOURCES = {
|
|
107
109
|
ban: fetchBan,
|
|
108
110
|
nad: fetchNAD,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/tools/fetch/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqFG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AACvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAA;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAEhD,cAAc,UAAU,CAAA;AACxB,cAAc,WAAW,CAAA;AACzB,cAAc,eAAe,CAAA;AAC7B,cAAc,UAAU,CAAA;AACxB,cAAc,YAAY,CAAA;AAC1B,cAAc,oBAAoB,CAAA;AAClC,cAAc,uBAAuB,CAAA;AACrC,cAAc,oBAAoB,CAAA;AAClC,cAAc,iBAAiB,CAAA;AAE/B
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/tools/fetch/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqFG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AACvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAA;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAEhD,cAAc,UAAU,CAAA;AACxB,cAAc,WAAW,CAAA;AACzB,cAAc,eAAe,CAAA;AAC7B,cAAc,UAAU,CAAA;AACxB,cAAc,YAAY,CAAA;AAC1B,cAAc,oBAAoB,CAAA;AAClC,cAAc,uBAAuB,CAAA;AACrC,cAAc,oBAAoB,CAAA;AAClC,cAAc,iBAAiB,CAAA;AAE/B;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC5B,GAAG,EAAE,QAAQ;IACb,GAAG,EAAE,QAAQ;IACb,IAAI,EAAE,SAAS;IACf,UAAU,EAAE,YAAY;IACxB,KAAK,EAAE,UAAU;IACjB,aAAa,EAAE,kBAAkB;IACjC,eAAe,EAAE,iBAAiB;IAClC,kBAAkB,EAAE,mBAAmB;IACvC,YAAY,EAAE,cAAc;CACnB,CAAA"}
|
|
@@ -37,19 +37,33 @@
|
|
|
37
37
|
*/
|
|
38
38
|
import type { BaseFetchOptions, FetchSummary } from "./download.ts";
|
|
39
39
|
export interface FetchNADOptions extends BaseFetchOptions {
|
|
40
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* Fetch strategy. Default `featureserver`.
|
|
42
|
+
*/
|
|
41
43
|
mode?: "featureserver" | "bulk";
|
|
42
|
-
/**
|
|
44
|
+
/**
|
|
45
|
+
* Pre-signed S3 URL for bulk mode.
|
|
46
|
+
*/
|
|
43
47
|
nadURL?: string;
|
|
44
|
-
/**
|
|
48
|
+
/**
|
|
49
|
+
* Records per output file. Default `100000`.
|
|
50
|
+
*/
|
|
45
51
|
chunkSize?: number;
|
|
46
|
-
/**
|
|
52
|
+
/**
|
|
53
|
+
* Records per HTTP request. Default `5000`.
|
|
54
|
+
*/
|
|
47
55
|
pageSize?: number;
|
|
48
|
-
/**
|
|
56
|
+
/**
|
|
57
|
+
* Parallel page fetches within a chunk. Default `4`.
|
|
58
|
+
*/
|
|
49
59
|
concurrency?: number;
|
|
50
|
-
/**
|
|
60
|
+
/**
|
|
61
|
+
* Start OBJECTID. Default `1`.
|
|
62
|
+
*/
|
|
51
63
|
startOID?: number;
|
|
52
|
-
/**
|
|
64
|
+
/**
|
|
65
|
+
* Stop before this OID. Default = total count.
|
|
66
|
+
*/
|
|
53
67
|
endOID?: number;
|
|
54
68
|
}
|
|
55
69
|
export declare function fetchNAD(options: FetchNADOptions, report?: (line: string) => void): Promise<FetchSummary>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"nad.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/nad.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAQH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"nad.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/nad.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAQH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAQnE,MAAM,WAAW,eAAgB,SAAQ,gBAAgB;IACxD;;OAEG;IACH,IAAI,CAAC,EAAE,eAAe,GAAG,MAAM,CAAA;IAC/B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AA6OD,wBAAsB,QAAQ,CAAC,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAY/G"}
|
|
@@ -98,9 +98,9 @@ async function fetchChunk(chunkPath, chunkStart, chunkEnd, pageSize, concurrency
|
|
|
98
98
|
try {
|
|
99
99
|
pageResults[slot].rows = await fetchPage(s, e, pageSize);
|
|
100
100
|
}
|
|
101
|
-
catch (
|
|
102
|
-
pageResults[slot].error =
|
|
103
|
-
report?.(` ✗ page ${s}-${e}: ${
|
|
101
|
+
catch (error) {
|
|
102
|
+
pageResults[slot].error = error;
|
|
103
|
+
report?.(` ✗ page ${s}-${e}: ${error.message}`);
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
106
|
});
|
|
@@ -117,12 +117,12 @@ async function fetchChunk(chunkPath, chunkStart, chunkEnd, pageSize, concurrency
|
|
|
117
117
|
lines.push(JSON.stringify(row));
|
|
118
118
|
}
|
|
119
119
|
}
|
|
120
|
-
await writeFile(chunkPath, lines.length
|
|
120
|
+
await writeFile(chunkPath, !lines.length ? "" : lines.join("\n") + "\n");
|
|
121
121
|
return { recordCount: lines.length, errors };
|
|
122
122
|
}
|
|
123
123
|
async function featureserverMode(options, report) {
|
|
124
124
|
const chunkSize = options.chunkSize ?? 100_000;
|
|
125
|
-
const pageSize = options.pageSize ??
|
|
125
|
+
const pageSize = options.pageSize ?? 5000;
|
|
126
126
|
const concurrency = options.concurrency ?? 4;
|
|
127
127
|
const startOID = options.startOID ?? 1;
|
|
128
128
|
const chunkDir = join(options.outRoot, SLUG, "featureserver");
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"nad.js","sourceRoot":"","sources":["../../../../src/tools/fetch/nad.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACzD,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE3E,MAAM,IAAI,GAAG,WAAW,CAAA;
|
|
1
|
+
{"version":3,"file":"nad.js","sourceRoot":"","sources":["../../../../src/tools/fetch/nad.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACzD,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE3E,MAAM,IAAI,GAAG,WAAW,CAAA;AAExB,MAAM,mBAAmB,GACxB,sIAAsI,CAAA;AA6CvI,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,MAAc,EAAE,QAAgB;IAC1E,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,mBAAmB,QAAQ,CAAC,CAAA;IACnD,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,oBAAoB,QAAQ,QAAQ,MAAM,EAAE,CAAC,CAAA;IAC3E,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,CAAA;IACtC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;IACjC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,mBAAmB,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAA;IAE3D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAC5B,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC;KACpC,CAAC,CAAA;IAEF,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,WAAW,QAAQ,IAAI,MAAM,EAAE,CAAC,CAAA;IACjG,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA+E,CAAA;IAE7G,IAAI,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,IAAI,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAA;IAEnG,OAAO,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAA;AACtD,CAAC;AAED,KAAK,UAAU,kBAAkB;IAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,mBAAmB,QAAQ,CAAC,CAAA;IACnD,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;IACpC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAA;IAC/C,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;IACjC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;IAErE,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;IACvF,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAuB,CAAA;IAErD,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAA;IAE9F,OAAO,IAAI,CAAC,KAAK,CAAA;AAClB,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,UAAU,CACxB,SAAiB,EACjB,UAAkB,EAClB,QAAgB,EAChB,QAAgB,EAChB,WAAmB,EACnB,MAA+B;IAE/B,MAAM,UAAU,GAA4B,EAAE,CAAA;IAE9C,KAAK,IAAI,MAAM,GAAG,UAAU,EAAE,MAAM,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,EAAE,CAAC;QACtE,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,QAAQ,GAAG,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAA;IACrE,CAAC;IAED,yFAAyF;IACzF,MAAM,WAAW,GAAoD,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1F,IAAI,EAAE,EAAE;QACR,KAAK,EAAE,IAAI;KACX,CAAC,CAAC,CAAA;IAEH,IAAI,QAAQ,GAAG,CAAC,CAAA;IAEhB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC3F,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAA;YAEvB,IAAI,IAAI,IAAI,UAAU,CAAC,MAAM;gBAAE,OAAM;YACrC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,IAAI,CAAE,CAAA;YAEhC,IAAI,CAAC;gBACJ,WAAW,CAAC,IAAI,CAAE,CAAC,IAAI,GAAG,MAAM,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAA;YAC1D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,WAAW,CAAC,IAAI,CAAE,CAAC,KAAK,GAAG,KAAc,CAAA;gBACzC,MAAM,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,KAAM,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YAC9D,CAAC;QACF,CAAC;IACF,CAAC,CAAC,CAAA;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAE1B,mFAAmF;IACnF,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,WAAW,EAAE,CAAC;QAC3C,IAAI,KAAK,EAAE,CAAC;YACX,MAAM,EAAE,CAAA;YAER,SAAQ;QACT,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAA;QAChC,CAAC;IACF,CAAC;IAED,MAAM,SAAS,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAA;IAExE,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,CAAA;AAC7C,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,OAAwB,EAAE,MAA+B;IACzF,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAA;IAC9C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAA;IACzC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAA;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAA;IAEtC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,EAAE,eAAe,CAAC,CAAA;IAC7D,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAExC,MAAM,EAAE,CAAC,OAAO,IAAI,kBAAkB,CAAC,CAAA;IACvC,MAAM,EAAE,CAAC,gCAAgC,CAAC,CAAA;IAC1C,MAAM,UAAU,GAAG,MAAM,kBAAkB,EAAE,CAAA;IAC7C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,UAAU,CAAA;IAC3C,MAAM,EAAE,CAAC,oBAAoB,UAAU,CAAC,cAAc,EAAE,EAAE,CAAC,CAAA;IAC3D,MAAM,EAAE,CAAC,gBAAgB,QAAQ,CAAC,cAAc,EAAE,OAAO,MAAM,CAAC,cAAc,EAAE,EAAE,CAAC,CAAA;IACnF,MAAM,EAAE,CAAC,iBAAiB,SAAS,gBAAgB,QAAQ,kBAAkB,WAAW,EAAE,CAAC,CAAA;IAE3F,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,IAAI,MAAM,GAAG,QAAQ,EAAE,MAAM,IAAI,MAAM,EAAE,MAAM,IAAI,SAAS,EAAE,CAAC;QACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,SAAS,GAAG,CAAC,EAAE,MAAM,CAAC,CAAA;QACzD,MAAM,SAAS,GAAG,QAAQ,MAAM,IAAI,QAAQ,EAAE,CAAA;QAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,SAAS,SAAS,CAAC,CAAA;QACvD,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,SAAS,gBAAgB,CAAC,CAAA;QAEjE,qFAAqF;QACrF,2FAA2F;QAC3F,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAgB,YAAY,CAAC,CAAA;YAEhE,IAAI,QAAQ,EAAE,QAAQ,EAAE,CAAC;gBACxB,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;QACF,CAAC;QAED,MAAM,EAAE,CAAC,cAAc,SAAS,MAAM,CAAC,CAAA;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACrB,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,MAAM,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,CAAA;QAC5G,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;QACrD,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,IAAI,CAAA;QACtC,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,CAAA;QAEvC,MAAM,QAAQ,GAAkB;YAC/B,UAAU,EAAE,mBAAmB;YAC/B,SAAS,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC;YAC7B,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,QAAQ,EAAE,GAAG,SAAS,SAAS;YAC/B,MAAM,EAAE,GAAG;YACX,KAAK;YACL,YAAY,EAAE,WAAW;YACzB,WAAW,EAAE,MAAM;YACnB,QAAQ,EAAE,MAAM,KAAK,CAAC;SACtB,CAAA;QAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;QAE3C,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,cAAc,CAAA;QAE7D,MAAM,EAAE,CACP,OAAO,MAAM,KAAK,WAAW,CAAC,cAAc,EAAE,eAAe,OAAO,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CACjH,CAAA;QAED,OAAO,EAAE,CAAA;QACT,YAAY,IAAI,WAAW,CAAA;QAC3B,WAAW,IAAI,MAAM,CAAA;QAErB,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YAChB,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAC5B,CAAC;IACF,CAAC;IAED,MAAM,EAAE,CAAC,+BAA+B,CAAC,CAAA;IACzC,MAAM,EAAE,CAAC,mBAAmB,OAAO,cAAc,OAAO,EAAE,CAAC,CAAA;IAC3D,MAAM,EAAE,CAAC,kBAAkB,YAAY,CAAC,cAAc,EAAE,EAAE,CAAC,CAAA;IAC3D,MAAM,EAAE,CAAC,kBAAkB,WAAW,EAAE,CAAC,CAAA;IACzC,MAAM,EAAE,CAAC,kBAAkB,QAAQ,EAAE,CAAC,CAAA;IAEtC,gGAAgG;IAChG,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,CAAA;AAC9D,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,OAAwB,EAAE,MAA+B;IAChF,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CACd,8DAA8D;YAC7D,mFAAmF;YACnF,+DAA+D,CAChE,CAAA;IACF,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACvC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,SAAS,CAAA;IAC/E,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;IAExC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,QAAQ,EAAE,CAAC,CAAA;IACrC,MAAM,EAAE,CAAC,UAAU,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IAE3F,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,EAAE,MAAM,EAAE,CAAC,CAAA;IAEnH,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAA;IAEtC,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE;QACnD,UAAU,EAAE,OAAO,CAAC,MAAM;QAC1B,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ;QACR,MAAM,EAAE,GAAG;QACX,KAAK;KACL,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;IAE5E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAwB,EAAE,MAA+B;IACvF,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,eAAe,CAAA;IAE5C,IAAI,IAAI,KAAK,eAAe,EAAE,CAAC;QAC9B,OAAO,iBAAiB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IAC1C,CAAC;IAED,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QACrB,OAAO,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACjC,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,iBAAiB,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAA;AAChF,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"nppes.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/nppes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;
|
|
1
|
+
{"version":3,"file":"nppes.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/nppes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAaH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AASnE,MAAM,MAAM,iBAAiB,GAAG,gBAAgB,CAAA;AA+ChD,wBAAsB,UAAU,CAAC,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAwFnH"}
|
|
@@ -20,6 +20,8 @@
|
|
|
20
20
|
* Invoke via `mailwoman corpus fetch nppes --out-root <path>`. Idempotent: if dest CSV exists and
|
|
21
21
|
* sha256 matches MANIFEST, skips download.
|
|
22
22
|
*/
|
|
23
|
+
/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
|
|
24
|
+
procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
|
|
23
25
|
import { execFile } from "node:child_process";
|
|
24
26
|
import { existsSync, mkdirSync, statSync } from "node:fs";
|
|
25
27
|
import { rm } from "node:fs/promises";
|
|
@@ -50,7 +52,9 @@ async function discoverLatestZip() {
|
|
|
50
52
|
}
|
|
51
53
|
return undefined;
|
|
52
54
|
}
|
|
53
|
-
/**
|
|
55
|
+
/**
|
|
56
|
+
* Extract the main registry CSV name (npidata_pfile_*.csv) from a ZIP's `unzip -l` listing.
|
|
57
|
+
*/
|
|
54
58
|
async function findNpidataCSV(zipPath) {
|
|
55
59
|
const listing = await execFileAsync("unzip", ["-l", zipPath]);
|
|
56
60
|
for (const line of listing.stdout.split("\n")) {
|
|
@@ -74,10 +78,8 @@ export async function fetchNPPES(options, report) {
|
|
|
74
78
|
const zipURL = `${BASE_URL}/${zipFilename}`;
|
|
75
79
|
const zipDest = join(destDir, zipFilename);
|
|
76
80
|
report?.(` Latest full file: ${zipFilename}`);
|
|
77
|
-
// ------------------------------------------------------------------
|
|
78
81
|
// Idempotency check: if the main CSV already exists and sha matches,
|
|
79
82
|
// skip re-download.
|
|
80
|
-
// ------------------------------------------------------------------
|
|
81
83
|
const recorded = await readManifest(manifestPath);
|
|
82
84
|
if (recorded?.sha256 && recorded.filename) {
|
|
83
85
|
const recordedPath = join(destDir, recorded.filename);
|
|
@@ -86,9 +88,7 @@ export async function fetchNPPES(options, report) {
|
|
|
86
88
|
return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] };
|
|
87
89
|
}
|
|
88
90
|
}
|
|
89
|
-
//
|
|
90
|
-
// Download ZIP (large; 60-minute timeout)
|
|
91
|
-
// ------------------------------------------------------------------
|
|
91
|
+
// MARK: Download ZIP (large; 60-minute timeout)
|
|
92
92
|
report?.(` Downloading ${zipURL} ...`);
|
|
93
93
|
const { bytes: zipSize } = await downloadToFile({
|
|
94
94
|
url: zipURL,
|
|
@@ -98,9 +98,7 @@ export async function fetchNPPES(options, report) {
|
|
|
98
98
|
report,
|
|
99
99
|
});
|
|
100
100
|
report?.(` Downloaded: ${(zipSize / 1024 / 1024).toFixed(1)} MB`);
|
|
101
|
-
//
|
|
102
|
-
// Extract only the main registry CSV (npidata_pfile_*.csv)
|
|
103
|
-
// ------------------------------------------------------------------
|
|
101
|
+
// MARK: Extract only the main registry CSV (npidata_pfile_*.csv)
|
|
104
102
|
report?.(" Extracting npidata_pfile CSV from ZIP ...");
|
|
105
103
|
const csvName = await findNpidataCSV(zipDest);
|
|
106
104
|
if (!csvName) {
|
|
@@ -113,14 +111,10 @@ export async function fetchNPPES(options, report) {
|
|
|
113
111
|
const csvSize = statSync(csvDest).size;
|
|
114
112
|
const csvSha = await sha256File(csvDest);
|
|
115
113
|
report?.(` CSV size: ${(csvSize / 1024 / 1024).toFixed(1)} MB`);
|
|
116
|
-
//
|
|
117
|
-
// Remove the ZIP to reclaim ~1 GB (the CSV is what adapters consume)
|
|
118
|
-
// ------------------------------------------------------------------
|
|
114
|
+
// MARK: Remove the ZIP to reclaim ~1 GB
|
|
119
115
|
await rm(zipDest, { force: true });
|
|
120
116
|
report?.(" Removed ZIP (CSV kept)");
|
|
121
|
-
//
|
|
122
|
-
// Write MANIFEST (records the extracted CSV, not the ZIP)
|
|
123
|
-
// ------------------------------------------------------------------
|
|
117
|
+
// MARK: Write MANIFEST (records the extracted CSV, not the ZIP)
|
|
124
118
|
const manifest = {
|
|
125
119
|
source_url: zipURL,
|
|
126
120
|
downloaded_at: new Date().toISOString(),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"nppes.js","sourceRoot":"","sources":["../../../../src/tools/fetch/nppes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACzD,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAA;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAErC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE3E,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAA;AAEzC,MAAM,SAAS,GAAG,+CAA+C,CAAA;AACjE,MAAM,QAAQ,GAAG,gCAAgC,CAAA;AACjD,MAAM,IAAI,GAAG,aAAa,CAAA;AAY1B;;;GAGG;AACH,KAAK,UAAU,iBAAiB;IAC/B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;QAClC,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;KACnC,CAAC,CAAA;IAEF,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,OAAO,SAAS,EAAE,CAAC,CAAA;IACpF,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAA;IAE7B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,qDAAqD,CAAC,EAAE,CAAC;QAC1F,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QAErB,IAAI,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;IACnD,CAAC;IAED,OAAO,SAAS,CAAA;AACjB,CAAC;AAED
|
|
1
|
+
{"version":3,"file":"nppes.js","sourceRoot":"","sources":["../../../../src/tools/fetch/nppes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH;kGACkG;AAElG,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACzD,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAA;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAErC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE3E,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAA;AAEzC,MAAM,SAAS,GAAG,+CAA+C,CAAA;AACjE,MAAM,QAAQ,GAAG,gCAAgC,CAAA;AACjD,MAAM,IAAI,GAAG,aAAa,CAAA;AAY1B;;;GAGG;AACH,KAAK,UAAU,iBAAiB;IAC/B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;QAClC,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;KACnC,CAAC,CAAA;IAEF,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,OAAO,SAAS,EAAE,CAAC,CAAA;IACpF,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAA;IAE7B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,qDAAqD,CAAC,EAAE,CAAC;QAC1F,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QAErB,IAAI,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;IACnD,CAAC;IAED,OAAO,SAAS,CAAA;AACjB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,OAAe;IAC5C,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAA;IAE7D,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAG,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEjD,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAA;IAChC,CAAC;IAED,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAA0B,EAAE,MAA+B;IAC3F,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACvC,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAEnD,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IACvB,MAAM,EAAE,CAAC,kDAAkD,SAAS,MAAM,CAAC,CAAA;IAE3E,MAAM,WAAW,GAAG,MAAM,iBAAiB,EAAE,CAAA;IAE7C,IAAI,CAAC,WAAW,EAAE,CAAC;QAClB,MAAM,EAAE,CAAC,4CAA4C,SAAS,EAAE,CAAC,CAAA;QAEjE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,MAAM,GAAG,GAAG,QAAQ,IAAI,WAAW,EAAE,CAAA;IAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;IAC1C,MAAM,EAAE,CAAC,uBAAuB,WAAW,EAAE,CAAC,CAAA;IAE9C,qEAAqE;IACrE,oBAAoB;IACpB,MAAM,QAAQ,GAAG,MAAM,YAAY,CAA0B,YAAY,CAAC,CAAA;IAE1E,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAErD,IAAI,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;YACtF,MAAM,EAAE,CAAC,oEAAoE,CAAC,CAAA;YAE9E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;QAC9D,CAAC;IACF,CAAC;IAED,gDAAgD;IAEhD,MAAM,EAAE,CAAC,iBAAiB,MAAM,MAAM,CAAC,CAAA;IAEvC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,cAAc,CAAC;QAC/C,GAAG,EAAE,MAAM;QACX,IAAI,EAAE,OAAO;QACb,SAAS,EAAE,SAAS;QACpB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,iBAAiB,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IAElE,iEAAiE;IAEjE,MAAM,EAAE,CAAC,6CAA6C,CAAC,CAAA;IACvD,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAA;IAE7C,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,EAAE,CAAC,iDAAiD,CAAC,CAAA;QAE3D,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,EAAE,CAAC,iBAAiB,OAAO,EAAE,CAAC,CAAA;IACpC,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,CAAA;IAE3E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;IACtC,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAA;IACtC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IACxC,MAAM,EAAE,CAAC,eAAe,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IAEhE,wCAAwC;IAExC,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAClC,MAAM,EAAE,CAAC,0BAA0B,CAAC,CAAA;IAEpC,gEAAgE;IAEhE,MAAM,QAAQ,GAAmB;QAChC,UAAU,EAAE,MAAM;QAClB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,OAAO;QACjB,MAAM,EAAE,MAAM;QACd,KAAK,EAAE,OAAO;KACd,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,MAAM,EAAE,CAAC,CAAA;IAC1E,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IAEjD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}
|
|
@@ -52,7 +52,9 @@
|
|
|
52
52
|
*/
|
|
53
53
|
import type { BaseFetchOptions, FetchSummary } from "./download.ts";
|
|
54
54
|
export interface FetchOpenAddressesOptions extends BaseFetchOptions {
|
|
55
|
-
/**
|
|
55
|
+
/**
|
|
56
|
+
* OA country collection code. Default `ca`.
|
|
57
|
+
*/
|
|
56
58
|
country?: string;
|
|
57
59
|
}
|
|
58
60
|
export declare function fetchOpenAddresses(options: FetchOpenAddressesOptions, report?: (line: string) => void): Promise<FetchSummary>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openaddresses.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/openaddresses.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;
|
|
1
|
+
{"version":3,"file":"openaddresses.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/openaddresses.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AAgBH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAoCnE,MAAM,WAAW,yBAA0B,SAAQ,gBAAgB;IAClE;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB;AA2GD,wBAAsB,kBAAkB,CACvC,OAAO,EAAE,yBAAyB,EAClC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA6LvB"}
|
|
@@ -50,6 +50,8 @@
|
|
|
50
50
|
* mailwoman corpus fetch openaddresses --country ca
|
|
51
51
|
* ```
|
|
52
52
|
*/
|
|
53
|
+
/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
|
|
54
|
+
procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
|
|
53
55
|
import { execFile, spawn } from "node:child_process";
|
|
54
56
|
import { createReadStream, createWriteStream, existsSync, mkdirSync, renameSync, rmSync, statSync } from "node:fs";
|
|
55
57
|
import { join } from "node:path";
|
|
@@ -60,6 +62,19 @@ import { promisify } from "node:util";
|
|
|
60
62
|
import { $private } from "@mailwoman/core/env";
|
|
61
63
|
import { sha256File } from "@mailwoman/core/utils";
|
|
62
64
|
import { isTransientStatus, writeManifest } from "./download.js";
|
|
65
|
+
/**
|
|
66
|
+
* Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
|
|
67
|
+
* than data.
|
|
68
|
+
*/
|
|
69
|
+
/**
|
|
70
|
+
* A successful fetch; anything else is an error page or a redirect we did not follow.
|
|
71
|
+
*/
|
|
72
|
+
const HTTP_OK = 200;
|
|
73
|
+
/**
|
|
74
|
+
* Smallest plausible OpenAddresses shard. Below 10 KiB the file is a stub or an error body.
|
|
75
|
+
*/
|
|
76
|
+
const MIN_PLAUSIBLE_SHARD_BYTES = 10_240;
|
|
77
|
+
const BYTES_PER_KIB = 1024;
|
|
63
78
|
const execFileAsync = promisify(execFile);
|
|
64
79
|
const OA_BASE = "https://batch.openaddresses.io";
|
|
65
80
|
/**
|
|
@@ -74,12 +89,14 @@ const OA_COLLECTION_IDS = {
|
|
|
74
89
|
"us-midwest": 5,
|
|
75
90
|
global: 1,
|
|
76
91
|
};
|
|
77
|
-
/**
|
|
92
|
+
/**
|
|
93
|
+
* Stream-count newlines, matching `wc -l` (memory-safe for the multi-GB collection).
|
|
94
|
+
*/
|
|
78
95
|
async function countLines(path) {
|
|
79
96
|
let count = 0;
|
|
80
97
|
for await (const chunk of createReadStream(path)) {
|
|
81
|
-
for (
|
|
82
|
-
if (
|
|
98
|
+
for (const byte of chunk) {
|
|
99
|
+
if (byte === 0x0a) {
|
|
83
100
|
count++;
|
|
84
101
|
}
|
|
85
102
|
}
|
|
@@ -90,7 +107,7 @@ function humanBytes(bytes) {
|
|
|
90
107
|
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
91
108
|
let value = bytes;
|
|
92
109
|
let unit = 0;
|
|
93
|
-
while (value >=
|
|
110
|
+
while (value >= BYTES_PER_KIB && unit < units.length - 1) {
|
|
94
111
|
value /= 1024;
|
|
95
112
|
unit++;
|
|
96
113
|
}
|
|
@@ -132,7 +149,9 @@ async function streamDownload(url, dest, opts) {
|
|
|
132
149
|
}
|
|
133
150
|
return 0;
|
|
134
151
|
}
|
|
135
|
-
/**
|
|
152
|
+
/**
|
|
153
|
+
* Decompress `src` → `dest` with the same deprioritized subprocess the old fetcher used.
|
|
154
|
+
*/
|
|
136
155
|
async function gunzipToFile(src, dest) {
|
|
137
156
|
const child = spawn("nice", ["-n", "15", "ionice", "-c", "3", "gunzip", "-c", src], {
|
|
138
157
|
stdio: ["ignore", "pipe", "inherit"],
|
|
@@ -153,9 +172,7 @@ export async function fetchOpenAddresses(options, report) {
|
|
|
153
172
|
report?.(`=== fetch openaddresses: country=${country}`);
|
|
154
173
|
report?.(` dest: ${destDir}`);
|
|
155
174
|
mkdirSync(destDir, { recursive: true });
|
|
156
|
-
//
|
|
157
|
-
// Authentication check
|
|
158
|
-
// -------------------------------------------------------------------------
|
|
175
|
+
// MARK: Authentication check
|
|
159
176
|
if (!token) {
|
|
160
177
|
report?.(`
|
|
161
178
|
ERROR: OA_BATCH_TOKEN is not set.
|
|
@@ -177,9 +194,7 @@ The Canada collection (ca) is ~2 GiB compressed / ~7 GiB uncompressed
|
|
|
177
194
|
`);
|
|
178
195
|
return fail("OA_BATCH_TOKEN");
|
|
179
196
|
}
|
|
180
|
-
//
|
|
181
|
-
// Determine collection ID
|
|
182
|
-
// -------------------------------------------------------------------------
|
|
197
|
+
// MARK: Determine collection ID
|
|
183
198
|
let collectionID = OA_COLLECTION_IDS[country];
|
|
184
199
|
if (collectionID === undefined) {
|
|
185
200
|
report?.(`Unknown country code '${country}'. Fetching collection list to find ID...`);
|
|
@@ -205,9 +220,7 @@ The Canada collection (ca) is ~2 GiB compressed / ~7 GiB uncompressed
|
|
|
205
220
|
collectionID = match.id;
|
|
206
221
|
report?.(` Found collection id=${collectionID} for '${country}'`);
|
|
207
222
|
}
|
|
208
|
-
//
|
|
209
|
-
// Download via the collections download endpoint (302s to S3)
|
|
210
|
-
// -------------------------------------------------------------------------
|
|
223
|
+
// MARK: Download via the collections download endpoint (302s to S3)
|
|
211
224
|
report?.(` Resolving download URL for collection id=${collectionID}...`);
|
|
212
225
|
report?.(` Attempting authenticated download...`);
|
|
213
226
|
const tmpGz = join(destDir, "collection.geojsonl.gz.tmp");
|
|
@@ -219,7 +232,7 @@ The Canada collection (ca) is ~2 GiB compressed / ~7 GiB uncompressed
|
|
|
219
232
|
retries: 3,
|
|
220
233
|
retryDelayMs: 30_000,
|
|
221
234
|
});
|
|
222
|
-
if (httpStatus !==
|
|
235
|
+
if (httpStatus !== HTTP_OK) {
|
|
223
236
|
// Try the geojsonl.gz directly with token as query param (alternate URL shape).
|
|
224
237
|
httpStatus = await streamDownload(`${OA_BASE}/api/collections/${collectionID}/geojsonl.gz?token=${token}`, tmpGz, {
|
|
225
238
|
timeoutMs: 7_200_000,
|
|
@@ -227,7 +240,7 @@ The Canada collection (ca) is ~2 GiB compressed / ~7 GiB uncompressed
|
|
|
227
240
|
retryDelayMs: 30_000,
|
|
228
241
|
});
|
|
229
242
|
}
|
|
230
|
-
if (httpStatus !==
|
|
243
|
+
if (httpStatus !== HTTP_OK) {
|
|
231
244
|
rmSync(tmpGz, { force: true });
|
|
232
245
|
report?.(`
|
|
233
246
|
ERROR: Download returned HTTP ${httpStatus}.
|
|
@@ -247,9 +260,7 @@ URL tried: ${OA_BASE}/api/collections/${collectionID}/download
|
|
|
247
260
|
`);
|
|
248
261
|
return fail(country);
|
|
249
262
|
}
|
|
250
|
-
//
|
|
251
|
-
// Decompress if the downloaded file is gzipped
|
|
252
|
-
// -------------------------------------------------------------------------
|
|
263
|
+
// MARK: Decompress if the downloaded file is gzipped
|
|
253
264
|
const fileMagic = (await execFileAsync("file", ["--brief", tmpGz]).catch(() => ({ stdout: "" }))).stdout;
|
|
254
265
|
if (/gzip|compressed/i.test(fileMagic)) {
|
|
255
266
|
report?.(` Decompressing gzip archive...`);
|
|
@@ -267,15 +278,13 @@ URL tried: ${OA_BASE}/api/collections/${collectionID}/download
|
|
|
267
278
|
renameSync(tmpGz, outputFile);
|
|
268
279
|
report?.(` WARNING: Downloaded file type is '${fileMagic.trim()}' — may need manual decompression.`);
|
|
269
280
|
}
|
|
270
|
-
//
|
|
271
|
-
// Verify + write MANIFEST
|
|
272
|
-
// -------------------------------------------------------------------------
|
|
281
|
+
// MARK: Verify + write MANIFEST
|
|
273
282
|
if (!existsSync(outputFile)) {
|
|
274
283
|
report?.(`ERROR: Output file not found at ${outputFile} after download.`);
|
|
275
284
|
return fail(country);
|
|
276
285
|
}
|
|
277
286
|
const size = statSync(outputFile).size;
|
|
278
|
-
if (size <
|
|
287
|
+
if (size < MIN_PLAUSIBLE_SHARD_BYTES) {
|
|
279
288
|
report?.(`ERROR: File is suspiciously small (${size} bytes) — likely an error response.`);
|
|
280
289
|
return fail(country);
|
|
281
290
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openaddresses.js","sourceRoot":"","sources":["../../../../src/tools/fetch/openaddresses.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AAEH,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAA;AACpD,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAClH,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAErC,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAEhE,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAA;AAEzC,MAAM,OAAO,GAAG,gCAAgC,CAAA;AAEhD;;;GAGG;AACH,MAAM,iBAAiB,GAA2B;IACjD,EAAE,EAAE,CAAC;IACL,SAAS,EAAE,CAAC;IACZ,UAAU,EAAE,CAAC;IACb,cAAc,EAAE,CAAC;IACjB,YAAY,EAAE,CAAC;IACf,MAAM,EAAE,CAAC;CACT,CAAA;
|
|
1
|
+
{"version":3,"file":"openaddresses.js","sourceRoot":"","sources":["../../../../src/tools/fetch/openaddresses.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AAEH;kGACkG;AAElG,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAA;AACpD,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAClH,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAErC,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAEhE;;;GAGG;AACH;;GAEG;AACH,MAAM,OAAO,GAAG,GAAG,CAAA;AAEnB;;GAEG;AACH,MAAM,yBAAyB,GAAG,MAAM,CAAA;AAExC,MAAM,aAAa,GAAG,IAAI,CAAA;AAE1B,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAA;AAEzC,MAAM,OAAO,GAAG,gCAAgC,CAAA;AAEhD;;;GAGG;AACH,MAAM,iBAAiB,GAA2B;IACjD,EAAE,EAAE,CAAC;IACL,SAAS,EAAE,CAAC;IACZ,UAAU,EAAE,CAAC;IACb,cAAc,EAAE,CAAC;IACjB,YAAY,EAAE,CAAC;IACf,MAAM,EAAE,CAAC;CACT,CAAA;AAgBD;;GAEG;AACH,KAAK,UAAU,UAAU,CAAC,IAAY;IACrC,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,gBAAgB,CAAC,IAAI,CAA0B,EAAE,CAAC;QAC3E,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;gBACnB,KAAK,EAAE,CAAA;YACR,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAChC,MAAM,KAAK,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAA;IAC/C,IAAI,KAAK,GAAG,KAAK,CAAA;IACjB,IAAI,IAAI,GAAG,CAAC,CAAA;IAEZ,OAAO,KAAK,IAAI,aAAa,IAAI,IAAI,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1D,KAAK,IAAI,IAAI,CAAA;QAEb,IAAI,EAAE,CAAA;IACP,CAAC;IAED,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAA;AAC5D,CAAC;AASD;;;;;;;GAOG;AACH,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,IAAY,EAAE,IAAwB;IAChF,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QAC1D,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC5B,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;gBAC3B,QAAQ,EAAE,QAAQ;gBAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;aAC3C,CAAC,CAAA;YAEF,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACxB,MAAM,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAA;gBAEnE,OAAO,GAAG,CAAC,MAAM,CAAA;YAClB,CAAC;YAED,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7D,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAE9B,SAAQ;YACT,CAAC;YAED,OAAO,GAAG,CAAC,MAAM,CAAA;QAClB,CAAC;QAAC,MAAM,CAAC;YACR,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC5B,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAE9B,SAAQ;YACT,CAAC;YAED,OAAO,CAAC,CAAA;QACT,CAAC;IACF,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,YAAY,CAAC,GAAW,EAAE,IAAY;IACpD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,EAAE;QACnF,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC;KACpC,CAAC,CAAA;IAEF,MAAM,QAAQ,CAAC,KAAK,CAAC,MAAO,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAA;IAEtD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;QAC5G,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IAC1B,CAAC,CAAC,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,OAAkC,EAClC,MAA+B;IAE/B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAA;IACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,cAAc,CAAA;IAErC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,CAAC,CAAA;IAC/D,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IACnD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAA;IAEvD,MAAM,IAAI,GAAG,CAAC,IAAY,EAAgB,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAEzG,MAAM,EAAE,CAAC,oCAAoC,OAAO,EAAE,CAAC,CAAA;IACvD,MAAM,EAAE,CAAC,aAAa,OAAO,EAAE,CAAC,CAAA;IAEhC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvC,6BAA6B;IAE7B,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC;;;;;;;;;;;;;;;;;CAiBV,CAAC,CAAA;QAEA,OAAO,IAAI,CAAC,gBAAgB,CAAC,CAAA;IAC9B,CAAC;IAED,gCAAgC;IAEhC,IAAI,YAAY,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;IAE7C,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QAChC,MAAM,EAAE,CAAC,yBAAyB,OAAO,2CAA2C,CAAC,CAAA;QAErF,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,kBAAkB,EAAE;YACrD,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,KAAK,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;YAC5E,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;SACnC,CAAC,CAAA;QAEF,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,EAAE,CAAC,6CAA6C,GAAG,CAAC,MAAM,GAAG,CAAC,CAAA;YAEpE,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;QACrB,CAAC;QAED,MAAM,WAAW,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAmB,CAAA;QACxD,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,CAAA;QAE/D,IAAI,KAAK,EAAE,EAAE,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,EAAE,CAAC,6CAA6C,OAAO,4BAA4B,CAAC,CAAA;YAC1F,MAAM,EAAE,CAAC,wBAAwB,CAAC,CAAA;YAElC,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;gBAChC,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,cAAc,EAAE,CAAA;gBAC9C,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,KAAK,IAAI,EAAE,UAAU,IAAI,QAAQ,CAAC,CAAA;YACtG,CAAC;YAED,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;QACrB,CAAC;QAED,YAAY,GAAG,KAAK,CAAC,EAAE,CAAA;QACvB,MAAM,EAAE,CAAC,yBAAyB,YAAY,SAAS,OAAO,GAAG,CAAC,CAAA;IACnE,CAAC;IAED,oEAAoE;IAEpE,MAAM,EAAE,CAAC,8CAA8C,YAAY,KAAK,CAAC,CAAA;IACzE,MAAM,EAAE,CAAC,wCAAwC,CAAC,CAAA;IAElD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,4BAA4B,CAAC,CAAA;IACzD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,yBAAyB,CAAC,CAAA;IACvD,MAAM,SAAS,GAAG,GAAG,OAAO,oBAAoB,YAAY,WAAW,CAAA;IAEvE,IAAI,UAAU,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,KAAK,EAAE;QACvD,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,KAAK,EAAE,EAAE;QAC7C,SAAS,EAAE,SAAS;QACpB,OAAO,EAAE,CAAC;QACV,YAAY,EAAE,MAAM;KACpB,CAAC,CAAA;IAEF,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;QAC5B,gFAAgF;QAChF,UAAU,GAAG,MAAM,cAAc,CAAC,GAAG,OAAO,oBAAoB,YAAY,sBAAsB,KAAK,EAAE,EAAE,KAAK,EAAE;YACjH,SAAS,EAAE,SAAS;YACpB,OAAO,EAAE,CAAC;YACV,YAAY,EAAE,MAAM;SACpB,CAAC,CAAA;IACH,CAAC;IAED,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;QAC5B,MAAM,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QAE9B,MAAM,EAAE,CAAC;gCACqB,UAAU;;;;;;;;;4DASkB,YAAY;;eAEzD,UAAU;;aAEZ,OAAO,oBAAoB,YAAY;CACnD,CAAC,CAAA;QAEA,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;IACrB,CAAC;IAED,qDAAqD;IAErD,MAAM,SAAS,GAAG,CAAC,MAAM,aAAa,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;IAExG,IAAI,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACxC,MAAM,EAAE,CAAC,iCAAiC,CAAC,CAAA;QAC3C,MAAM,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACjC,MAAM,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QAC9B,UAAU,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;IAC/B,CAAC;SAAM,IAAI,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QAChD,kCAAkC;QAClC,UAAU,CAAC,KAAK,EAAE,UAAU,CAAC,CAAA;QAC7B,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAChC,CAAC;SAAM,CAAC;QACP,0DAA0D;QAC1D,UAAU,CAAC,KAAK,EAAE,UAAU,CAAC,CAAA;QAC7B,MAAM,EAAE,CAAC,uCAAuC,SAAS,CAAC,IAAI,EAAE,oCAAoC,CAAC,CAAA;IACtG,CAAC;IAED,gCAAgC;IAEhC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC7B,MAAM,EAAE,CAAC,mCAAmC,UAAU,kBAAkB,CAAC,CAAA;QAEzE,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;IACrB,CAAC;IAED,MAAM,IAAI,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,CAAA;IAEtC,IAAI,IAAI,GAAG,yBAAyB,EAAE,CAAC;QACtC,MAAM,EAAE,CAAC,sCAAsC,IAAI,qCAAqC,CAAC,CAAA;QAEzF,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;IACrB,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,CAAA;IACxC,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,CAAA;IAC7C,MAAM,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAA;IAEvE,MAAM,QAAQ,GAAG;QAChB,UAAU,EAAE,SAAS;QACrB,aAAa,EAAE,YAAY;QAC3B,OAAO;QACP,QAAQ,EAAE,qBAAqB;QAC/B,aAAa,EAAE,YAAY;QAC3B,MAAM,EAAE,GAAG;QACX,KAAK,EAAE,IAAI;QACX,SAAS,EAAE,QAAQ;QACnB,KAAK,EACJ,8LAA8L;KAC/L,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,UAAU,CAAC,IAAI,CAAC,UAAU,QAAQ,YAAY,GAAG,EAAE,CAAC,CAAA;IACpE,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IACjD,MAAM,EAAE,CAAC,UAAU,CAAC,CAAA;IACpB,MAAM,EAAE,CAAC,sBAAsB,CAAC,CAAA;IAChC,MAAM,EAAE,CAAC,yCAAyC,CAAC,CAAA;IACnD,MAAM,EAAE,CAAC,eAAe,UAAU,KAAK,CAAC,CAAA;IACxC,MAAM,EAAE,CAAC,iBAAiB,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,gBAAgB,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;IAE3C,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}
|