@mailwoman/corpus 10.1.0 → 10.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/licenses/cdla-permissive-2.0.md +1 -1
- package/lib/adapters/cordis/adapter.ts +13 -7
- package/lib/adapters/dedup-key.ts +1 -1
- package/lib/adapters/geonames/adapter.ts +1 -1
- package/lib/adapters/gleif/adapter.ts +17 -16
- package/lib/adapters/libpostal-dictionary.ts +0 -2
- package/lib/adapters/openaddresses/adapter.ts +4 -3
- package/lib/adapters/osm/adapter.ts +22 -21
- package/lib/adapters/po-box/adapter.ts +10 -5
- package/lib/adapters/registry.ts +4 -4
- package/lib/adapters/source-id.ts +2 -2
- package/lib/adapters/street-line.ts +7 -6
- package/lib/adapters/wof/admin/json/adapter.ts +2 -2
- package/lib/adapters/wof/json-rows.ts +2 -1
- package/lib/adapters/wof/postalcode-json/adapter.ts +6 -2
- package/lib/adapters.ts +5 -5
- package/lib/au/adapters/gnaf/adapter.ts +1 -1
- package/lib/be/adapters/brussels/adapter.ts +22 -22
- package/lib/be/adapters/vlaanderen/adapter.ts +54 -57
- package/lib/be/adapters/wallonie/adapter.ts +17 -17
- package/lib/be/tools/fetch/brussels.ts +13 -10
- package/lib/be/tools/fetch/wallonie.ts +8 -7
- package/lib/build/adapters.ts +2 -1
- package/lib/build/align.ts +5 -4
- package/lib/build/checkpoint.ts +5 -3
- package/lib/build/eligibility.ts +3 -3
- package/lib/build/types.ts +8 -6
- package/lib/cz/adapters/cuzk/adapter.ts +41 -38
- package/lib/cz/tools/fetch/cuzk.ts +42 -31
- package/lib/de/recipes/locale.ts +1 -1
- package/lib/dk/adapters/inspire/adapter.ts +38 -38
- package/lib/dk/tools/fetch/inspire.ts +16 -14
- package/lib/ee/adapters/ads/adapter.ts +26 -32
- package/lib/ee/tools/fetch/ads.ts +10 -8
- package/lib/es/adapters/bizkaia/adapter.ts +47 -47
- package/lib/es/adapters/catastro/adapter.ts +36 -34
- package/lib/es/adapters/gipuzkoa/adapter.ts +38 -45
- package/lib/es/adapters/navarra/adapter.ts +28 -27
- package/lib/es/cadastre.ts +31 -31
- package/lib/es/tools/fetch/bizkaia.ts +40 -36
- package/lib/es/tools/fetch/catastro.ts +51 -40
- package/lib/es/tools/fetch/gipuzkoa.ts +13 -11
- package/lib/es/tools/fetch/navarra.ts +33 -25
- package/lib/fi/adapters/ryhti/adapter.ts +22 -26
- package/lib/fi/tools/fetch/ryhti.ts +31 -31
- package/lib/fr/adapters/annuaire-education/adapter.ts +13 -11
- package/lib/fr/adapters/ban/adapter.ts +6 -6
- package/lib/fr/adapters/finess/adapter.ts +16 -15
- package/lib/fr/adapters/finess/voie-type.ts +4 -3
- package/lib/fr/insee-country.ts +2 -2
- package/lib/fr/recipes/lieudit.ts +8 -6
- package/lib/fr/tools/fetch/annuaire-education.ts +11 -8
- package/lib/fr/tools/fetch/ban.ts +3 -2
- package/lib/fr/tools/fetch/finess.ts +14 -13
- package/lib/inspire/address.ts +86 -78
- package/lib/inspire/errors.ts +9 -8
- package/lib/inspire/stream.ts +29 -26
- package/lib/international/recipes/locale.ts +13 -9
- package/lib/it/adapters/anac/adapter.ts +51 -46
- package/lib/it/tools/fetch/anac.ts +22 -25
- package/lib/jp/adapters/wof-admin/adapter.ts +2 -6
- package/lib/jp/tools/fetch/houjin.ts +3 -3
- package/lib/kr/adapters/localdata/permit-rows.ts +1 -1
- package/lib/kr/tools/fetch/juso.ts +1 -1
- package/lib/kr/tools/fetch/localdata.ts +2 -2
- package/lib/lu/adapters/bd-adresses/adapter.ts +10 -10
- package/lib/lu/tools/fetch/bd-adresses.ts +15 -13
- package/lib/lv/adapters/varis/adapter.ts +24 -23
- package/lib/lv/tools/fetch/varis.ts +3 -2
- package/lib/nl/adapters/kadaster/adapter.ts +29 -29
- package/lib/nl/tools/fetch/kadaster.ts +16 -12
- package/lib/no/adapters/matrikkelen/adapter.ts +6 -5
- package/lib/no/tools/fetch/matrikkelen.ts +59 -47
- package/lib/parquet/duckdb.ts +7 -4
- package/lib/parquet/readers.ts +3 -3
- package/lib/parquet/schema.ts +1 -1
- package/lib/parquet/writers.ts +3 -2
- package/lib/pl/adapters/emuia/adapter.ts +12 -12
- package/lib/pl/tools/fetch/emuia.ts +10 -9
- package/lib/recipes/bare/postcode.ts +1 -1
- package/lib/recipes/boundary-stress.ts +1 -1
- package/lib/recipes/house-venue.ts +7 -3
- package/lib/recipes/intersection.ts +2 -2
- package/lib/recipes/po/box/cedex/recipe.ts +3 -3
- package/lib/recipes/po/box/cedex/sources.ts +13 -1
- package/lib/recipes/scaffold.ts +31 -21
- package/lib/recipes/sources.ts +16 -3
- package/lib/recipes/street/bare.ts +2 -2
- package/lib/recipes/street.ts +3 -1
- package/lib/recipes/sub/venue/context.ts +2 -0
- package/lib/recipes/sub/venue/render.ts +1 -1
- package/lib/recipes/sub/venue/sources.ts +24 -29
- package/lib/recipes/sub/venue.ts +12 -18
- package/lib/recipes/trailing-region.ts +6 -2
- package/lib/recipes/unit.ts +3 -3
- package/lib/recipes/venue.ts +347 -0
- package/lib/recipes.ts +4 -2
- package/lib/registers.ts +8 -6
- package/lib/ro/adapters/retea-scolara/adapter.ts +5 -5
- package/lib/ro/tools/fetch/retea-scolara.ts +1 -1
- package/lib/runner.ts +4 -3
- package/lib/sg/recipes/register.ts +1 -1
- package/lib/sg/tools/fetch/acra.ts +3 -3
- package/lib/sk/adapters/inspire/adapter.ts +25 -27
- package/lib/sk/tools/fetch/inspire.ts +15 -14
- package/lib/source-register/effective-manifest.ts +3 -3
- package/lib/source-register/register.ts +3 -3
- package/lib/source-register/training-manifest.ts +1 -1
- package/lib/source-register/types.ts +12 -13
- package/lib/subvenue/promotions.ts +1 -1
- package/lib/subvenue/table.ts +4 -4
- package/lib/surfaces/locale.ts +2 -0
- package/lib/synthesizers/anchor-absorption.ts +2 -1
- package/lib/synthesizers/boundary-stress.ts +20 -26
- package/lib/synthesizers/house-venue.ts +26 -30
- package/lib/synthesizers/no-street.ts +2 -1
- package/lib/synthesizers/po-box.ts +6 -3
- package/lib/synthesizers/tail.ts +55 -0
- package/lib/synthesizers/utils.ts +19 -2
- package/lib/test-kit.ts +2 -2
- package/lib/tools/audit.ts +7 -5
- package/lib/tools/corpus-stats.ts +1 -1
- package/lib/tools/fetch/atom.ts +10 -10
- package/lib/tools/fetch/cordis.ts +11 -9
- package/lib/tools/fetch/download/manifest.ts +2 -1
- package/lib/tools/fetch/download/network.ts +4 -4
- package/lib/tools/fetch/download/retry.ts +4 -4
- package/lib/tools/fetch/geonames/dump.ts +5 -3
- package/lib/tools/fetch/geonames/postal.ts +2 -2
- package/lib/tools/fetch/gleif.ts +6 -4
- package/lib/tools/fetch/header.ts +8 -6
- package/lib/tools/fetch/inspire-addresses.ts +41 -36
- package/lib/tools/fetch/openaddresses.ts +1 -1
- package/lib/tools/fetch/vlaanderen-ad.ts +17 -17
- package/lib/tools/fetch/wfs-harvest.ts +39 -28
- package/lib/tools/fetch.ts +0 -42
- package/lib/tools/golden/expand.ts +8 -8
- package/lib/tools/golden/promote.ts +12 -8
- package/lib/tools/golden/relabel-street/directory.ts +7 -6
- package/lib/tools/golden/relabel-street/row.ts +7 -6
- package/lib/tools/ingest-csv.ts +3 -3
- package/lib/tools/jsonl-to-parquet.ts +1 -1
- package/lib/tools/lint/recipe-output/vocab.ts +10 -5
- package/lib/tools/lint/recipe-output.ts +26 -8
- package/lib/tools/overlay/kryptonite.ts +4 -3
- package/lib/tools/overlay/manifest.ts +4 -2
- package/lib/tools/overlay/merge-source.ts +2 -1
- package/lib/tools/overlay/split-slice.ts +2 -2
- package/lib/tools/overlay/translit.ts +6 -4
- package/lib/tools/overture-subvenue.ts +1 -1
- package/lib/tools/postcode-triples.ts +5 -4
- package/lib/tools/source-register/build.ts +17 -14
- package/lib/tools/sub/venue/lexicon.ts +4 -8
- package/lib/tw/tools/fetch/gcis.ts +4 -4
- package/lib/types.ts +4 -3
- package/lib/us/adapters/state/hi-schools/workbook.ts +1 -1
- package/lib/us/adapters/usgov/nad/adapter.ts +11 -11
- package/lib/us/adapters/usgov/samhsa-treatment-locator/adapter.ts +4 -4
- package/lib/us/fips-state.ts +2 -2
- package/lib/{synthesizers → us/synthesizers}/intersection.ts +12 -13
- package/lib/{synthesizers → us/synthesizers}/street.ts +16 -15
- package/lib/us/tools/fetch/imls-pls.ts +1 -1
- package/lib/us/tools/fetch/nad.ts +7 -7
- package/lib/us/tools/fetch/nppes.ts +5 -5
- package/lib/us/tools/fetch/state/hi-schools.ts +3 -2
- package/lib/us/tools/fetch/tiger-full.ts +1 -1
- package/lib/utils/align.ts +13 -10
- package/lib/utils/golden.ts +2 -2
- package/lib/utils/license.ts +3 -3
- package/lib/utils/split.ts +19 -15
- package/lib/utils/tokenize.ts +7 -26
- package/lib/utils/wof-json.ts +4 -4
- package/out/adapters/cordis/adapter.d.ts +5 -3
- package/out/adapters/cordis/adapter.d.ts.map +1 -1
- package/out/adapters/cordis/adapter.js +13 -7
- package/out/adapters/cordis/adapter.js.map +1 -1
- package/out/adapters/dedup-key.d.ts +1 -1
- package/out/adapters/dedup-key.js +1 -1
- package/out/adapters/geonames/adapter.js +1 -1
- package/out/adapters/geonames/adapter.js.map +1 -1
- package/out/adapters/gleif/adapter.d.ts +9 -9
- package/out/adapters/gleif/adapter.d.ts.map +1 -1
- package/out/adapters/gleif/adapter.js +15 -14
- package/out/adapters/gleif/adapter.js.map +1 -1
- package/out/adapters/libpostal-dictionary.d.ts +0 -2
- package/out/adapters/libpostal-dictionary.d.ts.map +1 -1
- package/out/adapters/libpostal-dictionary.js +0 -2
- package/out/adapters/libpostal-dictionary.js.map +1 -1
- package/out/adapters/openaddresses/adapter.d.ts +3 -2
- package/out/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/adapters/openaddresses/adapter.js +1 -1
- package/out/adapters/openaddresses/adapter.js.map +1 -1
- package/out/adapters/osm/adapter.d.ts +17 -16
- package/out/adapters/osm/adapter.d.ts.map +1 -1
- package/out/adapters/osm/adapter.js +10 -9
- package/out/adapters/osm/adapter.js.map +1 -1
- package/out/adapters/po-box/adapter.d.ts +6 -3
- package/out/adapters/po-box/adapter.d.ts.map +1 -1
- package/out/adapters/po-box/adapter.js +4 -2
- package/out/adapters/po-box/adapter.js.map +1 -1
- package/out/adapters/registry.d.ts +3 -3
- package/out/adapters/registry.d.ts.map +1 -1
- package/out/adapters/registry.js +1 -1
- package/out/adapters/registry.js.map +1 -1
- package/out/adapters/source-id.d.ts +2 -2
- package/out/adapters/source-id.js +2 -2
- package/out/adapters/street-line.d.ts +7 -6
- package/out/adapters/street-line.d.ts.map +1 -1
- package/out/adapters/street-line.js +7 -6
- package/out/adapters/street-line.js.map +1 -1
- package/out/adapters/wof/admin/json/adapter.js +1 -1
- package/out/adapters/wof/admin/json/adapter.js.map +1 -1
- package/out/adapters/wof/json-rows.d.ts +2 -1
- package/out/adapters/wof/json-rows.d.ts.map +1 -1
- package/out/adapters/wof/json-rows.js.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.d.ts.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.js +5 -1
- package/out/adapters/wof/postalcode-json/adapter.js.map +1 -1
- package/out/adapters.d.ts +5 -5
- package/out/adapters.js +5 -5
- package/out/au/adapters/gnaf/adapter.js +1 -1
- package/out/au/adapters/gnaf/adapter.js.map +1 -1
- package/out/be/adapters/brussels/adapter.d.ts +9 -9
- package/out/be/adapters/brussels/adapter.d.ts.map +1 -1
- package/out/be/adapters/brussels/adapter.js +21 -21
- package/out/be/adapters/brussels/adapter.js.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.d.ts +34 -37
- package/out/be/adapters/vlaanderen/adapter.d.ts.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.js +40 -38
- package/out/be/adapters/vlaanderen/adapter.js.map +1 -1
- package/out/be/adapters/wallonie/adapter.d.ts +2 -2
- package/out/be/adapters/wallonie/adapter.js +17 -17
- package/out/be/adapters/wallonie/adapter.js.map +1 -1
- package/out/be/tools/fetch/brussels.d.ts +11 -8
- package/out/be/tools/fetch/brussels.d.ts.map +1 -1
- package/out/be/tools/fetch/brussels.js +10 -8
- package/out/be/tools/fetch/brussels.js.map +1 -1
- package/out/be/tools/fetch/wallonie.d.ts +6 -5
- package/out/be/tools/fetch/wallonie.d.ts.map +1 -1
- package/out/be/tools/fetch/wallonie.js +4 -4
- package/out/be/tools/fetch/wallonie.js.map +1 -1
- package/out/build/adapters.d.ts.map +1 -1
- package/out/build/adapters.js +2 -1
- package/out/build/adapters.js.map +1 -1
- package/out/build/align.d.ts +3 -2
- package/out/build/align.d.ts.map +1 -1
- package/out/build/align.js +2 -2
- package/out/build/align.js.map +1 -1
- package/out/build/checkpoint.d.ts +1 -1
- package/out/build/checkpoint.d.ts.map +1 -1
- package/out/build/checkpoint.js +4 -2
- package/out/build/checkpoint.js.map +1 -1
- package/out/build/eligibility.d.ts +2 -2
- package/out/build/eligibility.js +3 -3
- package/out/build/eligibility.js.map +1 -1
- package/out/build/types.d.ts +8 -6
- package/out/build/types.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.d.ts +14 -14
- package/out/cz/adapters/cuzk/adapter.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.js +38 -35
- package/out/cz/adapters/cuzk/adapter.js.map +1 -1
- package/out/cz/tools/fetch/cuzk.d.ts +31 -22
- package/out/cz/tools/fetch/cuzk.d.ts.map +1 -1
- package/out/cz/tools/fetch/cuzk.js +21 -19
- package/out/cz/tools/fetch/cuzk.js.map +1 -1
- package/out/de/recipes/locale.js +1 -1
- package/out/de/recipes/locale.js.map +1 -1
- package/out/dk/adapters/inspire/adapter.d.ts +32 -33
- package/out/dk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/dk/adapters/inspire/adapter.js +38 -38
- package/out/dk/adapters/inspire/adapter.js.map +1 -1
- package/out/dk/tools/fetch/inspire.d.ts +6 -5
- package/out/dk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/dk/tools/fetch/inspire.js +15 -13
- package/out/dk/tools/fetch/inspire.js.map +1 -1
- package/out/ee/adapters/ads/adapter.d.ts +19 -26
- package/out/ee/adapters/ads/adapter.d.ts.map +1 -1
- package/out/ee/adapters/ads/adapter.js +22 -28
- package/out/ee/adapters/ads/adapter.js.map +1 -1
- package/out/ee/tools/fetch/ads.d.ts +3 -3
- package/out/ee/tools/fetch/ads.d.ts.map +1 -1
- package/out/ee/tools/fetch/ads.js +10 -8
- package/out/ee/tools/fetch/ads.js.map +1 -1
- package/out/es/adapters/bizkaia/adapter.d.ts +28 -28
- package/out/es/adapters/bizkaia/adapter.d.ts.map +1 -1
- package/out/es/adapters/bizkaia/adapter.js +43 -43
- package/out/es/adapters/bizkaia/adapter.js.map +1 -1
- package/out/es/adapters/catastro/adapter.d.ts +22 -23
- package/out/es/adapters/catastro/adapter.d.ts.map +1 -1
- package/out/es/adapters/catastro/adapter.js +35 -33
- package/out/es/adapters/catastro/adapter.js.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.d.ts +31 -38
- package/out/es/adapters/gipuzkoa/adapter.d.ts.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.js +40 -47
- package/out/es/adapters/gipuzkoa/adapter.js.map +1 -1
- package/out/es/adapters/navarra/adapter.d.ts +20 -19
- package/out/es/adapters/navarra/adapter.d.ts.map +1 -1
- package/out/es/adapters/navarra/adapter.js +23 -22
- package/out/es/adapters/navarra/adapter.js.map +1 -1
- package/out/es/cadastre.d.ts +21 -20
- package/out/es/cadastre.d.ts.map +1 -1
- package/out/es/cadastre.js +19 -21
- package/out/es/cadastre.js.map +1 -1
- package/out/es/tools/fetch/bizkaia.d.ts +28 -25
- package/out/es/tools/fetch/bizkaia.d.ts.map +1 -1
- package/out/es/tools/fetch/bizkaia.js +25 -24
- package/out/es/tools/fetch/bizkaia.js.map +1 -1
- package/out/es/tools/fetch/catastro.d.ts +28 -20
- package/out/es/tools/fetch/catastro.d.ts.map +1 -1
- package/out/es/tools/fetch/catastro.js +36 -31
- package/out/es/tools/fetch/catastro.js.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.d.ts +11 -9
- package/out/es/tools/fetch/gipuzkoa.d.ts.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.js +9 -8
- package/out/es/tools/fetch/gipuzkoa.js.map +1 -1
- package/out/es/tools/fetch/navarra.d.ts +20 -15
- package/out/es/tools/fetch/navarra.d.ts.map +1 -1
- package/out/es/tools/fetch/navarra.js +21 -16
- package/out/es/tools/fetch/navarra.js.map +1 -1
- package/out/fi/adapters/ryhti/adapter.d.ts +16 -20
- package/out/fi/adapters/ryhti/adapter.d.ts.map +1 -1
- package/out/fi/adapters/ryhti/adapter.js +21 -25
- package/out/fi/adapters/ryhti/adapter.js.map +1 -1
- package/out/fi/tools/fetch/ryhti.d.ts +24 -24
- package/out/fi/tools/fetch/ryhti.d.ts.map +1 -1
- package/out/fi/tools/fetch/ryhti.js +21 -24
- package/out/fi/tools/fetch/ryhti.js.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.d.ts +8 -8
- package/out/fr/adapters/annuaire-education/adapter.d.ts.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.js +13 -11
- package/out/fr/adapters/annuaire-education/adapter.js.map +1 -1
- package/out/fr/adapters/ban/adapter.d.ts +4 -4
- package/out/fr/adapters/ban/adapter.js +6 -6
- package/out/fr/adapters/ban/adapter.js.map +1 -1
- package/out/fr/adapters/finess/adapter.d.ts +12 -11
- package/out/fr/adapters/finess/adapter.d.ts.map +1 -1
- package/out/fr/adapters/finess/adapter.js +16 -15
- package/out/fr/adapters/finess/adapter.js.map +1 -1
- package/out/fr/adapters/finess/voie-type.d.ts +2 -2
- package/out/fr/adapters/finess/voie-type.d.ts.map +1 -1
- package/out/fr/adapters/finess/voie-type.js +4 -3
- package/out/fr/adapters/finess/voie-type.js.map +1 -1
- package/out/fr/insee-country.d.ts +2 -2
- package/out/fr/insee-country.js +2 -2
- package/out/fr/recipes/lieudit.d.ts.map +1 -1
- package/out/fr/recipes/lieudit.js +5 -4
- package/out/fr/recipes/lieudit.js.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.d.ts +7 -4
- package/out/fr/tools/fetch/annuaire-education.d.ts.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.js +9 -7
- package/out/fr/tools/fetch/annuaire-education.js.map +1 -1
- package/out/fr/tools/fetch/ban.d.ts +1 -1
- package/out/fr/tools/fetch/ban.d.ts.map +1 -1
- package/out/fr/tools/fetch/ban.js +3 -2
- package/out/fr/tools/fetch/ban.js.map +1 -1
- package/out/fr/tools/fetch/finess.d.ts +5 -5
- package/out/fr/tools/fetch/finess.d.ts.map +1 -1
- package/out/fr/tools/fetch/finess.js +13 -12
- package/out/fr/tools/fetch/finess.js.map +1 -1
- package/out/inspire/address.d.ts +62 -52
- package/out/inspire/address.d.ts.map +1 -1
- package/out/inspire/address.js +71 -60
- package/out/inspire/address.js.map +1 -1
- package/out/inspire/errors.d.ts +9 -8
- package/out/inspire/errors.d.ts.map +1 -1
- package/out/inspire/errors.js +7 -6
- package/out/inspire/errors.js.map +1 -1
- package/out/inspire/stream.d.ts +25 -22
- package/out/inspire/stream.d.ts.map +1 -1
- package/out/inspire/stream.js +10 -12
- package/out/inspire/stream.js.map +1 -1
- package/out/international/recipes/locale.d.ts +2 -2
- package/out/international/recipes/locale.d.ts.map +1 -1
- package/out/international/recipes/locale.js +11 -7
- package/out/international/recipes/locale.js.map +1 -1
- package/out/it/adapters/anac/adapter.d.ts +45 -43
- package/out/it/adapters/anac/adapter.d.ts.map +1 -1
- package/out/it/adapters/anac/adapter.js +48 -43
- package/out/it/adapters/anac/adapter.js.map +1 -1
- package/out/it/tools/fetch/anac.d.ts +16 -19
- package/out/it/tools/fetch/anac.d.ts.map +1 -1
- package/out/it/tools/fetch/anac.js +20 -23
- package/out/it/tools/fetch/anac.js.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.d.ts.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.js.map +1 -1
- package/out/jp/tools/fetch/houjin.js +2 -2
- package/out/jp/tools/fetch/houjin.js.map +1 -1
- package/out/kr/adapters/localdata/permit-rows.js +1 -1
- package/out/kr/adapters/localdata/permit-rows.js.map +1 -1
- package/out/kr/tools/fetch/juso.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.js +1 -1
- package/out/kr/tools/fetch/localdata.js.map +1 -1
- package/out/lu/adapters/bd-adresses/adapter.d.ts +9 -9
- package/out/lu/adapters/bd-adresses/adapter.js +10 -10
- package/out/lu/tools/fetch/bd-adresses.d.ts +11 -9
- package/out/lu/tools/fetch/bd-adresses.d.ts.map +1 -1
- package/out/lu/tools/fetch/bd-adresses.js +11 -11
- package/out/lu/tools/fetch/bd-adresses.js.map +1 -1
- package/out/lv/adapters/varis/adapter.d.ts +24 -23
- package/out/lv/adapters/varis/adapter.d.ts.map +1 -1
- package/out/lv/adapters/varis/adapter.js +24 -23
- package/out/lv/adapters/varis/adapter.js.map +1 -1
- package/out/lv/tools/fetch/varis.d.ts +2 -1
- package/out/lv/tools/fetch/varis.d.ts.map +1 -1
- package/out/lv/tools/fetch/varis.js +1 -1
- package/out/lv/tools/fetch/varis.js.map +1 -1
- package/out/nl/adapters/kadaster/adapter.d.ts +12 -12
- package/out/nl/adapters/kadaster/adapter.d.ts.map +1 -1
- package/out/nl/adapters/kadaster/adapter.js +22 -23
- package/out/nl/adapters/kadaster/adapter.js.map +1 -1
- package/out/nl/tools/fetch/kadaster.d.ts +14 -10
- package/out/nl/tools/fetch/kadaster.d.ts.map +1 -1
- package/out/nl/tools/fetch/kadaster.js +12 -9
- package/out/nl/tools/fetch/kadaster.js.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.js +3 -3
- package/out/no/adapters/matrikkelen/adapter.js.map +1 -1
- package/out/no/tools/fetch/matrikkelen.d.ts +50 -38
- package/out/no/tools/fetch/matrikkelen.d.ts.map +1 -1
- package/out/no/tools/fetch/matrikkelen.js +44 -36
- package/out/no/tools/fetch/matrikkelen.js.map +1 -1
- package/out/parquet/duckdb.d.ts +7 -4
- package/out/parquet/duckdb.d.ts.map +1 -1
- package/out/parquet/duckdb.js +2 -1
- package/out/parquet/duckdb.js.map +1 -1
- package/out/parquet/readers.js +3 -3
- package/out/parquet/readers.js.map +1 -1
- package/out/parquet/schema.js +1 -1
- package/out/parquet/schema.js.map +1 -1
- package/out/parquet/writers.d.ts +1 -1
- package/out/parquet/writers.d.ts.map +1 -1
- package/out/parquet/writers.js +2 -1
- package/out/parquet/writers.js.map +1 -1
- package/out/pl/adapters/emuia/adapter.d.ts +8 -8
- package/out/pl/adapters/emuia/adapter.js +12 -12
- package/out/pl/adapters/emuia/adapter.js.map +1 -1
- package/out/pl/tools/fetch/emuia.d.ts +5 -4
- package/out/pl/tools/fetch/emuia.d.ts.map +1 -1
- package/out/pl/tools/fetch/emuia.js +10 -9
- package/out/pl/tools/fetch/emuia.js.map +1 -1
- package/out/recipes/bare/postcode.js +1 -1
- package/out/recipes/bare/postcode.js.map +1 -1
- package/out/recipes/boundary-stress.js +1 -1
- package/out/recipes/boundary-stress.js.map +1 -1
- package/out/recipes/house-venue.d.ts.map +1 -1
- package/out/recipes/house-venue.js +6 -3
- package/out/recipes/house-venue.js.map +1 -1
- package/out/recipes/intersection.js +2 -2
- package/out/recipes/intersection.js.map +1 -1
- package/out/recipes/po/box/cedex/recipe.js +2 -1
- package/out/recipes/po/box/cedex/recipe.js.map +1 -1
- package/out/recipes/po/box/cedex/sources.d.ts +8 -2
- package/out/recipes/po/box/cedex/sources.d.ts.map +1 -1
- package/out/recipes/po/box/cedex/sources.js +1 -1
- package/out/recipes/po/box/cedex/sources.js.map +1 -1
- package/out/recipes/scaffold.d.ts +28 -19
- package/out/recipes/scaffold.d.ts.map +1 -1
- package/out/recipes/scaffold.js +3 -2
- package/out/recipes/scaffold.js.map +1 -1
- package/out/recipes/sources.d.ts +1 -1
- package/out/recipes/sources.d.ts.map +1 -1
- package/out/recipes/sources.js +15 -2
- package/out/recipes/sources.js.map +1 -1
- package/out/recipes/street/bare.js +2 -2
- package/out/recipes/street/bare.js.map +1 -1
- package/out/recipes/street.d.ts.map +1 -1
- package/out/recipes/street.js +3 -1
- package/out/recipes/street.js.map +1 -1
- package/out/recipes/sub/venue/context.d.ts.map +1 -1
- package/out/recipes/sub/venue/context.js +2 -0
- package/out/recipes/sub/venue/context.js.map +1 -1
- package/out/recipes/sub/venue/render.js +1 -1
- package/out/recipes/sub/venue/render.js.map +1 -1
- package/out/recipes/sub/venue/sources.d.ts +6 -3
- package/out/recipes/sub/venue/sources.d.ts.map +1 -1
- package/out/recipes/sub/venue/sources.js +16 -20
- package/out/recipes/sub/venue/sources.js.map +1 -1
- package/out/recipes/sub/venue.d.ts +2 -5
- package/out/recipes/sub/venue.d.ts.map +1 -1
- package/out/recipes/sub/venue.js +10 -14
- package/out/recipes/sub/venue.js.map +1 -1
- package/out/recipes/trailing-region.d.ts.map +1 -1
- package/out/recipes/trailing-region.js +6 -1
- package/out/recipes/trailing-region.js.map +1 -1
- package/out/recipes/unit.js +2 -2
- package/out/recipes/unit.js.map +1 -1
- package/out/recipes/venue.d.ts +119 -0
- package/out/recipes/venue.d.ts.map +1 -0
- package/out/recipes/venue.js +245 -0
- package/out/recipes/venue.js.map +1 -0
- package/out/recipes.d.ts +1 -1
- package/out/recipes.d.ts.map +1 -1
- package/out/recipes.js +3 -1
- package/out/recipes.js.map +1 -1
- package/out/registers.d.ts +8 -6
- package/out/registers.d.ts.map +1 -1
- package/out/registers.js +8 -6
- package/out/registers.js.map +1 -1
- package/out/ro/adapters/retea-scolara/adapter.d.ts +5 -5
- package/out/ro/adapters/retea-scolara/adapter.js +5 -5
- package/out/ro/tools/fetch/retea-scolara.js +1 -1
- package/out/ro/tools/fetch/retea-scolara.js.map +1 -1
- package/out/runner.d.ts +3 -2
- package/out/runner.d.ts.map +1 -1
- package/out/runner.js +1 -1
- package/out/runner.js.map +1 -1
- package/out/sg/recipes/register.js +1 -1
- package/out/sg/recipes/register.js.map +1 -1
- package/out/sg/tools/fetch/acra.js +2 -2
- package/out/sg/tools/fetch/acra.js.map +1 -1
- package/out/sk/adapters/inspire/adapter.d.ts +5 -5
- package/out/sk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/sk/adapters/inspire/adapter.js +18 -18
- package/out/sk/adapters/inspire/adapter.js.map +1 -1
- package/out/sk/tools/fetch/inspire.d.ts +11 -10
- package/out/sk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/sk/tools/fetch/inspire.js +15 -14
- package/out/sk/tools/fetch/inspire.js.map +1 -1
- package/out/source-register/effective-manifest.js +3 -3
- package/out/source-register/effective-manifest.js.map +1 -1
- package/out/source-register/register.d.ts +2 -2
- package/out/source-register/register.d.ts.map +1 -1
- package/out/source-register/register.js +2 -2
- package/out/source-register/register.js.map +1 -1
- package/out/source-register/training-manifest.d.ts +1 -1
- package/out/source-register/types.d.ts +12 -13
- package/out/source-register/types.d.ts.map +1 -1
- package/out/source-register/types.js +2 -2
- package/out/source-register/types.js.map +1 -1
- package/out/subvenue/promotions.d.ts +1 -1
- package/out/subvenue/promotions.js +1 -1
- package/out/subvenue/table.d.ts +4 -4
- package/out/subvenue/table.js +3 -3
- package/out/surfaces/locale.d.ts.map +1 -1
- package/out/surfaces/locale.js +2 -0
- package/out/surfaces/locale.js.map +1 -1
- package/out/synthesizers/anchor-absorption.d.ts +2 -1
- package/out/synthesizers/anchor-absorption.d.ts.map +1 -1
- package/out/synthesizers/anchor-absorption.js.map +1 -1
- package/out/synthesizers/boundary-stress.d.ts +1 -1
- package/out/synthesizers/boundary-stress.d.ts.map +1 -1
- package/out/synthesizers/boundary-stress.js +20 -23
- package/out/synthesizers/boundary-stress.js.map +1 -1
- package/out/synthesizers/house-venue.d.ts +3 -2
- package/out/synthesizers/house-venue.d.ts.map +1 -1
- package/out/synthesizers/house-venue.js +24 -26
- package/out/synthesizers/house-venue.js.map +1 -1
- package/out/synthesizers/no-street.d.ts.map +1 -1
- package/out/synthesizers/no-street.js.map +1 -1
- package/out/synthesizers/po-box.d.ts +4 -2
- package/out/synthesizers/po-box.d.ts.map +1 -1
- package/out/synthesizers/po-box.js.map +1 -1
- package/out/synthesizers/tail.d.ts +36 -0
- package/out/synthesizers/tail.d.ts.map +1 -0
- package/out/synthesizers/tail.js +32 -0
- package/out/synthesizers/tail.js.map +1 -0
- package/out/synthesizers/utils.d.ts +2 -1
- package/out/synthesizers/utils.d.ts.map +1 -1
- package/out/synthesizers/utils.js +17 -1
- package/out/synthesizers/utils.js.map +1 -1
- package/out/test-kit.js +2 -2
- package/out/test-kit.js.map +1 -1
- package/out/tools/audit.d.ts +2 -1
- package/out/tools/audit.d.ts.map +1 -1
- package/out/tools/audit.js +4 -3
- package/out/tools/audit.js.map +1 -1
- package/out/tools/corpus-stats.js +1 -1
- package/out/tools/corpus-stats.js.map +1 -1
- package/out/tools/fetch/atom.d.ts +8 -8
- package/out/tools/fetch/atom.d.ts.map +1 -1
- package/out/tools/fetch/atom.js +7 -8
- package/out/tools/fetch/atom.js.map +1 -1
- package/out/tools/fetch/cordis.d.ts +4 -3
- package/out/tools/fetch/cordis.d.ts.map +1 -1
- package/out/tools/fetch/cordis.js +9 -7
- package/out/tools/fetch/cordis.js.map +1 -1
- package/out/tools/fetch/download/manifest.d.ts.map +1 -1
- package/out/tools/fetch/download/manifest.js +2 -1
- package/out/tools/fetch/download/manifest.js.map +1 -1
- package/out/tools/fetch/download/network.d.ts +4 -4
- package/out/tools/fetch/download/retry.js +4 -4
- package/out/tools/fetch/download/retry.js.map +1 -1
- package/out/tools/fetch/geonames/dump.d.ts +3 -2
- package/out/tools/fetch/geonames/dump.d.ts.map +1 -1
- package/out/tools/fetch/geonames/dump.js +2 -1
- package/out/tools/fetch/geonames/dump.js.map +1 -1
- package/out/tools/fetch/geonames/postal.d.ts +2 -2
- package/out/tools/fetch/gleif.d.ts +3 -2
- package/out/tools/fetch/gleif.d.ts.map +1 -1
- package/out/tools/fetch/gleif.js +6 -4
- package/out/tools/fetch/gleif.js.map +1 -1
- package/out/tools/fetch/header.d.ts +7 -5
- package/out/tools/fetch/header.d.ts.map +1 -1
- package/out/tools/fetch/header.js +8 -6
- package/out/tools/fetch/header.js.map +1 -1
- package/out/tools/fetch/inspire-addresses.d.ts +32 -27
- package/out/tools/fetch/inspire-addresses.d.ts.map +1 -1
- package/out/tools/fetch/inspire-addresses.js +25 -24
- package/out/tools/fetch/inspire-addresses.js.map +1 -1
- package/out/tools/fetch/openaddresses.js +1 -1
- package/out/tools/fetch/openaddresses.js.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.d.ts +5 -5
- package/out/tools/fetch/vlaanderen-ad.d.ts.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.js +16 -16
- package/out/tools/fetch/vlaanderen-ad.js.map +1 -1
- package/out/tools/fetch/wfs-harvest.d.ts +26 -16
- package/out/tools/fetch/wfs-harvest.d.ts.map +1 -1
- package/out/tools/fetch/wfs-harvest.js +21 -18
- package/out/tools/fetch/wfs-harvest.js.map +1 -1
- package/out/tools/fetch.d.ts +0 -41
- package/out/tools/fetch.d.ts.map +1 -1
- package/out/tools/fetch.js +0 -41
- package/out/tools/fetch.js.map +1 -1
- package/out/tools/golden/expand.d.ts +7 -7
- package/out/tools/golden/expand.js +1 -1
- package/out/tools/golden/expand.js.map +1 -1
- package/out/tools/golden/promote.d.ts +2 -2
- package/out/tools/golden/promote.d.ts.map +1 -1
- package/out/tools/golden/promote.js +9 -6
- package/out/tools/golden/promote.js.map +1 -1
- package/out/tools/golden/relabel-street/directory.d.ts +4 -4
- package/out/tools/golden/relabel-street/directory.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/directory.js +3 -2
- package/out/tools/golden/relabel-street/directory.js.map +1 -1
- package/out/tools/golden/relabel-street/row.d.ts +4 -3
- package/out/tools/golden/relabel-street/row.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/row.js +3 -3
- package/out/tools/golden/relabel-street/row.js.map +1 -1
- package/out/tools/ingest-csv.js +3 -3
- package/out/tools/ingest-csv.js.map +1 -1
- package/out/tools/jsonl-to-parquet.d.ts +1 -1
- package/out/tools/lint/recipe-output/vocab.d.ts +10 -5
- package/out/tools/lint/recipe-output/vocab.d.ts.map +1 -1
- package/out/tools/lint/recipe-output/vocab.js.map +1 -1
- package/out/tools/lint/recipe-output.d.ts +8 -8
- package/out/tools/lint/recipe-output.d.ts.map +1 -1
- package/out/tools/lint/recipe-output.js +17 -0
- package/out/tools/lint/recipe-output.js.map +1 -1
- package/out/tools/overlay/kryptonite.d.ts +2 -2
- package/out/tools/overlay/kryptonite.d.ts.map +1 -1
- package/out/tools/overlay/kryptonite.js +2 -1
- package/out/tools/overlay/kryptonite.js.map +1 -1
- package/out/tools/overlay/manifest.d.ts +2 -1
- package/out/tools/overlay/manifest.d.ts.map +1 -1
- package/out/tools/overlay/manifest.js +2 -1
- package/out/tools/overlay/manifest.js.map +1 -1
- package/out/tools/overlay/merge-source.d.ts +2 -1
- package/out/tools/overlay/merge-source.d.ts.map +1 -1
- package/out/tools/overlay/merge-source.js.map +1 -1
- package/out/tools/overlay/split-slice.d.ts +1 -1
- package/out/tools/overlay/split-slice.js +1 -1
- package/out/tools/overlay/split-slice.js.map +1 -1
- package/out/tools/overlay/translit.d.ts +4 -3
- package/out/tools/overlay/translit.d.ts.map +1 -1
- package/out/tools/overlay/translit.js +2 -1
- package/out/tools/overlay/translit.js.map +1 -1
- package/out/tools/overture-subvenue.d.ts +1 -1
- package/out/tools/overture-subvenue.d.ts.map +1 -1
- package/out/tools/postcode-triples.d.ts +1 -1
- package/out/tools/postcode-triples.d.ts.map +1 -1
- package/out/tools/postcode-triples.js +4 -3
- package/out/tools/postcode-triples.js.map +1 -1
- package/out/tools/source-register/build.d.ts +10 -8
- package/out/tools/source-register/build.d.ts.map +1 -1
- package/out/tools/source-register/build.js +10 -8
- package/out/tools/source-register/build.js.map +1 -1
- package/out/tools/sub/venue/lexicon.d.ts +2 -6
- package/out/tools/sub/venue/lexicon.d.ts.map +1 -1
- package/out/tools/sub/venue/lexicon.js +2 -6
- package/out/tools/sub/venue/lexicon.js.map +1 -1
- package/out/tw/tools/fetch/gcis.js +3 -3
- package/out/tw/tools/fetch/gcis.js.map +1 -1
- package/out/types.d.ts +4 -3
- package/out/types.d.ts.map +1 -1
- package/out/types.js.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.js.map +1 -1
- package/out/us/adapters/usgov/nad/adapter.js +6 -6
- package/out/us/adapters/usgov/nad/adapter.js.map +1 -1
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js +2 -2
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/us/fips-state.d.ts +2 -2
- package/out/us/fips-state.d.ts.map +1 -1
- package/out/us/fips-state.js.map +1 -1
- package/out/us/synthesizers/intersection.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/intersection.js +13 -9
- package/out/us/synthesizers/intersection.js.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.d.ts +3 -2
- package/out/us/synthesizers/street.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.js +12 -12
- package/out/us/synthesizers/street.js.map +1 -0
- package/out/us/tools/fetch/imls-pls.js +1 -1
- package/out/us/tools/fetch/imls-pls.js.map +1 -1
- package/out/us/tools/fetch/nad.d.ts +6 -6
- package/out/us/tools/fetch/nad.js +1 -1
- package/out/us/tools/fetch/nad.js.map +1 -1
- package/out/us/tools/fetch/nppes.js +3 -3
- package/out/us/tools/fetch/nppes.js.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts +2 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.js +1 -1
- package/out/us/tools/fetch/state/hi-schools.js.map +1 -1
- package/out/us/tools/fetch/tiger-full.js +1 -1
- package/out/us/tools/fetch/tiger-full.js.map +1 -1
- package/out/utils/align.d.ts +5 -3
- package/out/utils/align.d.ts.map +1 -1
- package/out/utils/align.js +5 -5
- package/out/utils/align.js.map +1 -1
- package/out/utils/golden.js +2 -2
- package/out/utils/golden.js.map +1 -1
- package/out/utils/license.d.ts +3 -3
- package/out/utils/license.d.ts.map +1 -1
- package/out/utils/license.js.map +1 -1
- package/out/utils/split.d.ts +1 -1
- package/out/utils/split.d.ts.map +1 -1
- package/out/utils/split.js +18 -14
- package/out/utils/split.js.map +1 -1
- package/out/utils/tokenize.d.ts +3 -20
- package/out/utils/tokenize.d.ts.map +1 -1
- package/out/utils/tokenize.js.map +1 -1
- package/out/utils/wof-json.js +3 -3
- package/out/utils/wof-json.js.map +1 -1
- package/package.json +18 -27
- package/lib/tools.ts +0 -31
- package/out/synthesizers/intersection.d.ts.map +0 -1
- package/out/synthesizers/intersection.js.map +0 -1
- package/out/synthesizers/street.d.ts.map +0 -1
- package/out/synthesizers/street.js.map +0 -1
- package/out/tools.d.ts +0 -31
- package/out/tools.d.ts.map +0 -1
- package/out/tools.js +0 -31
- package/out/tools.js.map +0 -1
- /package/out/{synthesizers → us/synthesizers}/intersection.d.ts +0 -0
|
@@ -6,17 +6,17 @@
|
|
|
6
6
|
* Harvests the INSPIRE Addresses theme the Diputación Foral de Bizkaia publishes into the
|
|
7
7
|
* directory `#es/adapters/bizkaia/adapter` reads.
|
|
8
8
|
*
|
|
9
|
-
* ## Two services, because the archives
|
|
9
|
+
* ## Two services, because the archives hold only addresses
|
|
10
10
|
*
|
|
11
11
|
* The ATOM download service at `https://apli.bizkaia.eus/apps/Danok/INSPIRE/addresses.xml` lists
|
|
12
12
|
* one entry per municipality, each offering that municipality's addresses as a zipped GML through a
|
|
13
13
|
* `rel="enclosure"` link. Those archives hold no `ad:ThoroughfareName`, `ad:PostalDescriptor` or
|
|
14
14
|
* `ad:AdminUnitName` feature: every `ad:component` reference an address writes is a stored-query
|
|
15
|
-
* URL on the publisher's WFS
|
|
16
|
-
* archives
|
|
17
|
-
* its municipality
|
|
15
|
+
* URL on the publisher's WFS that the ATOM service does not republish. An adapter handed only those
|
|
16
|
+
* archives therefore reads every address in the province without its street, its postcode or
|
|
17
|
+
* its municipality. This harvest acquires both services into one directory for that reason.
|
|
18
18
|
*
|
|
19
|
-
* Each component type is read in one request,
|
|
19
|
+
* Each component type is read in one request, and that is the only way this service serves it. Its
|
|
20
20
|
* capabilities document states `ImplementsResultPaging` as `FALSE` and `CountDefault` as `1000`, so
|
|
21
21
|
* a reader that paged a type would receive the first page again. The harvest therefore asks
|
|
22
22
|
* `RESULTTYPE=hits` for the type's count and then asks for that many features in one `GetFeature`,
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
* count. A truncated component document is the failure that would otherwise reach the adapter as
|
|
25
25
|
* an unresolved reference per address.
|
|
26
26
|
*
|
|
27
|
-
* A component document's own bytes are not reproducible. Its `wfs:FeatureCollection`
|
|
27
|
+
* A component document's own bytes are not reproducible. Its `wfs:FeatureCollection` holds a
|
|
28
28
|
* `timeStamp` attribute holding the moment of the request, so two identical harvests differ in
|
|
29
29
|
* that element and therefore in their digests. The manifest records each document's byte count and
|
|
30
30
|
* sha256 as what arrived, never as a value a later run compares against.
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
* The component documents state no version at all. The WFS answers them under `Cache-Control:
|
|
45
45
|
* no-cache` with no `Last-Modified`, so a run holds no value to compare and every run that
|
|
46
46
|
* harvests them downloads them again. {@linkcode HarvestESBizkaiaOptions.components} is how a run that wants
|
|
47
|
-
* the archives
|
|
47
|
+
* only the archives declines the 14 MB.
|
|
48
48
|
*/
|
|
49
49
|
|
|
50
50
|
/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
|
|
@@ -52,7 +52,7 @@
|
|
|
52
52
|
|
|
53
53
|
import { APIClient, assertNoOGCServiceException, readWFSFeatureCount } from "@mailwoman/core/api"
|
|
54
54
|
import { ByteFormatter } from "@mailwoman/core/fs/formatters"
|
|
55
|
-
import { tryStat } from "@mailwoman/core/fs/readers"
|
|
55
|
+
import { tryStat } from "@mailwoman/core/fs/readers/stat"
|
|
56
56
|
import { makeDirectories, writeLocalFile } from "@mailwoman/core/fs/writers"
|
|
57
57
|
import { sha256File, sha256Hex } from "@mailwoman/core/hash"
|
|
58
58
|
import { rootAttribute } from "@mailwoman/core/html/document"
|
|
@@ -74,8 +74,8 @@ export const ES_BIZKAIA_SERVICE_URL = "https://apli.bizkaia.eus/apps/Danok/INSPI
|
|
|
74
74
|
/**
|
|
75
75
|
* The WFS the municipality archives reference their components on.
|
|
76
76
|
*
|
|
77
|
-
* Read from the `xlink:href` the archives themselves write
|
|
78
|
-
* endpoint through the `GetFeatureById` stored query.
|
|
77
|
+
* Read from the `xlink:href` the archives themselves write.
|
|
78
|
+
* That link addresses this endpoint through the `GetFeatureById` stored query.
|
|
79
79
|
*/
|
|
80
80
|
export const ES_BIZKAIA_WFS_URL =
|
|
81
81
|
"https://geo.bizkaia.eus/arcgisserverinspire/rest/services/Catastro/Annex1/MapServer/exts/InspireFeatureDownload/service"
|
|
@@ -115,8 +115,8 @@ export const ES_BIZKAIA_REQUEST_INTERVAL_MS = 250
|
|
|
115
115
|
/**
|
|
116
116
|
* Per-request timeout for a body, in milliseconds.
|
|
117
117
|
*
|
|
118
|
-
* Ten minutes
|
|
119
|
-
* document
|
|
118
|
+
* Ten minutes.
|
|
119
|
+
* That covers both a municipality archive and the one component document with every street in the province.
|
|
120
120
|
*/
|
|
121
121
|
const BODY_TIMEOUT_MS = 600_000
|
|
122
122
|
|
|
@@ -174,7 +174,8 @@ export interface ESBizkaiaMunicipalityDataset {
|
|
|
174
174
|
*/
|
|
175
175
|
downloadURL: string
|
|
176
176
|
/**
|
|
177
|
-
* The file name the archive is written under
|
|
177
|
+
* The file name the archive is written under.
|
|
178
|
+
* The adapter globs this name.
|
|
178
179
|
*/
|
|
179
180
|
filename: string
|
|
180
181
|
}
|
|
@@ -187,9 +188,9 @@ export interface ESBizkaiaMunicipalityDataset {
|
|
|
187
188
|
* the archive is stored under: a disagreement between the publisher's two statements of
|
|
188
189
|
* a municipality would store one municipality's addresses under another's name.
|
|
189
190
|
*
|
|
190
|
-
* @throws When the feed lists no entry, when an entry states neither spelling of its
|
|
191
|
-
* when the two spellings disagree, when two entries claim one file name
|
|
192
|
-
* one archive overwrite the other
|
|
191
|
+
* @throws When the feed lists no entry, when an entry states neither spelling of its
|
|
192
|
+
* municipality, when the two spellings disagree, when two entries claim one file name
|
|
193
|
+
* and one archive would overwrite the other, or when an entry has no `rel="enclosure"` link.
|
|
193
194
|
*/
|
|
194
195
|
export function readESBizkaiaServiceFeed(feed: AtomFeed): readonly ESBizkaiaMunicipalityDataset[] {
|
|
195
196
|
if (!feed.entries.length) {
|
|
@@ -220,7 +221,7 @@ export function readESBizkaiaServiceFeed(feed: AtomFeed): readonly ESBizkaiaMuni
|
|
|
220
221
|
)
|
|
221
222
|
}
|
|
222
223
|
|
|
223
|
-
// The identifier
|
|
224
|
+
// The identifier holds the three-digit tail of the five-digit municipality code:
|
|
224
225
|
// `48915` is published as `ES.BFA.AD.915.zip`.
|
|
225
226
|
if (!fromTitle.endsWith(identified[2]!)) {
|
|
226
227
|
throw new Error(
|
|
@@ -288,8 +289,9 @@ export interface ESBizkaiaComponentEntry extends SourceManifest {
|
|
|
288
289
|
*/
|
|
289
290
|
feature_count: number
|
|
290
291
|
/**
|
|
291
|
-
* The `numberReturned` the saved document states
|
|
292
|
-
*
|
|
292
|
+
* The `numberReturned` the saved document states.
|
|
293
|
+
*
|
|
294
|
+
* The harvest requires it to equal {@linkcode ESBizkaiaComponentEntry.feature_count}.
|
|
293
295
|
*/
|
|
294
296
|
features_returned: number
|
|
295
297
|
}
|
|
@@ -311,7 +313,7 @@ export interface ESBizkaiaHarvestManifest extends SourceCollectionManifest {
|
|
|
311
313
|
municipalities_listed: number
|
|
312
314
|
files: ESBizkaiaArchiveEntry[]
|
|
313
315
|
/**
|
|
314
|
-
* The saved WFS documents, kept apart from `files` because they
|
|
316
|
+
* The saved WFS documents, kept apart from `files` because they have no
|
|
315
317
|
* municipality and no stated version.
|
|
316
318
|
*/
|
|
317
319
|
components: ESBizkaiaComponentEntry[]
|
|
@@ -319,12 +321,12 @@ export interface ESBizkaiaHarvestManifest extends SourceCollectionManifest {
|
|
|
319
321
|
|
|
320
322
|
export interface HarvestESBizkaiaOptions {
|
|
321
323
|
/**
|
|
322
|
-
* Where the archives, the component documents and the manifest are written
|
|
323
|
-
*
|
|
324
|
+
* Where the archives, the component documents and the manifest are written.
|
|
325
|
+
* This path is the adapter's `inputPath`.
|
|
324
326
|
*/
|
|
325
327
|
outputDir: PathBuilderLike
|
|
326
328
|
/**
|
|
327
|
-
* Harvest only these five-digit municipality codes, for a probe or
|
|
329
|
+
* Harvest only these five-digit municipality codes, for a probe or to repair specific ones.
|
|
328
330
|
*
|
|
329
331
|
* A code the service document does not list is reported as a failure rather than ignored.
|
|
330
332
|
*/
|
|
@@ -335,10 +337,11 @@ export interface HarvestESBizkaiaOptions {
|
|
|
335
337
|
limit?: number
|
|
336
338
|
/**
|
|
337
339
|
* Save the WFS component documents as well as the archives.
|
|
338
|
-
* Defaults to true.
|
|
339
340
|
*
|
|
340
341
|
* An adapter run needs them: an archive references its street, its postcode
|
|
341
|
-
* and its municipality and
|
|
342
|
+
* and its municipality and has none of the three.
|
|
343
|
+
*
|
|
344
|
+
* @defaultValue `true`
|
|
342
345
|
*/
|
|
343
346
|
components?: boolean
|
|
344
347
|
/**
|
|
@@ -348,7 +351,7 @@ export interface HarvestESBizkaiaOptions {
|
|
|
348
351
|
/**
|
|
349
352
|
* Re-read the sha256 of every archive already on disk instead of comparing its byte count.
|
|
350
353
|
*
|
|
351
|
-
* The default compares the recorded byte count against the file's size,
|
|
354
|
+
* The default compares the recorded byte count against the file's size, one `stat`.
|
|
352
355
|
*/
|
|
353
356
|
verifyDigests?: boolean
|
|
354
357
|
signal?: AbortSignal
|
|
@@ -364,7 +367,7 @@ export interface HarvestESBizkaiaOptions {
|
|
|
364
367
|
* so a service that stops stating a version is re-fetched rather than frozen.
|
|
365
368
|
*/
|
|
366
369
|
async function archiveIsCurrent(
|
|
367
|
-
recorded: ESBizkaiaArchiveEntry |
|
|
370
|
+
recorded: ESBizkaiaArchiveEntry | null,
|
|
368
371
|
dataset: ESBizkaiaMunicipalityDataset,
|
|
369
372
|
path: PathBuilderLike,
|
|
370
373
|
verifyDigests: boolean
|
|
@@ -377,7 +380,8 @@ async function archiveIsCurrent(
|
|
|
377
380
|
|
|
378
381
|
if (!stat || stat.size !== recorded.bytes) return false
|
|
379
382
|
|
|
380
|
-
// The digest check reads the whole file rather than its metadata
|
|
383
|
+
// The digest check reads the whole file rather than its metadata.
|
|
384
|
+
// It is opt-in.
|
|
381
385
|
if (!verifyDigests) return true
|
|
382
386
|
|
|
383
387
|
return (await sha256File(path)) === recorded.sha256
|
|
@@ -433,7 +437,7 @@ export async function fetchBizkaiaComponentDocument(
|
|
|
433
437
|
// count is read from `numberReturned` and compared against the hits request's figure.
|
|
434
438
|
const returned = rootAttribute(document, "numberReturned", { xml: true })
|
|
435
439
|
|
|
436
|
-
if (returned
|
|
440
|
+
if (!returned || !/^\d+$/u.test(returned)) {
|
|
437
441
|
throw new Error(
|
|
438
442
|
`${context}: the GetFeature response states numberReturned=${stringifyJSON(returned ?? null)} rather than ` +
|
|
439
443
|
`a count, so what the document holds against the ${featureCount} features the service matched is unknown`
|
|
@@ -514,8 +518,8 @@ export async function harvestESBizkaia(
|
|
|
514
518
|
const recordedManifest = await readManifest<ESBizkaiaHarvestManifest>(manifestPath)
|
|
515
519
|
const files = new Map<string, ESBizkaiaArchiveEntry>()
|
|
516
520
|
|
|
517
|
-
// An entry for a municipality this run does not consider is
|
|
518
|
-
//
|
|
521
|
+
// An entry for a municipality this run does not consider is kept, so a bounded
|
|
522
|
+
// run never drops what an earlier run recorded.
|
|
519
523
|
for (const [filename, entry] of previous) {
|
|
520
524
|
files.set(filename, entry as ESBizkaiaArchiveEntry)
|
|
521
525
|
}
|
|
@@ -558,7 +562,7 @@ export async function harvestESBizkaia(
|
|
|
558
562
|
|
|
559
563
|
if (
|
|
560
564
|
!options.force &&
|
|
561
|
-
(await archiveIsCurrent(files.get(dataset.filename), dataset, dest, options.verifyDigests ?? false))
|
|
565
|
+
(await archiveIsCurrent(files.get(dataset.filename) ?? null, dataset, dest, options.verifyDigests ?? false))
|
|
562
566
|
) {
|
|
563
567
|
skipped++
|
|
564
568
|
|
|
@@ -625,7 +629,7 @@ export async function harvestESBizkaia(
|
|
|
625
629
|
}
|
|
626
630
|
|
|
627
631
|
/**
|
|
628
|
-
* The municipalities this run considers, and the codes a caller
|
|
632
|
+
* The municipalities this run considers, and the codes a caller asked for that the feed does not list.
|
|
629
633
|
*
|
|
630
634
|
* A code the caller asked for and the service document does not list is returned
|
|
631
635
|
* rather than dropped, because a municipality that is not published is a different
|
|
@@ -695,8 +699,8 @@ async function fetchArchive(
|
|
|
695
699
|
/**
|
|
696
700
|
* The path `#es/adapters/bizkaia/adapter` reads, given the root a fetch wrote under.
|
|
697
701
|
*
|
|
698
|
-
* The directory rather than a file: the adapter reads every archive and every saved
|
|
699
|
-
* document under it, and
|
|
702
|
+
* The directory rather than a file: the adapter reads every archive and every saved
|
|
703
|
+
* WFS document under it, and the component documents hold the streets.
|
|
700
704
|
*/
|
|
701
705
|
export function esBizkaiaInputPath(outRoot: BaseFetchOptions["outRoot"]): PathBuilderLike {
|
|
702
706
|
return outRoot(ES_BIZKAIA_ADAPTER_ID)
|
|
@@ -712,7 +716,7 @@ export interface FetchESBizkaiaOptions
|
|
|
712
716
|
/**
|
|
713
717
|
* The minimum spacing between two requests, in milliseconds.
|
|
714
718
|
*
|
|
715
|
-
*
|
|
719
|
+
* @defaultValue {@linkcode ES_BIZKAIA_REQUEST_INTERVAL_MS}
|
|
716
720
|
*/
|
|
717
721
|
minRequestIntervalMs?: number
|
|
718
722
|
}
|
|
@@ -27,16 +27,16 @@
|
|
|
27
27
|
* excluding three names. A feed that renames its offices then matches zero titles and raises.
|
|
28
28
|
* 2. **A province feed declares ISO-8859-1 and means it, while the national feed declares UTF-8.**
|
|
29
29
|
* `ES.SDGC.ad.atom_15.xml` writes A Coruña's directory name with the byte `0xD1` for `Ñ`.
|
|
30
|
-
*
|
|
31
|
-
* answers an html error page under http 200 rather than the archive.
|
|
32
|
-
* bytes and decoded through {@linkcode decodeDeclaredXML}
|
|
33
|
-
* declaration.
|
|
30
|
+
* A UTF-8 decode of that feed yields U+FFFD. The URL built from the replacement character then
|
|
31
|
+
* answers an html error page under http 200 rather than the archive. Both levels are therefore
|
|
32
|
+
* read as bytes and decoded through {@linkcode decodeDeclaredXML}. That function reads each
|
|
33
|
+
* document's own declaration.
|
|
34
34
|
* 3. **An archive URL cannot be composed, and the href it is read from is not request-ready.** The
|
|
35
35
|
* directory segment is the municipality's own name, so Madrid sits under `28900-MADRID` while
|
|
36
36
|
* the composable `28079` answers an html page. The publisher writes that segment with literal
|
|
37
37
|
* spaces and with non-ASCII letters, and {@linkcode archiveRequestURL} percent-encodes both.
|
|
38
38
|
* 4. **The municipality code is stated twice, and the two are checked against each other.** An
|
|
39
|
-
* entry's title reads `55101-CEUTA addresses` and its archive is
|
|
39
|
+
* entry's title reads `55101-CEUTA addresses` and its archive file is
|
|
40
40
|
* `A.ES.SDGC.AD.55101.zip`. The file keeps the publisher's own name rather than one built from
|
|
41
41
|
* the code, so the code decides which entry is read and never where the file lands. A
|
|
42
42
|
* disagreement between the publisher's two statements of it is a change in the feed's layout
|
|
@@ -62,7 +62,7 @@
|
|
|
62
62
|
|
|
63
63
|
import { APIClient, assertNoOGCServiceException } from "@mailwoman/core/api"
|
|
64
64
|
import { ByteFormatter } from "@mailwoman/core/fs/formatters"
|
|
65
|
-
import { tryStat } from "@mailwoman/core/fs/readers"
|
|
65
|
+
import { tryStat } from "@mailwoman/core/fs/readers/stat"
|
|
66
66
|
import { decodeBytes } from "@mailwoman/core/fs/streams"
|
|
67
67
|
import { makeDirectories } from "@mailwoman/core/fs/writers"
|
|
68
68
|
import { sha256File } from "@mailwoman/core/hash"
|
|
@@ -77,7 +77,7 @@ import { loadCollectionFiles, writeManifest } from "#tools/fetch/download"
|
|
|
77
77
|
import { downloadZipArchive } from "#tools/fetch/zip-archive"
|
|
78
78
|
|
|
79
79
|
/**
|
|
80
|
-
* The national ATOM service document
|
|
80
|
+
* The national ATOM service document that lists one province feed per territorial office.
|
|
81
81
|
*/
|
|
82
82
|
export const ES_CATASTRO_SERVICE_FEED_URL = "https://www.catastro.hacienda.gob.es/INSPIRE/Addresses/ES.SDGC.AD.atom.xml"
|
|
83
83
|
|
|
@@ -100,7 +100,7 @@ export const ES_CATASTRO_ATTRIBUTION = "Dirección General del Catastro (Ministe
|
|
|
100
100
|
export const ES_CATASTRO_REQUEST_INTERVAL_MS = 250
|
|
101
101
|
|
|
102
102
|
/**
|
|
103
|
-
* The directory the harvest writes under
|
|
103
|
+
* The directory the harvest writes under and the adapter reads as `inputPath`.
|
|
104
104
|
*/
|
|
105
105
|
const SLUG = ES_CATASTRO_ADAPTER_ID
|
|
106
106
|
|
|
@@ -126,8 +126,8 @@ const DECLARED_ENCODING = /<\?xml[^>]*encoding="([^"]+)"/iu
|
|
|
126
126
|
/**
|
|
127
127
|
* A national entry's title, `Territorial office 02 Albacete`.
|
|
128
128
|
*
|
|
129
|
-
* The three
|
|
130
|
-
*
|
|
129
|
+
* The three foral-cadastre entries are titled `Provincial Council of <name>` and do not match.
|
|
130
|
+
* Their own adapters read those feeds.
|
|
131
131
|
*/
|
|
132
132
|
const TERRITORIAL_OFFICE_TITLE = /^Territorial office\s+(\d{2})\s+(\S.*)$/u
|
|
133
133
|
|
|
@@ -151,7 +151,8 @@ const ARCHIVE_FILENAME = /^A\.ES\.SDGC\.AD\.(\d{5})\.zip$/iu
|
|
|
151
151
|
*
|
|
152
152
|
* The national feed declares UTF-8 and a province feed declares ISO-8859-1, so a reader
|
|
153
153
|
* that fixed either encoding would mojibake the other publisher's accented place names.
|
|
154
|
-
* A document that declares no encoding is decoded as UTF-8
|
|
154
|
+
* A document that declares no encoding is decoded as UTF-8.
|
|
155
|
+
* XML states that default.
|
|
155
156
|
*/
|
|
156
157
|
export function decodeDeclaredXML(bytes: Uint8Array): string {
|
|
157
158
|
const head = Buffer.from(bytes.subarray(0, DECLARATION_WINDOW)).toString("latin1")
|
|
@@ -164,7 +165,8 @@ export function decodeDeclaredXML(bytes: Uint8Array): string {
|
|
|
164
165
|
*
|
|
165
166
|
* A province feed writes a municipality's directory segment as the publisher spells it,
|
|
166
167
|
* with literal spaces and with non-ASCII letters: `…/15/15900-A CORUÑA/A.ES.SDGC.AD.15900.zip`.
|
|
167
|
-
* `URL` percent-encodes both
|
|
168
|
+
* `URL` percent-encodes both.
|
|
169
|
+
* The host serves the archive at that encoded form.
|
|
168
170
|
*/
|
|
169
171
|
export function archiveRequestURL(href: string): string {
|
|
170
172
|
return new URL(href).toString()
|
|
@@ -175,7 +177,9 @@ export function archiveRequestURL(href: string): string {
|
|
|
175
177
|
*/
|
|
176
178
|
export interface ESCatastroProvince {
|
|
177
179
|
/**
|
|
178
|
-
* The two-digit province code
|
|
180
|
+
* The two-digit province code.
|
|
181
|
+
*
|
|
182
|
+
* Each municipality code under it starts with these two digits.
|
|
179
183
|
*/
|
|
180
184
|
code: string
|
|
181
185
|
/**
|
|
@@ -184,7 +188,8 @@ export interface ESCatastroProvince {
|
|
|
184
188
|
name: string
|
|
185
189
|
title: string
|
|
186
190
|
/**
|
|
187
|
-
* This office's province feed
|
|
191
|
+
* This office's province feed.
|
|
192
|
+
* It lists one archive per municipality.
|
|
188
193
|
*/
|
|
189
194
|
provinceFeedURL: string
|
|
190
195
|
/**
|
|
@@ -216,8 +221,9 @@ export interface ESCatastroMunicipalityDataset {
|
|
|
216
221
|
*/
|
|
217
222
|
updated: string
|
|
218
223
|
/**
|
|
219
|
-
* The file name the archive is written under
|
|
220
|
-
*
|
|
224
|
+
* The file name the archive is written under.
|
|
225
|
+
*
|
|
226
|
+
* It is the publisher's own name and the file the adapter globs.
|
|
221
227
|
*/
|
|
222
228
|
filename: string
|
|
223
229
|
}
|
|
@@ -226,7 +232,7 @@ export interface ESCatastroMunicipalityDataset {
|
|
|
226
232
|
* The territorial offices the national service document lists.
|
|
227
233
|
*
|
|
228
234
|
* @throws When the document holds no entry, when it holds no territorial-office entry at all,
|
|
229
|
-
* when an office
|
|
235
|
+
* when an office has no province-feed link, or when two offices claim one province code.
|
|
230
236
|
* Each of those would otherwise write zero archives while reporting a completed run.
|
|
231
237
|
*/
|
|
232
238
|
export function readESCatastroServiceFeed(feed: AtomFeed): readonly ESCatastroProvince[] {
|
|
@@ -264,7 +270,7 @@ export function readESCatastroServiceFeed(feed: AtomFeed): readonly ESCatastroPr
|
|
|
264
270
|
|
|
265
271
|
const fromHref = PROVINCE_FEED_CODE.exec(link.href)?.[1]
|
|
266
272
|
|
|
267
|
-
if (fromHref
|
|
273
|
+
if (fromHref && fromHref !== code) {
|
|
268
274
|
throw new Error(
|
|
269
275
|
`${ES_CATASTRO_SERVICE_FEED_URL}: the entry titled ${stringifyJSON(entry.title)} states province ` +
|
|
270
276
|
`${code} in its title and links ${link.href}, which names province ${fromHref}`
|
|
@@ -303,9 +309,9 @@ export function readESCatastroServiceFeed(feed: AtomFeed): readonly ESCatastroPr
|
|
|
303
309
|
/**
|
|
304
310
|
* The municipalities one province feed lists.
|
|
305
311
|
*
|
|
306
|
-
* @throws When the feed holds no entry, when an entry
|
|
307
|
-
* when the municipality code its title states disagrees with the
|
|
308
|
-
*
|
|
312
|
+
* @throws When the feed holds no entry, when an entry has no archive link,
|
|
313
|
+
* when the municipality code its title states disagrees with the code in its archive
|
|
314
|
+
* file name, or when two entries reference one archive.
|
|
309
315
|
*/
|
|
310
316
|
export function readESCatastroProvinceFeed(
|
|
311
317
|
feed: AtomFeed,
|
|
@@ -331,8 +337,8 @@ export function readESCatastroProvinceFeed(
|
|
|
331
337
|
)
|
|
332
338
|
}
|
|
333
339
|
|
|
334
|
-
// The publisher's own file name, taken off the href before
|
|
335
|
-
// so the file on disk
|
|
340
|
+
// The publisher's own file name, taken off the href before percent-encoding,
|
|
341
|
+
// so the file on disk keeps the publisher's spelling.
|
|
336
342
|
const filename = link.href.slice(link.href.lastIndexOf("/") + 1)
|
|
337
343
|
const fromFilename = ARCHIVE_FILENAME.exec(filename)?.[1]
|
|
338
344
|
|
|
@@ -395,7 +401,7 @@ export interface ESCatastroArchiveEntry extends SourceManifest {
|
|
|
395
401
|
/**
|
|
396
402
|
* The `<updated>` the province feed stated when this archive was fetched.
|
|
397
403
|
*
|
|
398
|
-
* A later run that reads the same value for this municipality
|
|
404
|
+
* A later run that reads the same value for this municipality reuses the file.
|
|
399
405
|
* The publisher moves it when it publishes a new edition.
|
|
400
406
|
*/
|
|
401
407
|
feed_updated: string
|
|
@@ -429,7 +435,8 @@ export interface ESCatastroHarvestManifest extends SourceCollectionManifest {
|
|
|
429
435
|
|
|
430
436
|
export interface HarvestESCatastroOptions {
|
|
431
437
|
/**
|
|
432
|
-
* Where the archives and the manifest are written
|
|
438
|
+
* Where the archives and the manifest are written.
|
|
439
|
+
* The adapter reads this directory as `inputPath`.
|
|
433
440
|
*/
|
|
434
441
|
outputDir: PathBuilderLike
|
|
435
442
|
/**
|
|
@@ -452,7 +459,8 @@ export interface HarvestESCatastroOptions {
|
|
|
452
459
|
/**
|
|
453
460
|
* Re-read the sha256 of every archive already on disk instead of comparing its byte count.
|
|
454
461
|
*
|
|
455
|
-
* The default compares the recorded byte count against the file's size
|
|
462
|
+
* The default compares the recorded byte count against the file's size.
|
|
463
|
+
* That check costs one `stat`.
|
|
456
464
|
*/
|
|
457
465
|
verifyDigests?: boolean
|
|
458
466
|
signal?: AbortSignal
|
|
@@ -463,11 +471,13 @@ export interface HarvestESCatastroOptions {
|
|
|
463
471
|
* Whether the archive recorded for one municipality is still the one the province feed describes.
|
|
464
472
|
*
|
|
465
473
|
* A skip requires the feed to state an `<updated>` value.
|
|
466
|
-
* Where it states none, both sides read an empty string
|
|
467
|
-
*
|
|
474
|
+
* Where it states none, both sides read an empty string.
|
|
475
|
+
*
|
|
476
|
+
* An equality test would then match, keeping an archive of unknown age for
|
|
477
|
+
* as long as the feed stayed silent.
|
|
468
478
|
*/
|
|
469
479
|
async function isCurrent(
|
|
470
|
-
recorded: ESCatastroArchiveEntry |
|
|
480
|
+
recorded: ESCatastroArchiveEntry | null,
|
|
471
481
|
dataset: ESCatastroMunicipalityDataset,
|
|
472
482
|
path: PathBuilderLike,
|
|
473
483
|
verifyDigests: boolean
|
|
@@ -478,7 +488,8 @@ async function isCurrent(
|
|
|
478
488
|
|
|
479
489
|
if (!stat || stat.size !== recorded.bytes) return false
|
|
480
490
|
|
|
481
|
-
// The digest check reads the whole file rather than its metadata
|
|
491
|
+
// The digest check reads the whole file rather than its metadata.
|
|
492
|
+
// That cost is why it is opt-in.
|
|
482
493
|
if (!verifyDigests) return true
|
|
483
494
|
|
|
484
495
|
return (await sha256File(path)) === recorded.sha256
|
|
@@ -504,11 +515,11 @@ async function readFeed(
|
|
|
504
515
|
// A feed shares the http 200 an exception report arrives on.
|
|
505
516
|
assertNoOGCServiceException(text, options.context)
|
|
506
517
|
|
|
507
|
-
return
|
|
518
|
+
return readAtomFeed(feedChunks(text))
|
|
508
519
|
}
|
|
509
520
|
|
|
510
521
|
/**
|
|
511
|
-
* The provinces this run reads, and the codes a caller
|
|
522
|
+
* The provinces this run reads, and the codes a caller requested that the service document omits.
|
|
512
523
|
*/
|
|
513
524
|
function selectProvinces(
|
|
514
525
|
listed: readonly ESCatastroProvince[],
|
|
@@ -578,8 +589,8 @@ export async function harvestESCatastro(
|
|
|
578
589
|
const previous = await loadCollectionFiles(manifestPath)
|
|
579
590
|
const files = new Map<string, ESCatastroArchiveEntry>()
|
|
580
591
|
|
|
581
|
-
//
|
|
582
|
-
//
|
|
592
|
+
// The harvest keeps an entry for a municipality this run skips, so a bounded
|
|
593
|
+
// run never drops what an earlier run recorded.
|
|
583
594
|
for (const [filename, entry] of previous) {
|
|
584
595
|
files.set(filename, entry as ESCatastroArchiveEntry)
|
|
585
596
|
}
|
|
@@ -589,7 +600,7 @@ export async function harvestESCatastro(
|
|
|
589
600
|
let municipalitiesListed = 0
|
|
590
601
|
let provincesHarvested = 0
|
|
591
602
|
const failedCodes: string[] = [...provinces.unknown]
|
|
592
|
-
const requested = options.municipalities
|
|
603
|
+
const requested = options.municipalities ? new Set(options.municipalities) : undefined
|
|
593
604
|
const found = new Set<string>()
|
|
594
605
|
|
|
595
606
|
const writeHarvestManifest = async (): Promise<void> => {
|
|
@@ -641,7 +652,7 @@ export async function harvestESCatastro(
|
|
|
641
652
|
|
|
642
653
|
const dest = destDir(dataset.filename)
|
|
643
654
|
|
|
644
|
-
if (await isCurrent(files.get(dataset.filename), dataset, dest, options.verifyDigests ?? false)) {
|
|
655
|
+
if (await isCurrent(files.get(dataset.filename) ?? null, dataset, dest, options.verifyDigests ?? false)) {
|
|
645
656
|
skipped++
|
|
646
657
|
|
|
647
658
|
continue
|
|
@@ -683,9 +694,9 @@ export async function harvestESCatastro(
|
|
|
683
694
|
}
|
|
684
695
|
}
|
|
685
696
|
|
|
686
|
-
// A municipality a caller
|
|
687
|
-
//
|
|
688
|
-
//
|
|
697
|
+
// A municipality a caller requested and no selected province feed listed is reported rather than dropped.
|
|
698
|
+
// A municipality that is not published is a different condition from a
|
|
699
|
+
// selection that lists no municipality.
|
|
689
700
|
for (const code of requested ?? []) {
|
|
690
701
|
if (found.has(code)) continue
|
|
691
702
|
|
|
@@ -719,7 +730,7 @@ export interface FetchESCatastroOptions
|
|
|
719
730
|
/**
|
|
720
731
|
* The minimum spacing between two requests, in milliseconds.
|
|
721
732
|
*
|
|
722
|
-
*
|
|
733
|
+
* @defaultValue {@linkcode ES_CATASTRO_REQUEST_INTERVAL_MS}
|
|
723
734
|
*/
|
|
724
735
|
minRequestIntervalMs?: number
|
|
725
736
|
}
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
*
|
|
15
15
|
* addresses.xml → GML/ES.GFA.AD.zip
|
|
16
16
|
*
|
|
17
|
-
* That entry links its dataset through `rel="alternate"`. It
|
|
17
|
+
* That entry links its dataset through `rel="alternate"`. It holds no `rel="enclosure"` link at
|
|
18
18
|
* all, and the `rel="describedby"` link beside the alternate is the ISO 19139 metadata record. A
|
|
19
19
|
* reader that looks for an enclosure therefore finds no dataset here, and a reader that takes the
|
|
20
20
|
* first link of any relation downloads the metadata record instead of 7,959,103 bytes of
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
* keep one archive forever. The archive's HTTP `last-modified` is the signal this fetcher compares,
|
|
29
29
|
* and {@linkcode gipuzkoaPublicationIsRecorded} requires the publisher to state one before it
|
|
30
30
|
* agrees to skip a download. The feed's value is recorded beside it, unused, because it is what
|
|
31
|
-
* the publisher
|
|
31
|
+
* the publisher states about the dataset.
|
|
32
32
|
*
|
|
33
33
|
* ## Why the archive stays compressed
|
|
34
34
|
*
|
|
@@ -39,7 +39,7 @@
|
|
|
39
39
|
|
|
40
40
|
import { APIClient, assertNoOGCServiceException } from "@mailwoman/core/api"
|
|
41
41
|
import { ByteFormatter } from "@mailwoman/core/fs/formatters"
|
|
42
|
-
import { tryStat } from "@mailwoman/core/fs/readers"
|
|
42
|
+
import { tryStat } from "@mailwoman/core/fs/readers/stat"
|
|
43
43
|
import { makeDirectories } from "@mailwoman/core/fs/writers"
|
|
44
44
|
import { sha256File } from "@mailwoman/core/hash"
|
|
45
45
|
import { PathBuilder, type PathBuilderLike } from "path-ts"
|
|
@@ -55,7 +55,8 @@ import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/downlo
|
|
|
55
55
|
export const ES_GIPUZKOA_SERVICE_URL = "https://b5m.gipuzkoa.eus/inspire/download/addresses.xml"
|
|
56
56
|
|
|
57
57
|
/**
|
|
58
|
-
* The name the archive is written under
|
|
58
|
+
* The name the archive is written under.
|
|
59
|
+
* The adapter's `inputPath` states that name.
|
|
59
60
|
*/
|
|
60
61
|
export const ES_GIPUZKOA_ARCHIVE_FILENAME = "ES.GFA.AD.zip"
|
|
61
62
|
|
|
@@ -104,7 +105,7 @@ export interface DownloadGipuzkoaOptions {
|
|
|
104
105
|
export interface GipuzkoaArchiveReference {
|
|
105
106
|
archiveURL: string
|
|
106
107
|
/**
|
|
107
|
-
* The entry's title
|
|
108
|
+
* The entry's title states the theme rather than a municipality.
|
|
108
109
|
*/
|
|
109
110
|
entryTitle: string
|
|
110
111
|
/**
|
|
@@ -116,7 +117,7 @@ export interface GipuzkoaArchiveReference {
|
|
|
116
117
|
/**
|
|
117
118
|
* Resolves the download service to the one archive its single entry links.
|
|
118
119
|
*
|
|
119
|
-
* Raises where the feed lists no entry, and where the entry
|
|
120
|
+
* Raises where the feed lists no entry, and where the entry holds no `rel="alternate"` link.
|
|
120
121
|
* The alternative would return `{fetched: 0, skipped: 0, failed: 0}`, which a caller
|
|
121
122
|
* reads as a fetch that completed and found the publisher empty.
|
|
122
123
|
*
|
|
@@ -176,8 +177,9 @@ export async function resolveGipuzkoaArchive(
|
|
|
176
177
|
* What the publisher's HEAD response states about the archive it holds.
|
|
177
178
|
*
|
|
178
179
|
* Both fields read `null` where the header is absent, rather than an empty string or zero.
|
|
179
|
-
* An absent `last-modified` is the service declining to state a version
|
|
180
|
-
*
|
|
180
|
+
* An absent `last-modified` is the service declining to state a version.
|
|
181
|
+
*
|
|
182
|
+
* That differs from a version that happens to match the one on disk.
|
|
181
183
|
*/
|
|
182
184
|
export interface GipuzkoaPublication {
|
|
183
185
|
lastModified: string | null
|
|
@@ -188,7 +190,7 @@ export interface GipuzkoaPublication {
|
|
|
188
190
|
* Reads the publisher's HEAD response for one archive URL.
|
|
189
191
|
*
|
|
190
192
|
* Separate from {@linkcode downloadGipuzkoa} for the same reason {@linkcode resolveGipuzkoaArchive}
|
|
191
|
-
* is: this request
|
|
193
|
+
* is: this request holds the whole freshness decision.
|
|
192
194
|
*/
|
|
193
195
|
export async function readGipuzkoaPublication(
|
|
194
196
|
client: Pick<APIClient, "fetch">,
|
|
@@ -215,7 +217,7 @@ export async function readGipuzkoaPublication(
|
|
|
215
217
|
* Whether the archive on disk is the one the publisher now serves.
|
|
216
218
|
*
|
|
217
219
|
* A publisher that states no `last-modified` answers false.
|
|
218
|
-
*
|
|
220
|
+
* An absent header compared against an absent recorded value makes `null === null` hold,
|
|
219
221
|
* and that reading keeps a stale archive for as long as the service stays silent.
|
|
220
222
|
*/
|
|
221
223
|
export function gipuzkoaPublicationIsRecorded(
|
|
@@ -223,7 +225,7 @@ export function gipuzkoaPublicationIsRecorded(
|
|
|
223
225
|
bytesOnDisk: number,
|
|
224
226
|
publication: GipuzkoaPublication
|
|
225
227
|
): boolean {
|
|
226
|
-
if (publication.lastModified
|
|
228
|
+
if (!publication.lastModified) return false
|
|
227
229
|
|
|
228
230
|
return recorded.last_modified === publication.lastModified && recorded.bytes === bytesOnDisk
|
|
229
231
|
}
|