@mailwoman/corpus 10.1.0 → 10.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/licenses/cdla-permissive-2.0.md +1 -1
- package/lib/adapters/cordis/adapter.ts +13 -7
- package/lib/adapters/dedup-key.ts +1 -1
- package/lib/adapters/geonames/adapter.ts +1 -1
- package/lib/adapters/gleif/adapter.ts +17 -16
- package/lib/adapters/libpostal-dictionary.ts +0 -2
- package/lib/adapters/openaddresses/adapter.ts +4 -3
- package/lib/adapters/osm/adapter.ts +22 -21
- package/lib/adapters/po-box/adapter.ts +10 -5
- package/lib/adapters/registry.ts +4 -4
- package/lib/adapters/source-id.ts +2 -2
- package/lib/adapters/street-line.ts +7 -6
- package/lib/adapters/wof/admin/json/adapter.ts +2 -2
- package/lib/adapters/wof/json-rows.ts +2 -1
- package/lib/adapters/wof/postalcode-json/adapter.ts +6 -2
- package/lib/adapters.ts +5 -5
- package/lib/au/adapters/gnaf/adapter.ts +1 -1
- package/lib/be/adapters/brussels/adapter.ts +22 -22
- package/lib/be/adapters/vlaanderen/adapter.ts +54 -57
- package/lib/be/adapters/wallonie/adapter.ts +17 -17
- package/lib/be/tools/fetch/brussels.ts +13 -10
- package/lib/be/tools/fetch/wallonie.ts +8 -7
- package/lib/build/adapters.ts +2 -1
- package/lib/build/align.ts +5 -4
- package/lib/build/checkpoint.ts +5 -3
- package/lib/build/eligibility.ts +3 -3
- package/lib/build/types.ts +8 -6
- package/lib/cz/adapters/cuzk/adapter.ts +41 -38
- package/lib/cz/tools/fetch/cuzk.ts +42 -31
- package/lib/de/recipes/locale.ts +1 -1
- package/lib/dk/adapters/inspire/adapter.ts +38 -38
- package/lib/dk/tools/fetch/inspire.ts +16 -14
- package/lib/ee/adapters/ads/adapter.ts +26 -32
- package/lib/ee/tools/fetch/ads.ts +10 -8
- package/lib/es/adapters/bizkaia/adapter.ts +47 -47
- package/lib/es/adapters/catastro/adapter.ts +36 -34
- package/lib/es/adapters/gipuzkoa/adapter.ts +38 -45
- package/lib/es/adapters/navarra/adapter.ts +28 -27
- package/lib/es/cadastre.ts +31 -31
- package/lib/es/tools/fetch/bizkaia.ts +40 -36
- package/lib/es/tools/fetch/catastro.ts +51 -40
- package/lib/es/tools/fetch/gipuzkoa.ts +13 -11
- package/lib/es/tools/fetch/navarra.ts +33 -25
- package/lib/fi/adapters/ryhti/adapter.ts +22 -26
- package/lib/fi/tools/fetch/ryhti.ts +31 -31
- package/lib/fr/adapters/annuaire-education/adapter.ts +13 -11
- package/lib/fr/adapters/ban/adapter.ts +6 -6
- package/lib/fr/adapters/finess/adapter.ts +16 -15
- package/lib/fr/adapters/finess/voie-type.ts +4 -3
- package/lib/fr/insee-country.ts +2 -2
- package/lib/fr/recipes/lieudit.ts +8 -6
- package/lib/fr/tools/fetch/annuaire-education.ts +11 -8
- package/lib/fr/tools/fetch/ban.ts +3 -2
- package/lib/fr/tools/fetch/finess.ts +14 -13
- package/lib/inspire/address.ts +86 -78
- package/lib/inspire/errors.ts +9 -8
- package/lib/inspire/stream.ts +29 -26
- package/lib/international/recipes/locale.ts +13 -9
- package/lib/it/adapters/anac/adapter.ts +51 -46
- package/lib/it/tools/fetch/anac.ts +22 -25
- package/lib/jp/adapters/wof-admin/adapter.ts +2 -6
- package/lib/jp/tools/fetch/houjin.ts +3 -3
- package/lib/kr/adapters/localdata/permit-rows.ts +1 -1
- package/lib/kr/tools/fetch/juso.ts +1 -1
- package/lib/kr/tools/fetch/localdata.ts +2 -2
- package/lib/lu/adapters/bd-adresses/adapter.ts +10 -10
- package/lib/lu/tools/fetch/bd-adresses.ts +15 -13
- package/lib/lv/adapters/varis/adapter.ts +24 -23
- package/lib/lv/tools/fetch/varis.ts +3 -2
- package/lib/nl/adapters/kadaster/adapter.ts +29 -29
- package/lib/nl/tools/fetch/kadaster.ts +16 -12
- package/lib/no/adapters/matrikkelen/adapter.ts +6 -5
- package/lib/no/tools/fetch/matrikkelen.ts +59 -47
- package/lib/parquet/duckdb.ts +7 -4
- package/lib/parquet/readers.ts +3 -3
- package/lib/parquet/schema.ts +1 -1
- package/lib/parquet/writers.ts +3 -2
- package/lib/pl/adapters/emuia/adapter.ts +12 -12
- package/lib/pl/tools/fetch/emuia.ts +10 -9
- package/lib/recipes/bare/postcode.ts +1 -1
- package/lib/recipes/boundary-stress.ts +1 -1
- package/lib/recipes/house-venue.ts +7 -3
- package/lib/recipes/intersection.ts +2 -2
- package/lib/recipes/po/box/cedex/recipe.ts +3 -3
- package/lib/recipes/po/box/cedex/sources.ts +13 -1
- package/lib/recipes/scaffold.ts +31 -21
- package/lib/recipes/sources.ts +16 -3
- package/lib/recipes/street/bare.ts +2 -2
- package/lib/recipes/street.ts +3 -1
- package/lib/recipes/sub/venue/context.ts +2 -0
- package/lib/recipes/sub/venue/render.ts +1 -1
- package/lib/recipes/sub/venue/sources.ts +24 -29
- package/lib/recipes/sub/venue.ts +12 -18
- package/lib/recipes/trailing-region.ts +6 -2
- package/lib/recipes/unit.ts +3 -3
- package/lib/recipes/venue.ts +347 -0
- package/lib/recipes.ts +4 -2
- package/lib/registers.ts +8 -6
- package/lib/ro/adapters/retea-scolara/adapter.ts +5 -5
- package/lib/ro/tools/fetch/retea-scolara.ts +1 -1
- package/lib/runner.ts +4 -3
- package/lib/sg/recipes/register.ts +1 -1
- package/lib/sg/tools/fetch/acra.ts +3 -3
- package/lib/sk/adapters/inspire/adapter.ts +25 -27
- package/lib/sk/tools/fetch/inspire.ts +15 -14
- package/lib/source-register/effective-manifest.ts +3 -3
- package/lib/source-register/register.ts +3 -3
- package/lib/source-register/training-manifest.ts +1 -1
- package/lib/source-register/types.ts +12 -13
- package/lib/subvenue/promotions.ts +1 -1
- package/lib/subvenue/table.ts +4 -4
- package/lib/surfaces/locale.ts +2 -0
- package/lib/synthesizers/anchor-absorption.ts +2 -1
- package/lib/synthesizers/boundary-stress.ts +20 -26
- package/lib/synthesizers/house-venue.ts +26 -30
- package/lib/synthesizers/no-street.ts +2 -1
- package/lib/synthesizers/po-box.ts +6 -3
- package/lib/synthesizers/tail.ts +55 -0
- package/lib/synthesizers/utils.ts +19 -2
- package/lib/test-kit.ts +2 -2
- package/lib/tools/audit.ts +7 -5
- package/lib/tools/corpus-stats.ts +1 -1
- package/lib/tools/fetch/atom.ts +10 -10
- package/lib/tools/fetch/cordis.ts +11 -9
- package/lib/tools/fetch/download/manifest.ts +2 -1
- package/lib/tools/fetch/download/network.ts +4 -4
- package/lib/tools/fetch/download/retry.ts +4 -4
- package/lib/tools/fetch/geonames/dump.ts +5 -3
- package/lib/tools/fetch/geonames/postal.ts +2 -2
- package/lib/tools/fetch/gleif.ts +6 -4
- package/lib/tools/fetch/header.ts +8 -6
- package/lib/tools/fetch/inspire-addresses.ts +41 -36
- package/lib/tools/fetch/openaddresses.ts +1 -1
- package/lib/tools/fetch/vlaanderen-ad.ts +17 -17
- package/lib/tools/fetch/wfs-harvest.ts +39 -28
- package/lib/tools/fetch.ts +0 -42
- package/lib/tools/golden/expand.ts +8 -8
- package/lib/tools/golden/promote.ts +12 -8
- package/lib/tools/golden/relabel-street/directory.ts +7 -6
- package/lib/tools/golden/relabel-street/row.ts +7 -6
- package/lib/tools/ingest-csv.ts +3 -3
- package/lib/tools/jsonl-to-parquet.ts +1 -1
- package/lib/tools/lint/recipe-output/vocab.ts +10 -5
- package/lib/tools/lint/recipe-output.ts +26 -8
- package/lib/tools/overlay/kryptonite.ts +4 -3
- package/lib/tools/overlay/manifest.ts +4 -2
- package/lib/tools/overlay/merge-source.ts +2 -1
- package/lib/tools/overlay/split-slice.ts +2 -2
- package/lib/tools/overlay/translit.ts +6 -4
- package/lib/tools/overture-subvenue.ts +1 -1
- package/lib/tools/postcode-triples.ts +5 -4
- package/lib/tools/source-register/build.ts +17 -14
- package/lib/tools/sub/venue/lexicon.ts +4 -8
- package/lib/tw/tools/fetch/gcis.ts +4 -4
- package/lib/types.ts +4 -3
- package/lib/us/adapters/state/hi-schools/workbook.ts +1 -1
- package/lib/us/adapters/usgov/nad/adapter.ts +11 -11
- package/lib/us/adapters/usgov/samhsa-treatment-locator/adapter.ts +4 -4
- package/lib/us/fips-state.ts +2 -2
- package/lib/{synthesizers → us/synthesizers}/intersection.ts +12 -13
- package/lib/{synthesizers → us/synthesizers}/street.ts +16 -15
- package/lib/us/tools/fetch/imls-pls.ts +1 -1
- package/lib/us/tools/fetch/nad.ts +7 -7
- package/lib/us/tools/fetch/nppes.ts +5 -5
- package/lib/us/tools/fetch/state/hi-schools.ts +3 -2
- package/lib/us/tools/fetch/tiger-full.ts +1 -1
- package/lib/utils/align.ts +13 -10
- package/lib/utils/golden.ts +2 -2
- package/lib/utils/license.ts +3 -3
- package/lib/utils/split.ts +19 -15
- package/lib/utils/tokenize.ts +7 -26
- package/lib/utils/wof-json.ts +4 -4
- package/out/adapters/cordis/adapter.d.ts +5 -3
- package/out/adapters/cordis/adapter.d.ts.map +1 -1
- package/out/adapters/cordis/adapter.js +13 -7
- package/out/adapters/cordis/adapter.js.map +1 -1
- package/out/adapters/dedup-key.d.ts +1 -1
- package/out/adapters/dedup-key.js +1 -1
- package/out/adapters/geonames/adapter.js +1 -1
- package/out/adapters/geonames/adapter.js.map +1 -1
- package/out/adapters/gleif/adapter.d.ts +9 -9
- package/out/adapters/gleif/adapter.d.ts.map +1 -1
- package/out/adapters/gleif/adapter.js +15 -14
- package/out/adapters/gleif/adapter.js.map +1 -1
- package/out/adapters/libpostal-dictionary.d.ts +0 -2
- package/out/adapters/libpostal-dictionary.d.ts.map +1 -1
- package/out/adapters/libpostal-dictionary.js +0 -2
- package/out/adapters/libpostal-dictionary.js.map +1 -1
- package/out/adapters/openaddresses/adapter.d.ts +3 -2
- package/out/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/adapters/openaddresses/adapter.js +1 -1
- package/out/adapters/openaddresses/adapter.js.map +1 -1
- package/out/adapters/osm/adapter.d.ts +17 -16
- package/out/adapters/osm/adapter.d.ts.map +1 -1
- package/out/adapters/osm/adapter.js +10 -9
- package/out/adapters/osm/adapter.js.map +1 -1
- package/out/adapters/po-box/adapter.d.ts +6 -3
- package/out/adapters/po-box/adapter.d.ts.map +1 -1
- package/out/adapters/po-box/adapter.js +4 -2
- package/out/adapters/po-box/adapter.js.map +1 -1
- package/out/adapters/registry.d.ts +3 -3
- package/out/adapters/registry.d.ts.map +1 -1
- package/out/adapters/registry.js +1 -1
- package/out/adapters/registry.js.map +1 -1
- package/out/adapters/source-id.d.ts +2 -2
- package/out/adapters/source-id.js +2 -2
- package/out/adapters/street-line.d.ts +7 -6
- package/out/adapters/street-line.d.ts.map +1 -1
- package/out/adapters/street-line.js +7 -6
- package/out/adapters/street-line.js.map +1 -1
- package/out/adapters/wof/admin/json/adapter.js +1 -1
- package/out/adapters/wof/admin/json/adapter.js.map +1 -1
- package/out/adapters/wof/json-rows.d.ts +2 -1
- package/out/adapters/wof/json-rows.d.ts.map +1 -1
- package/out/adapters/wof/json-rows.js.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.d.ts.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.js +5 -1
- package/out/adapters/wof/postalcode-json/adapter.js.map +1 -1
- package/out/adapters.d.ts +5 -5
- package/out/adapters.js +5 -5
- package/out/au/adapters/gnaf/adapter.js +1 -1
- package/out/au/adapters/gnaf/adapter.js.map +1 -1
- package/out/be/adapters/brussels/adapter.d.ts +9 -9
- package/out/be/adapters/brussels/adapter.d.ts.map +1 -1
- package/out/be/adapters/brussels/adapter.js +21 -21
- package/out/be/adapters/brussels/adapter.js.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.d.ts +34 -37
- package/out/be/adapters/vlaanderen/adapter.d.ts.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.js +40 -38
- package/out/be/adapters/vlaanderen/adapter.js.map +1 -1
- package/out/be/adapters/wallonie/adapter.d.ts +2 -2
- package/out/be/adapters/wallonie/adapter.js +17 -17
- package/out/be/adapters/wallonie/adapter.js.map +1 -1
- package/out/be/tools/fetch/brussels.d.ts +11 -8
- package/out/be/tools/fetch/brussels.d.ts.map +1 -1
- package/out/be/tools/fetch/brussels.js +10 -8
- package/out/be/tools/fetch/brussels.js.map +1 -1
- package/out/be/tools/fetch/wallonie.d.ts +6 -5
- package/out/be/tools/fetch/wallonie.d.ts.map +1 -1
- package/out/be/tools/fetch/wallonie.js +4 -4
- package/out/be/tools/fetch/wallonie.js.map +1 -1
- package/out/build/adapters.d.ts.map +1 -1
- package/out/build/adapters.js +2 -1
- package/out/build/adapters.js.map +1 -1
- package/out/build/align.d.ts +3 -2
- package/out/build/align.d.ts.map +1 -1
- package/out/build/align.js +2 -2
- package/out/build/align.js.map +1 -1
- package/out/build/checkpoint.d.ts +1 -1
- package/out/build/checkpoint.d.ts.map +1 -1
- package/out/build/checkpoint.js +4 -2
- package/out/build/checkpoint.js.map +1 -1
- package/out/build/eligibility.d.ts +2 -2
- package/out/build/eligibility.js +3 -3
- package/out/build/eligibility.js.map +1 -1
- package/out/build/types.d.ts +8 -6
- package/out/build/types.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.d.ts +14 -14
- package/out/cz/adapters/cuzk/adapter.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.js +38 -35
- package/out/cz/adapters/cuzk/adapter.js.map +1 -1
- package/out/cz/tools/fetch/cuzk.d.ts +31 -22
- package/out/cz/tools/fetch/cuzk.d.ts.map +1 -1
- package/out/cz/tools/fetch/cuzk.js +21 -19
- package/out/cz/tools/fetch/cuzk.js.map +1 -1
- package/out/de/recipes/locale.js +1 -1
- package/out/de/recipes/locale.js.map +1 -1
- package/out/dk/adapters/inspire/adapter.d.ts +32 -33
- package/out/dk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/dk/adapters/inspire/adapter.js +38 -38
- package/out/dk/adapters/inspire/adapter.js.map +1 -1
- package/out/dk/tools/fetch/inspire.d.ts +6 -5
- package/out/dk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/dk/tools/fetch/inspire.js +15 -13
- package/out/dk/tools/fetch/inspire.js.map +1 -1
- package/out/ee/adapters/ads/adapter.d.ts +19 -26
- package/out/ee/adapters/ads/adapter.d.ts.map +1 -1
- package/out/ee/adapters/ads/adapter.js +22 -28
- package/out/ee/adapters/ads/adapter.js.map +1 -1
- package/out/ee/tools/fetch/ads.d.ts +3 -3
- package/out/ee/tools/fetch/ads.d.ts.map +1 -1
- package/out/ee/tools/fetch/ads.js +10 -8
- package/out/ee/tools/fetch/ads.js.map +1 -1
- package/out/es/adapters/bizkaia/adapter.d.ts +28 -28
- package/out/es/adapters/bizkaia/adapter.d.ts.map +1 -1
- package/out/es/adapters/bizkaia/adapter.js +43 -43
- package/out/es/adapters/bizkaia/adapter.js.map +1 -1
- package/out/es/adapters/catastro/adapter.d.ts +22 -23
- package/out/es/adapters/catastro/adapter.d.ts.map +1 -1
- package/out/es/adapters/catastro/adapter.js +35 -33
- package/out/es/adapters/catastro/adapter.js.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.d.ts +31 -38
- package/out/es/adapters/gipuzkoa/adapter.d.ts.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.js +40 -47
- package/out/es/adapters/gipuzkoa/adapter.js.map +1 -1
- package/out/es/adapters/navarra/adapter.d.ts +20 -19
- package/out/es/adapters/navarra/adapter.d.ts.map +1 -1
- package/out/es/adapters/navarra/adapter.js +23 -22
- package/out/es/adapters/navarra/adapter.js.map +1 -1
- package/out/es/cadastre.d.ts +21 -20
- package/out/es/cadastre.d.ts.map +1 -1
- package/out/es/cadastre.js +19 -21
- package/out/es/cadastre.js.map +1 -1
- package/out/es/tools/fetch/bizkaia.d.ts +28 -25
- package/out/es/tools/fetch/bizkaia.d.ts.map +1 -1
- package/out/es/tools/fetch/bizkaia.js +25 -24
- package/out/es/tools/fetch/bizkaia.js.map +1 -1
- package/out/es/tools/fetch/catastro.d.ts +28 -20
- package/out/es/tools/fetch/catastro.d.ts.map +1 -1
- package/out/es/tools/fetch/catastro.js +36 -31
- package/out/es/tools/fetch/catastro.js.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.d.ts +11 -9
- package/out/es/tools/fetch/gipuzkoa.d.ts.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.js +9 -8
- package/out/es/tools/fetch/gipuzkoa.js.map +1 -1
- package/out/es/tools/fetch/navarra.d.ts +20 -15
- package/out/es/tools/fetch/navarra.d.ts.map +1 -1
- package/out/es/tools/fetch/navarra.js +21 -16
- package/out/es/tools/fetch/navarra.js.map +1 -1
- package/out/fi/adapters/ryhti/adapter.d.ts +16 -20
- package/out/fi/adapters/ryhti/adapter.d.ts.map +1 -1
- package/out/fi/adapters/ryhti/adapter.js +21 -25
- package/out/fi/adapters/ryhti/adapter.js.map +1 -1
- package/out/fi/tools/fetch/ryhti.d.ts +24 -24
- package/out/fi/tools/fetch/ryhti.d.ts.map +1 -1
- package/out/fi/tools/fetch/ryhti.js +21 -24
- package/out/fi/tools/fetch/ryhti.js.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.d.ts +8 -8
- package/out/fr/adapters/annuaire-education/adapter.d.ts.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.js +13 -11
- package/out/fr/adapters/annuaire-education/adapter.js.map +1 -1
- package/out/fr/adapters/ban/adapter.d.ts +4 -4
- package/out/fr/adapters/ban/adapter.js +6 -6
- package/out/fr/adapters/ban/adapter.js.map +1 -1
- package/out/fr/adapters/finess/adapter.d.ts +12 -11
- package/out/fr/adapters/finess/adapter.d.ts.map +1 -1
- package/out/fr/adapters/finess/adapter.js +16 -15
- package/out/fr/adapters/finess/adapter.js.map +1 -1
- package/out/fr/adapters/finess/voie-type.d.ts +2 -2
- package/out/fr/adapters/finess/voie-type.d.ts.map +1 -1
- package/out/fr/adapters/finess/voie-type.js +4 -3
- package/out/fr/adapters/finess/voie-type.js.map +1 -1
- package/out/fr/insee-country.d.ts +2 -2
- package/out/fr/insee-country.js +2 -2
- package/out/fr/recipes/lieudit.d.ts.map +1 -1
- package/out/fr/recipes/lieudit.js +5 -4
- package/out/fr/recipes/lieudit.js.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.d.ts +7 -4
- package/out/fr/tools/fetch/annuaire-education.d.ts.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.js +9 -7
- package/out/fr/tools/fetch/annuaire-education.js.map +1 -1
- package/out/fr/tools/fetch/ban.d.ts +1 -1
- package/out/fr/tools/fetch/ban.d.ts.map +1 -1
- package/out/fr/tools/fetch/ban.js +3 -2
- package/out/fr/tools/fetch/ban.js.map +1 -1
- package/out/fr/tools/fetch/finess.d.ts +5 -5
- package/out/fr/tools/fetch/finess.d.ts.map +1 -1
- package/out/fr/tools/fetch/finess.js +13 -12
- package/out/fr/tools/fetch/finess.js.map +1 -1
- package/out/inspire/address.d.ts +62 -52
- package/out/inspire/address.d.ts.map +1 -1
- package/out/inspire/address.js +71 -60
- package/out/inspire/address.js.map +1 -1
- package/out/inspire/errors.d.ts +9 -8
- package/out/inspire/errors.d.ts.map +1 -1
- package/out/inspire/errors.js +7 -6
- package/out/inspire/errors.js.map +1 -1
- package/out/inspire/stream.d.ts +25 -22
- package/out/inspire/stream.d.ts.map +1 -1
- package/out/inspire/stream.js +10 -12
- package/out/inspire/stream.js.map +1 -1
- package/out/international/recipes/locale.d.ts +2 -2
- package/out/international/recipes/locale.d.ts.map +1 -1
- package/out/international/recipes/locale.js +11 -7
- package/out/international/recipes/locale.js.map +1 -1
- package/out/it/adapters/anac/adapter.d.ts +45 -43
- package/out/it/adapters/anac/adapter.d.ts.map +1 -1
- package/out/it/adapters/anac/adapter.js +48 -43
- package/out/it/adapters/anac/adapter.js.map +1 -1
- package/out/it/tools/fetch/anac.d.ts +16 -19
- package/out/it/tools/fetch/anac.d.ts.map +1 -1
- package/out/it/tools/fetch/anac.js +20 -23
- package/out/it/tools/fetch/anac.js.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.d.ts.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.js.map +1 -1
- package/out/jp/tools/fetch/houjin.js +2 -2
- package/out/jp/tools/fetch/houjin.js.map +1 -1
- package/out/kr/adapters/localdata/permit-rows.js +1 -1
- package/out/kr/adapters/localdata/permit-rows.js.map +1 -1
- package/out/kr/tools/fetch/juso.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.js +1 -1
- package/out/kr/tools/fetch/localdata.js.map +1 -1
- package/out/lu/adapters/bd-adresses/adapter.d.ts +9 -9
- package/out/lu/adapters/bd-adresses/adapter.js +10 -10
- package/out/lu/tools/fetch/bd-adresses.d.ts +11 -9
- package/out/lu/tools/fetch/bd-adresses.d.ts.map +1 -1
- package/out/lu/tools/fetch/bd-adresses.js +11 -11
- package/out/lu/tools/fetch/bd-adresses.js.map +1 -1
- package/out/lv/adapters/varis/adapter.d.ts +24 -23
- package/out/lv/adapters/varis/adapter.d.ts.map +1 -1
- package/out/lv/adapters/varis/adapter.js +24 -23
- package/out/lv/adapters/varis/adapter.js.map +1 -1
- package/out/lv/tools/fetch/varis.d.ts +2 -1
- package/out/lv/tools/fetch/varis.d.ts.map +1 -1
- package/out/lv/tools/fetch/varis.js +1 -1
- package/out/lv/tools/fetch/varis.js.map +1 -1
- package/out/nl/adapters/kadaster/adapter.d.ts +12 -12
- package/out/nl/adapters/kadaster/adapter.d.ts.map +1 -1
- package/out/nl/adapters/kadaster/adapter.js +22 -23
- package/out/nl/adapters/kadaster/adapter.js.map +1 -1
- package/out/nl/tools/fetch/kadaster.d.ts +14 -10
- package/out/nl/tools/fetch/kadaster.d.ts.map +1 -1
- package/out/nl/tools/fetch/kadaster.js +12 -9
- package/out/nl/tools/fetch/kadaster.js.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.js +3 -3
- package/out/no/adapters/matrikkelen/adapter.js.map +1 -1
- package/out/no/tools/fetch/matrikkelen.d.ts +50 -38
- package/out/no/tools/fetch/matrikkelen.d.ts.map +1 -1
- package/out/no/tools/fetch/matrikkelen.js +44 -36
- package/out/no/tools/fetch/matrikkelen.js.map +1 -1
- package/out/parquet/duckdb.d.ts +7 -4
- package/out/parquet/duckdb.d.ts.map +1 -1
- package/out/parquet/duckdb.js +2 -1
- package/out/parquet/duckdb.js.map +1 -1
- package/out/parquet/readers.js +3 -3
- package/out/parquet/readers.js.map +1 -1
- package/out/parquet/schema.js +1 -1
- package/out/parquet/schema.js.map +1 -1
- package/out/parquet/writers.d.ts +1 -1
- package/out/parquet/writers.d.ts.map +1 -1
- package/out/parquet/writers.js +2 -1
- package/out/parquet/writers.js.map +1 -1
- package/out/pl/adapters/emuia/adapter.d.ts +8 -8
- package/out/pl/adapters/emuia/adapter.js +12 -12
- package/out/pl/adapters/emuia/adapter.js.map +1 -1
- package/out/pl/tools/fetch/emuia.d.ts +5 -4
- package/out/pl/tools/fetch/emuia.d.ts.map +1 -1
- package/out/pl/tools/fetch/emuia.js +10 -9
- package/out/pl/tools/fetch/emuia.js.map +1 -1
- package/out/recipes/bare/postcode.js +1 -1
- package/out/recipes/bare/postcode.js.map +1 -1
- package/out/recipes/boundary-stress.js +1 -1
- package/out/recipes/boundary-stress.js.map +1 -1
- package/out/recipes/house-venue.d.ts.map +1 -1
- package/out/recipes/house-venue.js +6 -3
- package/out/recipes/house-venue.js.map +1 -1
- package/out/recipes/intersection.js +2 -2
- package/out/recipes/intersection.js.map +1 -1
- package/out/recipes/po/box/cedex/recipe.js +2 -1
- package/out/recipes/po/box/cedex/recipe.js.map +1 -1
- package/out/recipes/po/box/cedex/sources.d.ts +8 -2
- package/out/recipes/po/box/cedex/sources.d.ts.map +1 -1
- package/out/recipes/po/box/cedex/sources.js +1 -1
- package/out/recipes/po/box/cedex/sources.js.map +1 -1
- package/out/recipes/scaffold.d.ts +28 -19
- package/out/recipes/scaffold.d.ts.map +1 -1
- package/out/recipes/scaffold.js +3 -2
- package/out/recipes/scaffold.js.map +1 -1
- package/out/recipes/sources.d.ts +1 -1
- package/out/recipes/sources.d.ts.map +1 -1
- package/out/recipes/sources.js +15 -2
- package/out/recipes/sources.js.map +1 -1
- package/out/recipes/street/bare.js +2 -2
- package/out/recipes/street/bare.js.map +1 -1
- package/out/recipes/street.d.ts.map +1 -1
- package/out/recipes/street.js +3 -1
- package/out/recipes/street.js.map +1 -1
- package/out/recipes/sub/venue/context.d.ts.map +1 -1
- package/out/recipes/sub/venue/context.js +2 -0
- package/out/recipes/sub/venue/context.js.map +1 -1
- package/out/recipes/sub/venue/render.js +1 -1
- package/out/recipes/sub/venue/render.js.map +1 -1
- package/out/recipes/sub/venue/sources.d.ts +6 -3
- package/out/recipes/sub/venue/sources.d.ts.map +1 -1
- package/out/recipes/sub/venue/sources.js +16 -20
- package/out/recipes/sub/venue/sources.js.map +1 -1
- package/out/recipes/sub/venue.d.ts +2 -5
- package/out/recipes/sub/venue.d.ts.map +1 -1
- package/out/recipes/sub/venue.js +10 -14
- package/out/recipes/sub/venue.js.map +1 -1
- package/out/recipes/trailing-region.d.ts.map +1 -1
- package/out/recipes/trailing-region.js +6 -1
- package/out/recipes/trailing-region.js.map +1 -1
- package/out/recipes/unit.js +2 -2
- package/out/recipes/unit.js.map +1 -1
- package/out/recipes/venue.d.ts +119 -0
- package/out/recipes/venue.d.ts.map +1 -0
- package/out/recipes/venue.js +245 -0
- package/out/recipes/venue.js.map +1 -0
- package/out/recipes.d.ts +1 -1
- package/out/recipes.d.ts.map +1 -1
- package/out/recipes.js +3 -1
- package/out/recipes.js.map +1 -1
- package/out/registers.d.ts +8 -6
- package/out/registers.d.ts.map +1 -1
- package/out/registers.js +8 -6
- package/out/registers.js.map +1 -1
- package/out/ro/adapters/retea-scolara/adapter.d.ts +5 -5
- package/out/ro/adapters/retea-scolara/adapter.js +5 -5
- package/out/ro/tools/fetch/retea-scolara.js +1 -1
- package/out/ro/tools/fetch/retea-scolara.js.map +1 -1
- package/out/runner.d.ts +3 -2
- package/out/runner.d.ts.map +1 -1
- package/out/runner.js +1 -1
- package/out/runner.js.map +1 -1
- package/out/sg/recipes/register.js +1 -1
- package/out/sg/recipes/register.js.map +1 -1
- package/out/sg/tools/fetch/acra.js +2 -2
- package/out/sg/tools/fetch/acra.js.map +1 -1
- package/out/sk/adapters/inspire/adapter.d.ts +5 -5
- package/out/sk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/sk/adapters/inspire/adapter.js +18 -18
- package/out/sk/adapters/inspire/adapter.js.map +1 -1
- package/out/sk/tools/fetch/inspire.d.ts +11 -10
- package/out/sk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/sk/tools/fetch/inspire.js +15 -14
- package/out/sk/tools/fetch/inspire.js.map +1 -1
- package/out/source-register/effective-manifest.js +3 -3
- package/out/source-register/effective-manifest.js.map +1 -1
- package/out/source-register/register.d.ts +2 -2
- package/out/source-register/register.d.ts.map +1 -1
- package/out/source-register/register.js +2 -2
- package/out/source-register/register.js.map +1 -1
- package/out/source-register/training-manifest.d.ts +1 -1
- package/out/source-register/types.d.ts +12 -13
- package/out/source-register/types.d.ts.map +1 -1
- package/out/source-register/types.js +2 -2
- package/out/source-register/types.js.map +1 -1
- package/out/subvenue/promotions.d.ts +1 -1
- package/out/subvenue/promotions.js +1 -1
- package/out/subvenue/table.d.ts +4 -4
- package/out/subvenue/table.js +3 -3
- package/out/surfaces/locale.d.ts.map +1 -1
- package/out/surfaces/locale.js +2 -0
- package/out/surfaces/locale.js.map +1 -1
- package/out/synthesizers/anchor-absorption.d.ts +2 -1
- package/out/synthesizers/anchor-absorption.d.ts.map +1 -1
- package/out/synthesizers/anchor-absorption.js.map +1 -1
- package/out/synthesizers/boundary-stress.d.ts +1 -1
- package/out/synthesizers/boundary-stress.d.ts.map +1 -1
- package/out/synthesizers/boundary-stress.js +20 -23
- package/out/synthesizers/boundary-stress.js.map +1 -1
- package/out/synthesizers/house-venue.d.ts +3 -2
- package/out/synthesizers/house-venue.d.ts.map +1 -1
- package/out/synthesizers/house-venue.js +24 -26
- package/out/synthesizers/house-venue.js.map +1 -1
- package/out/synthesizers/no-street.d.ts.map +1 -1
- package/out/synthesizers/no-street.js.map +1 -1
- package/out/synthesizers/po-box.d.ts +4 -2
- package/out/synthesizers/po-box.d.ts.map +1 -1
- package/out/synthesizers/po-box.js.map +1 -1
- package/out/synthesizers/tail.d.ts +36 -0
- package/out/synthesizers/tail.d.ts.map +1 -0
- package/out/synthesizers/tail.js +32 -0
- package/out/synthesizers/tail.js.map +1 -0
- package/out/synthesizers/utils.d.ts +2 -1
- package/out/synthesizers/utils.d.ts.map +1 -1
- package/out/synthesizers/utils.js +17 -1
- package/out/synthesizers/utils.js.map +1 -1
- package/out/test-kit.js +2 -2
- package/out/test-kit.js.map +1 -1
- package/out/tools/audit.d.ts +2 -1
- package/out/tools/audit.d.ts.map +1 -1
- package/out/tools/audit.js +4 -3
- package/out/tools/audit.js.map +1 -1
- package/out/tools/corpus-stats.js +1 -1
- package/out/tools/corpus-stats.js.map +1 -1
- package/out/tools/fetch/atom.d.ts +8 -8
- package/out/tools/fetch/atom.d.ts.map +1 -1
- package/out/tools/fetch/atom.js +7 -8
- package/out/tools/fetch/atom.js.map +1 -1
- package/out/tools/fetch/cordis.d.ts +4 -3
- package/out/tools/fetch/cordis.d.ts.map +1 -1
- package/out/tools/fetch/cordis.js +9 -7
- package/out/tools/fetch/cordis.js.map +1 -1
- package/out/tools/fetch/download/manifest.d.ts.map +1 -1
- package/out/tools/fetch/download/manifest.js +2 -1
- package/out/tools/fetch/download/manifest.js.map +1 -1
- package/out/tools/fetch/download/network.d.ts +4 -4
- package/out/tools/fetch/download/retry.js +4 -4
- package/out/tools/fetch/download/retry.js.map +1 -1
- package/out/tools/fetch/geonames/dump.d.ts +3 -2
- package/out/tools/fetch/geonames/dump.d.ts.map +1 -1
- package/out/tools/fetch/geonames/dump.js +2 -1
- package/out/tools/fetch/geonames/dump.js.map +1 -1
- package/out/tools/fetch/geonames/postal.d.ts +2 -2
- package/out/tools/fetch/gleif.d.ts +3 -2
- package/out/tools/fetch/gleif.d.ts.map +1 -1
- package/out/tools/fetch/gleif.js +6 -4
- package/out/tools/fetch/gleif.js.map +1 -1
- package/out/tools/fetch/header.d.ts +7 -5
- package/out/tools/fetch/header.d.ts.map +1 -1
- package/out/tools/fetch/header.js +8 -6
- package/out/tools/fetch/header.js.map +1 -1
- package/out/tools/fetch/inspire-addresses.d.ts +32 -27
- package/out/tools/fetch/inspire-addresses.d.ts.map +1 -1
- package/out/tools/fetch/inspire-addresses.js +25 -24
- package/out/tools/fetch/inspire-addresses.js.map +1 -1
- package/out/tools/fetch/openaddresses.js +1 -1
- package/out/tools/fetch/openaddresses.js.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.d.ts +5 -5
- package/out/tools/fetch/vlaanderen-ad.d.ts.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.js +16 -16
- package/out/tools/fetch/vlaanderen-ad.js.map +1 -1
- package/out/tools/fetch/wfs-harvest.d.ts +26 -16
- package/out/tools/fetch/wfs-harvest.d.ts.map +1 -1
- package/out/tools/fetch/wfs-harvest.js +21 -18
- package/out/tools/fetch/wfs-harvest.js.map +1 -1
- package/out/tools/fetch.d.ts +0 -41
- package/out/tools/fetch.d.ts.map +1 -1
- package/out/tools/fetch.js +0 -41
- package/out/tools/fetch.js.map +1 -1
- package/out/tools/golden/expand.d.ts +7 -7
- package/out/tools/golden/expand.js +1 -1
- package/out/tools/golden/expand.js.map +1 -1
- package/out/tools/golden/promote.d.ts +2 -2
- package/out/tools/golden/promote.d.ts.map +1 -1
- package/out/tools/golden/promote.js +9 -6
- package/out/tools/golden/promote.js.map +1 -1
- package/out/tools/golden/relabel-street/directory.d.ts +4 -4
- package/out/tools/golden/relabel-street/directory.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/directory.js +3 -2
- package/out/tools/golden/relabel-street/directory.js.map +1 -1
- package/out/tools/golden/relabel-street/row.d.ts +4 -3
- package/out/tools/golden/relabel-street/row.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/row.js +3 -3
- package/out/tools/golden/relabel-street/row.js.map +1 -1
- package/out/tools/ingest-csv.js +3 -3
- package/out/tools/ingest-csv.js.map +1 -1
- package/out/tools/jsonl-to-parquet.d.ts +1 -1
- package/out/tools/lint/recipe-output/vocab.d.ts +10 -5
- package/out/tools/lint/recipe-output/vocab.d.ts.map +1 -1
- package/out/tools/lint/recipe-output/vocab.js.map +1 -1
- package/out/tools/lint/recipe-output.d.ts +8 -8
- package/out/tools/lint/recipe-output.d.ts.map +1 -1
- package/out/tools/lint/recipe-output.js +17 -0
- package/out/tools/lint/recipe-output.js.map +1 -1
- package/out/tools/overlay/kryptonite.d.ts +2 -2
- package/out/tools/overlay/kryptonite.d.ts.map +1 -1
- package/out/tools/overlay/kryptonite.js +2 -1
- package/out/tools/overlay/kryptonite.js.map +1 -1
- package/out/tools/overlay/manifest.d.ts +2 -1
- package/out/tools/overlay/manifest.d.ts.map +1 -1
- package/out/tools/overlay/manifest.js +2 -1
- package/out/tools/overlay/manifest.js.map +1 -1
- package/out/tools/overlay/merge-source.d.ts +2 -1
- package/out/tools/overlay/merge-source.d.ts.map +1 -1
- package/out/tools/overlay/merge-source.js.map +1 -1
- package/out/tools/overlay/split-slice.d.ts +1 -1
- package/out/tools/overlay/split-slice.js +1 -1
- package/out/tools/overlay/split-slice.js.map +1 -1
- package/out/tools/overlay/translit.d.ts +4 -3
- package/out/tools/overlay/translit.d.ts.map +1 -1
- package/out/tools/overlay/translit.js +2 -1
- package/out/tools/overlay/translit.js.map +1 -1
- package/out/tools/overture-subvenue.d.ts +1 -1
- package/out/tools/overture-subvenue.d.ts.map +1 -1
- package/out/tools/postcode-triples.d.ts +1 -1
- package/out/tools/postcode-triples.d.ts.map +1 -1
- package/out/tools/postcode-triples.js +4 -3
- package/out/tools/postcode-triples.js.map +1 -1
- package/out/tools/source-register/build.d.ts +10 -8
- package/out/tools/source-register/build.d.ts.map +1 -1
- package/out/tools/source-register/build.js +10 -8
- package/out/tools/source-register/build.js.map +1 -1
- package/out/tools/sub/venue/lexicon.d.ts +2 -6
- package/out/tools/sub/venue/lexicon.d.ts.map +1 -1
- package/out/tools/sub/venue/lexicon.js +2 -6
- package/out/tools/sub/venue/lexicon.js.map +1 -1
- package/out/tw/tools/fetch/gcis.js +3 -3
- package/out/tw/tools/fetch/gcis.js.map +1 -1
- package/out/types.d.ts +4 -3
- package/out/types.d.ts.map +1 -1
- package/out/types.js.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.js.map +1 -1
- package/out/us/adapters/usgov/nad/adapter.js +6 -6
- package/out/us/adapters/usgov/nad/adapter.js.map +1 -1
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js +2 -2
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/us/fips-state.d.ts +2 -2
- package/out/us/fips-state.d.ts.map +1 -1
- package/out/us/fips-state.js.map +1 -1
- package/out/us/synthesizers/intersection.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/intersection.js +13 -9
- package/out/us/synthesizers/intersection.js.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.d.ts +3 -2
- package/out/us/synthesizers/street.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.js +12 -12
- package/out/us/synthesizers/street.js.map +1 -0
- package/out/us/tools/fetch/imls-pls.js +1 -1
- package/out/us/tools/fetch/imls-pls.js.map +1 -1
- package/out/us/tools/fetch/nad.d.ts +6 -6
- package/out/us/tools/fetch/nad.js +1 -1
- package/out/us/tools/fetch/nad.js.map +1 -1
- package/out/us/tools/fetch/nppes.js +3 -3
- package/out/us/tools/fetch/nppes.js.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts +2 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.js +1 -1
- package/out/us/tools/fetch/state/hi-schools.js.map +1 -1
- package/out/us/tools/fetch/tiger-full.js +1 -1
- package/out/us/tools/fetch/tiger-full.js.map +1 -1
- package/out/utils/align.d.ts +5 -3
- package/out/utils/align.d.ts.map +1 -1
- package/out/utils/align.js +5 -5
- package/out/utils/align.js.map +1 -1
- package/out/utils/golden.js +2 -2
- package/out/utils/golden.js.map +1 -1
- package/out/utils/license.d.ts +3 -3
- package/out/utils/license.d.ts.map +1 -1
- package/out/utils/license.js.map +1 -1
- package/out/utils/split.d.ts +1 -1
- package/out/utils/split.d.ts.map +1 -1
- package/out/utils/split.js +18 -14
- package/out/utils/split.js.map +1 -1
- package/out/utils/tokenize.d.ts +3 -20
- package/out/utils/tokenize.d.ts.map +1 -1
- package/out/utils/tokenize.js.map +1 -1
- package/out/utils/wof-json.js +3 -3
- package/out/utils/wof-json.js.map +1 -1
- package/package.json +18 -27
- package/lib/tools.ts +0 -31
- package/out/synthesizers/intersection.d.ts.map +0 -1
- package/out/synthesizers/intersection.js.map +0 -1
- package/out/synthesizers/street.d.ts.map +0 -1
- package/out/synthesizers/street.js.map +0 -1
- package/out/tools.d.ts +0 -31
- package/out/tools.d.ts.map +0 -1
- package/out/tools.js +0 -31
- package/out/tools.js.map +0 -1
- /package/out/{synthesizers → us/synthesizers}/intersection.d.ts +0 -0
|
@@ -15,11 +15,11 @@
|
|
|
15
15
|
* Four properties of this service decide the harvest's shape.
|
|
16
16
|
*
|
|
17
17
|
* 1. **The archive link is the `enclosure` link, and the `alternate` link is not it.** Each entry
|
|
18
|
-
*
|
|
18
|
+
* lists six links, three of which address the same archive under `enclosure`, `section` and
|
|
19
19
|
* `alternate`. The entry's *first* `alternate` link is the relative href
|
|
20
20
|
* `Addresses_DatasetATOM_Navarra.xml`, which is the dataset feed, so a reader taking the first
|
|
21
21
|
* `alternate` link downloads an ATOM document in place of the archive. This harvest takes
|
|
22
|
-
* `enclosure` and refuses an entry that
|
|
22
|
+
* `enclosure` and refuses an entry that lists none.
|
|
23
23
|
* 2. **An entry leaves its own area unstated.** Every one of the 272 entries is titled
|
|
24
24
|
* `Address Navarra` and every `<id>` is empty, so the archive's own file name,
|
|
25
25
|
* `AD_Navarra_<n>.gml.zip`, is the only statement of which partition an entry offers. The
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
*/
|
|
47
47
|
import { APIClient, assertNoOGCServiceException } from "@mailwoman/core/api";
|
|
48
48
|
import { ByteFormatter } from "@mailwoman/core/fs/formatters";
|
|
49
|
-
import { tryStat } from "@mailwoman/core/fs/readers";
|
|
49
|
+
import { tryStat } from "@mailwoman/core/fs/readers/stat";
|
|
50
50
|
import { makeDirectories } from "@mailwoman/core/fs/writers";
|
|
51
51
|
import { sha256File } from "@mailwoman/core/hash";
|
|
52
52
|
import { stringifyJSON } from "@mailwoman/core/json";
|
|
@@ -56,7 +56,8 @@ import { feedChunks, linkWithRel, readAtomFeed } from "#tools/fetch/atom";
|
|
|
56
56
|
import { loadCollectionFiles, writeManifest } from "#tools/fetch/download";
|
|
57
57
|
import { downloadZipArchive } from "#tools/fetch/zip-archive";
|
|
58
58
|
/**
|
|
59
|
-
* The ATOM service document
|
|
59
|
+
* The ATOM service document.
|
|
60
|
+
* It lists one zipped GML partition per entry.
|
|
60
61
|
*/
|
|
61
62
|
export const ES_NAVARRA_SERVICE_FEED_URL = "https://filescartografia.navarra.es/2_CARTOGRAFIA_TEMATICA/2_7_CATASTRO/2_7_3_INSPIRE_ATOM/2_7_3_3_AD/Addresses_ServiceATOM_Navarra.xml";
|
|
62
63
|
/**
|
|
@@ -75,7 +76,8 @@ export const ES_NAVARRA_ATTRIBUTION = "Gobierno de Navarra";
|
|
|
75
76
|
*/
|
|
76
77
|
export const ES_NAVARRA_REQUEST_INTERVAL_MS = 250;
|
|
77
78
|
/**
|
|
78
|
-
* The directory the harvest writes under
|
|
79
|
+
* The directory the harvest writes under.
|
|
80
|
+
* The adapter reads this directory as its `inputPath`.
|
|
79
81
|
*/
|
|
80
82
|
const SLUG = ES_NAVARRA_ADAPTER_ID;
|
|
81
83
|
/**
|
|
@@ -92,8 +94,8 @@ const ARCHIVE_FILENAME = /^AD_Navarra_(\d+)\.gml\.zip$/iu;
|
|
|
92
94
|
/**
|
|
93
95
|
* The partitions the service document lists.
|
|
94
96
|
*
|
|
95
|
-
* @throws When the document holds no entry, when an entry
|
|
96
|
-
*
|
|
97
|
+
* @throws When the document holds no entry, when an entry lists no `enclosure` link, when a linked
|
|
98
|
+
* archive's file name is not `AD_Navarra_<n>.gml.zip`, or when two entries point to one archive.
|
|
97
99
|
* Each of those would otherwise write zero archives while reporting a completed run.
|
|
98
100
|
*/
|
|
99
101
|
export function readESNavarraServiceFeed(feed) {
|
|
@@ -138,8 +140,9 @@ export function readESNavarraServiceFeed(feed) {
|
|
|
138
140
|
* Whether the archive recorded for one partition is still the one the feed describes.
|
|
139
141
|
*
|
|
140
142
|
* A skip requires the feed to state an `<updated>` value.
|
|
141
|
-
* Where it states none, both sides read an empty string and an equality test would hold
|
|
142
|
-
*
|
|
143
|
+
* Where it states none, both sides read an empty string and an equality test would hold.
|
|
144
|
+
*
|
|
145
|
+
* That test would keep an archive of unknown age for as long as the feed stayed silent.
|
|
143
146
|
*/
|
|
144
147
|
async function isCurrent(recorded, partition, path, verifyDigests) {
|
|
145
148
|
if (!recorded || !partition.updated || recorded.feed_updated !== partition.updated)
|
|
@@ -147,13 +150,14 @@ async function isCurrent(recorded, partition, path, verifyDigests) {
|
|
|
147
150
|
const stat = await tryStat(path);
|
|
148
151
|
if (!stat || stat.size !== recorded.bytes)
|
|
149
152
|
return false;
|
|
150
|
-
// The digest check reads the whole file rather than its metadata
|
|
153
|
+
// The digest check reads the whole file rather than its metadata.
|
|
154
|
+
// The caller opts in.
|
|
151
155
|
if (!verifyDigests)
|
|
152
156
|
return true;
|
|
153
157
|
return (await sha256File(path)) === recorded.sha256;
|
|
154
158
|
}
|
|
155
159
|
/**
|
|
156
|
-
* The partitions this run considers, and the numbers a caller
|
|
160
|
+
* The partitions this run considers, and the numbers a caller supplied that the feed does not list.
|
|
157
161
|
*
|
|
158
162
|
* A number the caller asked for and the service document does not list is returned
|
|
159
163
|
* rather than dropped, because a partition that is not published is a different
|
|
@@ -188,8 +192,9 @@ function selectPartitions(listed, options) {
|
|
|
188
192
|
* the pacing. {@linkcode fetchESNavarra} is the registry entry point and supplies both.
|
|
189
193
|
*
|
|
190
194
|
* @returns What was fetched, what the feed states is already recorded, and which partitions failed.
|
|
191
|
-
* @throws When the service document lists no partition
|
|
192
|
-
* `{fetched: 0, skipped: 0, failed: 0}`
|
|
195
|
+
* @throws When the service document lists no partition.
|
|
196
|
+
* The call would otherwise return `{fetched: 0, skipped: 0, failed: 0}`
|
|
197
|
+
* and read to a caller as a completed fetch of an empty publisher.
|
|
193
198
|
*/
|
|
194
199
|
export async function harvestESNavarra(client, options) {
|
|
195
200
|
const { report, signal } = options;
|
|
@@ -215,8 +220,8 @@ export async function harvestESNavarra(client, options) {
|
|
|
215
220
|
await makeDirectories(destDir);
|
|
216
221
|
const previous = await loadCollectionFiles(manifestPath);
|
|
217
222
|
const files = new Map();
|
|
218
|
-
// An entry for a partition this run does not consider is
|
|
219
|
-
//
|
|
223
|
+
// An entry for a partition this run does not consider is kept, so a bounded
|
|
224
|
+
// run never drops what an earlier run recorded.
|
|
220
225
|
for (const [filename, entry] of previous) {
|
|
221
226
|
files.set(filename, entry);
|
|
222
227
|
}
|
|
@@ -242,7 +247,7 @@ export async function harvestESNavarra(client, options) {
|
|
|
242
247
|
if (signal?.aborted)
|
|
243
248
|
break;
|
|
244
249
|
const dest = destDir(partition.filename);
|
|
245
|
-
if (await isCurrent(files.get(partition.filename), partition, dest, options.verifyDigests ?? false)) {
|
|
250
|
+
if (await isCurrent(files.get(partition.filename) ?? null, partition, dest, options.verifyDigests ?? false)) {
|
|
246
251
|
skipped++;
|
|
247
252
|
continue;
|
|
248
253
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"navarra.js","sourceRoot":"","sources":["../../../../lib/es/tools/fetch/navarra.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AAEH,OAAO,EAAE,SAAS,EAAE,2BAA2B,EAAE,MAAM,qBAAqB,CAAA;AAC5E,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"navarra.js","sourceRoot":"","sources":["../../../../lib/es/tools/fetch/navarra.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AAEH,OAAO,EAAE,SAAS,EAAE,2BAA2B,EAAE,MAAM,qBAAqB,CAAA;AAC5E,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,iCAAiC,CAAA;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAA;AAE3D,OAAO,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAA;AAExF,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAEzE,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAC1E,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AAE7D;;;GAGG;AACH,MAAM,CAAC,MAAM,2BAA2B,GACvC,yIAAyI,CAAA;AAE1I;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,qBAAqB,CAAA;AAE3D;;;;;GAKG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAG,GAAG,CAAA;AAEjD;;;GAGG;AACH,MAAM,IAAI,GAAG,qBAAqB,CAAA;AAElC;;;;;GAKG;AACH,MAAM,iBAAiB,GAAG,EAAE,CAAA;AAE5B;;GAEG;AACH,MAAM,gBAAgB,GAAG,gCAAgC,CAAA;AA8BzD;;;;;;GAMG;AACH,MAAM,UAAU,wBAAwB,CAAC,IAAc;IACtD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACd,GAAG,2BAA2B,uEAAuE;YACpG,mBAAmB,CACpB,CAAA;IACF,CAAC;IAED,MAAM,UAAU,GAAyB,EAAE,CAAA;IAC3C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,gFAAgF;QAChF,+EAA+E;QAC/E,kDAAkD;QAClD,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,KAAK,EAAE,WAAW,CAAC,CAAA;QAElD,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CACd,GAAG,2BAA2B,qBAAqB,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc;gBAC1F,wDAAwD,CACzD,CAAA;QACF,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;QAChE,MAAM,QAAQ,GAAG,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAEhD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACd,GAAG,2BAA2B,oBAAoB,aAAa,CAAC,QAAQ,CAAC,uBAAuB;gBAC/F,iDAAiD,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,qBAAqB;gBAChG,4DAA4D,CAC7D,CAAA;QACF,CAAC;QAED,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACd,GAAG,2BAA2B,KAAK,aAAa,CAAC,QAAQ,CAAC,sCAAsC;gBAC/F,oCAAoC,CACrC,CAAA;QACF,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAElB,UAAU,CAAC,IAAI,CAAC;YACf,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAE;YACvB,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE;YACzB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,UAAU,EAAE,IAAI,CAAC,IAAI;YACrB,QAAQ;SACR,CAAC,CAAA;IACH,CAAC;IAED,OAAO,UAAU,CAAA;AAClB,CAAC;AA4DD;;;;;;;GAOG;AACH,KAAK,UAAU,SAAS,CACvB,QAAsC,EACtC,SAA6B,EAC7B,IAAqB,EACrB,aAAsB;IAEtB,IAAI,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,OAAO,IAAI,QAAQ,CAAC,YAAY,KAAK,SAAS,CAAC,OAAO;QAAE,OAAO,KAAK,CAAA;IAEhG,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAEhC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,KAAK;QAAE,OAAO,KAAK,CAAA;IAEvD,kEAAkE;IAClE,sBAAsB;IACtB,IAAI,CAAC,aAAa;QAAE,OAAO,IAAI,CAAA;IAE/B,OAAO,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAA;AACpD,CAAC;AAED;;;;;;GAMG;AACH,SAAS,gBAAgB,CACxB,MAAqC,EACrC,OAA8D;IAE9D,IAAI,QAAQ,GAAG,MAAM,CAAA;IACrB,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAA;QACrF,MAAM,KAAK,GAAyB,EAAE,CAAA;QAEtC,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACzC,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YAEtC,IAAI,SAAS,EAAE,CAAC;gBACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACtB,CAAC;iBAAM,CAAC;gBACP,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YACrB,CAAC;QACF,CAAC;QAED,QAAQ,GAAG,KAAK,CAAA;IACjB,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QACjC,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;IAC5C,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAA;AAC7B,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACrC,MAAgC,EAChC,OAAgC;IAEhC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IAClC,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IACnD,MAAM,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC,CAAA;IAE7C,MAAM,EAAE,CAAC,OAAO,IAAI,aAAa,2BAA2B,EAAE,CAAC,CAAA;IAE/D,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAAS;QACvD,MAAM,EAAE,KAAK;QACb,GAAG,EAAE,2BAA2B;QAChC,YAAY,EAAE,MAAM;QACpB,OAAO,EAAE,OAAO;QAChB,MAAM;KACN,CAAC,CAAA;IAEF,2EAA2E;IAC3E,2BAA2B,CAAC,UAAU,EAAE,GAAG,IAAI,mBAAmB,CAAC,CAAA;IAEnE,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAA;IACvD,MAAM,MAAM,GAAG,wBAAwB,CAAC,IAAI,CAAC,CAAA;IAE7C,MAAM,EAAE,CAAC,KAAK,MAAM,CAAC,MAAM,oBAAoB,CAAC,CAAA;IAEhD,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAEhD,MAAM,EAAE,CAAC,KAAK,MAAM,CAAC,QAAQ,CAAC,MAAM,wBAAwB,CAAC,CAAA;IAE7D,kFAAkF;IAClF,kDAAkD;IAClD,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,YAAY,CAAC,CAAA;IACxD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAiC,CAAA;IAEtD,4EAA4E;IAC5E,gDAAgD;IAChD,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,QAAQ,EAAE,CAAC;QAC1C,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,KAA8B,CAAC,CAAA;IACpD,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,MAAM,WAAW,GAAa,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAA;IAEjD,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,MAAM,EAAE,CAAC,iBAAiB,OAAO,wCAAwC,CAAC,CAAA;IAC3E,CAAC;IAED,MAAM,oBAAoB,GAAG,KAAK,IAAmB,EAAE;QACtD,MAAM,QAAQ,GAA6B;YAC1C,MAAM,EAAE,IAAI;YACZ,UAAU,EAAE,2BAA2B;YACvC,OAAO,EAAE,kBAAkB;YAC3B,WAAW,EAAE,sBAAsB;YACnC,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,iBAAiB,EAAE,MAAM,CAAC,MAAM;YAChC,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;SACtG,CAAA;QAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAC5C,CAAC,CAAA;IAED,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACzC,IAAI,MAAM,EAAE,OAAO;YAAE,MAAK;QAE1B,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAA;QAExC,IAAI,MAAM,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,aAAa,IAAI,KAAK,CAAC,EAAE,CAAC;YAC7G,OAAO,EAAE,CAAA;YAET,SAAQ;QACT,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,SAAS,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE;gBAClD,GAAG,EAAE,SAAS,CAAC,UAAU;gBACzB,IAAI;gBACJ,OAAO,EAAE,OAAO;gBAChB,MAAM;aACN,CAAC,CAAA;YAEF,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,EAAE;gBAC7B,SAAS,EAAE,SAAS,CAAC,SAAS;gBAC9B,QAAQ,EAAE,SAAS,CAAC,QAAQ;gBAC5B,UAAU,EAAE,SAAS,CAAC,UAAU;gBAChC,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACvC,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,YAAY,EAAE,SAAS,CAAC,OAAO;gBAC/B,aAAa,EAAE,SAAS,CAAC,YAAY;aACrC,CAAC,CAAA;YAEF,OAAO,EAAE,CAAA;YAET,MAAM,EAAE,CAAC,OAAO,SAAS,CAAC,QAAQ,IAAI,aAAa,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,WAAW,SAAS,CAAC,MAAM,EAAE,CAAC,CAAA;QAC7G,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,OAAO,SAAS,CAAC,SAAS,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YACjG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;YAErC,SAAQ;QACT,CAAC;QAED,IAAI,OAAO,GAAG,iBAAiB,KAAK,CAAC,EAAE,CAAC;YACvC,MAAM,oBAAoB,EAAE,CAAA;QAC7B,CAAC;IACF,CAAC;IAED,MAAM,oBAAoB,EAAE,CAAA;IAE5B,MAAM,EAAE,CAAC,OAAO,IAAI,UAAU,CAAC,CAAA;IAC/B,MAAM,EAAE,CAAC,YAAY,OAAO,EAAE,CAAC,CAAA;IAC/B,MAAM,EAAE,CAAC,YAAY,OAAO,0DAA0D,CAAC,CAAA;IACvF,MAAM,EAAE,CAAC,YAAY,WAAW,CAAC,MAAM,EAAE,CAAC,CAAA;IAE1C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,CAAA;AACrE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAoC;IACtE,OAAO,OAAO,CAAC,IAAI,CAAC,CAAA;AACrB,CAAC;AAYD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,OAA8B,EAC9B,MAA+B;IAE/B,YAAY,MAAM,GAAG,IAAI,SAAS,CAAC;QAClC,WAAW,EAAE,IAAI;QACjB,oBAAoB,EAAE,OAAO,CAAC,oBAAoB,IAAI,8BAA8B;QACpF,KAAK,EAAE,IAAI;KACX,CAAC,CAAA;IAEF,yFAAyF;IACzF,0DAA0D;IAC1D,OAAO,MAAM,gBAAgB,CAAC,MAAM,EAAE;QACrC,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;QAChC,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM;KACN,CAAC,CAAA;AACH,CAAC"}
|
|
@@ -17,43 +17,40 @@
|
|
|
17
17
|
* every name column bare.
|
|
18
18
|
*
|
|
19
19
|
* A row's country comes from `municipality_number` rather than from the caller. Åland is part of
|
|
20
|
-
* Finland and its sixteen municipalities
|
|
21
|
-
* holds both jurisdictions and {@linkcode ALAND_MUNICIPALITIES}
|
|
20
|
+
* Finland and its sixteen municipalities have Finnish municipality numbers, so one national file
|
|
21
|
+
* holds both jurisdictions and {@linkcode ALAND_MUNICIPALITIES} separates them.
|
|
22
22
|
*
|
|
23
23
|
* The street and the house number are read from the publisher's own part columns rather than from
|
|
24
24
|
* its inline address text.
|
|
25
25
|
* `address_name_fin` is the whole of `address_fin` or a space-delimited prefix of it, and the four
|
|
26
26
|
* number-part columns reproduce the publisher's inline number.
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
* composed number equals the inline remainder byte for byte on all 265,830 numbered rows.
|
|
30
|
-
* No row publishes inline address text with both name columns empty,
|
|
31
|
-
* so reading the name columns loses no address.
|
|
27
|
+
* No row publishes inline address text with both name columns empty, so the part columns lose no
|
|
28
|
+
* address.
|
|
32
29
|
*
|
|
33
|
-
* `address_number` is not a house number. It is a 1-based ordinal over the addresses
|
|
34
|
-
* `building_key
|
|
35
|
-
*
|
|
30
|
+
* `address_number` is not a house number. It is a 1-based ordinal over the addresses that share one
|
|
31
|
+
* `building_key`. That is how a corner building's several frontages are numbered. The adapter
|
|
32
|
+
* ignores it.
|
|
36
33
|
*
|
|
37
|
-
* The file is bilingual, and which column
|
|
34
|
+
* The file is bilingual, and which column holds which language depends on the municipality. On
|
|
38
35
|
* mainland Finland `address_name_fin` is the Finnish name and `address_name_swe` the Swedish one. On
|
|
39
36
|
* Åland the `*_fin` columns, where populated at all, hold the **Swedish** name: `address_fin` reads
|
|
40
37
|
* `Skogshyddsvägen 11`.
|
|
41
38
|
* So the adapter prefers `*_swe` on Åland and `*_fin` on the mainland, and emits one row per
|
|
42
39
|
* published record from the preferred language's columns.
|
|
43
|
-
* `locale`
|
|
44
|
-
* name came from
|
|
45
|
-
* `fi-FI` is a mainland record
|
|
46
|
-
* `sv-FI` is a mainland record that populates `address_name_swe`
|
|
40
|
+
* `locale` states the language of the name it emitted rather than the language of the column that
|
|
41
|
+
* name came from. That gives three values.
|
|
42
|
+
* `fi-FI` is a mainland record whose name came from `address_name_fin`.
|
|
43
|
+
* `sv-FI` is a mainland record that populates `address_name_swe` only. That is the form a
|
|
47
44
|
* Swedish-speaking municipality publishes.
|
|
48
45
|
* `sv-AX` is every Åland record, because the Swedish columns are the populated ones there and the
|
|
49
46
|
* `*_fin` columns hold the Swedish name where they are populated at all.
|
|
50
47
|
* A mainland record's Swedish name, where the record publishes both, is not emitted as a second
|
|
51
|
-
* row: a record is one premise, and `CanonicalRow`
|
|
48
|
+
* row: a record is one premise, and `CanonicalRow` holds one written surface.
|
|
52
49
|
*
|
|
53
50
|
* SYKE's metadata record `{DBD610F4-3392-44CD-B601-BAE8FA547A57}` grants CC BY 4.0 for its open data
|
|
54
51
|
* and the register's two elected decisions for this file record `CC-BY-4.0` in their `spdx` field, so
|
|
55
52
|
* the adapter records that license on every row. SYKE states the attribution it wants as
|
|
56
|
-
* `Lähde: Syke Ryhti`, and the model card must
|
|
53
|
+
* `Lähde: Syke Ryhti`, and the model card must state it.
|
|
57
54
|
*
|
|
58
55
|
* The adapter streams with `CSVSpliterator.fromAsync`, so the 3.8M-row file never sits in memory. It
|
|
59
56
|
* honors `opts.limit`, `opts.signal` and `opts.country`.
|
|
@@ -69,7 +66,6 @@ export declare const RYHTI_ADAPTER_ID = "ryhti";
|
|
|
69
66
|
* The sixteen `municipality_number` values of Åland's municipalities.
|
|
70
67
|
*
|
|
71
68
|
* Every other number in the file belongs to a mainland Finnish municipality.
|
|
72
|
-
* The file holds 308 distinct values, and these sixteen cover 32,518 of its 3,862,609 rows.
|
|
73
69
|
*
|
|
74
70
|
* The column is a zero-padded three-digit string, so a reader compares the padded form.
|
|
75
71
|
*/
|
|
@@ -102,8 +98,8 @@ export declare function countryOfFinnishMunicipality(municipalityNumber: string)
|
|
|
102
98
|
* as `184-183b` across a second pair of the same two columns.
|
|
103
99
|
* The second pair is appended after a hyphen when either of its halves is populated.
|
|
104
100
|
*
|
|
105
|
-
* @returns The composed number, or an empty string for a row with no number at all
|
|
106
|
-
*
|
|
101
|
+
* @returns The composed number, or an empty string for a row with no number at all.
|
|
102
|
+
* A caller treats that empty string as a place name rather than a number.
|
|
107
103
|
*/
|
|
108
104
|
export declare function composeRyhtiHouseNumber(record: Partial<RyhtiNumberParts>): string;
|
|
109
105
|
export declare function createRyhtiAdapter(): CorpusAdapter;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../lib/fi/adapters/ryhti/adapter.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../lib/fi/adapters/ryhti/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwDG;AASH,OAAO,EAAuD,KAAK,aAAa,EAAiB,MAAM,QAAQ,CAAA;AAE/G;;;;GAIG;AACH,eAAO,MAAM,gBAAgB,UAAU,CAAA;AAEvC;;;;;;GAMG;AACH,eAAO,MAAM,oBAAoB,EAAE,WAAW,CAAC,MAAM,CAiBnD,CAAA;AAEF;;GAEG;AACH,eAAO,MAAM,eAAe,EAAE,SAAS,MAAM,EAAiB,CAAA;AAmB9D;;;GAGG;AACH,UAAU,gBAAgB;IACzB,6BAA6B,EAAE,MAAM,CAAA;IACrC,8BAA8B,EAAE,MAAM,CAAA;IACtC,oCAAoC,EAAE,MAAM,CAAA;IAC5C,qCAAqC,EAAE,MAAM,CAAA;CAC7C;AAED;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAAC,kBAAkB,EAAE,MAAM,GAAG,MAAM,CAI/E;AAED;;;;;;;;;GASG;AACH,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,MAAM,CAcjF;AAkDD,wBAAgB,kBAAkB,IAAI,aAAa,CA8FlD;AAED;;GAEG;AACH,eAAO,MAAM,YAAY,eAAuB,CAAA"}
|
|
@@ -17,43 +17,40 @@
|
|
|
17
17
|
* every name column bare.
|
|
18
18
|
*
|
|
19
19
|
* A row's country comes from `municipality_number` rather than from the caller. Åland is part of
|
|
20
|
-
* Finland and its sixteen municipalities
|
|
21
|
-
* holds both jurisdictions and {@linkcode ALAND_MUNICIPALITIES}
|
|
20
|
+
* Finland and its sixteen municipalities have Finnish municipality numbers, so one national file
|
|
21
|
+
* holds both jurisdictions and {@linkcode ALAND_MUNICIPALITIES} separates them.
|
|
22
22
|
*
|
|
23
23
|
* The street and the house number are read from the publisher's own part columns rather than from
|
|
24
24
|
* its inline address text.
|
|
25
25
|
* `address_name_fin` is the whole of `address_fin` or a space-delimited prefix of it, and the four
|
|
26
26
|
* number-part columns reproduce the publisher's inline number.
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
* composed number equals the inline remainder byte for byte on all 265,830 numbered rows.
|
|
30
|
-
* No row publishes inline address text with both name columns empty,
|
|
31
|
-
* so reading the name columns loses no address.
|
|
27
|
+
* No row publishes inline address text with both name columns empty, so the part columns lose no
|
|
28
|
+
* address.
|
|
32
29
|
*
|
|
33
|
-
* `address_number` is not a house number. It is a 1-based ordinal over the addresses
|
|
34
|
-
* `building_key
|
|
35
|
-
*
|
|
30
|
+
* `address_number` is not a house number. It is a 1-based ordinal over the addresses that share one
|
|
31
|
+
* `building_key`. That is how a corner building's several frontages are numbered. The adapter
|
|
32
|
+
* ignores it.
|
|
36
33
|
*
|
|
37
|
-
* The file is bilingual, and which column
|
|
34
|
+
* The file is bilingual, and which column holds which language depends on the municipality. On
|
|
38
35
|
* mainland Finland `address_name_fin` is the Finnish name and `address_name_swe` the Swedish one. On
|
|
39
36
|
* Åland the `*_fin` columns, where populated at all, hold the **Swedish** name: `address_fin` reads
|
|
40
37
|
* `Skogshyddsvägen 11`.
|
|
41
38
|
* So the adapter prefers `*_swe` on Åland and `*_fin` on the mainland, and emits one row per
|
|
42
39
|
* published record from the preferred language's columns.
|
|
43
|
-
* `locale`
|
|
44
|
-
* name came from
|
|
45
|
-
* `fi-FI` is a mainland record
|
|
46
|
-
* `sv-FI` is a mainland record that populates `address_name_swe`
|
|
40
|
+
* `locale` states the language of the name it emitted rather than the language of the column that
|
|
41
|
+
* name came from. That gives three values.
|
|
42
|
+
* `fi-FI` is a mainland record whose name came from `address_name_fin`.
|
|
43
|
+
* `sv-FI` is a mainland record that populates `address_name_swe` only. That is the form a
|
|
47
44
|
* Swedish-speaking municipality publishes.
|
|
48
45
|
* `sv-AX` is every Åland record, because the Swedish columns are the populated ones there and the
|
|
49
46
|
* `*_fin` columns hold the Swedish name where they are populated at all.
|
|
50
47
|
* A mainland record's Swedish name, where the record publishes both, is not emitted as a second
|
|
51
|
-
* row: a record is one premise, and `CanonicalRow`
|
|
48
|
+
* row: a record is one premise, and `CanonicalRow` holds one written surface.
|
|
52
49
|
*
|
|
53
50
|
* SYKE's metadata record `{DBD610F4-3392-44CD-B601-BAE8FA547A57}` grants CC BY 4.0 for its open data
|
|
54
51
|
* and the register's two elected decisions for this file record `CC-BY-4.0` in their `spdx` field, so
|
|
55
52
|
* the adapter records that license on every row. SYKE states the attribution it wants as
|
|
56
|
-
* `Lähde: Syke Ryhti`, and the model card must
|
|
53
|
+
* `Lähde: Syke Ryhti`, and the model card must state it.
|
|
57
54
|
*
|
|
58
55
|
* The adapter streams with `CSVSpliterator.fromAsync`, so the 3.8M-row file never sits in memory. It
|
|
59
56
|
* honors `opts.limit`, `opts.signal` and `opts.country`.
|
|
@@ -75,7 +72,6 @@ export const RYHTI_ADAPTER_ID = "ryhti";
|
|
|
75
72
|
* The sixteen `municipality_number` values of Åland's municipalities.
|
|
76
73
|
*
|
|
77
74
|
* Every other number in the file belongs to a mainland Finnish municipality.
|
|
78
|
-
* The file holds 308 distinct values, and these sixteen cover 32,518 of its 3,862,609 rows.
|
|
79
75
|
*
|
|
80
76
|
* The column is a zero-padded three-digit string, so a reader compares the padded form.
|
|
81
77
|
*/
|
|
@@ -118,8 +114,8 @@ export function countryOfFinnishMunicipality(municipalityNumber) {
|
|
|
118
114
|
* as `184-183b` across a second pair of the same two columns.
|
|
119
115
|
* The second pair is appended after a hyphen when either of its halves is populated.
|
|
120
116
|
*
|
|
121
|
-
* @returns The composed number, or an empty string for a row with no number at all
|
|
122
|
-
*
|
|
117
|
+
* @returns The composed number, or an empty string for a row with no number at all.
|
|
118
|
+
* A caller treats that empty string as a place name rather than a number.
|
|
123
119
|
*/
|
|
124
120
|
export function composeRyhtiHouseNumber(record) {
|
|
125
121
|
const first = composeHouseNumber(record.number_part_of_address_number ?? "", record.subdivision_letter_of_address_number ?? "");
|
|
@@ -132,7 +128,7 @@ export function composeRyhtiHouseNumber(record) {
|
|
|
132
128
|
return `${first}-${secondNumber}${secondLetter}`;
|
|
133
129
|
}
|
|
134
130
|
/**
|
|
135
|
-
* The surface to emit for one record, or `
|
|
131
|
+
* The surface to emit for one record, or `null` for a record with no street.
|
|
136
132
|
*
|
|
137
133
|
* On Åland the Swedish columns are preferred, and the Finnish ones are read as Swedish.
|
|
138
134
|
* That is the language they hold there.
|
|
@@ -141,7 +137,7 @@ export function composeRyhtiHouseNumber(record) {
|
|
|
141
137
|
* The Swedish ones stand in where `address_name_fin` is empty.
|
|
142
138
|
* That is the form a Swedish-speaking municipality publishes.
|
|
143
139
|
*
|
|
144
|
-
* A language whose name column is empty
|
|
140
|
+
* A language whose name column is empty has no street.
|
|
145
141
|
* Seven mainland rows publish an `address_swe` of a bare number with an empty `address_name_swe`.
|
|
146
142
|
* This refuses them rather than giving them an empty street.
|
|
147
143
|
*/
|
|
@@ -156,11 +152,11 @@ function surfaceOf(record, country) {
|
|
|
156
152
|
};
|
|
157
153
|
if (country === "AX") {
|
|
158
154
|
const preferred = swedish.street ? swedish : finnish;
|
|
159
|
-
return preferred.street ? { ...preferred, language: "sv" } :
|
|
155
|
+
return preferred.street ? { ...preferred, language: "sv" } : null;
|
|
160
156
|
}
|
|
161
157
|
if (finnish.street)
|
|
162
158
|
return { ...finnish, language: "fi" };
|
|
163
|
-
return swedish.street ? { ...swedish, language: "sv" } :
|
|
159
|
+
return swedish.street ? { ...swedish, language: "sv" } : null;
|
|
164
160
|
}
|
|
165
161
|
export function createRyhtiAdapter() {
|
|
166
162
|
return {
|
|
@@ -186,7 +182,7 @@ export function createRyhtiAdapter() {
|
|
|
186
182
|
if (opts.limit !== undefined && emitted >= opts.limit)
|
|
187
183
|
break;
|
|
188
184
|
const country = countryOfFinnishMunicipality(record.municipality_number ?? "");
|
|
189
|
-
// A caller that
|
|
185
|
+
// A caller that requested one jurisdiction gets that jurisdiction's rows,
|
|
190
186
|
// so the one national file can be read once per country.
|
|
191
187
|
if (opts.country && country !== opts.country)
|
|
192
188
|
continue;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../lib/fi/adapters/ryhti/adapter.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../lib/fi/adapters/ryhti/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwDG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAA;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,OAAO,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAA;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAA;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,EAAE,WAAW,EAA8D,aAAa,EAAE,MAAM,QAAQ,CAAA;AAE/G;;;;GAIG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,OAAO,CAAA;AAEvC;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAwB,IAAI,GAAG,CAAC;IAChE,KAAK,EAAE,SAAS;IAChB,KAAK,EAAE,SAAS;IAChB,KAAK,EAAE,WAAW;IAClB,KAAK,EAAE,QAAQ;IACf,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,aAAa;IACpB,KAAK,EAAE,SAAS;IAChB,KAAK,EAAE,WAAW;IAClB,KAAK,EAAE,QAAQ;IACf,KAAK,EAAE,UAAU;IACjB,KAAK,EAAE,aAAa;IACpB,KAAK,EAAE,YAAY;IACnB,KAAK,EAAE,UAAU;IACjB,KAAK,EAAE,WAAW;IAClB,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,QAAQ;CACf,CAAC,CAAA;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAsB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;AA8B9D;;;;;GAKG;AACH,MAAM,UAAU,4BAA4B,CAAC,kBAA0B;IACtE,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;IAEzD,OAAO,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAA;AACtD,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,uBAAuB,CAAC,MAAiC;IACxE,MAAM,KAAK,GAAG,kBAAkB,CAC/B,MAAM,CAAC,6BAA6B,IAAI,EAAE,EAC1C,MAAM,CAAC,oCAAoC,IAAI,EAAE,CACjD,CAAA;IAED,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,CAAA;IAErB,MAAM,YAAY,GAAG,CAAC,MAAM,CAAC,8BAA8B,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IACzE,MAAM,YAAY,GAAG,CAAC,MAAM,CAAC,qCAAqC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAEhF,IAAI,CAAC,YAAY,IAAI,CAAC,YAAY;QAAE,OAAO,KAAK,CAAA;IAEhD,OAAO,GAAG,KAAK,IAAI,YAAY,GAAG,YAAY,EAAE,CAAA;AACjD,CAAC;AAcD;;;;;;;;;;;;;GAaG;AACH,SAAS,SAAS,CAAC,MAAgB,EAAE,OAAe;IACnD,MAAM,OAAO,GAAG;QACf,MAAM,EAAE,CAAC,MAAM,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;QAC9C,QAAQ,EAAE,CAAC,MAAM,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;KACjD,CAAA;IAED,MAAM,OAAO,GAAG;QACf,MAAM,EAAE,CAAC,MAAM,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;QAC9C,QAAQ,EAAE,CAAC,MAAM,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;KACjD,CAAA;IAED,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACtB,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAA;QAEpD,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;IAClE,CAAC;IAED,IAAI,OAAO,CAAC,MAAM;QAAE,OAAO,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;IAEzD,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;AAC9D,CAAC;AAED,MAAM,UAAU,kBAAkB;IACjC,OAAO;QACN,EAAE,EAAE,gBAAgB;QACpB,cAAc,EAAE,WAAW;QAC3B,WAAW,EAAE,WAAW,CAAC,OAAO;QAChC,QAAQ,EAAE,cAAc,CAAC,oBAAoB;QAC7C,OAAO,EAAE,aAAa,CAAC,QAAQ;QAC/B,WAAW,EACV,uHAAuH;QAExH,KAAK,CAAC,CAAC,IAAI,CAAC,IAAoB;YAC/B,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC7D,MAAM,IAAI,uBAAuB,CAAC,gBAAgB,EAAE,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;YACnF,CAAC;YAED,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE;gBACrD,aAAa,EAAE,KAAK;aACpB,CAAC,CAAA;YAEF,IAAI,OAAO,GAAG,CAAC,CAAA;YACf,IAAI,OAAO,GAAG,CAAC,CAAA;YAEf,IAAI,CAAC;gBACJ,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,IAA+B,EAAE,CAAC;oBAC5D,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;wBAAE,MAAK;oBAE/B,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,CAAC,KAAK;wBAAE,MAAK;oBAE5D,MAAM,OAAO,GAAG,4BAA4B,CAAC,MAAM,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAA;oBAE9E,0EAA0E;oBAC1E,yDAAyD;oBACzD,IAAI,IAAI,CAAC,OAAO,IAAI,OAAO,KAAK,IAAI,CAAC,OAAO;wBAAE,SAAQ;oBAEtD,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;oBAE1C,IAAI,CAAC,OAAO,EAAE,CAAC;wBACd,OAAO,IAAI,CAAC,CAAA;wBAEZ,SAAQ;oBACT,CAAC;oBAED,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;oBAElD,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,QAAQ;wBAAE,SAAQ;oBAE5C,MAAM,KAAK,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAA;oBAC7C,MAAM,UAAU,GAA+B,EAAE,CAAA;oBAEjD,IAAI,KAAK,EAAE,CAAC;wBACX,UAAU,CAAC,YAAY,GAAG,KAAK,CAAA;oBAChC,CAAC;oBAED,UAAU,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAA;oBAElC,IAAI,QAAQ,EAAE,CAAC;wBACd,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAA;oBAC/B,CAAC;oBAED,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;wBACtB,UAAU,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAA;oBACvC,CAAC;oBAED,MAAM,QAAQ,GAAG,gBAAgB,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;oBAE5E,IAAI,CAAC,QAAQ;wBAAE,SAAQ;oBAEvB,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAA;oBAC7C,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;oBAE7C,MAAM,QAAQ,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,gBAAgB,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAA;oBAE/F,MAAM;wBACL,GAAG;wBACH,UAAU,EAAE,OAAO;wBACnB,OAAO;wBACP,MAAM,EAAE,GAAG,OAAO,CAAC,QAAQ,IAAI,OAAO,EAAE;wBACxC,MAAM,EAAE,gBAAgB;wBACxB,SAAS,EAAE,QAAQ;wBACnB,cAAc,EAAE,EAAE;wBAClB,OAAO,EAAE,WAAW;qBACpB,CAAA;oBAED,OAAO,EAAE,CAAA;gBACV,CAAC;YACF,CAAC;oBAAS,CAAC;gBACV,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;oBACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,YAAY,OAAO,6DAA6D,OAAO,SAAS,CAChG,CAAA;gBACF,CAAC;YACF,CAAC;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,kBAAkB,EAAE,CAAA"}
|
|
@@ -14,31 +14,27 @@
|
|
|
14
14
|
* this module sits beside `#fr/tools/fetch/ban` rather than in an `sdk/` root the workspace does
|
|
15
15
|
* not declare.
|
|
16
16
|
*
|
|
17
|
-
* SYKE's metadata record `{DBD610F4-3392-44CD-B601-BAE8FA547A57}` grants CC BY 4.0
|
|
18
|
-
*
|
|
17
|
+
* SYKE's metadata record `{DBD610F4-3392-44CD-B601-BAE8FA547A57}` grants CC BY 4.0 and states the
|
|
18
|
+
* attribution as `Lähde: Syke Ryhti`. The address-source register elects that license. The manifest
|
|
19
19
|
* records both, and `#fi/adapters/ryhti/adapter` records the license on every row.
|
|
20
20
|
*
|
|
21
21
|
* Four measured properties decide what this module does:
|
|
22
22
|
*
|
|
23
|
-
* 1. **The file is republished daily and its length changes with it.** `HEAD`
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
* a `HEAD` compared against the manifest, which costs one small request instead of 351 MB, and
|
|
27
|
-
* neither number is written into this module as a constant.
|
|
23
|
+
* 1. **The file is republished daily and its length changes with it.** The re-run check is a `HEAD`
|
|
24
|
+
* compared against the manifest, so one small request replaces the whole body and no length is a
|
|
25
|
+
* constant here.
|
|
28
26
|
* 2. **The host ignores `Range`.** `bytes=0-4095` answers HTTP 200 with no `content-range` and the
|
|
29
27
|
* whole `content-length`, and `accept-ranges` is absent, so a resumed transfer is not available
|
|
30
28
|
* and an interrupted one restarts. {@linkcode resumableDownload} would refuse the response.
|
|
31
|
-
* 3. **This module moves bytes and decodes none of them.** The body is streamed to disk compressed
|
|
32
|
-
* decompressed stream to stream, so no chunk boundary is
|
|
33
|
-
* boundary. A `chunk.toString("utf8")` per gzip chunk
|
|
34
|
-
*
|
|
35
|
-
* `EF BF BD` with 0 found: the replacement characters such a reader reports are its own.
|
|
29
|
+
* 3. **This module moves bytes and decodes none of them.** The body is streamed to disk compressed
|
|
30
|
+
* and then decompressed stream to stream, so no chunk boundary is interpreted as a character
|
|
31
|
+
* boundary. A `chunk.toString("utf8")` per gzip chunk would write U+FFFD at every split
|
|
32
|
+
* multi-byte character.
|
|
36
33
|
* 4. **The decompressed CSV is what the adapter can read.** `#fi/adapters/ryhti/adapter` hands
|
|
37
|
-
* `opts.inputPath` to `CSVSpliterator.fromAsync`, which reads the bytes
|
|
38
|
-
*
|
|
39
|
-
* written: 793 MB of readable input instead of 351 MB plus a reader the adapter does not have.
|
|
34
|
+
* `opts.inputPath` to `CSVSpliterator.fromAsync`, which reads the bytes on disk. A path to the
|
|
35
|
+
* `.gz` would be parsed as CSV over gzip bytes, so the `.gz` is removed once the CSV is written.
|
|
40
36
|
* A caller that wants the compressed copy kept passes
|
|
41
|
-
* {@linkcode FetchRyhtiOptions.keepCompressed}
|
|
37
|
+
* {@linkcode FetchRyhtiOptions.keepCompressed}.
|
|
42
38
|
*/
|
|
43
39
|
import { APIClient } from "@mailwoman/core/api";
|
|
44
40
|
import { type PathBuilderLike } from "path-ts";
|
|
@@ -56,7 +52,8 @@ export declare const FI_RYHTI_CSV_FILENAME = "open_address.csv";
|
|
|
56
52
|
*/
|
|
57
53
|
export declare const FI_RYHTI_LICENSE = "CC-BY-4.0";
|
|
58
54
|
/**
|
|
59
|
-
* The attribution SYKE states for its open data
|
|
55
|
+
* The attribution SYKE states for its open data.
|
|
56
|
+
* The model card must record it.
|
|
60
57
|
*/
|
|
61
58
|
export declare const FI_RYHTI_ATTRIBUTION = "L\u00E4hde: Syke Ryhti";
|
|
62
59
|
/**
|
|
@@ -73,8 +70,9 @@ export declare const FI_RYHTI_REQUIRED_COLUMNS: readonly string[];
|
|
|
73
70
|
/**
|
|
74
71
|
* What the manifest records about the one file.
|
|
75
72
|
*
|
|
76
|
-
* {@linkcode SourceManifest}'s five fields describe the decompressed CSV
|
|
77
|
-
*
|
|
73
|
+
* {@linkcode SourceManifest}'s five fields describe the decompressed CSV.
|
|
74
|
+
* That file is what the adapter reads and what the digest covers.
|
|
75
|
+
*
|
|
78
76
|
* The three added fields describe the transfer, and they are what the re-run
|
|
79
77
|
* check compares before it downloads 351 MB.
|
|
80
78
|
*/
|
|
@@ -90,7 +88,7 @@ export interface RyhtiManifest extends SourceManifest {
|
|
|
90
88
|
*/
|
|
91
89
|
compressed_bytes: number | null;
|
|
92
90
|
/**
|
|
93
|
-
* The columns the header
|
|
91
|
+
* The columns the header held when this file was downloaded, in order.
|
|
94
92
|
*
|
|
95
93
|
* Recorded rather than counted, so a later edition's added, removed or renamed
|
|
96
94
|
* column is a diff against this list rather than a count that moved.
|
|
@@ -141,13 +139,14 @@ export interface DownloadRyhtiOptions {
|
|
|
141
139
|
* Keep the compressed copy beside the CSV.
|
|
142
140
|
*
|
|
143
141
|
* The adapter reads the CSV, so the default removes the archive once the CSV is written.
|
|
144
|
-
*
|
|
142
|
+
* Both together hold 1.14 GB for the one file.
|
|
145
143
|
*/
|
|
146
144
|
keepCompressed?: boolean;
|
|
147
145
|
/**
|
|
148
146
|
* Re-read the CSV's sha256 on a re-run instead of comparing its byte count.
|
|
149
147
|
*
|
|
150
|
-
* The default compares the recorded byte count against the file's size
|
|
148
|
+
* The default compares the recorded byte count against the file's size.
|
|
149
|
+
* That check is one `stat`.
|
|
151
150
|
* This re-hashes 793 MB.
|
|
152
151
|
*/
|
|
153
152
|
verifyDigest?: boolean;
|
|
@@ -157,14 +156,15 @@ export interface DownloadRyhtiOptions {
|
|
|
157
156
|
force?: boolean;
|
|
158
157
|
/**
|
|
159
158
|
* Attempts after the first for the one transfer.
|
|
160
|
-
* Defaults to three.
|
|
161
159
|
*
|
|
162
160
|
* The host ignores `Range`, so each attempt starts the 351 MB again.
|
|
161
|
+
*
|
|
162
|
+
* @defaultValue three.
|
|
163
163
|
*/
|
|
164
164
|
retries?: number;
|
|
165
165
|
/**
|
|
166
166
|
* Pause between attempts, in milliseconds.
|
|
167
|
-
*
|
|
167
|
+
* @defaultValue {@linkcode DEFAULT_RETRY_DELAY_MS}
|
|
168
168
|
*/
|
|
169
169
|
retryDelayMs?: number;
|
|
170
170
|
signal?: AbortSignal;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ryhti.d.ts","sourceRoot":"","sources":["../../../../lib/fi/tools/fetch/ryhti.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"ryhti.d.ts","sourceRoot":"","sources":["../../../../lib/fi/tools/fetch/ryhti.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAKH,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAO/C,OAAO,EAAe,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAG3D,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AAI3F;;GAEG;AACH,eAAO,MAAM,gBAAgB,wEAAwE,CAAA;AAOrG;;GAEG;AACH,eAAO,MAAM,qBAAqB,qBAAqB,CAAA;AAOvD;;GAEG;AACH,eAAO,MAAM,gBAAgB,cAAc,CAAA;AAE3C;;;GAGG;AACH,eAAO,MAAM,oBAAoB,2BAAsB,CAAA;AAEvD;;;;;;;;;GASG;AACH,eAAO,MAAM,yBAAyB,EAAE,SAAS,MAAM,EAYtD,CAAA;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,aAAc,SAAQ,cAAc;IACpD,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B;;OAEG;IACH,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAA;IAC/B;;;;;OAKG;IACH,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;CAC1B;AAED;;;;;;;;GAQG;AACH,wBAAsB,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,GAAG,SAAmB,GAAG,OAAO,CAAC,SAAS,CAAC,CAUhH;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,SAAS,MAAM,EAAE,CAAC,CAElF;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAEpF;AAED,MAAM,WAAW,oBAAoB;IACpC;;;OAGG;IACH,SAAS,EAAE,eAAe,CAAA;IAC1B;;;;;OAKG;IACH,cAAc,CAAC,EAAE,OAAO,CAAA;IACxB;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,OAAO,CAAA;IACtB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;IACf;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B;AAED,MAAM,MAAM,iBAAiB,GAAG,gBAAgB,GAAG,IAAI,CAAC,oBAAoB,EAAE,WAAW,GAAG,QAAQ,CAAC,CAAA;AAErG;;GAEG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,gBAAgB,CAAC,SAAS,CAAC,GAAG,eAAe,CAEpF;AAkCD;;;;;;;;;;;GAWG;AACH,wBAAsB,aAAa,CAClC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAChC,OAAO,EAAE,oBAAoB,GAC3B,OAAO,CAAC,YAAY,CAAC,CAqJvB;AAED;;;;;;GAMG;AACH,wBAAsB,UAAU,CAAC,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAenH;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CAAC,MAAM,EAAE,eAAe,EAAE,WAAW,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAI1G"}
|
|
@@ -14,38 +14,34 @@
|
|
|
14
14
|
* this module sits beside `#fr/tools/fetch/ban` rather than in an `sdk/` root the workspace does
|
|
15
15
|
* not declare.
|
|
16
16
|
*
|
|
17
|
-
* SYKE's metadata record `{DBD610F4-3392-44CD-B601-BAE8FA547A57}` grants CC BY 4.0
|
|
18
|
-
*
|
|
17
|
+
* SYKE's metadata record `{DBD610F4-3392-44CD-B601-BAE8FA547A57}` grants CC BY 4.0 and states the
|
|
18
|
+
* attribution as `Lähde: Syke Ryhti`. The address-source register elects that license. The manifest
|
|
19
19
|
* records both, and `#fi/adapters/ryhti/adapter` records the license on every row.
|
|
20
20
|
*
|
|
21
21
|
* Four measured properties decide what this module does:
|
|
22
22
|
*
|
|
23
|
-
* 1. **The file is republished daily and its length changes with it.** `HEAD`
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
* a `HEAD` compared against the manifest, which costs one small request instead of 351 MB, and
|
|
27
|
-
* neither number is written into this module as a constant.
|
|
23
|
+
* 1. **The file is republished daily and its length changes with it.** The re-run check is a `HEAD`
|
|
24
|
+
* compared against the manifest, so one small request replaces the whole body and no length is a
|
|
25
|
+
* constant here.
|
|
28
26
|
* 2. **The host ignores `Range`.** `bytes=0-4095` answers HTTP 200 with no `content-range` and the
|
|
29
27
|
* whole `content-length`, and `accept-ranges` is absent, so a resumed transfer is not available
|
|
30
28
|
* and an interrupted one restarts. {@linkcode resumableDownload} would refuse the response.
|
|
31
|
-
* 3. **This module moves bytes and decodes none of them.** The body is streamed to disk compressed
|
|
32
|
-
* decompressed stream to stream, so no chunk boundary is
|
|
33
|
-
* boundary. A `chunk.toString("utf8")` per gzip chunk
|
|
34
|
-
*
|
|
35
|
-
* `EF BF BD` with 0 found: the replacement characters such a reader reports are its own.
|
|
29
|
+
* 3. **This module moves bytes and decodes none of them.** The body is streamed to disk compressed
|
|
30
|
+
* and then decompressed stream to stream, so no chunk boundary is interpreted as a character
|
|
31
|
+
* boundary. A `chunk.toString("utf8")` per gzip chunk would write U+FFFD at every split
|
|
32
|
+
* multi-byte character.
|
|
36
33
|
* 4. **The decompressed CSV is what the adapter can read.** `#fi/adapters/ryhti/adapter` hands
|
|
37
|
-
* `opts.inputPath` to `CSVSpliterator.fromAsync`, which reads the bytes
|
|
38
|
-
*
|
|
39
|
-
* written: 793 MB of readable input instead of 351 MB plus a reader the adapter does not have.
|
|
34
|
+
* `opts.inputPath` to `CSVSpliterator.fromAsync`, which reads the bytes on disk. A path to the
|
|
35
|
+
* `.gz` would be parsed as CSV over gzip bytes, so the `.gz` is removed once the CSV is written.
|
|
40
36
|
* A caller that wants the compressed copy kept passes
|
|
41
|
-
* {@linkcode FetchRyhtiOptions.keepCompressed}
|
|
37
|
+
* {@linkcode FetchRyhtiOptions.keepCompressed}.
|
|
42
38
|
*/
|
|
43
39
|
/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
|
|
44
40
|
procedure rather than sections of declarations. A region there folds no element a reader wants folded. */
|
|
45
41
|
import { APIClient } from "@mailwoman/core/api";
|
|
46
42
|
import { gunzipChunks } from "@mailwoman/core/fs/compression";
|
|
47
43
|
import { ByteFormatter } from "@mailwoman/core/fs/formatters";
|
|
48
|
-
import { tryStat } from "@mailwoman/core/fs/readers";
|
|
44
|
+
import { tryStat } from "@mailwoman/core/fs/readers/stat";
|
|
49
45
|
import { openReadStream, openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams";
|
|
50
46
|
import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers";
|
|
51
47
|
import { sha256File } from "@mailwoman/core/hash";
|
|
@@ -58,7 +54,7 @@ import { assertHeaderColumns, readDelimitedHeader } from "#tools/fetch/header";
|
|
|
58
54
|
*/
|
|
59
55
|
export const FI_RYHTI_CSV_URL = "https://paikkatiedot.ymparisto.fi/geoserver/www/open_address.csv.gz";
|
|
60
56
|
/**
|
|
61
|
-
* The
|
|
57
|
+
* The download directory, the adapter's `inputPath`'s parent.
|
|
62
58
|
*/
|
|
63
59
|
const SLUG = RYHTI_ADAPTER_ID;
|
|
64
60
|
/**
|
|
@@ -74,7 +70,8 @@ const COMPRESSED_FILENAME = "open_address.csv.gz";
|
|
|
74
70
|
*/
|
|
75
71
|
export const FI_RYHTI_LICENSE = "CC-BY-4.0";
|
|
76
72
|
/**
|
|
77
|
-
* The attribution SYKE states for its open data
|
|
73
|
+
* The attribution SYKE states for its open data.
|
|
74
|
+
* The model card must record it.
|
|
78
75
|
*/
|
|
79
76
|
export const FI_RYHTI_ATTRIBUTION = "Lähde: Syke Ryhti";
|
|
80
77
|
/**
|
|
@@ -115,7 +112,7 @@ export async function readRyhtiHead(client, url = FI_RYHTI_CSV_URL) {
|
|
|
115
112
|
const length = headers?.["content-length"];
|
|
116
113
|
return {
|
|
117
114
|
lastModified: headers?.["last-modified"] ?? null,
|
|
118
|
-
contentLength: length
|
|
115
|
+
contentLength: length && /^\d+$/u.test(length) ? Number(length) : null,
|
|
119
116
|
contentType: headers?.["content-type"] ?? null,
|
|
120
117
|
};
|
|
121
118
|
}
|
|
@@ -157,9 +154,9 @@ async function isCurrent(recorded, head, path, verifyDigest) {
|
|
|
157
154
|
if (!recorded)
|
|
158
155
|
return false;
|
|
159
156
|
// A host that states neither cannot be compared against, so the file is downloaded.
|
|
160
|
-
if (head.lastModified
|
|
157
|
+
if (!head.lastModified && head.contentLength === null)
|
|
161
158
|
return false;
|
|
162
|
-
if (head.lastModified
|
|
159
|
+
if (head.lastModified && head.lastModified !== recorded.last_modified)
|
|
163
160
|
return false;
|
|
164
161
|
if (head.contentLength !== null && head.contentLength !== recorded.compressed_bytes)
|
|
165
162
|
return false;
|
|
@@ -221,8 +218,8 @@ export async function downloadRyhti(client, options) {
|
|
|
221
218
|
throw new Error(`HTTP ${response.status} ${response.statusText} — ${FI_RYHTI_CSV_URL}`);
|
|
222
219
|
const contentType = response.headers.get("content-type") ?? "";
|
|
223
220
|
// The host serves `application/x-gzip`.
|
|
224
|
-
// An `text/html` body under http 200 is a portal page
|
|
225
|
-
//
|
|
221
|
+
// An `text/html` body under http 200 is a portal page.
|
|
222
|
+
// A portal page saved to disk would fail later as a corrupt archive.
|
|
226
223
|
if (contentType.includes("text/html")) {
|
|
227
224
|
throw new Error(`${FI_RYHTI_CSV_URL} answered ${contentType}, which is a page rather than the file`);
|
|
228
225
|
}
|