@mailwoman/corpus 10.1.0 → 10.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/licenses/cdla-permissive-2.0.md +1 -1
- package/lib/adapters/cordis/adapter.ts +13 -7
- package/lib/adapters/dedup-key.ts +1 -1
- package/lib/adapters/geonames/adapter.ts +1 -1
- package/lib/adapters/gleif/adapter.ts +17 -16
- package/lib/adapters/libpostal-dictionary.ts +0 -2
- package/lib/adapters/openaddresses/adapter.ts +4 -3
- package/lib/adapters/osm/adapter.ts +22 -21
- package/lib/adapters/po-box/adapter.ts +10 -5
- package/lib/adapters/registry.ts +4 -4
- package/lib/adapters/source-id.ts +2 -2
- package/lib/adapters/street-line.ts +7 -6
- package/lib/adapters/wof/admin/json/adapter.ts +2 -2
- package/lib/adapters/wof/json-rows.ts +2 -1
- package/lib/adapters/wof/postalcode-json/adapter.ts +6 -2
- package/lib/adapters.ts +5 -5
- package/lib/au/adapters/gnaf/adapter.ts +1 -1
- package/lib/be/adapters/brussels/adapter.ts +22 -22
- package/lib/be/adapters/vlaanderen/adapter.ts +54 -57
- package/lib/be/adapters/wallonie/adapter.ts +17 -17
- package/lib/be/tools/fetch/brussels.ts +13 -10
- package/lib/be/tools/fetch/wallonie.ts +8 -7
- package/lib/build/adapters.ts +2 -1
- package/lib/build/align.ts +5 -4
- package/lib/build/checkpoint.ts +5 -3
- package/lib/build/eligibility.ts +3 -3
- package/lib/build/types.ts +8 -6
- package/lib/cz/adapters/cuzk/adapter.ts +41 -38
- package/lib/cz/tools/fetch/cuzk.ts +42 -31
- package/lib/de/recipes/locale.ts +1 -1
- package/lib/dk/adapters/inspire/adapter.ts +38 -38
- package/lib/dk/tools/fetch/inspire.ts +16 -14
- package/lib/ee/adapters/ads/adapter.ts +26 -32
- package/lib/ee/tools/fetch/ads.ts +10 -8
- package/lib/es/adapters/bizkaia/adapter.ts +47 -47
- package/lib/es/adapters/catastro/adapter.ts +36 -34
- package/lib/es/adapters/gipuzkoa/adapter.ts +38 -45
- package/lib/es/adapters/navarra/adapter.ts +28 -27
- package/lib/es/cadastre.ts +31 -31
- package/lib/es/tools/fetch/bizkaia.ts +40 -36
- package/lib/es/tools/fetch/catastro.ts +51 -40
- package/lib/es/tools/fetch/gipuzkoa.ts +13 -11
- package/lib/es/tools/fetch/navarra.ts +33 -25
- package/lib/fi/adapters/ryhti/adapter.ts +22 -26
- package/lib/fi/tools/fetch/ryhti.ts +31 -31
- package/lib/fr/adapters/annuaire-education/adapter.ts +13 -11
- package/lib/fr/adapters/ban/adapter.ts +6 -6
- package/lib/fr/adapters/finess/adapter.ts +16 -15
- package/lib/fr/adapters/finess/voie-type.ts +4 -3
- package/lib/fr/insee-country.ts +2 -2
- package/lib/fr/recipes/lieudit.ts +8 -6
- package/lib/fr/tools/fetch/annuaire-education.ts +11 -8
- package/lib/fr/tools/fetch/ban.ts +3 -2
- package/lib/fr/tools/fetch/finess.ts +14 -13
- package/lib/inspire/address.ts +86 -78
- package/lib/inspire/errors.ts +9 -8
- package/lib/inspire/stream.ts +29 -26
- package/lib/international/recipes/locale.ts +13 -9
- package/lib/it/adapters/anac/adapter.ts +51 -46
- package/lib/it/tools/fetch/anac.ts +22 -25
- package/lib/jp/adapters/wof-admin/adapter.ts +2 -6
- package/lib/jp/tools/fetch/houjin.ts +3 -3
- package/lib/kr/adapters/localdata/permit-rows.ts +1 -1
- package/lib/kr/tools/fetch/juso.ts +1 -1
- package/lib/kr/tools/fetch/localdata.ts +2 -2
- package/lib/lu/adapters/bd-adresses/adapter.ts +10 -10
- package/lib/lu/tools/fetch/bd-adresses.ts +15 -13
- package/lib/lv/adapters/varis/adapter.ts +24 -23
- package/lib/lv/tools/fetch/varis.ts +3 -2
- package/lib/nl/adapters/kadaster/adapter.ts +29 -29
- package/lib/nl/tools/fetch/kadaster.ts +16 -12
- package/lib/no/adapters/matrikkelen/adapter.ts +6 -5
- package/lib/no/tools/fetch/matrikkelen.ts +59 -47
- package/lib/parquet/duckdb.ts +7 -4
- package/lib/parquet/readers.ts +3 -3
- package/lib/parquet/schema.ts +1 -1
- package/lib/parquet/writers.ts +3 -2
- package/lib/pl/adapters/emuia/adapter.ts +12 -12
- package/lib/pl/tools/fetch/emuia.ts +10 -9
- package/lib/recipes/bare/postcode.ts +1 -1
- package/lib/recipes/boundary-stress.ts +1 -1
- package/lib/recipes/house-venue.ts +7 -3
- package/lib/recipes/intersection.ts +2 -2
- package/lib/recipes/po/box/cedex/recipe.ts +3 -3
- package/lib/recipes/po/box/cedex/sources.ts +13 -1
- package/lib/recipes/scaffold.ts +31 -21
- package/lib/recipes/sources.ts +16 -3
- package/lib/recipes/street/bare.ts +2 -2
- package/lib/recipes/street.ts +3 -1
- package/lib/recipes/sub/venue/context.ts +2 -0
- package/lib/recipes/sub/venue/render.ts +1 -1
- package/lib/recipes/sub/venue/sources.ts +24 -29
- package/lib/recipes/sub/venue.ts +12 -18
- package/lib/recipes/trailing-region.ts +6 -2
- package/lib/recipes/unit.ts +3 -3
- package/lib/recipes/venue.ts +347 -0
- package/lib/recipes.ts +4 -2
- package/lib/registers.ts +8 -6
- package/lib/ro/adapters/retea-scolara/adapter.ts +5 -5
- package/lib/ro/tools/fetch/retea-scolara.ts +1 -1
- package/lib/runner.ts +4 -3
- package/lib/sg/recipes/register.ts +1 -1
- package/lib/sg/tools/fetch/acra.ts +3 -3
- package/lib/sk/adapters/inspire/adapter.ts +25 -27
- package/lib/sk/tools/fetch/inspire.ts +15 -14
- package/lib/source-register/effective-manifest.ts +3 -3
- package/lib/source-register/register.ts +3 -3
- package/lib/source-register/training-manifest.ts +1 -1
- package/lib/source-register/types.ts +12 -13
- package/lib/subvenue/promotions.ts +1 -1
- package/lib/subvenue/table.ts +4 -4
- package/lib/surfaces/locale.ts +2 -0
- package/lib/synthesizers/anchor-absorption.ts +2 -1
- package/lib/synthesizers/boundary-stress.ts +20 -26
- package/lib/synthesizers/house-venue.ts +26 -30
- package/lib/synthesizers/no-street.ts +2 -1
- package/lib/synthesizers/po-box.ts +6 -3
- package/lib/synthesizers/tail.ts +55 -0
- package/lib/synthesizers/utils.ts +19 -2
- package/lib/test-kit.ts +2 -2
- package/lib/tools/audit.ts +7 -5
- package/lib/tools/corpus-stats.ts +1 -1
- package/lib/tools/fetch/atom.ts +10 -10
- package/lib/tools/fetch/cordis.ts +11 -9
- package/lib/tools/fetch/download/manifest.ts +2 -1
- package/lib/tools/fetch/download/network.ts +4 -4
- package/lib/tools/fetch/download/retry.ts +4 -4
- package/lib/tools/fetch/geonames/dump.ts +5 -3
- package/lib/tools/fetch/geonames/postal.ts +2 -2
- package/lib/tools/fetch/gleif.ts +6 -4
- package/lib/tools/fetch/header.ts +8 -6
- package/lib/tools/fetch/inspire-addresses.ts +41 -36
- package/lib/tools/fetch/openaddresses.ts +1 -1
- package/lib/tools/fetch/vlaanderen-ad.ts +17 -17
- package/lib/tools/fetch/wfs-harvest.ts +39 -28
- package/lib/tools/fetch.ts +0 -42
- package/lib/tools/golden/expand.ts +8 -8
- package/lib/tools/golden/promote.ts +12 -8
- package/lib/tools/golden/relabel-street/directory.ts +7 -6
- package/lib/tools/golden/relabel-street/row.ts +7 -6
- package/lib/tools/ingest-csv.ts +3 -3
- package/lib/tools/jsonl-to-parquet.ts +1 -1
- package/lib/tools/lint/recipe-output/vocab.ts +10 -5
- package/lib/tools/lint/recipe-output.ts +26 -8
- package/lib/tools/overlay/kryptonite.ts +4 -3
- package/lib/tools/overlay/manifest.ts +4 -2
- package/lib/tools/overlay/merge-source.ts +2 -1
- package/lib/tools/overlay/split-slice.ts +2 -2
- package/lib/tools/overlay/translit.ts +6 -4
- package/lib/tools/overture-subvenue.ts +1 -1
- package/lib/tools/postcode-triples.ts +5 -4
- package/lib/tools/source-register/build.ts +17 -14
- package/lib/tools/sub/venue/lexicon.ts +4 -8
- package/lib/tw/tools/fetch/gcis.ts +4 -4
- package/lib/types.ts +4 -3
- package/lib/us/adapters/state/hi-schools/workbook.ts +1 -1
- package/lib/us/adapters/usgov/nad/adapter.ts +11 -11
- package/lib/us/adapters/usgov/samhsa-treatment-locator/adapter.ts +4 -4
- package/lib/us/fips-state.ts +2 -2
- package/lib/{synthesizers → us/synthesizers}/intersection.ts +12 -13
- package/lib/{synthesizers → us/synthesizers}/street.ts +16 -15
- package/lib/us/tools/fetch/imls-pls.ts +1 -1
- package/lib/us/tools/fetch/nad.ts +7 -7
- package/lib/us/tools/fetch/nppes.ts +5 -5
- package/lib/us/tools/fetch/state/hi-schools.ts +3 -2
- package/lib/us/tools/fetch/tiger-full.ts +1 -1
- package/lib/utils/align.ts +13 -10
- package/lib/utils/golden.ts +2 -2
- package/lib/utils/license.ts +3 -3
- package/lib/utils/split.ts +19 -15
- package/lib/utils/tokenize.ts +7 -26
- package/lib/utils/wof-json.ts +4 -4
- package/out/adapters/cordis/adapter.d.ts +5 -3
- package/out/adapters/cordis/adapter.d.ts.map +1 -1
- package/out/adapters/cordis/adapter.js +13 -7
- package/out/adapters/cordis/adapter.js.map +1 -1
- package/out/adapters/dedup-key.d.ts +1 -1
- package/out/adapters/dedup-key.js +1 -1
- package/out/adapters/geonames/adapter.js +1 -1
- package/out/adapters/geonames/adapter.js.map +1 -1
- package/out/adapters/gleif/adapter.d.ts +9 -9
- package/out/adapters/gleif/adapter.d.ts.map +1 -1
- package/out/adapters/gleif/adapter.js +15 -14
- package/out/adapters/gleif/adapter.js.map +1 -1
- package/out/adapters/libpostal-dictionary.d.ts +0 -2
- package/out/adapters/libpostal-dictionary.d.ts.map +1 -1
- package/out/adapters/libpostal-dictionary.js +0 -2
- package/out/adapters/libpostal-dictionary.js.map +1 -1
- package/out/adapters/openaddresses/adapter.d.ts +3 -2
- package/out/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/adapters/openaddresses/adapter.js +1 -1
- package/out/adapters/openaddresses/adapter.js.map +1 -1
- package/out/adapters/osm/adapter.d.ts +17 -16
- package/out/adapters/osm/adapter.d.ts.map +1 -1
- package/out/adapters/osm/adapter.js +10 -9
- package/out/adapters/osm/adapter.js.map +1 -1
- package/out/adapters/po-box/adapter.d.ts +6 -3
- package/out/adapters/po-box/adapter.d.ts.map +1 -1
- package/out/adapters/po-box/adapter.js +4 -2
- package/out/adapters/po-box/adapter.js.map +1 -1
- package/out/adapters/registry.d.ts +3 -3
- package/out/adapters/registry.d.ts.map +1 -1
- package/out/adapters/registry.js +1 -1
- package/out/adapters/registry.js.map +1 -1
- package/out/adapters/source-id.d.ts +2 -2
- package/out/adapters/source-id.js +2 -2
- package/out/adapters/street-line.d.ts +7 -6
- package/out/adapters/street-line.d.ts.map +1 -1
- package/out/adapters/street-line.js +7 -6
- package/out/adapters/street-line.js.map +1 -1
- package/out/adapters/wof/admin/json/adapter.js +1 -1
- package/out/adapters/wof/admin/json/adapter.js.map +1 -1
- package/out/adapters/wof/json-rows.d.ts +2 -1
- package/out/adapters/wof/json-rows.d.ts.map +1 -1
- package/out/adapters/wof/json-rows.js.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.d.ts.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.js +5 -1
- package/out/adapters/wof/postalcode-json/adapter.js.map +1 -1
- package/out/adapters.d.ts +5 -5
- package/out/adapters.js +5 -5
- package/out/au/adapters/gnaf/adapter.js +1 -1
- package/out/au/adapters/gnaf/adapter.js.map +1 -1
- package/out/be/adapters/brussels/adapter.d.ts +9 -9
- package/out/be/adapters/brussels/adapter.d.ts.map +1 -1
- package/out/be/adapters/brussels/adapter.js +21 -21
- package/out/be/adapters/brussels/adapter.js.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.d.ts +34 -37
- package/out/be/adapters/vlaanderen/adapter.d.ts.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.js +40 -38
- package/out/be/adapters/vlaanderen/adapter.js.map +1 -1
- package/out/be/adapters/wallonie/adapter.d.ts +2 -2
- package/out/be/adapters/wallonie/adapter.js +17 -17
- package/out/be/adapters/wallonie/adapter.js.map +1 -1
- package/out/be/tools/fetch/brussels.d.ts +11 -8
- package/out/be/tools/fetch/brussels.d.ts.map +1 -1
- package/out/be/tools/fetch/brussels.js +10 -8
- package/out/be/tools/fetch/brussels.js.map +1 -1
- package/out/be/tools/fetch/wallonie.d.ts +6 -5
- package/out/be/tools/fetch/wallonie.d.ts.map +1 -1
- package/out/be/tools/fetch/wallonie.js +4 -4
- package/out/be/tools/fetch/wallonie.js.map +1 -1
- package/out/build/adapters.d.ts.map +1 -1
- package/out/build/adapters.js +2 -1
- package/out/build/adapters.js.map +1 -1
- package/out/build/align.d.ts +3 -2
- package/out/build/align.d.ts.map +1 -1
- package/out/build/align.js +2 -2
- package/out/build/align.js.map +1 -1
- package/out/build/checkpoint.d.ts +1 -1
- package/out/build/checkpoint.d.ts.map +1 -1
- package/out/build/checkpoint.js +4 -2
- package/out/build/checkpoint.js.map +1 -1
- package/out/build/eligibility.d.ts +2 -2
- package/out/build/eligibility.js +3 -3
- package/out/build/eligibility.js.map +1 -1
- package/out/build/types.d.ts +8 -6
- package/out/build/types.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.d.ts +14 -14
- package/out/cz/adapters/cuzk/adapter.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.js +38 -35
- package/out/cz/adapters/cuzk/adapter.js.map +1 -1
- package/out/cz/tools/fetch/cuzk.d.ts +31 -22
- package/out/cz/tools/fetch/cuzk.d.ts.map +1 -1
- package/out/cz/tools/fetch/cuzk.js +21 -19
- package/out/cz/tools/fetch/cuzk.js.map +1 -1
- package/out/de/recipes/locale.js +1 -1
- package/out/de/recipes/locale.js.map +1 -1
- package/out/dk/adapters/inspire/adapter.d.ts +32 -33
- package/out/dk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/dk/adapters/inspire/adapter.js +38 -38
- package/out/dk/adapters/inspire/adapter.js.map +1 -1
- package/out/dk/tools/fetch/inspire.d.ts +6 -5
- package/out/dk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/dk/tools/fetch/inspire.js +15 -13
- package/out/dk/tools/fetch/inspire.js.map +1 -1
- package/out/ee/adapters/ads/adapter.d.ts +19 -26
- package/out/ee/adapters/ads/adapter.d.ts.map +1 -1
- package/out/ee/adapters/ads/adapter.js +22 -28
- package/out/ee/adapters/ads/adapter.js.map +1 -1
- package/out/ee/tools/fetch/ads.d.ts +3 -3
- package/out/ee/tools/fetch/ads.d.ts.map +1 -1
- package/out/ee/tools/fetch/ads.js +10 -8
- package/out/ee/tools/fetch/ads.js.map +1 -1
- package/out/es/adapters/bizkaia/adapter.d.ts +28 -28
- package/out/es/adapters/bizkaia/adapter.d.ts.map +1 -1
- package/out/es/adapters/bizkaia/adapter.js +43 -43
- package/out/es/adapters/bizkaia/adapter.js.map +1 -1
- package/out/es/adapters/catastro/adapter.d.ts +22 -23
- package/out/es/adapters/catastro/adapter.d.ts.map +1 -1
- package/out/es/adapters/catastro/adapter.js +35 -33
- package/out/es/adapters/catastro/adapter.js.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.d.ts +31 -38
- package/out/es/adapters/gipuzkoa/adapter.d.ts.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.js +40 -47
- package/out/es/adapters/gipuzkoa/adapter.js.map +1 -1
- package/out/es/adapters/navarra/adapter.d.ts +20 -19
- package/out/es/adapters/navarra/adapter.d.ts.map +1 -1
- package/out/es/adapters/navarra/adapter.js +23 -22
- package/out/es/adapters/navarra/adapter.js.map +1 -1
- package/out/es/cadastre.d.ts +21 -20
- package/out/es/cadastre.d.ts.map +1 -1
- package/out/es/cadastre.js +19 -21
- package/out/es/cadastre.js.map +1 -1
- package/out/es/tools/fetch/bizkaia.d.ts +28 -25
- package/out/es/tools/fetch/bizkaia.d.ts.map +1 -1
- package/out/es/tools/fetch/bizkaia.js +25 -24
- package/out/es/tools/fetch/bizkaia.js.map +1 -1
- package/out/es/tools/fetch/catastro.d.ts +28 -20
- package/out/es/tools/fetch/catastro.d.ts.map +1 -1
- package/out/es/tools/fetch/catastro.js +36 -31
- package/out/es/tools/fetch/catastro.js.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.d.ts +11 -9
- package/out/es/tools/fetch/gipuzkoa.d.ts.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.js +9 -8
- package/out/es/tools/fetch/gipuzkoa.js.map +1 -1
- package/out/es/tools/fetch/navarra.d.ts +20 -15
- package/out/es/tools/fetch/navarra.d.ts.map +1 -1
- package/out/es/tools/fetch/navarra.js +21 -16
- package/out/es/tools/fetch/navarra.js.map +1 -1
- package/out/fi/adapters/ryhti/adapter.d.ts +16 -20
- package/out/fi/adapters/ryhti/adapter.d.ts.map +1 -1
- package/out/fi/adapters/ryhti/adapter.js +21 -25
- package/out/fi/adapters/ryhti/adapter.js.map +1 -1
- package/out/fi/tools/fetch/ryhti.d.ts +24 -24
- package/out/fi/tools/fetch/ryhti.d.ts.map +1 -1
- package/out/fi/tools/fetch/ryhti.js +21 -24
- package/out/fi/tools/fetch/ryhti.js.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.d.ts +8 -8
- package/out/fr/adapters/annuaire-education/adapter.d.ts.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.js +13 -11
- package/out/fr/adapters/annuaire-education/adapter.js.map +1 -1
- package/out/fr/adapters/ban/adapter.d.ts +4 -4
- package/out/fr/adapters/ban/adapter.js +6 -6
- package/out/fr/adapters/ban/adapter.js.map +1 -1
- package/out/fr/adapters/finess/adapter.d.ts +12 -11
- package/out/fr/adapters/finess/adapter.d.ts.map +1 -1
- package/out/fr/adapters/finess/adapter.js +16 -15
- package/out/fr/adapters/finess/adapter.js.map +1 -1
- package/out/fr/adapters/finess/voie-type.d.ts +2 -2
- package/out/fr/adapters/finess/voie-type.d.ts.map +1 -1
- package/out/fr/adapters/finess/voie-type.js +4 -3
- package/out/fr/adapters/finess/voie-type.js.map +1 -1
- package/out/fr/insee-country.d.ts +2 -2
- package/out/fr/insee-country.js +2 -2
- package/out/fr/recipes/lieudit.d.ts.map +1 -1
- package/out/fr/recipes/lieudit.js +5 -4
- package/out/fr/recipes/lieudit.js.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.d.ts +7 -4
- package/out/fr/tools/fetch/annuaire-education.d.ts.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.js +9 -7
- package/out/fr/tools/fetch/annuaire-education.js.map +1 -1
- package/out/fr/tools/fetch/ban.d.ts +1 -1
- package/out/fr/tools/fetch/ban.d.ts.map +1 -1
- package/out/fr/tools/fetch/ban.js +3 -2
- package/out/fr/tools/fetch/ban.js.map +1 -1
- package/out/fr/tools/fetch/finess.d.ts +5 -5
- package/out/fr/tools/fetch/finess.d.ts.map +1 -1
- package/out/fr/tools/fetch/finess.js +13 -12
- package/out/fr/tools/fetch/finess.js.map +1 -1
- package/out/inspire/address.d.ts +62 -52
- package/out/inspire/address.d.ts.map +1 -1
- package/out/inspire/address.js +71 -60
- package/out/inspire/address.js.map +1 -1
- package/out/inspire/errors.d.ts +9 -8
- package/out/inspire/errors.d.ts.map +1 -1
- package/out/inspire/errors.js +7 -6
- package/out/inspire/errors.js.map +1 -1
- package/out/inspire/stream.d.ts +25 -22
- package/out/inspire/stream.d.ts.map +1 -1
- package/out/inspire/stream.js +10 -12
- package/out/inspire/stream.js.map +1 -1
- package/out/international/recipes/locale.d.ts +2 -2
- package/out/international/recipes/locale.d.ts.map +1 -1
- package/out/international/recipes/locale.js +11 -7
- package/out/international/recipes/locale.js.map +1 -1
- package/out/it/adapters/anac/adapter.d.ts +45 -43
- package/out/it/adapters/anac/adapter.d.ts.map +1 -1
- package/out/it/adapters/anac/adapter.js +48 -43
- package/out/it/adapters/anac/adapter.js.map +1 -1
- package/out/it/tools/fetch/anac.d.ts +16 -19
- package/out/it/tools/fetch/anac.d.ts.map +1 -1
- package/out/it/tools/fetch/anac.js +20 -23
- package/out/it/tools/fetch/anac.js.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.d.ts.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.js.map +1 -1
- package/out/jp/tools/fetch/houjin.js +2 -2
- package/out/jp/tools/fetch/houjin.js.map +1 -1
- package/out/kr/adapters/localdata/permit-rows.js +1 -1
- package/out/kr/adapters/localdata/permit-rows.js.map +1 -1
- package/out/kr/tools/fetch/juso.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.js +1 -1
- package/out/kr/tools/fetch/localdata.js.map +1 -1
- package/out/lu/adapters/bd-adresses/adapter.d.ts +9 -9
- package/out/lu/adapters/bd-adresses/adapter.js +10 -10
- package/out/lu/tools/fetch/bd-adresses.d.ts +11 -9
- package/out/lu/tools/fetch/bd-adresses.d.ts.map +1 -1
- package/out/lu/tools/fetch/bd-adresses.js +11 -11
- package/out/lu/tools/fetch/bd-adresses.js.map +1 -1
- package/out/lv/adapters/varis/adapter.d.ts +24 -23
- package/out/lv/adapters/varis/adapter.d.ts.map +1 -1
- package/out/lv/adapters/varis/adapter.js +24 -23
- package/out/lv/adapters/varis/adapter.js.map +1 -1
- package/out/lv/tools/fetch/varis.d.ts +2 -1
- package/out/lv/tools/fetch/varis.d.ts.map +1 -1
- package/out/lv/tools/fetch/varis.js +1 -1
- package/out/lv/tools/fetch/varis.js.map +1 -1
- package/out/nl/adapters/kadaster/adapter.d.ts +12 -12
- package/out/nl/adapters/kadaster/adapter.d.ts.map +1 -1
- package/out/nl/adapters/kadaster/adapter.js +22 -23
- package/out/nl/adapters/kadaster/adapter.js.map +1 -1
- package/out/nl/tools/fetch/kadaster.d.ts +14 -10
- package/out/nl/tools/fetch/kadaster.d.ts.map +1 -1
- package/out/nl/tools/fetch/kadaster.js +12 -9
- package/out/nl/tools/fetch/kadaster.js.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.js +3 -3
- package/out/no/adapters/matrikkelen/adapter.js.map +1 -1
- package/out/no/tools/fetch/matrikkelen.d.ts +50 -38
- package/out/no/tools/fetch/matrikkelen.d.ts.map +1 -1
- package/out/no/tools/fetch/matrikkelen.js +44 -36
- package/out/no/tools/fetch/matrikkelen.js.map +1 -1
- package/out/parquet/duckdb.d.ts +7 -4
- package/out/parquet/duckdb.d.ts.map +1 -1
- package/out/parquet/duckdb.js +2 -1
- package/out/parquet/duckdb.js.map +1 -1
- package/out/parquet/readers.js +3 -3
- package/out/parquet/readers.js.map +1 -1
- package/out/parquet/schema.js +1 -1
- package/out/parquet/schema.js.map +1 -1
- package/out/parquet/writers.d.ts +1 -1
- package/out/parquet/writers.d.ts.map +1 -1
- package/out/parquet/writers.js +2 -1
- package/out/parquet/writers.js.map +1 -1
- package/out/pl/adapters/emuia/adapter.d.ts +8 -8
- package/out/pl/adapters/emuia/adapter.js +12 -12
- package/out/pl/adapters/emuia/adapter.js.map +1 -1
- package/out/pl/tools/fetch/emuia.d.ts +5 -4
- package/out/pl/tools/fetch/emuia.d.ts.map +1 -1
- package/out/pl/tools/fetch/emuia.js +10 -9
- package/out/pl/tools/fetch/emuia.js.map +1 -1
- package/out/recipes/bare/postcode.js +1 -1
- package/out/recipes/bare/postcode.js.map +1 -1
- package/out/recipes/boundary-stress.js +1 -1
- package/out/recipes/boundary-stress.js.map +1 -1
- package/out/recipes/house-venue.d.ts.map +1 -1
- package/out/recipes/house-venue.js +6 -3
- package/out/recipes/house-venue.js.map +1 -1
- package/out/recipes/intersection.js +2 -2
- package/out/recipes/intersection.js.map +1 -1
- package/out/recipes/po/box/cedex/recipe.js +2 -1
- package/out/recipes/po/box/cedex/recipe.js.map +1 -1
- package/out/recipes/po/box/cedex/sources.d.ts +8 -2
- package/out/recipes/po/box/cedex/sources.d.ts.map +1 -1
- package/out/recipes/po/box/cedex/sources.js +1 -1
- package/out/recipes/po/box/cedex/sources.js.map +1 -1
- package/out/recipes/scaffold.d.ts +28 -19
- package/out/recipes/scaffold.d.ts.map +1 -1
- package/out/recipes/scaffold.js +3 -2
- package/out/recipes/scaffold.js.map +1 -1
- package/out/recipes/sources.d.ts +1 -1
- package/out/recipes/sources.d.ts.map +1 -1
- package/out/recipes/sources.js +15 -2
- package/out/recipes/sources.js.map +1 -1
- package/out/recipes/street/bare.js +2 -2
- package/out/recipes/street/bare.js.map +1 -1
- package/out/recipes/street.d.ts.map +1 -1
- package/out/recipes/street.js +3 -1
- package/out/recipes/street.js.map +1 -1
- package/out/recipes/sub/venue/context.d.ts.map +1 -1
- package/out/recipes/sub/venue/context.js +2 -0
- package/out/recipes/sub/venue/context.js.map +1 -1
- package/out/recipes/sub/venue/render.js +1 -1
- package/out/recipes/sub/venue/render.js.map +1 -1
- package/out/recipes/sub/venue/sources.d.ts +6 -3
- package/out/recipes/sub/venue/sources.d.ts.map +1 -1
- package/out/recipes/sub/venue/sources.js +16 -20
- package/out/recipes/sub/venue/sources.js.map +1 -1
- package/out/recipes/sub/venue.d.ts +2 -5
- package/out/recipes/sub/venue.d.ts.map +1 -1
- package/out/recipes/sub/venue.js +10 -14
- package/out/recipes/sub/venue.js.map +1 -1
- package/out/recipes/trailing-region.d.ts.map +1 -1
- package/out/recipes/trailing-region.js +6 -1
- package/out/recipes/trailing-region.js.map +1 -1
- package/out/recipes/unit.js +2 -2
- package/out/recipes/unit.js.map +1 -1
- package/out/recipes/venue.d.ts +119 -0
- package/out/recipes/venue.d.ts.map +1 -0
- package/out/recipes/venue.js +245 -0
- package/out/recipes/venue.js.map +1 -0
- package/out/recipes.d.ts +1 -1
- package/out/recipes.d.ts.map +1 -1
- package/out/recipes.js +3 -1
- package/out/recipes.js.map +1 -1
- package/out/registers.d.ts +8 -6
- package/out/registers.d.ts.map +1 -1
- package/out/registers.js +8 -6
- package/out/registers.js.map +1 -1
- package/out/ro/adapters/retea-scolara/adapter.d.ts +5 -5
- package/out/ro/adapters/retea-scolara/adapter.js +5 -5
- package/out/ro/tools/fetch/retea-scolara.js +1 -1
- package/out/ro/tools/fetch/retea-scolara.js.map +1 -1
- package/out/runner.d.ts +3 -2
- package/out/runner.d.ts.map +1 -1
- package/out/runner.js +1 -1
- package/out/runner.js.map +1 -1
- package/out/sg/recipes/register.js +1 -1
- package/out/sg/recipes/register.js.map +1 -1
- package/out/sg/tools/fetch/acra.js +2 -2
- package/out/sg/tools/fetch/acra.js.map +1 -1
- package/out/sk/adapters/inspire/adapter.d.ts +5 -5
- package/out/sk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/sk/adapters/inspire/adapter.js +18 -18
- package/out/sk/adapters/inspire/adapter.js.map +1 -1
- package/out/sk/tools/fetch/inspire.d.ts +11 -10
- package/out/sk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/sk/tools/fetch/inspire.js +15 -14
- package/out/sk/tools/fetch/inspire.js.map +1 -1
- package/out/source-register/effective-manifest.js +3 -3
- package/out/source-register/effective-manifest.js.map +1 -1
- package/out/source-register/register.d.ts +2 -2
- package/out/source-register/register.d.ts.map +1 -1
- package/out/source-register/register.js +2 -2
- package/out/source-register/register.js.map +1 -1
- package/out/source-register/training-manifest.d.ts +1 -1
- package/out/source-register/types.d.ts +12 -13
- package/out/source-register/types.d.ts.map +1 -1
- package/out/source-register/types.js +2 -2
- package/out/source-register/types.js.map +1 -1
- package/out/subvenue/promotions.d.ts +1 -1
- package/out/subvenue/promotions.js +1 -1
- package/out/subvenue/table.d.ts +4 -4
- package/out/subvenue/table.js +3 -3
- package/out/surfaces/locale.d.ts.map +1 -1
- package/out/surfaces/locale.js +2 -0
- package/out/surfaces/locale.js.map +1 -1
- package/out/synthesizers/anchor-absorption.d.ts +2 -1
- package/out/synthesizers/anchor-absorption.d.ts.map +1 -1
- package/out/synthesizers/anchor-absorption.js.map +1 -1
- package/out/synthesizers/boundary-stress.d.ts +1 -1
- package/out/synthesizers/boundary-stress.d.ts.map +1 -1
- package/out/synthesizers/boundary-stress.js +20 -23
- package/out/synthesizers/boundary-stress.js.map +1 -1
- package/out/synthesizers/house-venue.d.ts +3 -2
- package/out/synthesizers/house-venue.d.ts.map +1 -1
- package/out/synthesizers/house-venue.js +24 -26
- package/out/synthesizers/house-venue.js.map +1 -1
- package/out/synthesizers/no-street.d.ts.map +1 -1
- package/out/synthesizers/no-street.js.map +1 -1
- package/out/synthesizers/po-box.d.ts +4 -2
- package/out/synthesizers/po-box.d.ts.map +1 -1
- package/out/synthesizers/po-box.js.map +1 -1
- package/out/synthesizers/tail.d.ts +36 -0
- package/out/synthesizers/tail.d.ts.map +1 -0
- package/out/synthesizers/tail.js +32 -0
- package/out/synthesizers/tail.js.map +1 -0
- package/out/synthesizers/utils.d.ts +2 -1
- package/out/synthesizers/utils.d.ts.map +1 -1
- package/out/synthesizers/utils.js +17 -1
- package/out/synthesizers/utils.js.map +1 -1
- package/out/test-kit.js +2 -2
- package/out/test-kit.js.map +1 -1
- package/out/tools/audit.d.ts +2 -1
- package/out/tools/audit.d.ts.map +1 -1
- package/out/tools/audit.js +4 -3
- package/out/tools/audit.js.map +1 -1
- package/out/tools/corpus-stats.js +1 -1
- package/out/tools/corpus-stats.js.map +1 -1
- package/out/tools/fetch/atom.d.ts +8 -8
- package/out/tools/fetch/atom.d.ts.map +1 -1
- package/out/tools/fetch/atom.js +7 -8
- package/out/tools/fetch/atom.js.map +1 -1
- package/out/tools/fetch/cordis.d.ts +4 -3
- package/out/tools/fetch/cordis.d.ts.map +1 -1
- package/out/tools/fetch/cordis.js +9 -7
- package/out/tools/fetch/cordis.js.map +1 -1
- package/out/tools/fetch/download/manifest.d.ts.map +1 -1
- package/out/tools/fetch/download/manifest.js +2 -1
- package/out/tools/fetch/download/manifest.js.map +1 -1
- package/out/tools/fetch/download/network.d.ts +4 -4
- package/out/tools/fetch/download/retry.js +4 -4
- package/out/tools/fetch/download/retry.js.map +1 -1
- package/out/tools/fetch/geonames/dump.d.ts +3 -2
- package/out/tools/fetch/geonames/dump.d.ts.map +1 -1
- package/out/tools/fetch/geonames/dump.js +2 -1
- package/out/tools/fetch/geonames/dump.js.map +1 -1
- package/out/tools/fetch/geonames/postal.d.ts +2 -2
- package/out/tools/fetch/gleif.d.ts +3 -2
- package/out/tools/fetch/gleif.d.ts.map +1 -1
- package/out/tools/fetch/gleif.js +6 -4
- package/out/tools/fetch/gleif.js.map +1 -1
- package/out/tools/fetch/header.d.ts +7 -5
- package/out/tools/fetch/header.d.ts.map +1 -1
- package/out/tools/fetch/header.js +8 -6
- package/out/tools/fetch/header.js.map +1 -1
- package/out/tools/fetch/inspire-addresses.d.ts +32 -27
- package/out/tools/fetch/inspire-addresses.d.ts.map +1 -1
- package/out/tools/fetch/inspire-addresses.js +25 -24
- package/out/tools/fetch/inspire-addresses.js.map +1 -1
- package/out/tools/fetch/openaddresses.js +1 -1
- package/out/tools/fetch/openaddresses.js.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.d.ts +5 -5
- package/out/tools/fetch/vlaanderen-ad.d.ts.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.js +16 -16
- package/out/tools/fetch/vlaanderen-ad.js.map +1 -1
- package/out/tools/fetch/wfs-harvest.d.ts +26 -16
- package/out/tools/fetch/wfs-harvest.d.ts.map +1 -1
- package/out/tools/fetch/wfs-harvest.js +21 -18
- package/out/tools/fetch/wfs-harvest.js.map +1 -1
- package/out/tools/fetch.d.ts +0 -41
- package/out/tools/fetch.d.ts.map +1 -1
- package/out/tools/fetch.js +0 -41
- package/out/tools/fetch.js.map +1 -1
- package/out/tools/golden/expand.d.ts +7 -7
- package/out/tools/golden/expand.js +1 -1
- package/out/tools/golden/expand.js.map +1 -1
- package/out/tools/golden/promote.d.ts +2 -2
- package/out/tools/golden/promote.d.ts.map +1 -1
- package/out/tools/golden/promote.js +9 -6
- package/out/tools/golden/promote.js.map +1 -1
- package/out/tools/golden/relabel-street/directory.d.ts +4 -4
- package/out/tools/golden/relabel-street/directory.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/directory.js +3 -2
- package/out/tools/golden/relabel-street/directory.js.map +1 -1
- package/out/tools/golden/relabel-street/row.d.ts +4 -3
- package/out/tools/golden/relabel-street/row.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/row.js +3 -3
- package/out/tools/golden/relabel-street/row.js.map +1 -1
- package/out/tools/ingest-csv.js +3 -3
- package/out/tools/ingest-csv.js.map +1 -1
- package/out/tools/jsonl-to-parquet.d.ts +1 -1
- package/out/tools/lint/recipe-output/vocab.d.ts +10 -5
- package/out/tools/lint/recipe-output/vocab.d.ts.map +1 -1
- package/out/tools/lint/recipe-output/vocab.js.map +1 -1
- package/out/tools/lint/recipe-output.d.ts +8 -8
- package/out/tools/lint/recipe-output.d.ts.map +1 -1
- package/out/tools/lint/recipe-output.js +17 -0
- package/out/tools/lint/recipe-output.js.map +1 -1
- package/out/tools/overlay/kryptonite.d.ts +2 -2
- package/out/tools/overlay/kryptonite.d.ts.map +1 -1
- package/out/tools/overlay/kryptonite.js +2 -1
- package/out/tools/overlay/kryptonite.js.map +1 -1
- package/out/tools/overlay/manifest.d.ts +2 -1
- package/out/tools/overlay/manifest.d.ts.map +1 -1
- package/out/tools/overlay/manifest.js +2 -1
- package/out/tools/overlay/manifest.js.map +1 -1
- package/out/tools/overlay/merge-source.d.ts +2 -1
- package/out/tools/overlay/merge-source.d.ts.map +1 -1
- package/out/tools/overlay/merge-source.js.map +1 -1
- package/out/tools/overlay/split-slice.d.ts +1 -1
- package/out/tools/overlay/split-slice.js +1 -1
- package/out/tools/overlay/split-slice.js.map +1 -1
- package/out/tools/overlay/translit.d.ts +4 -3
- package/out/tools/overlay/translit.d.ts.map +1 -1
- package/out/tools/overlay/translit.js +2 -1
- package/out/tools/overlay/translit.js.map +1 -1
- package/out/tools/overture-subvenue.d.ts +1 -1
- package/out/tools/overture-subvenue.d.ts.map +1 -1
- package/out/tools/postcode-triples.d.ts +1 -1
- package/out/tools/postcode-triples.d.ts.map +1 -1
- package/out/tools/postcode-triples.js +4 -3
- package/out/tools/postcode-triples.js.map +1 -1
- package/out/tools/source-register/build.d.ts +10 -8
- package/out/tools/source-register/build.d.ts.map +1 -1
- package/out/tools/source-register/build.js +10 -8
- package/out/tools/source-register/build.js.map +1 -1
- package/out/tools/sub/venue/lexicon.d.ts +2 -6
- package/out/tools/sub/venue/lexicon.d.ts.map +1 -1
- package/out/tools/sub/venue/lexicon.js +2 -6
- package/out/tools/sub/venue/lexicon.js.map +1 -1
- package/out/tw/tools/fetch/gcis.js +3 -3
- package/out/tw/tools/fetch/gcis.js.map +1 -1
- package/out/types.d.ts +4 -3
- package/out/types.d.ts.map +1 -1
- package/out/types.js.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.js.map +1 -1
- package/out/us/adapters/usgov/nad/adapter.js +6 -6
- package/out/us/adapters/usgov/nad/adapter.js.map +1 -1
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js +2 -2
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/us/fips-state.d.ts +2 -2
- package/out/us/fips-state.d.ts.map +1 -1
- package/out/us/fips-state.js.map +1 -1
- package/out/us/synthesizers/intersection.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/intersection.js +13 -9
- package/out/us/synthesizers/intersection.js.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.d.ts +3 -2
- package/out/us/synthesizers/street.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.js +12 -12
- package/out/us/synthesizers/street.js.map +1 -0
- package/out/us/tools/fetch/imls-pls.js +1 -1
- package/out/us/tools/fetch/imls-pls.js.map +1 -1
- package/out/us/tools/fetch/nad.d.ts +6 -6
- package/out/us/tools/fetch/nad.js +1 -1
- package/out/us/tools/fetch/nad.js.map +1 -1
- package/out/us/tools/fetch/nppes.js +3 -3
- package/out/us/tools/fetch/nppes.js.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts +2 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.js +1 -1
- package/out/us/tools/fetch/state/hi-schools.js.map +1 -1
- package/out/us/tools/fetch/tiger-full.js +1 -1
- package/out/us/tools/fetch/tiger-full.js.map +1 -1
- package/out/utils/align.d.ts +5 -3
- package/out/utils/align.d.ts.map +1 -1
- package/out/utils/align.js +5 -5
- package/out/utils/align.js.map +1 -1
- package/out/utils/golden.js +2 -2
- package/out/utils/golden.js.map +1 -1
- package/out/utils/license.d.ts +3 -3
- package/out/utils/license.d.ts.map +1 -1
- package/out/utils/license.js.map +1 -1
- package/out/utils/split.d.ts +1 -1
- package/out/utils/split.d.ts.map +1 -1
- package/out/utils/split.js +18 -14
- package/out/utils/split.js.map +1 -1
- package/out/utils/tokenize.d.ts +3 -20
- package/out/utils/tokenize.d.ts.map +1 -1
- package/out/utils/tokenize.js.map +1 -1
- package/out/utils/wof-json.js +3 -3
- package/out/utils/wof-json.js.map +1 -1
- package/package.json +18 -27
- package/lib/tools.ts +0 -31
- package/out/synthesizers/intersection.d.ts.map +0 -1
- package/out/synthesizers/intersection.js.map +0 -1
- package/out/synthesizers/street.d.ts.map +0 -1
- package/out/synthesizers/street.js.map +0 -1
- package/out/tools.d.ts +0 -31
- package/out/tools.d.ts.map +0 -1
- package/out/tools.js +0 -31
- package/out/tools.js.map +0 -1
- /package/out/{synthesizers → us/synthesizers}/intersection.d.ts +0 -0
|
@@ -15,11 +15,11 @@
|
|
|
15
15
|
* Four properties of this service decide the harvest's shape.
|
|
16
16
|
*
|
|
17
17
|
* 1. **The archive link is the `enclosure` link, and the `alternate` link is not it.** Each entry
|
|
18
|
-
*
|
|
18
|
+
* lists six links, three of which address the same archive under `enclosure`, `section` and
|
|
19
19
|
* `alternate`. The entry's *first* `alternate` link is the relative href
|
|
20
20
|
* `Addresses_DatasetATOM_Navarra.xml`, which is the dataset feed, so a reader taking the first
|
|
21
21
|
* `alternate` link downloads an ATOM document in place of the archive. This harvest takes
|
|
22
|
-
* `enclosure` and refuses an entry that
|
|
22
|
+
* `enclosure` and refuses an entry that lists none.
|
|
23
23
|
* 2. **An entry leaves its own area unstated.** Every one of the 272 entries is titled
|
|
24
24
|
* `Address Navarra` and every `<id>` is empty, so the archive's own file name,
|
|
25
25
|
* `AD_Navarra_<n>.gml.zip`, is the only statement of which partition an entry offers. The
|
|
@@ -47,7 +47,7 @@
|
|
|
47
47
|
|
|
48
48
|
import { APIClient, assertNoOGCServiceException } from "@mailwoman/core/api"
|
|
49
49
|
import { ByteFormatter } from "@mailwoman/core/fs/formatters"
|
|
50
|
-
import { tryStat } from "@mailwoman/core/fs/readers"
|
|
50
|
+
import { tryStat } from "@mailwoman/core/fs/readers/stat"
|
|
51
51
|
import { makeDirectories } from "@mailwoman/core/fs/writers"
|
|
52
52
|
import { sha256File } from "@mailwoman/core/hash"
|
|
53
53
|
import { stringifyJSON } from "@mailwoman/core/json"
|
|
@@ -61,7 +61,8 @@ import { loadCollectionFiles, writeManifest } from "#tools/fetch/download"
|
|
|
61
61
|
import { downloadZipArchive } from "#tools/fetch/zip-archive"
|
|
62
62
|
|
|
63
63
|
/**
|
|
64
|
-
* The ATOM service document
|
|
64
|
+
* The ATOM service document.
|
|
65
|
+
* It lists one zipped GML partition per entry.
|
|
65
66
|
*/
|
|
66
67
|
export const ES_NAVARRA_SERVICE_FEED_URL =
|
|
67
68
|
"https://filescartografia.navarra.es/2_CARTOGRAFIA_TEMATICA/2_7_CATASTRO/2_7_3_INSPIRE_ATOM/2_7_3_3_AD/Addresses_ServiceATOM_Navarra.xml"
|
|
@@ -84,7 +85,8 @@ export const ES_NAVARRA_ATTRIBUTION = "Gobierno de Navarra"
|
|
|
84
85
|
export const ES_NAVARRA_REQUEST_INTERVAL_MS = 250
|
|
85
86
|
|
|
86
87
|
/**
|
|
87
|
-
* The directory the harvest writes under
|
|
88
|
+
* The directory the harvest writes under.
|
|
89
|
+
* The adapter reads this directory as its `inputPath`.
|
|
88
90
|
*/
|
|
89
91
|
const SLUG = ES_NAVARRA_ADAPTER_ID
|
|
90
92
|
|
|
@@ -113,7 +115,8 @@ export interface ESNavarraPartition {
|
|
|
113
115
|
*/
|
|
114
116
|
partition: string
|
|
115
117
|
/**
|
|
116
|
-
* The entry's title
|
|
118
|
+
* The entry's title.
|
|
119
|
+
* Every entry states `Address Navarra`.
|
|
117
120
|
*/
|
|
118
121
|
title: string
|
|
119
122
|
/**
|
|
@@ -122,8 +125,8 @@ export interface ESNavarraPartition {
|
|
|
122
125
|
updated: string
|
|
123
126
|
archiveURL: string
|
|
124
127
|
/**
|
|
125
|
-
* The file name the archive is written under
|
|
126
|
-
*
|
|
128
|
+
* The file name the archive is written under.
|
|
129
|
+
* The publisher states it and the adapter globs it.
|
|
127
130
|
*/
|
|
128
131
|
filename: string
|
|
129
132
|
}
|
|
@@ -131,8 +134,8 @@ export interface ESNavarraPartition {
|
|
|
131
134
|
/**
|
|
132
135
|
* The partitions the service document lists.
|
|
133
136
|
*
|
|
134
|
-
* @throws When the document holds no entry, when an entry
|
|
135
|
-
*
|
|
137
|
+
* @throws When the document holds no entry, when an entry lists no `enclosure` link, when a linked
|
|
138
|
+
* archive's file name is not `AD_Navarra_<n>.gml.zip`, or when two entries point to one archive.
|
|
136
139
|
* Each of those would otherwise write zero archives while reporting a completed run.
|
|
137
140
|
*/
|
|
138
141
|
export function readESNavarraServiceFeed(feed: AtomFeed): readonly ESNavarraPartition[] {
|
|
@@ -199,7 +202,7 @@ export interface ESNavarraArchiveEntry extends SourceManifest {
|
|
|
199
202
|
/**
|
|
200
203
|
* The `<updated>` the service document stated when this archive was fetched.
|
|
201
204
|
*
|
|
202
|
-
* A later run that reads the same value
|
|
205
|
+
* A later run that reads the same value skips the archive.
|
|
203
206
|
* The publisher moves it when it publishes a new edition.
|
|
204
207
|
*/
|
|
205
208
|
feed_updated: string
|
|
@@ -224,11 +227,12 @@ export interface ESNavarraHarvestManifest extends SourceCollectionManifest {
|
|
|
224
227
|
|
|
225
228
|
export interface HarvestESNavarraOptions {
|
|
226
229
|
/**
|
|
227
|
-
* Where the archives and the manifest are written
|
|
230
|
+
* Where the archives and the manifest are written.
|
|
231
|
+
* The adapter reads it as its `inputPath`.
|
|
228
232
|
*/
|
|
229
233
|
outputDir: PathBuilderLike
|
|
230
234
|
/**
|
|
231
|
-
* Harvest only these partition numbers, for a probe or
|
|
235
|
+
* Harvest only these partition numbers, for a probe or to repair specific partitions.
|
|
232
236
|
*
|
|
233
237
|
* A number the service document does not list is reported as a failure rather than ignored.
|
|
234
238
|
*/
|
|
@@ -240,7 +244,8 @@ export interface HarvestESNavarraOptions {
|
|
|
240
244
|
/**
|
|
241
245
|
* Re-read the sha256 of every archive already on disk instead of comparing its byte count.
|
|
242
246
|
*
|
|
243
|
-
*
|
|
247
|
+
* By default the harvest compares the recorded byte count against the file's size.
|
|
248
|
+
* That comparison costs one `stat`.
|
|
244
249
|
*/
|
|
245
250
|
verifyDigests?: boolean
|
|
246
251
|
signal?: AbortSignal
|
|
@@ -251,11 +256,12 @@ export interface HarvestESNavarraOptions {
|
|
|
251
256
|
* Whether the archive recorded for one partition is still the one the feed describes.
|
|
252
257
|
*
|
|
253
258
|
* A skip requires the feed to state an `<updated>` value.
|
|
254
|
-
* Where it states none, both sides read an empty string and an equality test would hold
|
|
255
|
-
*
|
|
259
|
+
* Where it states none, both sides read an empty string and an equality test would hold.
|
|
260
|
+
*
|
|
261
|
+
* That test would keep an archive of unknown age for as long as the feed stayed silent.
|
|
256
262
|
*/
|
|
257
263
|
async function isCurrent(
|
|
258
|
-
recorded: ESNavarraArchiveEntry |
|
|
264
|
+
recorded: ESNavarraArchiveEntry | null,
|
|
259
265
|
partition: ESNavarraPartition,
|
|
260
266
|
path: PathBuilderLike,
|
|
261
267
|
verifyDigests: boolean
|
|
@@ -266,14 +272,15 @@ async function isCurrent(
|
|
|
266
272
|
|
|
267
273
|
if (!stat || stat.size !== recorded.bytes) return false
|
|
268
274
|
|
|
269
|
-
// The digest check reads the whole file rather than its metadata
|
|
275
|
+
// The digest check reads the whole file rather than its metadata.
|
|
276
|
+
// The caller opts in.
|
|
270
277
|
if (!verifyDigests) return true
|
|
271
278
|
|
|
272
279
|
return (await sha256File(path)) === recorded.sha256
|
|
273
280
|
}
|
|
274
281
|
|
|
275
282
|
/**
|
|
276
|
-
* The partitions this run considers, and the numbers a caller
|
|
283
|
+
* The partitions this run considers, and the numbers a caller supplied that the feed does not list.
|
|
277
284
|
*
|
|
278
285
|
* A number the caller asked for and the service document does not list is returned
|
|
279
286
|
* rather than dropped, because a partition that is not published is a different
|
|
@@ -317,8 +324,9 @@ function selectPartitions(
|
|
|
317
324
|
* the pacing. {@linkcode fetchESNavarra} is the registry entry point and supplies both.
|
|
318
325
|
*
|
|
319
326
|
* @returns What was fetched, what the feed states is already recorded, and which partitions failed.
|
|
320
|
-
* @throws When the service document lists no partition
|
|
321
|
-
* `{fetched: 0, skipped: 0, failed: 0}`
|
|
327
|
+
* @throws When the service document lists no partition.
|
|
328
|
+
* The call would otherwise return `{fetched: 0, skipped: 0, failed: 0}`
|
|
329
|
+
* and read to a caller as a completed fetch of an empty publisher.
|
|
322
330
|
*/
|
|
323
331
|
export async function harvestESNavarra(
|
|
324
332
|
client: Pick<APIClient, "fetch">,
|
|
@@ -357,8 +365,8 @@ export async function harvestESNavarra(
|
|
|
357
365
|
const previous = await loadCollectionFiles(manifestPath)
|
|
358
366
|
const files = new Map<string, ESNavarraArchiveEntry>()
|
|
359
367
|
|
|
360
|
-
// An entry for a partition this run does not consider is
|
|
361
|
-
//
|
|
368
|
+
// An entry for a partition this run does not consider is kept, so a bounded
|
|
369
|
+
// run never drops what an earlier run recorded.
|
|
362
370
|
for (const [filename, entry] of previous) {
|
|
363
371
|
files.set(filename, entry as ESNavarraArchiveEntry)
|
|
364
372
|
}
|
|
@@ -390,7 +398,7 @@ export async function harvestESNavarra(
|
|
|
390
398
|
|
|
391
399
|
const dest = destDir(partition.filename)
|
|
392
400
|
|
|
393
|
-
if (await isCurrent(files.get(partition.filename), partition, dest, options.verifyDigests ?? false)) {
|
|
401
|
+
if (await isCurrent(files.get(partition.filename) ?? null, partition, dest, options.verifyDigests ?? false)) {
|
|
394
402
|
skipped++
|
|
395
403
|
|
|
396
404
|
continue
|
|
@@ -454,7 +462,7 @@ export interface FetchESNavarraOptions
|
|
|
454
462
|
/**
|
|
455
463
|
* The minimum spacing between two requests, in milliseconds.
|
|
456
464
|
*
|
|
457
|
-
*
|
|
465
|
+
* @defaultValue {@linkcode ES_NAVARRA_REQUEST_INTERVAL_MS}
|
|
458
466
|
*/
|
|
459
467
|
minRequestIntervalMs?: number
|
|
460
468
|
}
|
|
@@ -17,43 +17,40 @@
|
|
|
17
17
|
* every name column bare.
|
|
18
18
|
*
|
|
19
19
|
* A row's country comes from `municipality_number` rather than from the caller. Åland is part of
|
|
20
|
-
* Finland and its sixteen municipalities
|
|
21
|
-
* holds both jurisdictions and {@linkcode ALAND_MUNICIPALITIES}
|
|
20
|
+
* Finland and its sixteen municipalities have Finnish municipality numbers, so one national file
|
|
21
|
+
* holds both jurisdictions and {@linkcode ALAND_MUNICIPALITIES} separates them.
|
|
22
22
|
*
|
|
23
23
|
* The street and the house number are read from the publisher's own part columns rather than from
|
|
24
24
|
* its inline address text.
|
|
25
25
|
* `address_name_fin` is the whole of `address_fin` or a space-delimited prefix of it, and the four
|
|
26
26
|
* number-part columns reproduce the publisher's inline number.
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
* composed number equals the inline remainder byte for byte on all 265,830 numbered rows.
|
|
30
|
-
* No row publishes inline address text with both name columns empty,
|
|
31
|
-
* so reading the name columns loses no address.
|
|
27
|
+
* No row publishes inline address text with both name columns empty, so the part columns lose no
|
|
28
|
+
* address.
|
|
32
29
|
*
|
|
33
|
-
* `address_number` is not a house number. It is a 1-based ordinal over the addresses
|
|
34
|
-
* `building_key
|
|
35
|
-
*
|
|
30
|
+
* `address_number` is not a house number. It is a 1-based ordinal over the addresses that share one
|
|
31
|
+
* `building_key`. That is how a corner building's several frontages are numbered. The adapter
|
|
32
|
+
* ignores it.
|
|
36
33
|
*
|
|
37
|
-
* The file is bilingual, and which column
|
|
34
|
+
* The file is bilingual, and which column holds which language depends on the municipality. On
|
|
38
35
|
* mainland Finland `address_name_fin` is the Finnish name and `address_name_swe` the Swedish one. On
|
|
39
36
|
* Åland the `*_fin` columns, where populated at all, hold the **Swedish** name: `address_fin` reads
|
|
40
37
|
* `Skogshyddsvägen 11`.
|
|
41
38
|
* So the adapter prefers `*_swe` on Åland and `*_fin` on the mainland, and emits one row per
|
|
42
39
|
* published record from the preferred language's columns.
|
|
43
|
-
* `locale`
|
|
44
|
-
* name came from
|
|
45
|
-
* `fi-FI` is a mainland record
|
|
46
|
-
* `sv-FI` is a mainland record that populates `address_name_swe`
|
|
40
|
+
* `locale` states the language of the name it emitted rather than the language of the column that
|
|
41
|
+
* name came from. That gives three values.
|
|
42
|
+
* `fi-FI` is a mainland record whose name came from `address_name_fin`.
|
|
43
|
+
* `sv-FI` is a mainland record that populates `address_name_swe` only. That is the form a
|
|
47
44
|
* Swedish-speaking municipality publishes.
|
|
48
45
|
* `sv-AX` is every Åland record, because the Swedish columns are the populated ones there and the
|
|
49
46
|
* `*_fin` columns hold the Swedish name where they are populated at all.
|
|
50
47
|
* A mainland record's Swedish name, where the record publishes both, is not emitted as a second
|
|
51
|
-
* row: a record is one premise, and `CanonicalRow`
|
|
48
|
+
* row: a record is one premise, and `CanonicalRow` holds one written surface.
|
|
52
49
|
*
|
|
53
50
|
* SYKE's metadata record `{DBD610F4-3392-44CD-B601-BAE8FA547A57}` grants CC BY 4.0 for its open data
|
|
54
51
|
* and the register's two elected decisions for this file record `CC-BY-4.0` in their `spdx` field, so
|
|
55
52
|
* the adapter records that license on every row. SYKE states the attribution it wants as
|
|
56
|
-
* `Lähde: Syke Ryhti`, and the model card must
|
|
53
|
+
* `Lähde: Syke Ryhti`, and the model card must state it.
|
|
57
54
|
*
|
|
58
55
|
* The adapter streams with `CSVSpliterator.fromAsync`, so the 3.8M-row file never sits in memory. It
|
|
59
56
|
* honors `opts.limit`, `opts.signal` and `opts.country`.
|
|
@@ -79,7 +76,6 @@ export const RYHTI_ADAPTER_ID = "ryhti"
|
|
|
79
76
|
* The sixteen `municipality_number` values of Åland's municipalities.
|
|
80
77
|
*
|
|
81
78
|
* Every other number in the file belongs to a mainland Finnish municipality.
|
|
82
|
-
* The file holds 308 distinct values, and these sixteen cover 32,518 of its 3,862,609 rows.
|
|
83
79
|
*
|
|
84
80
|
* The column is a zero-padded three-digit string, so a reader compares the padded form.
|
|
85
81
|
*/
|
|
@@ -154,8 +150,8 @@ export function countryOfFinnishMunicipality(municipalityNumber: string): string
|
|
|
154
150
|
* as `184-183b` across a second pair of the same two columns.
|
|
155
151
|
* The second pair is appended after a hyphen when either of its halves is populated.
|
|
156
152
|
*
|
|
157
|
-
* @returns The composed number, or an empty string for a row with no number at all
|
|
158
|
-
*
|
|
153
|
+
* @returns The composed number, or an empty string for a row with no number at all.
|
|
154
|
+
* A caller treats that empty string as a place name rather than a number.
|
|
159
155
|
*/
|
|
160
156
|
export function composeRyhtiHouseNumber(record: Partial<RyhtiNumberParts>): string {
|
|
161
157
|
const first = composeHouseNumber(
|
|
@@ -186,7 +182,7 @@ interface NamedSurface {
|
|
|
186
182
|
}
|
|
187
183
|
|
|
188
184
|
/**
|
|
189
|
-
* The surface to emit for one record, or `
|
|
185
|
+
* The surface to emit for one record, or `null` for a record with no street.
|
|
190
186
|
*
|
|
191
187
|
* On Åland the Swedish columns are preferred, and the Finnish ones are read as Swedish.
|
|
192
188
|
* That is the language they hold there.
|
|
@@ -195,11 +191,11 @@ interface NamedSurface {
|
|
|
195
191
|
* The Swedish ones stand in where `address_name_fin` is empty.
|
|
196
192
|
* That is the form a Swedish-speaking municipality publishes.
|
|
197
193
|
*
|
|
198
|
-
* A language whose name column is empty
|
|
194
|
+
* A language whose name column is empty has no street.
|
|
199
195
|
* Seven mainland rows publish an `address_swe` of a bare number with an empty `address_name_swe`.
|
|
200
196
|
* This refuses them rather than giving them an empty street.
|
|
201
197
|
*/
|
|
202
|
-
function surfaceOf(record: RyhtiRow, country: string): NamedSurface |
|
|
198
|
+
function surfaceOf(record: RyhtiRow, country: string): NamedSurface | null {
|
|
203
199
|
const finnish = {
|
|
204
200
|
street: (record.address_name_fin ?? "").trim(),
|
|
205
201
|
locality: (record.postal_office_fin ?? "").trim(),
|
|
@@ -213,12 +209,12 @@ function surfaceOf(record: RyhtiRow, country: string): NamedSurface | undefined
|
|
|
213
209
|
if (country === "AX") {
|
|
214
210
|
const preferred = swedish.street ? swedish : finnish
|
|
215
211
|
|
|
216
|
-
return preferred.street ? { ...preferred, language: "sv" } :
|
|
212
|
+
return preferred.street ? { ...preferred, language: "sv" } : null
|
|
217
213
|
}
|
|
218
214
|
|
|
219
215
|
if (finnish.street) return { ...finnish, language: "fi" }
|
|
220
216
|
|
|
221
|
-
return swedish.street ? { ...swedish, language: "sv" } :
|
|
217
|
+
return swedish.street ? { ...swedish, language: "sv" } : null
|
|
222
218
|
}
|
|
223
219
|
|
|
224
220
|
export function createRyhtiAdapter(): CorpusAdapter {
|
|
@@ -251,7 +247,7 @@ export function createRyhtiAdapter(): CorpusAdapter {
|
|
|
251
247
|
|
|
252
248
|
const country = countryOfFinnishMunicipality(record.municipality_number ?? "")
|
|
253
249
|
|
|
254
|
-
// A caller that
|
|
250
|
+
// A caller that requested one jurisdiction gets that jurisdiction's rows,
|
|
255
251
|
// so the one national file can be read once per country.
|
|
256
252
|
if (opts.country && country !== opts.country) continue
|
|
257
253
|
|
|
@@ -14,31 +14,27 @@
|
|
|
14
14
|
* this module sits beside `#fr/tools/fetch/ban` rather than in an `sdk/` root the workspace does
|
|
15
15
|
* not declare.
|
|
16
16
|
*
|
|
17
|
-
* SYKE's metadata record `{DBD610F4-3392-44CD-B601-BAE8FA547A57}` grants CC BY 4.0
|
|
18
|
-
*
|
|
17
|
+
* SYKE's metadata record `{DBD610F4-3392-44CD-B601-BAE8FA547A57}` grants CC BY 4.0 and states the
|
|
18
|
+
* attribution as `Lähde: Syke Ryhti`. The address-source register elects that license. The manifest
|
|
19
19
|
* records both, and `#fi/adapters/ryhti/adapter` records the license on every row.
|
|
20
20
|
*
|
|
21
21
|
* Four measured properties decide what this module does:
|
|
22
22
|
*
|
|
23
|
-
* 1. **The file is republished daily and its length changes with it.** `HEAD`
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
* a `HEAD` compared against the manifest, which costs one small request instead of 351 MB, and
|
|
27
|
-
* neither number is written into this module as a constant.
|
|
23
|
+
* 1. **The file is republished daily and its length changes with it.** The re-run check is a `HEAD`
|
|
24
|
+
* compared against the manifest, so one small request replaces the whole body and no length is a
|
|
25
|
+
* constant here.
|
|
28
26
|
* 2. **The host ignores `Range`.** `bytes=0-4095` answers HTTP 200 with no `content-range` and the
|
|
29
27
|
* whole `content-length`, and `accept-ranges` is absent, so a resumed transfer is not available
|
|
30
28
|
* and an interrupted one restarts. {@linkcode resumableDownload} would refuse the response.
|
|
31
|
-
* 3. **This module moves bytes and decodes none of them.** The body is streamed to disk compressed
|
|
32
|
-
* decompressed stream to stream, so no chunk boundary is
|
|
33
|
-
* boundary. A `chunk.toString("utf8")` per gzip chunk
|
|
34
|
-
*
|
|
35
|
-
* `EF BF BD` with 0 found: the replacement characters such a reader reports are its own.
|
|
29
|
+
* 3. **This module moves bytes and decodes none of them.** The body is streamed to disk compressed
|
|
30
|
+
* and then decompressed stream to stream, so no chunk boundary is interpreted as a character
|
|
31
|
+
* boundary. A `chunk.toString("utf8")` per gzip chunk would write U+FFFD at every split
|
|
32
|
+
* multi-byte character.
|
|
36
33
|
* 4. **The decompressed CSV is what the adapter can read.** `#fi/adapters/ryhti/adapter` hands
|
|
37
|
-
* `opts.inputPath` to `CSVSpliterator.fromAsync`, which reads the bytes
|
|
38
|
-
*
|
|
39
|
-
* written: 793 MB of readable input instead of 351 MB plus a reader the adapter does not have.
|
|
34
|
+
* `opts.inputPath` to `CSVSpliterator.fromAsync`, which reads the bytes on disk. A path to the
|
|
35
|
+
* `.gz` would be parsed as CSV over gzip bytes, so the `.gz` is removed once the CSV is written.
|
|
40
36
|
* A caller that wants the compressed copy kept passes
|
|
41
|
-
* {@linkcode FetchRyhtiOptions.keepCompressed}
|
|
37
|
+
* {@linkcode FetchRyhtiOptions.keepCompressed}.
|
|
42
38
|
*/
|
|
43
39
|
|
|
44
40
|
/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
|
|
@@ -47,7 +43,7 @@
|
|
|
47
43
|
import { APIClient } from "@mailwoman/core/api"
|
|
48
44
|
import { gunzipChunks } from "@mailwoman/core/fs/compression"
|
|
49
45
|
import { ByteFormatter } from "@mailwoman/core/fs/formatters"
|
|
50
|
-
import { tryStat } from "@mailwoman/core/fs/readers"
|
|
46
|
+
import { tryStat } from "@mailwoman/core/fs/readers/stat"
|
|
51
47
|
import { openReadStream, openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams"
|
|
52
48
|
import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
|
|
53
49
|
import { sha256File } from "@mailwoman/core/hash"
|
|
@@ -64,7 +60,7 @@ import { assertHeaderColumns, readDelimitedHeader } from "#tools/fetch/header"
|
|
|
64
60
|
export const FI_RYHTI_CSV_URL = "https://paikkatiedot.ymparisto.fi/geoserver/www/open_address.csv.gz"
|
|
65
61
|
|
|
66
62
|
/**
|
|
67
|
-
* The
|
|
63
|
+
* The download directory, the adapter's `inputPath`'s parent.
|
|
68
64
|
*/
|
|
69
65
|
const SLUG = RYHTI_ADAPTER_ID
|
|
70
66
|
|
|
@@ -84,7 +80,8 @@ const COMPRESSED_FILENAME = "open_address.csv.gz"
|
|
|
84
80
|
export const FI_RYHTI_LICENSE = "CC-BY-4.0"
|
|
85
81
|
|
|
86
82
|
/**
|
|
87
|
-
* The attribution SYKE states for its open data
|
|
83
|
+
* The attribution SYKE states for its open data.
|
|
84
|
+
* The model card must record it.
|
|
88
85
|
*/
|
|
89
86
|
export const FI_RYHTI_ATTRIBUTION = "Lähde: Syke Ryhti"
|
|
90
87
|
|
|
@@ -115,8 +112,9 @@ export const FI_RYHTI_REQUIRED_COLUMNS: readonly string[] = [
|
|
|
115
112
|
/**
|
|
116
113
|
* What the manifest records about the one file.
|
|
117
114
|
*
|
|
118
|
-
* {@linkcode SourceManifest}'s five fields describe the decompressed CSV
|
|
119
|
-
*
|
|
115
|
+
* {@linkcode SourceManifest}'s five fields describe the decompressed CSV.
|
|
116
|
+
* That file is what the adapter reads and what the digest covers.
|
|
117
|
+
*
|
|
120
118
|
* The three added fields describe the transfer, and they are what the re-run
|
|
121
119
|
* check compares before it downloads 351 MB.
|
|
122
120
|
*/
|
|
@@ -132,7 +130,7 @@ export interface RyhtiManifest extends SourceManifest {
|
|
|
132
130
|
*/
|
|
133
131
|
compressed_bytes: number | null
|
|
134
132
|
/**
|
|
135
|
-
* The columns the header
|
|
133
|
+
* The columns the header held when this file was downloaded, in order.
|
|
136
134
|
*
|
|
137
135
|
* Recorded rather than counted, so a later edition's added, removed or renamed
|
|
138
136
|
* column is a diff against this list rather than a count that moved.
|
|
@@ -165,7 +163,7 @@ export async function readRyhtiHead(client: Pick<APIClient, "fetch">, url = FI_R
|
|
|
165
163
|
|
|
166
164
|
return {
|
|
167
165
|
lastModified: headers?.["last-modified"] ?? null,
|
|
168
|
-
contentLength: length
|
|
166
|
+
contentLength: length && /^\d+$/u.test(length) ? Number(length) : null,
|
|
169
167
|
contentType: headers?.["content-type"] ?? null,
|
|
170
168
|
}
|
|
171
169
|
}
|
|
@@ -202,13 +200,14 @@ export interface DownloadRyhtiOptions {
|
|
|
202
200
|
* Keep the compressed copy beside the CSV.
|
|
203
201
|
*
|
|
204
202
|
* The adapter reads the CSV, so the default removes the archive once the CSV is written.
|
|
205
|
-
*
|
|
203
|
+
* Both together hold 1.14 GB for the one file.
|
|
206
204
|
*/
|
|
207
205
|
keepCompressed?: boolean
|
|
208
206
|
/**
|
|
209
207
|
* Re-read the CSV's sha256 on a re-run instead of comparing its byte count.
|
|
210
208
|
*
|
|
211
|
-
* The default compares the recorded byte count against the file's size
|
|
209
|
+
* The default compares the recorded byte count against the file's size.
|
|
210
|
+
* That check is one `stat`.
|
|
212
211
|
* This re-hashes 793 MB.
|
|
213
212
|
*/
|
|
214
213
|
verifyDigest?: boolean
|
|
@@ -218,14 +217,15 @@ export interface DownloadRyhtiOptions {
|
|
|
218
217
|
force?: boolean
|
|
219
218
|
/**
|
|
220
219
|
* Attempts after the first for the one transfer.
|
|
221
|
-
* Defaults to three.
|
|
222
220
|
*
|
|
223
221
|
* The host ignores `Range`, so each attempt starts the 351 MB again.
|
|
222
|
+
*
|
|
223
|
+
* @defaultValue three.
|
|
224
224
|
*/
|
|
225
225
|
retries?: number
|
|
226
226
|
/**
|
|
227
227
|
* Pause between attempts, in milliseconds.
|
|
228
|
-
*
|
|
228
|
+
* @defaultValue {@linkcode DEFAULT_RETRY_DELAY_MS}
|
|
229
229
|
*/
|
|
230
230
|
retryDelayMs?: number
|
|
231
231
|
signal?: AbortSignal
|
|
@@ -258,9 +258,9 @@ async function isCurrent(
|
|
|
258
258
|
if (!recorded) return false
|
|
259
259
|
|
|
260
260
|
// A host that states neither cannot be compared against, so the file is downloaded.
|
|
261
|
-
if (head.lastModified
|
|
261
|
+
if (!head.lastModified && head.contentLength === null) return false
|
|
262
262
|
|
|
263
|
-
if (head.lastModified
|
|
263
|
+
if (head.lastModified && head.lastModified !== recorded.last_modified) return false
|
|
264
264
|
|
|
265
265
|
if (head.contentLength !== null && head.contentLength !== recorded.compressed_bytes) return false
|
|
266
266
|
|
|
@@ -344,8 +344,8 @@ export async function downloadRyhti(
|
|
|
344
344
|
const contentType = response.headers.get("content-type") ?? ""
|
|
345
345
|
|
|
346
346
|
// The host serves `application/x-gzip`.
|
|
347
|
-
// An `text/html` body under http 200 is a portal page
|
|
348
|
-
//
|
|
347
|
+
// An `text/html` body under http 200 is a portal page.
|
|
348
|
+
// A portal page saved to disk would fail later as a corrupt archive.
|
|
349
349
|
if (contentType.includes("text/html")) {
|
|
350
350
|
throw new Error(`${FI_RYHTI_CSV_URL} answered ${contentType}, which is a page rather than the file`)
|
|
351
351
|
}
|
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
* `/api/explore/v2.1/catalog/datasets/fr-en-annuaire-education/exports/jsonl`. The export keys each
|
|
12
12
|
* record by the dataset's field names (`nom_etablissement`, `adresse_1`, `adresse_2`, `adresse_3`,
|
|
13
13
|
* `code_postal`, `nom_commune`, `code_departement`, `etat`), and {@linkcode AnnuaireRecord} declares
|
|
14
|
-
* the ones this adapter reads. `fetchAnnuaireEducationOverseas` asks the export for the overseas
|
|
15
|
-
*
|
|
14
|
+
* the ones this adapter reads. `fetchAnnuaireEducationOverseas` asks the export for the overseas
|
|
15
|
+
* departments only, and the adapter filters again, so a whole-country export reads the same.
|
|
16
16
|
*
|
|
17
17
|
* ## Which rows are read
|
|
18
18
|
*
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
* ## Organizations only
|
|
26
26
|
*
|
|
27
27
|
* Every record is an education establishment or office. One private school among the overseas
|
|
28
|
-
* records writes its address care of
|
|
28
|
+
* records writes its address care of one person (`Chez …`), which places it at that person's
|
|
29
29
|
* premises, so a line opening with `Chez` refuses the row under {@linkcode AnnuaireRefusal.CareOfPerson}.
|
|
30
30
|
*
|
|
31
31
|
* ## The three address lines
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
* what it holds rather than by its position:
|
|
36
36
|
*
|
|
37
37
|
* - **A street.** `5 rue Maréchal Leclerc` opens with a number and a street-type word, and
|
|
38
|
-
* `Rue Marcel Bonin` with the word
|
|
38
|
+
* `Rue Marcel Bonin` with the word only. The word becomes `street_prefix` through
|
|
39
39
|
* {@linkcode splitFrenchStreetType}, which reads the BAN reader's dictionary and then TRE-R35's
|
|
40
40
|
* street types for the words it lacks (`quartier`, `lotissement`, `ZAC`). A number of `0` is the
|
|
41
41
|
* publisher's placeholder for none. `Bourg` or `Village` written by itself refers to the
|
|
@@ -79,8 +79,8 @@
|
|
|
79
79
|
* transformer, pour créer des « Informations dérivées », des produits ou des services », « Sous
|
|
80
80
|
* réserve de : mentionner la paternité de l'« Information » : sa source (au moins le nom du «
|
|
81
81
|
* Concédant ») et la date de dernière mise à jour de l'« Information » réutilisée. » Every row
|
|
82
|
-
* records the license, and {@linkcode FR_ANNUAIRE_EDUCATION_ATTRIBUTION} is the credit
|
|
83
|
-
*
|
|
82
|
+
* records the license, and {@linkcode FR_ANNUAIRE_EDUCATION_ATTRIBUTION} is the credit. The model
|
|
83
|
+
* card completes it with the update date the fetch manifest records.
|
|
84
84
|
*
|
|
85
85
|
* The adapter honors `opts.limit`, `opts.signal` and `opts.country`, and counts every refusal
|
|
86
86
|
* under its reason in `opts.dropped`.
|
|
@@ -125,7 +125,7 @@ export const FR_ANNUAIRE_EDUCATION_ATTRIBUTION =
|
|
|
125
125
|
|
|
126
126
|
/**
|
|
127
127
|
* The overseas INSEE departments this adapter reads: every prefix in `COUNTRY_BY_INSEE_DEPARTMENT`
|
|
128
|
-
* except `984`, the French Southern Territories,
|
|
128
|
+
* except `984`, the French Southern Territories, where there are no schools.
|
|
129
129
|
*/
|
|
130
130
|
export const FR_ANNUAIRE_EDUCATION_DEPARTEMENTS: readonly string[] = [
|
|
131
131
|
"971",
|
|
@@ -206,7 +206,7 @@ export const AnnuaireRefusal = {
|
|
|
206
206
|
*/
|
|
207
207
|
DeliveryLineAbsent: "row:delivery-line-absent",
|
|
208
208
|
/**
|
|
209
|
-
* The jurisdiction's layout does not print every component the row
|
|
209
|
+
* The jurisdiction's layout does not print every component the row holds.
|
|
210
210
|
*/
|
|
211
211
|
Unrenderable: "row:unrenderable",
|
|
212
212
|
/**
|
|
@@ -252,7 +252,8 @@ const ARCHIPELAGO_CODE = /^(?:idv|islv|tg|marq|australes)(?![\p{L}\d])/iu
|
|
|
252
252
|
const LEADING_NUMBER = /^(\d+)(?:\s*(bis|ter|quater)(?![\p{L}])|\s?([A-Za-z])(?![\p{L}'’]))?[\s,]+(.+)$/iu
|
|
253
253
|
|
|
254
254
|
/**
|
|
255
|
-
* A leading `Lieu-dit` marker
|
|
255
|
+
* A leading `Lieu-dit` marker.
|
|
256
|
+
* A place line drops it.
|
|
256
257
|
*/
|
|
257
258
|
const LIEU_DIT_MARKER = /^lieu[- ]dit\s+/iu
|
|
258
259
|
|
|
@@ -299,7 +300,7 @@ const THOROUGHFARE_WORDS: ReadonlySet<string> = new Set([
|
|
|
299
300
|
])
|
|
300
301
|
|
|
301
302
|
/**
|
|
302
|
-
* Street-type words that
|
|
303
|
+
* Street-type words that stand for a settlement's center: `Bourg`, `Village`.
|
|
303
304
|
*/
|
|
304
305
|
const LONE_PLACE_WORDS: ReadonlySet<string> = new Set(["bourg", "village", "le bourg"])
|
|
305
306
|
|
|
@@ -347,7 +348,8 @@ export function readAnnuaireLine(line: string): LineReading {
|
|
|
347
348
|
|
|
348
349
|
if (split) {
|
|
349
350
|
if (!split.street) {
|
|
350
|
-
// `Bourg` and `Village`
|
|
351
|
+
// `Bourg` and `Village` on their own refer to the settlement's center.
|
|
352
|
+
// That center is a place.
|
|
351
353
|
if (!house && split.street_prefix && LONE_PLACE_WORDS.has(foldToken(split.street_prefix))) {
|
|
352
354
|
return { kind: "place", place: split.street_prefix }
|
|
353
355
|
}
|
|
@@ -7,12 +7,12 @@
|
|
|
7
7
|
*
|
|
8
8
|
* Input is a semicolon-separated CSV dump from `adresse.data.gouv.fr`. The adapter reads `numero`
|
|
9
9
|
* as `house_number`, `rep` as a repetition index appended to it, `nom_voie` as `street`,
|
|
10
|
-
* `code_postal` as `postcode`, and `nom_commune` as `locality`. BAN
|
|
10
|
+
* `code_postal` as `postcode`, and `nom_commune` as `locality`. BAN holds no region column, so
|
|
11
11
|
* region is left to the wof-postalcode and wof-admin cross-reference at corpus build time.
|
|
12
12
|
*
|
|
13
13
|
* IGN publishes one file per INSEE department beside the whole-country extract, and the overseas
|
|
14
|
-
* departments
|
|
15
|
-
* derived from `code_insee` rather than asserted by the caller
|
|
14
|
+
* departments hold the same 23-column header as the metropolitan rows. A row's country is therefore
|
|
15
|
+
* derived from `code_insee` rather than asserted by the caller. That derivation reads the same whether the
|
|
16
16
|
* input is one department's file or an extract spanning several.
|
|
17
17
|
*
|
|
18
18
|
* The official BAN is dual-licensed under Licence Ouverte 2.0 and ODbL.
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
*
|
|
22
22
|
* The adapter streams with `CSVSpliterator.fromAsync`, so a 25M-row file never sits in memory. It
|
|
23
23
|
* honors `opts.limit` and `opts.signal`. `opts.country` is optional: when set it both rejects a
|
|
24
|
-
* jurisdiction BAN does not publish and keeps only the rows of the one
|
|
24
|
+
* jurisdiction BAN does not publish and keeps only the rows of the one requested.
|
|
25
25
|
*/
|
|
26
26
|
|
|
27
27
|
import { formatAddressRow } from "@mailwoman/codex/address/format"
|
|
@@ -94,7 +94,7 @@ export function createBanAdapter(): CorpusAdapter {
|
|
|
94
94
|
|
|
95
95
|
const country = countryOfInseeCode(record.code_insee ?? "")
|
|
96
96
|
|
|
97
|
-
// A caller that
|
|
97
|
+
// A caller that requested one jurisdiction gets that jurisdiction's rows,
|
|
98
98
|
// so one extract spanning several departments can be read once per country.
|
|
99
99
|
if (opts.country && country !== opts.country) continue
|
|
100
100
|
|
|
@@ -167,7 +167,7 @@ export function createBanAdapter(): CorpusAdapter {
|
|
|
167
167
|
components: aligned,
|
|
168
168
|
country,
|
|
169
169
|
// Every jurisdiction BAN publishes writes its addresses in French,
|
|
170
|
-
// so the language is constant and the region
|
|
170
|
+
// so the language is constant and the region holds the jurisdiction.
|
|
171
171
|
locale: `fr-${country}`,
|
|
172
172
|
source: BAN_ADAPTER_ID,
|
|
173
173
|
source_id: sourceID,
|