@mailwoman/corpus 10.1.0 → 10.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/licenses/cdla-permissive-2.0.md +1 -1
- package/lib/adapters/cordis/adapter.ts +13 -7
- package/lib/adapters/dedup-key.ts +1 -1
- package/lib/adapters/geonames/adapter.ts +1 -1
- package/lib/adapters/gleif/adapter.ts +17 -16
- package/lib/adapters/libpostal-dictionary.ts +0 -2
- package/lib/adapters/openaddresses/adapter.ts +4 -3
- package/lib/adapters/osm/adapter.ts +22 -21
- package/lib/adapters/po-box/adapter.ts +10 -5
- package/lib/adapters/registry.ts +4 -4
- package/lib/adapters/source-id.ts +2 -2
- package/lib/adapters/street-line.ts +7 -6
- package/lib/adapters/wof/admin/json/adapter.ts +2 -2
- package/lib/adapters/wof/json-rows.ts +2 -1
- package/lib/adapters/wof/postalcode-json/adapter.ts +6 -2
- package/lib/adapters.ts +5 -5
- package/lib/au/adapters/gnaf/adapter.ts +1 -1
- package/lib/be/adapters/brussels/adapter.ts +22 -22
- package/lib/be/adapters/vlaanderen/adapter.ts +54 -57
- package/lib/be/adapters/wallonie/adapter.ts +17 -17
- package/lib/be/tools/fetch/brussels.ts +13 -10
- package/lib/be/tools/fetch/wallonie.ts +8 -7
- package/lib/build/adapters.ts +2 -1
- package/lib/build/align.ts +5 -4
- package/lib/build/checkpoint.ts +5 -3
- package/lib/build/eligibility.ts +3 -3
- package/lib/build/types.ts +8 -6
- package/lib/cz/adapters/cuzk/adapter.ts +41 -38
- package/lib/cz/tools/fetch/cuzk.ts +42 -31
- package/lib/de/recipes/locale.ts +1 -1
- package/lib/dk/adapters/inspire/adapter.ts +38 -38
- package/lib/dk/tools/fetch/inspire.ts +16 -14
- package/lib/ee/adapters/ads/adapter.ts +26 -32
- package/lib/ee/tools/fetch/ads.ts +10 -8
- package/lib/es/adapters/bizkaia/adapter.ts +47 -47
- package/lib/es/adapters/catastro/adapter.ts +36 -34
- package/lib/es/adapters/gipuzkoa/adapter.ts +38 -45
- package/lib/es/adapters/navarra/adapter.ts +28 -27
- package/lib/es/cadastre.ts +31 -31
- package/lib/es/tools/fetch/bizkaia.ts +40 -36
- package/lib/es/tools/fetch/catastro.ts +51 -40
- package/lib/es/tools/fetch/gipuzkoa.ts +13 -11
- package/lib/es/tools/fetch/navarra.ts +33 -25
- package/lib/fi/adapters/ryhti/adapter.ts +22 -26
- package/lib/fi/tools/fetch/ryhti.ts +31 -31
- package/lib/fr/adapters/annuaire-education/adapter.ts +13 -11
- package/lib/fr/adapters/ban/adapter.ts +6 -6
- package/lib/fr/adapters/finess/adapter.ts +16 -15
- package/lib/fr/adapters/finess/voie-type.ts +4 -3
- package/lib/fr/insee-country.ts +2 -2
- package/lib/fr/recipes/lieudit.ts +8 -6
- package/lib/fr/tools/fetch/annuaire-education.ts +11 -8
- package/lib/fr/tools/fetch/ban.ts +3 -2
- package/lib/fr/tools/fetch/finess.ts +14 -13
- package/lib/inspire/address.ts +86 -78
- package/lib/inspire/errors.ts +9 -8
- package/lib/inspire/stream.ts +29 -26
- package/lib/international/recipes/locale.ts +13 -9
- package/lib/it/adapters/anac/adapter.ts +51 -46
- package/lib/it/tools/fetch/anac.ts +22 -25
- package/lib/jp/adapters/wof-admin/adapter.ts +2 -6
- package/lib/jp/tools/fetch/houjin.ts +3 -3
- package/lib/kr/adapters/localdata/permit-rows.ts +1 -1
- package/lib/kr/tools/fetch/juso.ts +1 -1
- package/lib/kr/tools/fetch/localdata.ts +2 -2
- package/lib/lu/adapters/bd-adresses/adapter.ts +10 -10
- package/lib/lu/tools/fetch/bd-adresses.ts +15 -13
- package/lib/lv/adapters/varis/adapter.ts +24 -23
- package/lib/lv/tools/fetch/varis.ts +3 -2
- package/lib/nl/adapters/kadaster/adapter.ts +29 -29
- package/lib/nl/tools/fetch/kadaster.ts +16 -12
- package/lib/no/adapters/matrikkelen/adapter.ts +6 -5
- package/lib/no/tools/fetch/matrikkelen.ts +59 -47
- package/lib/parquet/duckdb.ts +7 -4
- package/lib/parquet/readers.ts +3 -3
- package/lib/parquet/schema.ts +1 -1
- package/lib/parquet/writers.ts +3 -2
- package/lib/pl/adapters/emuia/adapter.ts +12 -12
- package/lib/pl/tools/fetch/emuia.ts +10 -9
- package/lib/recipes/bare/postcode.ts +1 -1
- package/lib/recipes/boundary-stress.ts +1 -1
- package/lib/recipes/house-venue.ts +7 -3
- package/lib/recipes/intersection.ts +2 -2
- package/lib/recipes/po/box/cedex/recipe.ts +3 -3
- package/lib/recipes/po/box/cedex/sources.ts +13 -1
- package/lib/recipes/scaffold.ts +31 -21
- package/lib/recipes/sources.ts +16 -3
- package/lib/recipes/street/bare.ts +2 -2
- package/lib/recipes/street.ts +3 -1
- package/lib/recipes/sub/venue/context.ts +2 -0
- package/lib/recipes/sub/venue/render.ts +1 -1
- package/lib/recipes/sub/venue/sources.ts +24 -29
- package/lib/recipes/sub/venue.ts +12 -18
- package/lib/recipes/trailing-region.ts +6 -2
- package/lib/recipes/unit.ts +3 -3
- package/lib/recipes/venue.ts +347 -0
- package/lib/recipes.ts +4 -2
- package/lib/registers.ts +8 -6
- package/lib/ro/adapters/retea-scolara/adapter.ts +5 -5
- package/lib/ro/tools/fetch/retea-scolara.ts +1 -1
- package/lib/runner.ts +4 -3
- package/lib/sg/recipes/register.ts +1 -1
- package/lib/sg/tools/fetch/acra.ts +3 -3
- package/lib/sk/adapters/inspire/adapter.ts +25 -27
- package/lib/sk/tools/fetch/inspire.ts +15 -14
- package/lib/source-register/effective-manifest.ts +3 -3
- package/lib/source-register/register.ts +3 -3
- package/lib/source-register/training-manifest.ts +1 -1
- package/lib/source-register/types.ts +12 -13
- package/lib/subvenue/promotions.ts +1 -1
- package/lib/subvenue/table.ts +4 -4
- package/lib/surfaces/locale.ts +2 -0
- package/lib/synthesizers/anchor-absorption.ts +2 -1
- package/lib/synthesizers/boundary-stress.ts +20 -26
- package/lib/synthesizers/house-venue.ts +26 -30
- package/lib/synthesizers/no-street.ts +2 -1
- package/lib/synthesizers/po-box.ts +6 -3
- package/lib/synthesizers/tail.ts +55 -0
- package/lib/synthesizers/utils.ts +19 -2
- package/lib/test-kit.ts +2 -2
- package/lib/tools/audit.ts +7 -5
- package/lib/tools/corpus-stats.ts +1 -1
- package/lib/tools/fetch/atom.ts +10 -10
- package/lib/tools/fetch/cordis.ts +11 -9
- package/lib/tools/fetch/download/manifest.ts +2 -1
- package/lib/tools/fetch/download/network.ts +4 -4
- package/lib/tools/fetch/download/retry.ts +4 -4
- package/lib/tools/fetch/geonames/dump.ts +5 -3
- package/lib/tools/fetch/geonames/postal.ts +2 -2
- package/lib/tools/fetch/gleif.ts +6 -4
- package/lib/tools/fetch/header.ts +8 -6
- package/lib/tools/fetch/inspire-addresses.ts +41 -36
- package/lib/tools/fetch/openaddresses.ts +1 -1
- package/lib/tools/fetch/vlaanderen-ad.ts +17 -17
- package/lib/tools/fetch/wfs-harvest.ts +39 -28
- package/lib/tools/fetch.ts +0 -42
- package/lib/tools/golden/expand.ts +8 -8
- package/lib/tools/golden/promote.ts +12 -8
- package/lib/tools/golden/relabel-street/directory.ts +7 -6
- package/lib/tools/golden/relabel-street/row.ts +7 -6
- package/lib/tools/ingest-csv.ts +3 -3
- package/lib/tools/jsonl-to-parquet.ts +1 -1
- package/lib/tools/lint/recipe-output/vocab.ts +10 -5
- package/lib/tools/lint/recipe-output.ts +26 -8
- package/lib/tools/overlay/kryptonite.ts +4 -3
- package/lib/tools/overlay/manifest.ts +4 -2
- package/lib/tools/overlay/merge-source.ts +2 -1
- package/lib/tools/overlay/split-slice.ts +2 -2
- package/lib/tools/overlay/translit.ts +6 -4
- package/lib/tools/overture-subvenue.ts +1 -1
- package/lib/tools/postcode-triples.ts +5 -4
- package/lib/tools/source-register/build.ts +17 -14
- package/lib/tools/sub/venue/lexicon.ts +4 -8
- package/lib/tw/tools/fetch/gcis.ts +4 -4
- package/lib/types.ts +4 -3
- package/lib/us/adapters/state/hi-schools/workbook.ts +1 -1
- package/lib/us/adapters/usgov/nad/adapter.ts +11 -11
- package/lib/us/adapters/usgov/samhsa-treatment-locator/adapter.ts +4 -4
- package/lib/us/fips-state.ts +2 -2
- package/lib/{synthesizers → us/synthesizers}/intersection.ts +12 -13
- package/lib/{synthesizers → us/synthesizers}/street.ts +16 -15
- package/lib/us/tools/fetch/imls-pls.ts +1 -1
- package/lib/us/tools/fetch/nad.ts +7 -7
- package/lib/us/tools/fetch/nppes.ts +5 -5
- package/lib/us/tools/fetch/state/hi-schools.ts +3 -2
- package/lib/us/tools/fetch/tiger-full.ts +1 -1
- package/lib/utils/align.ts +13 -10
- package/lib/utils/golden.ts +2 -2
- package/lib/utils/license.ts +3 -3
- package/lib/utils/split.ts +19 -15
- package/lib/utils/tokenize.ts +7 -26
- package/lib/utils/wof-json.ts +4 -4
- package/out/adapters/cordis/adapter.d.ts +5 -3
- package/out/adapters/cordis/adapter.d.ts.map +1 -1
- package/out/adapters/cordis/adapter.js +13 -7
- package/out/adapters/cordis/adapter.js.map +1 -1
- package/out/adapters/dedup-key.d.ts +1 -1
- package/out/adapters/dedup-key.js +1 -1
- package/out/adapters/geonames/adapter.js +1 -1
- package/out/adapters/geonames/adapter.js.map +1 -1
- package/out/adapters/gleif/adapter.d.ts +9 -9
- package/out/adapters/gleif/adapter.d.ts.map +1 -1
- package/out/adapters/gleif/adapter.js +15 -14
- package/out/adapters/gleif/adapter.js.map +1 -1
- package/out/adapters/libpostal-dictionary.d.ts +0 -2
- package/out/adapters/libpostal-dictionary.d.ts.map +1 -1
- package/out/adapters/libpostal-dictionary.js +0 -2
- package/out/adapters/libpostal-dictionary.js.map +1 -1
- package/out/adapters/openaddresses/adapter.d.ts +3 -2
- package/out/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/adapters/openaddresses/adapter.js +1 -1
- package/out/adapters/openaddresses/adapter.js.map +1 -1
- package/out/adapters/osm/adapter.d.ts +17 -16
- package/out/adapters/osm/adapter.d.ts.map +1 -1
- package/out/adapters/osm/adapter.js +10 -9
- package/out/adapters/osm/adapter.js.map +1 -1
- package/out/adapters/po-box/adapter.d.ts +6 -3
- package/out/adapters/po-box/adapter.d.ts.map +1 -1
- package/out/adapters/po-box/adapter.js +4 -2
- package/out/adapters/po-box/adapter.js.map +1 -1
- package/out/adapters/registry.d.ts +3 -3
- package/out/adapters/registry.d.ts.map +1 -1
- package/out/adapters/registry.js +1 -1
- package/out/adapters/registry.js.map +1 -1
- package/out/adapters/source-id.d.ts +2 -2
- package/out/adapters/source-id.js +2 -2
- package/out/adapters/street-line.d.ts +7 -6
- package/out/adapters/street-line.d.ts.map +1 -1
- package/out/adapters/street-line.js +7 -6
- package/out/adapters/street-line.js.map +1 -1
- package/out/adapters/wof/admin/json/adapter.js +1 -1
- package/out/adapters/wof/admin/json/adapter.js.map +1 -1
- package/out/adapters/wof/json-rows.d.ts +2 -1
- package/out/adapters/wof/json-rows.d.ts.map +1 -1
- package/out/adapters/wof/json-rows.js.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.d.ts.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.js +5 -1
- package/out/adapters/wof/postalcode-json/adapter.js.map +1 -1
- package/out/adapters.d.ts +5 -5
- package/out/adapters.js +5 -5
- package/out/au/adapters/gnaf/adapter.js +1 -1
- package/out/au/adapters/gnaf/adapter.js.map +1 -1
- package/out/be/adapters/brussels/adapter.d.ts +9 -9
- package/out/be/adapters/brussels/adapter.d.ts.map +1 -1
- package/out/be/adapters/brussels/adapter.js +21 -21
- package/out/be/adapters/brussels/adapter.js.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.d.ts +34 -37
- package/out/be/adapters/vlaanderen/adapter.d.ts.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.js +40 -38
- package/out/be/adapters/vlaanderen/adapter.js.map +1 -1
- package/out/be/adapters/wallonie/adapter.d.ts +2 -2
- package/out/be/adapters/wallonie/adapter.js +17 -17
- package/out/be/adapters/wallonie/adapter.js.map +1 -1
- package/out/be/tools/fetch/brussels.d.ts +11 -8
- package/out/be/tools/fetch/brussels.d.ts.map +1 -1
- package/out/be/tools/fetch/brussels.js +10 -8
- package/out/be/tools/fetch/brussels.js.map +1 -1
- package/out/be/tools/fetch/wallonie.d.ts +6 -5
- package/out/be/tools/fetch/wallonie.d.ts.map +1 -1
- package/out/be/tools/fetch/wallonie.js +4 -4
- package/out/be/tools/fetch/wallonie.js.map +1 -1
- package/out/build/adapters.d.ts.map +1 -1
- package/out/build/adapters.js +2 -1
- package/out/build/adapters.js.map +1 -1
- package/out/build/align.d.ts +3 -2
- package/out/build/align.d.ts.map +1 -1
- package/out/build/align.js +2 -2
- package/out/build/align.js.map +1 -1
- package/out/build/checkpoint.d.ts +1 -1
- package/out/build/checkpoint.d.ts.map +1 -1
- package/out/build/checkpoint.js +4 -2
- package/out/build/checkpoint.js.map +1 -1
- package/out/build/eligibility.d.ts +2 -2
- package/out/build/eligibility.js +3 -3
- package/out/build/eligibility.js.map +1 -1
- package/out/build/types.d.ts +8 -6
- package/out/build/types.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.d.ts +14 -14
- package/out/cz/adapters/cuzk/adapter.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.js +38 -35
- package/out/cz/adapters/cuzk/adapter.js.map +1 -1
- package/out/cz/tools/fetch/cuzk.d.ts +31 -22
- package/out/cz/tools/fetch/cuzk.d.ts.map +1 -1
- package/out/cz/tools/fetch/cuzk.js +21 -19
- package/out/cz/tools/fetch/cuzk.js.map +1 -1
- package/out/de/recipes/locale.js +1 -1
- package/out/de/recipes/locale.js.map +1 -1
- package/out/dk/adapters/inspire/adapter.d.ts +32 -33
- package/out/dk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/dk/adapters/inspire/adapter.js +38 -38
- package/out/dk/adapters/inspire/adapter.js.map +1 -1
- package/out/dk/tools/fetch/inspire.d.ts +6 -5
- package/out/dk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/dk/tools/fetch/inspire.js +15 -13
- package/out/dk/tools/fetch/inspire.js.map +1 -1
- package/out/ee/adapters/ads/adapter.d.ts +19 -26
- package/out/ee/adapters/ads/adapter.d.ts.map +1 -1
- package/out/ee/adapters/ads/adapter.js +22 -28
- package/out/ee/adapters/ads/adapter.js.map +1 -1
- package/out/ee/tools/fetch/ads.d.ts +3 -3
- package/out/ee/tools/fetch/ads.d.ts.map +1 -1
- package/out/ee/tools/fetch/ads.js +10 -8
- package/out/ee/tools/fetch/ads.js.map +1 -1
- package/out/es/adapters/bizkaia/adapter.d.ts +28 -28
- package/out/es/adapters/bizkaia/adapter.d.ts.map +1 -1
- package/out/es/adapters/bizkaia/adapter.js +43 -43
- package/out/es/adapters/bizkaia/adapter.js.map +1 -1
- package/out/es/adapters/catastro/adapter.d.ts +22 -23
- package/out/es/adapters/catastro/adapter.d.ts.map +1 -1
- package/out/es/adapters/catastro/adapter.js +35 -33
- package/out/es/adapters/catastro/adapter.js.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.d.ts +31 -38
- package/out/es/adapters/gipuzkoa/adapter.d.ts.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.js +40 -47
- package/out/es/adapters/gipuzkoa/adapter.js.map +1 -1
- package/out/es/adapters/navarra/adapter.d.ts +20 -19
- package/out/es/adapters/navarra/adapter.d.ts.map +1 -1
- package/out/es/adapters/navarra/adapter.js +23 -22
- package/out/es/adapters/navarra/adapter.js.map +1 -1
- package/out/es/cadastre.d.ts +21 -20
- package/out/es/cadastre.d.ts.map +1 -1
- package/out/es/cadastre.js +19 -21
- package/out/es/cadastre.js.map +1 -1
- package/out/es/tools/fetch/bizkaia.d.ts +28 -25
- package/out/es/tools/fetch/bizkaia.d.ts.map +1 -1
- package/out/es/tools/fetch/bizkaia.js +25 -24
- package/out/es/tools/fetch/bizkaia.js.map +1 -1
- package/out/es/tools/fetch/catastro.d.ts +28 -20
- package/out/es/tools/fetch/catastro.d.ts.map +1 -1
- package/out/es/tools/fetch/catastro.js +36 -31
- package/out/es/tools/fetch/catastro.js.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.d.ts +11 -9
- package/out/es/tools/fetch/gipuzkoa.d.ts.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.js +9 -8
- package/out/es/tools/fetch/gipuzkoa.js.map +1 -1
- package/out/es/tools/fetch/navarra.d.ts +20 -15
- package/out/es/tools/fetch/navarra.d.ts.map +1 -1
- package/out/es/tools/fetch/navarra.js +21 -16
- package/out/es/tools/fetch/navarra.js.map +1 -1
- package/out/fi/adapters/ryhti/adapter.d.ts +16 -20
- package/out/fi/adapters/ryhti/adapter.d.ts.map +1 -1
- package/out/fi/adapters/ryhti/adapter.js +21 -25
- package/out/fi/adapters/ryhti/adapter.js.map +1 -1
- package/out/fi/tools/fetch/ryhti.d.ts +24 -24
- package/out/fi/tools/fetch/ryhti.d.ts.map +1 -1
- package/out/fi/tools/fetch/ryhti.js +21 -24
- package/out/fi/tools/fetch/ryhti.js.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.d.ts +8 -8
- package/out/fr/adapters/annuaire-education/adapter.d.ts.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.js +13 -11
- package/out/fr/adapters/annuaire-education/adapter.js.map +1 -1
- package/out/fr/adapters/ban/adapter.d.ts +4 -4
- package/out/fr/adapters/ban/adapter.js +6 -6
- package/out/fr/adapters/ban/adapter.js.map +1 -1
- package/out/fr/adapters/finess/adapter.d.ts +12 -11
- package/out/fr/adapters/finess/adapter.d.ts.map +1 -1
- package/out/fr/adapters/finess/adapter.js +16 -15
- package/out/fr/adapters/finess/adapter.js.map +1 -1
- package/out/fr/adapters/finess/voie-type.d.ts +2 -2
- package/out/fr/adapters/finess/voie-type.d.ts.map +1 -1
- package/out/fr/adapters/finess/voie-type.js +4 -3
- package/out/fr/adapters/finess/voie-type.js.map +1 -1
- package/out/fr/insee-country.d.ts +2 -2
- package/out/fr/insee-country.js +2 -2
- package/out/fr/recipes/lieudit.d.ts.map +1 -1
- package/out/fr/recipes/lieudit.js +5 -4
- package/out/fr/recipes/lieudit.js.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.d.ts +7 -4
- package/out/fr/tools/fetch/annuaire-education.d.ts.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.js +9 -7
- package/out/fr/tools/fetch/annuaire-education.js.map +1 -1
- package/out/fr/tools/fetch/ban.d.ts +1 -1
- package/out/fr/tools/fetch/ban.d.ts.map +1 -1
- package/out/fr/tools/fetch/ban.js +3 -2
- package/out/fr/tools/fetch/ban.js.map +1 -1
- package/out/fr/tools/fetch/finess.d.ts +5 -5
- package/out/fr/tools/fetch/finess.d.ts.map +1 -1
- package/out/fr/tools/fetch/finess.js +13 -12
- package/out/fr/tools/fetch/finess.js.map +1 -1
- package/out/inspire/address.d.ts +62 -52
- package/out/inspire/address.d.ts.map +1 -1
- package/out/inspire/address.js +71 -60
- package/out/inspire/address.js.map +1 -1
- package/out/inspire/errors.d.ts +9 -8
- package/out/inspire/errors.d.ts.map +1 -1
- package/out/inspire/errors.js +7 -6
- package/out/inspire/errors.js.map +1 -1
- package/out/inspire/stream.d.ts +25 -22
- package/out/inspire/stream.d.ts.map +1 -1
- package/out/inspire/stream.js +10 -12
- package/out/inspire/stream.js.map +1 -1
- package/out/international/recipes/locale.d.ts +2 -2
- package/out/international/recipes/locale.d.ts.map +1 -1
- package/out/international/recipes/locale.js +11 -7
- package/out/international/recipes/locale.js.map +1 -1
- package/out/it/adapters/anac/adapter.d.ts +45 -43
- package/out/it/adapters/anac/adapter.d.ts.map +1 -1
- package/out/it/adapters/anac/adapter.js +48 -43
- package/out/it/adapters/anac/adapter.js.map +1 -1
- package/out/it/tools/fetch/anac.d.ts +16 -19
- package/out/it/tools/fetch/anac.d.ts.map +1 -1
- package/out/it/tools/fetch/anac.js +20 -23
- package/out/it/tools/fetch/anac.js.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.d.ts.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.js.map +1 -1
- package/out/jp/tools/fetch/houjin.js +2 -2
- package/out/jp/tools/fetch/houjin.js.map +1 -1
- package/out/kr/adapters/localdata/permit-rows.js +1 -1
- package/out/kr/adapters/localdata/permit-rows.js.map +1 -1
- package/out/kr/tools/fetch/juso.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.js +1 -1
- package/out/kr/tools/fetch/localdata.js.map +1 -1
- package/out/lu/adapters/bd-adresses/adapter.d.ts +9 -9
- package/out/lu/adapters/bd-adresses/adapter.js +10 -10
- package/out/lu/tools/fetch/bd-adresses.d.ts +11 -9
- package/out/lu/tools/fetch/bd-adresses.d.ts.map +1 -1
- package/out/lu/tools/fetch/bd-adresses.js +11 -11
- package/out/lu/tools/fetch/bd-adresses.js.map +1 -1
- package/out/lv/adapters/varis/adapter.d.ts +24 -23
- package/out/lv/adapters/varis/adapter.d.ts.map +1 -1
- package/out/lv/adapters/varis/adapter.js +24 -23
- package/out/lv/adapters/varis/adapter.js.map +1 -1
- package/out/lv/tools/fetch/varis.d.ts +2 -1
- package/out/lv/tools/fetch/varis.d.ts.map +1 -1
- package/out/lv/tools/fetch/varis.js +1 -1
- package/out/lv/tools/fetch/varis.js.map +1 -1
- package/out/nl/adapters/kadaster/adapter.d.ts +12 -12
- package/out/nl/adapters/kadaster/adapter.d.ts.map +1 -1
- package/out/nl/adapters/kadaster/adapter.js +22 -23
- package/out/nl/adapters/kadaster/adapter.js.map +1 -1
- package/out/nl/tools/fetch/kadaster.d.ts +14 -10
- package/out/nl/tools/fetch/kadaster.d.ts.map +1 -1
- package/out/nl/tools/fetch/kadaster.js +12 -9
- package/out/nl/tools/fetch/kadaster.js.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.js +3 -3
- package/out/no/adapters/matrikkelen/adapter.js.map +1 -1
- package/out/no/tools/fetch/matrikkelen.d.ts +50 -38
- package/out/no/tools/fetch/matrikkelen.d.ts.map +1 -1
- package/out/no/tools/fetch/matrikkelen.js +44 -36
- package/out/no/tools/fetch/matrikkelen.js.map +1 -1
- package/out/parquet/duckdb.d.ts +7 -4
- package/out/parquet/duckdb.d.ts.map +1 -1
- package/out/parquet/duckdb.js +2 -1
- package/out/parquet/duckdb.js.map +1 -1
- package/out/parquet/readers.js +3 -3
- package/out/parquet/readers.js.map +1 -1
- package/out/parquet/schema.js +1 -1
- package/out/parquet/schema.js.map +1 -1
- package/out/parquet/writers.d.ts +1 -1
- package/out/parquet/writers.d.ts.map +1 -1
- package/out/parquet/writers.js +2 -1
- package/out/parquet/writers.js.map +1 -1
- package/out/pl/adapters/emuia/adapter.d.ts +8 -8
- package/out/pl/adapters/emuia/adapter.js +12 -12
- package/out/pl/adapters/emuia/adapter.js.map +1 -1
- package/out/pl/tools/fetch/emuia.d.ts +5 -4
- package/out/pl/tools/fetch/emuia.d.ts.map +1 -1
- package/out/pl/tools/fetch/emuia.js +10 -9
- package/out/pl/tools/fetch/emuia.js.map +1 -1
- package/out/recipes/bare/postcode.js +1 -1
- package/out/recipes/bare/postcode.js.map +1 -1
- package/out/recipes/boundary-stress.js +1 -1
- package/out/recipes/boundary-stress.js.map +1 -1
- package/out/recipes/house-venue.d.ts.map +1 -1
- package/out/recipes/house-venue.js +6 -3
- package/out/recipes/house-venue.js.map +1 -1
- package/out/recipes/intersection.js +2 -2
- package/out/recipes/intersection.js.map +1 -1
- package/out/recipes/po/box/cedex/recipe.js +2 -1
- package/out/recipes/po/box/cedex/recipe.js.map +1 -1
- package/out/recipes/po/box/cedex/sources.d.ts +8 -2
- package/out/recipes/po/box/cedex/sources.d.ts.map +1 -1
- package/out/recipes/po/box/cedex/sources.js +1 -1
- package/out/recipes/po/box/cedex/sources.js.map +1 -1
- package/out/recipes/scaffold.d.ts +28 -19
- package/out/recipes/scaffold.d.ts.map +1 -1
- package/out/recipes/scaffold.js +3 -2
- package/out/recipes/scaffold.js.map +1 -1
- package/out/recipes/sources.d.ts +1 -1
- package/out/recipes/sources.d.ts.map +1 -1
- package/out/recipes/sources.js +15 -2
- package/out/recipes/sources.js.map +1 -1
- package/out/recipes/street/bare.js +2 -2
- package/out/recipes/street/bare.js.map +1 -1
- package/out/recipes/street.d.ts.map +1 -1
- package/out/recipes/street.js +3 -1
- package/out/recipes/street.js.map +1 -1
- package/out/recipes/sub/venue/context.d.ts.map +1 -1
- package/out/recipes/sub/venue/context.js +2 -0
- package/out/recipes/sub/venue/context.js.map +1 -1
- package/out/recipes/sub/venue/render.js +1 -1
- package/out/recipes/sub/venue/render.js.map +1 -1
- package/out/recipes/sub/venue/sources.d.ts +6 -3
- package/out/recipes/sub/venue/sources.d.ts.map +1 -1
- package/out/recipes/sub/venue/sources.js +16 -20
- package/out/recipes/sub/venue/sources.js.map +1 -1
- package/out/recipes/sub/venue.d.ts +2 -5
- package/out/recipes/sub/venue.d.ts.map +1 -1
- package/out/recipes/sub/venue.js +10 -14
- package/out/recipes/sub/venue.js.map +1 -1
- package/out/recipes/trailing-region.d.ts.map +1 -1
- package/out/recipes/trailing-region.js +6 -1
- package/out/recipes/trailing-region.js.map +1 -1
- package/out/recipes/unit.js +2 -2
- package/out/recipes/unit.js.map +1 -1
- package/out/recipes/venue.d.ts +119 -0
- package/out/recipes/venue.d.ts.map +1 -0
- package/out/recipes/venue.js +245 -0
- package/out/recipes/venue.js.map +1 -0
- package/out/recipes.d.ts +1 -1
- package/out/recipes.d.ts.map +1 -1
- package/out/recipes.js +3 -1
- package/out/recipes.js.map +1 -1
- package/out/registers.d.ts +8 -6
- package/out/registers.d.ts.map +1 -1
- package/out/registers.js +8 -6
- package/out/registers.js.map +1 -1
- package/out/ro/adapters/retea-scolara/adapter.d.ts +5 -5
- package/out/ro/adapters/retea-scolara/adapter.js +5 -5
- package/out/ro/tools/fetch/retea-scolara.js +1 -1
- package/out/ro/tools/fetch/retea-scolara.js.map +1 -1
- package/out/runner.d.ts +3 -2
- package/out/runner.d.ts.map +1 -1
- package/out/runner.js +1 -1
- package/out/runner.js.map +1 -1
- package/out/sg/recipes/register.js +1 -1
- package/out/sg/recipes/register.js.map +1 -1
- package/out/sg/tools/fetch/acra.js +2 -2
- package/out/sg/tools/fetch/acra.js.map +1 -1
- package/out/sk/adapters/inspire/adapter.d.ts +5 -5
- package/out/sk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/sk/adapters/inspire/adapter.js +18 -18
- package/out/sk/adapters/inspire/adapter.js.map +1 -1
- package/out/sk/tools/fetch/inspire.d.ts +11 -10
- package/out/sk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/sk/tools/fetch/inspire.js +15 -14
- package/out/sk/tools/fetch/inspire.js.map +1 -1
- package/out/source-register/effective-manifest.js +3 -3
- package/out/source-register/effective-manifest.js.map +1 -1
- package/out/source-register/register.d.ts +2 -2
- package/out/source-register/register.d.ts.map +1 -1
- package/out/source-register/register.js +2 -2
- package/out/source-register/register.js.map +1 -1
- package/out/source-register/training-manifest.d.ts +1 -1
- package/out/source-register/types.d.ts +12 -13
- package/out/source-register/types.d.ts.map +1 -1
- package/out/source-register/types.js +2 -2
- package/out/source-register/types.js.map +1 -1
- package/out/subvenue/promotions.d.ts +1 -1
- package/out/subvenue/promotions.js +1 -1
- package/out/subvenue/table.d.ts +4 -4
- package/out/subvenue/table.js +3 -3
- package/out/surfaces/locale.d.ts.map +1 -1
- package/out/surfaces/locale.js +2 -0
- package/out/surfaces/locale.js.map +1 -1
- package/out/synthesizers/anchor-absorption.d.ts +2 -1
- package/out/synthesizers/anchor-absorption.d.ts.map +1 -1
- package/out/synthesizers/anchor-absorption.js.map +1 -1
- package/out/synthesizers/boundary-stress.d.ts +1 -1
- package/out/synthesizers/boundary-stress.d.ts.map +1 -1
- package/out/synthesizers/boundary-stress.js +20 -23
- package/out/synthesizers/boundary-stress.js.map +1 -1
- package/out/synthesizers/house-venue.d.ts +3 -2
- package/out/synthesizers/house-venue.d.ts.map +1 -1
- package/out/synthesizers/house-venue.js +24 -26
- package/out/synthesizers/house-venue.js.map +1 -1
- package/out/synthesizers/no-street.d.ts.map +1 -1
- package/out/synthesizers/no-street.js.map +1 -1
- package/out/synthesizers/po-box.d.ts +4 -2
- package/out/synthesizers/po-box.d.ts.map +1 -1
- package/out/synthesizers/po-box.js.map +1 -1
- package/out/synthesizers/tail.d.ts +36 -0
- package/out/synthesizers/tail.d.ts.map +1 -0
- package/out/synthesizers/tail.js +32 -0
- package/out/synthesizers/tail.js.map +1 -0
- package/out/synthesizers/utils.d.ts +2 -1
- package/out/synthesizers/utils.d.ts.map +1 -1
- package/out/synthesizers/utils.js +17 -1
- package/out/synthesizers/utils.js.map +1 -1
- package/out/test-kit.js +2 -2
- package/out/test-kit.js.map +1 -1
- package/out/tools/audit.d.ts +2 -1
- package/out/tools/audit.d.ts.map +1 -1
- package/out/tools/audit.js +4 -3
- package/out/tools/audit.js.map +1 -1
- package/out/tools/corpus-stats.js +1 -1
- package/out/tools/corpus-stats.js.map +1 -1
- package/out/tools/fetch/atom.d.ts +8 -8
- package/out/tools/fetch/atom.d.ts.map +1 -1
- package/out/tools/fetch/atom.js +7 -8
- package/out/tools/fetch/atom.js.map +1 -1
- package/out/tools/fetch/cordis.d.ts +4 -3
- package/out/tools/fetch/cordis.d.ts.map +1 -1
- package/out/tools/fetch/cordis.js +9 -7
- package/out/tools/fetch/cordis.js.map +1 -1
- package/out/tools/fetch/download/manifest.d.ts.map +1 -1
- package/out/tools/fetch/download/manifest.js +2 -1
- package/out/tools/fetch/download/manifest.js.map +1 -1
- package/out/tools/fetch/download/network.d.ts +4 -4
- package/out/tools/fetch/download/retry.js +4 -4
- package/out/tools/fetch/download/retry.js.map +1 -1
- package/out/tools/fetch/geonames/dump.d.ts +3 -2
- package/out/tools/fetch/geonames/dump.d.ts.map +1 -1
- package/out/tools/fetch/geonames/dump.js +2 -1
- package/out/tools/fetch/geonames/dump.js.map +1 -1
- package/out/tools/fetch/geonames/postal.d.ts +2 -2
- package/out/tools/fetch/gleif.d.ts +3 -2
- package/out/tools/fetch/gleif.d.ts.map +1 -1
- package/out/tools/fetch/gleif.js +6 -4
- package/out/tools/fetch/gleif.js.map +1 -1
- package/out/tools/fetch/header.d.ts +7 -5
- package/out/tools/fetch/header.d.ts.map +1 -1
- package/out/tools/fetch/header.js +8 -6
- package/out/tools/fetch/header.js.map +1 -1
- package/out/tools/fetch/inspire-addresses.d.ts +32 -27
- package/out/tools/fetch/inspire-addresses.d.ts.map +1 -1
- package/out/tools/fetch/inspire-addresses.js +25 -24
- package/out/tools/fetch/inspire-addresses.js.map +1 -1
- package/out/tools/fetch/openaddresses.js +1 -1
- package/out/tools/fetch/openaddresses.js.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.d.ts +5 -5
- package/out/tools/fetch/vlaanderen-ad.d.ts.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.js +16 -16
- package/out/tools/fetch/vlaanderen-ad.js.map +1 -1
- package/out/tools/fetch/wfs-harvest.d.ts +26 -16
- package/out/tools/fetch/wfs-harvest.d.ts.map +1 -1
- package/out/tools/fetch/wfs-harvest.js +21 -18
- package/out/tools/fetch/wfs-harvest.js.map +1 -1
- package/out/tools/fetch.d.ts +0 -41
- package/out/tools/fetch.d.ts.map +1 -1
- package/out/tools/fetch.js +0 -41
- package/out/tools/fetch.js.map +1 -1
- package/out/tools/golden/expand.d.ts +7 -7
- package/out/tools/golden/expand.js +1 -1
- package/out/tools/golden/expand.js.map +1 -1
- package/out/tools/golden/promote.d.ts +2 -2
- package/out/tools/golden/promote.d.ts.map +1 -1
- package/out/tools/golden/promote.js +9 -6
- package/out/tools/golden/promote.js.map +1 -1
- package/out/tools/golden/relabel-street/directory.d.ts +4 -4
- package/out/tools/golden/relabel-street/directory.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/directory.js +3 -2
- package/out/tools/golden/relabel-street/directory.js.map +1 -1
- package/out/tools/golden/relabel-street/row.d.ts +4 -3
- package/out/tools/golden/relabel-street/row.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/row.js +3 -3
- package/out/tools/golden/relabel-street/row.js.map +1 -1
- package/out/tools/ingest-csv.js +3 -3
- package/out/tools/ingest-csv.js.map +1 -1
- package/out/tools/jsonl-to-parquet.d.ts +1 -1
- package/out/tools/lint/recipe-output/vocab.d.ts +10 -5
- package/out/tools/lint/recipe-output/vocab.d.ts.map +1 -1
- package/out/tools/lint/recipe-output/vocab.js.map +1 -1
- package/out/tools/lint/recipe-output.d.ts +8 -8
- package/out/tools/lint/recipe-output.d.ts.map +1 -1
- package/out/tools/lint/recipe-output.js +17 -0
- package/out/tools/lint/recipe-output.js.map +1 -1
- package/out/tools/overlay/kryptonite.d.ts +2 -2
- package/out/tools/overlay/kryptonite.d.ts.map +1 -1
- package/out/tools/overlay/kryptonite.js +2 -1
- package/out/tools/overlay/kryptonite.js.map +1 -1
- package/out/tools/overlay/manifest.d.ts +2 -1
- package/out/tools/overlay/manifest.d.ts.map +1 -1
- package/out/tools/overlay/manifest.js +2 -1
- package/out/tools/overlay/manifest.js.map +1 -1
- package/out/tools/overlay/merge-source.d.ts +2 -1
- package/out/tools/overlay/merge-source.d.ts.map +1 -1
- package/out/tools/overlay/merge-source.js.map +1 -1
- package/out/tools/overlay/split-slice.d.ts +1 -1
- package/out/tools/overlay/split-slice.js +1 -1
- package/out/tools/overlay/split-slice.js.map +1 -1
- package/out/tools/overlay/translit.d.ts +4 -3
- package/out/tools/overlay/translit.d.ts.map +1 -1
- package/out/tools/overlay/translit.js +2 -1
- package/out/tools/overlay/translit.js.map +1 -1
- package/out/tools/overture-subvenue.d.ts +1 -1
- package/out/tools/overture-subvenue.d.ts.map +1 -1
- package/out/tools/postcode-triples.d.ts +1 -1
- package/out/tools/postcode-triples.d.ts.map +1 -1
- package/out/tools/postcode-triples.js +4 -3
- package/out/tools/postcode-triples.js.map +1 -1
- package/out/tools/source-register/build.d.ts +10 -8
- package/out/tools/source-register/build.d.ts.map +1 -1
- package/out/tools/source-register/build.js +10 -8
- package/out/tools/source-register/build.js.map +1 -1
- package/out/tools/sub/venue/lexicon.d.ts +2 -6
- package/out/tools/sub/venue/lexicon.d.ts.map +1 -1
- package/out/tools/sub/venue/lexicon.js +2 -6
- package/out/tools/sub/venue/lexicon.js.map +1 -1
- package/out/tw/tools/fetch/gcis.js +3 -3
- package/out/tw/tools/fetch/gcis.js.map +1 -1
- package/out/types.d.ts +4 -3
- package/out/types.d.ts.map +1 -1
- package/out/types.js.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.js.map +1 -1
- package/out/us/adapters/usgov/nad/adapter.js +6 -6
- package/out/us/adapters/usgov/nad/adapter.js.map +1 -1
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js +2 -2
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/us/fips-state.d.ts +2 -2
- package/out/us/fips-state.d.ts.map +1 -1
- package/out/us/fips-state.js.map +1 -1
- package/out/us/synthesizers/intersection.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/intersection.js +13 -9
- package/out/us/synthesizers/intersection.js.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.d.ts +3 -2
- package/out/us/synthesizers/street.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.js +12 -12
- package/out/us/synthesizers/street.js.map +1 -0
- package/out/us/tools/fetch/imls-pls.js +1 -1
- package/out/us/tools/fetch/imls-pls.js.map +1 -1
- package/out/us/tools/fetch/nad.d.ts +6 -6
- package/out/us/tools/fetch/nad.js +1 -1
- package/out/us/tools/fetch/nad.js.map +1 -1
- package/out/us/tools/fetch/nppes.js +3 -3
- package/out/us/tools/fetch/nppes.js.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts +2 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.js +1 -1
- package/out/us/tools/fetch/state/hi-schools.js.map +1 -1
- package/out/us/tools/fetch/tiger-full.js +1 -1
- package/out/us/tools/fetch/tiger-full.js.map +1 -1
- package/out/utils/align.d.ts +5 -3
- package/out/utils/align.d.ts.map +1 -1
- package/out/utils/align.js +5 -5
- package/out/utils/align.js.map +1 -1
- package/out/utils/golden.js +2 -2
- package/out/utils/golden.js.map +1 -1
- package/out/utils/license.d.ts +3 -3
- package/out/utils/license.d.ts.map +1 -1
- package/out/utils/license.js.map +1 -1
- package/out/utils/split.d.ts +1 -1
- package/out/utils/split.d.ts.map +1 -1
- package/out/utils/split.js +18 -14
- package/out/utils/split.js.map +1 -1
- package/out/utils/tokenize.d.ts +3 -20
- package/out/utils/tokenize.d.ts.map +1 -1
- package/out/utils/tokenize.js.map +1 -1
- package/out/utils/wof-json.js +3 -3
- package/out/utils/wof-json.js.map +1 -1
- package/package.json +18 -27
- package/lib/tools.ts +0 -31
- package/out/synthesizers/intersection.d.ts.map +0 -1
- package/out/synthesizers/intersection.js.map +0 -1
- package/out/synthesizers/street.d.ts.map +0 -1
- package/out/synthesizers/street.js.map +0 -1
- package/out/tools.d.ts +0 -31
- package/out/tools.d.ts.map +0 -1
- package/out/tools.js +0 -31
- package/out/tools.js.map +0 -1
- /package/out/{synthesizers → us/synthesizers}/intersection.d.ts +0 -0
package/lib/build/checkpoint.ts
CHANGED
|
@@ -15,8 +15,10 @@
|
|
|
15
15
|
* `build.test.ts` compares four output digests across two builds over one intermediate directory.
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import { readLocalJSONFile
|
|
18
|
+
import { readLocalJSONFile } from "@mailwoman/core/fs/readers"
|
|
19
|
+
import { pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
19
20
|
import { writeLocalJSONFile } from "@mailwoman/core/fs/writers"
|
|
21
|
+
import { isIdentical } from "@mailwoman/core/identical"
|
|
20
22
|
import { stringifyJSON } from "@mailwoman/core/json"
|
|
21
23
|
import type { PathBuilderLike } from "path-ts"
|
|
22
24
|
|
|
@@ -71,7 +73,7 @@ export interface AlignCheckpoint {
|
|
|
71
73
|
license_counts: Record<string, number>
|
|
72
74
|
refused_by_kind: Record<string, number>
|
|
73
75
|
/**
|
|
74
|
-
* Rows the license policy refused, counted per `source`, so a resumed build
|
|
76
|
+
* Rows the license policy refused, counted per `source`, so a resumed build keeps the tally
|
|
75
77
|
* rather than restarting it at zero and understating the refusal in its manifest.
|
|
76
78
|
*/
|
|
77
79
|
refused_by_source: Record<string, number>
|
|
@@ -110,7 +112,7 @@ export async function readAlignCheckpoint(
|
|
|
110
112
|
|
|
111
113
|
const prefix = expected.adapterOrder.slice(0, checkpoint.completed_adapters.length)
|
|
112
114
|
|
|
113
|
-
if (
|
|
115
|
+
if (!isIdentical(prefix, checkpoint.completed_adapters)) {
|
|
114
116
|
throw new Error(
|
|
115
117
|
`resume refused: ${path.toString()} recorded ${stringifyJSON(checkpoint.completed_adapters)} as aligned ` +
|
|
116
118
|
`and this build's adapter order begins ${stringifyJSON(prefix)}. Alignment appends, so the rows would ` +
|
package/lib/build/eligibility.ts
CHANGED
|
@@ -14,7 +14,7 @@ import type { CanonicalRow } from "#types"
|
|
|
14
14
|
* The key a row is looked up under: the adapter that emitted it and the jurisdiction it describes.
|
|
15
15
|
*
|
|
16
16
|
* The register scopes a source to one publisher in one jurisdiction,
|
|
17
|
-
* so an adapter id
|
|
17
|
+
* so an adapter id by itself cannot address a record.
|
|
18
18
|
* One adapter serves several: `ban` emits eleven jurisdictions from one schema
|
|
19
19
|
* and each is its own source under its own license decision.
|
|
20
20
|
*/
|
|
@@ -24,7 +24,7 @@ export function sourceEligibilityKey(adapterID: string, iso2: string): string {
|
|
|
24
24
|
|
|
25
25
|
/**
|
|
26
26
|
* Every register source's ingest-eligibility reasons, keyed by the adapter
|
|
27
|
-
* and jurisdiction of the rows that
|
|
27
|
+
* and jurisdiction of the rows that recorded it.
|
|
28
28
|
*
|
|
29
29
|
* A source declaring no `adapterID` is absent from the map, because no adapter emits it
|
|
30
30
|
* and no row can arrive under it.
|
|
@@ -36,7 +36,7 @@ export async function readSourceEligibility(): Promise<ReadonlyMap<string, reado
|
|
|
36
36
|
const keyed = new Map<string, readonly string[]>()
|
|
37
37
|
|
|
38
38
|
for (const source of register.sources) {
|
|
39
|
-
if (source.adapterID
|
|
39
|
+
if (!source.adapterID) continue
|
|
40
40
|
|
|
41
41
|
keyed.set(sourceEligibilityKey(source.adapterID, source.iso2), ingestEligibilityProblems(source, register))
|
|
42
42
|
}
|
package/lib/build/types.ts
CHANGED
|
@@ -67,7 +67,7 @@ export interface BuildCorpusOptions {
|
|
|
67
67
|
/**
|
|
68
68
|
* Adapters to drive, in order.
|
|
69
69
|
*
|
|
70
|
-
*
|
|
70
|
+
* @defaultValue `defaultAdapterRegistry.list()`, and an explicit list filters the run.
|
|
71
71
|
*/
|
|
72
72
|
adapters?: readonly CorpusAdapter[]
|
|
73
73
|
|
|
@@ -81,13 +81,14 @@ export interface BuildCorpusOptions {
|
|
|
81
81
|
/**
|
|
82
82
|
* Enable the synthesis pass.
|
|
83
83
|
*
|
|
84
|
-
*
|
|
84
|
+
* @defaultValue `true`, and a fixture-driven smoke test passes `false`.
|
|
85
85
|
*/
|
|
86
86
|
synthesize?: boolean
|
|
87
87
|
|
|
88
88
|
/**
|
|
89
89
|
* Max rows per `.parquet` file, forwarded to `writeParquetSplits`.
|
|
90
|
-
*
|
|
90
|
+
*
|
|
91
|
+
* @defaultValue `1_000_000`
|
|
91
92
|
*/
|
|
92
93
|
rowsPerFile?: number
|
|
93
94
|
|
|
@@ -100,7 +101,7 @@ export interface BuildCorpusOptions {
|
|
|
100
101
|
* An unshuffled row group therefore holds rows from one to eleven countries in its source.
|
|
101
102
|
* A bounded epoch draw reads only those countries.
|
|
102
103
|
*
|
|
103
|
-
*
|
|
104
|
+
* @defaultValue `DEFAULT_SHUFFLE_WINDOW`, where `0` or `1` writes arrival
|
|
104
105
|
* order unchanged and consumes no random draw.
|
|
105
106
|
*/
|
|
106
107
|
shuffleWindow?: number
|
|
@@ -129,7 +130,7 @@ export interface BuildCorpusOptions {
|
|
|
129
130
|
/**
|
|
130
131
|
* Which rows this build admits on the evidence of their license obligations.
|
|
131
132
|
*
|
|
132
|
-
*
|
|
133
|
+
* @defaultValue `LicensePolicy.All`, admitting every row an adapter yields.
|
|
133
134
|
*
|
|
134
135
|
* A weights build that must have no share-alike obligation passes `LicensePolicy.ShareAlikeFree`.
|
|
135
136
|
*/
|
|
@@ -137,7 +138,8 @@ export interface BuildCorpusOptions {
|
|
|
137
138
|
|
|
138
139
|
/**
|
|
139
140
|
* What this corpus is being built for.
|
|
140
|
-
*
|
|
141
|
+
*
|
|
142
|
+
* @defaultValue {@linkcode BuildProfile.Exploratory}
|
|
141
143
|
*/
|
|
142
144
|
profile?: BuildProfile
|
|
143
145
|
}
|
|
@@ -6,24 +6,24 @@
|
|
|
6
6
|
* `cz-cuzk`: the INSPIRE Addresses theme ČÚZK publishes, read from its per-municipality GML.
|
|
7
7
|
*
|
|
8
8
|
* ČÚZK, the Czech Office for Surveying, Mapping and Cadastre, publishes one zipped GML per
|
|
9
|
-
* municipality through the ATOM service at `https://atom.cuzk.gov.cz/AD/AD.xml
|
|
10
|
-
* dataset feeds. Each feed offers the same data in EPSG:5514 and EPSG:4258. The adapter reads
|
|
11
|
-
* single archive or a directory of them, so a caller harvests municipality by municipality rather
|
|
9
|
+
* municipality through the ATOM service at `https://atom.cuzk.gov.cz/AD/AD.xml`. That service lists
|
|
10
|
+
* 6,258 dataset feeds. Each feed offers the same data in EPSG:5514 and EPSG:4258. The adapter reads
|
|
11
|
+
* a single archive or a directory of them, so a caller harvests municipality by municipality rather
|
|
12
12
|
* than waiting on a national file.
|
|
13
13
|
*
|
|
14
|
-
* Every address and every feature it references sit in the same archive,
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
14
|
+
* Every address and every feature it references sit in the same archive, so this publisher is cheap
|
|
15
|
+
* to read. The `ad:component` hrefs look external, because ČÚZK writes an absolute WFS stored-query
|
|
16
|
+
* URL on `services.cuzk.cz`. The `Id=` parameter of each one matches a `gml:id` in the file at hand.
|
|
17
|
+
* Measured over `584282.xml`: 5,460 component references across 1,365 addresses, 0 of them reaching
|
|
18
|
+
* outside the archive. Only `ad:parcel` and `ad:building` point at genuinely separate themes, and
|
|
19
|
+
* the adapter reads neither.
|
|
20
20
|
*
|
|
21
21
|
* ČÚZK states one condition of use and states it uniformly. Every one of the 6,259 `rights` elements
|
|
22
|
-
* in the ATOM service document reads `žádné podmínky neplatí`,
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
22
|
+
* in the ATOM service document reads `žádné podmínky neplatí`, the INSPIRE controlled value
|
|
23
|
+
* `no conditions apply to access and use`. The address-source register elects that and records no
|
|
24
|
+
* license instrument, so no SPDX identifier exists to record and the rows store the controlled value
|
|
25
|
+
* itself. The register also holds a caution this adapter cannot resolve: that value states the
|
|
26
|
+
* absence of a restriction on access and use rather than granting copyright, so what it permits
|
|
27
27
|
* beyond access and use rests on Czech law.
|
|
28
28
|
*/
|
|
29
29
|
|
|
@@ -70,9 +70,10 @@ export const CZ_CUZK_COUNTRIES: readonly string[] = ["CZ"]
|
|
|
70
70
|
export const CZ_CUZK_LICENSE = "no conditions apply to access and use"
|
|
71
71
|
|
|
72
72
|
/**
|
|
73
|
-
* The feature types an address references
|
|
73
|
+
* The feature types an address references.
|
|
74
|
+
* The adapter indexes them as they arrive.
|
|
74
75
|
*
|
|
75
|
-
* `ad:Address`
|
|
76
|
+
* `ad:Address` holds its number inline and is read by the driver rather than listed here.
|
|
76
77
|
* These four hold the street, the postcode and the two place names, and they are read in the same pass
|
|
77
78
|
* because an archive writes them interleaved with the addresses rather than ahead of them.
|
|
78
79
|
*/
|
|
@@ -100,7 +101,7 @@ const GML_MEMBER = /\.xml$/iu
|
|
|
100
101
|
* The series prefix a street name makes unnecessary.
|
|
101
102
|
*
|
|
102
103
|
* Czechia numbers a building in one of two series.
|
|
103
|
-
* A `číslo popisné` is the descriptive number an ordinary building
|
|
104
|
+
* A `číslo popisné` is the descriptive number an ordinary building holds, and a
|
|
104
105
|
* `číslo evidenční` numbers a recreational or temporary building in a separate series,
|
|
105
106
|
* so the same digits can name two buildings in one municipality.
|
|
106
107
|
*/
|
|
@@ -121,22 +122,24 @@ const DESCRIPTIVE_NUMBER_PREFIX = "č.p."
|
|
|
121
122
|
* | `č.ev.` | `Komenského č.ev. 502` | `č.ev. 38, 66463 Unkovice` |
|
|
122
123
|
*
|
|
123
124
|
* So the descriptive prefix is dropped only when a street name already places the number.
|
|
124
|
-
* A number
|
|
125
|
-
* and the digits
|
|
125
|
+
* A number with no street keeps its prefix, because the two series overlap
|
|
126
|
+
* and the digits by themselves would not say which building is meant.
|
|
126
127
|
*
|
|
127
128
|
* An earlier rule that dropped `č.p.` everywhere disagreed with the publisher on 254 of those 1,669 rows.
|
|
128
129
|
*
|
|
129
|
-
* A prefix other than `č.p.` is kept in every case, including one these two municipalities
|
|
130
|
-
* not
|
|
131
|
-
* unrecognized prefix
|
|
130
|
+
* A prefix other than `č.p.` is kept in every case, including one these two municipalities
|
|
131
|
+
* did not show, because the observed values come from 2 of 6,258 municipalities.
|
|
132
|
+
* An unrecognized prefix stays.
|
|
133
|
+
*
|
|
134
|
+
* That preserves a distinction the publisher drew rather than discarding it.
|
|
132
135
|
*/
|
|
133
136
|
function czechHouseNumber(
|
|
134
137
|
byType: ReadonlyMap<string, readonly string[]>,
|
|
135
|
-
options: { street
|
|
136
|
-
): string |
|
|
138
|
+
options: { street: string | null }
|
|
139
|
+
): string | null {
|
|
137
140
|
const number = designator(byType, "buildingIdentifier")
|
|
138
141
|
|
|
139
|
-
if (!number) return
|
|
142
|
+
if (!number) return null
|
|
140
143
|
|
|
141
144
|
const prefix = designator(byType, "buildingIdentifierPrefix")
|
|
142
145
|
|
|
@@ -190,11 +193,11 @@ export function createCzCuzkAdapter(): CorpusAdapter {
|
|
|
190
193
|
}
|
|
191
194
|
|
|
192
195
|
/**
|
|
193
|
-
* One address's row, `
|
|
194
|
-
* when a reference it
|
|
196
|
+
* One address's row, `null` when it holds too little to render, or `"deferred"`
|
|
197
|
+
* when a reference it cites is not indexed yet.
|
|
195
198
|
*
|
|
196
|
-
*
|
|
197
|
-
*
|
|
199
|
+
* The caller defers rather than drops because an unresolved reference and an absent one
|
|
200
|
+
* are different conditions, and only the archive's own ordering tells them apart.
|
|
198
201
|
* With `final` set the whole member has been read, so an unresolved reference is an
|
|
199
202
|
* absence the publisher is responsible for and the row is rendered without it.
|
|
200
203
|
*/
|
|
@@ -202,13 +205,13 @@ function composeRow(
|
|
|
202
205
|
address: MarkupElement,
|
|
203
206
|
referenced: ReadonlyMap<string, MarkupElement>,
|
|
204
207
|
options: { final?: boolean } = {}
|
|
205
|
-
): CanonicalRow |
|
|
208
|
+
): CanonicalRow | null | "deferred" {
|
|
206
209
|
const byType = designatorsByType(address)
|
|
207
210
|
|
|
208
|
-
let street: string |
|
|
209
|
-
let postcode: string |
|
|
210
|
-
let locality: string |
|
|
211
|
-
let settlement: string |
|
|
211
|
+
let street: string | null = null
|
|
212
|
+
let postcode: string | null = null
|
|
213
|
+
let locality: string | null = null
|
|
214
|
+
let settlement: string | null = null
|
|
212
215
|
let unresolved = false
|
|
213
216
|
|
|
214
217
|
for (const href of componentHrefs(address)) {
|
|
@@ -220,7 +223,7 @@ function composeRow(
|
|
|
220
223
|
|
|
221
224
|
if (!feature) {
|
|
222
225
|
// Only the four themes this adapter reads are expected in the archive.
|
|
223
|
-
// A parcel or building reference
|
|
226
|
+
// A parcel or building reference holds no `Id` the loop above accepted.
|
|
224
227
|
if (/^(?:TF|PD|AU|AA)\./u.test(key)) {
|
|
225
228
|
unresolved = true
|
|
226
229
|
}
|
|
@@ -245,9 +248,9 @@ function composeRow(
|
|
|
245
248
|
const house = czechHouseNumber(byType, { street })
|
|
246
249
|
const place = locality ?? settlement
|
|
247
250
|
|
|
248
|
-
if (!place) return
|
|
251
|
+
if (!place) return null
|
|
249
252
|
|
|
250
|
-
if (!postcode && !street) return
|
|
253
|
+
if (!postcode && !street) return null
|
|
251
254
|
|
|
252
255
|
const components: CanonicalRow["components"] = {}
|
|
253
256
|
|
|
@@ -272,7 +275,7 @@ function composeRow(
|
|
|
272
275
|
|
|
273
276
|
const rendered = formatAddressRow(components, "CZ", { singleLine: true })
|
|
274
277
|
|
|
275
|
-
if (!rendered) return
|
|
278
|
+
if (!rendered) return null
|
|
276
279
|
|
|
277
280
|
const { raw, components: aligned } = rendered
|
|
278
281
|
const seed = address.attributes["gml:id"]?.trim()
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
* Four properties of this service decide the harvest's shape, and each was measured rather than
|
|
25
25
|
* assumed:
|
|
26
26
|
*
|
|
27
|
-
* 1. **The service document
|
|
27
|
+
* 1. **The service document states each municipality's file modification time.** An entry's
|
|
28
28
|
* `<updated>` equals the archive's `Last-Modified` to the second, checked on six municipalities
|
|
29
29
|
* on 2026-10-02 — `584061` reads `2026-06-04T02:19:33+02:00` against
|
|
30
30
|
* `Thu, 04 Jun 2026 00:19:33 GMT`, and `531529`, `551481`, `545031`, `587044` and `584282`
|
|
@@ -34,8 +34,8 @@
|
|
|
34
34
|
* 2. **The download URL is composed from a base the feed itself declares.** The service document's
|
|
35
35
|
* own `rel="next"` links are `https://services.cuzk.gov.cz/gml/inspire/ad/epsg-4258` and
|
|
36
36
|
* `…/epsg-5514`, and a dataset feed's `alternate` link for the same municipality is that base
|
|
37
|
-
* plus `/<code>.zip`.
|
|
38
|
-
* halves
|
|
37
|
+
* plus `/<code>.zip`. The harvest composes from the declared base instead of reading 6,258
|
|
38
|
+
* dataset feeds. That halves its requests. {@linkcode HarvestCzCuzkOptions.resolveThroughDatasetFeed}
|
|
39
39
|
* reads the publisher's own href instead, for a run that will not compose a URL.
|
|
40
40
|
* 3. **The `type` attribute describes the data rather than the file.** A dataset feed's link
|
|
41
41
|
* advertises `application/gml+xml` while the host serves `application/zip`, so the GML is inside
|
|
@@ -43,15 +43,14 @@
|
|
|
43
43
|
* arrives: the adapter opens it with `adm-zip`, because at least one archive's central directory
|
|
44
44
|
* records its member's size as the ZIP64 sentinel `0xFFFFFFFF`.
|
|
45
45
|
* 4. **A dataset feed's `length` is accurate here, and is still not used.** `584061.zip` claims
|
|
46
|
-
* 24,846 bytes and the host's `content-length`
|
|
46
|
+
* 24,846 bytes and the host's `content-length` reports 24,846. Denmark's feed understates its file
|
|
47
47
|
* by 23 times, so every byte count this module records is counted off the delivered body.
|
|
48
48
|
*
|
|
49
49
|
* The archives are small — a systematic sample of 50 of the 6,258 municipalities on 2026-10-02 ran
|
|
50
50
|
* from 5,236 to 1,642,586 bytes with a median of 16,591 — so the harvest is bound by round trips
|
|
51
51
|
* rather than by bandwidth, and it is dispatched serially through one `APIClient`. ČÚZK publishes
|
|
52
52
|
* no rate limit. {@linkcode CZ_CUZK_REQUEST_INTERVAL_MS} holds the harvest to four requests per
|
|
53
|
-
* second against one government host
|
|
54
|
-
* requests measured, and a caller that has asked ČÚZK for more raises it.
|
|
53
|
+
* second against one government host. A caller that has asked ČÚZK for more raises it.
|
|
55
54
|
*/
|
|
56
55
|
|
|
57
56
|
/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
|
|
@@ -59,7 +58,7 @@
|
|
|
59
58
|
|
|
60
59
|
import { APIClient, assertNoOGCServiceException } from "@mailwoman/core/api"
|
|
61
60
|
import { ByteFormatter } from "@mailwoman/core/fs/formatters"
|
|
62
|
-
import { tryStat } from "@mailwoman/core/fs/readers"
|
|
61
|
+
import { tryStat } from "@mailwoman/core/fs/readers/stat"
|
|
63
62
|
import { makeDirectories, writeLocalFile } from "@mailwoman/core/fs/writers"
|
|
64
63
|
import { sha256File, sha256Hex } from "@mailwoman/core/hash"
|
|
65
64
|
import { stringifyJSON } from "@mailwoman/core/json"
|
|
@@ -77,12 +76,13 @@ import { loadCollectionFiles, writeManifest } from "#tools/fetch/download"
|
|
|
77
76
|
export const CZ_CUZK_SERVICE_FEED_URL = "https://atom.cuzk.gov.cz/AD/AD.xml"
|
|
78
77
|
|
|
79
78
|
/**
|
|
80
|
-
* The directory the harvest writes under
|
|
79
|
+
* The directory the harvest writes under.
|
|
80
|
+
* This is the adapter's `inputPath`.
|
|
81
81
|
*/
|
|
82
82
|
const SLUG = CZ_CUZK_ADAPTER_ID
|
|
83
83
|
|
|
84
84
|
/**
|
|
85
|
-
* The attribution the model card must
|
|
85
|
+
* The attribution the model card must state for this source.
|
|
86
86
|
*/
|
|
87
87
|
export const CZ_CUZK_ATTRIBUTION = "Český úřad zeměměřický a katastrální (ČÚZK), INSPIRE téma Adresy"
|
|
88
88
|
|
|
@@ -136,7 +136,9 @@ const MANIFEST_INTERVAL = 25
|
|
|
136
136
|
*/
|
|
137
137
|
export interface CzMunicipalityDataset {
|
|
138
138
|
/**
|
|
139
|
-
* The municipality code
|
|
139
|
+
* The municipality code.
|
|
140
|
+
*
|
|
141
|
+
* It is both the `obec` code and the archive's file name stem.
|
|
140
142
|
*/
|
|
141
143
|
code: string
|
|
142
144
|
/**
|
|
@@ -144,11 +146,13 @@ export interface CzMunicipalityDataset {
|
|
|
144
146
|
*/
|
|
145
147
|
title: string
|
|
146
148
|
/**
|
|
147
|
-
* The `<updated>` the service document states
|
|
149
|
+
* The `<updated>` the service document states.
|
|
150
|
+
* This value is the archive's modification time.
|
|
148
151
|
*/
|
|
149
152
|
updated: string
|
|
150
153
|
/**
|
|
151
|
-
* The dataset feed for this municipality
|
|
154
|
+
* The dataset feed for this municipality.
|
|
155
|
+
* It lists the same data in both projections.
|
|
152
156
|
*/
|
|
153
157
|
datasetFeedURL: string
|
|
154
158
|
/**
|
|
@@ -156,7 +160,8 @@ export interface CzMunicipalityDataset {
|
|
|
156
160
|
*/
|
|
157
161
|
downloadURL: string
|
|
158
162
|
/**
|
|
159
|
-
* The file name the archive is written under
|
|
163
|
+
* The file name the archive is written under.
|
|
164
|
+
* The adapter globs for this name.
|
|
160
165
|
*/
|
|
161
166
|
filename: string
|
|
162
167
|
}
|
|
@@ -179,8 +184,8 @@ const TITLE_CODE = /\[(\d+)\]\s*$/u
|
|
|
179
184
|
* and stored under, so a disagreement between the publisher's two statements of it
|
|
180
185
|
* would silently store one municipality's addresses under another's name.
|
|
181
186
|
*
|
|
182
|
-
* @throws When either statement is missing or the two disagree
|
|
183
|
-
* in the publisher's layout rather than as a municipality without a code.
|
|
187
|
+
* @throws When either statement is missing or the two disagree.
|
|
188
|
+
* That reads as a change in the publisher's layout rather than as a municipality without a code.
|
|
184
189
|
*/
|
|
185
190
|
export function municipalityCodeOf(entry: { title: string; identifierCode: string | null }): string {
|
|
186
191
|
const fromIdentifier = IDENTIFIER_CODE_SUFFIX.exec(entry.identifierCode ?? "")?.[1]
|
|
@@ -236,8 +241,9 @@ export function projectionBaseURL(feed: AtomFeed, projection: CzCuzkProjection):
|
|
|
236
241
|
/**
|
|
237
242
|
* The municipalities the service document lists, with the archive URL for one projection.
|
|
238
243
|
*
|
|
239
|
-
* @throws When the feed lists no entry, when two entries claim one municipality code
|
|
240
|
-
*
|
|
244
|
+
* @throws When the feed lists no entry, when two entries claim one municipality code,
|
|
245
|
+
* or when an entry states no dataset feed link.
|
|
246
|
+
* A repeated code would make one archive overwrite the other.
|
|
241
247
|
*/
|
|
242
248
|
export function readCzCuzkServiceFeed(feed: AtomFeed, projection: CzCuzkProjection): readonly CzMunicipalityDataset[] {
|
|
243
249
|
if (!feed.entries.length) {
|
|
@@ -265,7 +271,7 @@ export function readCzCuzkServiceFeed(feed: AtomFeed, projection: CzCuzkProjecti
|
|
|
265
271
|
|
|
266
272
|
// The entry's `alternate` link is its dataset feed.
|
|
267
273
|
// The `describedby` link beside it is the ISO 19139 metadata record.
|
|
268
|
-
//
|
|
274
|
+
// Either link read as the other costs a request that answers XML of the wrong kind.
|
|
269
275
|
const datasetFeed = linkWithRel(entry.links, "alternate")
|
|
270
276
|
|
|
271
277
|
if (!datasetFeed?.href) {
|
|
@@ -321,7 +327,7 @@ export interface CzCuzkArchiveEntry extends SourceManifest {
|
|
|
321
327
|
/**
|
|
322
328
|
* The `<updated>` the service document stated when this archive was fetched.
|
|
323
329
|
*
|
|
324
|
-
* A later run that reads the same value for this municipality
|
|
330
|
+
* A later run that reads the same value for this municipality does not fetch the archive.
|
|
325
331
|
* This is the field that makes the harvest resumable without a request per municipality.
|
|
326
332
|
*/
|
|
327
333
|
feed_updated: string
|
|
@@ -356,16 +362,18 @@ export interface CzCuzkHarvestManifest extends SourceCollectionManifest {
|
|
|
356
362
|
|
|
357
363
|
export interface HarvestCzCuzkOptions {
|
|
358
364
|
/**
|
|
359
|
-
* Where the archives and the manifest are written
|
|
365
|
+
* Where the archives and the manifest are written.
|
|
366
|
+
* This is the adapter's `inputPath`.
|
|
360
367
|
*/
|
|
361
368
|
outputDir: PathBuilderLike
|
|
362
369
|
/**
|
|
363
370
|
* Which projection to harvest.
|
|
364
|
-
*
|
|
371
|
+
*
|
|
372
|
+
* @defaultValue `epsg-4258`
|
|
365
373
|
*/
|
|
366
374
|
projection?: CzCuzkProjection
|
|
367
375
|
/**
|
|
368
|
-
* Harvest only these municipality codes, for a probe or
|
|
376
|
+
* Harvest only these municipality codes, for a probe or to repair specific municipalities.
|
|
369
377
|
*
|
|
370
378
|
* A code the service document does not list is reported as a failure rather than ignored.
|
|
371
379
|
*/
|
|
@@ -386,8 +394,10 @@ export interface HarvestCzCuzkOptions {
|
|
|
386
394
|
/**
|
|
387
395
|
* Re-read the sha256 of every archive already on disk instead of comparing its byte count.
|
|
388
396
|
*
|
|
389
|
-
* The default compares the recorded byte count against the file's size
|
|
390
|
-
*
|
|
397
|
+
* The default compares the recorded byte count against the file's size.
|
|
398
|
+
* That check takes one `stat`.
|
|
399
|
+
*
|
|
400
|
+
* This option re-hashes and reads every archive in the directory.
|
|
391
401
|
*/
|
|
392
402
|
verifyDigests?: boolean
|
|
393
403
|
signal?: AbortSignal
|
|
@@ -402,7 +412,7 @@ export interface HarvestCzCuzkOptions {
|
|
|
402
412
|
* the file is still there, and it is still the length that was recorded for it.
|
|
403
413
|
*/
|
|
404
414
|
async function isCurrent(
|
|
405
|
-
recorded: CzCuzkArchiveEntry |
|
|
415
|
+
recorded: CzCuzkArchiveEntry | null,
|
|
406
416
|
dataset: CzMunicipalityDataset,
|
|
407
417
|
path: PathBuilderLike,
|
|
408
418
|
verifyDigests: boolean
|
|
@@ -413,7 +423,8 @@ async function isCurrent(
|
|
|
413
423
|
|
|
414
424
|
if (!stat || stat.size !== recorded.bytes) return false
|
|
415
425
|
|
|
416
|
-
// The digest check reads the whole file rather than its metadata
|
|
426
|
+
// The digest check reads the whole file rather than its metadata.
|
|
427
|
+
// This option is opt-in for that reason.
|
|
417
428
|
if (!verifyDigests) return true
|
|
418
429
|
|
|
419
430
|
return (await sha256File(path)) === recorded.sha256
|
|
@@ -467,8 +478,8 @@ export async function harvestCzCuzk(
|
|
|
467
478
|
const previous = await loadCollectionFiles(manifestPath)
|
|
468
479
|
const files = new Map<string, CzCuzkArchiveEntry>()
|
|
469
480
|
|
|
470
|
-
// An entry for a municipality this run does not consider is
|
|
471
|
-
//
|
|
481
|
+
// An entry for a municipality this run does not consider is kept, so a bounded
|
|
482
|
+
// run never drops what an earlier run recorded.
|
|
472
483
|
for (const [filename, entry] of previous) {
|
|
473
484
|
files.set(filename, entry as CzCuzkArchiveEntry)
|
|
474
485
|
}
|
|
@@ -503,7 +514,7 @@ export async function harvestCzCuzk(
|
|
|
503
514
|
|
|
504
515
|
const dest = destDir(dataset.filename)
|
|
505
516
|
|
|
506
|
-
if (await isCurrent(files.get(dataset.filename), dataset, dest, options.verifyDigests ?? false)) {
|
|
517
|
+
if (await isCurrent(files.get(dataset.filename) ?? null, dataset, dest, options.verifyDigests ?? false)) {
|
|
507
518
|
skipped++
|
|
508
519
|
|
|
509
520
|
continue
|
|
@@ -544,7 +555,7 @@ export async function harvestCzCuzk(
|
|
|
544
555
|
}
|
|
545
556
|
|
|
546
557
|
/**
|
|
547
|
-
* The municipalities this run considers, and the codes a caller
|
|
558
|
+
* The municipalities this run considers, and the codes a caller supplied that the feed does not list.
|
|
548
559
|
*
|
|
549
560
|
* A code the caller asked for and the service document does not list is returned
|
|
550
561
|
* rather than dropped, because a municipality that is not published is a different
|
|
@@ -660,7 +671,7 @@ export interface FetchCzCuzkOptions
|
|
|
660
671
|
/**
|
|
661
672
|
* The minimum spacing between two requests, in milliseconds.
|
|
662
673
|
*
|
|
663
|
-
*
|
|
674
|
+
* @defaultValue {@linkcode CZ_CUZK_REQUEST_INTERVAL_MS}
|
|
664
675
|
*/
|
|
665
676
|
minRequestIntervalMs?: number
|
|
666
677
|
}
|
package/lib/de/recipes/locale.ts
CHANGED
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
*/
|
|
23
23
|
|
|
24
24
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
25
|
-
import { pathExists } from "@mailwoman/core/fs/readers"
|
|
25
|
+
import { pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
26
26
|
import { stringifyJSON } from "@mailwoman/core/json"
|
|
27
27
|
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
28
28
|
import { wofDatabasePath } from "@mailwoman/resolver-wof-sqlite/paths"
|