@mailwoman/corpus 10.1.0 → 10.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/licenses/cdla-permissive-2.0.md +1 -1
- package/lib/adapters/cordis/adapter.ts +13 -7
- package/lib/adapters/dedup-key.ts +1 -1
- package/lib/adapters/geonames/adapter.ts +1 -1
- package/lib/adapters/gleif/adapter.ts +17 -16
- package/lib/adapters/libpostal-dictionary.ts +0 -2
- package/lib/adapters/openaddresses/adapter.ts +4 -3
- package/lib/adapters/osm/adapter.ts +22 -21
- package/lib/adapters/po-box/adapter.ts +10 -5
- package/lib/adapters/registry.ts +4 -4
- package/lib/adapters/source-id.ts +2 -2
- package/lib/adapters/street-line.ts +7 -6
- package/lib/adapters/wof/admin/json/adapter.ts +2 -2
- package/lib/adapters/wof/json-rows.ts +2 -1
- package/lib/adapters/wof/postalcode-json/adapter.ts +6 -2
- package/lib/adapters.ts +5 -5
- package/lib/au/adapters/gnaf/adapter.ts +1 -1
- package/lib/be/adapters/brussels/adapter.ts +22 -22
- package/lib/be/adapters/vlaanderen/adapter.ts +54 -57
- package/lib/be/adapters/wallonie/adapter.ts +17 -17
- package/lib/be/tools/fetch/brussels.ts +13 -10
- package/lib/be/tools/fetch/wallonie.ts +8 -7
- package/lib/build/adapters.ts +2 -1
- package/lib/build/align.ts +5 -4
- package/lib/build/checkpoint.ts +5 -3
- package/lib/build/eligibility.ts +3 -3
- package/lib/build/types.ts +8 -6
- package/lib/cz/adapters/cuzk/adapter.ts +41 -38
- package/lib/cz/tools/fetch/cuzk.ts +42 -31
- package/lib/de/recipes/locale.ts +1 -1
- package/lib/dk/adapters/inspire/adapter.ts +38 -38
- package/lib/dk/tools/fetch/inspire.ts +16 -14
- package/lib/ee/adapters/ads/adapter.ts +26 -32
- package/lib/ee/tools/fetch/ads.ts +10 -8
- package/lib/es/adapters/bizkaia/adapter.ts +47 -47
- package/lib/es/adapters/catastro/adapter.ts +36 -34
- package/lib/es/adapters/gipuzkoa/adapter.ts +38 -45
- package/lib/es/adapters/navarra/adapter.ts +28 -27
- package/lib/es/cadastre.ts +31 -31
- package/lib/es/tools/fetch/bizkaia.ts +40 -36
- package/lib/es/tools/fetch/catastro.ts +51 -40
- package/lib/es/tools/fetch/gipuzkoa.ts +13 -11
- package/lib/es/tools/fetch/navarra.ts +33 -25
- package/lib/fi/adapters/ryhti/adapter.ts +22 -26
- package/lib/fi/tools/fetch/ryhti.ts +31 -31
- package/lib/fr/adapters/annuaire-education/adapter.ts +13 -11
- package/lib/fr/adapters/ban/adapter.ts +6 -6
- package/lib/fr/adapters/finess/adapter.ts +16 -15
- package/lib/fr/adapters/finess/voie-type.ts +4 -3
- package/lib/fr/insee-country.ts +2 -2
- package/lib/fr/recipes/lieudit.ts +8 -6
- package/lib/fr/tools/fetch/annuaire-education.ts +11 -8
- package/lib/fr/tools/fetch/ban.ts +3 -2
- package/lib/fr/tools/fetch/finess.ts +14 -13
- package/lib/inspire/address.ts +86 -78
- package/lib/inspire/errors.ts +9 -8
- package/lib/inspire/stream.ts +29 -26
- package/lib/international/recipes/locale.ts +13 -9
- package/lib/it/adapters/anac/adapter.ts +51 -46
- package/lib/it/tools/fetch/anac.ts +22 -25
- package/lib/jp/adapters/wof-admin/adapter.ts +2 -6
- package/lib/jp/tools/fetch/houjin.ts +3 -3
- package/lib/kr/adapters/localdata/permit-rows.ts +1 -1
- package/lib/kr/tools/fetch/juso.ts +1 -1
- package/lib/kr/tools/fetch/localdata.ts +2 -2
- package/lib/lu/adapters/bd-adresses/adapter.ts +10 -10
- package/lib/lu/tools/fetch/bd-adresses.ts +15 -13
- package/lib/lv/adapters/varis/adapter.ts +24 -23
- package/lib/lv/tools/fetch/varis.ts +3 -2
- package/lib/nl/adapters/kadaster/adapter.ts +29 -29
- package/lib/nl/tools/fetch/kadaster.ts +16 -12
- package/lib/no/adapters/matrikkelen/adapter.ts +6 -5
- package/lib/no/tools/fetch/matrikkelen.ts +59 -47
- package/lib/parquet/duckdb.ts +7 -4
- package/lib/parquet/readers.ts +3 -3
- package/lib/parquet/schema.ts +1 -1
- package/lib/parquet/writers.ts +3 -2
- package/lib/pl/adapters/emuia/adapter.ts +12 -12
- package/lib/pl/tools/fetch/emuia.ts +10 -9
- package/lib/recipes/bare/postcode.ts +1 -1
- package/lib/recipes/boundary-stress.ts +1 -1
- package/lib/recipes/house-venue.ts +7 -3
- package/lib/recipes/intersection.ts +2 -2
- package/lib/recipes/po/box/cedex/recipe.ts +3 -3
- package/lib/recipes/po/box/cedex/sources.ts +13 -1
- package/lib/recipes/scaffold.ts +31 -21
- package/lib/recipes/sources.ts +16 -3
- package/lib/recipes/street/bare.ts +2 -2
- package/lib/recipes/street.ts +3 -1
- package/lib/recipes/sub/venue/context.ts +2 -0
- package/lib/recipes/sub/venue/render.ts +1 -1
- package/lib/recipes/sub/venue/sources.ts +24 -29
- package/lib/recipes/sub/venue.ts +12 -18
- package/lib/recipes/trailing-region.ts +6 -2
- package/lib/recipes/unit.ts +3 -3
- package/lib/recipes/venue.ts +347 -0
- package/lib/recipes.ts +4 -2
- package/lib/registers.ts +8 -6
- package/lib/ro/adapters/retea-scolara/adapter.ts +5 -5
- package/lib/ro/tools/fetch/retea-scolara.ts +1 -1
- package/lib/runner.ts +4 -3
- package/lib/sg/recipes/register.ts +1 -1
- package/lib/sg/tools/fetch/acra.ts +3 -3
- package/lib/sk/adapters/inspire/adapter.ts +25 -27
- package/lib/sk/tools/fetch/inspire.ts +15 -14
- package/lib/source-register/effective-manifest.ts +3 -3
- package/lib/source-register/register.ts +3 -3
- package/lib/source-register/training-manifest.ts +1 -1
- package/lib/source-register/types.ts +12 -13
- package/lib/subvenue/promotions.ts +1 -1
- package/lib/subvenue/table.ts +4 -4
- package/lib/surfaces/locale.ts +2 -0
- package/lib/synthesizers/anchor-absorption.ts +2 -1
- package/lib/synthesizers/boundary-stress.ts +20 -26
- package/lib/synthesizers/house-venue.ts +26 -30
- package/lib/synthesizers/no-street.ts +2 -1
- package/lib/synthesizers/po-box.ts +6 -3
- package/lib/synthesizers/tail.ts +55 -0
- package/lib/synthesizers/utils.ts +19 -2
- package/lib/test-kit.ts +2 -2
- package/lib/tools/audit.ts +7 -5
- package/lib/tools/corpus-stats.ts +1 -1
- package/lib/tools/fetch/atom.ts +10 -10
- package/lib/tools/fetch/cordis.ts +11 -9
- package/lib/tools/fetch/download/manifest.ts +2 -1
- package/lib/tools/fetch/download/network.ts +4 -4
- package/lib/tools/fetch/download/retry.ts +4 -4
- package/lib/tools/fetch/geonames/dump.ts +5 -3
- package/lib/tools/fetch/geonames/postal.ts +2 -2
- package/lib/tools/fetch/gleif.ts +6 -4
- package/lib/tools/fetch/header.ts +8 -6
- package/lib/tools/fetch/inspire-addresses.ts +41 -36
- package/lib/tools/fetch/openaddresses.ts +1 -1
- package/lib/tools/fetch/vlaanderen-ad.ts +17 -17
- package/lib/tools/fetch/wfs-harvest.ts +39 -28
- package/lib/tools/fetch.ts +0 -42
- package/lib/tools/golden/expand.ts +8 -8
- package/lib/tools/golden/promote.ts +12 -8
- package/lib/tools/golden/relabel-street/directory.ts +7 -6
- package/lib/tools/golden/relabel-street/row.ts +7 -6
- package/lib/tools/ingest-csv.ts +3 -3
- package/lib/tools/jsonl-to-parquet.ts +1 -1
- package/lib/tools/lint/recipe-output/vocab.ts +10 -5
- package/lib/tools/lint/recipe-output.ts +26 -8
- package/lib/tools/overlay/kryptonite.ts +4 -3
- package/lib/tools/overlay/manifest.ts +4 -2
- package/lib/tools/overlay/merge-source.ts +2 -1
- package/lib/tools/overlay/split-slice.ts +2 -2
- package/lib/tools/overlay/translit.ts +6 -4
- package/lib/tools/overture-subvenue.ts +1 -1
- package/lib/tools/postcode-triples.ts +5 -4
- package/lib/tools/source-register/build.ts +17 -14
- package/lib/tools/sub/venue/lexicon.ts +4 -8
- package/lib/tw/tools/fetch/gcis.ts +4 -4
- package/lib/types.ts +4 -3
- package/lib/us/adapters/state/hi-schools/workbook.ts +1 -1
- package/lib/us/adapters/usgov/nad/adapter.ts +11 -11
- package/lib/us/adapters/usgov/samhsa-treatment-locator/adapter.ts +4 -4
- package/lib/us/fips-state.ts +2 -2
- package/lib/{synthesizers → us/synthesizers}/intersection.ts +12 -13
- package/lib/{synthesizers → us/synthesizers}/street.ts +16 -15
- package/lib/us/tools/fetch/imls-pls.ts +1 -1
- package/lib/us/tools/fetch/nad.ts +7 -7
- package/lib/us/tools/fetch/nppes.ts +5 -5
- package/lib/us/tools/fetch/state/hi-schools.ts +3 -2
- package/lib/us/tools/fetch/tiger-full.ts +1 -1
- package/lib/utils/align.ts +13 -10
- package/lib/utils/golden.ts +2 -2
- package/lib/utils/license.ts +3 -3
- package/lib/utils/split.ts +19 -15
- package/lib/utils/tokenize.ts +7 -26
- package/lib/utils/wof-json.ts +4 -4
- package/out/adapters/cordis/adapter.d.ts +5 -3
- package/out/adapters/cordis/adapter.d.ts.map +1 -1
- package/out/adapters/cordis/adapter.js +13 -7
- package/out/adapters/cordis/adapter.js.map +1 -1
- package/out/adapters/dedup-key.d.ts +1 -1
- package/out/adapters/dedup-key.js +1 -1
- package/out/adapters/geonames/adapter.js +1 -1
- package/out/adapters/geonames/adapter.js.map +1 -1
- package/out/adapters/gleif/adapter.d.ts +9 -9
- package/out/adapters/gleif/adapter.d.ts.map +1 -1
- package/out/adapters/gleif/adapter.js +15 -14
- package/out/adapters/gleif/adapter.js.map +1 -1
- package/out/adapters/libpostal-dictionary.d.ts +0 -2
- package/out/adapters/libpostal-dictionary.d.ts.map +1 -1
- package/out/adapters/libpostal-dictionary.js +0 -2
- package/out/adapters/libpostal-dictionary.js.map +1 -1
- package/out/adapters/openaddresses/adapter.d.ts +3 -2
- package/out/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/adapters/openaddresses/adapter.js +1 -1
- package/out/adapters/openaddresses/adapter.js.map +1 -1
- package/out/adapters/osm/adapter.d.ts +17 -16
- package/out/adapters/osm/adapter.d.ts.map +1 -1
- package/out/adapters/osm/adapter.js +10 -9
- package/out/adapters/osm/adapter.js.map +1 -1
- package/out/adapters/po-box/adapter.d.ts +6 -3
- package/out/adapters/po-box/adapter.d.ts.map +1 -1
- package/out/adapters/po-box/adapter.js +4 -2
- package/out/adapters/po-box/adapter.js.map +1 -1
- package/out/adapters/registry.d.ts +3 -3
- package/out/adapters/registry.d.ts.map +1 -1
- package/out/adapters/registry.js +1 -1
- package/out/adapters/registry.js.map +1 -1
- package/out/adapters/source-id.d.ts +2 -2
- package/out/adapters/source-id.js +2 -2
- package/out/adapters/street-line.d.ts +7 -6
- package/out/adapters/street-line.d.ts.map +1 -1
- package/out/adapters/street-line.js +7 -6
- package/out/adapters/street-line.js.map +1 -1
- package/out/adapters/wof/admin/json/adapter.js +1 -1
- package/out/adapters/wof/admin/json/adapter.js.map +1 -1
- package/out/adapters/wof/json-rows.d.ts +2 -1
- package/out/adapters/wof/json-rows.d.ts.map +1 -1
- package/out/adapters/wof/json-rows.js.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.d.ts.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.js +5 -1
- package/out/adapters/wof/postalcode-json/adapter.js.map +1 -1
- package/out/adapters.d.ts +5 -5
- package/out/adapters.js +5 -5
- package/out/au/adapters/gnaf/adapter.js +1 -1
- package/out/au/adapters/gnaf/adapter.js.map +1 -1
- package/out/be/adapters/brussels/adapter.d.ts +9 -9
- package/out/be/adapters/brussels/adapter.d.ts.map +1 -1
- package/out/be/adapters/brussels/adapter.js +21 -21
- package/out/be/adapters/brussels/adapter.js.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.d.ts +34 -37
- package/out/be/adapters/vlaanderen/adapter.d.ts.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.js +40 -38
- package/out/be/adapters/vlaanderen/adapter.js.map +1 -1
- package/out/be/adapters/wallonie/adapter.d.ts +2 -2
- package/out/be/adapters/wallonie/adapter.js +17 -17
- package/out/be/adapters/wallonie/adapter.js.map +1 -1
- package/out/be/tools/fetch/brussels.d.ts +11 -8
- package/out/be/tools/fetch/brussels.d.ts.map +1 -1
- package/out/be/tools/fetch/brussels.js +10 -8
- package/out/be/tools/fetch/brussels.js.map +1 -1
- package/out/be/tools/fetch/wallonie.d.ts +6 -5
- package/out/be/tools/fetch/wallonie.d.ts.map +1 -1
- package/out/be/tools/fetch/wallonie.js +4 -4
- package/out/be/tools/fetch/wallonie.js.map +1 -1
- package/out/build/adapters.d.ts.map +1 -1
- package/out/build/adapters.js +2 -1
- package/out/build/adapters.js.map +1 -1
- package/out/build/align.d.ts +3 -2
- package/out/build/align.d.ts.map +1 -1
- package/out/build/align.js +2 -2
- package/out/build/align.js.map +1 -1
- package/out/build/checkpoint.d.ts +1 -1
- package/out/build/checkpoint.d.ts.map +1 -1
- package/out/build/checkpoint.js +4 -2
- package/out/build/checkpoint.js.map +1 -1
- package/out/build/eligibility.d.ts +2 -2
- package/out/build/eligibility.js +3 -3
- package/out/build/eligibility.js.map +1 -1
- package/out/build/types.d.ts +8 -6
- package/out/build/types.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.d.ts +14 -14
- package/out/cz/adapters/cuzk/adapter.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.js +38 -35
- package/out/cz/adapters/cuzk/adapter.js.map +1 -1
- package/out/cz/tools/fetch/cuzk.d.ts +31 -22
- package/out/cz/tools/fetch/cuzk.d.ts.map +1 -1
- package/out/cz/tools/fetch/cuzk.js +21 -19
- package/out/cz/tools/fetch/cuzk.js.map +1 -1
- package/out/de/recipes/locale.js +1 -1
- package/out/de/recipes/locale.js.map +1 -1
- package/out/dk/adapters/inspire/adapter.d.ts +32 -33
- package/out/dk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/dk/adapters/inspire/adapter.js +38 -38
- package/out/dk/adapters/inspire/adapter.js.map +1 -1
- package/out/dk/tools/fetch/inspire.d.ts +6 -5
- package/out/dk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/dk/tools/fetch/inspire.js +15 -13
- package/out/dk/tools/fetch/inspire.js.map +1 -1
- package/out/ee/adapters/ads/adapter.d.ts +19 -26
- package/out/ee/adapters/ads/adapter.d.ts.map +1 -1
- package/out/ee/adapters/ads/adapter.js +22 -28
- package/out/ee/adapters/ads/adapter.js.map +1 -1
- package/out/ee/tools/fetch/ads.d.ts +3 -3
- package/out/ee/tools/fetch/ads.d.ts.map +1 -1
- package/out/ee/tools/fetch/ads.js +10 -8
- package/out/ee/tools/fetch/ads.js.map +1 -1
- package/out/es/adapters/bizkaia/adapter.d.ts +28 -28
- package/out/es/adapters/bizkaia/adapter.d.ts.map +1 -1
- package/out/es/adapters/bizkaia/adapter.js +43 -43
- package/out/es/adapters/bizkaia/adapter.js.map +1 -1
- package/out/es/adapters/catastro/adapter.d.ts +22 -23
- package/out/es/adapters/catastro/adapter.d.ts.map +1 -1
- package/out/es/adapters/catastro/adapter.js +35 -33
- package/out/es/adapters/catastro/adapter.js.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.d.ts +31 -38
- package/out/es/adapters/gipuzkoa/adapter.d.ts.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.js +40 -47
- package/out/es/adapters/gipuzkoa/adapter.js.map +1 -1
- package/out/es/adapters/navarra/adapter.d.ts +20 -19
- package/out/es/adapters/navarra/adapter.d.ts.map +1 -1
- package/out/es/adapters/navarra/adapter.js +23 -22
- package/out/es/adapters/navarra/adapter.js.map +1 -1
- package/out/es/cadastre.d.ts +21 -20
- package/out/es/cadastre.d.ts.map +1 -1
- package/out/es/cadastre.js +19 -21
- package/out/es/cadastre.js.map +1 -1
- package/out/es/tools/fetch/bizkaia.d.ts +28 -25
- package/out/es/tools/fetch/bizkaia.d.ts.map +1 -1
- package/out/es/tools/fetch/bizkaia.js +25 -24
- package/out/es/tools/fetch/bizkaia.js.map +1 -1
- package/out/es/tools/fetch/catastro.d.ts +28 -20
- package/out/es/tools/fetch/catastro.d.ts.map +1 -1
- package/out/es/tools/fetch/catastro.js +36 -31
- package/out/es/tools/fetch/catastro.js.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.d.ts +11 -9
- package/out/es/tools/fetch/gipuzkoa.d.ts.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.js +9 -8
- package/out/es/tools/fetch/gipuzkoa.js.map +1 -1
- package/out/es/tools/fetch/navarra.d.ts +20 -15
- package/out/es/tools/fetch/navarra.d.ts.map +1 -1
- package/out/es/tools/fetch/navarra.js +21 -16
- package/out/es/tools/fetch/navarra.js.map +1 -1
- package/out/fi/adapters/ryhti/adapter.d.ts +16 -20
- package/out/fi/adapters/ryhti/adapter.d.ts.map +1 -1
- package/out/fi/adapters/ryhti/adapter.js +21 -25
- package/out/fi/adapters/ryhti/adapter.js.map +1 -1
- package/out/fi/tools/fetch/ryhti.d.ts +24 -24
- package/out/fi/tools/fetch/ryhti.d.ts.map +1 -1
- package/out/fi/tools/fetch/ryhti.js +21 -24
- package/out/fi/tools/fetch/ryhti.js.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.d.ts +8 -8
- package/out/fr/adapters/annuaire-education/adapter.d.ts.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.js +13 -11
- package/out/fr/adapters/annuaire-education/adapter.js.map +1 -1
- package/out/fr/adapters/ban/adapter.d.ts +4 -4
- package/out/fr/adapters/ban/adapter.js +6 -6
- package/out/fr/adapters/ban/adapter.js.map +1 -1
- package/out/fr/adapters/finess/adapter.d.ts +12 -11
- package/out/fr/adapters/finess/adapter.d.ts.map +1 -1
- package/out/fr/adapters/finess/adapter.js +16 -15
- package/out/fr/adapters/finess/adapter.js.map +1 -1
- package/out/fr/adapters/finess/voie-type.d.ts +2 -2
- package/out/fr/adapters/finess/voie-type.d.ts.map +1 -1
- package/out/fr/adapters/finess/voie-type.js +4 -3
- package/out/fr/adapters/finess/voie-type.js.map +1 -1
- package/out/fr/insee-country.d.ts +2 -2
- package/out/fr/insee-country.js +2 -2
- package/out/fr/recipes/lieudit.d.ts.map +1 -1
- package/out/fr/recipes/lieudit.js +5 -4
- package/out/fr/recipes/lieudit.js.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.d.ts +7 -4
- package/out/fr/tools/fetch/annuaire-education.d.ts.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.js +9 -7
- package/out/fr/tools/fetch/annuaire-education.js.map +1 -1
- package/out/fr/tools/fetch/ban.d.ts +1 -1
- package/out/fr/tools/fetch/ban.d.ts.map +1 -1
- package/out/fr/tools/fetch/ban.js +3 -2
- package/out/fr/tools/fetch/ban.js.map +1 -1
- package/out/fr/tools/fetch/finess.d.ts +5 -5
- package/out/fr/tools/fetch/finess.d.ts.map +1 -1
- package/out/fr/tools/fetch/finess.js +13 -12
- package/out/fr/tools/fetch/finess.js.map +1 -1
- package/out/inspire/address.d.ts +62 -52
- package/out/inspire/address.d.ts.map +1 -1
- package/out/inspire/address.js +71 -60
- package/out/inspire/address.js.map +1 -1
- package/out/inspire/errors.d.ts +9 -8
- package/out/inspire/errors.d.ts.map +1 -1
- package/out/inspire/errors.js +7 -6
- package/out/inspire/errors.js.map +1 -1
- package/out/inspire/stream.d.ts +25 -22
- package/out/inspire/stream.d.ts.map +1 -1
- package/out/inspire/stream.js +10 -12
- package/out/inspire/stream.js.map +1 -1
- package/out/international/recipes/locale.d.ts +2 -2
- package/out/international/recipes/locale.d.ts.map +1 -1
- package/out/international/recipes/locale.js +11 -7
- package/out/international/recipes/locale.js.map +1 -1
- package/out/it/adapters/anac/adapter.d.ts +45 -43
- package/out/it/adapters/anac/adapter.d.ts.map +1 -1
- package/out/it/adapters/anac/adapter.js +48 -43
- package/out/it/adapters/anac/adapter.js.map +1 -1
- package/out/it/tools/fetch/anac.d.ts +16 -19
- package/out/it/tools/fetch/anac.d.ts.map +1 -1
- package/out/it/tools/fetch/anac.js +20 -23
- package/out/it/tools/fetch/anac.js.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.d.ts.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.js.map +1 -1
- package/out/jp/tools/fetch/houjin.js +2 -2
- package/out/jp/tools/fetch/houjin.js.map +1 -1
- package/out/kr/adapters/localdata/permit-rows.js +1 -1
- package/out/kr/adapters/localdata/permit-rows.js.map +1 -1
- package/out/kr/tools/fetch/juso.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.js +1 -1
- package/out/kr/tools/fetch/localdata.js.map +1 -1
- package/out/lu/adapters/bd-adresses/adapter.d.ts +9 -9
- package/out/lu/adapters/bd-adresses/adapter.js +10 -10
- package/out/lu/tools/fetch/bd-adresses.d.ts +11 -9
- package/out/lu/tools/fetch/bd-adresses.d.ts.map +1 -1
- package/out/lu/tools/fetch/bd-adresses.js +11 -11
- package/out/lu/tools/fetch/bd-adresses.js.map +1 -1
- package/out/lv/adapters/varis/adapter.d.ts +24 -23
- package/out/lv/adapters/varis/adapter.d.ts.map +1 -1
- package/out/lv/adapters/varis/adapter.js +24 -23
- package/out/lv/adapters/varis/adapter.js.map +1 -1
- package/out/lv/tools/fetch/varis.d.ts +2 -1
- package/out/lv/tools/fetch/varis.d.ts.map +1 -1
- package/out/lv/tools/fetch/varis.js +1 -1
- package/out/lv/tools/fetch/varis.js.map +1 -1
- package/out/nl/adapters/kadaster/adapter.d.ts +12 -12
- package/out/nl/adapters/kadaster/adapter.d.ts.map +1 -1
- package/out/nl/adapters/kadaster/adapter.js +22 -23
- package/out/nl/adapters/kadaster/adapter.js.map +1 -1
- package/out/nl/tools/fetch/kadaster.d.ts +14 -10
- package/out/nl/tools/fetch/kadaster.d.ts.map +1 -1
- package/out/nl/tools/fetch/kadaster.js +12 -9
- package/out/nl/tools/fetch/kadaster.js.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.js +3 -3
- package/out/no/adapters/matrikkelen/adapter.js.map +1 -1
- package/out/no/tools/fetch/matrikkelen.d.ts +50 -38
- package/out/no/tools/fetch/matrikkelen.d.ts.map +1 -1
- package/out/no/tools/fetch/matrikkelen.js +44 -36
- package/out/no/tools/fetch/matrikkelen.js.map +1 -1
- package/out/parquet/duckdb.d.ts +7 -4
- package/out/parquet/duckdb.d.ts.map +1 -1
- package/out/parquet/duckdb.js +2 -1
- package/out/parquet/duckdb.js.map +1 -1
- package/out/parquet/readers.js +3 -3
- package/out/parquet/readers.js.map +1 -1
- package/out/parquet/schema.js +1 -1
- package/out/parquet/schema.js.map +1 -1
- package/out/parquet/writers.d.ts +1 -1
- package/out/parquet/writers.d.ts.map +1 -1
- package/out/parquet/writers.js +2 -1
- package/out/parquet/writers.js.map +1 -1
- package/out/pl/adapters/emuia/adapter.d.ts +8 -8
- package/out/pl/adapters/emuia/adapter.js +12 -12
- package/out/pl/adapters/emuia/adapter.js.map +1 -1
- package/out/pl/tools/fetch/emuia.d.ts +5 -4
- package/out/pl/tools/fetch/emuia.d.ts.map +1 -1
- package/out/pl/tools/fetch/emuia.js +10 -9
- package/out/pl/tools/fetch/emuia.js.map +1 -1
- package/out/recipes/bare/postcode.js +1 -1
- package/out/recipes/bare/postcode.js.map +1 -1
- package/out/recipes/boundary-stress.js +1 -1
- package/out/recipes/boundary-stress.js.map +1 -1
- package/out/recipes/house-venue.d.ts.map +1 -1
- package/out/recipes/house-venue.js +6 -3
- package/out/recipes/house-venue.js.map +1 -1
- package/out/recipes/intersection.js +2 -2
- package/out/recipes/intersection.js.map +1 -1
- package/out/recipes/po/box/cedex/recipe.js +2 -1
- package/out/recipes/po/box/cedex/recipe.js.map +1 -1
- package/out/recipes/po/box/cedex/sources.d.ts +8 -2
- package/out/recipes/po/box/cedex/sources.d.ts.map +1 -1
- package/out/recipes/po/box/cedex/sources.js +1 -1
- package/out/recipes/po/box/cedex/sources.js.map +1 -1
- package/out/recipes/scaffold.d.ts +28 -19
- package/out/recipes/scaffold.d.ts.map +1 -1
- package/out/recipes/scaffold.js +3 -2
- package/out/recipes/scaffold.js.map +1 -1
- package/out/recipes/sources.d.ts +1 -1
- package/out/recipes/sources.d.ts.map +1 -1
- package/out/recipes/sources.js +15 -2
- package/out/recipes/sources.js.map +1 -1
- package/out/recipes/street/bare.js +2 -2
- package/out/recipes/street/bare.js.map +1 -1
- package/out/recipes/street.d.ts.map +1 -1
- package/out/recipes/street.js +3 -1
- package/out/recipes/street.js.map +1 -1
- package/out/recipes/sub/venue/context.d.ts.map +1 -1
- package/out/recipes/sub/venue/context.js +2 -0
- package/out/recipes/sub/venue/context.js.map +1 -1
- package/out/recipes/sub/venue/render.js +1 -1
- package/out/recipes/sub/venue/render.js.map +1 -1
- package/out/recipes/sub/venue/sources.d.ts +6 -3
- package/out/recipes/sub/venue/sources.d.ts.map +1 -1
- package/out/recipes/sub/venue/sources.js +16 -20
- package/out/recipes/sub/venue/sources.js.map +1 -1
- package/out/recipes/sub/venue.d.ts +2 -5
- package/out/recipes/sub/venue.d.ts.map +1 -1
- package/out/recipes/sub/venue.js +10 -14
- package/out/recipes/sub/venue.js.map +1 -1
- package/out/recipes/trailing-region.d.ts.map +1 -1
- package/out/recipes/trailing-region.js +6 -1
- package/out/recipes/trailing-region.js.map +1 -1
- package/out/recipes/unit.js +2 -2
- package/out/recipes/unit.js.map +1 -1
- package/out/recipes/venue.d.ts +119 -0
- package/out/recipes/venue.d.ts.map +1 -0
- package/out/recipes/venue.js +245 -0
- package/out/recipes/venue.js.map +1 -0
- package/out/recipes.d.ts +1 -1
- package/out/recipes.d.ts.map +1 -1
- package/out/recipes.js +3 -1
- package/out/recipes.js.map +1 -1
- package/out/registers.d.ts +8 -6
- package/out/registers.d.ts.map +1 -1
- package/out/registers.js +8 -6
- package/out/registers.js.map +1 -1
- package/out/ro/adapters/retea-scolara/adapter.d.ts +5 -5
- package/out/ro/adapters/retea-scolara/adapter.js +5 -5
- package/out/ro/tools/fetch/retea-scolara.js +1 -1
- package/out/ro/tools/fetch/retea-scolara.js.map +1 -1
- package/out/runner.d.ts +3 -2
- package/out/runner.d.ts.map +1 -1
- package/out/runner.js +1 -1
- package/out/runner.js.map +1 -1
- package/out/sg/recipes/register.js +1 -1
- package/out/sg/recipes/register.js.map +1 -1
- package/out/sg/tools/fetch/acra.js +2 -2
- package/out/sg/tools/fetch/acra.js.map +1 -1
- package/out/sk/adapters/inspire/adapter.d.ts +5 -5
- package/out/sk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/sk/adapters/inspire/adapter.js +18 -18
- package/out/sk/adapters/inspire/adapter.js.map +1 -1
- package/out/sk/tools/fetch/inspire.d.ts +11 -10
- package/out/sk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/sk/tools/fetch/inspire.js +15 -14
- package/out/sk/tools/fetch/inspire.js.map +1 -1
- package/out/source-register/effective-manifest.js +3 -3
- package/out/source-register/effective-manifest.js.map +1 -1
- package/out/source-register/register.d.ts +2 -2
- package/out/source-register/register.d.ts.map +1 -1
- package/out/source-register/register.js +2 -2
- package/out/source-register/register.js.map +1 -1
- package/out/source-register/training-manifest.d.ts +1 -1
- package/out/source-register/types.d.ts +12 -13
- package/out/source-register/types.d.ts.map +1 -1
- package/out/source-register/types.js +2 -2
- package/out/source-register/types.js.map +1 -1
- package/out/subvenue/promotions.d.ts +1 -1
- package/out/subvenue/promotions.js +1 -1
- package/out/subvenue/table.d.ts +4 -4
- package/out/subvenue/table.js +3 -3
- package/out/surfaces/locale.d.ts.map +1 -1
- package/out/surfaces/locale.js +2 -0
- package/out/surfaces/locale.js.map +1 -1
- package/out/synthesizers/anchor-absorption.d.ts +2 -1
- package/out/synthesizers/anchor-absorption.d.ts.map +1 -1
- package/out/synthesizers/anchor-absorption.js.map +1 -1
- package/out/synthesizers/boundary-stress.d.ts +1 -1
- package/out/synthesizers/boundary-stress.d.ts.map +1 -1
- package/out/synthesizers/boundary-stress.js +20 -23
- package/out/synthesizers/boundary-stress.js.map +1 -1
- package/out/synthesizers/house-venue.d.ts +3 -2
- package/out/synthesizers/house-venue.d.ts.map +1 -1
- package/out/synthesizers/house-venue.js +24 -26
- package/out/synthesizers/house-venue.js.map +1 -1
- package/out/synthesizers/no-street.d.ts.map +1 -1
- package/out/synthesizers/no-street.js.map +1 -1
- package/out/synthesizers/po-box.d.ts +4 -2
- package/out/synthesizers/po-box.d.ts.map +1 -1
- package/out/synthesizers/po-box.js.map +1 -1
- package/out/synthesizers/tail.d.ts +36 -0
- package/out/synthesizers/tail.d.ts.map +1 -0
- package/out/synthesizers/tail.js +32 -0
- package/out/synthesizers/tail.js.map +1 -0
- package/out/synthesizers/utils.d.ts +2 -1
- package/out/synthesizers/utils.d.ts.map +1 -1
- package/out/synthesizers/utils.js +17 -1
- package/out/synthesizers/utils.js.map +1 -1
- package/out/test-kit.js +2 -2
- package/out/test-kit.js.map +1 -1
- package/out/tools/audit.d.ts +2 -1
- package/out/tools/audit.d.ts.map +1 -1
- package/out/tools/audit.js +4 -3
- package/out/tools/audit.js.map +1 -1
- package/out/tools/corpus-stats.js +1 -1
- package/out/tools/corpus-stats.js.map +1 -1
- package/out/tools/fetch/atom.d.ts +8 -8
- package/out/tools/fetch/atom.d.ts.map +1 -1
- package/out/tools/fetch/atom.js +7 -8
- package/out/tools/fetch/atom.js.map +1 -1
- package/out/tools/fetch/cordis.d.ts +4 -3
- package/out/tools/fetch/cordis.d.ts.map +1 -1
- package/out/tools/fetch/cordis.js +9 -7
- package/out/tools/fetch/cordis.js.map +1 -1
- package/out/tools/fetch/download/manifest.d.ts.map +1 -1
- package/out/tools/fetch/download/manifest.js +2 -1
- package/out/tools/fetch/download/manifest.js.map +1 -1
- package/out/tools/fetch/download/network.d.ts +4 -4
- package/out/tools/fetch/download/retry.js +4 -4
- package/out/tools/fetch/download/retry.js.map +1 -1
- package/out/tools/fetch/geonames/dump.d.ts +3 -2
- package/out/tools/fetch/geonames/dump.d.ts.map +1 -1
- package/out/tools/fetch/geonames/dump.js +2 -1
- package/out/tools/fetch/geonames/dump.js.map +1 -1
- package/out/tools/fetch/geonames/postal.d.ts +2 -2
- package/out/tools/fetch/gleif.d.ts +3 -2
- package/out/tools/fetch/gleif.d.ts.map +1 -1
- package/out/tools/fetch/gleif.js +6 -4
- package/out/tools/fetch/gleif.js.map +1 -1
- package/out/tools/fetch/header.d.ts +7 -5
- package/out/tools/fetch/header.d.ts.map +1 -1
- package/out/tools/fetch/header.js +8 -6
- package/out/tools/fetch/header.js.map +1 -1
- package/out/tools/fetch/inspire-addresses.d.ts +32 -27
- package/out/tools/fetch/inspire-addresses.d.ts.map +1 -1
- package/out/tools/fetch/inspire-addresses.js +25 -24
- package/out/tools/fetch/inspire-addresses.js.map +1 -1
- package/out/tools/fetch/openaddresses.js +1 -1
- package/out/tools/fetch/openaddresses.js.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.d.ts +5 -5
- package/out/tools/fetch/vlaanderen-ad.d.ts.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.js +16 -16
- package/out/tools/fetch/vlaanderen-ad.js.map +1 -1
- package/out/tools/fetch/wfs-harvest.d.ts +26 -16
- package/out/tools/fetch/wfs-harvest.d.ts.map +1 -1
- package/out/tools/fetch/wfs-harvest.js +21 -18
- package/out/tools/fetch/wfs-harvest.js.map +1 -1
- package/out/tools/fetch.d.ts +0 -41
- package/out/tools/fetch.d.ts.map +1 -1
- package/out/tools/fetch.js +0 -41
- package/out/tools/fetch.js.map +1 -1
- package/out/tools/golden/expand.d.ts +7 -7
- package/out/tools/golden/expand.js +1 -1
- package/out/tools/golden/expand.js.map +1 -1
- package/out/tools/golden/promote.d.ts +2 -2
- package/out/tools/golden/promote.d.ts.map +1 -1
- package/out/tools/golden/promote.js +9 -6
- package/out/tools/golden/promote.js.map +1 -1
- package/out/tools/golden/relabel-street/directory.d.ts +4 -4
- package/out/tools/golden/relabel-street/directory.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/directory.js +3 -2
- package/out/tools/golden/relabel-street/directory.js.map +1 -1
- package/out/tools/golden/relabel-street/row.d.ts +4 -3
- package/out/tools/golden/relabel-street/row.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/row.js +3 -3
- package/out/tools/golden/relabel-street/row.js.map +1 -1
- package/out/tools/ingest-csv.js +3 -3
- package/out/tools/ingest-csv.js.map +1 -1
- package/out/tools/jsonl-to-parquet.d.ts +1 -1
- package/out/tools/lint/recipe-output/vocab.d.ts +10 -5
- package/out/tools/lint/recipe-output/vocab.d.ts.map +1 -1
- package/out/tools/lint/recipe-output/vocab.js.map +1 -1
- package/out/tools/lint/recipe-output.d.ts +8 -8
- package/out/tools/lint/recipe-output.d.ts.map +1 -1
- package/out/tools/lint/recipe-output.js +17 -0
- package/out/tools/lint/recipe-output.js.map +1 -1
- package/out/tools/overlay/kryptonite.d.ts +2 -2
- package/out/tools/overlay/kryptonite.d.ts.map +1 -1
- package/out/tools/overlay/kryptonite.js +2 -1
- package/out/tools/overlay/kryptonite.js.map +1 -1
- package/out/tools/overlay/manifest.d.ts +2 -1
- package/out/tools/overlay/manifest.d.ts.map +1 -1
- package/out/tools/overlay/manifest.js +2 -1
- package/out/tools/overlay/manifest.js.map +1 -1
- package/out/tools/overlay/merge-source.d.ts +2 -1
- package/out/tools/overlay/merge-source.d.ts.map +1 -1
- package/out/tools/overlay/merge-source.js.map +1 -1
- package/out/tools/overlay/split-slice.d.ts +1 -1
- package/out/tools/overlay/split-slice.js +1 -1
- package/out/tools/overlay/split-slice.js.map +1 -1
- package/out/tools/overlay/translit.d.ts +4 -3
- package/out/tools/overlay/translit.d.ts.map +1 -1
- package/out/tools/overlay/translit.js +2 -1
- package/out/tools/overlay/translit.js.map +1 -1
- package/out/tools/overture-subvenue.d.ts +1 -1
- package/out/tools/overture-subvenue.d.ts.map +1 -1
- package/out/tools/postcode-triples.d.ts +1 -1
- package/out/tools/postcode-triples.d.ts.map +1 -1
- package/out/tools/postcode-triples.js +4 -3
- package/out/tools/postcode-triples.js.map +1 -1
- package/out/tools/source-register/build.d.ts +10 -8
- package/out/tools/source-register/build.d.ts.map +1 -1
- package/out/tools/source-register/build.js +10 -8
- package/out/tools/source-register/build.js.map +1 -1
- package/out/tools/sub/venue/lexicon.d.ts +2 -6
- package/out/tools/sub/venue/lexicon.d.ts.map +1 -1
- package/out/tools/sub/venue/lexicon.js +2 -6
- package/out/tools/sub/venue/lexicon.js.map +1 -1
- package/out/tw/tools/fetch/gcis.js +3 -3
- package/out/tw/tools/fetch/gcis.js.map +1 -1
- package/out/types.d.ts +4 -3
- package/out/types.d.ts.map +1 -1
- package/out/types.js.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.js.map +1 -1
- package/out/us/adapters/usgov/nad/adapter.js +6 -6
- package/out/us/adapters/usgov/nad/adapter.js.map +1 -1
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js +2 -2
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/us/fips-state.d.ts +2 -2
- package/out/us/fips-state.d.ts.map +1 -1
- package/out/us/fips-state.js.map +1 -1
- package/out/us/synthesizers/intersection.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/intersection.js +13 -9
- package/out/us/synthesizers/intersection.js.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.d.ts +3 -2
- package/out/us/synthesizers/street.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.js +12 -12
- package/out/us/synthesizers/street.js.map +1 -0
- package/out/us/tools/fetch/imls-pls.js +1 -1
- package/out/us/tools/fetch/imls-pls.js.map +1 -1
- package/out/us/tools/fetch/nad.d.ts +6 -6
- package/out/us/tools/fetch/nad.js +1 -1
- package/out/us/tools/fetch/nad.js.map +1 -1
- package/out/us/tools/fetch/nppes.js +3 -3
- package/out/us/tools/fetch/nppes.js.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts +2 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.js +1 -1
- package/out/us/tools/fetch/state/hi-schools.js.map +1 -1
- package/out/us/tools/fetch/tiger-full.js +1 -1
- package/out/us/tools/fetch/tiger-full.js.map +1 -1
- package/out/utils/align.d.ts +5 -3
- package/out/utils/align.d.ts.map +1 -1
- package/out/utils/align.js +5 -5
- package/out/utils/align.js.map +1 -1
- package/out/utils/golden.js +2 -2
- package/out/utils/golden.js.map +1 -1
- package/out/utils/license.d.ts +3 -3
- package/out/utils/license.d.ts.map +1 -1
- package/out/utils/license.js.map +1 -1
- package/out/utils/split.d.ts +1 -1
- package/out/utils/split.d.ts.map +1 -1
- package/out/utils/split.js +18 -14
- package/out/utils/split.js.map +1 -1
- package/out/utils/tokenize.d.ts +3 -20
- package/out/utils/tokenize.d.ts.map +1 -1
- package/out/utils/tokenize.js.map +1 -1
- package/out/utils/wof-json.js +3 -3
- package/out/utils/wof-json.js.map +1 -1
- package/package.json +18 -27
- package/lib/tools.ts +0 -31
- package/out/synthesizers/intersection.d.ts.map +0 -1
- package/out/synthesizers/intersection.js.map +0 -1
- package/out/synthesizers/street.d.ts.map +0 -1
- package/out/synthesizers/street.js.map +0 -1
- package/out/tools.d.ts +0 -31
- package/out/tools.d.ts.map +0 -1
- package/out/tools.js +0 -31
- package/out/tools.js.map +0 -1
- /package/out/{synthesizers → us/synthesizers}/intersection.d.ts +0 -0
|
@@ -4,41 +4,41 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
6
|
* `it-anac`: the OCDS release Autorità Nazionale Anticorruzione publishes for Italy's National
|
|
7
|
-
* Database of Public Contracts, read for the addresses of the public bodies
|
|
7
|
+
* Database of Public Contracts, read for the addresses of the public bodies that appear on each contract.
|
|
8
8
|
*
|
|
9
9
|
* Input is the publisher's own JSON Lines: one OCDS release per line, as
|
|
10
10
|
* `https://data.open-contracting.org/en/publication/117/download?name=<year>.jsonl.gz` serves it
|
|
11
|
-
* once decompressed. A release
|
|
11
|
+
* once decompressed. A release has a `parties` array, and each party holds a `roles` list, a
|
|
12
12
|
* `name`, an `identifier` and an `address` whose four fields are `locality`, `postalCode`,
|
|
13
13
|
* `countryName` and `streetAddress`. The adapter streams the file through `JSONSpliterator`, so a
|
|
14
14
|
* year's release never sits in memory whole. `opts.inputPath` is one `.jsonl` edition or a directory
|
|
15
|
-
* holding several
|
|
15
|
+
* holding several. `fetchITANAC` writes the directory form.
|
|
16
16
|
*
|
|
17
17
|
* ## Which roles the adapter reads, and why those two
|
|
18
18
|
*
|
|
19
|
-
* The register's `addressRoles` for `it-procurement-grants-1`
|
|
19
|
+
* The register's `addressRoles` for `it-procurement-grants-1` records exactly two fields,
|
|
20
20
|
* `parties[roles=buyer].address` and `parties[roles=payer].address`.
|
|
21
|
-
* Those two are the only address fields the release
|
|
21
|
+
* Those two are the only address fields in the release. Measured over the whole 2025 edition, 9,405 releases and
|
|
22
22
|
* 29,823 party objects in three roles: a `supplier` party holds no `address` key at all on 11,075 of
|
|
23
23
|
* 11,075, a `buyer` holds one on 9,381 of 9,381, and a `payer` on 9,359 of 9,367. So every address
|
|
24
24
|
* in this release belongs to a contracting authority or to the office that pays.
|
|
25
25
|
* Both of those are Italian public entities, and a contracting counterparty holds none.
|
|
26
26
|
*
|
|
27
|
-
* That is what makes this source admissible under the policy that
|
|
27
|
+
* That is what makes this source admissible under the policy that admits the addresses of
|
|
28
28
|
* organizations and no address of an identifiable natural person. The two admitted roles are held by
|
|
29
29
|
* public bodies, and the one role whose parties can be sole traders publishes no address.
|
|
30
30
|
* {@linkcode readANACParty} still applies a name check and refuses a party whose name takes the
|
|
31
|
-
* `Surname, Firstname` form, because a later edition could
|
|
31
|
+
* `Surname, Firstname` form, because a later edition could add one. On the 2025 edition the check
|
|
32
32
|
* refuses no party: 0 of the 18,740 address-bearing buyer and payer parties take that form, while 56
|
|
33
33
|
* hold a comma inside an organization name — `PARCO NAZIONALE DEL CILENTO, VALLO DI DIANO E
|
|
34
|
-
* ALBURNI` and `SETTORE PATRIMONIO, AMBIENTE E LAVORI PUBBLICI` among them
|
|
35
|
-
* requires
|
|
34
|
+
* ALBURNI` and `SETTORE PATRIMONIO, AMBIENTE E LAVORI PUBBLICI` among them. The check therefore
|
|
35
|
+
* requires one word on each side of the comma rather than any comma.
|
|
36
36
|
*
|
|
37
37
|
* ## The street line
|
|
38
38
|
*
|
|
39
|
-
* Italy writes the house number after the street name
|
|
40
|
-
*
|
|
41
|
-
*
|
|
39
|
+
* Italy writes the house number after the street name, joined by a space. `STREET_ORDERS` records
|
|
40
|
+
* that order as `number-last`. `COMMA_JOINED_STREET_COUNTRIES` records it by leaving `IT` out.
|
|
41
|
+
* {@linkcode splitTrailingStreetLine} performs the split. Three Italian
|
|
42
42
|
* conventions are normalized before it runs, each measured over the 2025 edition's 11,930
|
|
43
43
|
* non-placeholder street lines:
|
|
44
44
|
*
|
|
@@ -52,23 +52,24 @@
|
|
|
52
52
|
* 3. **`senza numero`.** 182 lines end `SNC` or `SN`, which states that the premise has no civic
|
|
53
53
|
* number, so the tail is dropped and no `house_number` is composed from it.
|
|
54
54
|
*
|
|
55
|
-
* A
|
|
56
|
-
*
|
|
57
|
-
*
|
|
55
|
+
* A number preceded by `KM` names a point on a state highway rather than a house number, as
|
|
56
|
+
* `S.S. 7 APPIA KM 671` does. That number stays in the street line, and the rule below refuses the
|
|
57
|
+
* row.
|
|
58
58
|
*
|
|
59
59
|
* **A street line holding a digit the split could not place refuses the row.**
|
|
60
60
|
* {@linkcode ANACRefusal.StreetNumberUnplaced} covers 194 of those 11,930 lines, where the publisher
|
|
61
61
|
* appends a qualifier after the number: `VIA CIRCONVALLAZIONE, 1 PARCO IDROSCALO`, `VIA FELTRE 57
|
|
62
|
-
* 32100 BELLUNO (BL)`, `VIA SAN SECONDO 29 BIS`.
|
|
63
|
-
* a street name. A line with no digit at all is emitted as a
|
|
64
|
-
*
|
|
62
|
+
* 32100 BELLUNO (BL)`, `VIA SAN SECONDO 29 BIS`. The adapter refuses that line, because emitting it
|
|
63
|
+
* would label a civic number as part of a street name. A line with no digit at all is emitted as a
|
|
64
|
+
* street with no number. `PIAZZA CASTELLO` is the ordinary case, and 1,416 of the 11,930 lines take
|
|
65
|
+
* it. The remaining 10,320 state a number.
|
|
65
66
|
*
|
|
66
67
|
* Two limits of the split are the publisher's own ambiguity rather than this reader's. `PIAZZA
|
|
67
68
|
* UMBERTO 1` is Umberto I's square and reads identically to civic number 1, and the release gives no
|
|
68
69
|
* field that separates them. A number written twice, as `CORSO ITALIA 55 55` and `VIALE DEI PIANETI 1
|
|
69
|
-
* 1`, leaves the first copy in the street name. Together with
|
|
70
|
-
* NOVEMBRE`, `PIAZZA 1 MAGGIO`, `VIA 2 AGOSTO 1980` — 93 of the 10,320 numbered lines keep a
|
|
71
|
-
* inside the street, and most of those digits belong to the street's name.
|
|
70
|
+
* 1`, leaves the first copy in the street name. Together with streets that take a date for a name —
|
|
71
|
+
* `VIA 4 NOVEMBRE`, `PIAZZA 1 MAGGIO`, `VIA 2 AGOSTO 1980` — 93 of the 10,320 numbered lines keep a
|
|
72
|
+
* digit inside the street, and most of those digits belong to the street's name.
|
|
72
73
|
*
|
|
73
74
|
* ## What the split was checked against
|
|
74
75
|
*
|
|
@@ -82,15 +83,15 @@
|
|
|
82
83
|
* published token. Which side of the split each token belongs on is the part no field of this
|
|
83
84
|
* publisher's confirms.
|
|
84
85
|
*
|
|
85
|
-
* Every other component is a published value copied unchanged
|
|
86
|
-
* 18,473 rows the 2025 edition admits
|
|
86
|
+
* Every other component is a published value copied unchanged. The same pass checked this: 0 of the
|
|
87
|
+
* 18,473 rows the 2025 edition admits show a `locality`, `postcode` or `venue` that differs from the
|
|
87
88
|
* field it came from.
|
|
88
89
|
*
|
|
89
90
|
* ## Placeholders and the postcode
|
|
90
91
|
*
|
|
91
92
|
* The publisher writes the literal `N.A.` where it holds no value: 476 postcodes, 6 localities and 6
|
|
92
93
|
* street lines on the 2025 edition. {@linkcode isPlaceholderValue} treats it as absent, so no row
|
|
93
|
-
*
|
|
94
|
+
* records `N.A.` as a postcode.
|
|
94
95
|
*
|
|
95
96
|
* An Italian CAP is five digits. A postcode that is neither a placeholder nor five digits is not a
|
|
96
97
|
* CAP, and the row is refused rather than stripped of the field: the one such value on the 2025
|
|
@@ -102,14 +103,14 @@
|
|
|
102
103
|
* ## Identity and license
|
|
103
104
|
*
|
|
104
105
|
* The row id is content-addressed over the aligned components. The publisher's party identifier
|
|
105
|
-
* cannot serve: a buyer's `IT-CF` fiscal code
|
|
106
|
+
* cannot serve: a buyer's `IT-CF` fiscal code covers more than one address on 17 of 3,119 distinct
|
|
106
107
|
* codes, and one address recurs across hundreds of releases, so the identifier identifies the
|
|
107
108
|
* organization rather than the address.
|
|
108
109
|
*
|
|
109
|
-
* The OCP Data Registry publishes this release under CC BY 4.0
|
|
110
|
-
*
|
|
111
|
-
* statement that changes were made
|
|
112
|
-
* {@linkcode IT_ANAC_ATTRIBUTION} is the credit the model card
|
|
110
|
+
* The OCP Data Registry publishes this release under CC BY 4.0. The address-source register elected
|
|
111
|
+
* that license on 2026-09-30 with `spdx` `CC-BY-4.0`. CC BY requires credit, a link to the license
|
|
112
|
+
* and a statement that changes were made. Every row records the license, and
|
|
113
|
+
* {@linkcode IT_ANAC_ATTRIBUTION} is the credit the model card records.
|
|
113
114
|
*
|
|
114
115
|
* The adapter honors `opts.limit`, `opts.signal` and `opts.country`.
|
|
115
116
|
*/
|
|
@@ -148,15 +149,16 @@ export const IT_ANAC_COUNTRIES: readonly string[] = ["IT"]
|
|
|
148
149
|
export const IT_ANAC_LICENSE = "CC-BY-4.0"
|
|
149
150
|
|
|
150
151
|
/**
|
|
151
|
-
* The credit CC BY 4.0 obliges
|
|
152
|
-
*
|
|
152
|
+
* The credit CC BY 4.0 obliges.
|
|
153
|
+
*
|
|
154
|
+
* The model card records it beside the license and a statement that the data was changed.
|
|
153
155
|
*/
|
|
154
156
|
export const IT_ANAC_ATTRIBUTION = "Autorità Nazionale Anticorruzione (ANAC), via the OCP Data Registry"
|
|
155
157
|
|
|
156
158
|
/**
|
|
157
159
|
* The two party roles the adapter reads.
|
|
158
160
|
*
|
|
159
|
-
* The register's `addressRoles`
|
|
161
|
+
* The register's `addressRoles` records exactly these two fields.
|
|
160
162
|
* The `supplier` role is the one role whose parties are contracting counterparties
|
|
161
163
|
* rather than public bodies.
|
|
162
164
|
* That role publishes no `address` object.
|
|
@@ -172,12 +174,12 @@ export const IT_ANAC_PARTY_ROLES: readonly string[] = ["buyer", "payer"]
|
|
|
172
174
|
*/
|
|
173
175
|
export const ANACRefusal = {
|
|
174
176
|
/**
|
|
175
|
-
* The party holds a role other than `buyer` or `payer`, so its address is
|
|
176
|
-
* of scope even where
|
|
177
|
+
* The party holds a role other than `buyer` or `payer`, so its address is
|
|
178
|
+
* out of scope even where one appears.
|
|
177
179
|
*/
|
|
178
180
|
RoleNotAdmitted: "role-not-admitted",
|
|
179
181
|
/**
|
|
180
|
-
* The party
|
|
182
|
+
* The party has no `address` object, or every field in it is empty or a placeholder.
|
|
181
183
|
*/
|
|
182
184
|
AddressAbsent: "address-absent",
|
|
183
185
|
/**
|
|
@@ -205,10 +207,10 @@ export const ANACRefusal = {
|
|
|
205
207
|
*/
|
|
206
208
|
Unrenderable: "unrenderable",
|
|
207
209
|
/**
|
|
208
|
-
* The rendered line
|
|
210
|
+
* The rendered line holds one component, so it is a place name rather than an address.
|
|
209
211
|
*
|
|
210
|
-
* A party
|
|
211
|
-
* and
|
|
212
|
+
* A party leaves only the locality when its name holds no letter, its postcode is
|
|
213
|
+
* the publisher's placeholder and its street line is absent.
|
|
212
214
|
*/
|
|
213
215
|
ComponentsTooFew: "components-too-few",
|
|
214
216
|
} as const
|
|
@@ -245,9 +247,9 @@ export interface ANACParty {
|
|
|
245
247
|
/**
|
|
246
248
|
* One line of the input: an OCDS release.
|
|
247
249
|
*
|
|
248
|
-
* The adapter reads `parties
|
|
249
|
-
* A release also
|
|
250
|
-
*
|
|
250
|
+
* The adapter reads only `parties`.
|
|
251
|
+
* A release also has `tender`, `awards` and `contracts`, and no address sits
|
|
252
|
+
* on any of them in this publication.
|
|
251
253
|
*/
|
|
252
254
|
export interface ANACRelease {
|
|
253
255
|
ocid?: string
|
|
@@ -293,8 +295,9 @@ const CIVIC_MARKER = /(?<=^|[\s,])(?:n|nr|num|civ|civico)\s*\.?\s*(?=\d)/giu
|
|
|
293
295
|
const SENZA_NUMERO = /[\s,]+(?:snc|sn)\.?$/iu
|
|
294
296
|
|
|
295
297
|
/**
|
|
296
|
-
* A number introduced by a kilometer marker
|
|
297
|
-
*
|
|
298
|
+
* A number introduced by a kilometer marker.
|
|
299
|
+
*
|
|
300
|
+
* Such a number points at a place on a highway rather than a premise on a street.
|
|
298
301
|
*/
|
|
299
302
|
const KILOMETRE_POINT = /[\s,]+km\.?[\s,]*\d+(?:[.,]\d+)?$/iu
|
|
300
303
|
|
|
@@ -337,8 +340,8 @@ function fieldValue(value: string | null | undefined): string {
|
|
|
337
340
|
* The street line with the three Italian conventions normalized away: a trailing copy
|
|
338
341
|
* of the locality, a civic marker before the number, and a `senza numero` tail.
|
|
339
342
|
*
|
|
340
|
-
* `locality` is the publisher's own value for this party
|
|
341
|
-
* authorizes removing the trailing copy.
|
|
343
|
+
* `locality` is the publisher's own value for this party.
|
|
344
|
+
* That value authorizes removing the trailing copy.
|
|
342
345
|
* An empty string leaves the line as it is.
|
|
343
346
|
*/
|
|
344
347
|
export function normalizeANACStreetLine(line: string, locality: string): string {
|
|
@@ -497,7 +500,9 @@ export function createITANACAdapter(): CorpusAdapter {
|
|
|
497
500
|
* The registry serves one file per year beside an all-time `full.jsonl.gz`,
|
|
498
501
|
* and `fetchITANAC` writes whichever editions a caller asked for into one directory,
|
|
499
502
|
* so a build reading several years points at that directory.
|
|
500
|
-
* The editions are read in sorted order
|
|
503
|
+
* The editions are read in sorted order.
|
|
504
|
+
*
|
|
505
|
+
* That puts the years in chronological order and `full` after them.
|
|
501
506
|
*
|
|
502
507
|
* A directory holding no edition raises rather than yielding zero rows, so a wrong path
|
|
503
508
|
* reports itself instead of reading as a publisher with no contracts.
|
|
@@ -8,31 +8,29 @@
|
|
|
8
8
|
* The Open Contracting Partnership's Data Registry serves publication 117 at
|
|
9
9
|
* `https://data.open-contracting.org/en/publication/117/download?name=<edition>.jsonl.gz`, where an
|
|
10
10
|
* edition is a year, `undated`, or `full` for every year in one file. Each file is JSON Lines, one
|
|
11
|
-
* OCDS release per line
|
|
11
|
+
* OCDS release per line. That is the shape `#it/adapters/anac/adapter` reads.
|
|
12
12
|
*
|
|
13
13
|
* The acquisition belongs under a `tools/` root. `@mailwoman/corpus` is one of the
|
|
14
14
|
* `TOOLING_PACKAGES` in `dependency-cruiser.config.mjs`, which keep their tooling under `lib/`, so
|
|
15
15
|
* this module sits beside `#fi/tools/fetch/ryhti` rather than in an `sdk/` root the workspace does
|
|
16
16
|
* not declare.
|
|
17
17
|
*
|
|
18
|
-
* Three
|
|
18
|
+
* Three host behaviors decide what this module does:
|
|
19
19
|
*
|
|
20
20
|
* 1. **The host honors `Range`.** It answers `accept-ranges: bytes`, so an interrupted transfer
|
|
21
|
-
* resumes through {@linkcode resumableDownload} rather than restarting. That matters for
|
|
22
|
-
* `full` edition
|
|
23
|
-
* 2. **An edition's length and modification time are both served by `HEAD`.**
|
|
24
|
-
*
|
|
25
|
-
* Sep 2026 03:31:23 GMT` with `content-length` 3,963,957. The re-run check compares the pair
|
|
26
|
-
* against the manifest, which costs one small request per edition instead of the body.
|
|
21
|
+
* resumes through {@linkcode resumableDownload} rather than restarting. That matters most for
|
|
22
|
+
* the `full` edition.
|
|
23
|
+
* 2. **An edition's length and modification time are both served by `HEAD`.** The re-run check
|
|
24
|
+
* compares the pair against the manifest, so one small request per edition replaces the body.
|
|
27
25
|
* 3. **This module moves bytes and decodes none of them.** The compressed body is streamed to disk
|
|
28
26
|
* and then decompressed stream to stream, so no chunk boundary is interpreted as a character
|
|
29
|
-
* boundary. ANAC's data
|
|
30
|
-
* gzip chunk
|
|
27
|
+
* boundary. ANAC's data holds accented Italian place names, and a `chunk.toString("utf8")` per
|
|
28
|
+
* gzip chunk would write U+FFFD at every split multi-byte character.
|
|
31
29
|
*
|
|
32
|
-
* ANAC updates the publication monthly, on the second of the month
|
|
30
|
+
* ANAC updates the publication monthly, on the second of the month. The current year's
|
|
33
31
|
* edition is re-fetched on a `HEAD` disagreement rather than treated as final.
|
|
34
32
|
*
|
|
35
|
-
* The registry publishes this release under CC BY 4.0
|
|
33
|
+
* The registry publishes this release under CC BY 4.0. The address-source register elected it.
|
|
36
34
|
* The manifest records the license and the attribution, and the adapter records the license on every
|
|
37
35
|
* row.
|
|
38
36
|
*/
|
|
@@ -40,7 +38,7 @@
|
|
|
40
38
|
import { APIClient } from "@mailwoman/core/api"
|
|
41
39
|
import { gunzipChunks } from "@mailwoman/core/fs/compression"
|
|
42
40
|
import { ByteFormatter } from "@mailwoman/core/fs/formatters"
|
|
43
|
-
import { tryStat } from "@mailwoman/core/fs/readers"
|
|
41
|
+
import { tryStat } from "@mailwoman/core/fs/readers/stat"
|
|
44
42
|
import { openReadStream, openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams"
|
|
45
43
|
import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
|
|
46
44
|
import { sha256File } from "@mailwoman/core/hash"
|
|
@@ -54,10 +52,8 @@ import { loadCollectionFiles, resumableDownload, writeManifest } from "#tools/fe
|
|
|
54
52
|
* The publication the registry serves this release under.
|
|
55
53
|
*
|
|
56
54
|
* Italy has four publications on the registry and id 117 is ANAC's.
|
|
57
|
-
* Id 87 is OpenTender's Italian publication
|
|
58
|
-
*
|
|
59
|
-
* It runs to March 2024 and comes from a different body.
|
|
60
|
-
* Reading the country alone would take the wrong publication.
|
|
55
|
+
* Id 87 is OpenTender's Italian publication, so a reader that filters by country
|
|
56
|
+
* would take the wrong publication.
|
|
61
57
|
*/
|
|
62
58
|
export const IT_ANAC_PUBLICATION_ID = 117
|
|
63
59
|
|
|
@@ -69,14 +65,15 @@ export function anacEditionURL(edition: string): string {
|
|
|
69
65
|
}
|
|
70
66
|
|
|
71
67
|
/**
|
|
72
|
-
* The directory the editions are written under,
|
|
68
|
+
* The directory the editions are written under, the adapter's `inputPath`.
|
|
73
69
|
*/
|
|
74
70
|
const SLUG = IT_ANAC_ADAPTER_ID
|
|
75
71
|
|
|
76
72
|
/**
|
|
77
73
|
* The edition fetched when a caller names none.
|
|
78
74
|
*
|
|
79
|
-
* `full` is every year in one file
|
|
75
|
+
* `full` is every year in one file.
|
|
76
|
+
* A corpus build reads that edition.
|
|
80
77
|
* A caller measuring one year names that year instead.
|
|
81
78
|
*/
|
|
82
79
|
export const IT_ANAC_DEFAULT_EDITIONS: readonly string[] = ["full"]
|
|
@@ -125,7 +122,7 @@ export async function readANACEditionHead(client: Pick<APIClient, "fetch">, edit
|
|
|
125
122
|
|
|
126
123
|
return {
|
|
127
124
|
lastModified: headers?.["last-modified"] ?? null,
|
|
128
|
-
contentLength: length
|
|
125
|
+
contentLength: length && /^\d+$/u.test(length) ? Number(length) : null,
|
|
129
126
|
contentType: headers?.["content-type"] ?? null,
|
|
130
127
|
}
|
|
131
128
|
}
|
|
@@ -138,7 +135,7 @@ export interface DownloadANACOptions {
|
|
|
138
135
|
outputDir: PathBuilderLike
|
|
139
136
|
/**
|
|
140
137
|
* Which editions to fetch: a year, `undated`, or `full`.
|
|
141
|
-
*
|
|
138
|
+
* @defaultValue {@linkcode IT_ANAC_DEFAULT_EDITIONS}
|
|
142
139
|
*/
|
|
143
140
|
editions?: readonly string[]
|
|
144
141
|
/**
|
|
@@ -187,7 +184,7 @@ export function itANACInputPath(outRoot: BaseFetchOptions["outRoot"]): PathBuild
|
|
|
187
184
|
* letting {@linkcode downloadITANAC} reach the host to find out which branch it took.
|
|
188
185
|
*/
|
|
189
186
|
export async function isANACEditionCurrent(
|
|
190
|
-
recorded: ANACEditionManifest |
|
|
187
|
+
recorded: ANACEditionManifest | null,
|
|
191
188
|
head: ANACEditionHead,
|
|
192
189
|
path: PathBuilderLike,
|
|
193
190
|
verifyDigest: boolean
|
|
@@ -195,9 +192,9 @@ export async function isANACEditionCurrent(
|
|
|
195
192
|
if (!recorded) return false
|
|
196
193
|
|
|
197
194
|
// A host that states neither cannot be compared against, so the edition is downloaded.
|
|
198
|
-
if (head.lastModified
|
|
195
|
+
if (!head.lastModified && head.contentLength === null) return false
|
|
199
196
|
|
|
200
|
-
if (head.lastModified
|
|
197
|
+
if (head.lastModified && head.lastModified !== recorded.last_modified) return false
|
|
201
198
|
|
|
202
199
|
if (head.contentLength !== null && head.contentLength !== recorded.compressed_bytes) return false
|
|
203
200
|
|
|
@@ -259,7 +256,7 @@ export async function downloadITANAC(
|
|
|
259
256
|
|
|
260
257
|
const current =
|
|
261
258
|
!options.force &&
|
|
262
|
-
(await isANACEditionCurrent(recorded.get(filename), head, editionPath, options.verifyDigest ?? false))
|
|
259
|
+
(await isANACEditionCurrent(recorded.get(filename) ?? null, head, editionPath, options.verifyDigest ?? false))
|
|
263
260
|
|
|
264
261
|
if (current) {
|
|
265
262
|
report?.(` ✓ ${edition} already current — no download.`)
|
|
@@ -68,11 +68,7 @@ interface PlaceRow {
|
|
|
68
68
|
* That analogue stores the nearest ancestor's name for each tag.
|
|
69
69
|
* This function receives one `PlaceRow` per level and reads the chain directly.
|
|
70
70
|
*/
|
|
71
|
-
function chainOf(
|
|
72
|
-
byID: Map<number, PlaceRow>,
|
|
73
|
-
resolve: (id: number) => PlaceRow | undefined,
|
|
74
|
-
startID: number
|
|
75
|
-
): PlaceRow[] {
|
|
71
|
+
function chainOf(byID: Map<number, PlaceRow>, resolve: (id: number) => PlaceRow | null, startID: number): PlaceRow[] {
|
|
76
72
|
const out: PlaceRow[] = []
|
|
77
73
|
let id = startID
|
|
78
74
|
|
|
@@ -177,7 +173,7 @@ export function createWOFAdminJpAdapter(): CorpusAdapter {
|
|
|
177
173
|
|
|
178
174
|
const outsideStmt = db.prepare(`SELECT id, name, placetype, parent_id, country FROM spr WHERE id = ?`)
|
|
179
175
|
|
|
180
|
-
const resolveOutside = (id: number): PlaceRow |
|
|
176
|
+
const resolveOutside = (id: number): PlaceRow | null => {
|
|
181
177
|
const row = getRow<PlaceRow>(outsideStmt, id)
|
|
182
178
|
|
|
183
179
|
if (row) {
|
|
@@ -41,14 +41,14 @@ export type FetchHoujinJPOptions = BaseFetchOptions
|
|
|
41
41
|
/**
|
|
42
42
|
* The nationwide row of the "CSV形式・Unicode" table stores the file number in its `doDownload(N)` handler.
|
|
43
43
|
*/
|
|
44
|
-
function nationwideUnicodeFileNumber(html: string): string |
|
|
44
|
+
function nationwideUnicodeFileNumber(html: string): string | null {
|
|
45
45
|
const start = html.indexOf('id="csv-unicode"')
|
|
46
46
|
|
|
47
|
-
if (start === -1) return
|
|
47
|
+
if (start === -1) return null
|
|
48
48
|
const section = html.slice(start, html.indexOf('id="xml-unicode"', start))
|
|
49
49
|
const row = /全国[\s\S]{0,600}?doDownload\((\d+)\)/.exec(section)
|
|
50
50
|
|
|
51
|
-
return row?.[1]
|
|
51
|
+
return row?.[1] ?? null
|
|
52
52
|
}
|
|
53
53
|
|
|
54
54
|
export async function fetchHoujinJP(
|
|
@@ -73,7 +73,7 @@ function firstColumn(row: Record<string, string | undefined>, ...names: readonly
|
|
|
73
73
|
for (const name of names) {
|
|
74
74
|
const value = row[name]
|
|
75
75
|
|
|
76
|
-
if (value
|
|
76
|
+
if (value) return String(value).trim()
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
return ""
|
|
@@ -66,7 +66,7 @@ export interface FetchJusoKROptions extends BaseFetchOptions {
|
|
|
66
66
|
/**
|
|
67
67
|
* The month to fetch as `yyyymm`.
|
|
68
68
|
*
|
|
69
|
-
*
|
|
69
|
+
* @defaultValue the latest month the portal lists as present.
|
|
70
70
|
*/
|
|
71
71
|
month?: string
|
|
72
72
|
}
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
*/
|
|
27
27
|
|
|
28
28
|
import { BYTES_PER_KIB } from "@mailwoman/core/fs/formatters"
|
|
29
|
-
import { pathExists } from "@mailwoman/core/fs/readers"
|
|
29
|
+
import { pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
30
30
|
import { makeDirectories } from "@mailwoman/core/fs/writers"
|
|
31
31
|
import { sha256File } from "@mailwoman/core/hash"
|
|
32
32
|
import { sleep } from "@mailwoman/core/utils/sleep"
|
|
@@ -46,7 +46,7 @@ export interface FetchLocaldataKROptions extends BaseFetchOptions {
|
|
|
46
46
|
/**
|
|
47
47
|
* Category slugs to fetch (the path segment of `/file/<slug>/info`).
|
|
48
48
|
*
|
|
49
|
-
*
|
|
49
|
+
* @defaultValue every category the portal lists.
|
|
50
50
|
*/
|
|
51
51
|
categories?: string[]
|
|
52
52
|
}
|
|
@@ -19,11 +19,11 @@
|
|
|
19
19
|
* `CC0-1.0` for `lu-property-building-1` on that statement, so the adapter records that license on
|
|
20
20
|
* every row.
|
|
21
21
|
*
|
|
22
|
-
* Luxembourg writes the house number before the street
|
|
22
|
+
* Luxembourg writes the house number before the street. `STREET_ORDERS` records that order as
|
|
23
23
|
* `number-first`, and `COMMA_JOINED_STREET_COUNTRIES` does not list it. `formatAddressRow` therefore
|
|
24
24
|
* renders `20 Kaesfurterstrooss, L-9755 Hupperdange`.
|
|
25
25
|
*
|
|
26
|
-
* ## The file
|
|
26
|
+
* ## The file holds one street name, in no single language
|
|
27
27
|
*
|
|
28
28
|
* Luxembourg is trilingual, and the CSV declares one street column.
|
|
29
29
|
* There is no `rue_fr`, `rue_de` or `rue_lb` to choose between, so the adapter emits `rue` as the
|
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
* No column marks a row's language, so `locale` reads `und-LU`.
|
|
39
39
|
* `und` is BCP-47's undetermined language subtag, and the repository already stores it in
|
|
40
40
|
* `#recipes/bare/country` and `#tools/postcode-triples`.
|
|
41
|
-
*
|
|
41
|
+
* A `fr-LU` or `lb-LU` label would assert a language for a name the publisher did not
|
|
42
42
|
* tag.
|
|
43
43
|
*
|
|
44
44
|
* ## The number comes from `numero`, and never from the street line
|
|
@@ -46,10 +46,10 @@
|
|
|
46
46
|
* `rue` and `numero` are separate columns and the publisher never composes them, so neither
|
|
47
47
|
* `splitStreetLine` nor `composeHouseNumber` applies here: there is no line to split and no second
|
|
48
48
|
* column to join.
|
|
49
|
-
*
|
|
49
|
+
* A street split would also be wrong in both directions.
|
|
50
50
|
* 175 `rue` values end in a digit, among them `Route Nationale 1`, `Rue du 9 août 2019` and
|
|
51
51
|
* `Cité des Sacrifiés 1940-1945`.
|
|
52
|
-
* `numero` already
|
|
52
|
+
* `numero` already holds its own suffix as one token: 162,273 rows hold bare digits, 17,087 a
|
|
53
53
|
* trailing letter such as `12A`, 239 a hyphenated range such as `22-26`, eight the word `BIS` or
|
|
54
54
|
* `bis`, and 15 are empty.
|
|
55
55
|
*
|
|
@@ -59,15 +59,15 @@
|
|
|
59
59
|
* The file publishes no preformatted address line, so a rendered single line has no publisher-side
|
|
60
60
|
* counterpart to be asserted against.
|
|
61
61
|
*
|
|
62
|
-
* ##
|
|
62
|
+
* ## The file needs a quote-aware parser
|
|
63
63
|
*
|
|
64
64
|
* 22 rows RFC 4180-quote `rue` and double its interior quotes:
|
|
65
65
|
* `"Cité ""Pënscherbierg""";3;Wilwerwiltz;…`.
|
|
66
|
-
* Those rows
|
|
66
|
+
* Those rows hold no semicolon inside the quotes, so a naive split on `;` still yields 13 fields
|
|
67
67
|
* and passes a field-count check while handing the street to the corpus as
|
|
68
68
|
* `"Cité ""Pënscherbierg"""`.
|
|
69
69
|
* `CSVSpliterator` unescapes it to `Cité "Pënscherbierg"`.
|
|
70
|
-
* The same edition
|
|
70
|
+
* The same edition holds 0 fields with an embedded newline and 0 carriage returns in the whole
|
|
71
71
|
* file.
|
|
72
72
|
*
|
|
73
73
|
* The file opens with a UTF-8 byte-order mark, the three bytes `EF BB BF`, which the reader strips.
|
|
@@ -102,7 +102,7 @@ export const BD_ADRESSES_DEFAULT_LICENSE = "CC0-1.0"
|
|
|
102
102
|
* Every jurisdiction this adapter emits, checked against a caller's `--country`.
|
|
103
103
|
*
|
|
104
104
|
* BD-Adresses is one national file and Luxembourg has no dependency, so this is a single entry
|
|
105
|
-
* where `ban`, `matrikkelen` and `ryhti` each
|
|
105
|
+
* where `ban`, `matrikkelen` and `ryhti` each hold more than one.
|
|
106
106
|
*/
|
|
107
107
|
export const BD_ADRESSES_COUNTRIES: readonly string[] = ["LU"]
|
|
108
108
|
|
|
@@ -185,7 +185,7 @@ export function createBDAdressesAdapter(): CorpusAdapter {
|
|
|
185
185
|
|
|
186
186
|
// 15 of the 179,622 rows publish an empty `numero`, among them `Al Géidgen`
|
|
187
187
|
// in Goedange and `Jaanshaff` in Walferdange.
|
|
188
|
-
//
|
|
188
|
+
// These describe a place, so they keep their street and postcode
|
|
189
189
|
// rather than being assigned a number.
|
|
190
190
|
if (house) {
|
|
191
191
|
components.house_number = house
|
|
@@ -11,13 +11,13 @@
|
|
|
11
11
|
* this module sits beside `#fi/tools/fetch/ryhti` rather than in an `sdk/` root the workspace does
|
|
12
12
|
* not declare.
|
|
13
13
|
*
|
|
14
|
-
* `data.public.lu` states `cc-zero` for this dataset
|
|
15
|
-
* `CC0-1.0` under `lu-property-building-1`. CC0 asks for no attribution, and crediting the ACT
|
|
14
|
+
* `data.public.lu` states `cc-zero` for this dataset. The address-source register elects
|
|
15
|
+
* `CC0-1.0` for it under `lu-property-building-1`. CC0 asks for no attribution, and crediting the ACT
|
|
16
16
|
* remains good practice, so the manifest records the publisher beside the license.
|
|
17
17
|
*
|
|
18
18
|
* Four measured properties decide what this module does:
|
|
19
19
|
*
|
|
20
|
-
* 1. **The download URL
|
|
20
|
+
* 1. **The download URL holds the edition's own timestamp.** The dataset is republished weekly
|
|
21
21
|
* and the CSV resource's `url` reads
|
|
22
22
|
* `https://download.data.public.lu/resources/adresses-georeferencees-bd-adresses/20260928-023119/addresses.csv`,
|
|
23
23
|
* where the path segment is the publication time. A hardcoded URL therefore pins one edition and
|
|
@@ -26,14 +26,14 @@
|
|
|
26
26
|
* `https://data.public.lu/fr/datasets/r/<resource id>`, answers http 302 to the same dated path,
|
|
27
27
|
* and the dated path is the one downloaded, so the bytes and the checksum recorded beside them
|
|
28
28
|
* belong to one edition.
|
|
29
|
-
* 2. **The publisher states an md5 for the file.** The resource
|
|
29
|
+
* 2. **The publisher states an md5 for the file.** The resource records
|
|
30
30
|
* `checksum: {type: "md5", value: "…"}` and a `filesize`, measured as
|
|
31
31
|
* `c2df2f3c18b846144bb5d5dc57fc14ca` and 28,186,970 on the 2026-09-28 edition. The transfer is
|
|
32
32
|
* checked against both, so a truncated body is a reported failure rather than a short CSV that
|
|
33
33
|
* parses. A publisher that one day states a digest of another type is reported rather than
|
|
34
34
|
* skipped silently.
|
|
35
|
-
* 3. **The file opens with a UTF-8 byte-order mark**, `EF BB BF`. The portal's `filesize` counts it
|
|
36
|
-
*
|
|
35
|
+
* 3. **The file opens with a UTF-8 byte-order mark**, `EF BB BF`. The portal's `filesize` counts it.
|
|
36
|
+
* A reader that discards the mark therefore reports three bytes fewer than the portal does.
|
|
37
37
|
* The module writes the body to disk unaltered and checks the digest over the bytes as delivered.
|
|
38
38
|
* 4. **The re-run check is the stated checksum against the manifest.** The dataset API answer is
|
|
39
39
|
* roughly 15 KB, so comparing it costs one small request instead of 28 MB.
|
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
|
|
46
46
|
import { APIClient } from "@mailwoman/core/api"
|
|
47
47
|
import { ByteFormatter } from "@mailwoman/core/fs/formatters"
|
|
48
|
-
import { tryStat } from "@mailwoman/core/fs/readers"
|
|
48
|
+
import { tryStat } from "@mailwoman/core/fs/readers/stat"
|
|
49
49
|
import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
|
|
50
50
|
import { md5File, sha256File } from "@mailwoman/core/hash"
|
|
51
51
|
import { PathBuilder, type PathBuilderLike } from "path-ts"
|
|
@@ -65,7 +65,7 @@ import { assertHeaderColumns, readDelimitedHeader } from "#tools/fetch/header"
|
|
|
65
65
|
export const LU_BD_ADRESSES_DATASET_URL = "https://data.public.lu/api/1/datasets/adresses-georeferencees-bd-adresses/"
|
|
66
66
|
|
|
67
67
|
/**
|
|
68
|
-
* The directory the download is written under,
|
|
68
|
+
* The directory the download is written under, the adapter's `inputPath`'s parent.
|
|
69
69
|
*/
|
|
70
70
|
const SLUG = BD_ADRESSES_ADAPTER_ID
|
|
71
71
|
|
|
@@ -91,7 +91,8 @@ export const LU_BD_ADRESSES_DELIMITER = ";"
|
|
|
91
91
|
*/
|
|
92
92
|
export interface BDAdressesResource {
|
|
93
93
|
/**
|
|
94
|
-
* The dated download URL
|
|
94
|
+
* The dated download URL.
|
|
95
|
+
* It identifies one edition.
|
|
95
96
|
*/
|
|
96
97
|
url: string
|
|
97
98
|
/**
|
|
@@ -116,7 +117,8 @@ export interface BDAdressesManifest extends SourceManifest {
|
|
|
116
117
|
attribution: string
|
|
117
118
|
dataset_url: string
|
|
118
119
|
/**
|
|
119
|
-
* The md5 the dataset record stated
|
|
120
|
+
* The md5 the dataset record stated.
|
|
121
|
+
* The downloaded bytes were checked against it.
|
|
120
122
|
*/
|
|
121
123
|
publisher_md5: string | null
|
|
122
124
|
/**
|
|
@@ -143,7 +145,7 @@ interface DatasetRecord {
|
|
|
143
145
|
/**
|
|
144
146
|
* The CSV resource of a dataset record.
|
|
145
147
|
*
|
|
146
|
-
* @throws When the record
|
|
148
|
+
* @throws When the record holds no `csv` resource with a URL, because the publisher
|
|
147
149
|
* having moved the file is a reported failure rather than an empty transfer.
|
|
148
150
|
*/
|
|
149
151
|
export function readBDAdressesResource(record: DatasetRecord): BDAdressesResource {
|
|
@@ -225,7 +227,7 @@ export async function downloadBDAdresses(
|
|
|
225
227
|
!options.force &&
|
|
226
228
|
present &&
|
|
227
229
|
existing &&
|
|
228
|
-
resource.md5
|
|
230
|
+
resource.md5 &&
|
|
229
231
|
existing.publisher_md5 === resource.md5 &&
|
|
230
232
|
existing.source_url === resource.url
|
|
231
233
|
) {
|
|
@@ -270,7 +272,7 @@ export async function downloadBDAdresses(
|
|
|
270
272
|
|
|
271
273
|
const md5 = await md5File(csvPath)
|
|
272
274
|
|
|
273
|
-
if (resource.md5
|
|
275
|
+
if (resource.md5 && resource.md5 !== md5) {
|
|
274
276
|
report?.(` ✗ the dataset record states md5 ${resource.md5} and the bytes hash to ${md5}`)
|
|
275
277
|
await removePathIfPresent(csvPath)
|
|
276
278
|
|