@mailwoman/corpus 10.1.0 → 10.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/licenses/cdla-permissive-2.0.md +1 -1
- package/lib/adapters/cordis/adapter.ts +13 -7
- package/lib/adapters/dedup-key.ts +1 -1
- package/lib/adapters/geonames/adapter.ts +1 -1
- package/lib/adapters/gleif/adapter.ts +17 -16
- package/lib/adapters/libpostal-dictionary.ts +0 -2
- package/lib/adapters/openaddresses/adapter.ts +4 -3
- package/lib/adapters/osm/adapter.ts +22 -21
- package/lib/adapters/po-box/adapter.ts +10 -5
- package/lib/adapters/registry.ts +4 -4
- package/lib/adapters/source-id.ts +2 -2
- package/lib/adapters/street-line.ts +7 -6
- package/lib/adapters/wof/admin/json/adapter.ts +2 -2
- package/lib/adapters/wof/json-rows.ts +2 -1
- package/lib/adapters/wof/postalcode-json/adapter.ts +6 -2
- package/lib/adapters.ts +5 -5
- package/lib/au/adapters/gnaf/adapter.ts +1 -1
- package/lib/be/adapters/brussels/adapter.ts +22 -22
- package/lib/be/adapters/vlaanderen/adapter.ts +54 -57
- package/lib/be/adapters/wallonie/adapter.ts +17 -17
- package/lib/be/tools/fetch/brussels.ts +13 -10
- package/lib/be/tools/fetch/wallonie.ts +8 -7
- package/lib/build/adapters.ts +2 -1
- package/lib/build/align.ts +5 -4
- package/lib/build/checkpoint.ts +5 -3
- package/lib/build/eligibility.ts +3 -3
- package/lib/build/types.ts +8 -6
- package/lib/cz/adapters/cuzk/adapter.ts +41 -38
- package/lib/cz/tools/fetch/cuzk.ts +42 -31
- package/lib/de/recipes/locale.ts +1 -1
- package/lib/dk/adapters/inspire/adapter.ts +38 -38
- package/lib/dk/tools/fetch/inspire.ts +16 -14
- package/lib/ee/adapters/ads/adapter.ts +26 -32
- package/lib/ee/tools/fetch/ads.ts +10 -8
- package/lib/es/adapters/bizkaia/adapter.ts +47 -47
- package/lib/es/adapters/catastro/adapter.ts +36 -34
- package/lib/es/adapters/gipuzkoa/adapter.ts +38 -45
- package/lib/es/adapters/navarra/adapter.ts +28 -27
- package/lib/es/cadastre.ts +31 -31
- package/lib/es/tools/fetch/bizkaia.ts +40 -36
- package/lib/es/tools/fetch/catastro.ts +51 -40
- package/lib/es/tools/fetch/gipuzkoa.ts +13 -11
- package/lib/es/tools/fetch/navarra.ts +33 -25
- package/lib/fi/adapters/ryhti/adapter.ts +22 -26
- package/lib/fi/tools/fetch/ryhti.ts +31 -31
- package/lib/fr/adapters/annuaire-education/adapter.ts +13 -11
- package/lib/fr/adapters/ban/adapter.ts +6 -6
- package/lib/fr/adapters/finess/adapter.ts +16 -15
- package/lib/fr/adapters/finess/voie-type.ts +4 -3
- package/lib/fr/insee-country.ts +2 -2
- package/lib/fr/recipes/lieudit.ts +8 -6
- package/lib/fr/tools/fetch/annuaire-education.ts +11 -8
- package/lib/fr/tools/fetch/ban.ts +3 -2
- package/lib/fr/tools/fetch/finess.ts +14 -13
- package/lib/inspire/address.ts +86 -78
- package/lib/inspire/errors.ts +9 -8
- package/lib/inspire/stream.ts +29 -26
- package/lib/international/recipes/locale.ts +13 -9
- package/lib/it/adapters/anac/adapter.ts +51 -46
- package/lib/it/tools/fetch/anac.ts +22 -25
- package/lib/jp/adapters/wof-admin/adapter.ts +2 -6
- package/lib/jp/tools/fetch/houjin.ts +3 -3
- package/lib/kr/adapters/localdata/permit-rows.ts +1 -1
- package/lib/kr/tools/fetch/juso.ts +1 -1
- package/lib/kr/tools/fetch/localdata.ts +2 -2
- package/lib/lu/adapters/bd-adresses/adapter.ts +10 -10
- package/lib/lu/tools/fetch/bd-adresses.ts +15 -13
- package/lib/lv/adapters/varis/adapter.ts +24 -23
- package/lib/lv/tools/fetch/varis.ts +3 -2
- package/lib/nl/adapters/kadaster/adapter.ts +29 -29
- package/lib/nl/tools/fetch/kadaster.ts +16 -12
- package/lib/no/adapters/matrikkelen/adapter.ts +6 -5
- package/lib/no/tools/fetch/matrikkelen.ts +59 -47
- package/lib/parquet/duckdb.ts +7 -4
- package/lib/parquet/readers.ts +3 -3
- package/lib/parquet/schema.ts +1 -1
- package/lib/parquet/writers.ts +3 -2
- package/lib/pl/adapters/emuia/adapter.ts +12 -12
- package/lib/pl/tools/fetch/emuia.ts +10 -9
- package/lib/recipes/bare/postcode.ts +1 -1
- package/lib/recipes/boundary-stress.ts +1 -1
- package/lib/recipes/house-venue.ts +7 -3
- package/lib/recipes/intersection.ts +2 -2
- package/lib/recipes/po/box/cedex/recipe.ts +3 -3
- package/lib/recipes/po/box/cedex/sources.ts +13 -1
- package/lib/recipes/scaffold.ts +31 -21
- package/lib/recipes/sources.ts +16 -3
- package/lib/recipes/street/bare.ts +2 -2
- package/lib/recipes/street.ts +3 -1
- package/lib/recipes/sub/venue/context.ts +2 -0
- package/lib/recipes/sub/venue/render.ts +1 -1
- package/lib/recipes/sub/venue/sources.ts +24 -29
- package/lib/recipes/sub/venue.ts +12 -18
- package/lib/recipes/trailing-region.ts +6 -2
- package/lib/recipes/unit.ts +3 -3
- package/lib/recipes/venue.ts +347 -0
- package/lib/recipes.ts +4 -2
- package/lib/registers.ts +8 -6
- package/lib/ro/adapters/retea-scolara/adapter.ts +5 -5
- package/lib/ro/tools/fetch/retea-scolara.ts +1 -1
- package/lib/runner.ts +4 -3
- package/lib/sg/recipes/register.ts +1 -1
- package/lib/sg/tools/fetch/acra.ts +3 -3
- package/lib/sk/adapters/inspire/adapter.ts +25 -27
- package/lib/sk/tools/fetch/inspire.ts +15 -14
- package/lib/source-register/effective-manifest.ts +3 -3
- package/lib/source-register/register.ts +3 -3
- package/lib/source-register/training-manifest.ts +1 -1
- package/lib/source-register/types.ts +12 -13
- package/lib/subvenue/promotions.ts +1 -1
- package/lib/subvenue/table.ts +4 -4
- package/lib/surfaces/locale.ts +2 -0
- package/lib/synthesizers/anchor-absorption.ts +2 -1
- package/lib/synthesizers/boundary-stress.ts +20 -26
- package/lib/synthesizers/house-venue.ts +26 -30
- package/lib/synthesizers/no-street.ts +2 -1
- package/lib/synthesizers/po-box.ts +6 -3
- package/lib/synthesizers/tail.ts +55 -0
- package/lib/synthesizers/utils.ts +19 -2
- package/lib/test-kit.ts +2 -2
- package/lib/tools/audit.ts +7 -5
- package/lib/tools/corpus-stats.ts +1 -1
- package/lib/tools/fetch/atom.ts +10 -10
- package/lib/tools/fetch/cordis.ts +11 -9
- package/lib/tools/fetch/download/manifest.ts +2 -1
- package/lib/tools/fetch/download/network.ts +4 -4
- package/lib/tools/fetch/download/retry.ts +4 -4
- package/lib/tools/fetch/geonames/dump.ts +5 -3
- package/lib/tools/fetch/geonames/postal.ts +2 -2
- package/lib/tools/fetch/gleif.ts +6 -4
- package/lib/tools/fetch/header.ts +8 -6
- package/lib/tools/fetch/inspire-addresses.ts +41 -36
- package/lib/tools/fetch/openaddresses.ts +1 -1
- package/lib/tools/fetch/vlaanderen-ad.ts +17 -17
- package/lib/tools/fetch/wfs-harvest.ts +39 -28
- package/lib/tools/fetch.ts +0 -42
- package/lib/tools/golden/expand.ts +8 -8
- package/lib/tools/golden/promote.ts +12 -8
- package/lib/tools/golden/relabel-street/directory.ts +7 -6
- package/lib/tools/golden/relabel-street/row.ts +7 -6
- package/lib/tools/ingest-csv.ts +3 -3
- package/lib/tools/jsonl-to-parquet.ts +1 -1
- package/lib/tools/lint/recipe-output/vocab.ts +10 -5
- package/lib/tools/lint/recipe-output.ts +26 -8
- package/lib/tools/overlay/kryptonite.ts +4 -3
- package/lib/tools/overlay/manifest.ts +4 -2
- package/lib/tools/overlay/merge-source.ts +2 -1
- package/lib/tools/overlay/split-slice.ts +2 -2
- package/lib/tools/overlay/translit.ts +6 -4
- package/lib/tools/overture-subvenue.ts +1 -1
- package/lib/tools/postcode-triples.ts +5 -4
- package/lib/tools/source-register/build.ts +17 -14
- package/lib/tools/sub/venue/lexicon.ts +4 -8
- package/lib/tw/tools/fetch/gcis.ts +4 -4
- package/lib/types.ts +4 -3
- package/lib/us/adapters/state/hi-schools/workbook.ts +1 -1
- package/lib/us/adapters/usgov/nad/adapter.ts +11 -11
- package/lib/us/adapters/usgov/samhsa-treatment-locator/adapter.ts +4 -4
- package/lib/us/fips-state.ts +2 -2
- package/lib/{synthesizers → us/synthesizers}/intersection.ts +12 -13
- package/lib/{synthesizers → us/synthesizers}/street.ts +16 -15
- package/lib/us/tools/fetch/imls-pls.ts +1 -1
- package/lib/us/tools/fetch/nad.ts +7 -7
- package/lib/us/tools/fetch/nppes.ts +5 -5
- package/lib/us/tools/fetch/state/hi-schools.ts +3 -2
- package/lib/us/tools/fetch/tiger-full.ts +1 -1
- package/lib/utils/align.ts +13 -10
- package/lib/utils/golden.ts +2 -2
- package/lib/utils/license.ts +3 -3
- package/lib/utils/split.ts +19 -15
- package/lib/utils/tokenize.ts +7 -26
- package/lib/utils/wof-json.ts +4 -4
- package/out/adapters/cordis/adapter.d.ts +5 -3
- package/out/adapters/cordis/adapter.d.ts.map +1 -1
- package/out/adapters/cordis/adapter.js +13 -7
- package/out/adapters/cordis/adapter.js.map +1 -1
- package/out/adapters/dedup-key.d.ts +1 -1
- package/out/adapters/dedup-key.js +1 -1
- package/out/adapters/geonames/adapter.js +1 -1
- package/out/adapters/geonames/adapter.js.map +1 -1
- package/out/adapters/gleif/adapter.d.ts +9 -9
- package/out/adapters/gleif/adapter.d.ts.map +1 -1
- package/out/adapters/gleif/adapter.js +15 -14
- package/out/adapters/gleif/adapter.js.map +1 -1
- package/out/adapters/libpostal-dictionary.d.ts +0 -2
- package/out/adapters/libpostal-dictionary.d.ts.map +1 -1
- package/out/adapters/libpostal-dictionary.js +0 -2
- package/out/adapters/libpostal-dictionary.js.map +1 -1
- package/out/adapters/openaddresses/adapter.d.ts +3 -2
- package/out/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/adapters/openaddresses/adapter.js +1 -1
- package/out/adapters/openaddresses/adapter.js.map +1 -1
- package/out/adapters/osm/adapter.d.ts +17 -16
- package/out/adapters/osm/adapter.d.ts.map +1 -1
- package/out/adapters/osm/adapter.js +10 -9
- package/out/adapters/osm/adapter.js.map +1 -1
- package/out/adapters/po-box/adapter.d.ts +6 -3
- package/out/adapters/po-box/adapter.d.ts.map +1 -1
- package/out/adapters/po-box/adapter.js +4 -2
- package/out/adapters/po-box/adapter.js.map +1 -1
- package/out/adapters/registry.d.ts +3 -3
- package/out/adapters/registry.d.ts.map +1 -1
- package/out/adapters/registry.js +1 -1
- package/out/adapters/registry.js.map +1 -1
- package/out/adapters/source-id.d.ts +2 -2
- package/out/adapters/source-id.js +2 -2
- package/out/adapters/street-line.d.ts +7 -6
- package/out/adapters/street-line.d.ts.map +1 -1
- package/out/adapters/street-line.js +7 -6
- package/out/adapters/street-line.js.map +1 -1
- package/out/adapters/wof/admin/json/adapter.js +1 -1
- package/out/adapters/wof/admin/json/adapter.js.map +1 -1
- package/out/adapters/wof/json-rows.d.ts +2 -1
- package/out/adapters/wof/json-rows.d.ts.map +1 -1
- package/out/adapters/wof/json-rows.js.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.d.ts.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.js +5 -1
- package/out/adapters/wof/postalcode-json/adapter.js.map +1 -1
- package/out/adapters.d.ts +5 -5
- package/out/adapters.js +5 -5
- package/out/au/adapters/gnaf/adapter.js +1 -1
- package/out/au/adapters/gnaf/adapter.js.map +1 -1
- package/out/be/adapters/brussels/adapter.d.ts +9 -9
- package/out/be/adapters/brussels/adapter.d.ts.map +1 -1
- package/out/be/adapters/brussels/adapter.js +21 -21
- package/out/be/adapters/brussels/adapter.js.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.d.ts +34 -37
- package/out/be/adapters/vlaanderen/adapter.d.ts.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.js +40 -38
- package/out/be/adapters/vlaanderen/adapter.js.map +1 -1
- package/out/be/adapters/wallonie/adapter.d.ts +2 -2
- package/out/be/adapters/wallonie/adapter.js +17 -17
- package/out/be/adapters/wallonie/adapter.js.map +1 -1
- package/out/be/tools/fetch/brussels.d.ts +11 -8
- package/out/be/tools/fetch/brussels.d.ts.map +1 -1
- package/out/be/tools/fetch/brussels.js +10 -8
- package/out/be/tools/fetch/brussels.js.map +1 -1
- package/out/be/tools/fetch/wallonie.d.ts +6 -5
- package/out/be/tools/fetch/wallonie.d.ts.map +1 -1
- package/out/be/tools/fetch/wallonie.js +4 -4
- package/out/be/tools/fetch/wallonie.js.map +1 -1
- package/out/build/adapters.d.ts.map +1 -1
- package/out/build/adapters.js +2 -1
- package/out/build/adapters.js.map +1 -1
- package/out/build/align.d.ts +3 -2
- package/out/build/align.d.ts.map +1 -1
- package/out/build/align.js +2 -2
- package/out/build/align.js.map +1 -1
- package/out/build/checkpoint.d.ts +1 -1
- package/out/build/checkpoint.d.ts.map +1 -1
- package/out/build/checkpoint.js +4 -2
- package/out/build/checkpoint.js.map +1 -1
- package/out/build/eligibility.d.ts +2 -2
- package/out/build/eligibility.js +3 -3
- package/out/build/eligibility.js.map +1 -1
- package/out/build/types.d.ts +8 -6
- package/out/build/types.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.d.ts +14 -14
- package/out/cz/adapters/cuzk/adapter.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.js +38 -35
- package/out/cz/adapters/cuzk/adapter.js.map +1 -1
- package/out/cz/tools/fetch/cuzk.d.ts +31 -22
- package/out/cz/tools/fetch/cuzk.d.ts.map +1 -1
- package/out/cz/tools/fetch/cuzk.js +21 -19
- package/out/cz/tools/fetch/cuzk.js.map +1 -1
- package/out/de/recipes/locale.js +1 -1
- package/out/de/recipes/locale.js.map +1 -1
- package/out/dk/adapters/inspire/adapter.d.ts +32 -33
- package/out/dk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/dk/adapters/inspire/adapter.js +38 -38
- package/out/dk/adapters/inspire/adapter.js.map +1 -1
- package/out/dk/tools/fetch/inspire.d.ts +6 -5
- package/out/dk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/dk/tools/fetch/inspire.js +15 -13
- package/out/dk/tools/fetch/inspire.js.map +1 -1
- package/out/ee/adapters/ads/adapter.d.ts +19 -26
- package/out/ee/adapters/ads/adapter.d.ts.map +1 -1
- package/out/ee/adapters/ads/adapter.js +22 -28
- package/out/ee/adapters/ads/adapter.js.map +1 -1
- package/out/ee/tools/fetch/ads.d.ts +3 -3
- package/out/ee/tools/fetch/ads.d.ts.map +1 -1
- package/out/ee/tools/fetch/ads.js +10 -8
- package/out/ee/tools/fetch/ads.js.map +1 -1
- package/out/es/adapters/bizkaia/adapter.d.ts +28 -28
- package/out/es/adapters/bizkaia/adapter.d.ts.map +1 -1
- package/out/es/adapters/bizkaia/adapter.js +43 -43
- package/out/es/adapters/bizkaia/adapter.js.map +1 -1
- package/out/es/adapters/catastro/adapter.d.ts +22 -23
- package/out/es/adapters/catastro/adapter.d.ts.map +1 -1
- package/out/es/adapters/catastro/adapter.js +35 -33
- package/out/es/adapters/catastro/adapter.js.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.d.ts +31 -38
- package/out/es/adapters/gipuzkoa/adapter.d.ts.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.js +40 -47
- package/out/es/adapters/gipuzkoa/adapter.js.map +1 -1
- package/out/es/adapters/navarra/adapter.d.ts +20 -19
- package/out/es/adapters/navarra/adapter.d.ts.map +1 -1
- package/out/es/adapters/navarra/adapter.js +23 -22
- package/out/es/adapters/navarra/adapter.js.map +1 -1
- package/out/es/cadastre.d.ts +21 -20
- package/out/es/cadastre.d.ts.map +1 -1
- package/out/es/cadastre.js +19 -21
- package/out/es/cadastre.js.map +1 -1
- package/out/es/tools/fetch/bizkaia.d.ts +28 -25
- package/out/es/tools/fetch/bizkaia.d.ts.map +1 -1
- package/out/es/tools/fetch/bizkaia.js +25 -24
- package/out/es/tools/fetch/bizkaia.js.map +1 -1
- package/out/es/tools/fetch/catastro.d.ts +28 -20
- package/out/es/tools/fetch/catastro.d.ts.map +1 -1
- package/out/es/tools/fetch/catastro.js +36 -31
- package/out/es/tools/fetch/catastro.js.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.d.ts +11 -9
- package/out/es/tools/fetch/gipuzkoa.d.ts.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.js +9 -8
- package/out/es/tools/fetch/gipuzkoa.js.map +1 -1
- package/out/es/tools/fetch/navarra.d.ts +20 -15
- package/out/es/tools/fetch/navarra.d.ts.map +1 -1
- package/out/es/tools/fetch/navarra.js +21 -16
- package/out/es/tools/fetch/navarra.js.map +1 -1
- package/out/fi/adapters/ryhti/adapter.d.ts +16 -20
- package/out/fi/adapters/ryhti/adapter.d.ts.map +1 -1
- package/out/fi/adapters/ryhti/adapter.js +21 -25
- package/out/fi/adapters/ryhti/adapter.js.map +1 -1
- package/out/fi/tools/fetch/ryhti.d.ts +24 -24
- package/out/fi/tools/fetch/ryhti.d.ts.map +1 -1
- package/out/fi/tools/fetch/ryhti.js +21 -24
- package/out/fi/tools/fetch/ryhti.js.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.d.ts +8 -8
- package/out/fr/adapters/annuaire-education/adapter.d.ts.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.js +13 -11
- package/out/fr/adapters/annuaire-education/adapter.js.map +1 -1
- package/out/fr/adapters/ban/adapter.d.ts +4 -4
- package/out/fr/adapters/ban/adapter.js +6 -6
- package/out/fr/adapters/ban/adapter.js.map +1 -1
- package/out/fr/adapters/finess/adapter.d.ts +12 -11
- package/out/fr/adapters/finess/adapter.d.ts.map +1 -1
- package/out/fr/adapters/finess/adapter.js +16 -15
- package/out/fr/adapters/finess/adapter.js.map +1 -1
- package/out/fr/adapters/finess/voie-type.d.ts +2 -2
- package/out/fr/adapters/finess/voie-type.d.ts.map +1 -1
- package/out/fr/adapters/finess/voie-type.js +4 -3
- package/out/fr/adapters/finess/voie-type.js.map +1 -1
- package/out/fr/insee-country.d.ts +2 -2
- package/out/fr/insee-country.js +2 -2
- package/out/fr/recipes/lieudit.d.ts.map +1 -1
- package/out/fr/recipes/lieudit.js +5 -4
- package/out/fr/recipes/lieudit.js.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.d.ts +7 -4
- package/out/fr/tools/fetch/annuaire-education.d.ts.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.js +9 -7
- package/out/fr/tools/fetch/annuaire-education.js.map +1 -1
- package/out/fr/tools/fetch/ban.d.ts +1 -1
- package/out/fr/tools/fetch/ban.d.ts.map +1 -1
- package/out/fr/tools/fetch/ban.js +3 -2
- package/out/fr/tools/fetch/ban.js.map +1 -1
- package/out/fr/tools/fetch/finess.d.ts +5 -5
- package/out/fr/tools/fetch/finess.d.ts.map +1 -1
- package/out/fr/tools/fetch/finess.js +13 -12
- package/out/fr/tools/fetch/finess.js.map +1 -1
- package/out/inspire/address.d.ts +62 -52
- package/out/inspire/address.d.ts.map +1 -1
- package/out/inspire/address.js +71 -60
- package/out/inspire/address.js.map +1 -1
- package/out/inspire/errors.d.ts +9 -8
- package/out/inspire/errors.d.ts.map +1 -1
- package/out/inspire/errors.js +7 -6
- package/out/inspire/errors.js.map +1 -1
- package/out/inspire/stream.d.ts +25 -22
- package/out/inspire/stream.d.ts.map +1 -1
- package/out/inspire/stream.js +10 -12
- package/out/inspire/stream.js.map +1 -1
- package/out/international/recipes/locale.d.ts +2 -2
- package/out/international/recipes/locale.d.ts.map +1 -1
- package/out/international/recipes/locale.js +11 -7
- package/out/international/recipes/locale.js.map +1 -1
- package/out/it/adapters/anac/adapter.d.ts +45 -43
- package/out/it/adapters/anac/adapter.d.ts.map +1 -1
- package/out/it/adapters/anac/adapter.js +48 -43
- package/out/it/adapters/anac/adapter.js.map +1 -1
- package/out/it/tools/fetch/anac.d.ts +16 -19
- package/out/it/tools/fetch/anac.d.ts.map +1 -1
- package/out/it/tools/fetch/anac.js +20 -23
- package/out/it/tools/fetch/anac.js.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.d.ts.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.js.map +1 -1
- package/out/jp/tools/fetch/houjin.js +2 -2
- package/out/jp/tools/fetch/houjin.js.map +1 -1
- package/out/kr/adapters/localdata/permit-rows.js +1 -1
- package/out/kr/adapters/localdata/permit-rows.js.map +1 -1
- package/out/kr/tools/fetch/juso.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.js +1 -1
- package/out/kr/tools/fetch/localdata.js.map +1 -1
- package/out/lu/adapters/bd-adresses/adapter.d.ts +9 -9
- package/out/lu/adapters/bd-adresses/adapter.js +10 -10
- package/out/lu/tools/fetch/bd-adresses.d.ts +11 -9
- package/out/lu/tools/fetch/bd-adresses.d.ts.map +1 -1
- package/out/lu/tools/fetch/bd-adresses.js +11 -11
- package/out/lu/tools/fetch/bd-adresses.js.map +1 -1
- package/out/lv/adapters/varis/adapter.d.ts +24 -23
- package/out/lv/adapters/varis/adapter.d.ts.map +1 -1
- package/out/lv/adapters/varis/adapter.js +24 -23
- package/out/lv/adapters/varis/adapter.js.map +1 -1
- package/out/lv/tools/fetch/varis.d.ts +2 -1
- package/out/lv/tools/fetch/varis.d.ts.map +1 -1
- package/out/lv/tools/fetch/varis.js +1 -1
- package/out/lv/tools/fetch/varis.js.map +1 -1
- package/out/nl/adapters/kadaster/adapter.d.ts +12 -12
- package/out/nl/adapters/kadaster/adapter.d.ts.map +1 -1
- package/out/nl/adapters/kadaster/adapter.js +22 -23
- package/out/nl/adapters/kadaster/adapter.js.map +1 -1
- package/out/nl/tools/fetch/kadaster.d.ts +14 -10
- package/out/nl/tools/fetch/kadaster.d.ts.map +1 -1
- package/out/nl/tools/fetch/kadaster.js +12 -9
- package/out/nl/tools/fetch/kadaster.js.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.js +3 -3
- package/out/no/adapters/matrikkelen/adapter.js.map +1 -1
- package/out/no/tools/fetch/matrikkelen.d.ts +50 -38
- package/out/no/tools/fetch/matrikkelen.d.ts.map +1 -1
- package/out/no/tools/fetch/matrikkelen.js +44 -36
- package/out/no/tools/fetch/matrikkelen.js.map +1 -1
- package/out/parquet/duckdb.d.ts +7 -4
- package/out/parquet/duckdb.d.ts.map +1 -1
- package/out/parquet/duckdb.js +2 -1
- package/out/parquet/duckdb.js.map +1 -1
- package/out/parquet/readers.js +3 -3
- package/out/parquet/readers.js.map +1 -1
- package/out/parquet/schema.js +1 -1
- package/out/parquet/schema.js.map +1 -1
- package/out/parquet/writers.d.ts +1 -1
- package/out/parquet/writers.d.ts.map +1 -1
- package/out/parquet/writers.js +2 -1
- package/out/parquet/writers.js.map +1 -1
- package/out/pl/adapters/emuia/adapter.d.ts +8 -8
- package/out/pl/adapters/emuia/adapter.js +12 -12
- package/out/pl/adapters/emuia/adapter.js.map +1 -1
- package/out/pl/tools/fetch/emuia.d.ts +5 -4
- package/out/pl/tools/fetch/emuia.d.ts.map +1 -1
- package/out/pl/tools/fetch/emuia.js +10 -9
- package/out/pl/tools/fetch/emuia.js.map +1 -1
- package/out/recipes/bare/postcode.js +1 -1
- package/out/recipes/bare/postcode.js.map +1 -1
- package/out/recipes/boundary-stress.js +1 -1
- package/out/recipes/boundary-stress.js.map +1 -1
- package/out/recipes/house-venue.d.ts.map +1 -1
- package/out/recipes/house-venue.js +6 -3
- package/out/recipes/house-venue.js.map +1 -1
- package/out/recipes/intersection.js +2 -2
- package/out/recipes/intersection.js.map +1 -1
- package/out/recipes/po/box/cedex/recipe.js +2 -1
- package/out/recipes/po/box/cedex/recipe.js.map +1 -1
- package/out/recipes/po/box/cedex/sources.d.ts +8 -2
- package/out/recipes/po/box/cedex/sources.d.ts.map +1 -1
- package/out/recipes/po/box/cedex/sources.js +1 -1
- package/out/recipes/po/box/cedex/sources.js.map +1 -1
- package/out/recipes/scaffold.d.ts +28 -19
- package/out/recipes/scaffold.d.ts.map +1 -1
- package/out/recipes/scaffold.js +3 -2
- package/out/recipes/scaffold.js.map +1 -1
- package/out/recipes/sources.d.ts +1 -1
- package/out/recipes/sources.d.ts.map +1 -1
- package/out/recipes/sources.js +15 -2
- package/out/recipes/sources.js.map +1 -1
- package/out/recipes/street/bare.js +2 -2
- package/out/recipes/street/bare.js.map +1 -1
- package/out/recipes/street.d.ts.map +1 -1
- package/out/recipes/street.js +3 -1
- package/out/recipes/street.js.map +1 -1
- package/out/recipes/sub/venue/context.d.ts.map +1 -1
- package/out/recipes/sub/venue/context.js +2 -0
- package/out/recipes/sub/venue/context.js.map +1 -1
- package/out/recipes/sub/venue/render.js +1 -1
- package/out/recipes/sub/venue/render.js.map +1 -1
- package/out/recipes/sub/venue/sources.d.ts +6 -3
- package/out/recipes/sub/venue/sources.d.ts.map +1 -1
- package/out/recipes/sub/venue/sources.js +16 -20
- package/out/recipes/sub/venue/sources.js.map +1 -1
- package/out/recipes/sub/venue.d.ts +2 -5
- package/out/recipes/sub/venue.d.ts.map +1 -1
- package/out/recipes/sub/venue.js +10 -14
- package/out/recipes/sub/venue.js.map +1 -1
- package/out/recipes/trailing-region.d.ts.map +1 -1
- package/out/recipes/trailing-region.js +6 -1
- package/out/recipes/trailing-region.js.map +1 -1
- package/out/recipes/unit.js +2 -2
- package/out/recipes/unit.js.map +1 -1
- package/out/recipes/venue.d.ts +119 -0
- package/out/recipes/venue.d.ts.map +1 -0
- package/out/recipes/venue.js +245 -0
- package/out/recipes/venue.js.map +1 -0
- package/out/recipes.d.ts +1 -1
- package/out/recipes.d.ts.map +1 -1
- package/out/recipes.js +3 -1
- package/out/recipes.js.map +1 -1
- package/out/registers.d.ts +8 -6
- package/out/registers.d.ts.map +1 -1
- package/out/registers.js +8 -6
- package/out/registers.js.map +1 -1
- package/out/ro/adapters/retea-scolara/adapter.d.ts +5 -5
- package/out/ro/adapters/retea-scolara/adapter.js +5 -5
- package/out/ro/tools/fetch/retea-scolara.js +1 -1
- package/out/ro/tools/fetch/retea-scolara.js.map +1 -1
- package/out/runner.d.ts +3 -2
- package/out/runner.d.ts.map +1 -1
- package/out/runner.js +1 -1
- package/out/runner.js.map +1 -1
- package/out/sg/recipes/register.js +1 -1
- package/out/sg/recipes/register.js.map +1 -1
- package/out/sg/tools/fetch/acra.js +2 -2
- package/out/sg/tools/fetch/acra.js.map +1 -1
- package/out/sk/adapters/inspire/adapter.d.ts +5 -5
- package/out/sk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/sk/adapters/inspire/adapter.js +18 -18
- package/out/sk/adapters/inspire/adapter.js.map +1 -1
- package/out/sk/tools/fetch/inspire.d.ts +11 -10
- package/out/sk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/sk/tools/fetch/inspire.js +15 -14
- package/out/sk/tools/fetch/inspire.js.map +1 -1
- package/out/source-register/effective-manifest.js +3 -3
- package/out/source-register/effective-manifest.js.map +1 -1
- package/out/source-register/register.d.ts +2 -2
- package/out/source-register/register.d.ts.map +1 -1
- package/out/source-register/register.js +2 -2
- package/out/source-register/register.js.map +1 -1
- package/out/source-register/training-manifest.d.ts +1 -1
- package/out/source-register/types.d.ts +12 -13
- package/out/source-register/types.d.ts.map +1 -1
- package/out/source-register/types.js +2 -2
- package/out/source-register/types.js.map +1 -1
- package/out/subvenue/promotions.d.ts +1 -1
- package/out/subvenue/promotions.js +1 -1
- package/out/subvenue/table.d.ts +4 -4
- package/out/subvenue/table.js +3 -3
- package/out/surfaces/locale.d.ts.map +1 -1
- package/out/surfaces/locale.js +2 -0
- package/out/surfaces/locale.js.map +1 -1
- package/out/synthesizers/anchor-absorption.d.ts +2 -1
- package/out/synthesizers/anchor-absorption.d.ts.map +1 -1
- package/out/synthesizers/anchor-absorption.js.map +1 -1
- package/out/synthesizers/boundary-stress.d.ts +1 -1
- package/out/synthesizers/boundary-stress.d.ts.map +1 -1
- package/out/synthesizers/boundary-stress.js +20 -23
- package/out/synthesizers/boundary-stress.js.map +1 -1
- package/out/synthesizers/house-venue.d.ts +3 -2
- package/out/synthesizers/house-venue.d.ts.map +1 -1
- package/out/synthesizers/house-venue.js +24 -26
- package/out/synthesizers/house-venue.js.map +1 -1
- package/out/synthesizers/no-street.d.ts.map +1 -1
- package/out/synthesizers/no-street.js.map +1 -1
- package/out/synthesizers/po-box.d.ts +4 -2
- package/out/synthesizers/po-box.d.ts.map +1 -1
- package/out/synthesizers/po-box.js.map +1 -1
- package/out/synthesizers/tail.d.ts +36 -0
- package/out/synthesizers/tail.d.ts.map +1 -0
- package/out/synthesizers/tail.js +32 -0
- package/out/synthesizers/tail.js.map +1 -0
- package/out/synthesizers/utils.d.ts +2 -1
- package/out/synthesizers/utils.d.ts.map +1 -1
- package/out/synthesizers/utils.js +17 -1
- package/out/synthesizers/utils.js.map +1 -1
- package/out/test-kit.js +2 -2
- package/out/test-kit.js.map +1 -1
- package/out/tools/audit.d.ts +2 -1
- package/out/tools/audit.d.ts.map +1 -1
- package/out/tools/audit.js +4 -3
- package/out/tools/audit.js.map +1 -1
- package/out/tools/corpus-stats.js +1 -1
- package/out/tools/corpus-stats.js.map +1 -1
- package/out/tools/fetch/atom.d.ts +8 -8
- package/out/tools/fetch/atom.d.ts.map +1 -1
- package/out/tools/fetch/atom.js +7 -8
- package/out/tools/fetch/atom.js.map +1 -1
- package/out/tools/fetch/cordis.d.ts +4 -3
- package/out/tools/fetch/cordis.d.ts.map +1 -1
- package/out/tools/fetch/cordis.js +9 -7
- package/out/tools/fetch/cordis.js.map +1 -1
- package/out/tools/fetch/download/manifest.d.ts.map +1 -1
- package/out/tools/fetch/download/manifest.js +2 -1
- package/out/tools/fetch/download/manifest.js.map +1 -1
- package/out/tools/fetch/download/network.d.ts +4 -4
- package/out/tools/fetch/download/retry.js +4 -4
- package/out/tools/fetch/download/retry.js.map +1 -1
- package/out/tools/fetch/geonames/dump.d.ts +3 -2
- package/out/tools/fetch/geonames/dump.d.ts.map +1 -1
- package/out/tools/fetch/geonames/dump.js +2 -1
- package/out/tools/fetch/geonames/dump.js.map +1 -1
- package/out/tools/fetch/geonames/postal.d.ts +2 -2
- package/out/tools/fetch/gleif.d.ts +3 -2
- package/out/tools/fetch/gleif.d.ts.map +1 -1
- package/out/tools/fetch/gleif.js +6 -4
- package/out/tools/fetch/gleif.js.map +1 -1
- package/out/tools/fetch/header.d.ts +7 -5
- package/out/tools/fetch/header.d.ts.map +1 -1
- package/out/tools/fetch/header.js +8 -6
- package/out/tools/fetch/header.js.map +1 -1
- package/out/tools/fetch/inspire-addresses.d.ts +32 -27
- package/out/tools/fetch/inspire-addresses.d.ts.map +1 -1
- package/out/tools/fetch/inspire-addresses.js +25 -24
- package/out/tools/fetch/inspire-addresses.js.map +1 -1
- package/out/tools/fetch/openaddresses.js +1 -1
- package/out/tools/fetch/openaddresses.js.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.d.ts +5 -5
- package/out/tools/fetch/vlaanderen-ad.d.ts.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.js +16 -16
- package/out/tools/fetch/vlaanderen-ad.js.map +1 -1
- package/out/tools/fetch/wfs-harvest.d.ts +26 -16
- package/out/tools/fetch/wfs-harvest.d.ts.map +1 -1
- package/out/tools/fetch/wfs-harvest.js +21 -18
- package/out/tools/fetch/wfs-harvest.js.map +1 -1
- package/out/tools/fetch.d.ts +0 -41
- package/out/tools/fetch.d.ts.map +1 -1
- package/out/tools/fetch.js +0 -41
- package/out/tools/fetch.js.map +1 -1
- package/out/tools/golden/expand.d.ts +7 -7
- package/out/tools/golden/expand.js +1 -1
- package/out/tools/golden/expand.js.map +1 -1
- package/out/tools/golden/promote.d.ts +2 -2
- package/out/tools/golden/promote.d.ts.map +1 -1
- package/out/tools/golden/promote.js +9 -6
- package/out/tools/golden/promote.js.map +1 -1
- package/out/tools/golden/relabel-street/directory.d.ts +4 -4
- package/out/tools/golden/relabel-street/directory.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/directory.js +3 -2
- package/out/tools/golden/relabel-street/directory.js.map +1 -1
- package/out/tools/golden/relabel-street/row.d.ts +4 -3
- package/out/tools/golden/relabel-street/row.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/row.js +3 -3
- package/out/tools/golden/relabel-street/row.js.map +1 -1
- package/out/tools/ingest-csv.js +3 -3
- package/out/tools/ingest-csv.js.map +1 -1
- package/out/tools/jsonl-to-parquet.d.ts +1 -1
- package/out/tools/lint/recipe-output/vocab.d.ts +10 -5
- package/out/tools/lint/recipe-output/vocab.d.ts.map +1 -1
- package/out/tools/lint/recipe-output/vocab.js.map +1 -1
- package/out/tools/lint/recipe-output.d.ts +8 -8
- package/out/tools/lint/recipe-output.d.ts.map +1 -1
- package/out/tools/lint/recipe-output.js +17 -0
- package/out/tools/lint/recipe-output.js.map +1 -1
- package/out/tools/overlay/kryptonite.d.ts +2 -2
- package/out/tools/overlay/kryptonite.d.ts.map +1 -1
- package/out/tools/overlay/kryptonite.js +2 -1
- package/out/tools/overlay/kryptonite.js.map +1 -1
- package/out/tools/overlay/manifest.d.ts +2 -1
- package/out/tools/overlay/manifest.d.ts.map +1 -1
- package/out/tools/overlay/manifest.js +2 -1
- package/out/tools/overlay/manifest.js.map +1 -1
- package/out/tools/overlay/merge-source.d.ts +2 -1
- package/out/tools/overlay/merge-source.d.ts.map +1 -1
- package/out/tools/overlay/merge-source.js.map +1 -1
- package/out/tools/overlay/split-slice.d.ts +1 -1
- package/out/tools/overlay/split-slice.js +1 -1
- package/out/tools/overlay/split-slice.js.map +1 -1
- package/out/tools/overlay/translit.d.ts +4 -3
- package/out/tools/overlay/translit.d.ts.map +1 -1
- package/out/tools/overlay/translit.js +2 -1
- package/out/tools/overlay/translit.js.map +1 -1
- package/out/tools/overture-subvenue.d.ts +1 -1
- package/out/tools/overture-subvenue.d.ts.map +1 -1
- package/out/tools/postcode-triples.d.ts +1 -1
- package/out/tools/postcode-triples.d.ts.map +1 -1
- package/out/tools/postcode-triples.js +4 -3
- package/out/tools/postcode-triples.js.map +1 -1
- package/out/tools/source-register/build.d.ts +10 -8
- package/out/tools/source-register/build.d.ts.map +1 -1
- package/out/tools/source-register/build.js +10 -8
- package/out/tools/source-register/build.js.map +1 -1
- package/out/tools/sub/venue/lexicon.d.ts +2 -6
- package/out/tools/sub/venue/lexicon.d.ts.map +1 -1
- package/out/tools/sub/venue/lexicon.js +2 -6
- package/out/tools/sub/venue/lexicon.js.map +1 -1
- package/out/tw/tools/fetch/gcis.js +3 -3
- package/out/tw/tools/fetch/gcis.js.map +1 -1
- package/out/types.d.ts +4 -3
- package/out/types.d.ts.map +1 -1
- package/out/types.js.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.js.map +1 -1
- package/out/us/adapters/usgov/nad/adapter.js +6 -6
- package/out/us/adapters/usgov/nad/adapter.js.map +1 -1
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js +2 -2
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/us/fips-state.d.ts +2 -2
- package/out/us/fips-state.d.ts.map +1 -1
- package/out/us/fips-state.js.map +1 -1
- package/out/us/synthesizers/intersection.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/intersection.js +13 -9
- package/out/us/synthesizers/intersection.js.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.d.ts +3 -2
- package/out/us/synthesizers/street.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.js +12 -12
- package/out/us/synthesizers/street.js.map +1 -0
- package/out/us/tools/fetch/imls-pls.js +1 -1
- package/out/us/tools/fetch/imls-pls.js.map +1 -1
- package/out/us/tools/fetch/nad.d.ts +6 -6
- package/out/us/tools/fetch/nad.js +1 -1
- package/out/us/tools/fetch/nad.js.map +1 -1
- package/out/us/tools/fetch/nppes.js +3 -3
- package/out/us/tools/fetch/nppes.js.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts +2 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.js +1 -1
- package/out/us/tools/fetch/state/hi-schools.js.map +1 -1
- package/out/us/tools/fetch/tiger-full.js +1 -1
- package/out/us/tools/fetch/tiger-full.js.map +1 -1
- package/out/utils/align.d.ts +5 -3
- package/out/utils/align.d.ts.map +1 -1
- package/out/utils/align.js +5 -5
- package/out/utils/align.js.map +1 -1
- package/out/utils/golden.js +2 -2
- package/out/utils/golden.js.map +1 -1
- package/out/utils/license.d.ts +3 -3
- package/out/utils/license.d.ts.map +1 -1
- package/out/utils/license.js.map +1 -1
- package/out/utils/split.d.ts +1 -1
- package/out/utils/split.d.ts.map +1 -1
- package/out/utils/split.js +18 -14
- package/out/utils/split.js.map +1 -1
- package/out/utils/tokenize.d.ts +3 -20
- package/out/utils/tokenize.d.ts.map +1 -1
- package/out/utils/tokenize.js.map +1 -1
- package/out/utils/wof-json.js +3 -3
- package/out/utils/wof-json.js.map +1 -1
- package/package.json +18 -27
- package/lib/tools.ts +0 -31
- package/out/synthesizers/intersection.d.ts.map +0 -1
- package/out/synthesizers/intersection.js.map +0 -1
- package/out/synthesizers/street.d.ts.map +0 -1
- package/out/synthesizers/street.js.map +0 -1
- package/out/tools.d.ts +0 -31
- package/out/tools.d.ts.map +0 -1
- package/out/tools.js +0 -31
- package/out/tools.js.map +0 -1
- /package/out/{synthesizers → us/synthesizers}/intersection.d.ts +0 -0
|
@@ -9,15 +9,14 @@
|
|
|
9
9
|
* service: a predefined-dataset ATOM feed and a Web Feature Service. Most member states publish the
|
|
10
10
|
* second, so a file download reaches only a minority of them.
|
|
11
11
|
*
|
|
12
|
-
* An `ad:Address` feature
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
* several feature types rather than read from one.
|
|
12
|
+
* An `ad:Address` feature holds its point, its lifecycle dates and its locator. Its street name,
|
|
13
|
+
* postcode and administrative units are separate feature types that the address references through
|
|
14
|
+
* `component`. A complete address therefore comes from several feature types.
|
|
16
15
|
*
|
|
17
16
|
* Measured on Slovakia's service, one address holds four component references that resolve to three
|
|
18
|
-
* `ad:AdminUnitName` features and one `ad:PostalDescriptor`.
|
|
19
|
-
*
|
|
20
|
-
*
|
|
17
|
+
* `ad:AdminUnitName` features and one `ad:PostalDescriptor`. One request per reference would cost
|
|
18
|
+
* roughly four requests per address, 6.8 million for that service's 1,704,196 addresses. This module
|
|
19
|
+
* therefore pages each feature type in bulk and joins locally.
|
|
21
20
|
*
|
|
22
21
|
* A service's feature count is read with `readCheckedWFSFeatureCount` from `@mailwoman/core/api`,
|
|
23
22
|
* which owns every WFS count this repository takes.
|
|
@@ -59,8 +58,8 @@ export type ADFeatureType = (typeof AD_FEATURE_TYPES)[number]
|
|
|
59
58
|
* Poland publishes `ms:AD.Address`.
|
|
60
59
|
*
|
|
61
60
|
* A reader that compares the part after the colon against `Address` finds a type
|
|
62
|
-
* on two services and none on the other two
|
|
63
|
-
* no addresses rather than as a naming difference.
|
|
61
|
+
* on two services and none on the other two.
|
|
62
|
+
* That reads as a service publishing no addresses rather than as a naming difference.
|
|
64
63
|
*/
|
|
65
64
|
export function adFeatureTypeOf(qualifiedName: string): ADFeatureType | null {
|
|
66
65
|
const local = qualifiedName.slice(qualifiedName.lastIndexOf(":") + 1)
|
|
@@ -85,8 +84,8 @@ export interface WFSCapabilities {
|
|
|
85
84
|
/**
|
|
86
85
|
* The advertised format this module would ask for, or `null` when the service offers no JSON.
|
|
87
86
|
*
|
|
88
|
-
* A service without one is readable through its GML
|
|
89
|
-
* to its caller rather than reporting as unreadable.
|
|
87
|
+
* A service without one is readable through its GML.
|
|
88
|
+
* This module leaves that to its caller rather than reporting it as unreadable.
|
|
90
89
|
*
|
|
91
90
|
* `application/json` is preferred over `application/geo+json` because a service
|
|
92
91
|
* may advertise the second and reject it.
|
|
@@ -107,14 +106,15 @@ export interface WFSCapabilities {
|
|
|
107
106
|
*
|
|
108
107
|
* A service that does not cannot be paged, so a caller has to take the whole
|
|
109
108
|
* type in one request or decline it.
|
|
110
|
-
*
|
|
109
|
+
* A request past the first page of such a service silently repeats page one.
|
|
111
110
|
*/
|
|
112
111
|
supportsPaging: boolean
|
|
113
112
|
/**
|
|
114
113
|
* The `CountDefault` the service advertises, or `null` where it advertises none.
|
|
115
114
|
*
|
|
116
|
-
* The largest page the service will serve
|
|
117
|
-
* `numberMatched` reports
|
|
115
|
+
* The largest page the service will serve.
|
|
116
|
+
* Its own `numberMatched` reports that number when the service caps the query rather than counting it.
|
|
117
|
+
*
|
|
118
118
|
* Estonia advertises 1000000 and Poland 1000, measured 2026-10-02.
|
|
119
119
|
*/
|
|
120
120
|
countDefault: number | null
|
|
@@ -131,8 +131,9 @@ export interface WFSCapabilities {
|
|
|
131
131
|
/**
|
|
132
132
|
* A value a GeoServer JSON response uses for a voidable property that holds no value.
|
|
133
133
|
*
|
|
134
|
-
* INSPIRE marks many attributes voidable
|
|
135
|
-
*
|
|
134
|
+
* INSPIRE marks many attributes voidable.
|
|
135
|
+
* A voidable property must be present with either a value or a void with a reason.
|
|
136
|
+
*
|
|
136
137
|
* GeoServer encodes that void as an object rather than as `null`, so `String(value)`
|
|
137
138
|
* on one yields `[object Object]` and stores it as though it were data.
|
|
138
139
|
*/
|
|
@@ -160,8 +161,8 @@ export function isVoided(value: unknown): value is VoidedValue {
|
|
|
160
161
|
* This collapses both to `null`, so a caller that needs the two apart reads the
|
|
161
162
|
* property's presence on `properties` itself.
|
|
162
163
|
*/
|
|
163
|
-
export function readVoidable<T>(value: T | VoidedValue |
|
|
164
|
-
if (value ===
|
|
164
|
+
export function readVoidable<T>(value: T | VoidedValue | null): T | null {
|
|
165
|
+
if (value === null || isVoided(value)) return null
|
|
165
166
|
|
|
166
167
|
return value as T
|
|
167
168
|
}
|
|
@@ -215,7 +216,7 @@ export async function readWFSCapabilities(
|
|
|
215
216
|
|
|
216
217
|
// `application/json` is GeoServer's GeoJSON writer, and `application/geo+json` is
|
|
217
218
|
// the alias a service may advertise without implementing.
|
|
218
|
-
// The other JSON-ish formats
|
|
219
|
+
// The other JSON-ish formats have a different envelope, so they are left out rather than tried.
|
|
219
220
|
const jsonFormats = ["application/json", "application/geo+json"].filter((format) => outputFormats.includes(format))
|
|
220
221
|
|
|
221
222
|
const countDefault = /<(?:\w+:)?Constraint\s+name="CountDefault"[\s\S]{0,240}?<(?:\w+:)?DefaultValue>\s*(\d+)\s*</iu
|
|
@@ -226,7 +227,7 @@ export async function readWFSCapabilities(
|
|
|
226
227
|
outputFormats,
|
|
227
228
|
jsonFormat: jsonFormats[0] ?? null,
|
|
228
229
|
jsonFormats,
|
|
229
|
-
countDefault: countDefault
|
|
230
|
+
countDefault: countDefault ? Number(countDefault) : null,
|
|
230
231
|
supportsPaging: /ImplementsResultPaging[\s\S]{0,120}?>\s*(?:TRUE|true)\s*</u.test(data),
|
|
231
232
|
typeNames,
|
|
232
233
|
}
|
|
@@ -247,8 +248,8 @@ export interface FeaturePage<Feature> {
|
|
|
247
248
|
/**
|
|
248
249
|
* The `timeStamp` the service dated the page with, or `null` where it stated none.
|
|
249
250
|
*
|
|
250
|
-
* The service's own clock rather than the caller's
|
|
251
|
-
* records to date a page against the service that served it.
|
|
251
|
+
* The service's own clock rather than the caller's.
|
|
252
|
+
* A manifest records it to date a page against the service that served it.
|
|
252
253
|
*/
|
|
253
254
|
timeStamp: string | null
|
|
254
255
|
}
|
|
@@ -283,8 +284,10 @@ export async function readFeaturePage(
|
|
|
283
284
|
/**
|
|
284
285
|
* The property the service orders the results by, where it honors one.
|
|
285
286
|
*
|
|
286
|
-
*
|
|
287
|
-
* the same order for every request
|
|
287
|
+
* A caller that pages without an ordering assumes the service returns the same
|
|
288
|
+
* features in the same order for every request.
|
|
289
|
+
* No WFS guarantees that.
|
|
290
|
+
*
|
|
288
291
|
* Estonia honors `sortBy=gml_id` and Flanders answers HTTP 504 for its identifier,
|
|
289
292
|
* so the parameter is the caller's to supply or leave out.
|
|
290
293
|
*/
|
|
@@ -362,19 +365,20 @@ const VOID_REASON_TEXT = new Set(["unpopulated", "unknown", "withheld"])
|
|
|
362
365
|
const FLATTENED_COMPONENT = /^component\d*_xlink_href$/iu
|
|
363
366
|
|
|
364
367
|
/**
|
|
365
|
-
* Every `component` reference an address
|
|
368
|
+
* Every `component` reference an address has, as the service wrote them.
|
|
366
369
|
*
|
|
367
370
|
* Two encodings appear across the services measured.
|
|
368
371
|
* Slovakia and Flanders write a `component` array of objects carrying `@href`.
|
|
369
372
|
*
|
|
370
373
|
* Estonia writes one flat property per slot, `component1_xlink_href` through `component6_xlink_href`,
|
|
371
374
|
* and fills an unused slot with the string `unpopulated` rather than with a void object.
|
|
372
|
-
*
|
|
375
|
+
* A read of only the array reported 0 references for every Estonian address.
|
|
376
|
+
* Each address has four.
|
|
373
377
|
*
|
|
374
378
|
* A reference may leave the service: Estonia's first three slots address its Administrative
|
|
375
379
|
* Units theme at `AU_haldusyksused` rather than its Addresses theme.
|
|
376
|
-
* That is a reference this module reads
|
|
377
|
-
* against Addresses features
|
|
380
|
+
* That is a reference this module reads. {@linkcode resolveComponents} reports it as
|
|
381
|
+
* unjoined against Addresses features rather than dropping it.
|
|
378
382
|
*/
|
|
379
383
|
export function componentReferences(feature: GeoJSONFeature): readonly string[] {
|
|
380
384
|
const properties = feature.properties ?? {}
|
|
@@ -402,25 +406,26 @@ export function componentReferences(feature: GeoJSONFeature): readonly string[]
|
|
|
402
406
|
/**
|
|
403
407
|
* The key a component feature publishes for an address to reference it by.
|
|
404
408
|
*
|
|
405
|
-
* `identifier.value` is the INSPIRE external object identifier
|
|
406
|
-
* Flanders writes on both sides of the join.
|
|
409
|
+
* `identifier.value` is the INSPIRE external object identifier.
|
|
410
|
+
* Flanders writes it on both sides of the join.
|
|
411
|
+
*
|
|
407
412
|
* `gml_id` is the fallback for a service that publishes no identifier,
|
|
408
413
|
* and a stored-query reference addresses that id.
|
|
409
414
|
*/
|
|
410
415
|
export function componentIdentifier(feature: GeoJSONFeature): string | null {
|
|
411
416
|
const identifier = feature.properties?.["identifier"]
|
|
412
417
|
|
|
413
|
-
if (typeof identifier === "object" && identifier
|
|
418
|
+
if (typeof identifier === "object" && identifier) {
|
|
414
419
|
const value = (identifier as Record<string, unknown>)["value"]
|
|
415
420
|
|
|
416
|
-
if (typeof value === "string" && value
|
|
421
|
+
if (typeof value === "string" && value) return value
|
|
417
422
|
}
|
|
418
423
|
|
|
419
|
-
if (typeof identifier === "string" && identifier
|
|
424
|
+
if (typeof identifier === "string" && identifier) return identifier
|
|
420
425
|
|
|
421
426
|
const gmlID = feature.properties?.["gml_id"] ?? feature.id
|
|
422
427
|
|
|
423
|
-
return gmlID
|
|
428
|
+
return gmlID ? String(gmlID) : null
|
|
424
429
|
}
|
|
425
430
|
|
|
426
431
|
/**
|
|
@@ -460,7 +465,7 @@ export function resolveComponents(
|
|
|
460
465
|
for (const feature of components) {
|
|
461
466
|
const identifier = componentIdentifier(feature)
|
|
462
467
|
|
|
463
|
-
if (identifier
|
|
468
|
+
if (identifier) {
|
|
464
469
|
identifiers.add(identifier)
|
|
465
470
|
}
|
|
466
471
|
}
|
|
@@ -474,7 +479,7 @@ export function resolveComponents(
|
|
|
474
479
|
for (const href of componentReferences(address)) {
|
|
475
480
|
const key = componentJoinKey(href)
|
|
476
481
|
|
|
477
|
-
if (key
|
|
482
|
+
if (!key) {
|
|
478
483
|
unreadable++
|
|
479
484
|
|
|
480
485
|
continue
|
|
@@ -9,7 +9,7 @@ import { APIClient, isSuccessStatus } from "@mailwoman/core/api"
|
|
|
9
9
|
/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
|
|
10
10
|
procedure rather than sections of declarations. A region there folds no element a reader wants folded. */
|
|
11
11
|
import { ByteFormatter } from "@mailwoman/core/fs/formatters"
|
|
12
|
-
import { statPath, pathExists } from "@mailwoman/core/fs/readers"
|
|
12
|
+
import { statPath, pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
13
13
|
import { openWriteStream, pipeline } from "@mailwoman/core/fs/streams"
|
|
14
14
|
import { movePath, removePathIfPresent, makeDirectories } from "@mailwoman/core/fs/writers"
|
|
15
15
|
import { sha256File } from "@mailwoman/core/hash"
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* adapter joins locally rather than resolving four references per address against 4,563,062 of
|
|
15
15
|
* them.
|
|
16
16
|
* 2. `sortBy=gml:identifier` answers HTTP 504, so the service grants no ordering. The manifest
|
|
17
|
-
* records the pages taken and their order
|
|
17
|
+
* records the pages taken and their order. That record makes a harvest reproducible even though
|
|
18
18
|
* a second harvest need not match the first.
|
|
19
19
|
* 3. A bare `resultType=hits` answers `numberMatched="10000"` whatever `count` asks for. The count
|
|
20
20
|
* has to be taken past index 0: `startIndex=1` answers `numberMatched="4563062"`, and the
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
* returns none.
|
|
23
23
|
*
|
|
24
24
|
* Article 3 of the elected `Modellicentie voor gratis hergebruik Vlaanderen` grants reproduction and
|
|
25
|
-
* transmission,
|
|
25
|
+
* transmission, and article 6 states that access is free. Those two grants cover paging. The
|
|
26
26
|
* capabilities document's `ows:Fees` and `ows:AccessConstraints` describe the service rather than
|
|
27
27
|
* the data, so neither is read here.
|
|
28
28
|
*/
|
|
@@ -48,7 +48,7 @@ export const VLAANDEREN_AD_WFS = "https://geo.api.vlaanderen.be/ad/wfs"
|
|
|
48
48
|
* The `outputFormat` the harvest is written in.
|
|
49
49
|
*
|
|
50
50
|
* GML is the format whose `ad:component/@xlink:href` the adapter reads.
|
|
51
|
-
* `application/json`
|
|
51
|
+
* `application/json` holds the same references under `@href` and serves the same lost characters,
|
|
52
52
|
* so the choice is a matter of which reader the adapter uses rather than of what the service knows.
|
|
53
53
|
*/
|
|
54
54
|
const OUTPUT_FORMAT = "application/gml+xml; version=3.2"
|
|
@@ -68,8 +68,7 @@ export const MAX_PAGE_SIZE = 10_000
|
|
|
68
68
|
* Measured 2026-10-02, `ad:ThoroughfareName` holds 167,218 features against a 10,000 page cap,
|
|
69
69
|
* so it needs 17 pages, while the other two hold 1,190 and 609 and fit in one.
|
|
70
70
|
*
|
|
71
|
-
*
|
|
72
|
-
* until one of them passed the cap.
|
|
71
|
+
* A single file for the small ones would leave the harvester correct only until one of them passed the cap.
|
|
73
72
|
*/
|
|
74
73
|
const COMPONENT_STEMS = {
|
|
75
74
|
ThoroughfareName: "thoroughfare-name",
|
|
@@ -103,8 +102,8 @@ export interface FeaturePageBody {
|
|
|
103
102
|
/**
|
|
104
103
|
* Issues one `GetFeature` against the service.
|
|
105
104
|
*
|
|
106
|
-
* `#tools/fetch/wfs-harvest` owns the request, the exception check and the counts
|
|
107
|
-
*
|
|
105
|
+
* `#tools/fetch/wfs-harvest` owns the request, the exception check and the counts.
|
|
106
|
+
* Every markup-serving WFS in this directory shares them.
|
|
108
107
|
* This states the service and its context.
|
|
109
108
|
*
|
|
110
109
|
* @throws When the service answers an OGC exception report, so a 400 carrying
|
|
@@ -162,7 +161,7 @@ export interface HarvestVlaanderenOptions {
|
|
|
162
161
|
* Features per component page, capped at {@link MAX_PAGE_SIZE} and defaulting to it.
|
|
163
162
|
*
|
|
164
163
|
* Only a test lowers this.
|
|
165
|
-
* A component type is always taken whole, so a smaller page
|
|
164
|
+
* A component type is always taken whole, so a smaller page yields only more requests.
|
|
166
165
|
*/
|
|
167
166
|
componentPageSize?: number
|
|
168
167
|
signal?: AbortSignal
|
|
@@ -173,14 +172,14 @@ export interface HarvestVlaanderenOptions {
|
|
|
173
172
|
*
|
|
174
173
|
* The component types are taken first and whole, so a harvest that stops early
|
|
175
174
|
* holds address pages whose references all resolve.
|
|
176
|
-
*
|
|
175
|
+
* Addresses first would leave every page unjoinable.
|
|
177
176
|
*
|
|
178
177
|
* @returns The manifest as written.
|
|
179
178
|
*/
|
|
180
179
|
/**
|
|
181
180
|
* How much of a page is read to find its root attributes.
|
|
182
181
|
*
|
|
183
|
-
* `numberReturned` sits on the `wfs:FeatureCollection` element
|
|
182
|
+
* `numberReturned` sits on the `wfs:FeatureCollection` element that opens the document.
|
|
184
183
|
* A page of this service is about 20 MB, and reading all of one to find an attribute in
|
|
185
184
|
* its first kilobyte would cost 1.6 GB of reads across a whole-region harvest.
|
|
186
185
|
*/
|
|
@@ -204,9 +203,10 @@ interface PageOnDisk {
|
|
|
204
203
|
*
|
|
205
204
|
* The feature count is read from each file's own `numberReturned` rather than assumed
|
|
206
205
|
* from the page size, so a page the service answered short is recorded as short.
|
|
207
|
-
* A file whose opening bytes state no `numberReturned` is left out
|
|
208
|
-
*
|
|
209
|
-
* harvest
|
|
206
|
+
* A file whose opening bytes state no `numberReturned` is left out.
|
|
207
|
+
*
|
|
208
|
+
* The harvest then requests that index again: a page of unknown extent is not a page
|
|
209
|
+
* this harvest can count, and a run killed mid-write leaves exactly that.
|
|
210
210
|
*
|
|
211
211
|
* A group small enough for one page is written under `<stem>.gml` with no index,
|
|
212
212
|
* so that name reads as `startIndex` 0.
|
|
@@ -222,7 +222,7 @@ async function pagesOnDisk(root: PathBuilder, stem: string): Promise<Map<number,
|
|
|
222
222
|
|
|
223
223
|
if (!matched) continue
|
|
224
224
|
|
|
225
|
-
const startIndex = matched[1]
|
|
225
|
+
const startIndex = matched[1] ? Number(matched[1]) : 0
|
|
226
226
|
|
|
227
227
|
if (!Number.isInteger(startIndex)) continue
|
|
228
228
|
|
|
@@ -236,7 +236,7 @@ async function pagesOnDisk(root: PathBuilder, stem: string): Promise<Map<number,
|
|
|
236
236
|
xml: true,
|
|
237
237
|
})
|
|
238
238
|
|
|
239
|
-
if (returned
|
|
239
|
+
if (!returned || !/^\d+$/u.test(returned)) continue
|
|
240
240
|
|
|
241
241
|
pages.set(startIndex, { file: name, numberReturned: Number(returned) })
|
|
242
242
|
} finally {
|
|
@@ -271,7 +271,7 @@ export async function harvestVlaanderenAD(
|
|
|
271
271
|
for (let startIndex = 0; startIndex < count; startIndex += componentPageSize) {
|
|
272
272
|
const held = existing.get(startIndex)
|
|
273
273
|
|
|
274
|
-
if (held
|
|
274
|
+
if (held) {
|
|
275
275
|
componentFiles[typeName].push(held.file)
|
|
276
276
|
read += held.numberReturned
|
|
277
277
|
|
|
@@ -331,7 +331,7 @@ export async function harvestVlaanderenAD(
|
|
|
331
331
|
|
|
332
332
|
const existing = onDisk.get(startIndex)
|
|
333
333
|
|
|
334
|
-
if (existing
|
|
334
|
+
if (existing) {
|
|
335
335
|
addressPages.push({ startIndex, file: existing.file, numberReturned: existing.numberReturned })
|
|
336
336
|
|
|
337
337
|
continue
|
|
@@ -22,10 +22,10 @@
|
|
|
22
22
|
*
|
|
23
23
|
* 1. A service may ignore `startIndex` and answer every page with the first. `countWFSFeaturesByPaging`
|
|
24
24
|
* in `@mailwoman/core/api` refuses such a service by comparing two pages' leading feature. A
|
|
25
|
-
* harvest sees the same defect as two consecutive pages with equal bytes
|
|
25
|
+
* harvest sees the same defect as two consecutive pages with equal bytes. It refuses that pair.
|
|
26
26
|
* 2. A service's own `numberMatched` may be a per-request cap rather than a count. The caller states
|
|
27
27
|
* what its count is and where it came from, and {@linkcode PagedWFSHarvestOptions.featureCount}
|
|
28
|
-
*
|
|
28
|
+
* holds both. No count is written into this module, and a harvest with no usable count ends on
|
|
29
29
|
* the first page that returns no features.
|
|
30
30
|
* 3. A page may return fewer features than `count` asks for. The next `startIndex` therefore advances
|
|
31
31
|
* by the features the service returned rather than by the page size, so a service that caps a page
|
|
@@ -34,7 +34,8 @@
|
|
|
34
34
|
|
|
35
35
|
import { APIClient, assertNoOGCServiceException, type CheckedWFSFeatureCount } from "@mailwoman/core/api"
|
|
36
36
|
import { ByteFormatter } from "@mailwoman/core/fs/formatters"
|
|
37
|
-
import { readFileRange
|
|
37
|
+
import { readFileRange } from "@mailwoman/core/fs/readers"
|
|
38
|
+
import { tryStat } from "@mailwoman/core/fs/readers/stat"
|
|
38
39
|
import { appendLocalTextFile, makeDirectories, removePathIfPresent, truncateFile } from "@mailwoman/core/fs/writers"
|
|
39
40
|
import { sha256File, sha256Hex } from "@mailwoman/core/hash"
|
|
40
41
|
import { stringifyJSON } from "@mailwoman/core/json"
|
|
@@ -69,7 +70,8 @@ export interface MarkupRoot {
|
|
|
69
70
|
*/
|
|
70
71
|
name: string
|
|
71
72
|
/**
|
|
72
|
-
* The text between the name and the tag's terminator
|
|
73
|
+
* The text between the name and the tag's terminator.
|
|
74
|
+
* It holds the attributes.
|
|
73
75
|
*/
|
|
74
76
|
attributes: string
|
|
75
77
|
/**
|
|
@@ -85,17 +87,18 @@ export interface MarkupRoot {
|
|
|
85
87
|
* opening tag verbatim to open the file it assembles, and parsing a 1.6 MB page per request
|
|
86
88
|
* to recover a tag that the first 400 bytes already hold costs a document parse per page.
|
|
87
89
|
*
|
|
88
|
-
* The leading `[A-Za-z_]` skips the XML declaration
|
|
90
|
+
* The leading `[A-Za-z_]` skips the XML declaration.
|
|
91
|
+
* That declaration opens `<?xml`.
|
|
89
92
|
*/
|
|
90
93
|
const ROOT_OPENING_TAG = /<([A-Za-z_][\w.:-]*)([^>]*)>/u
|
|
91
94
|
|
|
92
95
|
/**
|
|
93
96
|
* One attribute of an opening tag, by local name.
|
|
94
97
|
*/
|
|
95
|
-
function tagAttribute(root: MarkupRoot, attribute: string): string |
|
|
98
|
+
function tagAttribute(root: MarkupRoot, attribute: string): string | null {
|
|
96
99
|
const pattern = new RegExp(`(?:^|\\s)(?:[\\w.-]+:)?${attribute}\\s*=\\s*"([^"]*)"`, "iu")
|
|
97
100
|
|
|
98
|
-
return pattern.exec(root.attributes)?.[1]
|
|
101
|
+
return pattern.exec(root.attributes)?.[1] ?? null
|
|
99
102
|
}
|
|
100
103
|
|
|
101
104
|
/**
|
|
@@ -106,7 +109,7 @@ function tagAttribute(root: MarkupRoot, attribute: string): string | undefined {
|
|
|
106
109
|
export function rootCount(root: MarkupRoot, attribute: string): number | null {
|
|
107
110
|
const value = tagAttribute(root, attribute)
|
|
108
111
|
|
|
109
|
-
return value
|
|
112
|
+
return value && /^\d+$/u.test(value) ? Number(value) : null
|
|
110
113
|
}
|
|
111
114
|
|
|
112
115
|
/**
|
|
@@ -133,7 +136,8 @@ export function readMarkupRoot(body: string, context: string): MarkupRoot {
|
|
|
133
136
|
}
|
|
134
137
|
|
|
135
138
|
/**
|
|
136
|
-
* The content between a document's root tags
|
|
139
|
+
* The content between a document's root tags.
|
|
140
|
+
* For a `wfs:FeatureCollection` those are its members.
|
|
137
141
|
*
|
|
138
142
|
* @throws When the root element is not closed, so a truncated transfer reads as
|
|
139
143
|
* unreadable rather than as a page of no features.
|
|
@@ -214,7 +218,7 @@ export async function readWFSMarkupPage(
|
|
|
214
218
|
root,
|
|
215
219
|
numberReturned: rootCount(root, "numberReturned"),
|
|
216
220
|
numberMatched: rootCount(root, "numberMatched"),
|
|
217
|
-
timeStamp: tagAttribute(root, "timeStamp")
|
|
221
|
+
timeStamp: tagAttribute(root, "timeStamp"),
|
|
218
222
|
}
|
|
219
223
|
}
|
|
220
224
|
|
|
@@ -231,8 +235,9 @@ export interface WFSHarvestRequest {
|
|
|
231
235
|
/**
|
|
232
236
|
* The property the service was asked to order by, or `null` where it grants no ordering.
|
|
233
237
|
*
|
|
234
|
-
* Without one, a resumed harvest rests on the service returning the same
|
|
235
|
-
* in the same order as the earlier run
|
|
238
|
+
* Without one, a resumed harvest rests on the service returning the same
|
|
239
|
+
* features in the same order as the earlier run.
|
|
240
|
+
* No WFS guarantees that order.
|
|
236
241
|
*/
|
|
237
242
|
sortBy: string | null
|
|
238
243
|
}
|
|
@@ -252,8 +257,9 @@ export interface FeatureCountStatement {
|
|
|
252
257
|
/**
|
|
253
258
|
* The count a service states, or no count and the reason there is none.
|
|
254
259
|
*
|
|
255
|
-
* `readCheckedWFSFeatureCount` proves a reported `numberMatched` against a page of the
|
|
256
|
-
*
|
|
260
|
+
* `readCheckedWFSFeatureCount` proves a reported `numberMatched` against a page of the same type.
|
|
261
|
+
* That comparison catches a service whose count is smaller than one of its own pages.
|
|
262
|
+
*
|
|
257
263
|
* It cannot catch a count that equals the service's page cap: no page can return more
|
|
258
264
|
* features than the cap, so the page it asks for agrees with the cap every time.
|
|
259
265
|
*
|
|
@@ -263,7 +269,7 @@ export interface FeatureCountStatement {
|
|
|
263
269
|
*
|
|
264
270
|
* The reported number is the cap, and a harvest that read it as a count would store 1,000
|
|
265
271
|
* of the 8.6 million features the service holds and record the type as read whole.
|
|
266
|
-
*
|
|
272
|
+
* The harvest therefore records no count when the reported number equals the cap.
|
|
267
273
|
*
|
|
268
274
|
* @param pageCap The largest page the service will serve, from its advertised `CountDefault`.
|
|
269
275
|
*/
|
|
@@ -305,7 +311,8 @@ export interface HarvestedPayload {
|
|
|
305
311
|
*/
|
|
306
312
|
footer?: string
|
|
307
313
|
/**
|
|
308
|
-
* The features the service returned
|
|
314
|
+
* The features the service returned.
|
|
315
|
+
* The next `startIndex` advances by this number.
|
|
309
316
|
*
|
|
310
317
|
* `null` where the service stated none.
|
|
311
318
|
* The harvest refuses that rather than reading it as zero.
|
|
@@ -324,7 +331,8 @@ export interface HarvestedPayload {
|
|
|
324
331
|
export interface WFSHarvestPageRecord {
|
|
325
332
|
start_index: number
|
|
326
333
|
/**
|
|
327
|
-
* The page size requested
|
|
334
|
+
* The page size requested.
|
|
335
|
+
* A service may answer with fewer features.
|
|
328
336
|
*/
|
|
329
337
|
count: number
|
|
330
338
|
offset: number
|
|
@@ -508,15 +516,14 @@ function nextStartIndexOf(pages: readonly WFSHarvestPageRecord[]): number {
|
|
|
508
516
|
}
|
|
509
517
|
|
|
510
518
|
/**
|
|
511
|
-
* The recorded pages
|
|
519
|
+
* The recorded pages present on disk, with the file truncated back to the last of them.
|
|
512
520
|
*
|
|
513
521
|
* An interrupted run can leave a partial page at the end of the file,
|
|
514
522
|
* or a footer the next page has to be appended past.
|
|
515
|
-
* Both are resolved by
|
|
516
|
-
* which is a page boundary rather than an arbitrary offset.
|
|
523
|
+
* Both are resolved by truncating the file at the last page boundary it reaches.
|
|
517
524
|
*
|
|
518
|
-
* @throws When the last surviving page's bytes no longer hash to what the manifest recorded
|
|
519
|
-
*
|
|
525
|
+
* @throws When the last surviving page's bytes no longer hash to what the manifest recorded.
|
|
526
|
+
* A mismatch means the file was altered after the harvest recorded it.
|
|
520
527
|
*/
|
|
521
528
|
async function reconcileHarvestFile(
|
|
522
529
|
manifest: WFSHarvestManifest,
|
|
@@ -601,7 +608,7 @@ export interface HarvestCurrencyExpectation {
|
|
|
601
608
|
* Two harvests need none:
|
|
602
609
|
*
|
|
603
610
|
* - A complete one whose data file still hashes to the digest the manifest records.
|
|
604
|
-
* - A partial one that already holds `maxPages` pages,
|
|
611
|
+
* - A partial one that already holds `maxPages` pages, the cap a bounded probe requests.
|
|
605
612
|
*
|
|
606
613
|
* @throws When a page's recorded byte range no longer hashes to its recorded digest,
|
|
607
614
|
* so a file that was altered under its manifest is reported rather than used.
|
|
@@ -679,7 +686,7 @@ export async function harvestPagedWFS(options: PagedWFSHarvestOptions): Promise<
|
|
|
679
686
|
const recorded = await readManifest<WFSHarvestManifest>(manifestPath)
|
|
680
687
|
const resumable = recorded !== null && Array.isArray(recorded.pages) && describesSameRequest(recorded, options)
|
|
681
688
|
|
|
682
|
-
if (recorded
|
|
689
|
+
if (recorded && !resumable) {
|
|
683
690
|
options.report?.(" The manifest describes a different request — starting the harvest over")
|
|
684
691
|
|
|
685
692
|
await removePathIfPresent(dataPath)
|
|
@@ -833,7 +840,8 @@ export async function harvestPagedWFS(options: PagedWFSHarvestOptions): Promise<
|
|
|
833
840
|
*/
|
|
834
841
|
export interface RunWFSHarvestOptions {
|
|
835
842
|
/**
|
|
836
|
-
* The slug the harvest is written under and reported by
|
|
843
|
+
* The slug the harvest is written under and reported by.
|
|
844
|
+
* It matches the adapter's own id.
|
|
837
845
|
*/
|
|
838
846
|
slug: string
|
|
839
847
|
|
|
@@ -843,7 +851,8 @@ export interface RunWFSHarvestOptions {
|
|
|
843
851
|
outRoot: (segment: string) => PathBuilderLike
|
|
844
852
|
|
|
845
853
|
/**
|
|
846
|
-
* The data file inside the harvest directory
|
|
854
|
+
* The data file inside the harvest directory.
|
|
855
|
+
* The adapter's `inputPath` points here.
|
|
847
856
|
*/
|
|
848
857
|
filename: string
|
|
849
858
|
|
|
@@ -868,12 +877,14 @@ export interface RunWFSHarvestOptions {
|
|
|
868
877
|
harvest: (client: Pick<APIClient, "fetch">) => Promise<WFSHarvestManifest>
|
|
869
878
|
|
|
870
879
|
/**
|
|
871
|
-
* A client to use rather than constructing one
|
|
880
|
+
* A client to use rather than constructing one.
|
|
881
|
+
* A test supplies this.
|
|
872
882
|
*/
|
|
873
883
|
client?: Pick<APIClient, "fetch">
|
|
874
884
|
|
|
875
885
|
/**
|
|
876
|
-
* A page cap
|
|
886
|
+
* A page cap.
|
|
887
|
+
* A capped run treats an incomplete harvest as a success.
|
|
877
888
|
*/
|
|
878
889
|
maxPages?: number
|
|
879
890
|
}
|
package/lib/tools/fetch.ts
CHANGED
|
@@ -48,48 +48,6 @@ import { fetchStateHISchools } from "#us/tools/fetch/state/hi-schools"
|
|
|
48
48
|
import { fetchStateSources } from "#us/tools/fetch/state/sources"
|
|
49
49
|
import { fetchTigerFull } from "#us/tools/fetch/tiger-full"
|
|
50
50
|
|
|
51
|
-
export * from "#sg/tools/fetch/acra"
|
|
52
|
-
export * from "#ee/tools/fetch/ads"
|
|
53
|
-
export * from "#be/tools/fetch/brussels"
|
|
54
|
-
export * from "#tools/fetch/vlaanderen-ad"
|
|
55
|
-
export * from "#be/tools/fetch/wallonie"
|
|
56
|
-
export * from "#it/tools/fetch/anac"
|
|
57
|
-
export * from "#fr/tools/fetch/finess"
|
|
58
|
-
export * from "#fr/tools/fetch/annuaire-education"
|
|
59
|
-
export * from "#ro/tools/fetch/retea-scolara"
|
|
60
|
-
export * from "#tools/fetch/cordis"
|
|
61
|
-
export * from "#tools/fetch/gleif"
|
|
62
|
-
export * from "#fr/tools/fetch/ban"
|
|
63
|
-
export * from "#lu/tools/fetch/bd-adresses"
|
|
64
|
-
export * from "#lv/tools/fetch/varis"
|
|
65
|
-
export * from "#es/tools/fetch/bizkaia"
|
|
66
|
-
export * from "#es/tools/fetch/catastro"
|
|
67
|
-
export * from "#cz/tools/fetch/cuzk"
|
|
68
|
-
export * from "#es/tools/fetch/gipuzkoa"
|
|
69
|
-
export * from "#es/tools/fetch/navarra"
|
|
70
|
-
export * from "#pl/tools/fetch/emuia"
|
|
71
|
-
export * from "#dk/tools/fetch/inspire"
|
|
72
|
-
export * from "#sk/tools/fetch/inspire"
|
|
73
|
-
export * from "#tw/tools/fetch/gcis"
|
|
74
|
-
export * from "#tools/fetch/geonames/dump"
|
|
75
|
-
export * from "#tools/fetch/geonames/postal"
|
|
76
|
-
export * from "#jp/tools/fetch/houjin"
|
|
77
|
-
export * from "#us/tools/fetch/hrsa"
|
|
78
|
-
export * from "#us/tools/fetch/imls-pls"
|
|
79
|
-
export * from "#kr/tools/fetch/juso"
|
|
80
|
-
export * from "#nl/tools/fetch/kadaster"
|
|
81
|
-
export * from "#kr/tools/fetch/localdata"
|
|
82
|
-
export * from "#no/tools/fetch/matrikkelen"
|
|
83
|
-
export * from "#us/tools/fetch/nad"
|
|
84
|
-
export * from "#us/tools/fetch/nppes"
|
|
85
|
-
export * from "#tools/fetch/openaddresses"
|
|
86
|
-
export * from "#tools/fetch/ourairports"
|
|
87
|
-
export * from "#fi/tools/fetch/ryhti"
|
|
88
|
-
export * from "#us/tools/fetch/state/hi-schools"
|
|
89
|
-
export * from "#us/tools/fetch/state/sources"
|
|
90
|
-
export * from "#us/tools/fetch/tiger-full"
|
|
91
|
-
export * from "#tools/fetch/wikidata-subvenue"
|
|
92
|
-
|
|
93
51
|
/**
|
|
94
52
|
* The fetcher for each source id that `mailwoman corpus fetch` accepts.
|
|
95
53
|
*/
|
|
@@ -113,43 +113,43 @@ export interface ExpandGoldenOptions {
|
|
|
113
113
|
/**
|
|
114
114
|
* Corpus test-split parquet path(s), comma-separated.
|
|
115
115
|
*
|
|
116
|
-
*
|
|
116
|
+
* @defaultValue the v0.2.0 test parquet file under the data root.
|
|
117
117
|
*/
|
|
118
118
|
corpus?: string
|
|
119
119
|
/**
|
|
120
120
|
* Total seeds to process.
|
|
121
121
|
*
|
|
122
|
-
*
|
|
122
|
+
* @defaultValue `100` (pilot).
|
|
123
123
|
*/
|
|
124
124
|
count?: number
|
|
125
125
|
/**
|
|
126
126
|
* Variants requested per seed.
|
|
127
127
|
*
|
|
128
|
-
*
|
|
128
|
+
* @defaultValue `5`
|
|
129
129
|
*/
|
|
130
130
|
variants?: number
|
|
131
131
|
/**
|
|
132
132
|
* Jsonl output path.
|
|
133
133
|
*
|
|
134
|
-
*
|
|
134
|
+
* @defaultValue `data/eval/golden/candidates/expand-<ts>.jsonl`
|
|
135
135
|
*/
|
|
136
136
|
output?: string
|
|
137
137
|
/**
|
|
138
138
|
* LLM provider.
|
|
139
139
|
*
|
|
140
|
-
*
|
|
140
|
+
* @defaultValue `deepseek`
|
|
141
141
|
*/
|
|
142
142
|
provider?: "deepseek" | "anthropic"
|
|
143
143
|
/**
|
|
144
144
|
* Model id.
|
|
145
145
|
*
|
|
146
|
-
*
|
|
146
|
+
* @defaultValue depends on provider.
|
|
147
147
|
*/
|
|
148
148
|
model?: string
|
|
149
149
|
/**
|
|
150
150
|
* Parallel LLM calls.
|
|
151
151
|
*
|
|
152
|
-
*
|
|
152
|
+
* @defaultValue `4`
|
|
153
153
|
*/
|
|
154
154
|
concurrency?: number
|
|
155
155
|
/**
|
|
@@ -444,7 +444,7 @@ function parseCandidates(text: string): Candidate[] {
|
|
|
444
444
|
if (Array.isArray(parsed)) return parsed as Candidate[]
|
|
445
445
|
|
|
446
446
|
// Some providers wrap in {"variants": [...]} or {"candidates": [...]}
|
|
447
|
-
if (typeof parsed === "object" && parsed
|
|
447
|
+
if (typeof parsed === "object" && parsed) {
|
|
448
448
|
for (const key of ["variants", "candidates", "results"]) {
|
|
449
449
|
const v = (parsed as Record<string, unknown>)[key]
|
|
450
450
|
|