@mailwoman/corpus 10.1.0 → 10.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/licenses/cdla-permissive-2.0.md +1 -1
- package/lib/adapters/cordis/adapter.ts +13 -7
- package/lib/adapters/dedup-key.ts +1 -1
- package/lib/adapters/geonames/adapter.ts +1 -1
- package/lib/adapters/gleif/adapter.ts +17 -16
- package/lib/adapters/libpostal-dictionary.ts +0 -2
- package/lib/adapters/openaddresses/adapter.ts +4 -3
- package/lib/adapters/osm/adapter.ts +22 -21
- package/lib/adapters/po-box/adapter.ts +10 -5
- package/lib/adapters/registry.ts +4 -4
- package/lib/adapters/source-id.ts +2 -2
- package/lib/adapters/street-line.ts +7 -6
- package/lib/adapters/wof/admin/json/adapter.ts +2 -2
- package/lib/adapters/wof/json-rows.ts +2 -1
- package/lib/adapters/wof/postalcode-json/adapter.ts +6 -2
- package/lib/adapters.ts +5 -5
- package/lib/au/adapters/gnaf/adapter.ts +1 -1
- package/lib/be/adapters/brussels/adapter.ts +22 -22
- package/lib/be/adapters/vlaanderen/adapter.ts +54 -57
- package/lib/be/adapters/wallonie/adapter.ts +17 -17
- package/lib/be/tools/fetch/brussels.ts +13 -10
- package/lib/be/tools/fetch/wallonie.ts +8 -7
- package/lib/build/adapters.ts +2 -1
- package/lib/build/align.ts +5 -4
- package/lib/build/checkpoint.ts +5 -3
- package/lib/build/eligibility.ts +3 -3
- package/lib/build/types.ts +8 -6
- package/lib/cz/adapters/cuzk/adapter.ts +41 -38
- package/lib/cz/tools/fetch/cuzk.ts +42 -31
- package/lib/de/recipes/locale.ts +1 -1
- package/lib/dk/adapters/inspire/adapter.ts +38 -38
- package/lib/dk/tools/fetch/inspire.ts +16 -14
- package/lib/ee/adapters/ads/adapter.ts +26 -32
- package/lib/ee/tools/fetch/ads.ts +10 -8
- package/lib/es/adapters/bizkaia/adapter.ts +47 -47
- package/lib/es/adapters/catastro/adapter.ts +36 -34
- package/lib/es/adapters/gipuzkoa/adapter.ts +38 -45
- package/lib/es/adapters/navarra/adapter.ts +28 -27
- package/lib/es/cadastre.ts +31 -31
- package/lib/es/tools/fetch/bizkaia.ts +40 -36
- package/lib/es/tools/fetch/catastro.ts +51 -40
- package/lib/es/tools/fetch/gipuzkoa.ts +13 -11
- package/lib/es/tools/fetch/navarra.ts +33 -25
- package/lib/fi/adapters/ryhti/adapter.ts +22 -26
- package/lib/fi/tools/fetch/ryhti.ts +31 -31
- package/lib/fr/adapters/annuaire-education/adapter.ts +13 -11
- package/lib/fr/adapters/ban/adapter.ts +6 -6
- package/lib/fr/adapters/finess/adapter.ts +16 -15
- package/lib/fr/adapters/finess/voie-type.ts +4 -3
- package/lib/fr/insee-country.ts +2 -2
- package/lib/fr/recipes/lieudit.ts +8 -6
- package/lib/fr/tools/fetch/annuaire-education.ts +11 -8
- package/lib/fr/tools/fetch/ban.ts +3 -2
- package/lib/fr/tools/fetch/finess.ts +14 -13
- package/lib/inspire/address.ts +86 -78
- package/lib/inspire/errors.ts +9 -8
- package/lib/inspire/stream.ts +29 -26
- package/lib/international/recipes/locale.ts +13 -9
- package/lib/it/adapters/anac/adapter.ts +51 -46
- package/lib/it/tools/fetch/anac.ts +22 -25
- package/lib/jp/adapters/wof-admin/adapter.ts +2 -6
- package/lib/jp/tools/fetch/houjin.ts +3 -3
- package/lib/kr/adapters/localdata/permit-rows.ts +1 -1
- package/lib/kr/tools/fetch/juso.ts +1 -1
- package/lib/kr/tools/fetch/localdata.ts +2 -2
- package/lib/lu/adapters/bd-adresses/adapter.ts +10 -10
- package/lib/lu/tools/fetch/bd-adresses.ts +15 -13
- package/lib/lv/adapters/varis/adapter.ts +24 -23
- package/lib/lv/tools/fetch/varis.ts +3 -2
- package/lib/nl/adapters/kadaster/adapter.ts +29 -29
- package/lib/nl/tools/fetch/kadaster.ts +16 -12
- package/lib/no/adapters/matrikkelen/adapter.ts +6 -5
- package/lib/no/tools/fetch/matrikkelen.ts +59 -47
- package/lib/parquet/duckdb.ts +7 -4
- package/lib/parquet/readers.ts +3 -3
- package/lib/parquet/schema.ts +1 -1
- package/lib/parquet/writers.ts +3 -2
- package/lib/pl/adapters/emuia/adapter.ts +12 -12
- package/lib/pl/tools/fetch/emuia.ts +10 -9
- package/lib/recipes/bare/postcode.ts +1 -1
- package/lib/recipes/boundary-stress.ts +1 -1
- package/lib/recipes/house-venue.ts +7 -3
- package/lib/recipes/intersection.ts +2 -2
- package/lib/recipes/po/box/cedex/recipe.ts +3 -3
- package/lib/recipes/po/box/cedex/sources.ts +13 -1
- package/lib/recipes/scaffold.ts +31 -21
- package/lib/recipes/sources.ts +16 -3
- package/lib/recipes/street/bare.ts +2 -2
- package/lib/recipes/street.ts +3 -1
- package/lib/recipes/sub/venue/context.ts +2 -0
- package/lib/recipes/sub/venue/render.ts +1 -1
- package/lib/recipes/sub/venue/sources.ts +24 -29
- package/lib/recipes/sub/venue.ts +12 -18
- package/lib/recipes/trailing-region.ts +6 -2
- package/lib/recipes/unit.ts +3 -3
- package/lib/recipes/venue.ts +347 -0
- package/lib/recipes.ts +4 -2
- package/lib/registers.ts +8 -6
- package/lib/ro/adapters/retea-scolara/adapter.ts +5 -5
- package/lib/ro/tools/fetch/retea-scolara.ts +1 -1
- package/lib/runner.ts +4 -3
- package/lib/sg/recipes/register.ts +1 -1
- package/lib/sg/tools/fetch/acra.ts +3 -3
- package/lib/sk/adapters/inspire/adapter.ts +25 -27
- package/lib/sk/tools/fetch/inspire.ts +15 -14
- package/lib/source-register/effective-manifest.ts +3 -3
- package/lib/source-register/register.ts +3 -3
- package/lib/source-register/training-manifest.ts +1 -1
- package/lib/source-register/types.ts +12 -13
- package/lib/subvenue/promotions.ts +1 -1
- package/lib/subvenue/table.ts +4 -4
- package/lib/surfaces/locale.ts +2 -0
- package/lib/synthesizers/anchor-absorption.ts +2 -1
- package/lib/synthesizers/boundary-stress.ts +20 -26
- package/lib/synthesizers/house-venue.ts +26 -30
- package/lib/synthesizers/no-street.ts +2 -1
- package/lib/synthesizers/po-box.ts +6 -3
- package/lib/synthesizers/tail.ts +55 -0
- package/lib/synthesizers/utils.ts +19 -2
- package/lib/test-kit.ts +2 -2
- package/lib/tools/audit.ts +7 -5
- package/lib/tools/corpus-stats.ts +1 -1
- package/lib/tools/fetch/atom.ts +10 -10
- package/lib/tools/fetch/cordis.ts +11 -9
- package/lib/tools/fetch/download/manifest.ts +2 -1
- package/lib/tools/fetch/download/network.ts +4 -4
- package/lib/tools/fetch/download/retry.ts +4 -4
- package/lib/tools/fetch/geonames/dump.ts +5 -3
- package/lib/tools/fetch/geonames/postal.ts +2 -2
- package/lib/tools/fetch/gleif.ts +6 -4
- package/lib/tools/fetch/header.ts +8 -6
- package/lib/tools/fetch/inspire-addresses.ts +41 -36
- package/lib/tools/fetch/openaddresses.ts +1 -1
- package/lib/tools/fetch/vlaanderen-ad.ts +17 -17
- package/lib/tools/fetch/wfs-harvest.ts +39 -28
- package/lib/tools/fetch.ts +0 -42
- package/lib/tools/golden/expand.ts +8 -8
- package/lib/tools/golden/promote.ts +12 -8
- package/lib/tools/golden/relabel-street/directory.ts +7 -6
- package/lib/tools/golden/relabel-street/row.ts +7 -6
- package/lib/tools/ingest-csv.ts +3 -3
- package/lib/tools/jsonl-to-parquet.ts +1 -1
- package/lib/tools/lint/recipe-output/vocab.ts +10 -5
- package/lib/tools/lint/recipe-output.ts +26 -8
- package/lib/tools/overlay/kryptonite.ts +4 -3
- package/lib/tools/overlay/manifest.ts +4 -2
- package/lib/tools/overlay/merge-source.ts +2 -1
- package/lib/tools/overlay/split-slice.ts +2 -2
- package/lib/tools/overlay/translit.ts +6 -4
- package/lib/tools/overture-subvenue.ts +1 -1
- package/lib/tools/postcode-triples.ts +5 -4
- package/lib/tools/source-register/build.ts +17 -14
- package/lib/tools/sub/venue/lexicon.ts +4 -8
- package/lib/tw/tools/fetch/gcis.ts +4 -4
- package/lib/types.ts +4 -3
- package/lib/us/adapters/state/hi-schools/workbook.ts +1 -1
- package/lib/us/adapters/usgov/nad/adapter.ts +11 -11
- package/lib/us/adapters/usgov/samhsa-treatment-locator/adapter.ts +4 -4
- package/lib/us/fips-state.ts +2 -2
- package/lib/{synthesizers → us/synthesizers}/intersection.ts +12 -13
- package/lib/{synthesizers → us/synthesizers}/street.ts +16 -15
- package/lib/us/tools/fetch/imls-pls.ts +1 -1
- package/lib/us/tools/fetch/nad.ts +7 -7
- package/lib/us/tools/fetch/nppes.ts +5 -5
- package/lib/us/tools/fetch/state/hi-schools.ts +3 -2
- package/lib/us/tools/fetch/tiger-full.ts +1 -1
- package/lib/utils/align.ts +13 -10
- package/lib/utils/golden.ts +2 -2
- package/lib/utils/license.ts +3 -3
- package/lib/utils/split.ts +19 -15
- package/lib/utils/tokenize.ts +7 -26
- package/lib/utils/wof-json.ts +4 -4
- package/out/adapters/cordis/adapter.d.ts +5 -3
- package/out/adapters/cordis/adapter.d.ts.map +1 -1
- package/out/adapters/cordis/adapter.js +13 -7
- package/out/adapters/cordis/adapter.js.map +1 -1
- package/out/adapters/dedup-key.d.ts +1 -1
- package/out/adapters/dedup-key.js +1 -1
- package/out/adapters/geonames/adapter.js +1 -1
- package/out/adapters/geonames/adapter.js.map +1 -1
- package/out/adapters/gleif/adapter.d.ts +9 -9
- package/out/adapters/gleif/adapter.d.ts.map +1 -1
- package/out/adapters/gleif/adapter.js +15 -14
- package/out/adapters/gleif/adapter.js.map +1 -1
- package/out/adapters/libpostal-dictionary.d.ts +0 -2
- package/out/adapters/libpostal-dictionary.d.ts.map +1 -1
- package/out/adapters/libpostal-dictionary.js +0 -2
- package/out/adapters/libpostal-dictionary.js.map +1 -1
- package/out/adapters/openaddresses/adapter.d.ts +3 -2
- package/out/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/adapters/openaddresses/adapter.js +1 -1
- package/out/adapters/openaddresses/adapter.js.map +1 -1
- package/out/adapters/osm/adapter.d.ts +17 -16
- package/out/adapters/osm/adapter.d.ts.map +1 -1
- package/out/adapters/osm/adapter.js +10 -9
- package/out/adapters/osm/adapter.js.map +1 -1
- package/out/adapters/po-box/adapter.d.ts +6 -3
- package/out/adapters/po-box/adapter.d.ts.map +1 -1
- package/out/adapters/po-box/adapter.js +4 -2
- package/out/adapters/po-box/adapter.js.map +1 -1
- package/out/adapters/registry.d.ts +3 -3
- package/out/adapters/registry.d.ts.map +1 -1
- package/out/adapters/registry.js +1 -1
- package/out/adapters/registry.js.map +1 -1
- package/out/adapters/source-id.d.ts +2 -2
- package/out/adapters/source-id.js +2 -2
- package/out/adapters/street-line.d.ts +7 -6
- package/out/adapters/street-line.d.ts.map +1 -1
- package/out/adapters/street-line.js +7 -6
- package/out/adapters/street-line.js.map +1 -1
- package/out/adapters/wof/admin/json/adapter.js +1 -1
- package/out/adapters/wof/admin/json/adapter.js.map +1 -1
- package/out/adapters/wof/json-rows.d.ts +2 -1
- package/out/adapters/wof/json-rows.d.ts.map +1 -1
- package/out/adapters/wof/json-rows.js.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.d.ts.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.js +5 -1
- package/out/adapters/wof/postalcode-json/adapter.js.map +1 -1
- package/out/adapters.d.ts +5 -5
- package/out/adapters.js +5 -5
- package/out/au/adapters/gnaf/adapter.js +1 -1
- package/out/au/adapters/gnaf/adapter.js.map +1 -1
- package/out/be/adapters/brussels/adapter.d.ts +9 -9
- package/out/be/adapters/brussels/adapter.d.ts.map +1 -1
- package/out/be/adapters/brussels/adapter.js +21 -21
- package/out/be/adapters/brussels/adapter.js.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.d.ts +34 -37
- package/out/be/adapters/vlaanderen/adapter.d.ts.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.js +40 -38
- package/out/be/adapters/vlaanderen/adapter.js.map +1 -1
- package/out/be/adapters/wallonie/adapter.d.ts +2 -2
- package/out/be/adapters/wallonie/adapter.js +17 -17
- package/out/be/adapters/wallonie/adapter.js.map +1 -1
- package/out/be/tools/fetch/brussels.d.ts +11 -8
- package/out/be/tools/fetch/brussels.d.ts.map +1 -1
- package/out/be/tools/fetch/brussels.js +10 -8
- package/out/be/tools/fetch/brussels.js.map +1 -1
- package/out/be/tools/fetch/wallonie.d.ts +6 -5
- package/out/be/tools/fetch/wallonie.d.ts.map +1 -1
- package/out/be/tools/fetch/wallonie.js +4 -4
- package/out/be/tools/fetch/wallonie.js.map +1 -1
- package/out/build/adapters.d.ts.map +1 -1
- package/out/build/adapters.js +2 -1
- package/out/build/adapters.js.map +1 -1
- package/out/build/align.d.ts +3 -2
- package/out/build/align.d.ts.map +1 -1
- package/out/build/align.js +2 -2
- package/out/build/align.js.map +1 -1
- package/out/build/checkpoint.d.ts +1 -1
- package/out/build/checkpoint.d.ts.map +1 -1
- package/out/build/checkpoint.js +4 -2
- package/out/build/checkpoint.js.map +1 -1
- package/out/build/eligibility.d.ts +2 -2
- package/out/build/eligibility.js +3 -3
- package/out/build/eligibility.js.map +1 -1
- package/out/build/types.d.ts +8 -6
- package/out/build/types.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.d.ts +14 -14
- package/out/cz/adapters/cuzk/adapter.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.js +38 -35
- package/out/cz/adapters/cuzk/adapter.js.map +1 -1
- package/out/cz/tools/fetch/cuzk.d.ts +31 -22
- package/out/cz/tools/fetch/cuzk.d.ts.map +1 -1
- package/out/cz/tools/fetch/cuzk.js +21 -19
- package/out/cz/tools/fetch/cuzk.js.map +1 -1
- package/out/de/recipes/locale.js +1 -1
- package/out/de/recipes/locale.js.map +1 -1
- package/out/dk/adapters/inspire/adapter.d.ts +32 -33
- package/out/dk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/dk/adapters/inspire/adapter.js +38 -38
- package/out/dk/adapters/inspire/adapter.js.map +1 -1
- package/out/dk/tools/fetch/inspire.d.ts +6 -5
- package/out/dk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/dk/tools/fetch/inspire.js +15 -13
- package/out/dk/tools/fetch/inspire.js.map +1 -1
- package/out/ee/adapters/ads/adapter.d.ts +19 -26
- package/out/ee/adapters/ads/adapter.d.ts.map +1 -1
- package/out/ee/adapters/ads/adapter.js +22 -28
- package/out/ee/adapters/ads/adapter.js.map +1 -1
- package/out/ee/tools/fetch/ads.d.ts +3 -3
- package/out/ee/tools/fetch/ads.d.ts.map +1 -1
- package/out/ee/tools/fetch/ads.js +10 -8
- package/out/ee/tools/fetch/ads.js.map +1 -1
- package/out/es/adapters/bizkaia/adapter.d.ts +28 -28
- package/out/es/adapters/bizkaia/adapter.d.ts.map +1 -1
- package/out/es/adapters/bizkaia/adapter.js +43 -43
- package/out/es/adapters/bizkaia/adapter.js.map +1 -1
- package/out/es/adapters/catastro/adapter.d.ts +22 -23
- package/out/es/adapters/catastro/adapter.d.ts.map +1 -1
- package/out/es/adapters/catastro/adapter.js +35 -33
- package/out/es/adapters/catastro/adapter.js.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.d.ts +31 -38
- package/out/es/adapters/gipuzkoa/adapter.d.ts.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.js +40 -47
- package/out/es/adapters/gipuzkoa/adapter.js.map +1 -1
- package/out/es/adapters/navarra/adapter.d.ts +20 -19
- package/out/es/adapters/navarra/adapter.d.ts.map +1 -1
- package/out/es/adapters/navarra/adapter.js +23 -22
- package/out/es/adapters/navarra/adapter.js.map +1 -1
- package/out/es/cadastre.d.ts +21 -20
- package/out/es/cadastre.d.ts.map +1 -1
- package/out/es/cadastre.js +19 -21
- package/out/es/cadastre.js.map +1 -1
- package/out/es/tools/fetch/bizkaia.d.ts +28 -25
- package/out/es/tools/fetch/bizkaia.d.ts.map +1 -1
- package/out/es/tools/fetch/bizkaia.js +25 -24
- package/out/es/tools/fetch/bizkaia.js.map +1 -1
- package/out/es/tools/fetch/catastro.d.ts +28 -20
- package/out/es/tools/fetch/catastro.d.ts.map +1 -1
- package/out/es/tools/fetch/catastro.js +36 -31
- package/out/es/tools/fetch/catastro.js.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.d.ts +11 -9
- package/out/es/tools/fetch/gipuzkoa.d.ts.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.js +9 -8
- package/out/es/tools/fetch/gipuzkoa.js.map +1 -1
- package/out/es/tools/fetch/navarra.d.ts +20 -15
- package/out/es/tools/fetch/navarra.d.ts.map +1 -1
- package/out/es/tools/fetch/navarra.js +21 -16
- package/out/es/tools/fetch/navarra.js.map +1 -1
- package/out/fi/adapters/ryhti/adapter.d.ts +16 -20
- package/out/fi/adapters/ryhti/adapter.d.ts.map +1 -1
- package/out/fi/adapters/ryhti/adapter.js +21 -25
- package/out/fi/adapters/ryhti/adapter.js.map +1 -1
- package/out/fi/tools/fetch/ryhti.d.ts +24 -24
- package/out/fi/tools/fetch/ryhti.d.ts.map +1 -1
- package/out/fi/tools/fetch/ryhti.js +21 -24
- package/out/fi/tools/fetch/ryhti.js.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.d.ts +8 -8
- package/out/fr/adapters/annuaire-education/adapter.d.ts.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.js +13 -11
- package/out/fr/adapters/annuaire-education/adapter.js.map +1 -1
- package/out/fr/adapters/ban/adapter.d.ts +4 -4
- package/out/fr/adapters/ban/adapter.js +6 -6
- package/out/fr/adapters/ban/adapter.js.map +1 -1
- package/out/fr/adapters/finess/adapter.d.ts +12 -11
- package/out/fr/adapters/finess/adapter.d.ts.map +1 -1
- package/out/fr/adapters/finess/adapter.js +16 -15
- package/out/fr/adapters/finess/adapter.js.map +1 -1
- package/out/fr/adapters/finess/voie-type.d.ts +2 -2
- package/out/fr/adapters/finess/voie-type.d.ts.map +1 -1
- package/out/fr/adapters/finess/voie-type.js +4 -3
- package/out/fr/adapters/finess/voie-type.js.map +1 -1
- package/out/fr/insee-country.d.ts +2 -2
- package/out/fr/insee-country.js +2 -2
- package/out/fr/recipes/lieudit.d.ts.map +1 -1
- package/out/fr/recipes/lieudit.js +5 -4
- package/out/fr/recipes/lieudit.js.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.d.ts +7 -4
- package/out/fr/tools/fetch/annuaire-education.d.ts.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.js +9 -7
- package/out/fr/tools/fetch/annuaire-education.js.map +1 -1
- package/out/fr/tools/fetch/ban.d.ts +1 -1
- package/out/fr/tools/fetch/ban.d.ts.map +1 -1
- package/out/fr/tools/fetch/ban.js +3 -2
- package/out/fr/tools/fetch/ban.js.map +1 -1
- package/out/fr/tools/fetch/finess.d.ts +5 -5
- package/out/fr/tools/fetch/finess.d.ts.map +1 -1
- package/out/fr/tools/fetch/finess.js +13 -12
- package/out/fr/tools/fetch/finess.js.map +1 -1
- package/out/inspire/address.d.ts +62 -52
- package/out/inspire/address.d.ts.map +1 -1
- package/out/inspire/address.js +71 -60
- package/out/inspire/address.js.map +1 -1
- package/out/inspire/errors.d.ts +9 -8
- package/out/inspire/errors.d.ts.map +1 -1
- package/out/inspire/errors.js +7 -6
- package/out/inspire/errors.js.map +1 -1
- package/out/inspire/stream.d.ts +25 -22
- package/out/inspire/stream.d.ts.map +1 -1
- package/out/inspire/stream.js +10 -12
- package/out/inspire/stream.js.map +1 -1
- package/out/international/recipes/locale.d.ts +2 -2
- package/out/international/recipes/locale.d.ts.map +1 -1
- package/out/international/recipes/locale.js +11 -7
- package/out/international/recipes/locale.js.map +1 -1
- package/out/it/adapters/anac/adapter.d.ts +45 -43
- package/out/it/adapters/anac/adapter.d.ts.map +1 -1
- package/out/it/adapters/anac/adapter.js +48 -43
- package/out/it/adapters/anac/adapter.js.map +1 -1
- package/out/it/tools/fetch/anac.d.ts +16 -19
- package/out/it/tools/fetch/anac.d.ts.map +1 -1
- package/out/it/tools/fetch/anac.js +20 -23
- package/out/it/tools/fetch/anac.js.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.d.ts.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.js.map +1 -1
- package/out/jp/tools/fetch/houjin.js +2 -2
- package/out/jp/tools/fetch/houjin.js.map +1 -1
- package/out/kr/adapters/localdata/permit-rows.js +1 -1
- package/out/kr/adapters/localdata/permit-rows.js.map +1 -1
- package/out/kr/tools/fetch/juso.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.js +1 -1
- package/out/kr/tools/fetch/localdata.js.map +1 -1
- package/out/lu/adapters/bd-adresses/adapter.d.ts +9 -9
- package/out/lu/adapters/bd-adresses/adapter.js +10 -10
- package/out/lu/tools/fetch/bd-adresses.d.ts +11 -9
- package/out/lu/tools/fetch/bd-adresses.d.ts.map +1 -1
- package/out/lu/tools/fetch/bd-adresses.js +11 -11
- package/out/lu/tools/fetch/bd-adresses.js.map +1 -1
- package/out/lv/adapters/varis/adapter.d.ts +24 -23
- package/out/lv/adapters/varis/adapter.d.ts.map +1 -1
- package/out/lv/adapters/varis/adapter.js +24 -23
- package/out/lv/adapters/varis/adapter.js.map +1 -1
- package/out/lv/tools/fetch/varis.d.ts +2 -1
- package/out/lv/tools/fetch/varis.d.ts.map +1 -1
- package/out/lv/tools/fetch/varis.js +1 -1
- package/out/lv/tools/fetch/varis.js.map +1 -1
- package/out/nl/adapters/kadaster/adapter.d.ts +12 -12
- package/out/nl/adapters/kadaster/adapter.d.ts.map +1 -1
- package/out/nl/adapters/kadaster/adapter.js +22 -23
- package/out/nl/adapters/kadaster/adapter.js.map +1 -1
- package/out/nl/tools/fetch/kadaster.d.ts +14 -10
- package/out/nl/tools/fetch/kadaster.d.ts.map +1 -1
- package/out/nl/tools/fetch/kadaster.js +12 -9
- package/out/nl/tools/fetch/kadaster.js.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.js +3 -3
- package/out/no/adapters/matrikkelen/adapter.js.map +1 -1
- package/out/no/tools/fetch/matrikkelen.d.ts +50 -38
- package/out/no/tools/fetch/matrikkelen.d.ts.map +1 -1
- package/out/no/tools/fetch/matrikkelen.js +44 -36
- package/out/no/tools/fetch/matrikkelen.js.map +1 -1
- package/out/parquet/duckdb.d.ts +7 -4
- package/out/parquet/duckdb.d.ts.map +1 -1
- package/out/parquet/duckdb.js +2 -1
- package/out/parquet/duckdb.js.map +1 -1
- package/out/parquet/readers.js +3 -3
- package/out/parquet/readers.js.map +1 -1
- package/out/parquet/schema.js +1 -1
- package/out/parquet/schema.js.map +1 -1
- package/out/parquet/writers.d.ts +1 -1
- package/out/parquet/writers.d.ts.map +1 -1
- package/out/parquet/writers.js +2 -1
- package/out/parquet/writers.js.map +1 -1
- package/out/pl/adapters/emuia/adapter.d.ts +8 -8
- package/out/pl/adapters/emuia/adapter.js +12 -12
- package/out/pl/adapters/emuia/adapter.js.map +1 -1
- package/out/pl/tools/fetch/emuia.d.ts +5 -4
- package/out/pl/tools/fetch/emuia.d.ts.map +1 -1
- package/out/pl/tools/fetch/emuia.js +10 -9
- package/out/pl/tools/fetch/emuia.js.map +1 -1
- package/out/recipes/bare/postcode.js +1 -1
- package/out/recipes/bare/postcode.js.map +1 -1
- package/out/recipes/boundary-stress.js +1 -1
- package/out/recipes/boundary-stress.js.map +1 -1
- package/out/recipes/house-venue.d.ts.map +1 -1
- package/out/recipes/house-venue.js +6 -3
- package/out/recipes/house-venue.js.map +1 -1
- package/out/recipes/intersection.js +2 -2
- package/out/recipes/intersection.js.map +1 -1
- package/out/recipes/po/box/cedex/recipe.js +2 -1
- package/out/recipes/po/box/cedex/recipe.js.map +1 -1
- package/out/recipes/po/box/cedex/sources.d.ts +8 -2
- package/out/recipes/po/box/cedex/sources.d.ts.map +1 -1
- package/out/recipes/po/box/cedex/sources.js +1 -1
- package/out/recipes/po/box/cedex/sources.js.map +1 -1
- package/out/recipes/scaffold.d.ts +28 -19
- package/out/recipes/scaffold.d.ts.map +1 -1
- package/out/recipes/scaffold.js +3 -2
- package/out/recipes/scaffold.js.map +1 -1
- package/out/recipes/sources.d.ts +1 -1
- package/out/recipes/sources.d.ts.map +1 -1
- package/out/recipes/sources.js +15 -2
- package/out/recipes/sources.js.map +1 -1
- package/out/recipes/street/bare.js +2 -2
- package/out/recipes/street/bare.js.map +1 -1
- package/out/recipes/street.d.ts.map +1 -1
- package/out/recipes/street.js +3 -1
- package/out/recipes/street.js.map +1 -1
- package/out/recipes/sub/venue/context.d.ts.map +1 -1
- package/out/recipes/sub/venue/context.js +2 -0
- package/out/recipes/sub/venue/context.js.map +1 -1
- package/out/recipes/sub/venue/render.js +1 -1
- package/out/recipes/sub/venue/render.js.map +1 -1
- package/out/recipes/sub/venue/sources.d.ts +6 -3
- package/out/recipes/sub/venue/sources.d.ts.map +1 -1
- package/out/recipes/sub/venue/sources.js +16 -20
- package/out/recipes/sub/venue/sources.js.map +1 -1
- package/out/recipes/sub/venue.d.ts +2 -5
- package/out/recipes/sub/venue.d.ts.map +1 -1
- package/out/recipes/sub/venue.js +10 -14
- package/out/recipes/sub/venue.js.map +1 -1
- package/out/recipes/trailing-region.d.ts.map +1 -1
- package/out/recipes/trailing-region.js +6 -1
- package/out/recipes/trailing-region.js.map +1 -1
- package/out/recipes/unit.js +2 -2
- package/out/recipes/unit.js.map +1 -1
- package/out/recipes/venue.d.ts +119 -0
- package/out/recipes/venue.d.ts.map +1 -0
- package/out/recipes/venue.js +245 -0
- package/out/recipes/venue.js.map +1 -0
- package/out/recipes.d.ts +1 -1
- package/out/recipes.d.ts.map +1 -1
- package/out/recipes.js +3 -1
- package/out/recipes.js.map +1 -1
- package/out/registers.d.ts +8 -6
- package/out/registers.d.ts.map +1 -1
- package/out/registers.js +8 -6
- package/out/registers.js.map +1 -1
- package/out/ro/adapters/retea-scolara/adapter.d.ts +5 -5
- package/out/ro/adapters/retea-scolara/adapter.js +5 -5
- package/out/ro/tools/fetch/retea-scolara.js +1 -1
- package/out/ro/tools/fetch/retea-scolara.js.map +1 -1
- package/out/runner.d.ts +3 -2
- package/out/runner.d.ts.map +1 -1
- package/out/runner.js +1 -1
- package/out/runner.js.map +1 -1
- package/out/sg/recipes/register.js +1 -1
- package/out/sg/recipes/register.js.map +1 -1
- package/out/sg/tools/fetch/acra.js +2 -2
- package/out/sg/tools/fetch/acra.js.map +1 -1
- package/out/sk/adapters/inspire/adapter.d.ts +5 -5
- package/out/sk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/sk/adapters/inspire/adapter.js +18 -18
- package/out/sk/adapters/inspire/adapter.js.map +1 -1
- package/out/sk/tools/fetch/inspire.d.ts +11 -10
- package/out/sk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/sk/tools/fetch/inspire.js +15 -14
- package/out/sk/tools/fetch/inspire.js.map +1 -1
- package/out/source-register/effective-manifest.js +3 -3
- package/out/source-register/effective-manifest.js.map +1 -1
- package/out/source-register/register.d.ts +2 -2
- package/out/source-register/register.d.ts.map +1 -1
- package/out/source-register/register.js +2 -2
- package/out/source-register/register.js.map +1 -1
- package/out/source-register/training-manifest.d.ts +1 -1
- package/out/source-register/types.d.ts +12 -13
- package/out/source-register/types.d.ts.map +1 -1
- package/out/source-register/types.js +2 -2
- package/out/source-register/types.js.map +1 -1
- package/out/subvenue/promotions.d.ts +1 -1
- package/out/subvenue/promotions.js +1 -1
- package/out/subvenue/table.d.ts +4 -4
- package/out/subvenue/table.js +3 -3
- package/out/surfaces/locale.d.ts.map +1 -1
- package/out/surfaces/locale.js +2 -0
- package/out/surfaces/locale.js.map +1 -1
- package/out/synthesizers/anchor-absorption.d.ts +2 -1
- package/out/synthesizers/anchor-absorption.d.ts.map +1 -1
- package/out/synthesizers/anchor-absorption.js.map +1 -1
- package/out/synthesizers/boundary-stress.d.ts +1 -1
- package/out/synthesizers/boundary-stress.d.ts.map +1 -1
- package/out/synthesizers/boundary-stress.js +20 -23
- package/out/synthesizers/boundary-stress.js.map +1 -1
- package/out/synthesizers/house-venue.d.ts +3 -2
- package/out/synthesizers/house-venue.d.ts.map +1 -1
- package/out/synthesizers/house-venue.js +24 -26
- package/out/synthesizers/house-venue.js.map +1 -1
- package/out/synthesizers/no-street.d.ts.map +1 -1
- package/out/synthesizers/no-street.js.map +1 -1
- package/out/synthesizers/po-box.d.ts +4 -2
- package/out/synthesizers/po-box.d.ts.map +1 -1
- package/out/synthesizers/po-box.js.map +1 -1
- package/out/synthesizers/tail.d.ts +36 -0
- package/out/synthesizers/tail.d.ts.map +1 -0
- package/out/synthesizers/tail.js +32 -0
- package/out/synthesizers/tail.js.map +1 -0
- package/out/synthesizers/utils.d.ts +2 -1
- package/out/synthesizers/utils.d.ts.map +1 -1
- package/out/synthesizers/utils.js +17 -1
- package/out/synthesizers/utils.js.map +1 -1
- package/out/test-kit.js +2 -2
- package/out/test-kit.js.map +1 -1
- package/out/tools/audit.d.ts +2 -1
- package/out/tools/audit.d.ts.map +1 -1
- package/out/tools/audit.js +4 -3
- package/out/tools/audit.js.map +1 -1
- package/out/tools/corpus-stats.js +1 -1
- package/out/tools/corpus-stats.js.map +1 -1
- package/out/tools/fetch/atom.d.ts +8 -8
- package/out/tools/fetch/atom.d.ts.map +1 -1
- package/out/tools/fetch/atom.js +7 -8
- package/out/tools/fetch/atom.js.map +1 -1
- package/out/tools/fetch/cordis.d.ts +4 -3
- package/out/tools/fetch/cordis.d.ts.map +1 -1
- package/out/tools/fetch/cordis.js +9 -7
- package/out/tools/fetch/cordis.js.map +1 -1
- package/out/tools/fetch/download/manifest.d.ts.map +1 -1
- package/out/tools/fetch/download/manifest.js +2 -1
- package/out/tools/fetch/download/manifest.js.map +1 -1
- package/out/tools/fetch/download/network.d.ts +4 -4
- package/out/tools/fetch/download/retry.js +4 -4
- package/out/tools/fetch/download/retry.js.map +1 -1
- package/out/tools/fetch/geonames/dump.d.ts +3 -2
- package/out/tools/fetch/geonames/dump.d.ts.map +1 -1
- package/out/tools/fetch/geonames/dump.js +2 -1
- package/out/tools/fetch/geonames/dump.js.map +1 -1
- package/out/tools/fetch/geonames/postal.d.ts +2 -2
- package/out/tools/fetch/gleif.d.ts +3 -2
- package/out/tools/fetch/gleif.d.ts.map +1 -1
- package/out/tools/fetch/gleif.js +6 -4
- package/out/tools/fetch/gleif.js.map +1 -1
- package/out/tools/fetch/header.d.ts +7 -5
- package/out/tools/fetch/header.d.ts.map +1 -1
- package/out/tools/fetch/header.js +8 -6
- package/out/tools/fetch/header.js.map +1 -1
- package/out/tools/fetch/inspire-addresses.d.ts +32 -27
- package/out/tools/fetch/inspire-addresses.d.ts.map +1 -1
- package/out/tools/fetch/inspire-addresses.js +25 -24
- package/out/tools/fetch/inspire-addresses.js.map +1 -1
- package/out/tools/fetch/openaddresses.js +1 -1
- package/out/tools/fetch/openaddresses.js.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.d.ts +5 -5
- package/out/tools/fetch/vlaanderen-ad.d.ts.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.js +16 -16
- package/out/tools/fetch/vlaanderen-ad.js.map +1 -1
- package/out/tools/fetch/wfs-harvest.d.ts +26 -16
- package/out/tools/fetch/wfs-harvest.d.ts.map +1 -1
- package/out/tools/fetch/wfs-harvest.js +21 -18
- package/out/tools/fetch/wfs-harvest.js.map +1 -1
- package/out/tools/fetch.d.ts +0 -41
- package/out/tools/fetch.d.ts.map +1 -1
- package/out/tools/fetch.js +0 -41
- package/out/tools/fetch.js.map +1 -1
- package/out/tools/golden/expand.d.ts +7 -7
- package/out/tools/golden/expand.js +1 -1
- package/out/tools/golden/expand.js.map +1 -1
- package/out/tools/golden/promote.d.ts +2 -2
- package/out/tools/golden/promote.d.ts.map +1 -1
- package/out/tools/golden/promote.js +9 -6
- package/out/tools/golden/promote.js.map +1 -1
- package/out/tools/golden/relabel-street/directory.d.ts +4 -4
- package/out/tools/golden/relabel-street/directory.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/directory.js +3 -2
- package/out/tools/golden/relabel-street/directory.js.map +1 -1
- package/out/tools/golden/relabel-street/row.d.ts +4 -3
- package/out/tools/golden/relabel-street/row.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/row.js +3 -3
- package/out/tools/golden/relabel-street/row.js.map +1 -1
- package/out/tools/ingest-csv.js +3 -3
- package/out/tools/ingest-csv.js.map +1 -1
- package/out/tools/jsonl-to-parquet.d.ts +1 -1
- package/out/tools/lint/recipe-output/vocab.d.ts +10 -5
- package/out/tools/lint/recipe-output/vocab.d.ts.map +1 -1
- package/out/tools/lint/recipe-output/vocab.js.map +1 -1
- package/out/tools/lint/recipe-output.d.ts +8 -8
- package/out/tools/lint/recipe-output.d.ts.map +1 -1
- package/out/tools/lint/recipe-output.js +17 -0
- package/out/tools/lint/recipe-output.js.map +1 -1
- package/out/tools/overlay/kryptonite.d.ts +2 -2
- package/out/tools/overlay/kryptonite.d.ts.map +1 -1
- package/out/tools/overlay/kryptonite.js +2 -1
- package/out/tools/overlay/kryptonite.js.map +1 -1
- package/out/tools/overlay/manifest.d.ts +2 -1
- package/out/tools/overlay/manifest.d.ts.map +1 -1
- package/out/tools/overlay/manifest.js +2 -1
- package/out/tools/overlay/manifest.js.map +1 -1
- package/out/tools/overlay/merge-source.d.ts +2 -1
- package/out/tools/overlay/merge-source.d.ts.map +1 -1
- package/out/tools/overlay/merge-source.js.map +1 -1
- package/out/tools/overlay/split-slice.d.ts +1 -1
- package/out/tools/overlay/split-slice.js +1 -1
- package/out/tools/overlay/split-slice.js.map +1 -1
- package/out/tools/overlay/translit.d.ts +4 -3
- package/out/tools/overlay/translit.d.ts.map +1 -1
- package/out/tools/overlay/translit.js +2 -1
- package/out/tools/overlay/translit.js.map +1 -1
- package/out/tools/overture-subvenue.d.ts +1 -1
- package/out/tools/overture-subvenue.d.ts.map +1 -1
- package/out/tools/postcode-triples.d.ts +1 -1
- package/out/tools/postcode-triples.d.ts.map +1 -1
- package/out/tools/postcode-triples.js +4 -3
- package/out/tools/postcode-triples.js.map +1 -1
- package/out/tools/source-register/build.d.ts +10 -8
- package/out/tools/source-register/build.d.ts.map +1 -1
- package/out/tools/source-register/build.js +10 -8
- package/out/tools/source-register/build.js.map +1 -1
- package/out/tools/sub/venue/lexicon.d.ts +2 -6
- package/out/tools/sub/venue/lexicon.d.ts.map +1 -1
- package/out/tools/sub/venue/lexicon.js +2 -6
- package/out/tools/sub/venue/lexicon.js.map +1 -1
- package/out/tw/tools/fetch/gcis.js +3 -3
- package/out/tw/tools/fetch/gcis.js.map +1 -1
- package/out/types.d.ts +4 -3
- package/out/types.d.ts.map +1 -1
- package/out/types.js.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.js.map +1 -1
- package/out/us/adapters/usgov/nad/adapter.js +6 -6
- package/out/us/adapters/usgov/nad/adapter.js.map +1 -1
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js +2 -2
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/us/fips-state.d.ts +2 -2
- package/out/us/fips-state.d.ts.map +1 -1
- package/out/us/fips-state.js.map +1 -1
- package/out/us/synthesizers/intersection.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/intersection.js +13 -9
- package/out/us/synthesizers/intersection.js.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.d.ts +3 -2
- package/out/us/synthesizers/street.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.js +12 -12
- package/out/us/synthesizers/street.js.map +1 -0
- package/out/us/tools/fetch/imls-pls.js +1 -1
- package/out/us/tools/fetch/imls-pls.js.map +1 -1
- package/out/us/tools/fetch/nad.d.ts +6 -6
- package/out/us/tools/fetch/nad.js +1 -1
- package/out/us/tools/fetch/nad.js.map +1 -1
- package/out/us/tools/fetch/nppes.js +3 -3
- package/out/us/tools/fetch/nppes.js.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts +2 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.js +1 -1
- package/out/us/tools/fetch/state/hi-schools.js.map +1 -1
- package/out/us/tools/fetch/tiger-full.js +1 -1
- package/out/us/tools/fetch/tiger-full.js.map +1 -1
- package/out/utils/align.d.ts +5 -3
- package/out/utils/align.d.ts.map +1 -1
- package/out/utils/align.js +5 -5
- package/out/utils/align.js.map +1 -1
- package/out/utils/golden.js +2 -2
- package/out/utils/golden.js.map +1 -1
- package/out/utils/license.d.ts +3 -3
- package/out/utils/license.d.ts.map +1 -1
- package/out/utils/license.js.map +1 -1
- package/out/utils/split.d.ts +1 -1
- package/out/utils/split.d.ts.map +1 -1
- package/out/utils/split.js +18 -14
- package/out/utils/split.js.map +1 -1
- package/out/utils/tokenize.d.ts +3 -20
- package/out/utils/tokenize.d.ts.map +1 -1
- package/out/utils/tokenize.js.map +1 -1
- package/out/utils/wof-json.js +3 -3
- package/out/utils/wof-json.js.map +1 -1
- package/package.json +18 -27
- package/lib/tools.ts +0 -31
- package/out/synthesizers/intersection.d.ts.map +0 -1
- package/out/synthesizers/intersection.js.map +0 -1
- package/out/synthesizers/street.d.ts.map +0 -1
- package/out/synthesizers/street.js.map +0 -1
- package/out/tools.d.ts +0 -31
- package/out/tools.d.ts.map +0 -1
- package/out/tools.js +0 -31
- package/out/tools.js.map +0 -1
- /package/out/{synthesizers → us/synthesizers}/intersection.d.ts +0 -0
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* `#no/adapters/matrikkelen/adapter` reads `kommunenummer` to decide a row's country, and
|
|
11
11
|
* municipality 2100 is Svalbard rather than the mainland. Geonorge publishes the dataset per area
|
|
12
12
|
* from one 375-entry area list, and the two entries this repository reads are `0000` `Hele landet`
|
|
13
|
-
* and `2100` `Svalbard`. The mainland extract
|
|
13
|
+
* and `2100` `Svalbard`. The mainland extract has no Svalbard row, so taking only `0000` would
|
|
14
14
|
* leave `SJ` with no rows while the adapter still claimed to cover it.
|
|
15
15
|
*
|
|
16
16
|
* Each area is a separate archive at the dataset's conventional download path, so no download
|
|
@@ -20,11 +20,11 @@
|
|
|
20
20
|
*
|
|
21
21
|
* ## Why the member is extracted
|
|
22
22
|
*
|
|
23
|
-
* The adapter opens `opts.inputPath` with `CSVSpliterator.fromAsync
|
|
24
|
-
* rather than an archive, so the member is written out beside the archive it came
|
|
25
|
-
* is
|
|
26
|
-
* archive
|
|
27
|
-
* name is kept.
|
|
23
|
+
* The adapter opens `opts.inputPath` with `CSVSpliterator.fromAsync`. That reader takes a
|
|
24
|
+
* delimited file rather than an archive, so the member is written out beside the archive it came
|
|
25
|
+
* from. The member is `matrikkelenAdresse.csv` in every area's archive, inside a directory whose
|
|
26
|
+
* name follows the archive. Each area is written under its own area-code directory, and the
|
|
27
|
+
* publisher's own file name is kept.
|
|
28
28
|
*
|
|
29
29
|
* The archive is kept rather than removed, because its sha256 is the value the address-source
|
|
30
30
|
* register records for the publication and the member's is not.
|
|
@@ -40,14 +40,14 @@
|
|
|
40
40
|
*
|
|
41
41
|
* A renamed column reaches the adapter as an empty string on every row rather than as an error, so
|
|
42
42
|
* the extract's header is read through the same `CSVSpliterator` the adapter uses and checked
|
|
43
|
-
* against the nine columns the adapter indexes by name. The file's first column name
|
|
44
|
-
* byte-order mark, and none of the nine is that column, so the check does not depend on
|
|
45
|
-
*
|
|
43
|
+
* against the nine columns the adapter indexes by name. The file's first column name has a
|
|
44
|
+
* byte-order mark, and none of the nine is that column, so the check does not depend on stripping
|
|
45
|
+
* the mark.
|
|
46
46
|
*/
|
|
47
47
|
|
|
48
48
|
import { APIClient } from "@mailwoman/core/api"
|
|
49
49
|
import { ByteFormatter } from "@mailwoman/core/fs/formatters"
|
|
50
|
-
import { tryStat } from "@mailwoman/core/fs/readers"
|
|
50
|
+
import { tryStat } from "@mailwoman/core/fs/readers/stat"
|
|
51
51
|
import { makeDirectories } from "@mailwoman/core/fs/writers"
|
|
52
52
|
import { extractZipEntry } from "@mailwoman/core/fs/zip"
|
|
53
53
|
import { sha256File } from "@mailwoman/core/hash"
|
|
@@ -63,22 +63,27 @@ import { assertHeaderColumns, readDelimitedHeader } from "#tools/fetch/header"
|
|
|
63
63
|
*/
|
|
64
64
|
export interface MatrikkelenArea {
|
|
65
65
|
/**
|
|
66
|
-
* The area code
|
|
66
|
+
* The area code.
|
|
67
|
+
* Each area is written under this directory.
|
|
67
68
|
*/
|
|
68
69
|
code: string
|
|
69
70
|
/**
|
|
70
|
-
* The area name as the download path spells it
|
|
71
|
-
*
|
|
71
|
+
* The area name as the download path spells it.
|
|
72
|
+
*
|
|
73
|
+
* That name differs from the list's `name`: the list calls area `0000` `Hele landet`,
|
|
74
|
+
* and the path calls it `Norge`.
|
|
72
75
|
*/
|
|
73
76
|
pathName: string
|
|
74
77
|
}
|
|
75
78
|
|
|
76
79
|
/**
|
|
77
|
-
* The areas a fetch takes when a caller
|
|
80
|
+
* The areas a fetch takes when a caller gives no codes.
|
|
78
81
|
*
|
|
79
|
-
* `0000` is the mainland and `2100` is Svalbard
|
|
80
|
-
* address-source register
|
|
81
|
-
*
|
|
82
|
+
* `0000` is the mainland and `2100` is Svalbard.
|
|
83
|
+
* These are the two areas the address-source register holds rows for.
|
|
84
|
+
*
|
|
85
|
+
* Svalbard is also published as `fylke` 21.
|
|
86
|
+
* That area holds the same rows as kommune 2100.
|
|
82
87
|
*/
|
|
83
88
|
export const MATRIKKELEN_AREAS: readonly MatrikkelenArea[] = [
|
|
84
89
|
{ code: "0000", pathName: "Norge" },
|
|
@@ -86,7 +91,8 @@ export const MATRIKKELEN_AREAS: readonly MatrikkelenArea[] = [
|
|
|
86
91
|
]
|
|
87
92
|
|
|
88
93
|
/**
|
|
89
|
-
* The area code of the whole-country extract
|
|
94
|
+
* The area code of the whole-country extract.
|
|
95
|
+
* That extract holds every mainland municipality.
|
|
90
96
|
*/
|
|
91
97
|
export const MATRIKKELEN_MAINLAND_AREA = "0000"
|
|
92
98
|
|
|
@@ -104,7 +110,8 @@ export const MATRIKKELEN_PROJECTION = "4258"
|
|
|
104
110
|
export const MATRIKKELEN_DOWNLOAD_ROOT = "https://nedlasting.geonorge.no/geonorge/Basisdata/MatrikkelenAdresse/CSV"
|
|
105
111
|
|
|
106
112
|
/**
|
|
107
|
-
* The dataset's metadata record
|
|
113
|
+
* The dataset's metadata record.
|
|
114
|
+
* That record states the elected license.
|
|
108
115
|
*/
|
|
109
116
|
export const MATRIKKELEN_METADATA_URL =
|
|
110
117
|
"https://kartkatalog.geonorge.no/api/getdata/f7df7a18-b30f-4745-bd64-d0863812350c"
|
|
@@ -153,7 +160,8 @@ export const MATRIKKELEN_LICENSE = "CC-BY-4.0"
|
|
|
153
160
|
const SLUG = MATRIKKELEN_ADAPTER_ID
|
|
154
161
|
|
|
155
162
|
/**
|
|
156
|
-
* The archive's name for one area
|
|
163
|
+
* The archive's name for one area.
|
|
164
|
+
* The fetch writes the archive under this name.
|
|
157
165
|
*/
|
|
158
166
|
export function matrikkelenArchiveFilename(area: MatrikkelenArea): string {
|
|
159
167
|
return `Basisdata_${area.code}_${area.pathName}_${MATRIKKELEN_PROJECTION}_MatrikkelenAdresse_CSV.zip`
|
|
@@ -171,8 +179,9 @@ export function matrikkelenArchiveURL(area: MatrikkelenArea): string {
|
|
|
171
179
|
*
|
|
172
180
|
* `last_modified` is the service's own header and the one freshness signal it offers,
|
|
173
181
|
* so a changed value is the one reason to download that area again.
|
|
174
|
-
* `bytes` and `sha256` describe the archive
|
|
175
|
-
*
|
|
182
|
+
* `bytes` and `sha256` describe the archive.
|
|
183
|
+
*
|
|
184
|
+
* The address-source register records a digest for that artifact.
|
|
176
185
|
*/
|
|
177
186
|
export interface MatrikkelenFileManifest extends SourceManifest {
|
|
178
187
|
area_code: string
|
|
@@ -183,18 +192,19 @@ export interface MatrikkelenFileManifest extends SourceManifest {
|
|
|
183
192
|
}
|
|
184
193
|
|
|
185
194
|
/**
|
|
186
|
-
* The recorded entry for one area, or `
|
|
195
|
+
* The recorded entry for one area, or `null` where the manifest holds none that can decide a skip.
|
|
196
|
+
*
|
|
197
|
+
* `loadCollectionFiles` reads the shared collection shape.
|
|
198
|
+
* That shape states the fields common to every source's manifest and omits this source's area fields.
|
|
187
199
|
*
|
|
188
|
-
* `loadCollectionFiles` reads the shared collection shape, which states what every
|
|
189
|
-
* source's manifest states and not this source's area fields.
|
|
190
200
|
* An entry written before those fields existed, or written with no stated `last_modified`,
|
|
191
201
|
* cannot answer whether the archive on disk is current, and this reports that as
|
|
192
202
|
* no recorded entry rather than as an entry that disagrees.
|
|
193
203
|
*/
|
|
194
|
-
export function recordedMatrikkelenArea(entry: SourceManifest |
|
|
195
|
-
const candidate = entry as (Partial<MatrikkelenFileManifest> & SourceManifest) |
|
|
204
|
+
export function recordedMatrikkelenArea(entry: SourceManifest | null): MatrikkelenFileManifest | null {
|
|
205
|
+
const candidate = entry as (Partial<MatrikkelenFileManifest> & SourceManifest) | null
|
|
196
206
|
|
|
197
|
-
if (!candidate) return
|
|
207
|
+
if (!candidate) return null
|
|
198
208
|
|
|
199
209
|
const complete =
|
|
200
210
|
typeof candidate.area_code === "string" &&
|
|
@@ -203,15 +213,15 @@ export function recordedMatrikkelenArea(entry: SourceManifest | undefined): Matr
|
|
|
203
213
|
typeof candidate.member_bytes === "number" &&
|
|
204
214
|
typeof candidate.member_sha256 === "string"
|
|
205
215
|
|
|
206
|
-
return complete ? (candidate as MatrikkelenFileManifest) :
|
|
216
|
+
return complete ? (candidate as MatrikkelenFileManifest) : null
|
|
207
217
|
}
|
|
208
218
|
|
|
209
219
|
/**
|
|
210
220
|
* What the service's HEAD response states about one area's archive.
|
|
211
221
|
*
|
|
212
222
|
* Both fields read `null` where the header is absent, rather than an empty string or zero.
|
|
213
|
-
* An absent `last-modified` is the service declining to state a version
|
|
214
|
-
*
|
|
223
|
+
* An absent `last-modified` is the service declining to state a version.
|
|
224
|
+
* That differs from a version that matches the one on disk.
|
|
215
225
|
*/
|
|
216
226
|
export interface MatrikkelenPublication {
|
|
217
227
|
lastModified: string | null
|
|
@@ -221,8 +231,8 @@ export interface MatrikkelenPublication {
|
|
|
221
231
|
/**
|
|
222
232
|
* Reads the service's HEAD response for one area.
|
|
223
233
|
*
|
|
224
|
-
* Separate from {@linkcode downloadMatrikkelen}
|
|
225
|
-
*
|
|
234
|
+
* Separate from {@linkcode downloadMatrikkelen}: this request supplies the whole freshness decision.
|
|
235
|
+
* The download itself runs on global `fetch`, so a unit test cannot intercept it.
|
|
226
236
|
*/
|
|
227
237
|
export async function readMatrikkelenPublication(
|
|
228
238
|
client: Pick<APIClient, "fetch">,
|
|
@@ -248,8 +258,9 @@ export async function readMatrikkelenPublication(
|
|
|
248
258
|
* Whether one area's archive and extract on disk are the ones the service currently serves.
|
|
249
259
|
*
|
|
250
260
|
* A skip requires the service to state a `last-modified` value.
|
|
251
|
-
* Where it states none, both sides read `null` and an equality test would hold
|
|
252
|
-
*
|
|
261
|
+
* Where it states none, both sides read `null` and an equality test would hold.
|
|
262
|
+
*
|
|
263
|
+
* That would keep an archive of unknown age for as long as the service stayed silent.
|
|
253
264
|
*/
|
|
254
265
|
export function matrikkelenPublicationIsRecorded(
|
|
255
266
|
recorded: MatrikkelenFileManifest,
|
|
@@ -257,7 +268,7 @@ export function matrikkelenPublicationIsRecorded(
|
|
|
257
268
|
memberBytesOnDisk: number,
|
|
258
269
|
publication: MatrikkelenPublication
|
|
259
270
|
): boolean {
|
|
260
|
-
if (publication.lastModified
|
|
271
|
+
if (!publication.lastModified) return false
|
|
261
272
|
|
|
262
273
|
return (
|
|
263
274
|
recorded.last_modified === publication.lastModified &&
|
|
@@ -267,12 +278,12 @@ export function matrikkelenPublicationIsRecorded(
|
|
|
267
278
|
}
|
|
268
279
|
|
|
269
280
|
/**
|
|
270
|
-
* The areas
|
|
281
|
+
* The areas for the given codes, or every area in {@linkcode MATRIKKELEN_AREAS}.
|
|
271
282
|
*
|
|
272
|
-
* @throws
|
|
273
|
-
* reports itself rather than reading as a fetch of no areas.
|
|
283
|
+
* @throws When the area list holds no matching area, the error states the unmatched codes.
|
|
284
|
+
* A typed code then reports itself rather than reading as a fetch of no areas.
|
|
274
285
|
*/
|
|
275
|
-
export function matrikkelenAreasFor(codes: readonly string[] |
|
|
286
|
+
export function matrikkelenAreasFor(codes: readonly string[] | null): readonly MatrikkelenArea[] {
|
|
276
287
|
if (!codes?.length) return MATRIKKELEN_AREAS
|
|
277
288
|
|
|
278
289
|
const wanted = codes.map((code) => code.trim()).filter((code) => code.length > 0)
|
|
@@ -309,14 +320,15 @@ export interface DownloadMatrikkelenOptions {
|
|
|
309
320
|
/**
|
|
310
321
|
* Downloads one area's archive, extracts the member the adapter reads and checks its header.
|
|
311
322
|
*
|
|
312
|
-
* @returns The manifest entry for the area
|
|
323
|
+
* @returns The manifest entry for the area.
|
|
324
|
+
* The collection manifest holds it.
|
|
313
325
|
*/
|
|
314
326
|
export async function downloadMatrikkelenArea(
|
|
315
327
|
client: Pick<APIClient, "fetch">,
|
|
316
328
|
area: MatrikkelenArea,
|
|
317
329
|
options: {
|
|
318
330
|
outputDir: PathBuilderLike
|
|
319
|
-
recorded?: MatrikkelenFileManifest
|
|
331
|
+
recorded?: MatrikkelenFileManifest | null
|
|
320
332
|
force?: boolean
|
|
321
333
|
retries?: number
|
|
322
334
|
retryDelayMs?: number
|
|
@@ -368,8 +380,8 @@ export async function downloadMatrikkelenArea(
|
|
|
368
380
|
|
|
369
381
|
report?.(` ✓ ${ByteFormatter.formatIEC(bytes)} sha256=${sha256}`)
|
|
370
382
|
|
|
371
|
-
// The member sits inside a directory
|
|
372
|
-
// the name's tail rather than the whole archive-internal path.
|
|
383
|
+
// The member sits inside a directory whose name follows the archive, so the selector
|
|
384
|
+
// matches the name's tail rather than the whole archive-internal path.
|
|
373
385
|
const memberBytes = await extractZipEntry(archivePath, /(?:^|\/)matrikkelenAdresse\.csv$/u, memberPath)
|
|
374
386
|
|
|
375
387
|
const header = await readDelimitedHeader(memberPath, MATRIKKELEN_DELIMITER)
|
|
@@ -428,7 +440,7 @@ export async function downloadMatrikkelen(
|
|
|
428
440
|
try {
|
|
429
441
|
const { entry, downloaded } = await downloadMatrikkelenArea(client, area, {
|
|
430
442
|
outputDir: destDir,
|
|
431
|
-
recorded: recordedMatrikkelenArea(entries.get(matrikkelenArchiveFilename(area))),
|
|
443
|
+
recorded: recordedMatrikkelenArea(entries.get(matrikkelenArchiveFilename(area)) ?? null),
|
|
432
444
|
force: options.force,
|
|
433
445
|
retries: options.retries,
|
|
434
446
|
retryDelayMs: options.retryDelayMs,
|
|
@@ -471,7 +483,7 @@ export async function downloadMatrikkelen(
|
|
|
471
483
|
* The path `#no/adapters/matrikkelen/adapter` reads for one area, given the root a fetch wrote under.
|
|
472
484
|
*
|
|
473
485
|
* The area is a parameter because the adapter covers two jurisdictions and reads one
|
|
474
|
-
* file at a time: `0000`
|
|
486
|
+
* file at a time: `0000` holds Norway's rows and `2100` holds Svalbard's.
|
|
475
487
|
*/
|
|
476
488
|
export function matrikkelenInputPath(
|
|
477
489
|
outRoot: BaseFetchOptions["outRoot"],
|
|
@@ -501,7 +513,7 @@ export async function fetchMatrikkelen(
|
|
|
501
513
|
options: FetchMatrikkelenOptions,
|
|
502
514
|
report?: (line: string) => void
|
|
503
515
|
): Promise<FetchSummary> {
|
|
504
|
-
const areas = matrikkelenAreasFor(options.areas)
|
|
516
|
+
const areas = matrikkelenAreasFor(options.areas ?? null)
|
|
505
517
|
|
|
506
518
|
await using client = new APIClient({ displayName: SLUG, retry: true })
|
|
507
519
|
|
package/lib/parquet/duckdb.ts
CHANGED
|
@@ -31,7 +31,6 @@ export type DisposableDuckDB = import("@duckdb/node-api").DuckDBConnection & Dis
|
|
|
31
31
|
* A `COUNT(DISTINCT …)` over `v0.6.0-register-surface`'s 697,675,170 train rows did
|
|
32
32
|
* that twice on 2026-09-28, with a second corpus tool running beside it.
|
|
33
33
|
*
|
|
34
|
-
* The default is a quarter because it has to bound the host rather than one query.
|
|
35
34
|
* Two instances at half the host exceed it together.
|
|
36
35
|
*
|
|
37
36
|
* That case took the machine down.
|
|
@@ -43,6 +42,8 @@ export type DisposableDuckDB = import("@duckdb/node-api").DuckDBConnection & Dis
|
|
|
43
42
|
* That is the failure to prefer.
|
|
44
43
|
*
|
|
45
44
|
* A caller that knows it holds the host by itself passes a larger `memoryLimitBytes`.
|
|
45
|
+
*
|
|
46
|
+
* @defaultValue a quarter because it has to bound the host rather than one query.
|
|
46
47
|
*/
|
|
47
48
|
export const DUCKDB_MEMORY_SHARE = 0.25
|
|
48
49
|
|
|
@@ -52,26 +53,28 @@ export const DUCKDB_MEMORY_SHARE = 0.25
|
|
|
52
53
|
export interface DuckDBLimits {
|
|
53
54
|
/**
|
|
54
55
|
* Bytes one instance may hold before it spills.
|
|
55
|
-
*
|
|
56
|
+
* @defaultValue {@linkcode DUCKDB_MEMORY_SHARE} of host memory.
|
|
56
57
|
*/
|
|
57
58
|
memoryLimitBytes?: number
|
|
58
59
|
|
|
59
60
|
/**
|
|
60
61
|
* Worker threads.
|
|
61
|
-
* Defaults to DuckDB's own choice.
|
|
62
62
|
*
|
|
63
63
|
* A build phase that runs DuckDB beside other work sets this, because each thread
|
|
64
64
|
* holds its own share of the memory limit and the default takes every core.
|
|
65
|
+
*
|
|
66
|
+
* @defaultValue DuckDB's own choice.
|
|
65
67
|
*/
|
|
66
68
|
threads?: number
|
|
67
69
|
|
|
68
70
|
/**
|
|
69
71
|
* Where a spill is written.
|
|
70
|
-
* Defaults to the data root's `tmp/duckdb`.
|
|
71
72
|
*
|
|
72
73
|
* The default sits under the data root rather than in the system temporary directory,
|
|
73
74
|
* because a spill from a corpus-sized query is tens of gigabytes and `/tmp` is commonly a
|
|
74
75
|
* memory-backed filesystem, where spilling to it consumes the memory the spill exists to release.
|
|
76
|
+
*
|
|
77
|
+
* @defaultValue the data root's `tmp/duckdb`.
|
|
75
78
|
*/
|
|
76
79
|
temporaryDirectory?: string
|
|
77
80
|
}
|
package/lib/parquet/readers.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* Both throw for an unreadable file, so a caller cannot mistake a read failure for an empty file.
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
import { pathExists } from "@mailwoman/core/fs/readers"
|
|
11
|
+
import { pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
12
12
|
import type { PathBuilderLike } from "path-ts"
|
|
13
13
|
|
|
14
14
|
import { escapeSQLString, openDuckDB } from "#parquet/duckdb"
|
|
@@ -26,7 +26,7 @@ export async function readParquetRows<T>(path: PathBuilderLike, options: Parquet
|
|
|
26
26
|
throw new Error(`No parquet file at ${path}`)
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
return
|
|
29
|
+
return Array.fromAsync(openParquetRowStream<T>(path, options))
|
|
30
30
|
}
|
|
31
31
|
|
|
32
32
|
/**
|
|
@@ -40,7 +40,7 @@ export async function tryReadParquetRows<T>(
|
|
|
40
40
|
): Promise<T[] | null> {
|
|
41
41
|
if (!(await pathExists(path))) return null
|
|
42
42
|
|
|
43
|
-
return
|
|
43
|
+
return Array.fromAsync(openParquetRowStream<T>(path, options))
|
|
44
44
|
}
|
|
45
45
|
|
|
46
46
|
/**
|
package/lib/parquet/schema.ts
CHANGED
|
@@ -163,7 +163,7 @@ export const LABELED_ROW_SCHEMA: ParquetSchemaDefinition<ParquetRow> = {
|
|
|
163
163
|
export function rowToParquet(row: LabeledRow): ParquetRow {
|
|
164
164
|
const { span_starts, span_ends, span_tags } = row
|
|
165
165
|
|
|
166
|
-
if (span_starts
|
|
166
|
+
if (!span_starts || !span_ends || !span_tags) {
|
|
167
167
|
throw new Error(
|
|
168
168
|
`rowToParquet: row is missing the char-offset span triple (#519) — ` +
|
|
169
169
|
`span_starts=${span_starts !== undefined} span_ends=${span_ends !== undefined} span_tags=${span_tags !== undefined} ` +
|
package/lib/parquet/writers.ts
CHANGED
|
@@ -31,7 +31,8 @@
|
|
|
31
31
|
* the head of the next. A reader must therefore take a file's sources from all of its rows.
|
|
32
32
|
*/
|
|
33
33
|
|
|
34
|
-
import {
|
|
34
|
+
import { readLocalJSONFile } from "@mailwoman/core/fs/readers"
|
|
35
|
+
import { pathExists, tryStat } from "@mailwoman/core/fs/readers/stat"
|
|
35
36
|
import { openWriteStream, type WriteStream } from "@mailwoman/core/fs/streams"
|
|
36
37
|
import { temporaryDirectory } from "@mailwoman/core/fs/temporary"
|
|
37
38
|
import { makeDirectories, writeLocalBuffer, writeLocalJSONFile } from "@mailwoman/core/fs/writers"
|
|
@@ -200,7 +201,7 @@ export interface WriteParquetSplitsOptions {
|
|
|
200
201
|
/**
|
|
201
202
|
* Max rows per `.parquet` file.
|
|
202
203
|
*
|
|
203
|
-
*
|
|
204
|
+
* @defaultValue `1,000,000`
|
|
204
205
|
*/
|
|
205
206
|
rowsPerFile?: number
|
|
206
207
|
|
|
@@ -15,25 +15,25 @@
|
|
|
15
15
|
* The payload is the Polish register's own schema rather than the harmonized INSPIRE model: a member's
|
|
16
16
|
* element names are the register's columns and every address value is inline, so there is no
|
|
17
17
|
* `ad:ThoroughfareName` to join to and no `xlink:href` to resolve. Read with that same function over
|
|
18
|
-
* 500 members across ten windows spread through the service, a member
|
|
19
|
-
* twenty-second
|
|
18
|
+
* 500 members across ten windows spread through the service, a member holds 21 children and never a
|
|
19
|
+
* twenty-second. The children are `gml:boundedBy`, `ms:msGeometry` and these nineteen columns: `ms:id`,
|
|
20
20
|
* `ms:gml_id`, `ms:lokalnyid`, `ms:jednostkaadmnistracyjna`, `ms:miejscowosc`,
|
|
21
21
|
* `ms:miejscowosc_lower`, `ms:czescmiejscowosci`, `ms:ulica`, `ms:ulica_pol`, `ms:ulica_skr`,
|
|
22
22
|
* `ms:numer`, `ms:kod`, `ms:teryt`, `ms:simc`, `ms:ulic`, `ms:status`, `ms:data_od`, `ms:wazny_od`
|
|
23
23
|
* and `ms:adres`. The adapter reads `ulica` as the street, `numer` as the house number, `kod` as the
|
|
24
24
|
* postcode and `miejscowosc` as the locality, with `czescmiejscowosci` as the part of the locality
|
|
25
|
-
* where the publisher populates it. Every column is written on every member, and three
|
|
25
|
+
* where the publisher populates it. Every column is written on every member, and three hold a value
|
|
26
26
|
* on every one of the 500: `numer`, `kod` and `miejscowosc`. Two are written and populated on none:
|
|
27
27
|
* `status` and `czescmiejscowosci`.
|
|
28
28
|
*
|
|
29
29
|
* A row with no street is the ordinary rural case rather than a defect. Over those 500 members 299
|
|
30
|
-
*
|
|
30
|
+
* hold a street and 201 leave `ms:ulica` written and empty, and the publisher's own `ms:adres` writes
|
|
31
31
|
* the streetless form as locality and number: `Księżyki 5`, `Jawor 19`. The adapter emits those rows
|
|
32
|
-
* with a locality and a number and no street
|
|
32
|
+
* with a locality and a number and no street. Poland delivers to that form.
|
|
33
33
|
*
|
|
34
|
-
* `ms:jednostkaadmnistracyjna`
|
|
34
|
+
* `ms:jednostkaadmnistracyjna` holds the administrative path as a PostgreSQL array literal,
|
|
35
35
|
* `{Polska,łódzkie,"Piotrków Trybunalski","Piotrków Trybunalski"}`, whose second element is the
|
|
36
|
-
* voivodeship. It is left unread: a Polish postal address does not
|
|
36
|
+
* voivodeship. It is left unread: a Polish postal address does not include the voivodeship, the codex
|
|
37
37
|
* layout for `PL` leaves a `region` component unplaced, and parsing a quoted array literal to reach a
|
|
38
38
|
* value the layout leaves unplaced would cost a parse and reach no rendered component.
|
|
39
39
|
*
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
* GUGiK's dataset record states one condition of use, in `otherConstraints`: `Brak warunków dostępu i
|
|
47
47
|
* użytkowania`, no conditions for access and use. The address-source register elected that and names
|
|
48
48
|
* no license instrument, so no SPDX identifier exists and the publisher's own words are the label the
|
|
49
|
-
* adapter stamps on every row.
|
|
49
|
+
* adapter stamps on every row. GUGiK credit remains good practice and the statement requires it
|
|
50
50
|
* nowhere.
|
|
51
51
|
*
|
|
52
52
|
* The adapter honors `opts.limit` and `opts.signal`. `opts.country` is optional and accepts only `PL`.
|
|
@@ -99,8 +99,8 @@ const ADDRESS_ELEMENT = "ms:AD.Address"
|
|
|
99
99
|
*
|
|
100
100
|
* `textAtPath` answers an empty string for an element the publisher wrote
|
|
101
101
|
* and left blank and `undefined` for one it omitted.
|
|
102
|
-
* Both are an absent value for this adapter's purposes, and all 500 members measured
|
|
103
|
-
* every one of the nineteen columns, so the distinction
|
|
102
|
+
* Both are an absent value for this adapter's purposes, and all 500 members measured held
|
|
103
|
+
* every one of the nineteen columns, so the distinction has no effect on this service.
|
|
104
104
|
*/
|
|
105
105
|
function column(feature: MarkupElement, name: string): string {
|
|
106
106
|
return (textAtPath(feature, `ms:${name}`) ?? "").trim()
|
|
@@ -170,8 +170,8 @@ export function createEMUiAAdapter(): CorpusAdapter {
|
|
|
170
170
|
|
|
171
171
|
const { raw, components: aligned } = rendered
|
|
172
172
|
|
|
173
|
-
// `ms:gml_id
|
|
174
|
-
//
|
|
173
|
+
// Keep `ms:gml_id`: it gives the upstream municipal register and its record.
|
|
174
|
+
// `ms:id` is this service's own row number.
|
|
175
175
|
const seed = column(feature, "gml_id") || column(feature, "id")
|
|
176
176
|
|
|
177
177
|
const sourceID = seed ? `${EMUIA_ADAPTER_ID}-${seed}` : stableSourceID(EMUIA_ADAPTER_ID, aligned)
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* Główny Urząd Geodezji i Kartografii publishes the Państwowy Rejestr Granic through MapServer at
|
|
10
10
|
* {@linkcode PL_EMUIA_WFS}. Its dataset record states one condition of use in `otherConstraints`,
|
|
11
11
|
* `Brak warunków dostępu i użytkowania` — no conditions for access and use — which the
|
|
12
|
-
* address-source register elected. No license instrument is
|
|
12
|
+
* address-source register elected. No license instrument is recorded, so the publisher's own words are
|
|
13
13
|
* the label, and that statement grants the reproduction a paged harvest performs.
|
|
14
14
|
*
|
|
15
15
|
* `#tools/fetch/wfs-harvest` owns the page read, the appending, the manifest and the resumption.
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
* harvest ends on the first page that returns no features. The size is read from the service when
|
|
36
36
|
* a caller asks for it, by `countWFSFeaturesByPaging`, and no count is written into this file.
|
|
37
37
|
* 5. The service honors a sort. `ImplementsSorting` is `TRUE`, `sortBy=ms:id` answers HTTP 200 and
|
|
38
|
-
* `sortBy=ms:id D` returns the register's last rows first
|
|
38
|
+
* `sortBy=ms:id D` returns the register's last rows first. That proves the parameter is applied.
|
|
39
39
|
* `PagingIsTransactionSafe` is `FALSE`, which is the service stating that a paged sequence may
|
|
40
40
|
* shift under edits made while it is being read. The sort bounds that shift to the rows whose
|
|
41
41
|
* `ms:id` changed rather than leaving the whole sequence unordered.
|
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
* fails reports the first gap rather than the extent. The figure is therefore reported with the date
|
|
46
46
|
* it was taken, never stored as a constant: the register recorded 8,625,921 on 2026-10-01, and on
|
|
47
47
|
* 2026-10-02 `startIndex=8626950` returned one feature while `startIndex=8626951` returned none, so
|
|
48
|
-
* the service then held 8,626,951.
|
|
48
|
+
* the service then held 8,626,951. Such growth is ordinary
|
|
49
49
|
* business for a municipal address register.
|
|
50
50
|
*
|
|
51
51
|
* At 1,000 features per page and 1,607,180 bytes per page, a full harvest at that size is 8,627
|
|
@@ -88,7 +88,8 @@ export const PL_EMUIA_WFS = "https://mapy.geoportal.gov.pl/wss/service/INSPIRE/A
|
|
|
88
88
|
export const PL_EMUIA_TYPE_NAME = "ms:AD.Address"
|
|
89
89
|
|
|
90
90
|
/**
|
|
91
|
-
* The output format the harvest asks for
|
|
91
|
+
* The output format the harvest asks for.
|
|
92
|
+
* The service advertises it first.
|
|
92
93
|
*/
|
|
93
94
|
export const PL_EMUIA_OUTPUT_FORMAT = "application/gml+xml; version=3.2"
|
|
94
95
|
|
|
@@ -268,7 +269,7 @@ export async function harvestEMUiAPL(
|
|
|
268
269
|
|
|
269
270
|
options.report?.(` Feature count: ${featureCount.count ?? "not stated"} — ${featureCount.because}`)
|
|
270
271
|
|
|
271
|
-
return
|
|
272
|
+
return harvestPagedWFS({
|
|
272
273
|
context: CONTEXT,
|
|
273
274
|
source: SLUG,
|
|
274
275
|
outputDir: options.outputDir,
|
|
@@ -280,8 +281,8 @@ export async function harvestEMUiAPL(
|
|
|
280
281
|
sortBy: PL_EMUIA_SORT_BY,
|
|
281
282
|
},
|
|
282
283
|
license: LICENSE,
|
|
283
|
-
// The elected statement
|
|
284
|
-
//
|
|
284
|
+
// The elected statement states no attribution clause.
|
|
285
|
+
// GUGiK credit stays good practice.
|
|
285
286
|
attribution: "",
|
|
286
287
|
featureCount,
|
|
287
288
|
pageSize: Math.min(options.pageSize ?? PL_EMUIA_PAGE_SIZE, PL_EMUIA_PAGE_SIZE),
|
|
@@ -335,7 +336,7 @@ export async function fetchEMUiAPL(
|
|
|
335
336
|
// and the manifest records what was asked for rather than what arrived.
|
|
336
337
|
const pageSize = Math.min(options.pageSize ?? PL_EMUIA_PAGE_SIZE, PL_EMUIA_PAGE_SIZE)
|
|
337
338
|
|
|
338
|
-
return
|
|
339
|
+
return runWFSHarvest(
|
|
339
340
|
{
|
|
340
341
|
slug: SLUG,
|
|
341
342
|
outRoot: options.outRoot,
|
|
@@ -346,7 +347,7 @@ export async function fetchEMUiAPL(
|
|
|
346
347
|
client: options.client,
|
|
347
348
|
maxPages: options.maxPages,
|
|
348
349
|
harvest: async (client) =>
|
|
349
|
-
|
|
350
|
+
harvestEMUiAPL(client, {
|
|
350
351
|
outputDir: options.outRoot(SLUG),
|
|
351
352
|
pageSize,
|
|
352
353
|
maxPages: options.maxPages,
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
|
|
11
11
|
import { isNLPostcodeKey } from "@mailwoman/codex/nl"
|
|
12
12
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
13
|
-
import { pathExists } from "@mailwoman/core/fs/readers"
|
|
13
|
+
import { pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
14
14
|
import { SeededRandom } from "@mailwoman/core/utils"
|
|
15
15
|
import { computeQueryShape } from "@mailwoman/query-shape"
|
|
16
16
|
import { isPostcodeFormat } from "@mailwoman/query-shape/known-formats"
|
|
@@ -73,7 +73,7 @@ export const boundaryStressRecipe: CorpusRecipe = {
|
|
|
73
73
|
let skipped = 0
|
|
74
74
|
|
|
75
75
|
for (let i = 0; i < count; i++) {
|
|
76
|
-
const row = synthesizeBoundaryStressRow(
|
|
76
|
+
const row = synthesizeBoundaryStressRow(null, { random, forceTemplate: pickTemplate(random) })
|
|
77
77
|
const country = row.locale.split("-")[1] ?? "US"
|
|
78
78
|
// One resolution for both, because `source_id` includes the source as its prefix
|
|
79
79
|
// and a pair that disagreed would name a source no row of this output declares.
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
* it replaced.
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
+
import { layoutForCountry, layoutPrintsTag } from "@mailwoman/codex/address/layouts"
|
|
13
14
|
import { makeLcg } from "@mailwoman/core/utils"
|
|
14
15
|
|
|
15
16
|
import {
|
|
@@ -58,9 +59,12 @@ export const houseVenueRecipe: CorpusRecipe = {
|
|
|
58
59
|
for await (const tuple of readTuples(opts.input)) {
|
|
59
60
|
read++
|
|
60
61
|
|
|
61
|
-
//
|
|
62
|
-
//
|
|
63
|
-
|
|
62
|
+
// A region is required where the country's layout prints one.
|
|
63
|
+
// The FR layout prints none, so an FR tuple without a region is whole.
|
|
64
|
+
const layout = tuple.country ? layoutForCountry(tuple.country) : null
|
|
65
|
+
const regionRequired = layout ? layoutPrintsTag(layout, "region") : true
|
|
66
|
+
|
|
67
|
+
if (!tuple.locality || !tuple.postcode || !tuple.country || (!tuple.region && regionRequired)) {
|
|
64
68
|
skipped++
|
|
65
69
|
|
|
66
70
|
continue
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
import type { DuckDBConnection } from "@duckdb/node-api"
|
|
36
36
|
import type { ComponentTag } from "@mailwoman/codex/component"
|
|
37
37
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
38
|
-
import { pathExists } from "@mailwoman/core/fs/readers"
|
|
38
|
+
import { pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
39
39
|
import { writeLocalJSONFile } from "@mailwoman/core/fs/writers"
|
|
40
40
|
import { stringifyJSON } from "@mailwoman/core/json"
|
|
41
41
|
import { repoRootPath } from "@mailwoman/core/paths"
|
|
@@ -396,7 +396,7 @@ function auditRow(row: LabeledRow, components: Partial<Record<ComponentTag, stri
|
|
|
396
396
|
}
|
|
397
397
|
|
|
398
398
|
for (const [tag, value] of Object.entries(components)) {
|
|
399
|
-
if (value
|
|
399
|
+
if (!value) continue
|
|
400
400
|
const indices = span_tags.map((t, i) => (t === tag ? i : -1)).filter((i) => i >= 0)
|
|
401
401
|
|
|
402
402
|
if (indices.length !== 1) {
|
|
@@ -23,7 +23,7 @@ import {
|
|
|
23
23
|
US_EVAL_SOURCE,
|
|
24
24
|
US_TRAIN_SOURCES,
|
|
25
25
|
} from "#recipes/po/box/cedex/sources"
|
|
26
|
-
import type {
|
|
26
|
+
import type { Rendered, USTuple } from "#recipes/po/box/cedex/types"
|
|
27
27
|
import { renderPmbUs, renderPoBoxUs } from "#recipes/po/box/cedex/us"
|
|
28
28
|
import { CLASS_MIX } from "#recipes/po/box/cedex/vocabulary"
|
|
29
29
|
import { recipeSourceID, type CanonicalRecipeRow, type CorpusRecipe } from "#recipes/scaffold"
|
|
@@ -86,8 +86,8 @@ export const poBoxCedexRecipe: CorpusRecipe = {
|
|
|
86
86
|
|
|
87
87
|
console.error(` GeoNames CA: QC ${qcAll.length}→${qcPool.length}, ON ${onAll.length}→${onPool.length}`)
|
|
88
88
|
|
|
89
|
-
const auAll =
|
|
90
|
-
|
|
89
|
+
const auAll = await readPostalTuples(GEONAMES_POSTAL_AU, { withState: true })
|
|
90
|
+
const nzAll = await readPostalTuples(GEONAMES_POSTAL_NZ, { withState: false })
|
|
91
91
|
|
|
92
92
|
const auPool = auAll.filter((tuple) => isHoldoutLocality(tuple.locality) === opts.golden),
|
|
93
93
|
nzPool = nzAll.filter((tuple) => isHoldoutLocality(tuple.locality) === opts.golden)
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
import { isAuPostcode, isAuStateAbbreviation } from "@mailwoman/codex/au"
|
|
6
6
|
import { isNZPostcode } from "@mailwoman/codex/nz"
|
|
7
7
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
8
|
-
import { pathExists } from "@mailwoman/core/fs/readers"
|
|
8
|
+
import { pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
9
9
|
import { readZipEntry } from "@mailwoman/core/fs/zip"
|
|
10
10
|
import type { PathBuilderLike } from "path-ts"
|
|
11
11
|
import { TextSpliterator, TSVSpliterator } from "spliterator"
|
|
@@ -156,6 +156,16 @@ export async function readCaLocalities(admin1: string): Promise<string[]> {
|
|
|
156
156
|
return [...localities]
|
|
157
157
|
}
|
|
158
158
|
|
|
159
|
+
export function readPostalTuples(
|
|
160
|
+
source: { zip: PathBuilderLike; txt: string },
|
|
161
|
+
opts: { withState: true }
|
|
162
|
+
): Promise<AUTuple[]>
|
|
163
|
+
|
|
164
|
+
export function readPostalTuples(
|
|
165
|
+
source: { zip: PathBuilderLike; txt: string },
|
|
166
|
+
opts: { withState: false }
|
|
167
|
+
): Promise<NZTuple[]>
|
|
168
|
+
|
|
159
169
|
export async function readPostalTuples(
|
|
160
170
|
source: { zip: PathBuilderLike; txt: string },
|
|
161
171
|
opts: { withState: boolean }
|
|
@@ -181,7 +191,9 @@ export async function readPostalTuples(
|
|
|
181
191
|
const key = `${locality}|${postcode}`.toLowerCase()
|
|
182
192
|
|
|
183
193
|
if (seen.has(key)) continue
|
|
194
|
+
|
|
184
195
|
seen.add(key)
|
|
196
|
+
|
|
185
197
|
tuples.push(opts.withState ? { locality, region, postcode } : { locality, postcode })
|
|
186
198
|
}
|
|
187
199
|
|