@mailwoman/corpus 10.1.0 → 10.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/licenses/cdla-permissive-2.0.md +1 -1
- package/lib/adapters/cordis/adapter.ts +13 -7
- package/lib/adapters/dedup-key.ts +1 -1
- package/lib/adapters/geonames/adapter.ts +1 -1
- package/lib/adapters/gleif/adapter.ts +17 -16
- package/lib/adapters/libpostal-dictionary.ts +0 -2
- package/lib/adapters/openaddresses/adapter.ts +4 -3
- package/lib/adapters/osm/adapter.ts +22 -21
- package/lib/adapters/po-box/adapter.ts +10 -5
- package/lib/adapters/registry.ts +4 -4
- package/lib/adapters/source-id.ts +2 -2
- package/lib/adapters/street-line.ts +7 -6
- package/lib/adapters/wof/admin/json/adapter.ts +2 -2
- package/lib/adapters/wof/json-rows.ts +2 -1
- package/lib/adapters/wof/postalcode-json/adapter.ts +6 -2
- package/lib/adapters.ts +5 -5
- package/lib/au/adapters/gnaf/adapter.ts +1 -1
- package/lib/be/adapters/brussels/adapter.ts +22 -22
- package/lib/be/adapters/vlaanderen/adapter.ts +54 -57
- package/lib/be/adapters/wallonie/adapter.ts +17 -17
- package/lib/be/tools/fetch/brussels.ts +13 -10
- package/lib/be/tools/fetch/wallonie.ts +8 -7
- package/lib/build/adapters.ts +2 -1
- package/lib/build/align.ts +5 -4
- package/lib/build/checkpoint.ts +5 -3
- package/lib/build/eligibility.ts +3 -3
- package/lib/build/types.ts +8 -6
- package/lib/cz/adapters/cuzk/adapter.ts +41 -38
- package/lib/cz/tools/fetch/cuzk.ts +42 -31
- package/lib/de/recipes/locale.ts +1 -1
- package/lib/dk/adapters/inspire/adapter.ts +38 -38
- package/lib/dk/tools/fetch/inspire.ts +16 -14
- package/lib/ee/adapters/ads/adapter.ts +26 -32
- package/lib/ee/tools/fetch/ads.ts +10 -8
- package/lib/es/adapters/bizkaia/adapter.ts +47 -47
- package/lib/es/adapters/catastro/adapter.ts +36 -34
- package/lib/es/adapters/gipuzkoa/adapter.ts +38 -45
- package/lib/es/adapters/navarra/adapter.ts +28 -27
- package/lib/es/cadastre.ts +31 -31
- package/lib/es/tools/fetch/bizkaia.ts +40 -36
- package/lib/es/tools/fetch/catastro.ts +51 -40
- package/lib/es/tools/fetch/gipuzkoa.ts +13 -11
- package/lib/es/tools/fetch/navarra.ts +33 -25
- package/lib/fi/adapters/ryhti/adapter.ts +22 -26
- package/lib/fi/tools/fetch/ryhti.ts +31 -31
- package/lib/fr/adapters/annuaire-education/adapter.ts +13 -11
- package/lib/fr/adapters/ban/adapter.ts +6 -6
- package/lib/fr/adapters/finess/adapter.ts +16 -15
- package/lib/fr/adapters/finess/voie-type.ts +4 -3
- package/lib/fr/insee-country.ts +2 -2
- package/lib/fr/recipes/lieudit.ts +8 -6
- package/lib/fr/tools/fetch/annuaire-education.ts +11 -8
- package/lib/fr/tools/fetch/ban.ts +3 -2
- package/lib/fr/tools/fetch/finess.ts +14 -13
- package/lib/inspire/address.ts +86 -78
- package/lib/inspire/errors.ts +9 -8
- package/lib/inspire/stream.ts +29 -26
- package/lib/international/recipes/locale.ts +13 -9
- package/lib/it/adapters/anac/adapter.ts +51 -46
- package/lib/it/tools/fetch/anac.ts +22 -25
- package/lib/jp/adapters/wof-admin/adapter.ts +2 -6
- package/lib/jp/tools/fetch/houjin.ts +3 -3
- package/lib/kr/adapters/localdata/permit-rows.ts +1 -1
- package/lib/kr/tools/fetch/juso.ts +1 -1
- package/lib/kr/tools/fetch/localdata.ts +2 -2
- package/lib/lu/adapters/bd-adresses/adapter.ts +10 -10
- package/lib/lu/tools/fetch/bd-adresses.ts +15 -13
- package/lib/lv/adapters/varis/adapter.ts +24 -23
- package/lib/lv/tools/fetch/varis.ts +3 -2
- package/lib/nl/adapters/kadaster/adapter.ts +29 -29
- package/lib/nl/tools/fetch/kadaster.ts +16 -12
- package/lib/no/adapters/matrikkelen/adapter.ts +6 -5
- package/lib/no/tools/fetch/matrikkelen.ts +59 -47
- package/lib/parquet/duckdb.ts +7 -4
- package/lib/parquet/readers.ts +3 -3
- package/lib/parquet/schema.ts +1 -1
- package/lib/parquet/writers.ts +3 -2
- package/lib/pl/adapters/emuia/adapter.ts +12 -12
- package/lib/pl/tools/fetch/emuia.ts +10 -9
- package/lib/recipes/bare/postcode.ts +1 -1
- package/lib/recipes/boundary-stress.ts +1 -1
- package/lib/recipes/house-venue.ts +7 -3
- package/lib/recipes/intersection.ts +2 -2
- package/lib/recipes/po/box/cedex/recipe.ts +3 -3
- package/lib/recipes/po/box/cedex/sources.ts +13 -1
- package/lib/recipes/scaffold.ts +31 -21
- package/lib/recipes/sources.ts +16 -3
- package/lib/recipes/street/bare.ts +2 -2
- package/lib/recipes/street.ts +3 -1
- package/lib/recipes/sub/venue/context.ts +2 -0
- package/lib/recipes/sub/venue/render.ts +1 -1
- package/lib/recipes/sub/venue/sources.ts +24 -29
- package/lib/recipes/sub/venue.ts +12 -18
- package/lib/recipes/trailing-region.ts +6 -2
- package/lib/recipes/unit.ts +3 -3
- package/lib/recipes/venue.ts +347 -0
- package/lib/recipes.ts +4 -2
- package/lib/registers.ts +8 -6
- package/lib/ro/adapters/retea-scolara/adapter.ts +5 -5
- package/lib/ro/tools/fetch/retea-scolara.ts +1 -1
- package/lib/runner.ts +4 -3
- package/lib/sg/recipes/register.ts +1 -1
- package/lib/sg/tools/fetch/acra.ts +3 -3
- package/lib/sk/adapters/inspire/adapter.ts +25 -27
- package/lib/sk/tools/fetch/inspire.ts +15 -14
- package/lib/source-register/effective-manifest.ts +3 -3
- package/lib/source-register/register.ts +3 -3
- package/lib/source-register/training-manifest.ts +1 -1
- package/lib/source-register/types.ts +12 -13
- package/lib/subvenue/promotions.ts +1 -1
- package/lib/subvenue/table.ts +4 -4
- package/lib/surfaces/locale.ts +2 -0
- package/lib/synthesizers/anchor-absorption.ts +2 -1
- package/lib/synthesizers/boundary-stress.ts +20 -26
- package/lib/synthesizers/house-venue.ts +26 -30
- package/lib/synthesizers/no-street.ts +2 -1
- package/lib/synthesizers/po-box.ts +6 -3
- package/lib/synthesizers/tail.ts +55 -0
- package/lib/synthesizers/utils.ts +19 -2
- package/lib/test-kit.ts +2 -2
- package/lib/tools/audit.ts +7 -5
- package/lib/tools/corpus-stats.ts +1 -1
- package/lib/tools/fetch/atom.ts +10 -10
- package/lib/tools/fetch/cordis.ts +11 -9
- package/lib/tools/fetch/download/manifest.ts +2 -1
- package/lib/tools/fetch/download/network.ts +4 -4
- package/lib/tools/fetch/download/retry.ts +4 -4
- package/lib/tools/fetch/geonames/dump.ts +5 -3
- package/lib/tools/fetch/geonames/postal.ts +2 -2
- package/lib/tools/fetch/gleif.ts +6 -4
- package/lib/tools/fetch/header.ts +8 -6
- package/lib/tools/fetch/inspire-addresses.ts +41 -36
- package/lib/tools/fetch/openaddresses.ts +1 -1
- package/lib/tools/fetch/vlaanderen-ad.ts +17 -17
- package/lib/tools/fetch/wfs-harvest.ts +39 -28
- package/lib/tools/fetch.ts +0 -42
- package/lib/tools/golden/expand.ts +8 -8
- package/lib/tools/golden/promote.ts +12 -8
- package/lib/tools/golden/relabel-street/directory.ts +7 -6
- package/lib/tools/golden/relabel-street/row.ts +7 -6
- package/lib/tools/ingest-csv.ts +3 -3
- package/lib/tools/jsonl-to-parquet.ts +1 -1
- package/lib/tools/lint/recipe-output/vocab.ts +10 -5
- package/lib/tools/lint/recipe-output.ts +26 -8
- package/lib/tools/overlay/kryptonite.ts +4 -3
- package/lib/tools/overlay/manifest.ts +4 -2
- package/lib/tools/overlay/merge-source.ts +2 -1
- package/lib/tools/overlay/split-slice.ts +2 -2
- package/lib/tools/overlay/translit.ts +6 -4
- package/lib/tools/overture-subvenue.ts +1 -1
- package/lib/tools/postcode-triples.ts +5 -4
- package/lib/tools/source-register/build.ts +17 -14
- package/lib/tools/sub/venue/lexicon.ts +4 -8
- package/lib/tw/tools/fetch/gcis.ts +4 -4
- package/lib/types.ts +4 -3
- package/lib/us/adapters/state/hi-schools/workbook.ts +1 -1
- package/lib/us/adapters/usgov/nad/adapter.ts +11 -11
- package/lib/us/adapters/usgov/samhsa-treatment-locator/adapter.ts +4 -4
- package/lib/us/fips-state.ts +2 -2
- package/lib/{synthesizers → us/synthesizers}/intersection.ts +12 -13
- package/lib/{synthesizers → us/synthesizers}/street.ts +16 -15
- package/lib/us/tools/fetch/imls-pls.ts +1 -1
- package/lib/us/tools/fetch/nad.ts +7 -7
- package/lib/us/tools/fetch/nppes.ts +5 -5
- package/lib/us/tools/fetch/state/hi-schools.ts +3 -2
- package/lib/us/tools/fetch/tiger-full.ts +1 -1
- package/lib/utils/align.ts +13 -10
- package/lib/utils/golden.ts +2 -2
- package/lib/utils/license.ts +3 -3
- package/lib/utils/split.ts +19 -15
- package/lib/utils/tokenize.ts +7 -26
- package/lib/utils/wof-json.ts +4 -4
- package/out/adapters/cordis/adapter.d.ts +5 -3
- package/out/adapters/cordis/adapter.d.ts.map +1 -1
- package/out/adapters/cordis/adapter.js +13 -7
- package/out/adapters/cordis/adapter.js.map +1 -1
- package/out/adapters/dedup-key.d.ts +1 -1
- package/out/adapters/dedup-key.js +1 -1
- package/out/adapters/geonames/adapter.js +1 -1
- package/out/adapters/geonames/adapter.js.map +1 -1
- package/out/adapters/gleif/adapter.d.ts +9 -9
- package/out/adapters/gleif/adapter.d.ts.map +1 -1
- package/out/adapters/gleif/adapter.js +15 -14
- package/out/adapters/gleif/adapter.js.map +1 -1
- package/out/adapters/libpostal-dictionary.d.ts +0 -2
- package/out/adapters/libpostal-dictionary.d.ts.map +1 -1
- package/out/adapters/libpostal-dictionary.js +0 -2
- package/out/adapters/libpostal-dictionary.js.map +1 -1
- package/out/adapters/openaddresses/adapter.d.ts +3 -2
- package/out/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/adapters/openaddresses/adapter.js +1 -1
- package/out/adapters/openaddresses/adapter.js.map +1 -1
- package/out/adapters/osm/adapter.d.ts +17 -16
- package/out/adapters/osm/adapter.d.ts.map +1 -1
- package/out/adapters/osm/adapter.js +10 -9
- package/out/adapters/osm/adapter.js.map +1 -1
- package/out/adapters/po-box/adapter.d.ts +6 -3
- package/out/adapters/po-box/adapter.d.ts.map +1 -1
- package/out/adapters/po-box/adapter.js +4 -2
- package/out/adapters/po-box/adapter.js.map +1 -1
- package/out/adapters/registry.d.ts +3 -3
- package/out/adapters/registry.d.ts.map +1 -1
- package/out/adapters/registry.js +1 -1
- package/out/adapters/registry.js.map +1 -1
- package/out/adapters/source-id.d.ts +2 -2
- package/out/adapters/source-id.js +2 -2
- package/out/adapters/street-line.d.ts +7 -6
- package/out/adapters/street-line.d.ts.map +1 -1
- package/out/adapters/street-line.js +7 -6
- package/out/adapters/street-line.js.map +1 -1
- package/out/adapters/wof/admin/json/adapter.js +1 -1
- package/out/adapters/wof/admin/json/adapter.js.map +1 -1
- package/out/adapters/wof/json-rows.d.ts +2 -1
- package/out/adapters/wof/json-rows.d.ts.map +1 -1
- package/out/adapters/wof/json-rows.js.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.d.ts.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.js +5 -1
- package/out/adapters/wof/postalcode-json/adapter.js.map +1 -1
- package/out/adapters.d.ts +5 -5
- package/out/adapters.js +5 -5
- package/out/au/adapters/gnaf/adapter.js +1 -1
- package/out/au/adapters/gnaf/adapter.js.map +1 -1
- package/out/be/adapters/brussels/adapter.d.ts +9 -9
- package/out/be/adapters/brussels/adapter.d.ts.map +1 -1
- package/out/be/adapters/brussels/adapter.js +21 -21
- package/out/be/adapters/brussels/adapter.js.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.d.ts +34 -37
- package/out/be/adapters/vlaanderen/adapter.d.ts.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.js +40 -38
- package/out/be/adapters/vlaanderen/adapter.js.map +1 -1
- package/out/be/adapters/wallonie/adapter.d.ts +2 -2
- package/out/be/adapters/wallonie/adapter.js +17 -17
- package/out/be/adapters/wallonie/adapter.js.map +1 -1
- package/out/be/tools/fetch/brussels.d.ts +11 -8
- package/out/be/tools/fetch/brussels.d.ts.map +1 -1
- package/out/be/tools/fetch/brussels.js +10 -8
- package/out/be/tools/fetch/brussels.js.map +1 -1
- package/out/be/tools/fetch/wallonie.d.ts +6 -5
- package/out/be/tools/fetch/wallonie.d.ts.map +1 -1
- package/out/be/tools/fetch/wallonie.js +4 -4
- package/out/be/tools/fetch/wallonie.js.map +1 -1
- package/out/build/adapters.d.ts.map +1 -1
- package/out/build/adapters.js +2 -1
- package/out/build/adapters.js.map +1 -1
- package/out/build/align.d.ts +3 -2
- package/out/build/align.d.ts.map +1 -1
- package/out/build/align.js +2 -2
- package/out/build/align.js.map +1 -1
- package/out/build/checkpoint.d.ts +1 -1
- package/out/build/checkpoint.d.ts.map +1 -1
- package/out/build/checkpoint.js +4 -2
- package/out/build/checkpoint.js.map +1 -1
- package/out/build/eligibility.d.ts +2 -2
- package/out/build/eligibility.js +3 -3
- package/out/build/eligibility.js.map +1 -1
- package/out/build/types.d.ts +8 -6
- package/out/build/types.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.d.ts +14 -14
- package/out/cz/adapters/cuzk/adapter.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.js +38 -35
- package/out/cz/adapters/cuzk/adapter.js.map +1 -1
- package/out/cz/tools/fetch/cuzk.d.ts +31 -22
- package/out/cz/tools/fetch/cuzk.d.ts.map +1 -1
- package/out/cz/tools/fetch/cuzk.js +21 -19
- package/out/cz/tools/fetch/cuzk.js.map +1 -1
- package/out/de/recipes/locale.js +1 -1
- package/out/de/recipes/locale.js.map +1 -1
- package/out/dk/adapters/inspire/adapter.d.ts +32 -33
- package/out/dk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/dk/adapters/inspire/adapter.js +38 -38
- package/out/dk/adapters/inspire/adapter.js.map +1 -1
- package/out/dk/tools/fetch/inspire.d.ts +6 -5
- package/out/dk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/dk/tools/fetch/inspire.js +15 -13
- package/out/dk/tools/fetch/inspire.js.map +1 -1
- package/out/ee/adapters/ads/adapter.d.ts +19 -26
- package/out/ee/adapters/ads/adapter.d.ts.map +1 -1
- package/out/ee/adapters/ads/adapter.js +22 -28
- package/out/ee/adapters/ads/adapter.js.map +1 -1
- package/out/ee/tools/fetch/ads.d.ts +3 -3
- package/out/ee/tools/fetch/ads.d.ts.map +1 -1
- package/out/ee/tools/fetch/ads.js +10 -8
- package/out/ee/tools/fetch/ads.js.map +1 -1
- package/out/es/adapters/bizkaia/adapter.d.ts +28 -28
- package/out/es/adapters/bizkaia/adapter.d.ts.map +1 -1
- package/out/es/adapters/bizkaia/adapter.js +43 -43
- package/out/es/adapters/bizkaia/adapter.js.map +1 -1
- package/out/es/adapters/catastro/adapter.d.ts +22 -23
- package/out/es/adapters/catastro/adapter.d.ts.map +1 -1
- package/out/es/adapters/catastro/adapter.js +35 -33
- package/out/es/adapters/catastro/adapter.js.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.d.ts +31 -38
- package/out/es/adapters/gipuzkoa/adapter.d.ts.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.js +40 -47
- package/out/es/adapters/gipuzkoa/adapter.js.map +1 -1
- package/out/es/adapters/navarra/adapter.d.ts +20 -19
- package/out/es/adapters/navarra/adapter.d.ts.map +1 -1
- package/out/es/adapters/navarra/adapter.js +23 -22
- package/out/es/adapters/navarra/adapter.js.map +1 -1
- package/out/es/cadastre.d.ts +21 -20
- package/out/es/cadastre.d.ts.map +1 -1
- package/out/es/cadastre.js +19 -21
- package/out/es/cadastre.js.map +1 -1
- package/out/es/tools/fetch/bizkaia.d.ts +28 -25
- package/out/es/tools/fetch/bizkaia.d.ts.map +1 -1
- package/out/es/tools/fetch/bizkaia.js +25 -24
- package/out/es/tools/fetch/bizkaia.js.map +1 -1
- package/out/es/tools/fetch/catastro.d.ts +28 -20
- package/out/es/tools/fetch/catastro.d.ts.map +1 -1
- package/out/es/tools/fetch/catastro.js +36 -31
- package/out/es/tools/fetch/catastro.js.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.d.ts +11 -9
- package/out/es/tools/fetch/gipuzkoa.d.ts.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.js +9 -8
- package/out/es/tools/fetch/gipuzkoa.js.map +1 -1
- package/out/es/tools/fetch/navarra.d.ts +20 -15
- package/out/es/tools/fetch/navarra.d.ts.map +1 -1
- package/out/es/tools/fetch/navarra.js +21 -16
- package/out/es/tools/fetch/navarra.js.map +1 -1
- package/out/fi/adapters/ryhti/adapter.d.ts +16 -20
- package/out/fi/adapters/ryhti/adapter.d.ts.map +1 -1
- package/out/fi/adapters/ryhti/adapter.js +21 -25
- package/out/fi/adapters/ryhti/adapter.js.map +1 -1
- package/out/fi/tools/fetch/ryhti.d.ts +24 -24
- package/out/fi/tools/fetch/ryhti.d.ts.map +1 -1
- package/out/fi/tools/fetch/ryhti.js +21 -24
- package/out/fi/tools/fetch/ryhti.js.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.d.ts +8 -8
- package/out/fr/adapters/annuaire-education/adapter.d.ts.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.js +13 -11
- package/out/fr/adapters/annuaire-education/adapter.js.map +1 -1
- package/out/fr/adapters/ban/adapter.d.ts +4 -4
- package/out/fr/adapters/ban/adapter.js +6 -6
- package/out/fr/adapters/ban/adapter.js.map +1 -1
- package/out/fr/adapters/finess/adapter.d.ts +12 -11
- package/out/fr/adapters/finess/adapter.d.ts.map +1 -1
- package/out/fr/adapters/finess/adapter.js +16 -15
- package/out/fr/adapters/finess/adapter.js.map +1 -1
- package/out/fr/adapters/finess/voie-type.d.ts +2 -2
- package/out/fr/adapters/finess/voie-type.d.ts.map +1 -1
- package/out/fr/adapters/finess/voie-type.js +4 -3
- package/out/fr/adapters/finess/voie-type.js.map +1 -1
- package/out/fr/insee-country.d.ts +2 -2
- package/out/fr/insee-country.js +2 -2
- package/out/fr/recipes/lieudit.d.ts.map +1 -1
- package/out/fr/recipes/lieudit.js +5 -4
- package/out/fr/recipes/lieudit.js.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.d.ts +7 -4
- package/out/fr/tools/fetch/annuaire-education.d.ts.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.js +9 -7
- package/out/fr/tools/fetch/annuaire-education.js.map +1 -1
- package/out/fr/tools/fetch/ban.d.ts +1 -1
- package/out/fr/tools/fetch/ban.d.ts.map +1 -1
- package/out/fr/tools/fetch/ban.js +3 -2
- package/out/fr/tools/fetch/ban.js.map +1 -1
- package/out/fr/tools/fetch/finess.d.ts +5 -5
- package/out/fr/tools/fetch/finess.d.ts.map +1 -1
- package/out/fr/tools/fetch/finess.js +13 -12
- package/out/fr/tools/fetch/finess.js.map +1 -1
- package/out/inspire/address.d.ts +62 -52
- package/out/inspire/address.d.ts.map +1 -1
- package/out/inspire/address.js +71 -60
- package/out/inspire/address.js.map +1 -1
- package/out/inspire/errors.d.ts +9 -8
- package/out/inspire/errors.d.ts.map +1 -1
- package/out/inspire/errors.js +7 -6
- package/out/inspire/errors.js.map +1 -1
- package/out/inspire/stream.d.ts +25 -22
- package/out/inspire/stream.d.ts.map +1 -1
- package/out/inspire/stream.js +10 -12
- package/out/inspire/stream.js.map +1 -1
- package/out/international/recipes/locale.d.ts +2 -2
- package/out/international/recipes/locale.d.ts.map +1 -1
- package/out/international/recipes/locale.js +11 -7
- package/out/international/recipes/locale.js.map +1 -1
- package/out/it/adapters/anac/adapter.d.ts +45 -43
- package/out/it/adapters/anac/adapter.d.ts.map +1 -1
- package/out/it/adapters/anac/adapter.js +48 -43
- package/out/it/adapters/anac/adapter.js.map +1 -1
- package/out/it/tools/fetch/anac.d.ts +16 -19
- package/out/it/tools/fetch/anac.d.ts.map +1 -1
- package/out/it/tools/fetch/anac.js +20 -23
- package/out/it/tools/fetch/anac.js.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.d.ts.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.js.map +1 -1
- package/out/jp/tools/fetch/houjin.js +2 -2
- package/out/jp/tools/fetch/houjin.js.map +1 -1
- package/out/kr/adapters/localdata/permit-rows.js +1 -1
- package/out/kr/adapters/localdata/permit-rows.js.map +1 -1
- package/out/kr/tools/fetch/juso.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.js +1 -1
- package/out/kr/tools/fetch/localdata.js.map +1 -1
- package/out/lu/adapters/bd-adresses/adapter.d.ts +9 -9
- package/out/lu/adapters/bd-adresses/adapter.js +10 -10
- package/out/lu/tools/fetch/bd-adresses.d.ts +11 -9
- package/out/lu/tools/fetch/bd-adresses.d.ts.map +1 -1
- package/out/lu/tools/fetch/bd-adresses.js +11 -11
- package/out/lu/tools/fetch/bd-adresses.js.map +1 -1
- package/out/lv/adapters/varis/adapter.d.ts +24 -23
- package/out/lv/adapters/varis/adapter.d.ts.map +1 -1
- package/out/lv/adapters/varis/adapter.js +24 -23
- package/out/lv/adapters/varis/adapter.js.map +1 -1
- package/out/lv/tools/fetch/varis.d.ts +2 -1
- package/out/lv/tools/fetch/varis.d.ts.map +1 -1
- package/out/lv/tools/fetch/varis.js +1 -1
- package/out/lv/tools/fetch/varis.js.map +1 -1
- package/out/nl/adapters/kadaster/adapter.d.ts +12 -12
- package/out/nl/adapters/kadaster/adapter.d.ts.map +1 -1
- package/out/nl/adapters/kadaster/adapter.js +22 -23
- package/out/nl/adapters/kadaster/adapter.js.map +1 -1
- package/out/nl/tools/fetch/kadaster.d.ts +14 -10
- package/out/nl/tools/fetch/kadaster.d.ts.map +1 -1
- package/out/nl/tools/fetch/kadaster.js +12 -9
- package/out/nl/tools/fetch/kadaster.js.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.js +3 -3
- package/out/no/adapters/matrikkelen/adapter.js.map +1 -1
- package/out/no/tools/fetch/matrikkelen.d.ts +50 -38
- package/out/no/tools/fetch/matrikkelen.d.ts.map +1 -1
- package/out/no/tools/fetch/matrikkelen.js +44 -36
- package/out/no/tools/fetch/matrikkelen.js.map +1 -1
- package/out/parquet/duckdb.d.ts +7 -4
- package/out/parquet/duckdb.d.ts.map +1 -1
- package/out/parquet/duckdb.js +2 -1
- package/out/parquet/duckdb.js.map +1 -1
- package/out/parquet/readers.js +3 -3
- package/out/parquet/readers.js.map +1 -1
- package/out/parquet/schema.js +1 -1
- package/out/parquet/schema.js.map +1 -1
- package/out/parquet/writers.d.ts +1 -1
- package/out/parquet/writers.d.ts.map +1 -1
- package/out/parquet/writers.js +2 -1
- package/out/parquet/writers.js.map +1 -1
- package/out/pl/adapters/emuia/adapter.d.ts +8 -8
- package/out/pl/adapters/emuia/adapter.js +12 -12
- package/out/pl/adapters/emuia/adapter.js.map +1 -1
- package/out/pl/tools/fetch/emuia.d.ts +5 -4
- package/out/pl/tools/fetch/emuia.d.ts.map +1 -1
- package/out/pl/tools/fetch/emuia.js +10 -9
- package/out/pl/tools/fetch/emuia.js.map +1 -1
- package/out/recipes/bare/postcode.js +1 -1
- package/out/recipes/bare/postcode.js.map +1 -1
- package/out/recipes/boundary-stress.js +1 -1
- package/out/recipes/boundary-stress.js.map +1 -1
- package/out/recipes/house-venue.d.ts.map +1 -1
- package/out/recipes/house-venue.js +6 -3
- package/out/recipes/house-venue.js.map +1 -1
- package/out/recipes/intersection.js +2 -2
- package/out/recipes/intersection.js.map +1 -1
- package/out/recipes/po/box/cedex/recipe.js +2 -1
- package/out/recipes/po/box/cedex/recipe.js.map +1 -1
- package/out/recipes/po/box/cedex/sources.d.ts +8 -2
- package/out/recipes/po/box/cedex/sources.d.ts.map +1 -1
- package/out/recipes/po/box/cedex/sources.js +1 -1
- package/out/recipes/po/box/cedex/sources.js.map +1 -1
- package/out/recipes/scaffold.d.ts +28 -19
- package/out/recipes/scaffold.d.ts.map +1 -1
- package/out/recipes/scaffold.js +3 -2
- package/out/recipes/scaffold.js.map +1 -1
- package/out/recipes/sources.d.ts +1 -1
- package/out/recipes/sources.d.ts.map +1 -1
- package/out/recipes/sources.js +15 -2
- package/out/recipes/sources.js.map +1 -1
- package/out/recipes/street/bare.js +2 -2
- package/out/recipes/street/bare.js.map +1 -1
- package/out/recipes/street.d.ts.map +1 -1
- package/out/recipes/street.js +3 -1
- package/out/recipes/street.js.map +1 -1
- package/out/recipes/sub/venue/context.d.ts.map +1 -1
- package/out/recipes/sub/venue/context.js +2 -0
- package/out/recipes/sub/venue/context.js.map +1 -1
- package/out/recipes/sub/venue/render.js +1 -1
- package/out/recipes/sub/venue/render.js.map +1 -1
- package/out/recipes/sub/venue/sources.d.ts +6 -3
- package/out/recipes/sub/venue/sources.d.ts.map +1 -1
- package/out/recipes/sub/venue/sources.js +16 -20
- package/out/recipes/sub/venue/sources.js.map +1 -1
- package/out/recipes/sub/venue.d.ts +2 -5
- package/out/recipes/sub/venue.d.ts.map +1 -1
- package/out/recipes/sub/venue.js +10 -14
- package/out/recipes/sub/venue.js.map +1 -1
- package/out/recipes/trailing-region.d.ts.map +1 -1
- package/out/recipes/trailing-region.js +6 -1
- package/out/recipes/trailing-region.js.map +1 -1
- package/out/recipes/unit.js +2 -2
- package/out/recipes/unit.js.map +1 -1
- package/out/recipes/venue.d.ts +119 -0
- package/out/recipes/venue.d.ts.map +1 -0
- package/out/recipes/venue.js +245 -0
- package/out/recipes/venue.js.map +1 -0
- package/out/recipes.d.ts +1 -1
- package/out/recipes.d.ts.map +1 -1
- package/out/recipes.js +3 -1
- package/out/recipes.js.map +1 -1
- package/out/registers.d.ts +8 -6
- package/out/registers.d.ts.map +1 -1
- package/out/registers.js +8 -6
- package/out/registers.js.map +1 -1
- package/out/ro/adapters/retea-scolara/adapter.d.ts +5 -5
- package/out/ro/adapters/retea-scolara/adapter.js +5 -5
- package/out/ro/tools/fetch/retea-scolara.js +1 -1
- package/out/ro/tools/fetch/retea-scolara.js.map +1 -1
- package/out/runner.d.ts +3 -2
- package/out/runner.d.ts.map +1 -1
- package/out/runner.js +1 -1
- package/out/runner.js.map +1 -1
- package/out/sg/recipes/register.js +1 -1
- package/out/sg/recipes/register.js.map +1 -1
- package/out/sg/tools/fetch/acra.js +2 -2
- package/out/sg/tools/fetch/acra.js.map +1 -1
- package/out/sk/adapters/inspire/adapter.d.ts +5 -5
- package/out/sk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/sk/adapters/inspire/adapter.js +18 -18
- package/out/sk/adapters/inspire/adapter.js.map +1 -1
- package/out/sk/tools/fetch/inspire.d.ts +11 -10
- package/out/sk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/sk/tools/fetch/inspire.js +15 -14
- package/out/sk/tools/fetch/inspire.js.map +1 -1
- package/out/source-register/effective-manifest.js +3 -3
- package/out/source-register/effective-manifest.js.map +1 -1
- package/out/source-register/register.d.ts +2 -2
- package/out/source-register/register.d.ts.map +1 -1
- package/out/source-register/register.js +2 -2
- package/out/source-register/register.js.map +1 -1
- package/out/source-register/training-manifest.d.ts +1 -1
- package/out/source-register/types.d.ts +12 -13
- package/out/source-register/types.d.ts.map +1 -1
- package/out/source-register/types.js +2 -2
- package/out/source-register/types.js.map +1 -1
- package/out/subvenue/promotions.d.ts +1 -1
- package/out/subvenue/promotions.js +1 -1
- package/out/subvenue/table.d.ts +4 -4
- package/out/subvenue/table.js +3 -3
- package/out/surfaces/locale.d.ts.map +1 -1
- package/out/surfaces/locale.js +2 -0
- package/out/surfaces/locale.js.map +1 -1
- package/out/synthesizers/anchor-absorption.d.ts +2 -1
- package/out/synthesizers/anchor-absorption.d.ts.map +1 -1
- package/out/synthesizers/anchor-absorption.js.map +1 -1
- package/out/synthesizers/boundary-stress.d.ts +1 -1
- package/out/synthesizers/boundary-stress.d.ts.map +1 -1
- package/out/synthesizers/boundary-stress.js +20 -23
- package/out/synthesizers/boundary-stress.js.map +1 -1
- package/out/synthesizers/house-venue.d.ts +3 -2
- package/out/synthesizers/house-venue.d.ts.map +1 -1
- package/out/synthesizers/house-venue.js +24 -26
- package/out/synthesizers/house-venue.js.map +1 -1
- package/out/synthesizers/no-street.d.ts.map +1 -1
- package/out/synthesizers/no-street.js.map +1 -1
- package/out/synthesizers/po-box.d.ts +4 -2
- package/out/synthesizers/po-box.d.ts.map +1 -1
- package/out/synthesizers/po-box.js.map +1 -1
- package/out/synthesizers/tail.d.ts +36 -0
- package/out/synthesizers/tail.d.ts.map +1 -0
- package/out/synthesizers/tail.js +32 -0
- package/out/synthesizers/tail.js.map +1 -0
- package/out/synthesizers/utils.d.ts +2 -1
- package/out/synthesizers/utils.d.ts.map +1 -1
- package/out/synthesizers/utils.js +17 -1
- package/out/synthesizers/utils.js.map +1 -1
- package/out/test-kit.js +2 -2
- package/out/test-kit.js.map +1 -1
- package/out/tools/audit.d.ts +2 -1
- package/out/tools/audit.d.ts.map +1 -1
- package/out/tools/audit.js +4 -3
- package/out/tools/audit.js.map +1 -1
- package/out/tools/corpus-stats.js +1 -1
- package/out/tools/corpus-stats.js.map +1 -1
- package/out/tools/fetch/atom.d.ts +8 -8
- package/out/tools/fetch/atom.d.ts.map +1 -1
- package/out/tools/fetch/atom.js +7 -8
- package/out/tools/fetch/atom.js.map +1 -1
- package/out/tools/fetch/cordis.d.ts +4 -3
- package/out/tools/fetch/cordis.d.ts.map +1 -1
- package/out/tools/fetch/cordis.js +9 -7
- package/out/tools/fetch/cordis.js.map +1 -1
- package/out/tools/fetch/download/manifest.d.ts.map +1 -1
- package/out/tools/fetch/download/manifest.js +2 -1
- package/out/tools/fetch/download/manifest.js.map +1 -1
- package/out/tools/fetch/download/network.d.ts +4 -4
- package/out/tools/fetch/download/retry.js +4 -4
- package/out/tools/fetch/download/retry.js.map +1 -1
- package/out/tools/fetch/geonames/dump.d.ts +3 -2
- package/out/tools/fetch/geonames/dump.d.ts.map +1 -1
- package/out/tools/fetch/geonames/dump.js +2 -1
- package/out/tools/fetch/geonames/dump.js.map +1 -1
- package/out/tools/fetch/geonames/postal.d.ts +2 -2
- package/out/tools/fetch/gleif.d.ts +3 -2
- package/out/tools/fetch/gleif.d.ts.map +1 -1
- package/out/tools/fetch/gleif.js +6 -4
- package/out/tools/fetch/gleif.js.map +1 -1
- package/out/tools/fetch/header.d.ts +7 -5
- package/out/tools/fetch/header.d.ts.map +1 -1
- package/out/tools/fetch/header.js +8 -6
- package/out/tools/fetch/header.js.map +1 -1
- package/out/tools/fetch/inspire-addresses.d.ts +32 -27
- package/out/tools/fetch/inspire-addresses.d.ts.map +1 -1
- package/out/tools/fetch/inspire-addresses.js +25 -24
- package/out/tools/fetch/inspire-addresses.js.map +1 -1
- package/out/tools/fetch/openaddresses.js +1 -1
- package/out/tools/fetch/openaddresses.js.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.d.ts +5 -5
- package/out/tools/fetch/vlaanderen-ad.d.ts.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.js +16 -16
- package/out/tools/fetch/vlaanderen-ad.js.map +1 -1
- package/out/tools/fetch/wfs-harvest.d.ts +26 -16
- package/out/tools/fetch/wfs-harvest.d.ts.map +1 -1
- package/out/tools/fetch/wfs-harvest.js +21 -18
- package/out/tools/fetch/wfs-harvest.js.map +1 -1
- package/out/tools/fetch.d.ts +0 -41
- package/out/tools/fetch.d.ts.map +1 -1
- package/out/tools/fetch.js +0 -41
- package/out/tools/fetch.js.map +1 -1
- package/out/tools/golden/expand.d.ts +7 -7
- package/out/tools/golden/expand.js +1 -1
- package/out/tools/golden/expand.js.map +1 -1
- package/out/tools/golden/promote.d.ts +2 -2
- package/out/tools/golden/promote.d.ts.map +1 -1
- package/out/tools/golden/promote.js +9 -6
- package/out/tools/golden/promote.js.map +1 -1
- package/out/tools/golden/relabel-street/directory.d.ts +4 -4
- package/out/tools/golden/relabel-street/directory.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/directory.js +3 -2
- package/out/tools/golden/relabel-street/directory.js.map +1 -1
- package/out/tools/golden/relabel-street/row.d.ts +4 -3
- package/out/tools/golden/relabel-street/row.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/row.js +3 -3
- package/out/tools/golden/relabel-street/row.js.map +1 -1
- package/out/tools/ingest-csv.js +3 -3
- package/out/tools/ingest-csv.js.map +1 -1
- package/out/tools/jsonl-to-parquet.d.ts +1 -1
- package/out/tools/lint/recipe-output/vocab.d.ts +10 -5
- package/out/tools/lint/recipe-output/vocab.d.ts.map +1 -1
- package/out/tools/lint/recipe-output/vocab.js.map +1 -1
- package/out/tools/lint/recipe-output.d.ts +8 -8
- package/out/tools/lint/recipe-output.d.ts.map +1 -1
- package/out/tools/lint/recipe-output.js +17 -0
- package/out/tools/lint/recipe-output.js.map +1 -1
- package/out/tools/overlay/kryptonite.d.ts +2 -2
- package/out/tools/overlay/kryptonite.d.ts.map +1 -1
- package/out/tools/overlay/kryptonite.js +2 -1
- package/out/tools/overlay/kryptonite.js.map +1 -1
- package/out/tools/overlay/manifest.d.ts +2 -1
- package/out/tools/overlay/manifest.d.ts.map +1 -1
- package/out/tools/overlay/manifest.js +2 -1
- package/out/tools/overlay/manifest.js.map +1 -1
- package/out/tools/overlay/merge-source.d.ts +2 -1
- package/out/tools/overlay/merge-source.d.ts.map +1 -1
- package/out/tools/overlay/merge-source.js.map +1 -1
- package/out/tools/overlay/split-slice.d.ts +1 -1
- package/out/tools/overlay/split-slice.js +1 -1
- package/out/tools/overlay/split-slice.js.map +1 -1
- package/out/tools/overlay/translit.d.ts +4 -3
- package/out/tools/overlay/translit.d.ts.map +1 -1
- package/out/tools/overlay/translit.js +2 -1
- package/out/tools/overlay/translit.js.map +1 -1
- package/out/tools/overture-subvenue.d.ts +1 -1
- package/out/tools/overture-subvenue.d.ts.map +1 -1
- package/out/tools/postcode-triples.d.ts +1 -1
- package/out/tools/postcode-triples.d.ts.map +1 -1
- package/out/tools/postcode-triples.js +4 -3
- package/out/tools/postcode-triples.js.map +1 -1
- package/out/tools/source-register/build.d.ts +10 -8
- package/out/tools/source-register/build.d.ts.map +1 -1
- package/out/tools/source-register/build.js +10 -8
- package/out/tools/source-register/build.js.map +1 -1
- package/out/tools/sub/venue/lexicon.d.ts +2 -6
- package/out/tools/sub/venue/lexicon.d.ts.map +1 -1
- package/out/tools/sub/venue/lexicon.js +2 -6
- package/out/tools/sub/venue/lexicon.js.map +1 -1
- package/out/tw/tools/fetch/gcis.js +3 -3
- package/out/tw/tools/fetch/gcis.js.map +1 -1
- package/out/types.d.ts +4 -3
- package/out/types.d.ts.map +1 -1
- package/out/types.js.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.js.map +1 -1
- package/out/us/adapters/usgov/nad/adapter.js +6 -6
- package/out/us/adapters/usgov/nad/adapter.js.map +1 -1
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js +2 -2
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/us/fips-state.d.ts +2 -2
- package/out/us/fips-state.d.ts.map +1 -1
- package/out/us/fips-state.js.map +1 -1
- package/out/us/synthesizers/intersection.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/intersection.js +13 -9
- package/out/us/synthesizers/intersection.js.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.d.ts +3 -2
- package/out/us/synthesizers/street.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.js +12 -12
- package/out/us/synthesizers/street.js.map +1 -0
- package/out/us/tools/fetch/imls-pls.js +1 -1
- package/out/us/tools/fetch/imls-pls.js.map +1 -1
- package/out/us/tools/fetch/nad.d.ts +6 -6
- package/out/us/tools/fetch/nad.js +1 -1
- package/out/us/tools/fetch/nad.js.map +1 -1
- package/out/us/tools/fetch/nppes.js +3 -3
- package/out/us/tools/fetch/nppes.js.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts +2 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.js +1 -1
- package/out/us/tools/fetch/state/hi-schools.js.map +1 -1
- package/out/us/tools/fetch/tiger-full.js +1 -1
- package/out/us/tools/fetch/tiger-full.js.map +1 -1
- package/out/utils/align.d.ts +5 -3
- package/out/utils/align.d.ts.map +1 -1
- package/out/utils/align.js +5 -5
- package/out/utils/align.js.map +1 -1
- package/out/utils/golden.js +2 -2
- package/out/utils/golden.js.map +1 -1
- package/out/utils/license.d.ts +3 -3
- package/out/utils/license.d.ts.map +1 -1
- package/out/utils/license.js.map +1 -1
- package/out/utils/split.d.ts +1 -1
- package/out/utils/split.d.ts.map +1 -1
- package/out/utils/split.js +18 -14
- package/out/utils/split.js.map +1 -1
- package/out/utils/tokenize.d.ts +3 -20
- package/out/utils/tokenize.d.ts.map +1 -1
- package/out/utils/tokenize.js.map +1 -1
- package/out/utils/wof-json.js +3 -3
- package/out/utils/wof-json.js.map +1 -1
- package/package.json +18 -27
- package/lib/tools.ts +0 -31
- package/out/synthesizers/intersection.d.ts.map +0 -1
- package/out/synthesizers/intersection.js.map +0 -1
- package/out/synthesizers/street.d.ts.map +0 -1
- package/out/synthesizers/street.js.map +0 -1
- package/out/tools.d.ts +0 -31
- package/out/tools.d.ts.map +0 -1
- package/out/tools.js +0 -31
- package/out/tools.js.map +0 -1
- /package/out/{synthesizers → us/synthesizers}/intersection.d.ts +0 -0
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `venue` recipe. A real venue name written as the whole query, and the same name followed by a
|
|
7
|
+
* locality drawn from the country's admin tuples. The rows teach the model that
|
|
8
|
+
* `Manchester Art Gallery` is a venue rather than a locality and a region, which the venue-head
|
|
9
|
+
* prior in `@mailwoman/neural` also asserts, so the model and the prior agree.
|
|
10
|
+
*
|
|
11
|
+
* Venue names come from the Overture place-names Parquet file the venue-head table is built
|
|
12
|
+
* from, which holds `name`, `category` and `country` for every place at confidence 0.85 or
|
|
13
|
+
* higher. The file has no locality column, so the locality row joins a name to a tuple of the
|
|
14
|
+
* same country and asserts no fact about where the venue stands, so its surface is
|
|
15
|
+
* `SurfaceOrigin.Composed`. The bare row is the publisher's own string, so its surface is
|
|
16
|
+
* `SurfaceOrigin.Attested`.
|
|
17
|
+
*
|
|
18
|
+
* A bare row needs a name of two or more words. A one-word name such as `Blackwell` or `Roadside`
|
|
19
|
+
* is written as a bare query only beside a locality, because a lone unknown word is the shape of a
|
|
20
|
+
* bare locality and a row that labels it `venue` would teach against bare-locality recall.
|
|
21
|
+
*
|
|
22
|
+
* A name that equals a locality of the country's tuples, or whose written row is an input on a
|
|
23
|
+
* gauntlet board, is refused.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
27
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
28
|
+
import { venueHeadLexicon } from "@mailwoman/poi-taxonomy/venue-heads"
|
|
29
|
+
import type { PathBuilderLike } from "path-ts"
|
|
30
|
+
|
|
31
|
+
import { normalizeGauntletSurface, readGauntletInputs } from "#gauntlet-inputs"
|
|
32
|
+
import { type DisposableDuckDB, escapeSQLString, openDuckDB } from "#parquet/duckdb"
|
|
33
|
+
import {
|
|
34
|
+
alignAndWrite,
|
|
35
|
+
readTuples,
|
|
36
|
+
recipeSourceID,
|
|
37
|
+
type CorpusRecipe,
|
|
38
|
+
type RecipeTuple,
|
|
39
|
+
type WriteRecipeLine,
|
|
40
|
+
} from "#recipes/scaffold"
|
|
41
|
+
import { defaultRecipeSource } from "#recipes/sources"
|
|
42
|
+
import { SourceRegister } from "#registers"
|
|
43
|
+
import { countryToLocale } from "#synthesizers/utils"
|
|
44
|
+
import { SurfaceOrigin } from "#types"
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* The Overture release the venue-head table is built from, so the two read one name population.
|
|
48
|
+
*/
|
|
49
|
+
export const DEFAULT_VENUE_NAMES: PathBuilderLike = dataRootPath(
|
|
50
|
+
"overture",
|
|
51
|
+
"2026-09-23.1",
|
|
52
|
+
"place-names",
|
|
53
|
+
"names.parquet"
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Venue names drawn per country when `--count` is absent.
|
|
58
|
+
*/
|
|
59
|
+
export const DEFAULT_NAMES_PER_COUNTRY = 20_000
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Share of locality rows that also carry the region when `--venue-region-fraction` is absent.
|
|
63
|
+
*/
|
|
64
|
+
export const DEFAULT_REGION_FRACTION = 0.5
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* The two written shapes.
|
|
68
|
+
*/
|
|
69
|
+
export const VenueTemplate = {
|
|
70
|
+
/**
|
|
71
|
+
* The venue name as the whole query.
|
|
72
|
+
*/
|
|
73
|
+
Bare: "venue-bare",
|
|
74
|
+
/**
|
|
75
|
+
* The venue name, a comma, and a locality of the same country.
|
|
76
|
+
*/
|
|
77
|
+
Locality: "venue-locality",
|
|
78
|
+
} as const
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* One of {@linkcode VenueTemplate}.
|
|
82
|
+
*/
|
|
83
|
+
export type VenueTemplate = (typeof VenueTemplate)[keyof typeof VenueTemplate]
|
|
84
|
+
|
|
85
|
+
const LICENSE =
|
|
86
|
+
"CDLA-Permissive-2.0 — Overture Maps places names; the locality comes from CC-BY / public-domain admin tuples"
|
|
87
|
+
|
|
88
|
+
const BARE_PROVENANCE = {
|
|
89
|
+
register: SourceRegister.Overture,
|
|
90
|
+
surface: SurfaceOrigin.Attested,
|
|
91
|
+
// The names file has no record id column.
|
|
92
|
+
baseSourceID: null,
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const LOCALITY_PROVENANCE = {
|
|
96
|
+
register: SourceRegister.Overture,
|
|
97
|
+
surface: SurfaceOrigin.Composed,
|
|
98
|
+
baseSourceID: null,
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* A head lookup for one country, the shape `venueHeadLexicon` returns.
|
|
103
|
+
*/
|
|
104
|
+
export interface VenueHeadSuffixLookup {
|
|
105
|
+
suffix(word: string): number | null
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const SPACELESS_SCRIPT_RE = /^[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}\p{Script=Thai}]+$/u
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Characters a spaceless-script name needs before it is written as a bare query.
|
|
112
|
+
*
|
|
113
|
+
* A two-character name such as `東京` is the length of a city name, and the head the table recognizes is
|
|
114
|
+
* the name's last character, so a name shorter than this is a head with one character of stem.
|
|
115
|
+
*/
|
|
116
|
+
export const MIN_SPACELESS_BARE_CHARACTERS = 3
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Whether a name may be written as the whole query.
|
|
120
|
+
*
|
|
121
|
+
* In a spaced script the name needs two or more words, and at least one word
|
|
122
|
+
* with a letter, so `24 7` and `A1` stay out.
|
|
123
|
+
* In a script written without spaces a name is one word, so it qualifies when it has three or more
|
|
124
|
+
* characters and ends in a venue head the table holds for the country, as `国立西洋美術館` ends in `館`.
|
|
125
|
+
*
|
|
126
|
+
* A spaceless name the table does not recognize is written beside a locality only.
|
|
127
|
+
*/
|
|
128
|
+
export function isBareVenueName(name: string, heads?: VenueHeadSuffixLookup): boolean {
|
|
129
|
+
const words = name.trim().split(/\s+/u)
|
|
130
|
+
|
|
131
|
+
if (words.length >= 2) return words.some((word) => /\p{L}{2,}/u.test(word))
|
|
132
|
+
|
|
133
|
+
const word = words[0]!
|
|
134
|
+
|
|
135
|
+
return (
|
|
136
|
+
Boolean(heads) &&
|
|
137
|
+
[...word].length >= MIN_SPACELESS_BARE_CHARACTERS &&
|
|
138
|
+
SPACELESS_SCRIPT_RE.test(word) &&
|
|
139
|
+
heads!.suffix(word) !== null
|
|
140
|
+
)
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* The comparison key for a name against the country's localities.
|
|
145
|
+
*/
|
|
146
|
+
export function venueNameKey(name: string): string {
|
|
147
|
+
return name.normalize("NFC").trim().toLowerCase().replaceAll(/\s+/gu, " ")
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Admin tuples grouped by country, with the locality set each country's names are checked against.
|
|
152
|
+
*/
|
|
153
|
+
export interface CountryTuples {
|
|
154
|
+
tuples: RecipeTuple[]
|
|
155
|
+
localities: Set<string>
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* The ISO 3166-1 alpha-2 code of a tuple's country.
|
|
160
|
+
*
|
|
161
|
+
* `corpus tuples` writes the code under `cc` and the display name under `country`.
|
|
162
|
+
* Older tuples files write the code under `country`.
|
|
163
|
+
* A tuple with neither form returns `null`.
|
|
164
|
+
*/
|
|
165
|
+
export function tupleCountryCode(tuple: RecipeTuple): string | null {
|
|
166
|
+
for (const value of [tuple.cc, tuple.country]) {
|
|
167
|
+
if (typeof value === "string" && /^[A-Za-z]{2}$/u.test(value.trim())) return value.trim().toUpperCase()
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
return null
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Groups the admin tuples by country and keeps those with a locality.
|
|
175
|
+
*/
|
|
176
|
+
export async function groupTuplesByCountry(input: PathBuilderLike): Promise<Map<string, CountryTuples>> {
|
|
177
|
+
const byCountry = new Map<string, CountryTuples>()
|
|
178
|
+
|
|
179
|
+
for await (const tuple of readTuples(input)) {
|
|
180
|
+
const country = tupleCountryCode(tuple)
|
|
181
|
+
|
|
182
|
+
if (!country || !tuple.locality) continue
|
|
183
|
+
let entry = byCountry.get(country)
|
|
184
|
+
|
|
185
|
+
if (!entry) {
|
|
186
|
+
entry = { tuples: [], localities: new Set() }
|
|
187
|
+
byCountry.set(country, entry)
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
entry.tuples.push(tuple)
|
|
191
|
+
entry.localities.add(venueNameKey(tuple.locality))
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
return byCountry
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* Reads up to `limit` distinct names of one country from the Overture names file.
|
|
199
|
+
*
|
|
200
|
+
* The order is a hash of the name and the seed, so one seed draws one sample and a second seed another.
|
|
201
|
+
*/
|
|
202
|
+
export async function readVenueNames(
|
|
203
|
+
db: DisposableDuckDB,
|
|
204
|
+
venueNames: PathBuilderLike,
|
|
205
|
+
country: string,
|
|
206
|
+
limit: number,
|
|
207
|
+
seed: number
|
|
208
|
+
): Promise<string[]> {
|
|
209
|
+
const result = await db.runAndReadAll(
|
|
210
|
+
`SELECT DISTINCT name FROM read_parquet('${escapeSQLString(String(venueNames))}')
|
|
211
|
+
WHERE country = '${escapeSQLString(country)}' AND name IS NOT NULL AND length(trim(name)) >= 3
|
|
212
|
+
ORDER BY hash(name, ${Math.floor(seed)}) LIMIT ${Math.floor(limit)}`
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
return result.getRows().map((row) => String(row[0]))
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
interface EmitContext {
|
|
219
|
+
write: WriteRecipeLine
|
|
220
|
+
source: { bare: string; locality: string }
|
|
221
|
+
boardInputs: ReadonlySet<string>
|
|
222
|
+
random: () => number
|
|
223
|
+
stats: { emitted: number; skipped: number; contaminated: number }
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function emit(
|
|
227
|
+
context: EmitContext,
|
|
228
|
+
raw: string,
|
|
229
|
+
components: Record<string, string>,
|
|
230
|
+
country: string,
|
|
231
|
+
template: VenueTemplate,
|
|
232
|
+
idParts: Record<string, string | undefined>
|
|
233
|
+
): void {
|
|
234
|
+
if (context.boardInputs.has(normalizeGauntletSurface(raw))) {
|
|
235
|
+
context.stats.contaminated++
|
|
236
|
+
|
|
237
|
+
return
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const source = template === VenueTemplate.Bare ? context.source.bare : context.source.locality
|
|
241
|
+
|
|
242
|
+
const ok = alignAndWrite(
|
|
243
|
+
context.write,
|
|
244
|
+
{
|
|
245
|
+
raw,
|
|
246
|
+
components,
|
|
247
|
+
country,
|
|
248
|
+
locale: countryToLocale(country),
|
|
249
|
+
source,
|
|
250
|
+
source_id: recipeSourceID(source, { ...idParts, template }),
|
|
251
|
+
corpus_version: "0.12.0",
|
|
252
|
+
license: LICENSE,
|
|
253
|
+
},
|
|
254
|
+
template,
|
|
255
|
+
template === VenueTemplate.Bare ? BARE_PROVENANCE : LOCALITY_PROVENANCE
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
if (ok) {
|
|
259
|
+
context.stats.emitted++
|
|
260
|
+
} else {
|
|
261
|
+
context.stats.skipped++
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/**
|
|
266
|
+
* Recipe registered with the corpus builder.
|
|
267
|
+
*
|
|
268
|
+
* `--input` supplies locality, region and country tuples, and every country
|
|
269
|
+
* in it with venue names is covered.
|
|
270
|
+
* `--count` caps the names drawn per country.
|
|
271
|
+
* `--venue-names` points at the Overture names file.
|
|
272
|
+
*/
|
|
273
|
+
export const venueRecipe: CorpusRecipe = {
|
|
274
|
+
name: "venue",
|
|
275
|
+
description: "A real venue name as the whole query, and the name beside a locality of its country",
|
|
276
|
+
mode: "tuples",
|
|
277
|
+
options: [
|
|
278
|
+
{ flag: "--venue-names <parquet>", description: `Overture place-names file (default ${DEFAULT_VENUE_NAMES})` },
|
|
279
|
+
{ flag: "--count <n>", description: `venue names per country (default ${DEFAULT_NAMES_PER_COUNTRY})` },
|
|
280
|
+
{
|
|
281
|
+
flag: "--venue-region-fraction <share>",
|
|
282
|
+
description: `locality rows that also carry the region (default ${DEFAULT_REGION_FRACTION})`,
|
|
283
|
+
},
|
|
284
|
+
],
|
|
285
|
+
async run(opts, write) {
|
|
286
|
+
if (!opts.input) throw new Error("venue recipe requires --input <tuples.jsonl>")
|
|
287
|
+
const random = makeMulberry32(opts.seed)
|
|
288
|
+
const venueNames = opts.venueNames ?? DEFAULT_VENUE_NAMES
|
|
289
|
+
const perCountry = opts.count ?? DEFAULT_NAMES_PER_COUNTRY
|
|
290
|
+
const regionFraction = opts.venueRegionFraction ?? DEFAULT_REGION_FRACTION
|
|
291
|
+
const byCountry = await groupTuplesByCountry(opts.input)
|
|
292
|
+
const boardInputs = await readGauntletInputs()
|
|
293
|
+
|
|
294
|
+
const context: EmitContext = {
|
|
295
|
+
write,
|
|
296
|
+
source: {
|
|
297
|
+
bare: opts.sourceName ? `${opts.sourceName}-bare` : defaultRecipeSource("synth-venue-bare"),
|
|
298
|
+
locality: opts.sourceName ? `${opts.sourceName}-locality` : defaultRecipeSource("synth-venue-locality"),
|
|
299
|
+
},
|
|
300
|
+
boardInputs,
|
|
301
|
+
random,
|
|
302
|
+
stats: { emitted: 0, skipped: 0, contaminated: 0 },
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
let read = 0
|
|
306
|
+
|
|
307
|
+
using db = await openDuckDB({ threads: 4 })
|
|
308
|
+
|
|
309
|
+
for (const [country, { tuples, localities }] of byCountry) {
|
|
310
|
+
const names = await readVenueNames(db, venueNames, country, perCountry, opts.seed)
|
|
311
|
+
const heads = venueHeadLexicon(country)
|
|
312
|
+
|
|
313
|
+
for (const name of names) {
|
|
314
|
+
read++
|
|
315
|
+
|
|
316
|
+
if (localities.has(venueNameKey(name))) {
|
|
317
|
+
context.stats.skipped++
|
|
318
|
+
|
|
319
|
+
continue
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
const tuple = tuples[Math.floor(random() * tuples.length)]!
|
|
323
|
+
const locality = tuple.locality!
|
|
324
|
+
const withRegion = Boolean(tuple.region) && random() < regionFraction
|
|
325
|
+
const components: Record<string, string> = { venue: name, locality }
|
|
326
|
+
let raw = `${name}, ${locality}`
|
|
327
|
+
|
|
328
|
+
if (withRegion) {
|
|
329
|
+
components.region = tuple.region!
|
|
330
|
+
raw = `${raw}, ${tuple.region}`
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
emit(context, raw, components, country, VenueTemplate.Locality, {
|
|
334
|
+
name,
|
|
335
|
+
locality,
|
|
336
|
+
region: withRegion ? tuple.region : undefined,
|
|
337
|
+
})
|
|
338
|
+
|
|
339
|
+
if (isBareVenueName(name, heads)) {
|
|
340
|
+
emit(context, name, { venue: name }, country, VenueTemplate.Bare, { name })
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
return { read, ...context.stats }
|
|
346
|
+
},
|
|
347
|
+
}
|
package/lib/recipes.ts
CHANGED
|
@@ -39,6 +39,7 @@ import { streetBareRecipe } from "#recipes/street/bare"
|
|
|
39
39
|
import { subVenueRecipe } from "#recipes/sub/venue"
|
|
40
40
|
import { trailingRegionRecipe } from "#recipes/trailing-region"
|
|
41
41
|
import { unitRecipe } from "#recipes/unit"
|
|
42
|
+
import { venueRecipe } from "#recipes/venue"
|
|
42
43
|
import { sgRegisterRecipe } from "#sg/recipes/register"
|
|
43
44
|
import { siBareVillageRecipe } from "#si/recipes/bare-village"
|
|
44
45
|
import { bdRegisterRecipe, pkRegisterRecipe } from "#south-asia/recipes/register"
|
|
@@ -84,6 +85,7 @@ const RECIPES: readonly CorpusRecipe[] = [
|
|
|
84
85
|
boundaryStressRecipe,
|
|
85
86
|
anchorAbsorptionRecipe,
|
|
86
87
|
subVenueRecipe,
|
|
88
|
+
venueRecipe,
|
|
87
89
|
]
|
|
88
90
|
|
|
89
91
|
/**
|
|
@@ -94,8 +96,8 @@ export const RECIPES_BY_NAME: ReadonlyMap<string, CorpusRecipe> = new Map(RECIPE
|
|
|
94
96
|
/**
|
|
95
97
|
* Look up a recipe by its `<recipe>` name.
|
|
96
98
|
*/
|
|
97
|
-
export function getRecipe(name: string): CorpusRecipe |
|
|
98
|
-
return RECIPES_BY_NAME.get(name)
|
|
99
|
+
export function getRecipe(name: string): CorpusRecipe | null {
|
|
100
|
+
return RECIPES_BY_NAME.get(name) ?? null
|
|
99
101
|
}
|
|
100
102
|
|
|
101
103
|
/**
|
package/lib/registers.ts
CHANGED
|
@@ -61,8 +61,10 @@ export const SourceRegister = {
|
|
|
61
61
|
*/
|
|
62
62
|
CuzkInspireAddresses: "cz-cuzk-inspire-addresses",
|
|
63
63
|
/**
|
|
64
|
-
* Suomen ympäristökeskus' Ryhti built-environment system
|
|
65
|
-
*
|
|
64
|
+
* Suomen ympäristökeskus' Ryhti built-environment system.
|
|
65
|
+
*
|
|
66
|
+
* Its building address data covers Åland's sixteen municipalities beside the
|
|
67
|
+
* Finnish mainland in one national file.
|
|
66
68
|
*
|
|
67
69
|
* The prefix is the publisher's own country, as `fr-ban`'s is.
|
|
68
70
|
*/
|
|
@@ -136,7 +138,7 @@ export const SourceRegister = {
|
|
|
136
138
|
* The INSPIRE Addresses theme of the Diputación Foral de Bizkaia.
|
|
137
139
|
*
|
|
138
140
|
* Coverage is the province of Bizkaia, as 112 municipality archives.
|
|
139
|
-
* The archives
|
|
141
|
+
* The archives hold addresses only, and the features they reference are served
|
|
140
142
|
* by the publisher's WFS rather than by the ATOM feed.
|
|
141
143
|
*/
|
|
142
144
|
BizkaiaInspireAddresses: "es-bizkaia-inspire-ad",
|
|
@@ -149,8 +151,8 @@ export const SourceRegister = {
|
|
|
149
151
|
/**
|
|
150
152
|
* The INSPIRE Addresses theme of the Gobierno de Navarra.
|
|
151
153
|
*
|
|
152
|
-
* The province is split across 272 zipped GML partitions, each titled
|
|
153
|
-
*
|
|
154
|
+
* The province is split across 272 zipped GML partitions, each titled
|
|
155
|
+
* `Address Navarra` The title states no municipality.
|
|
154
156
|
*/
|
|
155
157
|
NavarraInspireAddresses: "es-navarra-inspire-ad",
|
|
156
158
|
/**
|
|
@@ -166,7 +168,7 @@ export const SourceRegister = {
|
|
|
166
168
|
* of Public Contracts, reached through the Open Contracting Partnership's Data Registry.
|
|
167
169
|
*
|
|
168
170
|
* Every address in the release belongs to a contracting authority or a paying office,
|
|
169
|
-
* because the `supplier` role
|
|
171
|
+
* because the `supplier` role holds no address object.
|
|
170
172
|
*/
|
|
171
173
|
ANACContracts: "it-anac-ocds-contracts",
|
|
172
174
|
/**
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* `ro-retea-scolara`: Romania's school network (`Rețea școlară`), the Ministry of Education's list of every
|
|
7
7
|
* education unit for a school year, read for each unit's postal address.
|
|
8
8
|
*
|
|
9
|
-
* Input is the publisher's own workbook, one sheet
|
|
9
|
+
* Input is the publisher's own workbook, one sheet titled `Export`, as data.gov.ro serves it for the
|
|
10
10
|
* dataset `69392a50-5750-4a54-90c1-d461de44da6d`. Three rows precede the header. The adapter finds the
|
|
11
11
|
* header by its `Strada` cell rather than by position, because the preamble is a generation timestamp
|
|
12
12
|
* whose length the publisher does not fix.
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
* has none, on 601 and 180 units. Both are read as absent. A range such as `12-14` is a house number.
|
|
24
24
|
* - `Cod postal` holds a six-digit Romanian postcode on 17,484 units. A value of any other length is
|
|
25
25
|
* left out of the row rather than refusing the row, because the address is complete without it.
|
|
26
|
-
* - `Judet PJ` holds the county as a two-letter code. The row
|
|
26
|
+
* - `Judet PJ` holds the county as a two-letter code. The row holds no region, because libaddressinput's
|
|
27
27
|
* Romanian layout places the county between postcode and locality, and this reader has no source for
|
|
28
28
|
* the county name in the form an address prints it.
|
|
29
29
|
*
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
* ## Organizations only
|
|
33
33
|
*
|
|
34
34
|
* Every unit is a school, kindergarten, children's club, county inspectorate or similar public body, and
|
|
35
|
-
* the sheet
|
|
35
|
+
* the sheet holds no natural person's address. The venue is the institution's name.
|
|
36
36
|
*
|
|
37
37
|
* ## Identity and license
|
|
38
38
|
*
|
|
@@ -42,7 +42,7 @@
|
|
|
42
42
|
* data.gov.ro publishes the dataset under `CC-BY-4.0`, the license its CKAN record names for the
|
|
43
43
|
* publisher `Ministerul Educației`. CC BY 4.0 requires credit, a link to the license and a statement that
|
|
44
44
|
* changes were made, so every row records the license and {@linkcode RO_RETEA_SCOLARA_ATTRIBUTION} is the
|
|
45
|
-
* credit the model card
|
|
45
|
+
* credit the model card records.
|
|
46
46
|
*/
|
|
47
47
|
|
|
48
48
|
import { formatAddressRow } from "@mailwoman/codex/address/format"
|
|
@@ -90,7 +90,7 @@ export const ReteaScolaraRefusal = {
|
|
|
90
90
|
*/
|
|
91
91
|
LocalityAbsent: "row:locality-absent",
|
|
92
92
|
/**
|
|
93
|
-
* The unit states neither a street nor a house number, so the locality
|
|
93
|
+
* The unit states neither a street nor a house number, so the locality by itself would be a place name.
|
|
94
94
|
*/
|
|
95
95
|
PremiseAbsent: "row:premise-absent",
|
|
96
96
|
/**
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
import { APIClient } from "@mailwoman/core/api"
|
|
16
16
|
import { ByteFormatter } from "@mailwoman/core/fs/formatters"
|
|
17
|
-
import { tryStat } from "@mailwoman/core/fs/readers"
|
|
17
|
+
import { tryStat } from "@mailwoman/core/fs/readers/stat"
|
|
18
18
|
import { makeDirectories } from "@mailwoman/core/fs/writers"
|
|
19
19
|
import { sha256File } from "@mailwoman/core/hash"
|
|
20
20
|
import { PathBuilder, type PathBuilderLike } from "path-ts"
|
package/lib/runner.ts
CHANGED
|
@@ -97,7 +97,7 @@ export interface RunAdapterOptions {
|
|
|
97
97
|
/**
|
|
98
98
|
* Base-2 logarithm of the fingerprint table's slot count.
|
|
99
99
|
*
|
|
100
|
-
*
|
|
100
|
+
* @defaultValue {@linkcode DEFAULT_DEDUP_SLOTS_LOG2}
|
|
101
101
|
* A test sets it low to keep the allocation small.
|
|
102
102
|
*/
|
|
103
103
|
dedupSlotsLog2?: number
|
|
@@ -131,8 +131,9 @@ export interface RunAdapterOptions {
|
|
|
131
131
|
/**
|
|
132
132
|
* Yielded-row interval at which `onProgress` fires.
|
|
133
133
|
*
|
|
134
|
-
* Defaults to 1000 rows per callback.
|
|
135
134
|
* The runner always emits a terminal update.
|
|
135
|
+
*
|
|
136
|
+
* @defaultValue `1000` rows per callback.
|
|
136
137
|
*/
|
|
137
138
|
progressEvery?: number
|
|
138
139
|
}
|
|
@@ -194,7 +195,7 @@ export interface AdapterRunManifest {
|
|
|
194
195
|
export async function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunManifest> {
|
|
195
196
|
const { adapter, outputDir, corpusVersion } = opts
|
|
196
197
|
// A caller that passes its own map reads the counts back from it.
|
|
197
|
-
// Otherwise the runner supplies one, so every manifest
|
|
198
|
+
// Otherwise the runner supplies one, so every manifest records the adapter's drop reasons.
|
|
198
199
|
const dropped = opts.adapterOptions.dropped ?? new Map<string, number>()
|
|
199
200
|
const adapterOptions: AdapterOptions = { ...opts.adapterOptions, dropped }
|
|
200
201
|
const progressEvery = opts.progressEvery ?? 1000
|
|
@@ -125,7 +125,7 @@ export function renderSGRegister(row: SGRow, random: () => number, register?: SG
|
|
|
125
125
|
const street = random() < P_ABBREVIATE ? abbreviateSGStreet(street0) : street0
|
|
126
126
|
const drawn = random()
|
|
127
127
|
const building = isBuildingName(row.unit) ? titleCaseIfUpper(row.unit) : null
|
|
128
|
-
const draw = register
|
|
128
|
+
const draw = register ? REGISTER_DRAW[register] : drawn
|
|
129
129
|
|
|
130
130
|
let raw: string
|
|
131
131
|
let rendered: SGRegister
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import { BYTES_PER_KIB } from "@mailwoman/core/fs/formatters"
|
|
11
|
-
import { pathExists } from "@mailwoman/core/fs/readers"
|
|
11
|
+
import { pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
12
12
|
import { makeDirectories } from "@mailwoman/core/fs/writers"
|
|
13
13
|
import { sha256File } from "@mailwoman/core/hash"
|
|
14
14
|
import { sleep } from "@mailwoman/core/utils/sleep"
|
|
@@ -60,7 +60,7 @@ async function readJSON<T>(url: string): Promise<T> {
|
|
|
60
60
|
*
|
|
61
61
|
* The function returns `undefined` when polling runs out of attempts.
|
|
62
62
|
*/
|
|
63
|
-
async function signedURLFor(datasetID: string): Promise<string |
|
|
63
|
+
async function signedURLFor(datasetID: string): Promise<string | null> {
|
|
64
64
|
await readJSON(`${DOWNLOAD_API}/${datasetID}/initiate-download`)
|
|
65
65
|
|
|
66
66
|
for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt++) {
|
|
@@ -70,7 +70,7 @@ async function signedURLFor(datasetID: string): Promise<string | undefined> {
|
|
|
70
70
|
if (poll.data?.status === "DOWNLOAD_SUCCESS" && poll.data.url) return poll.data.url
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
-
return
|
|
73
|
+
return null
|
|
74
74
|
}
|
|
75
75
|
|
|
76
76
|
function filenameFor(name: string, datasetID: string): string {
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
*
|
|
16
16
|
* The reference form is a WFS stored-query URL: an `ad:component` href is
|
|
17
17
|
* `…/ad/ows?…&storedQuery_id=urn:ogc:def:query:OGC-WFS::GetFeatureById&id=AdminUnitName.199`, and
|
|
18
|
-
* that `id` parameter equals the target's `gml:id` exactly
|
|
18
|
+
* that `id` parameter equals the target's `gml:id` exactly. A `GetFeatureById` fetch of
|
|
19
19
|
* `AdminUnitName.199` confirms. `componentJoinKey` reads it. Each reference also repeats its target's
|
|
20
20
|
* value in `xlink:title`, as Czechia's does, and the adapter reads none of those titles. Three of an
|
|
21
21
|
* address's references are `ad:AdminUnitName` features whose titles are a region, a district and a
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
* 1,000 times, `buildingIdentifier` 334 times, and the street count was also exactly 334.
|
|
38
38
|
*
|
|
39
39
|
* The same publisher serves its register's own rendered address through `ra:address` on the same
|
|
40
|
-
* GeoServer
|
|
40
|
+
* GeoServer. That rendering pins the composition. Three records, matched by street, number and postcode:
|
|
41
41
|
*
|
|
42
42
|
* | `addressNumber` | `buildingIdentifier` | street | `fulladdress` |
|
|
43
43
|
* | --- | --- | --- | --- |
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
* | `142` | (absent) | (absent) | `142,Sikenička,94359` |
|
|
47
47
|
*
|
|
48
48
|
* So the number is `addressNumber` by itself, or `addressNumber/buildingIdentifier` where the
|
|
49
|
-
* orientation number exists
|
|
49
|
+
* orientation number exists. That is the publisher's own `annotation` field verbatim. A street-less
|
|
50
50
|
* address is ordinary rather than a defect: 666 of those 1,000 referenced no `ad:ThoroughfareName`.
|
|
51
51
|
*
|
|
52
52
|
* The place names come from two types. `ad:AdminUnitName` at `4thOrder` is the municipality and is
|
|
@@ -155,19 +155,20 @@ const MUNICIPALITY_LEVEL = "4thOrder"
|
|
|
155
155
|
*/
|
|
156
156
|
interface ReferencedValue {
|
|
157
157
|
/**
|
|
158
|
-
* The feature's element name
|
|
158
|
+
* The feature's element name.
|
|
159
|
+
* It decides where its value goes.
|
|
159
160
|
*/
|
|
160
161
|
readonly type: string
|
|
161
162
|
|
|
162
163
|
/**
|
|
163
|
-
* The street name, place name or postcode the feature states.
|
|
164
|
+
* The street name, place name or postcode the feature states, null when it states none.
|
|
164
165
|
*/
|
|
165
|
-
readonly value
|
|
166
|
+
readonly value: string | null
|
|
166
167
|
|
|
167
168
|
/**
|
|
168
169
|
* The administrative level an `ad:AdminUnitName` states, such as `4thOrder`.
|
|
169
170
|
*/
|
|
170
|
-
readonly level
|
|
171
|
+
readonly level: string | null
|
|
171
172
|
}
|
|
172
173
|
|
|
173
174
|
/**
|
|
@@ -175,18 +176,18 @@ interface ReferencedValue {
|
|
|
175
176
|
*/
|
|
176
177
|
function referencedValue(feature: MarkupElement): ReferencedValue {
|
|
177
178
|
if (feature.name === "ad:ThoroughfareName") {
|
|
178
|
-
return { type: feature.name, value: thoroughfareName(feature) }
|
|
179
|
+
return { type: feature.name, value: thoroughfareName(feature), level: null }
|
|
179
180
|
}
|
|
180
181
|
|
|
181
182
|
if (feature.name === "ad:PostalDescriptor") {
|
|
182
|
-
return { type: feature.name, value: postalDescriptorCode(feature) }
|
|
183
|
+
return { type: feature.name, value: postalDescriptorCode(feature), level: null }
|
|
183
184
|
}
|
|
184
185
|
|
|
185
186
|
if (feature.name === "ad:AdminUnitName") {
|
|
186
187
|
return { type: feature.name, value: placeName(feature), level: adminUnitLevel(feature) }
|
|
187
188
|
}
|
|
188
189
|
|
|
189
|
-
return { type: feature.name, value: placeName(feature) }
|
|
190
|
+
return { type: feature.name, value: placeName(feature), level: null }
|
|
190
191
|
}
|
|
191
192
|
|
|
192
193
|
/**
|
|
@@ -196,12 +197,12 @@ function referencedValue(feature: MarkupElement): ReferencedValue {
|
|
|
196
197
|
* number, and the two are joined by a solidus where it does.
|
|
197
198
|
* The table in this module's header reads that off three records of the publisher's `ra:address` layer.
|
|
198
199
|
*
|
|
199
|
-
* @returns The number, or
|
|
200
|
+
* @returns The number, or null when the publisher states no `addressNumber`.
|
|
200
201
|
*/
|
|
201
|
-
export function slovakHouseNumber(byType: ReadonlyMap<string, readonly string[]>): string |
|
|
202
|
+
export function slovakHouseNumber(byType: ReadonlyMap<string, readonly string[]>): string | null {
|
|
202
203
|
const descriptive = designator(byType, "addressNumber")
|
|
203
204
|
|
|
204
|
-
if (!descriptive) return
|
|
205
|
+
if (!descriptive) return null
|
|
205
206
|
|
|
206
207
|
const orientation = designator(byType, "buildingIdentifier")
|
|
207
208
|
|
|
@@ -273,26 +274,23 @@ export function createSKInspireAdapter(): CorpusAdapter {
|
|
|
273
274
|
}
|
|
274
275
|
|
|
275
276
|
/**
|
|
276
|
-
* One address's row, or `
|
|
277
|
+
* One address's row, or `null` when it holds too little to render.
|
|
277
278
|
*
|
|
278
279
|
* The component index is complete before this is called, so a reference that
|
|
279
280
|
* resolves to no feature raises rather than being deferred.
|
|
280
281
|
*/
|
|
281
|
-
function composeRow(
|
|
282
|
-
address:
|
|
283
|
-
referenced: ReadonlyMap<string, ReferencedValue>
|
|
284
|
-
): CanonicalRow | undefined {
|
|
285
|
-
const addressID = address.attributes["gml:id"]?.trim()
|
|
282
|
+
function composeRow(address: MarkupElement, referenced: ReadonlyMap<string, ReferencedValue>): CanonicalRow | null {
|
|
283
|
+
const addressID = address.attributes["gml:id"]?.trim() ?? null
|
|
286
284
|
const voided = voidDesignatorTypes(address)
|
|
287
285
|
|
|
288
286
|
for (const type of ["addressNumber", "buildingIdentifier"]) {
|
|
289
287
|
if (voided.has(type)) throw new VoidDesignatorError(SK_INSPIRE_ADAPTER_ID, addressID, type)
|
|
290
288
|
}
|
|
291
289
|
|
|
292
|
-
let street: string |
|
|
293
|
-
let postcode: string |
|
|
294
|
-
let municipality: string |
|
|
295
|
-
let settlement: string |
|
|
290
|
+
let street: string | null = null
|
|
291
|
+
let postcode: string | null = null
|
|
292
|
+
let municipality: string | null = null
|
|
293
|
+
let settlement: string | null = null
|
|
296
294
|
|
|
297
295
|
for (const href of componentHrefs(address)) {
|
|
298
296
|
const key = componentJoinKey(href)
|
|
@@ -313,7 +311,7 @@ function composeRow(
|
|
|
313
311
|
|
|
314
312
|
const place = municipality ?? settlement
|
|
315
313
|
|
|
316
|
-
if (!place) return
|
|
314
|
+
if (!place) return null
|
|
317
315
|
|
|
318
316
|
const components: CanonicalRow["components"] = {}
|
|
319
317
|
const house = slovakHouseNumber(designatorsByType(address))
|
|
@@ -326,8 +324,8 @@ function composeRow(
|
|
|
326
324
|
components.street = street
|
|
327
325
|
}
|
|
328
326
|
|
|
329
|
-
// The settlement part is a dependent locality only where it
|
|
330
|
-
//
|
|
327
|
+
// The settlement part is a dependent locality only where it differs from the
|
|
328
|
+
// municipality. 101 of 426 address-area references do.
|
|
331
329
|
if (settlement && municipality && settlement !== municipality) {
|
|
332
330
|
components.dependent_locality = settlement
|
|
333
331
|
}
|
|
@@ -340,7 +338,7 @@ function composeRow(
|
|
|
340
338
|
|
|
341
339
|
const rendered = formatAddressRow(components, "SK", { singleLine: true })
|
|
342
340
|
|
|
343
|
-
if (!rendered) return
|
|
341
|
+
if (!rendered) return null
|
|
344
342
|
|
|
345
343
|
const { raw, components: aligned } = rendered
|
|
346
344
|
|