@mailwoman/corpus 10.1.0 → 10.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/licenses/cdla-permissive-2.0.md +1 -1
- package/lib/adapters/cordis/adapter.ts +13 -7
- package/lib/adapters/dedup-key.ts +1 -1
- package/lib/adapters/geonames/adapter.ts +1 -1
- package/lib/adapters/gleif/adapter.ts +17 -16
- package/lib/adapters/libpostal-dictionary.ts +0 -2
- package/lib/adapters/openaddresses/adapter.ts +4 -3
- package/lib/adapters/osm/adapter.ts +22 -21
- package/lib/adapters/po-box/adapter.ts +10 -5
- package/lib/adapters/registry.ts +4 -4
- package/lib/adapters/source-id.ts +2 -2
- package/lib/adapters/street-line.ts +7 -6
- package/lib/adapters/wof/admin/json/adapter.ts +2 -2
- package/lib/adapters/wof/json-rows.ts +2 -1
- package/lib/adapters/wof/postalcode-json/adapter.ts +6 -2
- package/lib/adapters.ts +5 -5
- package/lib/au/adapters/gnaf/adapter.ts +1 -1
- package/lib/be/adapters/brussels/adapter.ts +22 -22
- package/lib/be/adapters/vlaanderen/adapter.ts +54 -57
- package/lib/be/adapters/wallonie/adapter.ts +17 -17
- package/lib/be/tools/fetch/brussels.ts +13 -10
- package/lib/be/tools/fetch/wallonie.ts +8 -7
- package/lib/build/adapters.ts +2 -1
- package/lib/build/align.ts +5 -4
- package/lib/build/checkpoint.ts +5 -3
- package/lib/build/eligibility.ts +3 -3
- package/lib/build/types.ts +8 -6
- package/lib/cz/adapters/cuzk/adapter.ts +41 -38
- package/lib/cz/tools/fetch/cuzk.ts +42 -31
- package/lib/de/recipes/locale.ts +1 -1
- package/lib/dk/adapters/inspire/adapter.ts +38 -38
- package/lib/dk/tools/fetch/inspire.ts +16 -14
- package/lib/ee/adapters/ads/adapter.ts +26 -32
- package/lib/ee/tools/fetch/ads.ts +10 -8
- package/lib/es/adapters/bizkaia/adapter.ts +47 -47
- package/lib/es/adapters/catastro/adapter.ts +36 -34
- package/lib/es/adapters/gipuzkoa/adapter.ts +38 -45
- package/lib/es/adapters/navarra/adapter.ts +28 -27
- package/lib/es/cadastre.ts +31 -31
- package/lib/es/tools/fetch/bizkaia.ts +40 -36
- package/lib/es/tools/fetch/catastro.ts +51 -40
- package/lib/es/tools/fetch/gipuzkoa.ts +13 -11
- package/lib/es/tools/fetch/navarra.ts +33 -25
- package/lib/fi/adapters/ryhti/adapter.ts +22 -26
- package/lib/fi/tools/fetch/ryhti.ts +31 -31
- package/lib/fr/adapters/annuaire-education/adapter.ts +13 -11
- package/lib/fr/adapters/ban/adapter.ts +6 -6
- package/lib/fr/adapters/finess/adapter.ts +16 -15
- package/lib/fr/adapters/finess/voie-type.ts +4 -3
- package/lib/fr/insee-country.ts +2 -2
- package/lib/fr/recipes/lieudit.ts +8 -6
- package/lib/fr/tools/fetch/annuaire-education.ts +11 -8
- package/lib/fr/tools/fetch/ban.ts +3 -2
- package/lib/fr/tools/fetch/finess.ts +14 -13
- package/lib/inspire/address.ts +86 -78
- package/lib/inspire/errors.ts +9 -8
- package/lib/inspire/stream.ts +29 -26
- package/lib/international/recipes/locale.ts +13 -9
- package/lib/it/adapters/anac/adapter.ts +51 -46
- package/lib/it/tools/fetch/anac.ts +22 -25
- package/lib/jp/adapters/wof-admin/adapter.ts +2 -6
- package/lib/jp/tools/fetch/houjin.ts +3 -3
- package/lib/kr/adapters/localdata/permit-rows.ts +1 -1
- package/lib/kr/tools/fetch/juso.ts +1 -1
- package/lib/kr/tools/fetch/localdata.ts +2 -2
- package/lib/lu/adapters/bd-adresses/adapter.ts +10 -10
- package/lib/lu/tools/fetch/bd-adresses.ts +15 -13
- package/lib/lv/adapters/varis/adapter.ts +24 -23
- package/lib/lv/tools/fetch/varis.ts +3 -2
- package/lib/nl/adapters/kadaster/adapter.ts +29 -29
- package/lib/nl/tools/fetch/kadaster.ts +16 -12
- package/lib/no/adapters/matrikkelen/adapter.ts +6 -5
- package/lib/no/tools/fetch/matrikkelen.ts +59 -47
- package/lib/parquet/duckdb.ts +7 -4
- package/lib/parquet/readers.ts +3 -3
- package/lib/parquet/schema.ts +1 -1
- package/lib/parquet/writers.ts +3 -2
- package/lib/pl/adapters/emuia/adapter.ts +12 -12
- package/lib/pl/tools/fetch/emuia.ts +10 -9
- package/lib/recipes/bare/postcode.ts +1 -1
- package/lib/recipes/boundary-stress.ts +1 -1
- package/lib/recipes/house-venue.ts +7 -3
- package/lib/recipes/intersection.ts +2 -2
- package/lib/recipes/po/box/cedex/recipe.ts +3 -3
- package/lib/recipes/po/box/cedex/sources.ts +13 -1
- package/lib/recipes/scaffold.ts +31 -21
- package/lib/recipes/sources.ts +16 -3
- package/lib/recipes/street/bare.ts +2 -2
- package/lib/recipes/street.ts +3 -1
- package/lib/recipes/sub/venue/context.ts +2 -0
- package/lib/recipes/sub/venue/render.ts +1 -1
- package/lib/recipes/sub/venue/sources.ts +24 -29
- package/lib/recipes/sub/venue.ts +12 -18
- package/lib/recipes/trailing-region.ts +6 -2
- package/lib/recipes/unit.ts +3 -3
- package/lib/recipes/venue.ts +347 -0
- package/lib/recipes.ts +4 -2
- package/lib/registers.ts +8 -6
- package/lib/ro/adapters/retea-scolara/adapter.ts +5 -5
- package/lib/ro/tools/fetch/retea-scolara.ts +1 -1
- package/lib/runner.ts +4 -3
- package/lib/sg/recipes/register.ts +1 -1
- package/lib/sg/tools/fetch/acra.ts +3 -3
- package/lib/sk/adapters/inspire/adapter.ts +25 -27
- package/lib/sk/tools/fetch/inspire.ts +15 -14
- package/lib/source-register/effective-manifest.ts +3 -3
- package/lib/source-register/register.ts +3 -3
- package/lib/source-register/training-manifest.ts +1 -1
- package/lib/source-register/types.ts +12 -13
- package/lib/subvenue/promotions.ts +1 -1
- package/lib/subvenue/table.ts +4 -4
- package/lib/surfaces/locale.ts +2 -0
- package/lib/synthesizers/anchor-absorption.ts +2 -1
- package/lib/synthesizers/boundary-stress.ts +20 -26
- package/lib/synthesizers/house-venue.ts +26 -30
- package/lib/synthesizers/no-street.ts +2 -1
- package/lib/synthesizers/po-box.ts +6 -3
- package/lib/synthesizers/tail.ts +55 -0
- package/lib/synthesizers/utils.ts +19 -2
- package/lib/test-kit.ts +2 -2
- package/lib/tools/audit.ts +7 -5
- package/lib/tools/corpus-stats.ts +1 -1
- package/lib/tools/fetch/atom.ts +10 -10
- package/lib/tools/fetch/cordis.ts +11 -9
- package/lib/tools/fetch/download/manifest.ts +2 -1
- package/lib/tools/fetch/download/network.ts +4 -4
- package/lib/tools/fetch/download/retry.ts +4 -4
- package/lib/tools/fetch/geonames/dump.ts +5 -3
- package/lib/tools/fetch/geonames/postal.ts +2 -2
- package/lib/tools/fetch/gleif.ts +6 -4
- package/lib/tools/fetch/header.ts +8 -6
- package/lib/tools/fetch/inspire-addresses.ts +41 -36
- package/lib/tools/fetch/openaddresses.ts +1 -1
- package/lib/tools/fetch/vlaanderen-ad.ts +17 -17
- package/lib/tools/fetch/wfs-harvest.ts +39 -28
- package/lib/tools/fetch.ts +0 -42
- package/lib/tools/golden/expand.ts +8 -8
- package/lib/tools/golden/promote.ts +12 -8
- package/lib/tools/golden/relabel-street/directory.ts +7 -6
- package/lib/tools/golden/relabel-street/row.ts +7 -6
- package/lib/tools/ingest-csv.ts +3 -3
- package/lib/tools/jsonl-to-parquet.ts +1 -1
- package/lib/tools/lint/recipe-output/vocab.ts +10 -5
- package/lib/tools/lint/recipe-output.ts +26 -8
- package/lib/tools/overlay/kryptonite.ts +4 -3
- package/lib/tools/overlay/manifest.ts +4 -2
- package/lib/tools/overlay/merge-source.ts +2 -1
- package/lib/tools/overlay/split-slice.ts +2 -2
- package/lib/tools/overlay/translit.ts +6 -4
- package/lib/tools/overture-subvenue.ts +1 -1
- package/lib/tools/postcode-triples.ts +5 -4
- package/lib/tools/source-register/build.ts +17 -14
- package/lib/tools/sub/venue/lexicon.ts +4 -8
- package/lib/tw/tools/fetch/gcis.ts +4 -4
- package/lib/types.ts +4 -3
- package/lib/us/adapters/state/hi-schools/workbook.ts +1 -1
- package/lib/us/adapters/usgov/nad/adapter.ts +11 -11
- package/lib/us/adapters/usgov/samhsa-treatment-locator/adapter.ts +4 -4
- package/lib/us/fips-state.ts +2 -2
- package/lib/{synthesizers → us/synthesizers}/intersection.ts +12 -13
- package/lib/{synthesizers → us/synthesizers}/street.ts +16 -15
- package/lib/us/tools/fetch/imls-pls.ts +1 -1
- package/lib/us/tools/fetch/nad.ts +7 -7
- package/lib/us/tools/fetch/nppes.ts +5 -5
- package/lib/us/tools/fetch/state/hi-schools.ts +3 -2
- package/lib/us/tools/fetch/tiger-full.ts +1 -1
- package/lib/utils/align.ts +13 -10
- package/lib/utils/golden.ts +2 -2
- package/lib/utils/license.ts +3 -3
- package/lib/utils/split.ts +19 -15
- package/lib/utils/tokenize.ts +7 -26
- package/lib/utils/wof-json.ts +4 -4
- package/out/adapters/cordis/adapter.d.ts +5 -3
- package/out/adapters/cordis/adapter.d.ts.map +1 -1
- package/out/adapters/cordis/adapter.js +13 -7
- package/out/adapters/cordis/adapter.js.map +1 -1
- package/out/adapters/dedup-key.d.ts +1 -1
- package/out/adapters/dedup-key.js +1 -1
- package/out/adapters/geonames/adapter.js +1 -1
- package/out/adapters/geonames/adapter.js.map +1 -1
- package/out/adapters/gleif/adapter.d.ts +9 -9
- package/out/adapters/gleif/adapter.d.ts.map +1 -1
- package/out/adapters/gleif/adapter.js +15 -14
- package/out/adapters/gleif/adapter.js.map +1 -1
- package/out/adapters/libpostal-dictionary.d.ts +0 -2
- package/out/adapters/libpostal-dictionary.d.ts.map +1 -1
- package/out/adapters/libpostal-dictionary.js +0 -2
- package/out/adapters/libpostal-dictionary.js.map +1 -1
- package/out/adapters/openaddresses/adapter.d.ts +3 -2
- package/out/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/adapters/openaddresses/adapter.js +1 -1
- package/out/adapters/openaddresses/adapter.js.map +1 -1
- package/out/adapters/osm/adapter.d.ts +17 -16
- package/out/adapters/osm/adapter.d.ts.map +1 -1
- package/out/adapters/osm/adapter.js +10 -9
- package/out/adapters/osm/adapter.js.map +1 -1
- package/out/adapters/po-box/adapter.d.ts +6 -3
- package/out/adapters/po-box/adapter.d.ts.map +1 -1
- package/out/adapters/po-box/adapter.js +4 -2
- package/out/adapters/po-box/adapter.js.map +1 -1
- package/out/adapters/registry.d.ts +3 -3
- package/out/adapters/registry.d.ts.map +1 -1
- package/out/adapters/registry.js +1 -1
- package/out/adapters/registry.js.map +1 -1
- package/out/adapters/source-id.d.ts +2 -2
- package/out/adapters/source-id.js +2 -2
- package/out/adapters/street-line.d.ts +7 -6
- package/out/adapters/street-line.d.ts.map +1 -1
- package/out/adapters/street-line.js +7 -6
- package/out/adapters/street-line.js.map +1 -1
- package/out/adapters/wof/admin/json/adapter.js +1 -1
- package/out/adapters/wof/admin/json/adapter.js.map +1 -1
- package/out/adapters/wof/json-rows.d.ts +2 -1
- package/out/adapters/wof/json-rows.d.ts.map +1 -1
- package/out/adapters/wof/json-rows.js.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.d.ts.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.js +5 -1
- package/out/adapters/wof/postalcode-json/adapter.js.map +1 -1
- package/out/adapters.d.ts +5 -5
- package/out/adapters.js +5 -5
- package/out/au/adapters/gnaf/adapter.js +1 -1
- package/out/au/adapters/gnaf/adapter.js.map +1 -1
- package/out/be/adapters/brussels/adapter.d.ts +9 -9
- package/out/be/adapters/brussels/adapter.d.ts.map +1 -1
- package/out/be/adapters/brussels/adapter.js +21 -21
- package/out/be/adapters/brussels/adapter.js.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.d.ts +34 -37
- package/out/be/adapters/vlaanderen/adapter.d.ts.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.js +40 -38
- package/out/be/adapters/vlaanderen/adapter.js.map +1 -1
- package/out/be/adapters/wallonie/adapter.d.ts +2 -2
- package/out/be/adapters/wallonie/adapter.js +17 -17
- package/out/be/adapters/wallonie/adapter.js.map +1 -1
- package/out/be/tools/fetch/brussels.d.ts +11 -8
- package/out/be/tools/fetch/brussels.d.ts.map +1 -1
- package/out/be/tools/fetch/brussels.js +10 -8
- package/out/be/tools/fetch/brussels.js.map +1 -1
- package/out/be/tools/fetch/wallonie.d.ts +6 -5
- package/out/be/tools/fetch/wallonie.d.ts.map +1 -1
- package/out/be/tools/fetch/wallonie.js +4 -4
- package/out/be/tools/fetch/wallonie.js.map +1 -1
- package/out/build/adapters.d.ts.map +1 -1
- package/out/build/adapters.js +2 -1
- package/out/build/adapters.js.map +1 -1
- package/out/build/align.d.ts +3 -2
- package/out/build/align.d.ts.map +1 -1
- package/out/build/align.js +2 -2
- package/out/build/align.js.map +1 -1
- package/out/build/checkpoint.d.ts +1 -1
- package/out/build/checkpoint.d.ts.map +1 -1
- package/out/build/checkpoint.js +4 -2
- package/out/build/checkpoint.js.map +1 -1
- package/out/build/eligibility.d.ts +2 -2
- package/out/build/eligibility.js +3 -3
- package/out/build/eligibility.js.map +1 -1
- package/out/build/types.d.ts +8 -6
- package/out/build/types.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.d.ts +14 -14
- package/out/cz/adapters/cuzk/adapter.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.js +38 -35
- package/out/cz/adapters/cuzk/adapter.js.map +1 -1
- package/out/cz/tools/fetch/cuzk.d.ts +31 -22
- package/out/cz/tools/fetch/cuzk.d.ts.map +1 -1
- package/out/cz/tools/fetch/cuzk.js +21 -19
- package/out/cz/tools/fetch/cuzk.js.map +1 -1
- package/out/de/recipes/locale.js +1 -1
- package/out/de/recipes/locale.js.map +1 -1
- package/out/dk/adapters/inspire/adapter.d.ts +32 -33
- package/out/dk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/dk/adapters/inspire/adapter.js +38 -38
- package/out/dk/adapters/inspire/adapter.js.map +1 -1
- package/out/dk/tools/fetch/inspire.d.ts +6 -5
- package/out/dk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/dk/tools/fetch/inspire.js +15 -13
- package/out/dk/tools/fetch/inspire.js.map +1 -1
- package/out/ee/adapters/ads/adapter.d.ts +19 -26
- package/out/ee/adapters/ads/adapter.d.ts.map +1 -1
- package/out/ee/adapters/ads/adapter.js +22 -28
- package/out/ee/adapters/ads/adapter.js.map +1 -1
- package/out/ee/tools/fetch/ads.d.ts +3 -3
- package/out/ee/tools/fetch/ads.d.ts.map +1 -1
- package/out/ee/tools/fetch/ads.js +10 -8
- package/out/ee/tools/fetch/ads.js.map +1 -1
- package/out/es/adapters/bizkaia/adapter.d.ts +28 -28
- package/out/es/adapters/bizkaia/adapter.d.ts.map +1 -1
- package/out/es/adapters/bizkaia/adapter.js +43 -43
- package/out/es/adapters/bizkaia/adapter.js.map +1 -1
- package/out/es/adapters/catastro/adapter.d.ts +22 -23
- package/out/es/adapters/catastro/adapter.d.ts.map +1 -1
- package/out/es/adapters/catastro/adapter.js +35 -33
- package/out/es/adapters/catastro/adapter.js.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.d.ts +31 -38
- package/out/es/adapters/gipuzkoa/adapter.d.ts.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.js +40 -47
- package/out/es/adapters/gipuzkoa/adapter.js.map +1 -1
- package/out/es/adapters/navarra/adapter.d.ts +20 -19
- package/out/es/adapters/navarra/adapter.d.ts.map +1 -1
- package/out/es/adapters/navarra/adapter.js +23 -22
- package/out/es/adapters/navarra/adapter.js.map +1 -1
- package/out/es/cadastre.d.ts +21 -20
- package/out/es/cadastre.d.ts.map +1 -1
- package/out/es/cadastre.js +19 -21
- package/out/es/cadastre.js.map +1 -1
- package/out/es/tools/fetch/bizkaia.d.ts +28 -25
- package/out/es/tools/fetch/bizkaia.d.ts.map +1 -1
- package/out/es/tools/fetch/bizkaia.js +25 -24
- package/out/es/tools/fetch/bizkaia.js.map +1 -1
- package/out/es/tools/fetch/catastro.d.ts +28 -20
- package/out/es/tools/fetch/catastro.d.ts.map +1 -1
- package/out/es/tools/fetch/catastro.js +36 -31
- package/out/es/tools/fetch/catastro.js.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.d.ts +11 -9
- package/out/es/tools/fetch/gipuzkoa.d.ts.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.js +9 -8
- package/out/es/tools/fetch/gipuzkoa.js.map +1 -1
- package/out/es/tools/fetch/navarra.d.ts +20 -15
- package/out/es/tools/fetch/navarra.d.ts.map +1 -1
- package/out/es/tools/fetch/navarra.js +21 -16
- package/out/es/tools/fetch/navarra.js.map +1 -1
- package/out/fi/adapters/ryhti/adapter.d.ts +16 -20
- package/out/fi/adapters/ryhti/adapter.d.ts.map +1 -1
- package/out/fi/adapters/ryhti/adapter.js +21 -25
- package/out/fi/adapters/ryhti/adapter.js.map +1 -1
- package/out/fi/tools/fetch/ryhti.d.ts +24 -24
- package/out/fi/tools/fetch/ryhti.d.ts.map +1 -1
- package/out/fi/tools/fetch/ryhti.js +21 -24
- package/out/fi/tools/fetch/ryhti.js.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.d.ts +8 -8
- package/out/fr/adapters/annuaire-education/adapter.d.ts.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.js +13 -11
- package/out/fr/adapters/annuaire-education/adapter.js.map +1 -1
- package/out/fr/adapters/ban/adapter.d.ts +4 -4
- package/out/fr/adapters/ban/adapter.js +6 -6
- package/out/fr/adapters/ban/adapter.js.map +1 -1
- package/out/fr/adapters/finess/adapter.d.ts +12 -11
- package/out/fr/adapters/finess/adapter.d.ts.map +1 -1
- package/out/fr/adapters/finess/adapter.js +16 -15
- package/out/fr/adapters/finess/adapter.js.map +1 -1
- package/out/fr/adapters/finess/voie-type.d.ts +2 -2
- package/out/fr/adapters/finess/voie-type.d.ts.map +1 -1
- package/out/fr/adapters/finess/voie-type.js +4 -3
- package/out/fr/adapters/finess/voie-type.js.map +1 -1
- package/out/fr/insee-country.d.ts +2 -2
- package/out/fr/insee-country.js +2 -2
- package/out/fr/recipes/lieudit.d.ts.map +1 -1
- package/out/fr/recipes/lieudit.js +5 -4
- package/out/fr/recipes/lieudit.js.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.d.ts +7 -4
- package/out/fr/tools/fetch/annuaire-education.d.ts.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.js +9 -7
- package/out/fr/tools/fetch/annuaire-education.js.map +1 -1
- package/out/fr/tools/fetch/ban.d.ts +1 -1
- package/out/fr/tools/fetch/ban.d.ts.map +1 -1
- package/out/fr/tools/fetch/ban.js +3 -2
- package/out/fr/tools/fetch/ban.js.map +1 -1
- package/out/fr/tools/fetch/finess.d.ts +5 -5
- package/out/fr/tools/fetch/finess.d.ts.map +1 -1
- package/out/fr/tools/fetch/finess.js +13 -12
- package/out/fr/tools/fetch/finess.js.map +1 -1
- package/out/inspire/address.d.ts +62 -52
- package/out/inspire/address.d.ts.map +1 -1
- package/out/inspire/address.js +71 -60
- package/out/inspire/address.js.map +1 -1
- package/out/inspire/errors.d.ts +9 -8
- package/out/inspire/errors.d.ts.map +1 -1
- package/out/inspire/errors.js +7 -6
- package/out/inspire/errors.js.map +1 -1
- package/out/inspire/stream.d.ts +25 -22
- package/out/inspire/stream.d.ts.map +1 -1
- package/out/inspire/stream.js +10 -12
- package/out/inspire/stream.js.map +1 -1
- package/out/international/recipes/locale.d.ts +2 -2
- package/out/international/recipes/locale.d.ts.map +1 -1
- package/out/international/recipes/locale.js +11 -7
- package/out/international/recipes/locale.js.map +1 -1
- package/out/it/adapters/anac/adapter.d.ts +45 -43
- package/out/it/adapters/anac/adapter.d.ts.map +1 -1
- package/out/it/adapters/anac/adapter.js +48 -43
- package/out/it/adapters/anac/adapter.js.map +1 -1
- package/out/it/tools/fetch/anac.d.ts +16 -19
- package/out/it/tools/fetch/anac.d.ts.map +1 -1
- package/out/it/tools/fetch/anac.js +20 -23
- package/out/it/tools/fetch/anac.js.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.d.ts.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.js.map +1 -1
- package/out/jp/tools/fetch/houjin.js +2 -2
- package/out/jp/tools/fetch/houjin.js.map +1 -1
- package/out/kr/adapters/localdata/permit-rows.js +1 -1
- package/out/kr/adapters/localdata/permit-rows.js.map +1 -1
- package/out/kr/tools/fetch/juso.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.js +1 -1
- package/out/kr/tools/fetch/localdata.js.map +1 -1
- package/out/lu/adapters/bd-adresses/adapter.d.ts +9 -9
- package/out/lu/adapters/bd-adresses/adapter.js +10 -10
- package/out/lu/tools/fetch/bd-adresses.d.ts +11 -9
- package/out/lu/tools/fetch/bd-adresses.d.ts.map +1 -1
- package/out/lu/tools/fetch/bd-adresses.js +11 -11
- package/out/lu/tools/fetch/bd-adresses.js.map +1 -1
- package/out/lv/adapters/varis/adapter.d.ts +24 -23
- package/out/lv/adapters/varis/adapter.d.ts.map +1 -1
- package/out/lv/adapters/varis/adapter.js +24 -23
- package/out/lv/adapters/varis/adapter.js.map +1 -1
- package/out/lv/tools/fetch/varis.d.ts +2 -1
- package/out/lv/tools/fetch/varis.d.ts.map +1 -1
- package/out/lv/tools/fetch/varis.js +1 -1
- package/out/lv/tools/fetch/varis.js.map +1 -1
- package/out/nl/adapters/kadaster/adapter.d.ts +12 -12
- package/out/nl/adapters/kadaster/adapter.d.ts.map +1 -1
- package/out/nl/adapters/kadaster/adapter.js +22 -23
- package/out/nl/adapters/kadaster/adapter.js.map +1 -1
- package/out/nl/tools/fetch/kadaster.d.ts +14 -10
- package/out/nl/tools/fetch/kadaster.d.ts.map +1 -1
- package/out/nl/tools/fetch/kadaster.js +12 -9
- package/out/nl/tools/fetch/kadaster.js.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.js +3 -3
- package/out/no/adapters/matrikkelen/adapter.js.map +1 -1
- package/out/no/tools/fetch/matrikkelen.d.ts +50 -38
- package/out/no/tools/fetch/matrikkelen.d.ts.map +1 -1
- package/out/no/tools/fetch/matrikkelen.js +44 -36
- package/out/no/tools/fetch/matrikkelen.js.map +1 -1
- package/out/parquet/duckdb.d.ts +7 -4
- package/out/parquet/duckdb.d.ts.map +1 -1
- package/out/parquet/duckdb.js +2 -1
- package/out/parquet/duckdb.js.map +1 -1
- package/out/parquet/readers.js +3 -3
- package/out/parquet/readers.js.map +1 -1
- package/out/parquet/schema.js +1 -1
- package/out/parquet/schema.js.map +1 -1
- package/out/parquet/writers.d.ts +1 -1
- package/out/parquet/writers.d.ts.map +1 -1
- package/out/parquet/writers.js +2 -1
- package/out/parquet/writers.js.map +1 -1
- package/out/pl/adapters/emuia/adapter.d.ts +8 -8
- package/out/pl/adapters/emuia/adapter.js +12 -12
- package/out/pl/adapters/emuia/adapter.js.map +1 -1
- package/out/pl/tools/fetch/emuia.d.ts +5 -4
- package/out/pl/tools/fetch/emuia.d.ts.map +1 -1
- package/out/pl/tools/fetch/emuia.js +10 -9
- package/out/pl/tools/fetch/emuia.js.map +1 -1
- package/out/recipes/bare/postcode.js +1 -1
- package/out/recipes/bare/postcode.js.map +1 -1
- package/out/recipes/boundary-stress.js +1 -1
- package/out/recipes/boundary-stress.js.map +1 -1
- package/out/recipes/house-venue.d.ts.map +1 -1
- package/out/recipes/house-venue.js +6 -3
- package/out/recipes/house-venue.js.map +1 -1
- package/out/recipes/intersection.js +2 -2
- package/out/recipes/intersection.js.map +1 -1
- package/out/recipes/po/box/cedex/recipe.js +2 -1
- package/out/recipes/po/box/cedex/recipe.js.map +1 -1
- package/out/recipes/po/box/cedex/sources.d.ts +8 -2
- package/out/recipes/po/box/cedex/sources.d.ts.map +1 -1
- package/out/recipes/po/box/cedex/sources.js +1 -1
- package/out/recipes/po/box/cedex/sources.js.map +1 -1
- package/out/recipes/scaffold.d.ts +28 -19
- package/out/recipes/scaffold.d.ts.map +1 -1
- package/out/recipes/scaffold.js +3 -2
- package/out/recipes/scaffold.js.map +1 -1
- package/out/recipes/sources.d.ts +1 -1
- package/out/recipes/sources.d.ts.map +1 -1
- package/out/recipes/sources.js +15 -2
- package/out/recipes/sources.js.map +1 -1
- package/out/recipes/street/bare.js +2 -2
- package/out/recipes/street/bare.js.map +1 -1
- package/out/recipes/street.d.ts.map +1 -1
- package/out/recipes/street.js +3 -1
- package/out/recipes/street.js.map +1 -1
- package/out/recipes/sub/venue/context.d.ts.map +1 -1
- package/out/recipes/sub/venue/context.js +2 -0
- package/out/recipes/sub/venue/context.js.map +1 -1
- package/out/recipes/sub/venue/render.js +1 -1
- package/out/recipes/sub/venue/render.js.map +1 -1
- package/out/recipes/sub/venue/sources.d.ts +6 -3
- package/out/recipes/sub/venue/sources.d.ts.map +1 -1
- package/out/recipes/sub/venue/sources.js +16 -20
- package/out/recipes/sub/venue/sources.js.map +1 -1
- package/out/recipes/sub/venue.d.ts +2 -5
- package/out/recipes/sub/venue.d.ts.map +1 -1
- package/out/recipes/sub/venue.js +10 -14
- package/out/recipes/sub/venue.js.map +1 -1
- package/out/recipes/trailing-region.d.ts.map +1 -1
- package/out/recipes/trailing-region.js +6 -1
- package/out/recipes/trailing-region.js.map +1 -1
- package/out/recipes/unit.js +2 -2
- package/out/recipes/unit.js.map +1 -1
- package/out/recipes/venue.d.ts +119 -0
- package/out/recipes/venue.d.ts.map +1 -0
- package/out/recipes/venue.js +245 -0
- package/out/recipes/venue.js.map +1 -0
- package/out/recipes.d.ts +1 -1
- package/out/recipes.d.ts.map +1 -1
- package/out/recipes.js +3 -1
- package/out/recipes.js.map +1 -1
- package/out/registers.d.ts +8 -6
- package/out/registers.d.ts.map +1 -1
- package/out/registers.js +8 -6
- package/out/registers.js.map +1 -1
- package/out/ro/adapters/retea-scolara/adapter.d.ts +5 -5
- package/out/ro/adapters/retea-scolara/adapter.js +5 -5
- package/out/ro/tools/fetch/retea-scolara.js +1 -1
- package/out/ro/tools/fetch/retea-scolara.js.map +1 -1
- package/out/runner.d.ts +3 -2
- package/out/runner.d.ts.map +1 -1
- package/out/runner.js +1 -1
- package/out/runner.js.map +1 -1
- package/out/sg/recipes/register.js +1 -1
- package/out/sg/recipes/register.js.map +1 -1
- package/out/sg/tools/fetch/acra.js +2 -2
- package/out/sg/tools/fetch/acra.js.map +1 -1
- package/out/sk/adapters/inspire/adapter.d.ts +5 -5
- package/out/sk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/sk/adapters/inspire/adapter.js +18 -18
- package/out/sk/adapters/inspire/adapter.js.map +1 -1
- package/out/sk/tools/fetch/inspire.d.ts +11 -10
- package/out/sk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/sk/tools/fetch/inspire.js +15 -14
- package/out/sk/tools/fetch/inspire.js.map +1 -1
- package/out/source-register/effective-manifest.js +3 -3
- package/out/source-register/effective-manifest.js.map +1 -1
- package/out/source-register/register.d.ts +2 -2
- package/out/source-register/register.d.ts.map +1 -1
- package/out/source-register/register.js +2 -2
- package/out/source-register/register.js.map +1 -1
- package/out/source-register/training-manifest.d.ts +1 -1
- package/out/source-register/types.d.ts +12 -13
- package/out/source-register/types.d.ts.map +1 -1
- package/out/source-register/types.js +2 -2
- package/out/source-register/types.js.map +1 -1
- package/out/subvenue/promotions.d.ts +1 -1
- package/out/subvenue/promotions.js +1 -1
- package/out/subvenue/table.d.ts +4 -4
- package/out/subvenue/table.js +3 -3
- package/out/surfaces/locale.d.ts.map +1 -1
- package/out/surfaces/locale.js +2 -0
- package/out/surfaces/locale.js.map +1 -1
- package/out/synthesizers/anchor-absorption.d.ts +2 -1
- package/out/synthesizers/anchor-absorption.d.ts.map +1 -1
- package/out/synthesizers/anchor-absorption.js.map +1 -1
- package/out/synthesizers/boundary-stress.d.ts +1 -1
- package/out/synthesizers/boundary-stress.d.ts.map +1 -1
- package/out/synthesizers/boundary-stress.js +20 -23
- package/out/synthesizers/boundary-stress.js.map +1 -1
- package/out/synthesizers/house-venue.d.ts +3 -2
- package/out/synthesizers/house-venue.d.ts.map +1 -1
- package/out/synthesizers/house-venue.js +24 -26
- package/out/synthesizers/house-venue.js.map +1 -1
- package/out/synthesizers/no-street.d.ts.map +1 -1
- package/out/synthesizers/no-street.js.map +1 -1
- package/out/synthesizers/po-box.d.ts +4 -2
- package/out/synthesizers/po-box.d.ts.map +1 -1
- package/out/synthesizers/po-box.js.map +1 -1
- package/out/synthesizers/tail.d.ts +36 -0
- package/out/synthesizers/tail.d.ts.map +1 -0
- package/out/synthesizers/tail.js +32 -0
- package/out/synthesizers/tail.js.map +1 -0
- package/out/synthesizers/utils.d.ts +2 -1
- package/out/synthesizers/utils.d.ts.map +1 -1
- package/out/synthesizers/utils.js +17 -1
- package/out/synthesizers/utils.js.map +1 -1
- package/out/test-kit.js +2 -2
- package/out/test-kit.js.map +1 -1
- package/out/tools/audit.d.ts +2 -1
- package/out/tools/audit.d.ts.map +1 -1
- package/out/tools/audit.js +4 -3
- package/out/tools/audit.js.map +1 -1
- package/out/tools/corpus-stats.js +1 -1
- package/out/tools/corpus-stats.js.map +1 -1
- package/out/tools/fetch/atom.d.ts +8 -8
- package/out/tools/fetch/atom.d.ts.map +1 -1
- package/out/tools/fetch/atom.js +7 -8
- package/out/tools/fetch/atom.js.map +1 -1
- package/out/tools/fetch/cordis.d.ts +4 -3
- package/out/tools/fetch/cordis.d.ts.map +1 -1
- package/out/tools/fetch/cordis.js +9 -7
- package/out/tools/fetch/cordis.js.map +1 -1
- package/out/tools/fetch/download/manifest.d.ts.map +1 -1
- package/out/tools/fetch/download/manifest.js +2 -1
- package/out/tools/fetch/download/manifest.js.map +1 -1
- package/out/tools/fetch/download/network.d.ts +4 -4
- package/out/tools/fetch/download/retry.js +4 -4
- package/out/tools/fetch/download/retry.js.map +1 -1
- package/out/tools/fetch/geonames/dump.d.ts +3 -2
- package/out/tools/fetch/geonames/dump.d.ts.map +1 -1
- package/out/tools/fetch/geonames/dump.js +2 -1
- package/out/tools/fetch/geonames/dump.js.map +1 -1
- package/out/tools/fetch/geonames/postal.d.ts +2 -2
- package/out/tools/fetch/gleif.d.ts +3 -2
- package/out/tools/fetch/gleif.d.ts.map +1 -1
- package/out/tools/fetch/gleif.js +6 -4
- package/out/tools/fetch/gleif.js.map +1 -1
- package/out/tools/fetch/header.d.ts +7 -5
- package/out/tools/fetch/header.d.ts.map +1 -1
- package/out/tools/fetch/header.js +8 -6
- package/out/tools/fetch/header.js.map +1 -1
- package/out/tools/fetch/inspire-addresses.d.ts +32 -27
- package/out/tools/fetch/inspire-addresses.d.ts.map +1 -1
- package/out/tools/fetch/inspire-addresses.js +25 -24
- package/out/tools/fetch/inspire-addresses.js.map +1 -1
- package/out/tools/fetch/openaddresses.js +1 -1
- package/out/tools/fetch/openaddresses.js.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.d.ts +5 -5
- package/out/tools/fetch/vlaanderen-ad.d.ts.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.js +16 -16
- package/out/tools/fetch/vlaanderen-ad.js.map +1 -1
- package/out/tools/fetch/wfs-harvest.d.ts +26 -16
- package/out/tools/fetch/wfs-harvest.d.ts.map +1 -1
- package/out/tools/fetch/wfs-harvest.js +21 -18
- package/out/tools/fetch/wfs-harvest.js.map +1 -1
- package/out/tools/fetch.d.ts +0 -41
- package/out/tools/fetch.d.ts.map +1 -1
- package/out/tools/fetch.js +0 -41
- package/out/tools/fetch.js.map +1 -1
- package/out/tools/golden/expand.d.ts +7 -7
- package/out/tools/golden/expand.js +1 -1
- package/out/tools/golden/expand.js.map +1 -1
- package/out/tools/golden/promote.d.ts +2 -2
- package/out/tools/golden/promote.d.ts.map +1 -1
- package/out/tools/golden/promote.js +9 -6
- package/out/tools/golden/promote.js.map +1 -1
- package/out/tools/golden/relabel-street/directory.d.ts +4 -4
- package/out/tools/golden/relabel-street/directory.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/directory.js +3 -2
- package/out/tools/golden/relabel-street/directory.js.map +1 -1
- package/out/tools/golden/relabel-street/row.d.ts +4 -3
- package/out/tools/golden/relabel-street/row.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/row.js +3 -3
- package/out/tools/golden/relabel-street/row.js.map +1 -1
- package/out/tools/ingest-csv.js +3 -3
- package/out/tools/ingest-csv.js.map +1 -1
- package/out/tools/jsonl-to-parquet.d.ts +1 -1
- package/out/tools/lint/recipe-output/vocab.d.ts +10 -5
- package/out/tools/lint/recipe-output/vocab.d.ts.map +1 -1
- package/out/tools/lint/recipe-output/vocab.js.map +1 -1
- package/out/tools/lint/recipe-output.d.ts +8 -8
- package/out/tools/lint/recipe-output.d.ts.map +1 -1
- package/out/tools/lint/recipe-output.js +17 -0
- package/out/tools/lint/recipe-output.js.map +1 -1
- package/out/tools/overlay/kryptonite.d.ts +2 -2
- package/out/tools/overlay/kryptonite.d.ts.map +1 -1
- package/out/tools/overlay/kryptonite.js +2 -1
- package/out/tools/overlay/kryptonite.js.map +1 -1
- package/out/tools/overlay/manifest.d.ts +2 -1
- package/out/tools/overlay/manifest.d.ts.map +1 -1
- package/out/tools/overlay/manifest.js +2 -1
- package/out/tools/overlay/manifest.js.map +1 -1
- package/out/tools/overlay/merge-source.d.ts +2 -1
- package/out/tools/overlay/merge-source.d.ts.map +1 -1
- package/out/tools/overlay/merge-source.js.map +1 -1
- package/out/tools/overlay/split-slice.d.ts +1 -1
- package/out/tools/overlay/split-slice.js +1 -1
- package/out/tools/overlay/split-slice.js.map +1 -1
- package/out/tools/overlay/translit.d.ts +4 -3
- package/out/tools/overlay/translit.d.ts.map +1 -1
- package/out/tools/overlay/translit.js +2 -1
- package/out/tools/overlay/translit.js.map +1 -1
- package/out/tools/overture-subvenue.d.ts +1 -1
- package/out/tools/overture-subvenue.d.ts.map +1 -1
- package/out/tools/postcode-triples.d.ts +1 -1
- package/out/tools/postcode-triples.d.ts.map +1 -1
- package/out/tools/postcode-triples.js +4 -3
- package/out/tools/postcode-triples.js.map +1 -1
- package/out/tools/source-register/build.d.ts +10 -8
- package/out/tools/source-register/build.d.ts.map +1 -1
- package/out/tools/source-register/build.js +10 -8
- package/out/tools/source-register/build.js.map +1 -1
- package/out/tools/sub/venue/lexicon.d.ts +2 -6
- package/out/tools/sub/venue/lexicon.d.ts.map +1 -1
- package/out/tools/sub/venue/lexicon.js +2 -6
- package/out/tools/sub/venue/lexicon.js.map +1 -1
- package/out/tw/tools/fetch/gcis.js +3 -3
- package/out/tw/tools/fetch/gcis.js.map +1 -1
- package/out/types.d.ts +4 -3
- package/out/types.d.ts.map +1 -1
- package/out/types.js.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.js.map +1 -1
- package/out/us/adapters/usgov/nad/adapter.js +6 -6
- package/out/us/adapters/usgov/nad/adapter.js.map +1 -1
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js +2 -2
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/us/fips-state.d.ts +2 -2
- package/out/us/fips-state.d.ts.map +1 -1
- package/out/us/fips-state.js.map +1 -1
- package/out/us/synthesizers/intersection.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/intersection.js +13 -9
- package/out/us/synthesizers/intersection.js.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.d.ts +3 -2
- package/out/us/synthesizers/street.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.js +12 -12
- package/out/us/synthesizers/street.js.map +1 -0
- package/out/us/tools/fetch/imls-pls.js +1 -1
- package/out/us/tools/fetch/imls-pls.js.map +1 -1
- package/out/us/tools/fetch/nad.d.ts +6 -6
- package/out/us/tools/fetch/nad.js +1 -1
- package/out/us/tools/fetch/nad.js.map +1 -1
- package/out/us/tools/fetch/nppes.js +3 -3
- package/out/us/tools/fetch/nppes.js.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts +2 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.js +1 -1
- package/out/us/tools/fetch/state/hi-schools.js.map +1 -1
- package/out/us/tools/fetch/tiger-full.js +1 -1
- package/out/us/tools/fetch/tiger-full.js.map +1 -1
- package/out/utils/align.d.ts +5 -3
- package/out/utils/align.d.ts.map +1 -1
- package/out/utils/align.js +5 -5
- package/out/utils/align.js.map +1 -1
- package/out/utils/golden.js +2 -2
- package/out/utils/golden.js.map +1 -1
- package/out/utils/license.d.ts +3 -3
- package/out/utils/license.d.ts.map +1 -1
- package/out/utils/license.js.map +1 -1
- package/out/utils/split.d.ts +1 -1
- package/out/utils/split.d.ts.map +1 -1
- package/out/utils/split.js +18 -14
- package/out/utils/split.js.map +1 -1
- package/out/utils/tokenize.d.ts +3 -20
- package/out/utils/tokenize.d.ts.map +1 -1
- package/out/utils/tokenize.js.map +1 -1
- package/out/utils/wof-json.js +3 -3
- package/out/utils/wof-json.js.map +1 -1
- package/package.json +18 -27
- package/lib/tools.ts +0 -31
- package/out/synthesizers/intersection.d.ts.map +0 -1
- package/out/synthesizers/intersection.js.map +0 -1
- package/out/synthesizers/street.d.ts.map +0 -1
- package/out/synthesizers/street.js.map +0 -1
- package/out/tools.d.ts +0 -31
- package/out/tools.d.ts.map +0 -1
- package/out/tools.js +0 -31
- package/out/tools.js.map +0 -1
- /package/out/{synthesizers → us/synthesizers}/intersection.d.ts +0 -0
|
@@ -31,7 +31,9 @@
|
|
|
31
31
|
* ```
|
|
32
32
|
*/
|
|
33
33
|
|
|
34
|
-
import {
|
|
34
|
+
import { layoutForCountry, layoutPrintsPostcodeBeforeLocality } from "@mailwoman/codex/address/layouts"
|
|
35
|
+
import { matchCountry } from "@mailwoman/codex/country"
|
|
36
|
+
import { pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
35
37
|
import { copyFileTo, makeDirectories, writeLocalJSONFile } from "@mailwoman/core/fs/writers"
|
|
36
38
|
import { sha256File } from "@mailwoman/core/hash"
|
|
37
39
|
import { stringifyJSON } from "@mailwoman/core/json"
|
|
@@ -78,13 +80,13 @@ export interface PromoteGoldenOptions {
|
|
|
78
80
|
/**
|
|
79
81
|
* Previous version to forward-copy + dedup against.
|
|
80
82
|
*
|
|
81
|
-
*
|
|
83
|
+
* @defaultValue `v0.1.0`
|
|
82
84
|
*/
|
|
83
85
|
prior?: string
|
|
84
86
|
/**
|
|
85
87
|
* Golden dir root.
|
|
86
88
|
*
|
|
87
|
-
*
|
|
89
|
+
* @defaultValue `data/eval/golden`
|
|
88
90
|
*/
|
|
89
91
|
goldenRoot?: PathBuilderLike
|
|
90
92
|
/**
|
|
@@ -113,16 +115,18 @@ function isComponentsGlued(entry: GoldenEntry): boolean {
|
|
|
113
115
|
}
|
|
114
116
|
|
|
115
117
|
/**
|
|
116
|
-
* Heuristic:
|
|
118
|
+
* Heuristic: where the country's layout prints the postcode after the locality, a postcode in the
|
|
119
|
+
* first third of a multi-component raw with 4+ components means the LLM reordered the address.
|
|
117
120
|
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
* FR is exempt (postcode often precedes locality there).
|
|
121
|
+
* A country whose layout prints the postcode before the locality (FR, DE) is exempt,
|
|
122
|
+
* as is a country with no layout or a layout that prints one of the two.
|
|
121
123
|
*/
|
|
122
124
|
function isPostcodeBadlyLeading(entry: GoldenEntry): boolean {
|
|
123
125
|
if (Object.keys(entry.components).length < MIN_PROMOTABLE_COMPONENTS) return false
|
|
124
126
|
|
|
125
|
-
|
|
127
|
+
const layout = layoutForCountry(matchCountry(entry.country)?.iso2 ?? entry.country)
|
|
128
|
+
|
|
129
|
+
if (!layout || layoutPrintsPostcodeBeforeLocality(layout) !== false) return false
|
|
126
130
|
const postcode = entry.components.postcode
|
|
127
131
|
|
|
128
132
|
if (!postcode) return false
|
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
* @file Rewrites a golden-set directory and renders its review deck.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
import {
|
|
8
|
+
import { readLocalBuffer, readLocalTextFile } from "@mailwoman/core/fs/readers"
|
|
9
|
+
import { pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
9
10
|
import {
|
|
10
11
|
makeDirectories,
|
|
11
12
|
writeLocalFile,
|
|
@@ -65,13 +66,13 @@ export interface RelabelGoldenOptions {
|
|
|
65
66
|
/**
|
|
66
67
|
* Review-deck jsonl path.
|
|
67
68
|
*
|
|
68
|
-
*
|
|
69
|
+
* @defaultValue `<output>/review-deck.jsonl`
|
|
69
70
|
*/
|
|
70
71
|
deck?: string
|
|
71
72
|
/**
|
|
72
73
|
* Parent version label recorded in the manifest.
|
|
73
74
|
*
|
|
74
|
-
*
|
|
75
|
+
* @defaultValue the input dir's basename.
|
|
75
76
|
*/
|
|
76
77
|
parentLabel?: string
|
|
77
78
|
/**
|
|
@@ -81,13 +82,13 @@ export interface RelabelGoldenOptions {
|
|
|
81
82
|
/**
|
|
82
83
|
* Passed through to {@linkcode relabelGoldenStreetRow}.
|
|
83
84
|
*
|
|
84
|
-
*
|
|
85
|
+
* @defaultValue `true`
|
|
85
86
|
*/
|
|
86
87
|
splitPrefix?: boolean
|
|
87
88
|
}
|
|
88
89
|
|
|
89
90
|
/**
|
|
90
|
-
* Aggregate
|
|
91
|
+
* Aggregate result for a whole golden version.
|
|
91
92
|
*/
|
|
92
93
|
export interface RelabelGoldenReport {
|
|
93
94
|
files: Record<
|
|
@@ -251,7 +252,7 @@ export async function relabelGoldenDirectory(
|
|
|
251
252
|
parent: options.parentLabel ?? basename(input),
|
|
252
253
|
generated_at: new Date().toISOString(),
|
|
253
254
|
tool: "corpus/src/tools/golden-relabel-street.ts (mailwoman corpus golden-relabel)",
|
|
254
|
-
|
|
255
|
+
commit: options.commit || null,
|
|
255
256
|
convention: {
|
|
256
257
|
street_convention: { US: "split", "*": "folded" },
|
|
257
258
|
declared:
|
|
@@ -29,7 +29,7 @@ export interface GoldenStreetRow {
|
|
|
29
29
|
}
|
|
30
30
|
|
|
31
31
|
/**
|
|
32
|
-
* The relabel
|
|
32
|
+
* The relabel result class for one row.
|
|
33
33
|
*/
|
|
34
34
|
export type GoldenRelabelClass =
|
|
35
35
|
| "split-suffix"
|
|
@@ -140,8 +140,9 @@ export interface RelabelStreetRowOptions {
|
|
|
140
140
|
/**
|
|
141
141
|
* Whether to also move a leading directional into `street_prefix`.
|
|
142
142
|
*
|
|
143
|
-
*
|
|
144
|
-
*
|
|
143
|
+
* Without it `N Desmet Avenue` grades as `street: "N Desmet"` against a model that emits a prefix.
|
|
144
|
+
*
|
|
145
|
+
* @defaultValue `true`
|
|
145
146
|
*/
|
|
146
147
|
splitPrefix?: boolean
|
|
147
148
|
}
|
|
@@ -178,9 +179,9 @@ export function relabelGoldenStreetRow(
|
|
|
178
179
|
const flags: GoldenRelabelFlag[] = []
|
|
179
180
|
let rowClass: GoldenRelabelClass
|
|
180
181
|
let name = street
|
|
181
|
-
let suffix: string |
|
|
182
|
+
let suffix: string | null = null
|
|
182
183
|
let suffixGap = ""
|
|
183
|
-
let canonical: USStreetSuffix |
|
|
184
|
+
let canonical: USStreetSuffix | null = null
|
|
184
185
|
|
|
185
186
|
const split = splitLastWord(street)
|
|
186
187
|
|
|
@@ -215,7 +216,7 @@ export function relabelGoldenStreetRow(
|
|
|
215
216
|
}
|
|
216
217
|
}
|
|
217
218
|
|
|
218
|
-
let prefix: string |
|
|
219
|
+
let prefix: string | null = null
|
|
219
220
|
let prefixGap = ""
|
|
220
221
|
|
|
221
222
|
if (splitPrefix && !row.components.street_prefix) {
|
package/lib/tools/ingest-csv.ts
CHANGED
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
*/
|
|
31
31
|
|
|
32
32
|
import { ByteFormatter } from "@mailwoman/core/fs/formatters"
|
|
33
|
-
import { tryStat, pathExists } from "@mailwoman/core/fs/readers"
|
|
33
|
+
import { tryStat, pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
34
34
|
import { makeDirectories, writeLocalJSONFile } from "@mailwoman/core/fs/writers"
|
|
35
35
|
import type { SQLInputValue } from "@mailwoman/sqlite/client"
|
|
36
36
|
import type { Database } from "@mailwoman/sqlite/database-schema"
|
|
@@ -99,7 +99,7 @@ function inferColumnType(samples: (string | null)[]): ColumnInfo {
|
|
|
99
99
|
let textCount = 0
|
|
100
100
|
|
|
101
101
|
for (const s of samples) {
|
|
102
|
-
if (s
|
|
102
|
+
if (!s) {
|
|
103
103
|
nullCount++
|
|
104
104
|
|
|
105
105
|
continue
|
|
@@ -263,7 +263,7 @@ async function runIngest(opts: IngestOptions): Promise<void> {
|
|
|
263
263
|
const values = fields.map((f, i) => {
|
|
264
264
|
const v = normalizeField(f)
|
|
265
265
|
|
|
266
|
-
if (v
|
|
266
|
+
if (!v) return null
|
|
267
267
|
const col = columns[i]
|
|
268
268
|
|
|
269
269
|
if (col?.type === "INTEGER" && /^-?\d+$/.test(v)) return Number.parseInt(v, 10)
|
|
@@ -167,27 +167,32 @@ export interface LintRecipeVocabOptions {
|
|
|
167
167
|
recipeOutputPath: string
|
|
168
168
|
/**
|
|
169
169
|
* Base corpus version.
|
|
170
|
-
*
|
|
170
|
+
*
|
|
171
|
+
* @defaultValue `v0.5.0`
|
|
171
172
|
*/
|
|
172
173
|
baseVersion?: string
|
|
173
174
|
/**
|
|
174
175
|
* Base corpus root.
|
|
175
|
-
*
|
|
176
|
+
*
|
|
177
|
+
* @defaultValue `$MAILWOMAN_DATA_ROOT/corpus/versioned`
|
|
176
178
|
*/
|
|
177
179
|
baseRoot?: string
|
|
178
180
|
/**
|
|
179
181
|
* Base-majority confidence floor for a contradiction.
|
|
180
|
-
*
|
|
182
|
+
*
|
|
183
|
+
* @defaultValue `0.7`
|
|
181
184
|
*/
|
|
182
185
|
threshold?: number
|
|
183
186
|
/**
|
|
184
187
|
* Minimum base support to judge a token.
|
|
185
|
-
*
|
|
188
|
+
*
|
|
189
|
+
* @defaultValue `50`
|
|
186
190
|
*/
|
|
187
191
|
minCount?: number
|
|
188
192
|
/**
|
|
189
193
|
* Fraction of base parts to scan, a proportional per-source sample below 1.0.
|
|
190
|
-
*
|
|
194
|
+
*
|
|
195
|
+
* @defaultValue `1.0`
|
|
191
196
|
*/
|
|
192
197
|
fraction?: number
|
|
193
198
|
}
|
|
@@ -73,7 +73,7 @@ export interface LintRecipeOutputOptions {
|
|
|
73
73
|
/**
|
|
74
74
|
* Anti-pattern rules JSON.
|
|
75
75
|
*
|
|
76
|
-
*
|
|
76
|
+
* @defaultValue the `lint-rules.json` beside this module.
|
|
77
77
|
*/
|
|
78
78
|
rulesPath?: string
|
|
79
79
|
/**
|
|
@@ -173,16 +173,26 @@ function majorityLabel(distribution: Map<string, number> | Record<string, number
|
|
|
173
173
|
export interface LintFlag {
|
|
174
174
|
check: string
|
|
175
175
|
severity: "error" | "warn"
|
|
176
|
-
token
|
|
177
|
-
bigram
|
|
178
|
-
outputLabel
|
|
179
|
-
corpusLabel
|
|
180
|
-
outputCount
|
|
181
|
-
corpusCount
|
|
176
|
+
token: string | null
|
|
177
|
+
bigram: string | null
|
|
178
|
+
outputLabel: string | null
|
|
179
|
+
corpusLabel: string | null
|
|
180
|
+
outputCount: number | null
|
|
181
|
+
corpusCount: number | null
|
|
182
182
|
detail: string
|
|
183
|
-
ruleID
|
|
183
|
+
ruleID: string | null
|
|
184
184
|
}
|
|
185
185
|
|
|
186
|
+
const EMPTY_FLAG_FIELDS = {
|
|
187
|
+
token: null,
|
|
188
|
+
bigram: null,
|
|
189
|
+
outputLabel: null,
|
|
190
|
+
corpusLabel: null,
|
|
191
|
+
outputCount: null,
|
|
192
|
+
corpusCount: null,
|
|
193
|
+
ruleID: null,
|
|
194
|
+
} as const satisfies Partial<LintFlag>
|
|
195
|
+
|
|
186
196
|
/**
|
|
187
197
|
* Findings summary returned by {@linkcode lintRecipeOutput}.
|
|
188
198
|
*/
|
|
@@ -213,6 +223,7 @@ function checkDistributionOutliers(output: RecipeOutputStats, corpus: CorpusStat
|
|
|
213
223
|
corpusMaj.total >= CORPUS_MIN_COUNT
|
|
214
224
|
) {
|
|
215
225
|
flags.push({
|
|
226
|
+
...EMPTY_FLAG_FIELDS,
|
|
216
227
|
check: "distribution-outlier",
|
|
217
228
|
severity: "error",
|
|
218
229
|
token,
|
|
@@ -244,6 +255,7 @@ function checkLabelVacuum(output: RecipeOutputStats, corpus: CorpusStats): LintF
|
|
|
244
255
|
|
|
245
256
|
if (corpusLabelMap[label] === undefined || corpusLabelMap[label] === 0) {
|
|
246
257
|
flags.push({
|
|
258
|
+
...EMPTY_FLAG_FIELDS,
|
|
247
259
|
check: "label-vacuum",
|
|
248
260
|
severity: "error",
|
|
249
261
|
token,
|
|
@@ -279,6 +291,7 @@ function checkBigramCollisions(output: RecipeOutputStats, corpus: CorpusStats):
|
|
|
279
291
|
const renderCorpusLabel = corpusMaj.label.split(SEP).join(" → ")
|
|
280
292
|
|
|
281
293
|
flags.push({
|
|
294
|
+
...EMPTY_FLAG_FIELDS,
|
|
282
295
|
check: "bigram-collision",
|
|
283
296
|
severity: "error",
|
|
284
297
|
bigram: renderBigram,
|
|
@@ -309,6 +322,7 @@ function checkRules(output: RecipeOutputStats, rulesFile: LintRulesFile): LintFl
|
|
|
309
322
|
for (const [label, count] of labelMap) {
|
|
310
323
|
if (rule.forbidden_labels.includes(label) && count >= FORBIDDEN_LABEL_REPORT_THRESHOLD) {
|
|
311
324
|
flags.push({
|
|
325
|
+
...EMPTY_FLAG_FIELDS,
|
|
312
326
|
check: "anti-pattern-rule",
|
|
313
327
|
severity: rule.severity,
|
|
314
328
|
ruleID: rule.id,
|
|
@@ -330,6 +344,7 @@ function checkSanity(output: RecipeOutputStats): LintFlag[] {
|
|
|
330
344
|
|
|
331
345
|
if (output.truncatedRows > 0) {
|
|
332
346
|
flags.push({
|
|
347
|
+
...EMPTY_FLAG_FIELDS,
|
|
333
348
|
check: "truncated-rows",
|
|
334
349
|
severity: "error",
|
|
335
350
|
detail: `${output.truncatedRows} row(s) have tokens.length !== labels.length. Pipeline alignment bug.`,
|
|
@@ -340,6 +355,7 @@ function checkSanity(output: RecipeOutputStats): LintFlag[] {
|
|
|
340
355
|
|
|
341
356
|
if (allORatio >= ALL_O_RATIO_CEILING) {
|
|
342
357
|
flags.push({
|
|
358
|
+
...EMPTY_FLAG_FIELDS,
|
|
343
359
|
check: "all-O-output",
|
|
344
360
|
severity: "warn",
|
|
345
361
|
detail: `${output.allORows}/${output.rowCount} rows (${(allORatio * 100).toFixed(0)}%) are entirely O-labeled. The recipe output contributes no signal.`,
|
|
@@ -349,6 +365,8 @@ function checkSanity(output: RecipeOutputStats): LintFlag[] {
|
|
|
349
365
|
return flags
|
|
350
366
|
}
|
|
351
367
|
|
|
368
|
+
// repo-health-ignore private-name-shadows-export -- renders a recipe-lint report.
|
|
369
|
+
// The dossier export renders a building dossier, an unrelated document.
|
|
352
370
|
function renderReport(
|
|
353
371
|
opts: { outputPath: string; statsPath: string; rulesPath: string },
|
|
354
372
|
output: RecipeOutputStats,
|
|
@@ -17,7 +17,8 @@
|
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
19
|
import { delimitedSource } from "@mailwoman/core/fs/delimited"
|
|
20
|
-
import {
|
|
20
|
+
import { readLocalJSONFile } from "@mailwoman/core/fs/readers"
|
|
21
|
+
import { pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
21
22
|
import { writeLocalJSONFile, writeLocalTextFile, makeDirectories } from "@mailwoman/core/fs/writers"
|
|
22
23
|
import { PathBuilder, type PathBuilderLike } from "path-ts"
|
|
23
24
|
import { JSONSpliterator } from "spliterator"
|
|
@@ -33,11 +34,11 @@ export interface KryptoniteOverlayOptions {
|
|
|
33
34
|
baseManifest: string
|
|
34
35
|
outDir: PathBuilderLike
|
|
35
36
|
/**
|
|
36
|
-
*
|
|
37
|
+
* @defaultValue `"0.4.0"`
|
|
37
38
|
*/
|
|
38
39
|
corpusVersion?: string
|
|
39
40
|
/**
|
|
40
|
-
*
|
|
41
|
+
* @defaultValue `"deepseek-kryptonite"`
|
|
41
42
|
*/
|
|
42
43
|
source?: string
|
|
43
44
|
}
|
|
@@ -7,7 +7,8 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
10
|
-
import { readLocalBuffer, readLocalJSONFile
|
|
10
|
+
import { readLocalBuffer, readLocalJSONFile } from "@mailwoman/core/fs/readers"
|
|
11
|
+
import { statPath } from "@mailwoman/core/fs/readers/stat"
|
|
11
12
|
import { writeLocalJSONFile } from "@mailwoman/core/fs/writers"
|
|
12
13
|
import { sha256Hex } from "@mailwoman/core/hash"
|
|
13
14
|
import { stringifyJSON } from "@mailwoman/core/json"
|
|
@@ -91,10 +92,11 @@ export interface OverlayFile {
|
|
|
91
92
|
source: string
|
|
92
93
|
/**
|
|
93
94
|
* The split that the file's rows belong to.
|
|
94
|
-
* The default is `train`.
|
|
95
95
|
*
|
|
96
96
|
* `val` and `test` are allowed only for files that `splitOverlaySlice` returned,
|
|
97
97
|
* so that the holdout policy chooses held-out rows.
|
|
98
|
+
*
|
|
99
|
+
* @defaultValue `train`
|
|
98
100
|
*/
|
|
99
101
|
split?: SplitName
|
|
100
102
|
}
|
|
@@ -32,7 +32,8 @@ export interface MergeSourceOptions {
|
|
|
32
32
|
output: PathBuilderLike
|
|
33
33
|
/**
|
|
34
34
|
* The number of rows held in memory while shuffling.
|
|
35
|
-
*
|
|
35
|
+
*
|
|
36
|
+
* @defaultValue the corpus writer's window.
|
|
36
37
|
*/
|
|
37
38
|
windowSize?: number
|
|
38
39
|
/**
|
|
@@ -63,7 +63,7 @@ export function holdoutComponents(row: SpannedRow, index: number, input: string)
|
|
|
63
63
|
const components: Record<string, string> = {}
|
|
64
64
|
|
|
65
65
|
for (const [span, tag] of tags.entries()) {
|
|
66
|
-
if (!HOLDOUT_TAGS.has(tag) || components[tag]
|
|
66
|
+
if (!HOLDOUT_TAGS.has(tag) || components[tag]) continue
|
|
67
67
|
|
|
68
68
|
components[tag] = raw.slice(starts[span]!, ends[span]!)
|
|
69
69
|
}
|
|
@@ -83,7 +83,7 @@ export interface SplitSliceOptions {
|
|
|
83
83
|
*/
|
|
84
84
|
outputDir: PathBuilderLike
|
|
85
85
|
/**
|
|
86
|
-
*
|
|
86
|
+
* @defaultValue `defaultHoldouts()`, which is the policy the base build applies.
|
|
87
87
|
*/
|
|
88
88
|
holdouts?: Record<string, CountryHoldout>
|
|
89
89
|
}
|
|
@@ -30,7 +30,8 @@
|
|
|
30
30
|
|
|
31
31
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
32
32
|
import { delimitedSource } from "@mailwoman/core/fs/delimited"
|
|
33
|
-
import {
|
|
33
|
+
import { readLocalJSONFile } from "@mailwoman/core/fs/readers"
|
|
34
|
+
import { pathExists, tryStat } from "@mailwoman/core/fs/readers/stat"
|
|
34
35
|
import { writeLocalJSONFile, writeLocalTextFile, makeDirectories } from "@mailwoman/core/fs/writers"
|
|
35
36
|
import { sha256File } from "@mailwoman/core/hash"
|
|
36
37
|
import { PathBuilder, type PathBuilderLike } from "path-ts"
|
|
@@ -48,20 +49,21 @@ export interface TranslitOverlayOptions {
|
|
|
48
49
|
baseManifest: string
|
|
49
50
|
outDir: PathBuilderLike
|
|
50
51
|
/**
|
|
51
|
-
*
|
|
52
|
+
* @defaultValue `"0.4.0"`
|
|
52
53
|
*/
|
|
53
54
|
corpusVersion?: string
|
|
54
55
|
/**
|
|
55
|
-
*
|
|
56
|
+
* @defaultValue `"/data/"`
|
|
56
57
|
*/
|
|
57
58
|
canonicalPathPrefix?: string
|
|
58
59
|
/**
|
|
59
60
|
* Prefix currently used in the base manifest's file paths, to be rewritten
|
|
60
61
|
* to {@link TranslitOverlayOptions.canonicalPathPrefix}.
|
|
61
62
|
*
|
|
62
|
-
* Defaults to `dataRootPath()` with a trailing slash — the root that wrote those paths.
|
|
63
63
|
* Pass it explicitly when translating a manifest generated under a different
|
|
64
64
|
* `$MAILWOMAN_DATA_ROOT` than the one you are running with.
|
|
65
|
+
*
|
|
66
|
+
* @defaultValue `dataRootPath()` with a trailing slash — the root that wrote those paths.
|
|
65
67
|
*/
|
|
66
68
|
legacyPathPrefix?: string
|
|
67
69
|
}
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
14
14
|
import type { PathBuilderLike } from "path-ts"
|
|
15
15
|
|
|
16
|
-
import type { SubVenueHarvestRow } from "#
|
|
16
|
+
import type { SubVenueHarvestRow } from "#subvenue/harvest"
|
|
17
17
|
|
|
18
18
|
/**
|
|
19
19
|
* Overture categories selected for sub-venue designator evidence.
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
import { officialLanguagesAlpha3, regionLanguagesAlpha3 } from "@mailwoman/codex/country"
|
|
12
12
|
import { foldName } from "@mailwoman/codex/normalize"
|
|
13
13
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
14
|
-
import { assertPathExists } from "@mailwoman/core/fs/readers"
|
|
14
|
+
import { assertPathExists } from "@mailwoman/core/fs/readers/stat"
|
|
15
15
|
import { wofDatabasePath } from "@mailwoman/resolver-wof-sqlite/paths"
|
|
16
16
|
import type { WOFDatabase } from "@mailwoman/resolver-wof-sqlite/schema"
|
|
17
17
|
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
@@ -32,7 +32,7 @@ export interface PostcodeTriple {
|
|
|
32
32
|
*
|
|
33
33
|
* A recipe output whose every row begins with the locality teaches that the first segment is the locality.
|
|
34
34
|
*/
|
|
35
|
-
dependentLocality
|
|
35
|
+
dependentLocality: string | null
|
|
36
36
|
locality: string
|
|
37
37
|
region: string
|
|
38
38
|
country: string
|
|
@@ -288,6 +288,7 @@ export async function readTriplesFromParentJoin(
|
|
|
288
288
|
for (const region of surfaces.region(cc, row.region_id, row.region)) {
|
|
289
289
|
out.push({
|
|
290
290
|
postcode: row.postcode,
|
|
291
|
+
dependentLocality: null,
|
|
291
292
|
locality,
|
|
292
293
|
region,
|
|
293
294
|
country: row.country ?? "",
|
|
@@ -429,7 +430,7 @@ export async function readPairsFromAdmin(
|
|
|
429
430
|
// The recipe drops that pair, so filtering it here keeps the country budget for rows that remain.
|
|
430
431
|
if (region === locality) continue
|
|
431
432
|
|
|
432
|
-
out.push({ locality, region, country: row.country ?? "", cc, locale })
|
|
433
|
+
out.push({ dependentLocality: null, locality, region, country: row.country ?? "", cc, locale })
|
|
433
434
|
}
|
|
434
435
|
}
|
|
435
436
|
}
|
|
@@ -534,7 +535,7 @@ export async function readTriplesFromGeonames(
|
|
|
534
535
|
|
|
535
536
|
out.push({
|
|
536
537
|
postcode,
|
|
537
|
-
|
|
538
|
+
dependentLocality: dep || null,
|
|
538
539
|
locality,
|
|
539
540
|
region,
|
|
540
541
|
country: countryName,
|
|
@@ -8,7 +8,8 @@
|
|
|
8
8
|
* The second input is mostly not input: the eight global discovery lookups repeat once per jurisdiction and are dropped, because they name a lookup rather than a national source.
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
import {
|
|
11
|
+
import { readLocalJSONFile } from "@mailwoman/core/fs/readers"
|
|
12
|
+
import { pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
12
13
|
import { writeLocalFile } from "@mailwoman/core/fs/writers"
|
|
13
14
|
import { prettyJSON, stringifyJSON } from "@mailwoman/core/json"
|
|
14
15
|
import type { PathBuilderLike } from "path-ts"
|
|
@@ -262,7 +263,7 @@ export interface BuildSourceRegisterOptions {
|
|
|
262
263
|
sourceVersion?: string
|
|
263
264
|
}
|
|
264
265
|
|
|
265
|
-
function required(value: string | undefined, field: string, row: number): string {
|
|
266
|
+
function required(value: string | null | undefined, field: string, row: number): string {
|
|
266
267
|
const trimmed = (value ?? "").trim()
|
|
267
268
|
|
|
268
269
|
if (!trimmed) throw new Error(`row ${row}: ${field} is empty, and the register has nowhere to put an unnamed value`)
|
|
@@ -543,17 +544,17 @@ async function readSources(
|
|
|
543
544
|
const UNRESOLVED_COLUMN_PLACEHOLDERS: ReadonlySet<string> = new Set(["varies", "country-specific", "unknown", "n/a"])
|
|
544
545
|
|
|
545
546
|
/**
|
|
546
|
-
* A column's value, or `
|
|
547
|
+
* A column's value, or `null` when the research pass left it unresolved.
|
|
547
548
|
*
|
|
548
549
|
* Every value outside the placeholder set is returned, so a value somebody fills in
|
|
549
550
|
* later reaches the register or fails the build rather than being lost.
|
|
550
551
|
*/
|
|
551
|
-
export function readUnresolvedColumn(value: string | undefined, column: string, row: number): string |
|
|
552
|
+
export function readUnresolvedColumn(value: string | null | undefined, column: string, row: number): string | null {
|
|
552
553
|
const trimmed = (value ?? "").trim()
|
|
553
554
|
|
|
554
|
-
if (!trimmed) return
|
|
555
|
+
if (!trimmed) return null
|
|
555
556
|
|
|
556
|
-
if (UNRESOLVED_COLUMN_PLACEHOLDERS.has(trimmed.toLowerCase())) return
|
|
557
|
+
if (UNRESOLVED_COLUMN_PLACEHOLDERS.has(trimmed.toLowerCase())) return null
|
|
557
558
|
|
|
558
559
|
if (column === "address_role" && !ADDRESS_ROLE_BY_NAME[trimmed.toLowerCase()]) {
|
|
559
560
|
throw new Error(
|
|
@@ -590,8 +591,8 @@ type RecordedDecision = Omit<ElectedLicense, "licenseID"> | Omit<RefusedLicense,
|
|
|
590
591
|
* One publisher can hold a license over several jurisdictions, and the register
|
|
591
592
|
* scopes a decision to one publisher in one jurisdiction, so reading that publisher's
|
|
592
593
|
* terms once produces several decisions with the same body.
|
|
593
|
-
*
|
|
594
|
-
*
|
|
594
|
+
* A shared body keeps each license id's own decision while the prose behind it has one home.
|
|
595
|
+
* Nine copies cannot drift apart under later editing.
|
|
595
596
|
*/
|
|
596
597
|
interface SharedDecisionReference {
|
|
597
598
|
sameAs: string
|
|
@@ -613,7 +614,8 @@ function isSharedReference(value: RecordedDecision | SharedDecisionReference): v
|
|
|
613
614
|
}
|
|
614
615
|
|
|
615
616
|
/**
|
|
616
|
-
* The source fields a review resolves
|
|
617
|
+
* The source fields a review resolves.
|
|
618
|
+
* The research CSV records placeholders for them.
|
|
617
619
|
*
|
|
618
620
|
* The CSV holds the research pass's findings and is a working document of that pass,
|
|
619
621
|
* so a later review records its own findings here rather than by editing the pass's record.
|
|
@@ -652,7 +654,8 @@ async function readSourceResolutions(
|
|
|
652
654
|
*
|
|
653
655
|
* @throws When a resolution's source id is absent from the register.
|
|
654
656
|
* Such an entry is a typo or a source that has been removed.
|
|
655
|
-
*
|
|
657
|
+
* An entry kept silently would leave a review whose fields reach no row.
|
|
658
|
+
* That reads as work already done.
|
|
656
659
|
*/
|
|
657
660
|
export function applySourceResolutions(
|
|
658
661
|
sources: readonly AddressSourceRecord[],
|
|
@@ -683,7 +686,7 @@ export function applySourceResolutions(
|
|
|
683
686
|
* because reading it as empty would silently drop somebody's recorded work.
|
|
684
687
|
* This function validates no decision it reads, because `auditAddressSourceRegister` already does.
|
|
685
688
|
*/
|
|
686
|
-
async function readLicenseDecisions(decisionsPath: PathBuilderLike |
|
|
689
|
+
async function readLicenseDecisions(decisionsPath: PathBuilderLike | null): Promise<Map<string, LicenseDecision>> {
|
|
687
690
|
if (!decisionsPath || !(await pathExists(decisionsPath))) return new Map()
|
|
688
691
|
|
|
689
692
|
return resolveRecordedDecisions(await readLocalJSONFile<LicenseDecisionsFile>(decisionsPath))
|
|
@@ -697,8 +700,8 @@ async function readLicenseDecisions(decisionsPath: PathBuilderLike | undefined):
|
|
|
697
700
|
* function checks only that the file declares a reading for each reference.
|
|
698
701
|
*
|
|
699
702
|
* @throws When a `sameAs` reads a reading absent from `sharedReadings`.
|
|
700
|
-
* Such a reference would otherwise produce a decision
|
|
701
|
-
* would report as an unknown review state rather than as the typo it is.
|
|
703
|
+
* Such a reference would otherwise produce a decision holding no terms.
|
|
704
|
+
* The audit would report that as an unknown review state rather than as the typo it is.
|
|
702
705
|
*/
|
|
703
706
|
export function resolveRecordedDecisions(file: LicenseDecisionsFile): Map<string, LicenseDecision> {
|
|
704
707
|
const shared = file.sharedReadings ?? {}
|
|
@@ -764,7 +767,7 @@ export async function buildSourceRegister(options: BuildSourceRegisterOptions):
|
|
|
764
767
|
})
|
|
765
768
|
.toSorted((left, right) => left.licenseID.localeCompare(right.licenseID))
|
|
766
769
|
|
|
767
|
-
const licenses = applyLicenseDecisions(generated, await readLicenseDecisions(options.decisionsPath))
|
|
770
|
+
const licenses = applyLicenseDecisions(generated, await readLicenseDecisions(options.decisionsPath ?? null))
|
|
768
771
|
|
|
769
772
|
const register: AddressSourceRegister = {
|
|
770
773
|
registerID: "address-source-register",
|
|
@@ -6,7 +6,8 @@
|
|
|
6
6
|
* @file Builds the sub-venue designator lexicon from Wikidata, OSM and Overture fetch outputs.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
-
import { readLocalJSONFile
|
|
9
|
+
import { readLocalJSONFile } from "@mailwoman/core/fs/readers"
|
|
10
|
+
import { statPath } from "@mailwoman/core/fs/readers/stat"
|
|
10
11
|
import { writeLocalFile } from "@mailwoman/core/fs/writers"
|
|
11
12
|
import { prettyJSON } from "@mailwoman/core/json"
|
|
12
13
|
import { isoDate } from "@mailwoman/core/utils"
|
|
@@ -32,12 +33,6 @@ import {
|
|
|
32
33
|
import { deriveHeadNounSurfaces } from "#tools/sub/venue/head-nouns"
|
|
33
34
|
import { surfacesFromWikidata } from "#tools/sub/venue/wikidata"
|
|
34
35
|
|
|
35
|
-
export * from "#subvenue/harvest"
|
|
36
|
-
export * from "#tools/sub/venue/head-nouns"
|
|
37
|
-
export * from "#subvenue/surfaces"
|
|
38
|
-
export * from "#subvenue/table"
|
|
39
|
-
export * from "#tools/sub/venue/wikidata"
|
|
40
|
-
|
|
41
36
|
/**
|
|
42
37
|
* Returns a copy of the surfaces with `curated: true` set on each surface that a promotion matches.
|
|
43
38
|
*
|
|
@@ -115,9 +110,10 @@ export interface BuildSubVenueLexiconInput {
|
|
|
115
110
|
*/
|
|
116
111
|
sources: readonly SubVenueLexiconSource[]
|
|
117
112
|
/**
|
|
118
|
-
* Defaults to {@link SUBVENUE_PROMOTIONS}.
|
|
119
113
|
*
|
|
120
114
|
* An empty array builds the uncurated table.
|
|
115
|
+
*
|
|
116
|
+
* @defaultValue {@linkcode SUBVENUE_PROMOTIONS}
|
|
121
117
|
*/
|
|
122
118
|
promotions?: readonly SubVenuePromotion[]
|
|
123
119
|
}
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
*/
|
|
26
26
|
|
|
27
27
|
import { BYTES_PER_KIB } from "@mailwoman/core/fs/formatters"
|
|
28
|
-
import { pathExists } from "@mailwoman/core/fs/readers"
|
|
28
|
+
import { pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
29
29
|
import { makeDirectories } from "@mailwoman/core/fs/writers"
|
|
30
30
|
import { sha256File } from "@mailwoman/core/hash"
|
|
31
31
|
import { sleep } from "@mailwoman/core/utils/sleep"
|
|
@@ -84,14 +84,14 @@ async function listBulkDatasets(): Promise<Dataset[]> {
|
|
|
84
84
|
/**
|
|
85
85
|
* The `/od/file?oid=…` link a dataset's detail page hands its download dialog.
|
|
86
86
|
*/
|
|
87
|
-
async function fileURLFor(dataset: Dataset): Promise<string |
|
|
87
|
+
async function fileURLFor(dataset: Dataset): Promise<string | null> {
|
|
88
88
|
const res = await fetch(`${PORTAL}${dataset.detailPath}`, { headers: { accept: "text/html" } })
|
|
89
89
|
|
|
90
|
-
if (!res.ok) return
|
|
90
|
+
if (!res.ok) return null
|
|
91
91
|
const html = await res.text()
|
|
92
92
|
const path = /showDialog\('(\/od\/file\?oid=[^']+)'\)/.exec(html)?.[1]
|
|
93
93
|
|
|
94
|
-
return path ? `${PORTAL}${path}` :
|
|
94
|
+
return path ? `${PORTAL}${path}` : null
|
|
95
95
|
}
|
|
96
96
|
|
|
97
97
|
export async function fetchGCISTW(options: FetchGCISTWOptions, report?: (line: string) => void): Promise<FetchSummary> {
|