@mailwoman/corpus 10.1.0 → 10.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/licenses/cdla-permissive-2.0.md +1 -1
- package/lib/adapters/cordis/adapter.ts +13 -7
- package/lib/adapters/dedup-key.ts +1 -1
- package/lib/adapters/geonames/adapter.ts +1 -1
- package/lib/adapters/gleif/adapter.ts +17 -16
- package/lib/adapters/libpostal-dictionary.ts +0 -2
- package/lib/adapters/openaddresses/adapter.ts +4 -3
- package/lib/adapters/osm/adapter.ts +22 -21
- package/lib/adapters/po-box/adapter.ts +10 -5
- package/lib/adapters/registry.ts +4 -4
- package/lib/adapters/source-id.ts +2 -2
- package/lib/adapters/street-line.ts +7 -6
- package/lib/adapters/wof/admin/json/adapter.ts +2 -2
- package/lib/adapters/wof/json-rows.ts +2 -1
- package/lib/adapters/wof/postalcode-json/adapter.ts +6 -2
- package/lib/adapters.ts +5 -5
- package/lib/au/adapters/gnaf/adapter.ts +1 -1
- package/lib/be/adapters/brussels/adapter.ts +22 -22
- package/lib/be/adapters/vlaanderen/adapter.ts +54 -57
- package/lib/be/adapters/wallonie/adapter.ts +17 -17
- package/lib/be/tools/fetch/brussels.ts +13 -10
- package/lib/be/tools/fetch/wallonie.ts +8 -7
- package/lib/build/adapters.ts +2 -1
- package/lib/build/align.ts +5 -4
- package/lib/build/checkpoint.ts +5 -3
- package/lib/build/eligibility.ts +3 -3
- package/lib/build/types.ts +8 -6
- package/lib/cz/adapters/cuzk/adapter.ts +41 -38
- package/lib/cz/tools/fetch/cuzk.ts +42 -31
- package/lib/de/recipes/locale.ts +1 -1
- package/lib/dk/adapters/inspire/adapter.ts +38 -38
- package/lib/dk/tools/fetch/inspire.ts +16 -14
- package/lib/ee/adapters/ads/adapter.ts +26 -32
- package/lib/ee/tools/fetch/ads.ts +10 -8
- package/lib/es/adapters/bizkaia/adapter.ts +47 -47
- package/lib/es/adapters/catastro/adapter.ts +36 -34
- package/lib/es/adapters/gipuzkoa/adapter.ts +38 -45
- package/lib/es/adapters/navarra/adapter.ts +28 -27
- package/lib/es/cadastre.ts +31 -31
- package/lib/es/tools/fetch/bizkaia.ts +40 -36
- package/lib/es/tools/fetch/catastro.ts +51 -40
- package/lib/es/tools/fetch/gipuzkoa.ts +13 -11
- package/lib/es/tools/fetch/navarra.ts +33 -25
- package/lib/fi/adapters/ryhti/adapter.ts +22 -26
- package/lib/fi/tools/fetch/ryhti.ts +31 -31
- package/lib/fr/adapters/annuaire-education/adapter.ts +13 -11
- package/lib/fr/adapters/ban/adapter.ts +6 -6
- package/lib/fr/adapters/finess/adapter.ts +16 -15
- package/lib/fr/adapters/finess/voie-type.ts +4 -3
- package/lib/fr/insee-country.ts +2 -2
- package/lib/fr/recipes/lieudit.ts +8 -6
- package/lib/fr/tools/fetch/annuaire-education.ts +11 -8
- package/lib/fr/tools/fetch/ban.ts +3 -2
- package/lib/fr/tools/fetch/finess.ts +14 -13
- package/lib/inspire/address.ts +86 -78
- package/lib/inspire/errors.ts +9 -8
- package/lib/inspire/stream.ts +29 -26
- package/lib/international/recipes/locale.ts +13 -9
- package/lib/it/adapters/anac/adapter.ts +51 -46
- package/lib/it/tools/fetch/anac.ts +22 -25
- package/lib/jp/adapters/wof-admin/adapter.ts +2 -6
- package/lib/jp/tools/fetch/houjin.ts +3 -3
- package/lib/kr/adapters/localdata/permit-rows.ts +1 -1
- package/lib/kr/tools/fetch/juso.ts +1 -1
- package/lib/kr/tools/fetch/localdata.ts +2 -2
- package/lib/lu/adapters/bd-adresses/adapter.ts +10 -10
- package/lib/lu/tools/fetch/bd-adresses.ts +15 -13
- package/lib/lv/adapters/varis/adapter.ts +24 -23
- package/lib/lv/tools/fetch/varis.ts +3 -2
- package/lib/nl/adapters/kadaster/adapter.ts +29 -29
- package/lib/nl/tools/fetch/kadaster.ts +16 -12
- package/lib/no/adapters/matrikkelen/adapter.ts +6 -5
- package/lib/no/tools/fetch/matrikkelen.ts +59 -47
- package/lib/parquet/duckdb.ts +7 -4
- package/lib/parquet/readers.ts +3 -3
- package/lib/parquet/schema.ts +1 -1
- package/lib/parquet/writers.ts +3 -2
- package/lib/pl/adapters/emuia/adapter.ts +12 -12
- package/lib/pl/tools/fetch/emuia.ts +10 -9
- package/lib/recipes/bare/postcode.ts +1 -1
- package/lib/recipes/boundary-stress.ts +1 -1
- package/lib/recipes/house-venue.ts +7 -3
- package/lib/recipes/intersection.ts +2 -2
- package/lib/recipes/po/box/cedex/recipe.ts +3 -3
- package/lib/recipes/po/box/cedex/sources.ts +13 -1
- package/lib/recipes/scaffold.ts +31 -21
- package/lib/recipes/sources.ts +16 -3
- package/lib/recipes/street/bare.ts +2 -2
- package/lib/recipes/street.ts +3 -1
- package/lib/recipes/sub/venue/context.ts +2 -0
- package/lib/recipes/sub/venue/render.ts +1 -1
- package/lib/recipes/sub/venue/sources.ts +24 -29
- package/lib/recipes/sub/venue.ts +12 -18
- package/lib/recipes/trailing-region.ts +6 -2
- package/lib/recipes/unit.ts +3 -3
- package/lib/recipes/venue.ts +347 -0
- package/lib/recipes.ts +4 -2
- package/lib/registers.ts +8 -6
- package/lib/ro/adapters/retea-scolara/adapter.ts +5 -5
- package/lib/ro/tools/fetch/retea-scolara.ts +1 -1
- package/lib/runner.ts +4 -3
- package/lib/sg/recipes/register.ts +1 -1
- package/lib/sg/tools/fetch/acra.ts +3 -3
- package/lib/sk/adapters/inspire/adapter.ts +25 -27
- package/lib/sk/tools/fetch/inspire.ts +15 -14
- package/lib/source-register/effective-manifest.ts +3 -3
- package/lib/source-register/register.ts +3 -3
- package/lib/source-register/training-manifest.ts +1 -1
- package/lib/source-register/types.ts +12 -13
- package/lib/subvenue/promotions.ts +1 -1
- package/lib/subvenue/table.ts +4 -4
- package/lib/surfaces/locale.ts +2 -0
- package/lib/synthesizers/anchor-absorption.ts +2 -1
- package/lib/synthesizers/boundary-stress.ts +20 -26
- package/lib/synthesizers/house-venue.ts +26 -30
- package/lib/synthesizers/no-street.ts +2 -1
- package/lib/synthesizers/po-box.ts +6 -3
- package/lib/synthesizers/tail.ts +55 -0
- package/lib/synthesizers/utils.ts +19 -2
- package/lib/test-kit.ts +2 -2
- package/lib/tools/audit.ts +7 -5
- package/lib/tools/corpus-stats.ts +1 -1
- package/lib/tools/fetch/atom.ts +10 -10
- package/lib/tools/fetch/cordis.ts +11 -9
- package/lib/tools/fetch/download/manifest.ts +2 -1
- package/lib/tools/fetch/download/network.ts +4 -4
- package/lib/tools/fetch/download/retry.ts +4 -4
- package/lib/tools/fetch/geonames/dump.ts +5 -3
- package/lib/tools/fetch/geonames/postal.ts +2 -2
- package/lib/tools/fetch/gleif.ts +6 -4
- package/lib/tools/fetch/header.ts +8 -6
- package/lib/tools/fetch/inspire-addresses.ts +41 -36
- package/lib/tools/fetch/openaddresses.ts +1 -1
- package/lib/tools/fetch/vlaanderen-ad.ts +17 -17
- package/lib/tools/fetch/wfs-harvest.ts +39 -28
- package/lib/tools/fetch.ts +0 -42
- package/lib/tools/golden/expand.ts +8 -8
- package/lib/tools/golden/promote.ts +12 -8
- package/lib/tools/golden/relabel-street/directory.ts +7 -6
- package/lib/tools/golden/relabel-street/row.ts +7 -6
- package/lib/tools/ingest-csv.ts +3 -3
- package/lib/tools/jsonl-to-parquet.ts +1 -1
- package/lib/tools/lint/recipe-output/vocab.ts +10 -5
- package/lib/tools/lint/recipe-output.ts +26 -8
- package/lib/tools/overlay/kryptonite.ts +4 -3
- package/lib/tools/overlay/manifest.ts +4 -2
- package/lib/tools/overlay/merge-source.ts +2 -1
- package/lib/tools/overlay/split-slice.ts +2 -2
- package/lib/tools/overlay/translit.ts +6 -4
- package/lib/tools/overture-subvenue.ts +1 -1
- package/lib/tools/postcode-triples.ts +5 -4
- package/lib/tools/source-register/build.ts +17 -14
- package/lib/tools/sub/venue/lexicon.ts +4 -8
- package/lib/tw/tools/fetch/gcis.ts +4 -4
- package/lib/types.ts +4 -3
- package/lib/us/adapters/state/hi-schools/workbook.ts +1 -1
- package/lib/us/adapters/usgov/nad/adapter.ts +11 -11
- package/lib/us/adapters/usgov/samhsa-treatment-locator/adapter.ts +4 -4
- package/lib/us/fips-state.ts +2 -2
- package/lib/{synthesizers → us/synthesizers}/intersection.ts +12 -13
- package/lib/{synthesizers → us/synthesizers}/street.ts +16 -15
- package/lib/us/tools/fetch/imls-pls.ts +1 -1
- package/lib/us/tools/fetch/nad.ts +7 -7
- package/lib/us/tools/fetch/nppes.ts +5 -5
- package/lib/us/tools/fetch/state/hi-schools.ts +3 -2
- package/lib/us/tools/fetch/tiger-full.ts +1 -1
- package/lib/utils/align.ts +13 -10
- package/lib/utils/golden.ts +2 -2
- package/lib/utils/license.ts +3 -3
- package/lib/utils/split.ts +19 -15
- package/lib/utils/tokenize.ts +7 -26
- package/lib/utils/wof-json.ts +4 -4
- package/out/adapters/cordis/adapter.d.ts +5 -3
- package/out/adapters/cordis/adapter.d.ts.map +1 -1
- package/out/adapters/cordis/adapter.js +13 -7
- package/out/adapters/cordis/adapter.js.map +1 -1
- package/out/adapters/dedup-key.d.ts +1 -1
- package/out/adapters/dedup-key.js +1 -1
- package/out/adapters/geonames/adapter.js +1 -1
- package/out/adapters/geonames/adapter.js.map +1 -1
- package/out/adapters/gleif/adapter.d.ts +9 -9
- package/out/adapters/gleif/adapter.d.ts.map +1 -1
- package/out/adapters/gleif/adapter.js +15 -14
- package/out/adapters/gleif/adapter.js.map +1 -1
- package/out/adapters/libpostal-dictionary.d.ts +0 -2
- package/out/adapters/libpostal-dictionary.d.ts.map +1 -1
- package/out/adapters/libpostal-dictionary.js +0 -2
- package/out/adapters/libpostal-dictionary.js.map +1 -1
- package/out/adapters/openaddresses/adapter.d.ts +3 -2
- package/out/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/adapters/openaddresses/adapter.js +1 -1
- package/out/adapters/openaddresses/adapter.js.map +1 -1
- package/out/adapters/osm/adapter.d.ts +17 -16
- package/out/adapters/osm/adapter.d.ts.map +1 -1
- package/out/adapters/osm/adapter.js +10 -9
- package/out/adapters/osm/adapter.js.map +1 -1
- package/out/adapters/po-box/adapter.d.ts +6 -3
- package/out/adapters/po-box/adapter.d.ts.map +1 -1
- package/out/adapters/po-box/adapter.js +4 -2
- package/out/adapters/po-box/adapter.js.map +1 -1
- package/out/adapters/registry.d.ts +3 -3
- package/out/adapters/registry.d.ts.map +1 -1
- package/out/adapters/registry.js +1 -1
- package/out/adapters/registry.js.map +1 -1
- package/out/adapters/source-id.d.ts +2 -2
- package/out/adapters/source-id.js +2 -2
- package/out/adapters/street-line.d.ts +7 -6
- package/out/adapters/street-line.d.ts.map +1 -1
- package/out/adapters/street-line.js +7 -6
- package/out/adapters/street-line.js.map +1 -1
- package/out/adapters/wof/admin/json/adapter.js +1 -1
- package/out/adapters/wof/admin/json/adapter.js.map +1 -1
- package/out/adapters/wof/json-rows.d.ts +2 -1
- package/out/adapters/wof/json-rows.d.ts.map +1 -1
- package/out/adapters/wof/json-rows.js.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.d.ts.map +1 -1
- package/out/adapters/wof/postalcode-json/adapter.js +5 -1
- package/out/adapters/wof/postalcode-json/adapter.js.map +1 -1
- package/out/adapters.d.ts +5 -5
- package/out/adapters.js +5 -5
- package/out/au/adapters/gnaf/adapter.js +1 -1
- package/out/au/adapters/gnaf/adapter.js.map +1 -1
- package/out/be/adapters/brussels/adapter.d.ts +9 -9
- package/out/be/adapters/brussels/adapter.d.ts.map +1 -1
- package/out/be/adapters/brussels/adapter.js +21 -21
- package/out/be/adapters/brussels/adapter.js.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.d.ts +34 -37
- package/out/be/adapters/vlaanderen/adapter.d.ts.map +1 -1
- package/out/be/adapters/vlaanderen/adapter.js +40 -38
- package/out/be/adapters/vlaanderen/adapter.js.map +1 -1
- package/out/be/adapters/wallonie/adapter.d.ts +2 -2
- package/out/be/adapters/wallonie/adapter.js +17 -17
- package/out/be/adapters/wallonie/adapter.js.map +1 -1
- package/out/be/tools/fetch/brussels.d.ts +11 -8
- package/out/be/tools/fetch/brussels.d.ts.map +1 -1
- package/out/be/tools/fetch/brussels.js +10 -8
- package/out/be/tools/fetch/brussels.js.map +1 -1
- package/out/be/tools/fetch/wallonie.d.ts +6 -5
- package/out/be/tools/fetch/wallonie.d.ts.map +1 -1
- package/out/be/tools/fetch/wallonie.js +4 -4
- package/out/be/tools/fetch/wallonie.js.map +1 -1
- package/out/build/adapters.d.ts.map +1 -1
- package/out/build/adapters.js +2 -1
- package/out/build/adapters.js.map +1 -1
- package/out/build/align.d.ts +3 -2
- package/out/build/align.d.ts.map +1 -1
- package/out/build/align.js +2 -2
- package/out/build/align.js.map +1 -1
- package/out/build/checkpoint.d.ts +1 -1
- package/out/build/checkpoint.d.ts.map +1 -1
- package/out/build/checkpoint.js +4 -2
- package/out/build/checkpoint.js.map +1 -1
- package/out/build/eligibility.d.ts +2 -2
- package/out/build/eligibility.js +3 -3
- package/out/build/eligibility.js.map +1 -1
- package/out/build/types.d.ts +8 -6
- package/out/build/types.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.d.ts +14 -14
- package/out/cz/adapters/cuzk/adapter.d.ts.map +1 -1
- package/out/cz/adapters/cuzk/adapter.js +38 -35
- package/out/cz/adapters/cuzk/adapter.js.map +1 -1
- package/out/cz/tools/fetch/cuzk.d.ts +31 -22
- package/out/cz/tools/fetch/cuzk.d.ts.map +1 -1
- package/out/cz/tools/fetch/cuzk.js +21 -19
- package/out/cz/tools/fetch/cuzk.js.map +1 -1
- package/out/de/recipes/locale.js +1 -1
- package/out/de/recipes/locale.js.map +1 -1
- package/out/dk/adapters/inspire/adapter.d.ts +32 -33
- package/out/dk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/dk/adapters/inspire/adapter.js +38 -38
- package/out/dk/adapters/inspire/adapter.js.map +1 -1
- package/out/dk/tools/fetch/inspire.d.ts +6 -5
- package/out/dk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/dk/tools/fetch/inspire.js +15 -13
- package/out/dk/tools/fetch/inspire.js.map +1 -1
- package/out/ee/adapters/ads/adapter.d.ts +19 -26
- package/out/ee/adapters/ads/adapter.d.ts.map +1 -1
- package/out/ee/adapters/ads/adapter.js +22 -28
- package/out/ee/adapters/ads/adapter.js.map +1 -1
- package/out/ee/tools/fetch/ads.d.ts +3 -3
- package/out/ee/tools/fetch/ads.d.ts.map +1 -1
- package/out/ee/tools/fetch/ads.js +10 -8
- package/out/ee/tools/fetch/ads.js.map +1 -1
- package/out/es/adapters/bizkaia/adapter.d.ts +28 -28
- package/out/es/adapters/bizkaia/adapter.d.ts.map +1 -1
- package/out/es/adapters/bizkaia/adapter.js +43 -43
- package/out/es/adapters/bizkaia/adapter.js.map +1 -1
- package/out/es/adapters/catastro/adapter.d.ts +22 -23
- package/out/es/adapters/catastro/adapter.d.ts.map +1 -1
- package/out/es/adapters/catastro/adapter.js +35 -33
- package/out/es/adapters/catastro/adapter.js.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.d.ts +31 -38
- package/out/es/adapters/gipuzkoa/adapter.d.ts.map +1 -1
- package/out/es/adapters/gipuzkoa/adapter.js +40 -47
- package/out/es/adapters/gipuzkoa/adapter.js.map +1 -1
- package/out/es/adapters/navarra/adapter.d.ts +20 -19
- package/out/es/adapters/navarra/adapter.d.ts.map +1 -1
- package/out/es/adapters/navarra/adapter.js +23 -22
- package/out/es/adapters/navarra/adapter.js.map +1 -1
- package/out/es/cadastre.d.ts +21 -20
- package/out/es/cadastre.d.ts.map +1 -1
- package/out/es/cadastre.js +19 -21
- package/out/es/cadastre.js.map +1 -1
- package/out/es/tools/fetch/bizkaia.d.ts +28 -25
- package/out/es/tools/fetch/bizkaia.d.ts.map +1 -1
- package/out/es/tools/fetch/bizkaia.js +25 -24
- package/out/es/tools/fetch/bizkaia.js.map +1 -1
- package/out/es/tools/fetch/catastro.d.ts +28 -20
- package/out/es/tools/fetch/catastro.d.ts.map +1 -1
- package/out/es/tools/fetch/catastro.js +36 -31
- package/out/es/tools/fetch/catastro.js.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.d.ts +11 -9
- package/out/es/tools/fetch/gipuzkoa.d.ts.map +1 -1
- package/out/es/tools/fetch/gipuzkoa.js +9 -8
- package/out/es/tools/fetch/gipuzkoa.js.map +1 -1
- package/out/es/tools/fetch/navarra.d.ts +20 -15
- package/out/es/tools/fetch/navarra.d.ts.map +1 -1
- package/out/es/tools/fetch/navarra.js +21 -16
- package/out/es/tools/fetch/navarra.js.map +1 -1
- package/out/fi/adapters/ryhti/adapter.d.ts +16 -20
- package/out/fi/adapters/ryhti/adapter.d.ts.map +1 -1
- package/out/fi/adapters/ryhti/adapter.js +21 -25
- package/out/fi/adapters/ryhti/adapter.js.map +1 -1
- package/out/fi/tools/fetch/ryhti.d.ts +24 -24
- package/out/fi/tools/fetch/ryhti.d.ts.map +1 -1
- package/out/fi/tools/fetch/ryhti.js +21 -24
- package/out/fi/tools/fetch/ryhti.js.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.d.ts +8 -8
- package/out/fr/adapters/annuaire-education/adapter.d.ts.map +1 -1
- package/out/fr/adapters/annuaire-education/adapter.js +13 -11
- package/out/fr/adapters/annuaire-education/adapter.js.map +1 -1
- package/out/fr/adapters/ban/adapter.d.ts +4 -4
- package/out/fr/adapters/ban/adapter.js +6 -6
- package/out/fr/adapters/ban/adapter.js.map +1 -1
- package/out/fr/adapters/finess/adapter.d.ts +12 -11
- package/out/fr/adapters/finess/adapter.d.ts.map +1 -1
- package/out/fr/adapters/finess/adapter.js +16 -15
- package/out/fr/adapters/finess/adapter.js.map +1 -1
- package/out/fr/adapters/finess/voie-type.d.ts +2 -2
- package/out/fr/adapters/finess/voie-type.d.ts.map +1 -1
- package/out/fr/adapters/finess/voie-type.js +4 -3
- package/out/fr/adapters/finess/voie-type.js.map +1 -1
- package/out/fr/insee-country.d.ts +2 -2
- package/out/fr/insee-country.js +2 -2
- package/out/fr/recipes/lieudit.d.ts.map +1 -1
- package/out/fr/recipes/lieudit.js +5 -4
- package/out/fr/recipes/lieudit.js.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.d.ts +7 -4
- package/out/fr/tools/fetch/annuaire-education.d.ts.map +1 -1
- package/out/fr/tools/fetch/annuaire-education.js +9 -7
- package/out/fr/tools/fetch/annuaire-education.js.map +1 -1
- package/out/fr/tools/fetch/ban.d.ts +1 -1
- package/out/fr/tools/fetch/ban.d.ts.map +1 -1
- package/out/fr/tools/fetch/ban.js +3 -2
- package/out/fr/tools/fetch/ban.js.map +1 -1
- package/out/fr/tools/fetch/finess.d.ts +5 -5
- package/out/fr/tools/fetch/finess.d.ts.map +1 -1
- package/out/fr/tools/fetch/finess.js +13 -12
- package/out/fr/tools/fetch/finess.js.map +1 -1
- package/out/inspire/address.d.ts +62 -52
- package/out/inspire/address.d.ts.map +1 -1
- package/out/inspire/address.js +71 -60
- package/out/inspire/address.js.map +1 -1
- package/out/inspire/errors.d.ts +9 -8
- package/out/inspire/errors.d.ts.map +1 -1
- package/out/inspire/errors.js +7 -6
- package/out/inspire/errors.js.map +1 -1
- package/out/inspire/stream.d.ts +25 -22
- package/out/inspire/stream.d.ts.map +1 -1
- package/out/inspire/stream.js +10 -12
- package/out/inspire/stream.js.map +1 -1
- package/out/international/recipes/locale.d.ts +2 -2
- package/out/international/recipes/locale.d.ts.map +1 -1
- package/out/international/recipes/locale.js +11 -7
- package/out/international/recipes/locale.js.map +1 -1
- package/out/it/adapters/anac/adapter.d.ts +45 -43
- package/out/it/adapters/anac/adapter.d.ts.map +1 -1
- package/out/it/adapters/anac/adapter.js +48 -43
- package/out/it/adapters/anac/adapter.js.map +1 -1
- package/out/it/tools/fetch/anac.d.ts +16 -19
- package/out/it/tools/fetch/anac.d.ts.map +1 -1
- package/out/it/tools/fetch/anac.js +20 -23
- package/out/it/tools/fetch/anac.js.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.d.ts.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.js.map +1 -1
- package/out/jp/tools/fetch/houjin.js +2 -2
- package/out/jp/tools/fetch/houjin.js.map +1 -1
- package/out/kr/adapters/localdata/permit-rows.js +1 -1
- package/out/kr/adapters/localdata/permit-rows.js.map +1 -1
- package/out/kr/tools/fetch/juso.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.d.ts +1 -1
- package/out/kr/tools/fetch/localdata.js +1 -1
- package/out/kr/tools/fetch/localdata.js.map +1 -1
- package/out/lu/adapters/bd-adresses/adapter.d.ts +9 -9
- package/out/lu/adapters/bd-adresses/adapter.js +10 -10
- package/out/lu/tools/fetch/bd-adresses.d.ts +11 -9
- package/out/lu/tools/fetch/bd-adresses.d.ts.map +1 -1
- package/out/lu/tools/fetch/bd-adresses.js +11 -11
- package/out/lu/tools/fetch/bd-adresses.js.map +1 -1
- package/out/lv/adapters/varis/adapter.d.ts +24 -23
- package/out/lv/adapters/varis/adapter.d.ts.map +1 -1
- package/out/lv/adapters/varis/adapter.js +24 -23
- package/out/lv/adapters/varis/adapter.js.map +1 -1
- package/out/lv/tools/fetch/varis.d.ts +2 -1
- package/out/lv/tools/fetch/varis.d.ts.map +1 -1
- package/out/lv/tools/fetch/varis.js +1 -1
- package/out/lv/tools/fetch/varis.js.map +1 -1
- package/out/nl/adapters/kadaster/adapter.d.ts +12 -12
- package/out/nl/adapters/kadaster/adapter.d.ts.map +1 -1
- package/out/nl/adapters/kadaster/adapter.js +22 -23
- package/out/nl/adapters/kadaster/adapter.js.map +1 -1
- package/out/nl/tools/fetch/kadaster.d.ts +14 -10
- package/out/nl/tools/fetch/kadaster.d.ts.map +1 -1
- package/out/nl/tools/fetch/kadaster.js +12 -9
- package/out/nl/tools/fetch/kadaster.js.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts +1 -1
- package/out/no/adapters/matrikkelen/adapter.d.ts.map +1 -1
- package/out/no/adapters/matrikkelen/adapter.js +3 -3
- package/out/no/adapters/matrikkelen/adapter.js.map +1 -1
- package/out/no/tools/fetch/matrikkelen.d.ts +50 -38
- package/out/no/tools/fetch/matrikkelen.d.ts.map +1 -1
- package/out/no/tools/fetch/matrikkelen.js +44 -36
- package/out/no/tools/fetch/matrikkelen.js.map +1 -1
- package/out/parquet/duckdb.d.ts +7 -4
- package/out/parquet/duckdb.d.ts.map +1 -1
- package/out/parquet/duckdb.js +2 -1
- package/out/parquet/duckdb.js.map +1 -1
- package/out/parquet/readers.js +3 -3
- package/out/parquet/readers.js.map +1 -1
- package/out/parquet/schema.js +1 -1
- package/out/parquet/schema.js.map +1 -1
- package/out/parquet/writers.d.ts +1 -1
- package/out/parquet/writers.d.ts.map +1 -1
- package/out/parquet/writers.js +2 -1
- package/out/parquet/writers.js.map +1 -1
- package/out/pl/adapters/emuia/adapter.d.ts +8 -8
- package/out/pl/adapters/emuia/adapter.js +12 -12
- package/out/pl/adapters/emuia/adapter.js.map +1 -1
- package/out/pl/tools/fetch/emuia.d.ts +5 -4
- package/out/pl/tools/fetch/emuia.d.ts.map +1 -1
- package/out/pl/tools/fetch/emuia.js +10 -9
- package/out/pl/tools/fetch/emuia.js.map +1 -1
- package/out/recipes/bare/postcode.js +1 -1
- package/out/recipes/bare/postcode.js.map +1 -1
- package/out/recipes/boundary-stress.js +1 -1
- package/out/recipes/boundary-stress.js.map +1 -1
- package/out/recipes/house-venue.d.ts.map +1 -1
- package/out/recipes/house-venue.js +6 -3
- package/out/recipes/house-venue.js.map +1 -1
- package/out/recipes/intersection.js +2 -2
- package/out/recipes/intersection.js.map +1 -1
- package/out/recipes/po/box/cedex/recipe.js +2 -1
- package/out/recipes/po/box/cedex/recipe.js.map +1 -1
- package/out/recipes/po/box/cedex/sources.d.ts +8 -2
- package/out/recipes/po/box/cedex/sources.d.ts.map +1 -1
- package/out/recipes/po/box/cedex/sources.js +1 -1
- package/out/recipes/po/box/cedex/sources.js.map +1 -1
- package/out/recipes/scaffold.d.ts +28 -19
- package/out/recipes/scaffold.d.ts.map +1 -1
- package/out/recipes/scaffold.js +3 -2
- package/out/recipes/scaffold.js.map +1 -1
- package/out/recipes/sources.d.ts +1 -1
- package/out/recipes/sources.d.ts.map +1 -1
- package/out/recipes/sources.js +15 -2
- package/out/recipes/sources.js.map +1 -1
- package/out/recipes/street/bare.js +2 -2
- package/out/recipes/street/bare.js.map +1 -1
- package/out/recipes/street.d.ts.map +1 -1
- package/out/recipes/street.js +3 -1
- package/out/recipes/street.js.map +1 -1
- package/out/recipes/sub/venue/context.d.ts.map +1 -1
- package/out/recipes/sub/venue/context.js +2 -0
- package/out/recipes/sub/venue/context.js.map +1 -1
- package/out/recipes/sub/venue/render.js +1 -1
- package/out/recipes/sub/venue/render.js.map +1 -1
- package/out/recipes/sub/venue/sources.d.ts +6 -3
- package/out/recipes/sub/venue/sources.d.ts.map +1 -1
- package/out/recipes/sub/venue/sources.js +16 -20
- package/out/recipes/sub/venue/sources.js.map +1 -1
- package/out/recipes/sub/venue.d.ts +2 -5
- package/out/recipes/sub/venue.d.ts.map +1 -1
- package/out/recipes/sub/venue.js +10 -14
- package/out/recipes/sub/venue.js.map +1 -1
- package/out/recipes/trailing-region.d.ts.map +1 -1
- package/out/recipes/trailing-region.js +6 -1
- package/out/recipes/trailing-region.js.map +1 -1
- package/out/recipes/unit.js +2 -2
- package/out/recipes/unit.js.map +1 -1
- package/out/recipes/venue.d.ts +119 -0
- package/out/recipes/venue.d.ts.map +1 -0
- package/out/recipes/venue.js +245 -0
- package/out/recipes/venue.js.map +1 -0
- package/out/recipes.d.ts +1 -1
- package/out/recipes.d.ts.map +1 -1
- package/out/recipes.js +3 -1
- package/out/recipes.js.map +1 -1
- package/out/registers.d.ts +8 -6
- package/out/registers.d.ts.map +1 -1
- package/out/registers.js +8 -6
- package/out/registers.js.map +1 -1
- package/out/ro/adapters/retea-scolara/adapter.d.ts +5 -5
- package/out/ro/adapters/retea-scolara/adapter.js +5 -5
- package/out/ro/tools/fetch/retea-scolara.js +1 -1
- package/out/ro/tools/fetch/retea-scolara.js.map +1 -1
- package/out/runner.d.ts +3 -2
- package/out/runner.d.ts.map +1 -1
- package/out/runner.js +1 -1
- package/out/runner.js.map +1 -1
- package/out/sg/recipes/register.js +1 -1
- package/out/sg/recipes/register.js.map +1 -1
- package/out/sg/tools/fetch/acra.js +2 -2
- package/out/sg/tools/fetch/acra.js.map +1 -1
- package/out/sk/adapters/inspire/adapter.d.ts +5 -5
- package/out/sk/adapters/inspire/adapter.d.ts.map +1 -1
- package/out/sk/adapters/inspire/adapter.js +18 -18
- package/out/sk/adapters/inspire/adapter.js.map +1 -1
- package/out/sk/tools/fetch/inspire.d.ts +11 -10
- package/out/sk/tools/fetch/inspire.d.ts.map +1 -1
- package/out/sk/tools/fetch/inspire.js +15 -14
- package/out/sk/tools/fetch/inspire.js.map +1 -1
- package/out/source-register/effective-manifest.js +3 -3
- package/out/source-register/effective-manifest.js.map +1 -1
- package/out/source-register/register.d.ts +2 -2
- package/out/source-register/register.d.ts.map +1 -1
- package/out/source-register/register.js +2 -2
- package/out/source-register/register.js.map +1 -1
- package/out/source-register/training-manifest.d.ts +1 -1
- package/out/source-register/types.d.ts +12 -13
- package/out/source-register/types.d.ts.map +1 -1
- package/out/source-register/types.js +2 -2
- package/out/source-register/types.js.map +1 -1
- package/out/subvenue/promotions.d.ts +1 -1
- package/out/subvenue/promotions.js +1 -1
- package/out/subvenue/table.d.ts +4 -4
- package/out/subvenue/table.js +3 -3
- package/out/surfaces/locale.d.ts.map +1 -1
- package/out/surfaces/locale.js +2 -0
- package/out/surfaces/locale.js.map +1 -1
- package/out/synthesizers/anchor-absorption.d.ts +2 -1
- package/out/synthesizers/anchor-absorption.d.ts.map +1 -1
- package/out/synthesizers/anchor-absorption.js.map +1 -1
- package/out/synthesizers/boundary-stress.d.ts +1 -1
- package/out/synthesizers/boundary-stress.d.ts.map +1 -1
- package/out/synthesizers/boundary-stress.js +20 -23
- package/out/synthesizers/boundary-stress.js.map +1 -1
- package/out/synthesizers/house-venue.d.ts +3 -2
- package/out/synthesizers/house-venue.d.ts.map +1 -1
- package/out/synthesizers/house-venue.js +24 -26
- package/out/synthesizers/house-venue.js.map +1 -1
- package/out/synthesizers/no-street.d.ts.map +1 -1
- package/out/synthesizers/no-street.js.map +1 -1
- package/out/synthesizers/po-box.d.ts +4 -2
- package/out/synthesizers/po-box.d.ts.map +1 -1
- package/out/synthesizers/po-box.js.map +1 -1
- package/out/synthesizers/tail.d.ts +36 -0
- package/out/synthesizers/tail.d.ts.map +1 -0
- package/out/synthesizers/tail.js +32 -0
- package/out/synthesizers/tail.js.map +1 -0
- package/out/synthesizers/utils.d.ts +2 -1
- package/out/synthesizers/utils.d.ts.map +1 -1
- package/out/synthesizers/utils.js +17 -1
- package/out/synthesizers/utils.js.map +1 -1
- package/out/test-kit.js +2 -2
- package/out/test-kit.js.map +1 -1
- package/out/tools/audit.d.ts +2 -1
- package/out/tools/audit.d.ts.map +1 -1
- package/out/tools/audit.js +4 -3
- package/out/tools/audit.js.map +1 -1
- package/out/tools/corpus-stats.js +1 -1
- package/out/tools/corpus-stats.js.map +1 -1
- package/out/tools/fetch/atom.d.ts +8 -8
- package/out/tools/fetch/atom.d.ts.map +1 -1
- package/out/tools/fetch/atom.js +7 -8
- package/out/tools/fetch/atom.js.map +1 -1
- package/out/tools/fetch/cordis.d.ts +4 -3
- package/out/tools/fetch/cordis.d.ts.map +1 -1
- package/out/tools/fetch/cordis.js +9 -7
- package/out/tools/fetch/cordis.js.map +1 -1
- package/out/tools/fetch/download/manifest.d.ts.map +1 -1
- package/out/tools/fetch/download/manifest.js +2 -1
- package/out/tools/fetch/download/manifest.js.map +1 -1
- package/out/tools/fetch/download/network.d.ts +4 -4
- package/out/tools/fetch/download/retry.js +4 -4
- package/out/tools/fetch/download/retry.js.map +1 -1
- package/out/tools/fetch/geonames/dump.d.ts +3 -2
- package/out/tools/fetch/geonames/dump.d.ts.map +1 -1
- package/out/tools/fetch/geonames/dump.js +2 -1
- package/out/tools/fetch/geonames/dump.js.map +1 -1
- package/out/tools/fetch/geonames/postal.d.ts +2 -2
- package/out/tools/fetch/gleif.d.ts +3 -2
- package/out/tools/fetch/gleif.d.ts.map +1 -1
- package/out/tools/fetch/gleif.js +6 -4
- package/out/tools/fetch/gleif.js.map +1 -1
- package/out/tools/fetch/header.d.ts +7 -5
- package/out/tools/fetch/header.d.ts.map +1 -1
- package/out/tools/fetch/header.js +8 -6
- package/out/tools/fetch/header.js.map +1 -1
- package/out/tools/fetch/inspire-addresses.d.ts +32 -27
- package/out/tools/fetch/inspire-addresses.d.ts.map +1 -1
- package/out/tools/fetch/inspire-addresses.js +25 -24
- package/out/tools/fetch/inspire-addresses.js.map +1 -1
- package/out/tools/fetch/openaddresses.js +1 -1
- package/out/tools/fetch/openaddresses.js.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.d.ts +5 -5
- package/out/tools/fetch/vlaanderen-ad.d.ts.map +1 -1
- package/out/tools/fetch/vlaanderen-ad.js +16 -16
- package/out/tools/fetch/vlaanderen-ad.js.map +1 -1
- package/out/tools/fetch/wfs-harvest.d.ts +26 -16
- package/out/tools/fetch/wfs-harvest.d.ts.map +1 -1
- package/out/tools/fetch/wfs-harvest.js +21 -18
- package/out/tools/fetch/wfs-harvest.js.map +1 -1
- package/out/tools/fetch.d.ts +0 -41
- package/out/tools/fetch.d.ts.map +1 -1
- package/out/tools/fetch.js +0 -41
- package/out/tools/fetch.js.map +1 -1
- package/out/tools/golden/expand.d.ts +7 -7
- package/out/tools/golden/expand.js +1 -1
- package/out/tools/golden/expand.js.map +1 -1
- package/out/tools/golden/promote.d.ts +2 -2
- package/out/tools/golden/promote.d.ts.map +1 -1
- package/out/tools/golden/promote.js +9 -6
- package/out/tools/golden/promote.js.map +1 -1
- package/out/tools/golden/relabel-street/directory.d.ts +4 -4
- package/out/tools/golden/relabel-street/directory.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/directory.js +3 -2
- package/out/tools/golden/relabel-street/directory.js.map +1 -1
- package/out/tools/golden/relabel-street/row.d.ts +4 -3
- package/out/tools/golden/relabel-street/row.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/row.js +3 -3
- package/out/tools/golden/relabel-street/row.js.map +1 -1
- package/out/tools/ingest-csv.js +3 -3
- package/out/tools/ingest-csv.js.map +1 -1
- package/out/tools/jsonl-to-parquet.d.ts +1 -1
- package/out/tools/lint/recipe-output/vocab.d.ts +10 -5
- package/out/tools/lint/recipe-output/vocab.d.ts.map +1 -1
- package/out/tools/lint/recipe-output/vocab.js.map +1 -1
- package/out/tools/lint/recipe-output.d.ts +8 -8
- package/out/tools/lint/recipe-output.d.ts.map +1 -1
- package/out/tools/lint/recipe-output.js +17 -0
- package/out/tools/lint/recipe-output.js.map +1 -1
- package/out/tools/overlay/kryptonite.d.ts +2 -2
- package/out/tools/overlay/kryptonite.d.ts.map +1 -1
- package/out/tools/overlay/kryptonite.js +2 -1
- package/out/tools/overlay/kryptonite.js.map +1 -1
- package/out/tools/overlay/manifest.d.ts +2 -1
- package/out/tools/overlay/manifest.d.ts.map +1 -1
- package/out/tools/overlay/manifest.js +2 -1
- package/out/tools/overlay/manifest.js.map +1 -1
- package/out/tools/overlay/merge-source.d.ts +2 -1
- package/out/tools/overlay/merge-source.d.ts.map +1 -1
- package/out/tools/overlay/merge-source.js.map +1 -1
- package/out/tools/overlay/split-slice.d.ts +1 -1
- package/out/tools/overlay/split-slice.js +1 -1
- package/out/tools/overlay/split-slice.js.map +1 -1
- package/out/tools/overlay/translit.d.ts +4 -3
- package/out/tools/overlay/translit.d.ts.map +1 -1
- package/out/tools/overlay/translit.js +2 -1
- package/out/tools/overlay/translit.js.map +1 -1
- package/out/tools/overture-subvenue.d.ts +1 -1
- package/out/tools/overture-subvenue.d.ts.map +1 -1
- package/out/tools/postcode-triples.d.ts +1 -1
- package/out/tools/postcode-triples.d.ts.map +1 -1
- package/out/tools/postcode-triples.js +4 -3
- package/out/tools/postcode-triples.js.map +1 -1
- package/out/tools/source-register/build.d.ts +10 -8
- package/out/tools/source-register/build.d.ts.map +1 -1
- package/out/tools/source-register/build.js +10 -8
- package/out/tools/source-register/build.js.map +1 -1
- package/out/tools/sub/venue/lexicon.d.ts +2 -6
- package/out/tools/sub/venue/lexicon.d.ts.map +1 -1
- package/out/tools/sub/venue/lexicon.js +2 -6
- package/out/tools/sub/venue/lexicon.js.map +1 -1
- package/out/tw/tools/fetch/gcis.js +3 -3
- package/out/tw/tools/fetch/gcis.js.map +1 -1
- package/out/types.d.ts +4 -3
- package/out/types.d.ts.map +1 -1
- package/out/types.js.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts +1 -1
- package/out/us/adapters/state/hi-schools/workbook.d.ts.map +1 -1
- package/out/us/adapters/state/hi-schools/workbook.js.map +1 -1
- package/out/us/adapters/usgov/nad/adapter.js +6 -6
- package/out/us/adapters/usgov/nad/adapter.js.map +1 -1
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js +2 -2
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/us/fips-state.d.ts +2 -2
- package/out/us/fips-state.d.ts.map +1 -1
- package/out/us/fips-state.js.map +1 -1
- package/out/us/synthesizers/intersection.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/intersection.js +13 -9
- package/out/us/synthesizers/intersection.js.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.d.ts +3 -2
- package/out/us/synthesizers/street.d.ts.map +1 -0
- package/out/{synthesizers → us/synthesizers}/street.js +12 -12
- package/out/us/synthesizers/street.js.map +1 -0
- package/out/us/tools/fetch/imls-pls.js +1 -1
- package/out/us/tools/fetch/imls-pls.js.map +1 -1
- package/out/us/tools/fetch/nad.d.ts +6 -6
- package/out/us/tools/fetch/nad.js +1 -1
- package/out/us/tools/fetch/nad.js.map +1 -1
- package/out/us/tools/fetch/nppes.js +3 -3
- package/out/us/tools/fetch/nppes.js.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts +2 -1
- package/out/us/tools/fetch/state/hi-schools.d.ts.map +1 -1
- package/out/us/tools/fetch/state/hi-schools.js +1 -1
- package/out/us/tools/fetch/state/hi-schools.js.map +1 -1
- package/out/us/tools/fetch/tiger-full.js +1 -1
- package/out/us/tools/fetch/tiger-full.js.map +1 -1
- package/out/utils/align.d.ts +5 -3
- package/out/utils/align.d.ts.map +1 -1
- package/out/utils/align.js +5 -5
- package/out/utils/align.js.map +1 -1
- package/out/utils/golden.js +2 -2
- package/out/utils/golden.js.map +1 -1
- package/out/utils/license.d.ts +3 -3
- package/out/utils/license.d.ts.map +1 -1
- package/out/utils/license.js.map +1 -1
- package/out/utils/split.d.ts +1 -1
- package/out/utils/split.d.ts.map +1 -1
- package/out/utils/split.js +18 -14
- package/out/utils/split.js.map +1 -1
- package/out/utils/tokenize.d.ts +3 -20
- package/out/utils/tokenize.d.ts.map +1 -1
- package/out/utils/tokenize.js.map +1 -1
- package/out/utils/wof-json.js +3 -3
- package/out/utils/wof-json.js.map +1 -1
- package/package.json +18 -27
- package/lib/tools.ts +0 -31
- package/out/synthesizers/intersection.d.ts.map +0 -1
- package/out/synthesizers/intersection.js.map +0 -1
- package/out/synthesizers/street.d.ts.map +0 -1
- package/out/synthesizers/street.js.map +0 -1
- package/out/tools.d.ts +0 -31
- package/out/tools.d.ts.map +0 -1
- package/out/tools.js +0 -31
- package/out/tools.js.map +0 -1
- /package/out/{synthesizers → us/synthesizers}/intersection.d.ts +0 -0
package/lib/types.ts
CHANGED
|
@@ -195,7 +195,7 @@ export interface SourceProvenance {
|
|
|
195
195
|
/**
|
|
196
196
|
* The license label or SPDX id for this row.
|
|
197
197
|
*
|
|
198
|
-
*
|
|
198
|
+
* @defaultValue the adapter's `defaultLicense`, and sources with per-record terms override it.
|
|
199
199
|
*/
|
|
200
200
|
license: string
|
|
201
201
|
}
|
|
@@ -382,8 +382,9 @@ export interface AdapterOptions {
|
|
|
382
382
|
*
|
|
383
383
|
* The runner writes the map into `MANIFEST.json` as `dropped`, so the gap between a
|
|
384
384
|
* source's record count and the rows written has a counted reason.
|
|
385
|
-
* An adapter that does not count leaves the map empty
|
|
386
|
-
*
|
|
385
|
+
* An adapter that does not count leaves the map empty.
|
|
386
|
+
*
|
|
387
|
+
* The manifest records that as an empty object rather than as zero drops of a specific kind.
|
|
387
388
|
*/
|
|
388
389
|
dropped?: Map<string, number>
|
|
389
390
|
}
|
|
@@ -35,6 +35,6 @@ export interface HiSchoolRow {
|
|
|
35
35
|
/**
|
|
36
36
|
* Convert a typed workbook cell or legacy CSV field to the trimmed text used by the corpus row.
|
|
37
37
|
*/
|
|
38
|
-
export function schoolCellText(value: XLSXCellValue | undefined): string {
|
|
38
|
+
export function schoolCellText(value: XLSXCellValue | null | undefined): string {
|
|
39
39
|
return value === null || value === undefined ? "" : String(value).trim()
|
|
40
40
|
}
|
|
@@ -76,14 +76,14 @@ interface NADRecord {
|
|
|
76
76
|
*/
|
|
77
77
|
const US_STATES_SET: ReadonlySet<string> = new Set(US_STATE_ABBREVIATIONS)
|
|
78
78
|
|
|
79
|
-
function nonEmpty(...values: Array<string | null | undefined>): string |
|
|
79
|
+
function nonEmpty(...values: Array<string | null | undefined>): string | null {
|
|
80
80
|
for (const v of values) {
|
|
81
81
|
const trimmed = (v ?? "").toString().trim()
|
|
82
82
|
|
|
83
83
|
if (trimmed) return trimmed
|
|
84
84
|
}
|
|
85
85
|
|
|
86
|
-
return
|
|
86
|
+
return null
|
|
87
87
|
}
|
|
88
88
|
|
|
89
89
|
/**
|
|
@@ -94,17 +94,17 @@ function nonEmpty(...values: Array<string | null | undefined>): string | undefin
|
|
|
94
94
|
* The shared `composeHouseNumber` in `#adapters/street-line` joins exactly two columns with
|
|
95
95
|
* one separator, so it fits a publisher that splits the number in two rather than in four.
|
|
96
96
|
*/
|
|
97
|
-
function nadHouseNumber(r: NADRecord): string |
|
|
97
|
+
function nadHouseNumber(r: NADRecord): string | null {
|
|
98
98
|
const full = (r.AddNo_Full ?? "").toString().trim()
|
|
99
99
|
|
|
100
100
|
if (full) return full
|
|
101
101
|
const num = r.Add_Number == null ? "" : String(r.Add_Number).trim()
|
|
102
102
|
|
|
103
|
-
if (!num) return
|
|
103
|
+
if (!num) return null
|
|
104
104
|
const pre = (r.AddNum_Pre ?? "").toString().trim()
|
|
105
105
|
const suf = (r.AddNum_Suf ?? "").toString().trim()
|
|
106
106
|
|
|
107
|
-
return [pre, num, suf].filter(isPresent).join(" ").trim() ||
|
|
107
|
+
return [pre, num, suf].filter(isPresent).join(" ").trim() || null
|
|
108
108
|
}
|
|
109
109
|
|
|
110
110
|
interface DecomposedNADStreet {
|
|
@@ -116,7 +116,7 @@ interface DecomposedNADStreet {
|
|
|
116
116
|
|
|
117
117
|
// Structured street fields take precedence.
|
|
118
118
|
// If they are empty, `StNam_Full` supplies the whole street.
|
|
119
|
-
function decomposeNADStreet(r: NADRecord): DecomposedNADStreet |
|
|
119
|
+
function decomposeNADStreet(r: NADRecord): DecomposedNADStreet | null {
|
|
120
120
|
const name = (r.St_Name ?? "").toString().trim()
|
|
121
121
|
|
|
122
122
|
if (name) {
|
|
@@ -136,18 +136,18 @@ function decomposeNADStreet(r: NADRecord): DecomposedNADStreet | undefined {
|
|
|
136
136
|
|
|
137
137
|
if (full) return { full, street: full }
|
|
138
138
|
|
|
139
|
-
return
|
|
139
|
+
return null
|
|
140
140
|
}
|
|
141
141
|
|
|
142
142
|
// The postal city comes first because it is the name that people write on mail.
|
|
143
|
-
function composeLocality(r: NADRecord): string |
|
|
143
|
+
function composeLocality(r: NADRecord): string | null {
|
|
144
144
|
return nonEmpty(r.Post_City, r.Inc_Muni, r.Census_Plc, r.Uninc_Comm)
|
|
145
145
|
}
|
|
146
146
|
|
|
147
|
-
function composePostcode(r: NADRecord): string |
|
|
147
|
+
function composePostcode(r: NADRecord): string | null {
|
|
148
148
|
const zip = (r.Zip_Code ?? "").toString().trim()
|
|
149
149
|
|
|
150
|
-
if (!zip) return
|
|
150
|
+
if (!zip) return null
|
|
151
151
|
const plus4 = (r.Plus_4 ?? "").toString().trim()
|
|
152
152
|
|
|
153
153
|
return plus4 ? `${zip}-${plus4}` : zip
|
|
@@ -196,7 +196,7 @@ export function createUsgovNADAdapter(): CorpusAdapter {
|
|
|
196
196
|
|
|
197
197
|
const record = tryParsingJSON<NADRecord>(line)
|
|
198
198
|
|
|
199
|
-
if (record
|
|
199
|
+
if (!record) continue
|
|
200
200
|
|
|
201
201
|
const state = (record.State ?? "").toString().trim().toUpperCase()
|
|
202
202
|
|
|
@@ -78,7 +78,7 @@ interface SamhsaSiteRow {
|
|
|
78
78
|
* The combined value is the `street` component surface form.
|
|
79
79
|
* Phase 1 does not break this out into the `unit` component — see the file-level comment.
|
|
80
80
|
*/
|
|
81
|
-
function joinTwoLineStreet(street1: string, street2: string |
|
|
81
|
+
function joinTwoLineStreet(street1: string, street2: string | null): string {
|
|
82
82
|
const s1 = street1.trim()
|
|
83
83
|
const s2 = (street2 ?? "").trim()
|
|
84
84
|
|
|
@@ -101,7 +101,7 @@ function joinTwoLineStreet(street1: string, street2: string | undefined): string
|
|
|
101
101
|
* Both render together as `"<name1> - <name2>"` when both are present — geocoder users
|
|
102
102
|
* typically type either form, so the model benefits from the joined surface.
|
|
103
103
|
*/
|
|
104
|
-
function composeVenue(name1: string, name2: string |
|
|
104
|
+
function composeVenue(name1: string, name2: string | null): string {
|
|
105
105
|
const n1 = name1.trim()
|
|
106
106
|
const n2 = (name2 ?? "").trim()
|
|
107
107
|
|
|
@@ -140,8 +140,8 @@ export function createUsgovSamhsaTreatmentLocatorAdapter(): CorpusAdapter {
|
|
|
140
140
|
|
|
141
141
|
if (opts.limit !== undefined && emitted >= opts.limit) break
|
|
142
142
|
|
|
143
|
-
const venue = composeVenue(record.name1 ?? "", record.name2)
|
|
144
|
-
const street = joinTwoLineStreet(record.street1 ?? "", record.street2)
|
|
143
|
+
const venue = composeVenue(record.name1 ?? "", record.name2 ?? null)
|
|
144
|
+
const street = joinTwoLineStreet(record.street1 ?? "", record.street2 ?? null)
|
|
145
145
|
const split = splitStreetLine(street)
|
|
146
146
|
const city = record.city ?? ""
|
|
147
147
|
const stateAbbr = record.state ?? ""
|
package/lib/us/fips-state.ts
CHANGED
|
@@ -92,7 +92,7 @@ export const US_FIPS_STATE: Readonly<Record<string, USStateInfo>> = Object.freez
|
|
|
92
92
|
*
|
|
93
93
|
* @returns Null when the FIPS code isn't recognized.
|
|
94
94
|
*/
|
|
95
|
-
export function lookupFipsState(statefp: string | null
|
|
95
|
+
export function lookupFipsState(statefp: string | null): USStateInfo | null {
|
|
96
96
|
if (!statefp) return null
|
|
97
97
|
|
|
98
98
|
return US_FIPS_STATE[statefp] ?? null
|
|
@@ -115,7 +115,7 @@ export const US_STATE_BY_ABBREVIATION: Readonly<Record<string, USStateInfo>> = O
|
|
|
115
115
|
* Case-folded.
|
|
116
116
|
* Null for any value outside the 50 states + DC + the five primary territories.
|
|
117
117
|
*/
|
|
118
|
-
export function lookupStateAbbreviation(abbreviation: string | null
|
|
118
|
+
export function lookupStateAbbreviation(abbreviation: string | null): USStateInfo | null {
|
|
119
119
|
if (!abbreviation) return null
|
|
120
120
|
|
|
121
121
|
return US_STATE_BY_ABBREVIATION[abbreviation.toUpperCase()] ?? null
|
|
@@ -19,13 +19,10 @@
|
|
|
19
19
|
* US-idiomatic only (the harness intersection cases are US: "X & Y, City, ST ZIP").
|
|
20
20
|
*/
|
|
21
21
|
|
|
22
|
-
/* oxlint-disable mailwoman/prefer-home -- the admin tails below are written as US templates because this synthesizer
|
|
23
|
-
refuses a non-US tuple outright (`if (base.country !== "US") return null`). A layout call would answer the same
|
|
24
|
-
string for the only country that reaches it. The function would read as though the file served more. */
|
|
25
|
-
|
|
26
22
|
import type { DirectionalAbbreviation } from "@mailwoman/codex/us"
|
|
27
23
|
import { sample } from "@mailwoman/core/random"
|
|
28
24
|
|
|
25
|
+
import { layoutAdminTail } from "#synthesizers/tail"
|
|
29
26
|
import type { CanonicalRow } from "#types"
|
|
30
27
|
|
|
31
28
|
/**
|
|
@@ -156,6 +153,9 @@ export function synthesizeIntersectionRow(
|
|
|
156
153
|
): SynthesizedIntersectionRow | null {
|
|
157
154
|
const random = opts.random ?? Math.random
|
|
158
155
|
|
|
156
|
+
// The junction connectors and the street vocabulary are US English,
|
|
157
|
+
// which is why this synthesizer sits under `us/`.
|
|
158
|
+
// The admin tail renders through the country's layout.
|
|
159
159
|
if (base.country !== "US") return null
|
|
160
160
|
|
|
161
161
|
const a = buildStreetName(random)
|
|
@@ -187,17 +187,16 @@ export function synthesizeIntersectionRow(
|
|
|
187
187
|
} else {
|
|
188
188
|
const includePostcode = base.postcode != null && random() < 0.7
|
|
189
189
|
|
|
190
|
-
const tail =
|
|
191
|
-
|
|
192
|
-
:
|
|
190
|
+
const tail = layoutAdminTail(base.country, {
|
|
191
|
+
locality: base.locality,
|
|
192
|
+
region: base.region,
|
|
193
|
+
postcode: includePostcode ? base.postcode : undefined,
|
|
194
|
+
})
|
|
193
195
|
|
|
194
|
-
|
|
195
|
-
components.locality = base.locality
|
|
196
|
-
components.region = base.region
|
|
196
|
+
if (!tail) return null
|
|
197
197
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
}
|
|
198
|
+
raw = `${cornerPrefix}${a}${connector}${b}, ${tail.raw}`
|
|
199
|
+
Object.assign(components, tail.components)
|
|
201
200
|
}
|
|
202
201
|
|
|
203
202
|
return { raw, components, locale: "en-US" }
|
|
@@ -19,13 +19,10 @@
|
|
|
19
19
|
* adapter.
|
|
20
20
|
*/
|
|
21
21
|
|
|
22
|
-
/* oxlint-disable mailwoman/prefer-home -- the admin tails below are written as US templates because this synthesizer
|
|
23
|
-
refuses a non-US tuple outright (`if (base.country !== "US") return null`). A layout call would answer the same
|
|
24
|
-
string for the only country that reaches it. The function would read as though the file served more. */
|
|
25
|
-
|
|
26
22
|
import { isPresent } from "@mailwoman/core/objects"
|
|
27
23
|
import { sample } from "@mailwoman/core/random"
|
|
28
24
|
|
|
25
|
+
import { layoutAdminTail } from "#synthesizers/tail"
|
|
29
26
|
import { tieredNumber } from "#synthesizers/utils"
|
|
30
27
|
import type { CanonicalRow } from "#types"
|
|
31
28
|
import { decomposeStreet } from "#us/adapters/tiger/street-decompose"
|
|
@@ -182,17 +179,18 @@ export interface StreetSynthesisOpts {
|
|
|
182
179
|
/**
|
|
183
180
|
* Probability of emitting house_number alongside the street.
|
|
184
181
|
*
|
|
185
|
-
*
|
|
182
|
+
* @defaultValue `0.85`
|
|
186
183
|
*/
|
|
187
184
|
includeHouseNumberProb?: number
|
|
188
185
|
/**
|
|
189
186
|
* Probability of emitting the street bare — no `, City, ST ZIP` tail and no region/locality/
|
|
190
187
|
* postcode components (just `street_prefix`/`street`/`street_suffix` + optional `house_number`).
|
|
191
188
|
*
|
|
192
|
-
* Default 0 (leaves the original full-address output exactly as it was, including the RNG sequence).
|
|
193
189
|
* Set >0 to teach the model that a bare `10th Ave` / `Main St` is a street rather than a
|
|
194
190
|
* locality — the functional-test failure cluster (bare streets mislabeled `locality`),
|
|
195
191
|
* the bare-format analogue of the v0.7.x intersection-bare fix.
|
|
192
|
+
*
|
|
193
|
+
* @defaultValue `0` (leaves the original full-address output exactly as it was, including the RNG sequence)
|
|
196
194
|
*/
|
|
197
195
|
bareProb?: number
|
|
198
196
|
}
|
|
@@ -223,6 +221,9 @@ export function synthesizeStreetRow(
|
|
|
223
221
|
const random = opts.random ?? Math.random
|
|
224
222
|
const includeHN = opts.includeHouseNumberProb ?? 0.85
|
|
225
223
|
|
|
224
|
+
// The street vocabulary (`STREET_NAMES`, `STREET_SUFFIXES`, the directionals) is
|
|
225
|
+
// US English, which is why this synthesizer sits under `us/`.
|
|
226
|
+
// The admin tail renders through the country's layout.
|
|
226
227
|
if (base.country !== "US") return null
|
|
227
228
|
|
|
228
229
|
const prefix = sample(DIRECTIONAL_PREFIXES, random)
|
|
@@ -247,13 +248,13 @@ export function synthesizeStreetRow(
|
|
|
247
248
|
const bareProb = opts.bareProb ?? 0
|
|
248
249
|
const bare = bareProb > 0 && random() < bareProb
|
|
249
250
|
|
|
250
|
-
const
|
|
251
|
-
?
|
|
252
|
-
: {
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
251
|
+
const tail = bare
|
|
252
|
+
? null
|
|
253
|
+
: layoutAdminTail(base.country, { locality: base.locality, region: base.region, postcode: base.postcode })
|
|
254
|
+
|
|
255
|
+
if (!bare && !tail) return null
|
|
256
|
+
|
|
257
|
+
const components: CanonicalRow["components"] = tail ? { ...tail.components } : {}
|
|
257
258
|
|
|
258
259
|
if (decomposed.prefix) {
|
|
259
260
|
components.street_prefix = decomposed.prefix
|
|
@@ -272,9 +273,9 @@ export function synthesizeStreetRow(
|
|
|
272
273
|
if (random() < includeHN) {
|
|
273
274
|
const hn = randomHouseNumber(random)
|
|
274
275
|
components.house_number = hn
|
|
275
|
-
raw =
|
|
276
|
+
raw = tail ? `${hn} ${fullStreet}, ${tail.raw}` : `${hn} ${fullStreet}`
|
|
276
277
|
} else {
|
|
277
|
-
raw =
|
|
278
|
+
raw = tail ? `${fullStreet}, ${tail.raw}` : fullStreet
|
|
278
279
|
}
|
|
279
280
|
|
|
280
281
|
return { raw, components, locale: "en-US" }
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
procedure rather than sections of declarations. A region there folds no element a reader wants folded. */
|
|
14
14
|
|
|
15
15
|
import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters"
|
|
16
|
-
import { statPath, pathExists } from "@mailwoman/core/fs/readers"
|
|
16
|
+
import { statPath, pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
17
17
|
import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
|
|
18
18
|
import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip"
|
|
19
19
|
import { sha256File } from "@mailwoman/core/hash"
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
*/
|
|
38
38
|
|
|
39
39
|
import { APIClient, pluckResponseData } from "@mailwoman/core/api"
|
|
40
|
-
import { statPath, pathExists } from "@mailwoman/core/fs/readers"
|
|
40
|
+
import { statPath, pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
41
41
|
import { makeDirectories, writeLocalTextFile } from "@mailwoman/core/fs/writers"
|
|
42
42
|
import { sha256File } from "@mailwoman/core/hash"
|
|
43
43
|
import { stringifyJSON } from "@mailwoman/core/json"
|
|
@@ -55,7 +55,7 @@ export interface FetchNADOptions extends BaseFetchOptions {
|
|
|
55
55
|
/**
|
|
56
56
|
* Fetch strategy.
|
|
57
57
|
*
|
|
58
|
-
*
|
|
58
|
+
* @defaultValue `featureserver`
|
|
59
59
|
*/
|
|
60
60
|
mode?: "featureserver" | "bulk"
|
|
61
61
|
/**
|
|
@@ -65,31 +65,31 @@ export interface FetchNADOptions extends BaseFetchOptions {
|
|
|
65
65
|
/**
|
|
66
66
|
* Records per output file.
|
|
67
67
|
*
|
|
68
|
-
*
|
|
68
|
+
* @defaultValue `100000`
|
|
69
69
|
*/
|
|
70
70
|
chunkSize?: number
|
|
71
71
|
/**
|
|
72
72
|
* Records per http request.
|
|
73
73
|
*
|
|
74
|
-
*
|
|
74
|
+
* @defaultValue `5000`
|
|
75
75
|
*/
|
|
76
76
|
pageSize?: number
|
|
77
77
|
/**
|
|
78
78
|
* Parallel page fetches within a chunk.
|
|
79
79
|
*
|
|
80
|
-
*
|
|
80
|
+
* @defaultValue `4`
|
|
81
81
|
*/
|
|
82
82
|
concurrency?: number
|
|
83
83
|
/**
|
|
84
84
|
* Start objectid.
|
|
85
85
|
*
|
|
86
|
-
*
|
|
86
|
+
* @defaultValue `1`
|
|
87
87
|
*/
|
|
88
88
|
startOID?: number
|
|
89
89
|
/**
|
|
90
90
|
* Stop before this OID.
|
|
91
91
|
*
|
|
92
|
-
*
|
|
92
|
+
* @defaultValue = total count.
|
|
93
93
|
*/
|
|
94
94
|
endOID?: number
|
|
95
95
|
}
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
procedure rather than sections of declarations. A region there folds no element a reader wants folded. */
|
|
16
16
|
|
|
17
17
|
import { APIClient, pluckResponseData } from "@mailwoman/core/api"
|
|
18
|
-
import { statPath, pathExists } from "@mailwoman/core/fs/readers"
|
|
18
|
+
import { statPath, pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
19
19
|
import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
|
|
20
20
|
import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip"
|
|
21
21
|
import { sha256File } from "@mailwoman/core/hash"
|
|
@@ -35,7 +35,7 @@ export type FetchNPPESOptions = BaseFetchOptions
|
|
|
35
35
|
* files match `NPPES_Data_Dissemination_<Month>_<Year>*.zip`, while weekly files
|
|
36
36
|
* include a `MMDDYY_MMDDYY` date range and are excluded.
|
|
37
37
|
*/
|
|
38
|
-
async function discoverLatestZip(): Promise<string |
|
|
38
|
+
async function discoverLatestZip(): Promise<string | null> {
|
|
39
39
|
const html = await new APIClient({
|
|
40
40
|
displayName: "nppes-index",
|
|
41
41
|
retry: true,
|
|
@@ -50,17 +50,17 @@ async function discoverLatestZip(): Promise<string | undefined> {
|
|
|
50
50
|
if (name && !/\d{6}_\d{6}/.test(name)) return name
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
-
return
|
|
53
|
+
return null
|
|
54
54
|
}
|
|
55
55
|
|
|
56
56
|
/**
|
|
57
57
|
* The main registry CSV (`npidata_pfile_*.csv`); the archive also includes a
|
|
58
58
|
* header file and a per-month change file.
|
|
59
59
|
*/
|
|
60
|
-
async function findNpidataCSV(zipPath: PathBuilderLike): Promise<string |
|
|
60
|
+
async function findNpidataCSV(zipPath: PathBuilderLike): Promise<string | null> {
|
|
61
61
|
const entries = await listZipEntries(zipPath)
|
|
62
62
|
|
|
63
|
-
return entries.find((entry) => /npidata_pfile\S+\.csv/i.test(entry.name))?.name
|
|
63
|
+
return entries.find((entry) => /npidata_pfile\S+\.csv/i.test(entry.name))?.name ?? null
|
|
64
64
|
}
|
|
65
65
|
|
|
66
66
|
export async function fetchNPPES(options: FetchNPPESOptions, report?: (line: string) => void): Promise<FetchSummary> {
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
19
|
import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters"
|
|
20
|
-
import { statPath, pathExists } from "@mailwoman/core/fs/readers"
|
|
20
|
+
import { statPath, pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
21
21
|
import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
|
|
22
22
|
import { sha256File } from "@mailwoman/core/hash"
|
|
23
23
|
import type { PathBuilderLike } from "path-ts"
|
|
@@ -35,8 +35,9 @@ export interface FetchStateHISchoolsOptions extends BaseFetchOptions {
|
|
|
35
35
|
/**
|
|
36
36
|
* Workbook URL.
|
|
37
37
|
*
|
|
38
|
-
* Defaults to the Hawaii DOE source.
|
|
39
38
|
* Overridable for an isolated fetch test.
|
|
39
|
+
*
|
|
40
|
+
* @defaultValue the Hawaii DOE source.
|
|
40
41
|
*/
|
|
41
42
|
sourceURL?: string
|
|
42
43
|
}
|
|
@@ -15,7 +15,7 @@ import { APIClient, pluckResponseData } from "@mailwoman/core/api"
|
|
|
15
15
|
/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
|
|
16
16
|
procedure rather than sections of declarations. A region there folds no element a reader wants folded. */
|
|
17
17
|
import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters"
|
|
18
|
-
import { statPath, pathExists } from "@mailwoman/core/fs/readers"
|
|
18
|
+
import { statPath, pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
19
19
|
import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
|
|
20
20
|
import { sha256File } from "@mailwoman/core/hash"
|
|
21
21
|
import { isoSeconds } from "@mailwoman/core/utils"
|
package/lib/utils/align.ts
CHANGED
|
@@ -33,10 +33,11 @@
|
|
|
33
33
|
*/
|
|
34
34
|
|
|
35
35
|
import type { BIOLabel, ComponentTag } from "@mailwoman/codex/component"
|
|
36
|
+
import type { TextSpan } from "@mailwoman/query-shape"
|
|
36
37
|
import { distance as levenshteinDistance } from "fastest-levenshtein"
|
|
37
38
|
|
|
38
39
|
import type { CanonicalRow, LabeledRow, QuarantinedRow } from "#types"
|
|
39
|
-
import { whitespaceTokenizer, type
|
|
40
|
+
import { whitespaceTokenizer, type Tokenizer } from "#utils/tokenize"
|
|
40
41
|
|
|
41
42
|
/**
|
|
42
43
|
* Options for `alignRow`.
|
|
@@ -45,7 +46,7 @@ export interface AlignOptions {
|
|
|
45
46
|
/**
|
|
46
47
|
* Tokenizer to use.
|
|
47
48
|
*
|
|
48
|
-
*
|
|
49
|
+
* @defaultValue `whitespaceTokenizer()`
|
|
49
50
|
*/
|
|
50
51
|
tokenizer?: Tokenizer
|
|
51
52
|
|
|
@@ -53,19 +54,21 @@ export interface AlignOptions {
|
|
|
53
54
|
* Max Levenshtein edit distance to accept when a verbatim substring match fails.
|
|
54
55
|
*
|
|
55
56
|
* Set `0` to require verbatim matches only.
|
|
56
|
-
* Default `2`.
|
|
57
57
|
*
|
|
58
58
|
* Distance is computed against same-length windows in `raw`, so the threshold
|
|
59
59
|
* scales naturally with the component value length.
|
|
60
|
+
*
|
|
61
|
+
* @defaultValue `2`
|
|
60
62
|
*/
|
|
61
63
|
maxEditDistance?: number
|
|
62
64
|
|
|
63
65
|
/**
|
|
64
66
|
* Case-insensitive comparison for substring search.
|
|
65
67
|
*
|
|
66
|
-
* Default `true`.
|
|
67
68
|
* The retained span in `raw` is the original case.
|
|
68
69
|
* Only matching is case-insensitive.
|
|
70
|
+
*
|
|
71
|
+
* @defaultValue `true`
|
|
69
72
|
*/
|
|
70
73
|
caseInsensitive?: boolean
|
|
71
74
|
}
|
|
@@ -221,17 +224,17 @@ function locateSpan(args: {
|
|
|
221
224
|
raw: string
|
|
222
225
|
claimed: Array<[number, number]>
|
|
223
226
|
maxEditDistance: number
|
|
224
|
-
}): { start: number; end: number } |
|
|
227
|
+
}): { start: number; end: number } | null {
|
|
225
228
|
const { haystack, needle, claimed, maxEditDistance } = args
|
|
226
229
|
|
|
227
|
-
if (!needle
|
|
230
|
+
if (!needle) return null
|
|
228
231
|
|
|
229
232
|
// Pass 1: verbatim substring.
|
|
230
233
|
// Word-boundary-aligned matches are preferred over intra-word ones, so a short
|
|
231
234
|
// value cannot claim the inside of an earlier word.
|
|
232
235
|
// Intra-word matches stay allowed as the fallback because they are essential for affix
|
|
233
236
|
// supervision (street_suffix "straße" inside "Hauptstraße" has no boundary-aligned occurrence).
|
|
234
|
-
let intraWord: { start: number; end: number } |
|
|
237
|
+
let intraWord: { start: number; end: number } | null = null
|
|
235
238
|
let from = 0
|
|
236
239
|
|
|
237
240
|
while (true) {
|
|
@@ -250,7 +253,7 @@ function locateSpan(args: {
|
|
|
250
253
|
|
|
251
254
|
if (intraWord) return intraWord
|
|
252
255
|
|
|
253
|
-
if (maxEditDistance <= 0) return
|
|
256
|
+
if (maxEditDistance <= 0) return null
|
|
254
257
|
|
|
255
258
|
// Pass 2: fuzzy sliding-window.
|
|
256
259
|
const len = needle.length
|
|
@@ -265,7 +268,7 @@ function locateSpan(args: {
|
|
|
265
268
|
if (d <= maxEditDistance) return { start: i, end: i + len }
|
|
266
269
|
}
|
|
267
270
|
|
|
268
|
-
return
|
|
271
|
+
return null
|
|
269
272
|
}
|
|
270
273
|
|
|
271
274
|
const WORD_CHAR = /[\p{L}\p{N}]/u
|
|
@@ -291,7 +294,7 @@ function overlapsClaimed(start: number, end: number, claimed: Array<[number, num
|
|
|
291
294
|
/**
|
|
292
295
|
* Assign BIO labels to tokens from component spans sorted by start offset.
|
|
293
296
|
*/
|
|
294
|
-
function labelTokens(tokens: readonly
|
|
297
|
+
function labelTokens(tokens: readonly TextSpan[], spans: readonly ComponentSpan[]): readonly BIOLabel[] {
|
|
295
298
|
const out: BIOLabel[] = []
|
|
296
299
|
const seenSpan = new Set<number>()
|
|
297
300
|
|
package/lib/utils/golden.ts
CHANGED
|
@@ -70,7 +70,7 @@ export interface GoldenReport {
|
|
|
70
70
|
export function parseGoldenLine(line: string): GoldenEntry {
|
|
71
71
|
const obj = parseJSONStrict<Partial<GoldenEntry> & Record<string, unknown>>(line)
|
|
72
72
|
|
|
73
|
-
if (typeof obj.raw !== "string" || !obj.raw
|
|
73
|
+
if (typeof obj.raw !== "string" || !obj.raw) {
|
|
74
74
|
throw new Error("missing/empty raw")
|
|
75
75
|
}
|
|
76
76
|
|
|
@@ -87,7 +87,7 @@ export function parseGoldenLine(line: string): GoldenEntry {
|
|
|
87
87
|
for (const [k, v] of Object.entries(components)) {
|
|
88
88
|
if (!TAG_SET.has(k)) throw new Error(`unknown ComponentTag: ${k}`)
|
|
89
89
|
|
|
90
|
-
if (typeof v !== "string" || !v
|
|
90
|
+
if (typeof v !== "string" || !v) {
|
|
91
91
|
throw new Error(`components.${k} must be a non-empty string`)
|
|
92
92
|
}
|
|
93
93
|
}
|
package/lib/utils/license.ts
CHANGED
|
@@ -129,7 +129,7 @@ export function compileLicenseExcludes(spec: string): RegExp[] {
|
|
|
129
129
|
/**
|
|
130
130
|
* True when `license` matches any of the operator's exclude `patterns`; empty patterns never exclude.
|
|
131
131
|
*/
|
|
132
|
-
export function licenseExcluded(license: string |
|
|
132
|
+
export function licenseExcluded(license: string | null, patterns: readonly RegExp[]): boolean {
|
|
133
133
|
const l = license ?? ""
|
|
134
134
|
|
|
135
135
|
return patterns.some((p) => p.test(l))
|
|
@@ -143,7 +143,7 @@ export function licenseExcluded(license: string | undefined, patterns: readonly
|
|
|
143
143
|
* so a row loop calls {@linkcode createLicenseVerdictCache} instead.
|
|
144
144
|
*/
|
|
145
145
|
export function licenseVerdict(
|
|
146
|
-
license: string |
|
|
146
|
+
license: string | null,
|
|
147
147
|
policy: LicensePolicy,
|
|
148
148
|
excluded: readonly RegExp[] = []
|
|
149
149
|
): LicenseVerdict {
|
|
@@ -217,7 +217,7 @@ export interface LicenseVerdictCache {
|
|
|
217
217
|
/**
|
|
218
218
|
* The verdict for this value under the policy this cache was built with.
|
|
219
219
|
*/
|
|
220
|
-
read(license: string |
|
|
220
|
+
read(license: string | null): LicenseVerdict
|
|
221
221
|
/**
|
|
222
222
|
* Every distinct value refused so far, with the class it was refused under,
|
|
223
223
|
* so a build that dropped rows reports which values caused each drop.
|
package/lib/utils/split.ts
CHANGED
|
@@ -109,10 +109,10 @@ export function defaultHoldouts(): Record<string, CountryHoldout> {
|
|
|
109
109
|
// Cornwall matches the admin rows for the same place `TR` covers.
|
|
110
110
|
regions: ["Cornwall"],
|
|
111
111
|
},
|
|
112
|
-
// The eight countries below enter training through OpenStreetMap rows
|
|
113
|
-
// region and many no postcode, so each holdout matches its place three ways: the region an
|
|
114
|
-
// row
|
|
115
|
-
// ones the 2026-10-02 extracts write.
|
|
112
|
+
// The eight countries below enter training through OpenStreetMap rows. Most hold no
|
|
113
|
+
// region and many no postcode, so each holdout matches its place three ways: the region an
|
|
114
|
+
// admin row holds, the postcode prefix, and the city as the OSM rows spell it. The spellings
|
|
115
|
+
// are the ones the 2026-10-02 extracts write.
|
|
116
116
|
CN: {
|
|
117
117
|
// Gansu by its province and capital, and Qinghai `81` and Tibet `85` by postcode.
|
|
118
118
|
regions: ["甘肃省", "甘肃", "Gansu"],
|
|
@@ -120,20 +120,22 @@ export function defaultHoldouts(): Record<string, CountryHoldout> {
|
|
|
120
120
|
localities: ["兰州市", "兰州", "Lanzhou"],
|
|
121
121
|
},
|
|
122
122
|
IN: {
|
|
123
|
-
// Kerala
|
|
123
|
+
// Kerala.
|
|
124
|
+
// Its PIN codes run 67 to 69.
|
|
124
125
|
regions: ["Kerala"],
|
|
125
126
|
postcodePrefixes: ["67", "68", "69"],
|
|
126
127
|
localities: ["Kochi", "Kollam", "Thiruvananthapuram", "Kozhikode"],
|
|
127
128
|
},
|
|
128
129
|
ID: {
|
|
129
|
-
// Bali
|
|
130
|
+
// Bali.
|
|
131
|
+
// Its postcodes begin `80`.
|
|
130
132
|
regions: ["Bali"],
|
|
131
133
|
postcodePrefixes: ["80"],
|
|
132
134
|
localities: ["Denpasar", "Bali"],
|
|
133
135
|
},
|
|
134
136
|
NG: {
|
|
135
137
|
// Enugu State.
|
|
136
|
-
// OSM Nigeria rows
|
|
138
|
+
// OSM Nigeria rows hold almost no postcode or region, so the city names supply the holdout.
|
|
137
139
|
regions: ["Enugu"],
|
|
138
140
|
localities: ["Enugu", "Nsukka"],
|
|
139
141
|
},
|
|
@@ -144,7 +146,8 @@ export function defaultHoldouts(): Record<string, CountryHoldout> {
|
|
|
144
146
|
localities: ["Islamabad", "islamabad", "اسلام آباد"],
|
|
145
147
|
},
|
|
146
148
|
BD: {
|
|
147
|
-
// Chattogram, Khulna and Sylhet
|
|
149
|
+
// Chattogram, Khulna and Sylhet.
|
|
150
|
+
// Their postcodes begin 4, 9 and 3 respectively.
|
|
148
151
|
postcodePrefixes: ["40", "41", "42", "43", "90", "91", "92", "31"],
|
|
149
152
|
localities: ["Chittagong", "Chattogram", "Khulna", "Sylhet"],
|
|
150
153
|
},
|
|
@@ -153,7 +156,8 @@ export function defaultHoldouts(): Record<string, CountryHoldout> {
|
|
|
153
156
|
localities: ["Cần Thơ", "Thành phố Cần Thơ", "Huế", "Thành phố Huế"],
|
|
154
157
|
},
|
|
155
158
|
RU: {
|
|
156
|
-
// Primorsky Krai
|
|
159
|
+
// Primorsky Krai.
|
|
160
|
+
// Its postcodes run 690 to 692.
|
|
157
161
|
regions: ["Приморский край"],
|
|
158
162
|
postcodePrefixes: ["690", "691", "692"],
|
|
159
163
|
localities: ["Владивосток", "Уссурийск", "Находка"],
|
|
@@ -169,13 +173,13 @@ export function defaultHoldouts(): Record<string, CountryHoldout> {
|
|
|
169
173
|
PM: { localities: ["Miquelon-Langlade"] },
|
|
170
174
|
YT: { localities: ["Dzaoudzi", "Pamandzi"] },
|
|
171
175
|
// Tumaraa holds 1,522 of PF's 6,755 rows with a street, more than a tenth,
|
|
172
|
-
// because Pirae's 958 rows
|
|
176
|
+
// because Pirae's 958 rows have no street name and Papeete has the remaining 5,233.
|
|
173
177
|
PF: { localities: ["Tumaraa"] },
|
|
174
178
|
NC: { localities: ["Koné", "Pouembout", "Kaala-Gomen"] },
|
|
175
179
|
LV: {
|
|
176
|
-
// Latgale's south-east: the Krāslava, Ludza, Balvi, Preiļi and Līvāni
|
|
177
|
-
//
|
|
178
|
-
//
|
|
180
|
+
// Latgale's south-east: the Krāslava, Ludza, Balvi, Preiļi and Līvāni municipalities.
|
|
181
|
+
// Their postcodes begin LV-56, LV-57, LV-45 and LV-53.
|
|
182
|
+
// The state city Rēzekne lies in no municipality.
|
|
179
183
|
// The address register writes `nov.`; other sources spell the municipality out.
|
|
180
184
|
regions: [
|
|
181
185
|
"Krāslavas nov.",
|
|
@@ -203,7 +207,7 @@ export function defaultHoldouts(): Record<string, CountryHoldout> {
|
|
|
203
207
|
/**
|
|
204
208
|
* Returns the matchers for one country, reading a bare array as a region list.
|
|
205
209
|
*/
|
|
206
|
-
export function holdoutPolicyFor(holdout: CountryHoldout |
|
|
210
|
+
export function holdoutPolicyFor(holdout: CountryHoldout | null): HoldoutPolicy {
|
|
207
211
|
if (!holdout) return {}
|
|
208
212
|
|
|
209
213
|
return Array.isArray(holdout) ? { regions: holdout } : (holdout as HoldoutPolicy)
|
|
@@ -221,7 +225,7 @@ export function splitForRow(
|
|
|
221
225
|
row: Pick<SplitInputRow, "source_id" | "country" | "components">,
|
|
222
226
|
holdouts: Record<string, CountryHoldout> = defaultHoldouts()
|
|
223
227
|
): SplitName {
|
|
224
|
-
const policy = holdoutPolicyFor(holdouts[row.country])
|
|
228
|
+
const policy = holdoutPolicyFor(holdouts[row.country] ?? null)
|
|
225
229
|
const { region, postcode, locality } = row.components
|
|
226
230
|
|
|
227
231
|
const isHeldOut =
|